feat(observability): add a local Prometheus and Grafana stack fed by a metrics simulator

Makefile

@@ -327,6 +327,36 @@ test-alerts: ## Lint and unit test the alert rules with promtool.
 	echo "Running unit tests..."; \
 	promtool test rules --diff "$$tmpdir"/tests/*.yaml
 
+OBS_DEV_NAMESPACE := demo
+OBS_DEV_OUT := $(OBS_DIR)/generated/dev
+# Extra flags for the simulator, e.g. SIMULATOR_ARGS="-leader=false".
+SIMULATOR_ARGS ?=
+
+.PHONY: observability-render-dev
+observability-render-dev: ## Render dashboards and plain rules for the dev stack, with every `for:` shortened to 2m.
+	@rm -rf $(OBS_DEV_OUT)/alerts/* $(OBS_DEV_OUT)/dashboards/*
+	@$(MAKE) --no-print-directory dashboards METRIC_NAMESPACE=$(OBS_DEV_NAMESPACE) OBS_OUT=$(OBS_DEV_OUT)
+	@$(MAKE) --no-print-directory alerts METRIC_NAMESPACE=$(OBS_DEV_NAMESPACE) OBS_OUT=$(OBS_DEV_OUT) ALERT_FORMAT=rules
+	@for file in $(OBS_DEV_OUT)/alerts/*.yaml; do \
+	  sed -E 's/^( *for: ).*/\12m/' "$$file" > "$$file.tmp" && mv "$$file.tmp" "$$file"; \
+	done
+
+.PHONY: observability-up
+observability-up: observability-render-dev ## Start Prometheus (:9090) and Grafana (:3000) and run the simulator on the host.
+	docker compose -f $(OBS_DIR)/dev/docker-compose.yaml up -d
+	@ready=0; for i in $$(seq 1 30); do \
+	  curl -fsS 127.0.0.1:9090/-/ready >/dev/null 2>&1 && { ready=1; break; }; \
+	  sleep 1; \
+	done; \
+	[ "$$ready" = "1" ] || { echo "Error: prometheus did not become ready"; exit 1; }; \
+	curl -fsS -X POST 127.0.0.1:9090/-/reload
+	@echo "Grafana: http://localhost:3000  Prometheus: http://localhost:9090/alerts"
+	go run ./$(OBS_DIR)/dev/simulator -metric-namespace=$(OBS_DEV_NAMESPACE) $(SIMULATOR_ARGS)
+
+.PHONY: observability-down
+observability-down: ## Stop the local Prometheus and Grafana.
+	docker compose -f $(OBS_DIR)/dev/docker-compose.yaml down
+
 
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/onsi/gomega v1.42.1
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/sourcehawk/go-crd-condition-metrics v1.1.0
 	github.com/spf13/cobra v1.10.2
 	github.com/stretchr/testify v1.11.1
@@ -46,7 +47,6 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.67.5 // indirect
 	github.com/prometheus/procfs v0.19.2 // indirect
 	github.com/sourcehawk/go-prometheus-gaugevecset v1.1.0 // indirect

observability/dev/docker-compose.yaml

@@ -0,0 +1,29 @@
+# Local Prometheus + Grafana for looking at the dashboards and alerts with the
+# simulator's data behind them. Start with `make observability-up`.
+services:
+  prometheus:
+    image: prom/prometheus:v3.14.0
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --web.enable-lifecycle
+    ports:
+      - "127.0.0.1:9090:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ../generated/dev/alerts:/etc/prometheus/rules:ro
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+
+  grafana:
+    image: grafana/grafana:13.1.4
+    ports:
+      - "127.0.0.1:3000:3000"
+    environment:
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+      GF_AUTH_DISABLE_LOGIN_FORM: "true"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ../generated/dev/dashboards:/var/lib/grafana/dashboards:ro
+    depends_on:
+      - prometheus

observability/dev/grafana/provisioning/dashboards/ocf.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+providers:
+  - name: ocf
+    folder: OCF
+    type: file
+    disableDeletion: true
+    options:
+      path: /var/lib/grafana/dashboards

observability/dev/grafana/provisioning/datasources/prometheus.yaml

@@ -0,0 +1,9 @@
+apiVersion: 1
+datasources:
+  - name: Prometheus
+    type: prometheus
+    uid: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false

observability/dev/prometheus/prometheus.yml

@@ -0,0 +1,18 @@
+# Scrapes the simulator running on the host. The static `namespace` and `pod`
+# target labels collide with the namespace label the condition gauge exports,
+# which Prometheus resolves by renaming the exported one to
+# `exported_namespace`, exactly as a ServiceMonitor scrape does in a cluster.
+global:
+  scrape_interval: 5s
+  evaluation_interval: 10s
+
+rule_files:
+  - /etc/prometheus/rules/*.yaml
+
+scrape_configs:
+  - job_name: demo-operator
+    static_configs:
+      - targets: ["host.docker.internal:8080"]
+        labels:
+          namespace: operators
+          pod: demo-operator-0

observability/dev/simulator/main.go

@@ -0,0 +1,77 @@
+// Command simulator exposes a synthetic operator's metrics on /metrics for the
+// local observability stack under observability/dev.
+//
+// The framework's own series (resource applies, conditions) are recorded
+// through the real pkg/metrics recorder; controller-runtime, workqueue, REST
+// client and leader election series are lookalikes guarded by runtime_test.go.
+// The world it plays is described in docs/observability.md.
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	ocm "github.com/sourcehawk/go-crd-condition-metrics/pkg/crd-condition-metrics"
+
+	"github.com/sourcehawk/operator-component-framework/pkg/metrics"
+)
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// run wires the registry the way an operator would, serves /metrics and plays
+// the world until the process is signalled. It returns an error when the
+// metrics endpoint failed to serve, so main exits non-zero and
+// `make observability-up` fails visibly instead of idling without metrics.
+func run() error {
+	listen := flag.String("listen", ":8080", "address to serve /metrics on")
+	metricNamespace := flag.String("metric-namespace", "demo", "metric namespace of the condition gauge")
+	leader := flag.Bool("leader", true, "report this replica as the leader; false shows OperatorLeaderMissing")
+	flag.Parse()
+
+	reg := prometheus.NewRegistry()
+	conditions := ocm.NewOperatorConditionsGauge(*metricNamespace)
+	collectors := metrics.NewCollectors()
+	reg.MustRegister(conditions, collectors)
+	rt := newRuntimeMetrics(reg)
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{}))
+	srv := &http.Server{Addr: *listen, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
+	serveErr := make(chan error, 1)
+	go func() {
+		log.Printf("serving metrics on %s/metrics", *listen)
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			serveErr <- err
+			stop()
+		}
+	}()
+
+	newWorld(conditions, collectors, rt, *leader).run(ctx)
+
+	shutdown, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	_ = srv.Shutdown(shutdown)
+	select {
+	case err := <-serveErr:
+		return fmt.Errorf("serving metrics: %w", err)
+	default:
+		return nil
+	}
+}

observability/dev/simulator/runtime.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+// runtimeMetrics defines the controller-runtime, workqueue, REST client and
+// leader election series an operator exposes, with the same names, label sets
+// and bucket layouts as the real ones. The real vectors live in
+// controller-runtime's internal packages; runtime_test.go guards this copy
+// against drift.
+type runtimeMetrics struct {
+	reconcileTotal      *prometheus.CounterVec
+	reconcileErrors     *prometheus.CounterVec
+	reconcilePanics     *prometheus.CounterVec
+	reconcileTime       *prometheus.HistogramVec
+	maxConcurrent       *prometheus.GaugeVec
+	activeWorkers       *prometheus.GaugeVec
+	queueDepth          *prometheus.GaugeVec
+	queueAdds           *prometheus.CounterVec
+	queueDuration       *prometheus.HistogramVec
+	workDuration        *prometheus.HistogramVec
+	queueUnfinished     *prometheus.GaugeVec
+	queueLongestRunning *prometheus.GaugeVec
+	queueRetries        *prometheus.CounterVec
+	restRequests        *prometheus.CounterVec
+	leader              *prometheus.GaugeVec
+}
+
+// Label names and the reconcile result value the lookalike series share with
+// controller-runtime.
+const (
+	labelController = "controller"
+	labelName       = "name"
+	resultError     = "error"
+)
+
+// reconcileTimeBuckets mirrors controller-runtime's ReconcileTime histogram.
+var reconcileTimeBuckets = []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
+	1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60}
+
+func newRuntimeMetrics(reg prometheus.Registerer) *runtimeMetrics {
+	m := &runtimeMetrics{
+		reconcileTotal: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "controller_runtime_reconcile_total",
+			Help: "Total number of reconciliations per controller",
+		}, []string{labelController, "result"}),
+		reconcileErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "controller_runtime_reconcile_errors_total",
+			Help: "Total number of reconciliation errors per controller",
+		}, []string{labelController}),
+		reconcilePanics: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "controller_runtime_reconcile_panics_total",
+			Help: "Total number of reconciliation panics per controller",
+		}, []string{labelController}),
+		reconcileTime: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "controller_runtime_reconcile_time_seconds",
+			Help:    "Length of time per reconciliation per controller",
+			Buckets: reconcileTimeBuckets,
+		}, []string{labelController}),
+		maxConcurrent: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "controller_runtime_max_concurrent_reconciles",
+			Help: "Maximum number of concurrent reconciles per controller",
+		}, []string{labelController}),
+		activeWorkers: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "controller_runtime_active_workers",
+			Help: "Number of currently used workers per controller",
+		}, []string{labelController}),
+		queueDepth: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.DepthKey,
+			Help: "Current depth of workqueue by workqueue and priority",
+		}, []string{labelName, labelController, "priority"}),
+		queueAdds: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.AddsKey,
+			Help: "Total number of adds handled by workqueue",
+		}, []string{labelName, labelController}),
+		queueDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.QueueLatencyKey,
+			Help:    "How long in seconds an item stays in workqueue before being requested",
+			Buckets: prometheus.ExponentialBuckets(10e-9, 10, 12),
+		}, []string{labelName, labelController}),
+		workDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.WorkDurationKey,
+			Help:    "How long in seconds processing an item from workqueue takes.",
+			Buckets: prometheus.ExponentialBuckets(10e-9, 10, 12),
+		}, []string{labelName, labelController}),
+		queueUnfinished: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.UnfinishedWorkKey,
+			Help: "How many seconds of work has been done that " +
+				"is in progress and hasn't been observed by work_duration. Large " +
+				"values indicate stuck threads. One can deduce the number of stuck " +
+				"threads by observing the rate at which this increases.",
+		}, []string{labelName, labelController}),
+		queueLongestRunning: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.LongestRunningProcessorKey,
+			Help: "How many seconds has the longest running " +
+				"processor for workqueue been running.",
+		}, []string{labelName, labelController}),
+		queueRetries: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Subsystem: ctrlmetrics.WorkQueueSubsystem, Name: ctrlmetrics.RetriesKey,
+			Help: "Total number of items added to the workqueue with a non-zero delay (rate-limited requeues, explicit RequeueAfter or AddAfter calls)",
+		}, []string{labelName, labelController}),
+		restRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "rest_client_requests_total",
+			Help: "Number of HTTP requests, partitioned by status code, method, and host.",
+		}, []string{"code", "method", "host"}),
+		leader: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "leader_election_master_status",
+			Help: "Gauge of if the reporting system is master of the relevant lease, 0 indicates backup, 1 indicates master. 'name' is the string used to identify the lease. Please make sure to group by name.",
+		}, []string{labelName}),
+	}
+	reg.MustRegister(
+		m.reconcileTotal, m.reconcileErrors, m.reconcilePanics, m.reconcileTime, m.maxConcurrent, m.activeWorkers,
+		m.queueDepth, m.queueAdds, m.queueDuration, m.workDuration, m.queueUnfinished, m.queueLongestRunning, m.queueRetries,
+		m.restRequests, m.leader,
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		collectors.NewGoCollector(),
+	)
+	return m
+}
+
+// initController creates the zero-valued series controller-runtime creates
+// when a controller starts, so panels show a flat line instead of no data.
+func (m *runtimeMetrics) initController(name string, maxConcurrent int) {
+	for _, result := range []string{resultError, "requeue_after", "requeue", "success"} {
+		m.reconcileTotal.WithLabelValues(name, result).Add(0)
+	}
+	m.reconcileErrors.WithLabelValues(name).Add(0)
+	m.reconcilePanics.WithLabelValues(name).Add(0)
+	m.reconcileTime.WithLabelValues(name)
+	m.maxConcurrent.WithLabelValues(name).Set(float64(maxConcurrent))
+	m.activeWorkers.WithLabelValues(name).Set(0)
+	m.queueDepth.WithLabelValues(name, name, "0").Set(0)
+	m.queueAdds.WithLabelValues(name, name).Add(0)
+	m.queueDuration.WithLabelValues(name, name)
+	m.workDuration.WithLabelValues(name, name)
+	m.queueUnfinished.WithLabelValues(name, name).Set(0)
+	m.queueLongestRunning.WithLabelValues(name, name).Set(0)
+	m.queueRetries.WithLabelValues(name, name).Add(0)
+}