From b8c7a22779bec4c72d2ddb66063a49fb4778d574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:52:56 +0200 Subject: [PATCH 1/9] feat(observability): add the alert render pipeline and the condition alerts (#184) Add an Observability section to the root Makefile with the alerts, dashboards and test-alerts targets, port the condition alert rules and their promtool unit tests from go-crd-condition-metrics, and ignore the rendered output under observability/generated. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .gitignore | 3 + Makefile | 95 +++++++ observability/README.md | 15 ++ observability/alerts/crd_conditions.tpl.yaml | 166 ++++++++++++ .../alerts/tests/crd_conditions_test.yaml | 250 ++++++++++++++++++ 5 files changed, 529 insertions(+) create mode 100644 observability/README.md create mode 100644 observability/alerts/crd_conditions.tpl.yaml create mode 100644 observability/alerts/tests/crd_conditions_test.yaml diff --git a/.gitignore b/.gitignore index f2d6818b..a1d509ba 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ docs/superpowers/ # MkDocs build output site/ + +# Rendered observability artifacts +observability/generated/ diff --git a/Makefile b/Makefile index 9e486268..12ac0982 100644 --- a/Makefile +++ b/Makefile @@ -205,6 +205,101 @@ e2e-component: ginkgo kind-create kind-set-context ## Run component E2E tests on .PHONY: e2e-full e2e-full: kind-create kind-set-context e2e kind-delete ## Full E2E lifecycle: create cluster, test, teardown. +##@ Observability + +OBS_DIR := observability +# Render output directory. The dev stack overrides it to keep its render apart. +OBS_OUT ?= $(OBS_DIR)/generated +# Prometheus metric namespace the condition gauge was created with +# (ocm.NewOperatorConditionsGauge("")). Required for rendering. +METRIC_NAMESPACE ?= unset +# Label carrying the namespace of the custom resource. The pod scraping the +# operator usually owns `namespace`, so the exported label arrives as +# `exported_namespace`; override with NAMESPACE_LABEL=namespace if yours does not. +NAMESPACE_LABEL ?= exported_namespace +# Shape of the rendered alert files: prometheusrule (one PrometheusRule object +# per template) or rules (plain files for prometheus' rule_files). +ALERT_FORMAT ?= prometheusrule +# Metric namespace the alert unit tests are written against. +ALERT_TEST_NAMESPACE := test_operator + +# Fail unless METRIC_NAMESPACE was given. $(1) is the target name for the hint. +define require_metric_namespace +@[ "$(METRIC_NAMESPACE)" != "unset" ] && [ -n "$(METRIC_NAMESPACE)" ] || { \ + echo "Error: METRIC_NAMESPACE is required."; \ + echo "Usage: make $(1) METRIC_NAMESPACE=my_operator"; \ + exit 1; \ +} +endef + +# Render a template to stdout. $(1) template path, $(2) metric namespace, +# $(3) namespace label. +define render_template +sed -e 's/{{operator_namespace}}/$(2)_/g' -e 's/{{namespace_label}}/$(3)/g' $(1) +endef + +.PHONY: dashboards +dashboards: ## Render the Grafana dashboards for METRIC_NAMESPACE into observability/generated/dashboards. + $(call require_metric_namespace,dashboards) + @echo "Rendering dashboards for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL))..." + @mkdir -p $(OBS_OUT)/dashboards + @for file in $(OBS_DIR)/dashboards/*.tpl.json; do \ + [ -e "$$file" ] || continue; \ + name=$$(basename "$$file" .tpl.json); \ + $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$(OBS_OUT)/dashboards/$$name.json"; \ + done + +.PHONY: alerts +alerts: ## Render the Prometheus alert rules for METRIC_NAMESPACE into observability/generated/alerts. + $(call require_metric_namespace,alerts) + @echo "Rendering alerts for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL), format: $(ALERT_FORMAT))..." + @mkdir -p $(OBS_OUT)/alerts + @for file in $(OBS_DIR)/alerts/*.tpl.yaml; do \ + name=$$(basename "$$file" .tpl.yaml); \ + out="$(OBS_OUT)/alerts/$$name.yaml"; \ + case "$(ALERT_FORMAT)" in \ + rules) \ + $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$$out" ;; \ + prometheusrule) \ + rule_name=$$(echo "$(METRIC_NAMESPACE)-$$name" | tr '[:upper:]' '[:lower:]' | tr '_:' '--'); \ + { \ + echo "apiVersion: monitoring.coreos.com/v1"; \ + echo "kind: PrometheusRule"; \ + echo "metadata:"; \ + echo " name: $$rule_name"; \ + echo "spec:"; \ + $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) \ + | sed -e 's/^/ /' -e 's/[[:space:]]*$$//'; \ + } > "$$out" ;; \ + *) \ + echo "Error: ALERT_FORMAT must be prometheusrule or rules, got '$(ALERT_FORMAT)'."; exit 1 ;; \ + esac; \ + done + +.PHONY: test-alerts +test-alerts: ## Lint and unit test the alert rule templates with promtool. + @command -v promtool >/dev/null 2>&1 || { \ + echo "Error: promtool is required to test the alert rules."; \ + echo "It ships with prometheus: https://prometheus.io/download/"; \ + exit 1; \ + } + @set -e; \ + tmpdir=$$(mktemp -d "$${TMPDIR:-/tmp}/ocf-alerts.XXXXXX"); \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + mkdir -p "$$tmpdir/tests" "$$tmpdir/namespace-label"; \ + for file in $(OBS_DIR)/alerts/*.tpl.yaml; do \ + name=$$(basename "$$file" .tpl.yaml); \ + $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),exported_namespace) > "$$tmpdir/$$name.yaml"; \ + $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),namespace) > "$$tmpdir/namespace-label/$$name.yaml"; \ + done; \ + cp $(OBS_DIR)/alerts/tests/*.yaml "$$tmpdir/tests/"; \ + echo "Linting rules..."; \ + promtool check rules --lint=all --lint-fatal "$$tmpdir"/*.yaml; \ + echo "Linting rules with NAMESPACE_LABEL=namespace..."; \ + promtool check rules --lint=all --lint-fatal "$$tmpdir"/namespace-label/*.yaml; \ + echo "Running unit tests..."; \ + promtool test rules --diff "$$tmpdir"/tests/*.yaml + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary diff --git a/observability/README.md b/observability/README.md new file mode 100644 index 00000000..40929deb --- /dev/null +++ b/observability/README.md @@ -0,0 +1,15 @@ +# Observability + +Grafana dashboards and Prometheus alert rules for operators built on the framework, plus a local stack to look at them. +Full documentation: [docs/observability.md](../docs/observability.md). + +Render for your operator, where `METRIC_NAMESPACE` is the argument you gave `ocm.NewOperatorConditionsGauge`: + + make dashboards METRIC_NAMESPACE=myoperator + make alerts METRIC_NAMESPACE=myoperator + +Output lands in `generated/`. Add `NAMESPACE_LABEL=namespace` if your scrape keeps the exported `namespace` label, and +`ALERT_FORMAT=rules` for plain rule files instead of `PrometheusRule` objects. + +Run the alert unit tests with `make test-alerts` (needs `promtool`), and bring up Prometheus and Grafana with the +simulator behind them with `make observability-up`. diff --git a/observability/alerts/crd_conditions.tpl.yaml b/observability/alerts/crd_conditions.tpl.yaml new file mode 100644 index 00000000..ed559f5b --- /dev/null +++ b/observability/alerts/crd_conditions.tpl.yaml @@ -0,0 +1,166 @@ +# Alerting rules for the status condition metrics the framework records through +# pkg/metrics (github.com/sourcehawk/go-crd-condition-metrics under the hood). +# +# See docs/observability.md for how these rules are rendered and which +# thresholds to tune. +# +# Every rule aggregates the metric instead of matching series directly. That is +# deliberate and load bearing: +# +# * it drops the reason, status and id labels, so a controller that keeps +# changing the reason while a resource stays unhealthy does not restart the +# `for:` clock every time, +# * max() keeps the value a lastTransitionTime, the freshest one, so a former +# leader pod still exporting a stale series cannot skew it the way sum() +# would. +groups: + - name: crd-conditions + rules: + # A resource whose Ready condition has been False for too long. + # + # Scoped to Ready on purpose. Matching `status="False"` across every + # condition would fire forever on negative polarity conditions such as + # Degraded or FailedToProvision, where False is the healthy state. Add + # your own positive polarity conditions to the matcher below. + - alert: CustomResourceNotReady + for: 30m + expr: > + max ( + {{operator_namespace}}controller_condition{condition="Ready", status="False"} + ) by (controller, kind, name, {{namespace_label}}) + labels: + severity: warning + annotations: + summary: >- + {{- if $labels.{{namespace_label}} -}} + {{ $labels.kind }} CR `{{ $labels.{{namespace_label}} }}/{{ $labels.name }}` has not been ready for 30 minutes + {{- else -}} + {{ $labels.kind }} CR `{{ $labels.name }}` has not been ready for 30 minutes + {{- end -}} + description: | + {{- if $labels.{{namespace_label}} -}} + Custom Resource of kind `{{ $labels.kind }}` named `{{ $labels.name }}` in namespace + `{{ $labels.{{namespace_label}} }}` has not been ready for more than 30 minutes. + {{- else -}} + Custom Resource of kind `{{ $labels.kind }}` named `{{ $labels.name }}` (cluster scoped + resource) has not been ready for more than 30 minutes. + {{- end }} + + The Ready condition last transitioned at {{ $value | humanizeTimestamp }}. + + Quick check with: + ``` + {{- if $labels.{{namespace_label}} }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} -n {{ $labels.{{namespace_label}} }} + {{- else }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} + {{- end }} + ``` + # Deep link into the CRD Conditions Browser dashboard shipped in this + # repository. The id label is "/", which for a cluster + # scoped resource is "/", so the empty namespace renders correctly + # here without a conditional. + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind + }}&var-condition=Ready&var-status=False&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} + + # A condition the controller cannot determine the state of. Unlike the two + # rules around it this one is not scoped to a single condition: Unknown is + # bad whatever the condition's polarity is. + - alert: CustomResourceConditionUnknown + for: 30m + expr: > + max ( + {{operator_namespace}}controller_condition{status="Unknown"} + ) by (controller, kind, name, condition, {{namespace_label}}) + labels: + severity: warning + annotations: + summary: >- + {{- if $labels.{{namespace_label}} -}} + {{ $labels.kind }} CR `{{ $labels.{{namespace_label}} }}/{{ $labels.name }}` has had an unknown {{ $labels.condition }} condition for 30 minutes + {{- else -}} + {{ $labels.kind }} CR `{{ $labels.name }}` has had an unknown {{ $labels.condition }} condition for 30 minutes + {{- end -}} + description: | + {{- if $labels.{{namespace_label}} -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` in + namespace `{{ $labels.{{namespace_label}} }}` has had status Unknown for more than 30 minutes. + {{- else -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` + (cluster scoped resource) has had status Unknown for more than 30 minutes. + {{- end }} + + The controller is failing to determine the state of this resource, so neither a healthy + nor an unhealthy signal can be trusted for it. The condition last transitioned at + {{ $value | humanizeTimestamp }}. + + Quick check with: + ``` + {{- if $labels.{{namespace_label}} }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} -n {{ $labels.{{namespace_label}} }} + {{- else }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} + {{- end }} + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + $labels.condition }}&var-status=Unknown&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} + + # A resource that has sat in a non-ready state for hours. + # + # This rule is the reason the metric value is a lastTransitionTime. It needs + # no `for:` clause because the expression is itself a duration comparison, + # and that difference is the point: a `for:` clause measures how long the + # *alert* has been true, which is bounded by how long the series has been + # continuously present, whereas this measures how long the *resource* has + # actually been in this state according to its own status. It therefore + # survives scrape gaps, operator restarts and ruler restarts, any of which + # silently restart a `for:` clock, and reports the real age of the state. + # It also covers status Unknown, which CustomResourceNotReady does not. + # + # Scoped to Ready for the same polarity reason as CustomResourceNotReady. + # The condition label is kept in the aggregation so widening the matcher to + # your own conditions stays a one line change. + - alert: CustomResourceConditionStuck + expr: > + ( + time() + - + max ( + {{operator_namespace}}controller_condition{condition="Ready", status!="True"} + ) by (controller, kind, name, condition, {{namespace_label}}) + ) > 21600 + labels: + severity: warning + annotations: + summary: >- + {{- if $labels.{{namespace_label}} -}} + {{ $labels.kind }} CR `{{ $labels.{{namespace_label}} }}/{{ $labels.name }}` has been stuck not ready for {{ $value | humanizeDuration }} + {{- else -}} + {{ $labels.kind }} CR `{{ $labels.name }}` has been stuck not ready for {{ $value | humanizeDuration }} + {{- end -}} + description: | + {{- if $labels.{{namespace_label}} -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` in + namespace `{{ $labels.{{namespace_label}} }}` has not been True for + {{ $value | humanizeDuration }}. + {{- else -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` + (cluster scoped resource) has not been True for {{ $value | humanizeDuration }}. + {{- end }} + + Nothing has moved for hours, so the controller is either wedged or waiting on something + that is never going to arrive. Reconciliation retries alone will not clear this. + + Quick check with: + ``` + {{- if $labels.{{namespace_label}} }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} -n {{ $labels.{{namespace_label}} }} + {{- else }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} + {{- end }} + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + $labels.condition }}&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} diff --git a/observability/alerts/tests/crd_conditions_test.yaml b/observability/alerts/tests/crd_conditions_test.yaml new file mode 100644 index 00000000..4e72524e --- /dev/null +++ b/observability/alerts/tests/crd_conditions_test.yaml @@ -0,0 +1,250 @@ +# Unit tests for the generic CRD condition alerting rules. +# +# Run with `make test-alerts`, which renders ../crd_conditions.tpl.yaml with the +# metric namespace pinned to `test_operator` and the namespace label left at its +# default before handing both files to promtool. +rule_files: + - ../crd_conditions.yaml + +evaluation_interval: 1m + +tests: + # A reason change while a resource stays not ready must not restart the 30m + # clock, and a resource with no namespace label (cluster scoped) must produce + # the alternate wording. + - name: TestCustomResourceNotReady + interval: 1m + input_series: + # Backup Ready=False reason=Error1 [0m-19m] + - series: 'test_operator_controller_condition{controller="backup",kind="Backup",name="nightly-abc123",exported_namespace="backups",condition="Ready",status="False",reason="Error1",id="backups/nightly-abc123"}' + values: '1760534009+0x19' + # Backup Ready=False reason=Error2 [20m-44m]. The Error1 series is still + # within the 5m lookback window until 24m, so both are present for a while + # and max() has to pick the freshest lastTransitionTime. + - series: 'test_operator_controller_condition{controller="backup",kind="Backup",name="nightly-abc123",exported_namespace="backups",condition="Ready",status="False",reason="Error2",id="backups/nightly-abc123"}' + values: '_x20 1760534348+0x24' + # Backup Ready=True [45m-60m] + - series: 'test_operator_controller_condition{controller="backup",kind="Backup",name="nightly-abc123",exported_namespace="backups",condition="Ready",status="True",reason="Ready",id="backups/nightly-abc123"}' + values: '_x45 1760534420+0x15' + # RemoteStorage Ready=False [15m-60m], cluster scoped so no namespace label + - series: 'test_operator_controller_condition{controller="remotestorage",kind="RemoteStorage",name="document-store",condition="Ready",status="False",reason="Error",id="/document-store"}' + values: '_x15 1760582000+0x45' + + alert_rule_test: + # Nothing fires immediately (for: 30m) + - eval_time: 0m + alertname: CustomResourceNotReady + exp_alerts: [] + + # Still pending at 25m: under the threshold, and the Error1 -> Error2 + # change at 20m must not have restarted the clock. + - eval_time: 25m + alertname: CustomResourceNotReady + exp_alerts: [] + + # Backup has been not ready for 30m. RemoteStorage only started at 15m so + # it is still pending. + - eval_time: 30m + alertname: CustomResourceNotReady + exp_alerts: + - exp_labels: + controller: backup + kind: Backup + name: nightly-abc123 + exported_namespace: backups + severity: warning + exp_annotations: + summary: >- + Backup CR `backups/nightly-abc123` has not been ready for 30 minutes + description: | + Custom Resource of kind `Backup` named `nightly-abc123` in namespace + `backups` has not been ready for more than 30 minutes. + + The Ready condition last transitioned at 2025-10-15 13:19:08 +0000 UTC. + + Quick check with: + ``` + kubectl describe Backup/nightly-abc123 -n backups + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=Backup&var-condition=Ready&var-status=False&var-resource_id=backups%2Fnightly-abc123 + + # Backup recovered at 45m, so it has resolved. RemoteStorage has now been + # not ready for 30m and is the only alert left. + - eval_time: 50m + alertname: CustomResourceNotReady + exp_alerts: + - exp_labels: + controller: remotestorage + kind: RemoteStorage + name: document-store + severity: warning + exp_annotations: + summary: >- + RemoteStorage CR `document-store` has not been ready for 30 minutes + description: | + Custom Resource of kind `RemoteStorage` named `document-store` (cluster scoped + resource) has not been ready for more than 30 minutes. + + The Ready condition last transitioned at 2025-10-16 02:33:20 +0000 UTC. + + Quick check with: + ``` + kubectl describe RemoteStorage/document-store + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=RemoteStorage&var-condition=Ready&var-status=False&var-resource_id=%2Fdocument-store + + # Unknown is bad whatever the condition's polarity is, so this rule is not + # scoped to a single condition: both fixtures below must fire. + - name: TestCustomResourceConditionUnknown + interval: 1m + input_series: + - series: 'test_operator_controller_condition{controller="app",kind="App",name="checkout",exported_namespace="shop",condition="Synced",status="Unknown",reason="ProbeFailed",id="shop/checkout"}' + values: '1760600000+0x60' + - series: 'test_operator_controller_condition{controller="app",kind="App",name="checkout",exported_namespace="shop",condition="Ready",status="Unknown",reason="ProbeFailed",id="shop/checkout"}' + values: '1760600000+0x60' + + alert_rule_test: + - eval_time: 25m + alertname: CustomResourceConditionUnknown + exp_alerts: [] + + - eval_time: 30m + alertname: CustomResourceConditionUnknown + exp_alerts: + - exp_labels: + controller: app + kind: App + name: checkout + condition: Ready + exported_namespace: shop + severity: warning + exp_annotations: + summary: >- + App CR `shop/checkout` has had an unknown Ready condition for 30 minutes + description: | + The `Ready` condition of `App` `checkout` in + namespace `shop` has had status Unknown for more than 30 minutes. + + The controller is failing to determine the state of this resource, so neither a healthy + nor an unhealthy signal can be trusted for it. The condition last transitioned at + 2025-10-16 07:33:20 +0000 UTC. + + Quick check with: + ``` + kubectl describe App/checkout -n shop + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Ready&var-status=Unknown&var-resource_id=shop%2Fcheckout + - exp_labels: + controller: app + kind: App + name: checkout + condition: Synced + exported_namespace: shop + severity: warning + exp_annotations: + summary: >- + App CR `shop/checkout` has had an unknown Synced condition for 30 minutes + description: | + The `Synced` condition of `App` `checkout` in + namespace `shop` has had status Unknown for more than 30 minutes. + + The controller is failing to determine the state of this resource, so neither a healthy + nor an unhealthy signal can be trusted for it. The condition last transitioned at + 2025-10-16 07:33:20 +0000 UTC. + + Quick check with: + ``` + kubectl describe App/checkout -n shop + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Synced&var-status=Unknown&var-resource_id=shop%2Fcheckout + + # The stuck rule compares against the metric value, so the fixture timestamps + # have to live on promtool's clock, which starts at the unix epoch. A Ready + # condition that transitioned at t=60s is 6h59m old at an eval_time of 7h. + - name: TestCustomResourceConditionStuck + interval: 1m + input_series: + - series: 'test_operator_controller_condition{controller="cluster",kind="Cluster",name="prod",exported_namespace="clusters",condition="Ready",status="False",reason="Provisioning",id="clusters/prod"}' + values: '60+0x480' + + alert_rule_test: + # 5h since the transition, under the 6h threshold + - eval_time: 5h + alertname: CustomResourceConditionStuck + exp_alerts: [] + + - eval_time: 7h + alertname: CustomResourceConditionStuck + exp_alerts: + - exp_labels: + controller: cluster + kind: Cluster + name: prod + condition: Ready + exported_namespace: clusters + severity: warning + exp_annotations: + summary: >- + Cluster CR `clusters/prod` has been stuck not ready for 6h 59m 0s + description: | + The `Ready` condition of `Cluster` `prod` in + namespace `clusters` has not been True for + 6h 59m 0s. + + Nothing has moved for hours, so the controller is either wedged or waiting on something + that is never going to arrive. Reconciliation retries alone will not clear this. + + Quick check with: + ``` + kubectl describe Cluster/prod -n clusters + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fprod + + # Why CustomResourceConditionStuck exists alongside CustomResourceNotReady: + # the series disappears between 1h and 6h (operator down, scrape gap), which + # restarts the `for: 30m` clock on its return. 20m after it comes back the + # `for:` based rule is still pending, while the value based rule correctly + # reports the resource as stuck for 6h19m. + - name: TestConditionStuckSurvivesSeriesGaps + interval: 1m + input_series: + - series: 'test_operator_controller_condition{controller="cluster",kind="Cluster",name="staging",exported_namespace="clusters",condition="Ready",status="False",reason="Provisioning",id="clusters/staging"}' + values: '60+0x60 _x300 60+0x60' + + alert_rule_test: + - eval_time: 6h20m + alertname: CustomResourceNotReady + exp_alerts: [] + + - eval_time: 6h20m + alertname: CustomResourceConditionStuck + exp_alerts: + - exp_labels: + controller: cluster + kind: Cluster + name: staging + condition: Ready + exported_namespace: clusters + severity: warning + exp_annotations: + summary: >- + Cluster CR `clusters/staging` has been stuck not ready for 6h 19m 0s + description: | + The `Ready` condition of `Cluster` `staging` in + namespace `clusters` has not been True for + 6h 19m 0s. + + Nothing has moved for hours, so the controller is either wedged or waiting on something + that is never going to arrive. Reconciliation retries alone will not clear this. + + Quick check with: + ``` + kubectl describe Cluster/staging -n clusters + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fstaging From 4f005962d7a513474e7a336dfd59ac32c43a70a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:56:51 +0200 Subject: [PATCH 2/9] feat(observability): alert on managed resources that never converge or fail to apply (#184) Add ManagedResourceNotConverging and ManagedResourceApplyFailing on the ocf_resource_apply_total and ocf_resource_apply_errors_total counters. Both are ratios of the resource's own applies with an absolute floor, keyed on the operator's static topology, so legitimate churn at scale and sporadic conflicts stay silent. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- observability/alerts/ocf_resources.tpl.yaml | 102 +++++++++++ .../alerts/tests/ocf_resources_test.yaml | 158 ++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 observability/alerts/ocf_resources.tpl.yaml create mode 100644 observability/alerts/tests/ocf_resources_test.yaml diff --git a/observability/alerts/ocf_resources.tpl.yaml b/observability/alerts/ocf_resources.tpl.yaml new file mode 100644 index 00000000..fd3a9673 --- /dev/null +++ b/observability/alerts/ocf_resources.tpl.yaml @@ -0,0 +1,102 @@ +# Alerting rules for the resource apply counters recorded by +# github.com/sourcehawk/operator-component-framework/pkg/metrics. +# +# Neither counter carries an owner name or namespace, only the operator's static +# topology (controller, owner kind, component, resource identifier, kind), so +# these rules fire per resource *type* rather than per owner. Per-owner signal +# comes from the condition rules in crd_conditions.tpl.yaml. +# +# Both rules are ratios of the resource's own applies rather than absolute +# rates. At scale, legitimate spec changes across many owners keep the absolute +# `updated` rate above zero indefinitely; what distinguishes a resource that +# never converges is that more than half of its applies rewrite it, at a rate +# no human produces. See docs/observability.md for the thresholds to tune. +groups: + - name: ocf-resources + rules: + # A managed resource rewritten on every reconcile. A converged resource + # applies as `none`; one whose applies keep coming back `updated` is in a + # hot loop (apply updates the object, the watch event requeues the owner, + # the next apply updates it again) or in a fight with a webhook or another + # controller. Legitimate churn is followed by a `none` apply on the next + # reconcile, which keeps its ratio at or below one half. The floor keeps a + # single edit on an idle resource from producing a ratio of one over a + # handful of samples. + - alert: ManagedResourceNotConverging + for: 15m + expr: > + ( + sum by (controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_total{operation="updated"}[15m]) + ) + / + sum by (controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_total[15m]) + ) + ) > 0.5 + and + sum by (controller, owner_kind, component, resource, kind) ( + increase(ocf_resource_apply_total{operation="updated"}[15m]) + ) > 15 + labels: + severity: warning + annotations: + summary: >- + {{ $labels.kind }} `{{ $labels.resource }}` of component `{{ $labels.component }}` ({{ $labels.owner_kind }}) is rewritten on every reconcile + description: | + Over the last 15 minutes, {{ $value | humanizePercentage }} of the applies of the `{{ $labels.kind }}` resource `{{ $labels.resource }}` in component `{{ $labels.component }}` of owner kind `{{ $labels.owner_kind }}` (controller `{{ $labels.controller }}`) updated the object. In steady state a converged resource applies as `none` on every pass, so a resource whose every apply is an update is being rewritten on every reconcile even though nothing changed. + + Usual causes: a mutation that is not idempotent (a timestamp, a random value, an unsorted list), a defaulting webhook or another controller fighting the operator, or a field the API server normalises. Events report the same thing, but client-go's spam filter truncates them within seconds under exactly these conditions. + + Quick check with: + ``` + kubectl get events -A --field-selector reason=Updated{{ $labels.kind }} + ``` + + # Applies of one resource type mostly failing. Transient conflicts among + # successful applies stay under the threshold, and a resource failing for + # a single owner among many is the condition rules' job. The `or … * 0` + # keeps the ratio defined for a resource that has never applied + # successfully, where no success series exists to add. + - alert: ManagedResourceApplyFailing + for: 15m + expr: > + ( + sum by (controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) + ) + / + ( + sum by (controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) + ) + + + ( + sum by (controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_total[15m]) + ) + or + sum by (controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) + ) * 0 + ) + ) + ) > 0.5 + and + sum by (controller, owner_kind, component, resource, kind) ( + increase(ocf_resource_apply_errors_total[15m]) + ) > 5 + labels: + severity: warning + annotations: + summary: >- + Applies of {{ $labels.kind }} `{{ $labels.resource }}` of component `{{ $labels.component }}` ({{ $labels.owner_kind }}) are failing + description: | + Over the last 15 minutes, {{ $value | humanizePercentage }} of the apply attempts of the `{{ $labels.kind }}` resource `{{ $labels.resource }}` in component `{{ $labels.component }}` of owner kind `{{ $labels.owner_kind }}` (controller `{{ $labels.controller }}`) failed. + + The counter covers every failure of an attempt: building the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition; this rule says the failure is systematic for the resource type rather than one owner's bad day. + + Quick check with: + ``` + kubectl get events -A --field-selector type=Warning | grep {{ $labels.kind }} + ``` diff --git a/observability/alerts/tests/ocf_resources_test.yaml b/observability/alerts/tests/ocf_resources_test.yaml new file mode 100644 index 00000000..0efa4593 --- /dev/null +++ b/observability/alerts/tests/ocf_resources_test.yaml @@ -0,0 +1,158 @@ +# Unit tests for the managed resource alerting rules. +# +# Run with `make test-alerts`. The counters carry no owner identity, so every +# series below is one resource *type* of one component, aggregated over all of +# its owners. Time series values are cumulative counter samples one minute apart. +rule_files: + - ../ocf_resources.yaml + +evaluation_interval: 1m + +tests: + # A hot loop: every apply of the configmap rewrites it, 4 updates a second. + # The other resources of the component converge with `none`, which must not + # mask the configmap's ratio. + - name: TestManagedResourceNotConvergingHotLoop + interval: 1m + input_series: + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="updated"}' + values: '0+240x60' + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="none"}' + values: '0+1x60' + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="deployment",kind="Deployment",operation="none"}' + values: '0+240x60' + alert_rule_test: + - eval_time: 10m + alertname: ManagedResourceNotConverging + exp_alerts: [] + # ratio 240/241 over the window, floor satisfied, for: 15m elapsed + - eval_time: 31m + alertname: ManagedResourceNotConverging + exp_alerts: + - exp_labels: + controller: webapp + owner_kind: WebApp + component: server + resource: configmap + kind: ConfigMap + severity: warning + exp_annotations: + summary: >- + ConfigMap `configmap` of component `server` (WebApp) is rewritten on every reconcile + description: | + Over the last 15 minutes, 99.59% of the applies of the `ConfigMap` resource `configmap` in component `server` of owner kind `WebApp` (controller `webapp`) updated the object. In steady state a converged resource applies as `none` on every pass, so a resource whose every apply is an update is being rewritten on every reconcile even though nothing changed. + + Usual causes: a mutation that is not idempotent (a timestamp, a random value, an unsorted list), a defaulting webhook or another controller fighting the operator, or a field the API server normalises. Events report the same thing, but client-go's spam filter truncates them within seconds under exactly these conditions. + + Quick check with: + ``` + kubectl get events -A --field-selector reason=UpdatedConfigMap + ``` + + # Legitimate churn at scale: many owners change spec, every `updated` apply is + # followed by a `none` apply on the reconcile the status update triggers. The + # absolute rate is high (2 updates a minute, well over the floor) but the ratio + # sits at 0.5, so the rule must stay silent. + - name: TestManagedResourceNotConvergingChurnDoesNotFire + interval: 1m + input_series: + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="updated"}' + values: '0+2x60' + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="none"}' + values: '0+2x60' + alert_rule_test: + - eval_time: 45m + alertname: ManagedResourceNotConverging + exp_alerts: [] + + # One edit on an otherwise idle resource: ratio is 1 but only one update in + # the window, under the floor. + - name: TestManagedResourceNotConvergingSingleEditDoesNotFire + interval: 1m + input_series: + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="updated"}' + values: '0 0 0 1+0x57' + alert_rule_test: + - eval_time: 30m + alertname: ManagedResourceNotConverging + exp_alerts: [] + + # A PVC whose applies fail three times out of four, on a resource that also + # has the occasional successful apply. + - name: TestManagedResourceApplyFailing + interval: 1m + input_series: + - series: 'ocf_resource_apply_errors_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim"}' + values: '0+3x60' + - series: 'ocf_resource_apply_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim",operation="none"}' + values: '0+1x60' + alert_rule_test: + - eval_time: 10m + alertname: ManagedResourceApplyFailing + exp_alerts: [] + - eval_time: 31m + alertname: ManagedResourceApplyFailing + exp_alerts: + - exp_labels: + controller: database + owner_kind: Database + component: storage + resource: pvc + kind: PersistentVolumeClaim + severity: warning + exp_annotations: + summary: >- + Applies of PersistentVolumeClaim `pvc` of component `storage` (Database) are failing + description: | + Over the last 15 minutes, 75% of the apply attempts of the `PersistentVolumeClaim` resource `pvc` in component `storage` of owner kind `Database` (controller `database`) failed. + + The counter covers every failure of an attempt: building the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition; this rule says the failure is systematic for the resource type rather than one owner's bad day. + + Quick check with: + ``` + kubectl get events -A --field-selector type=Warning | grep PersistentVolumeClaim + ``` + + # A resource that has only ever failed: no success series exists at all. The + # ratio must still evaluate (to 1) rather than vanish on a missing vector. + - name: TestManagedResourceApplyFailingWithoutSuccesses + interval: 1m + input_series: + - series: 'ocf_resource_apply_errors_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim"}' + values: '0+2x60' + alert_rule_test: + - eval_time: 31m + alertname: ManagedResourceApplyFailing + exp_alerts: + - exp_labels: + controller: database + owner_kind: Database + component: storage + resource: pvc + kind: PersistentVolumeClaim + severity: warning + exp_annotations: + summary: >- + Applies of PersistentVolumeClaim `pvc` of component `storage` (Database) are failing + description: | + Over the last 15 minutes, 100% of the apply attempts of the `PersistentVolumeClaim` resource `pvc` in component `storage` of owner kind `Database` (controller `database`) failed. + + The counter covers every failure of an attempt: building the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition; this rule says the failure is systematic for the resource type rather than one owner's bad day. + + Quick check with: + ``` + kubectl get events -A --field-selector type=Warning | grep PersistentVolumeClaim + ``` + + # Sporadic conflicts among many successful applies must not fire. + - name: TestManagedResourceApplyFailingSporadicDoesNotFire + interval: 1m + input_series: + - series: 'ocf_resource_apply_errors_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim"}' + values: '0+1x60' + - series: 'ocf_resource_apply_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim",operation="none"}' + values: '0+9x60' + alert_rule_test: + - eval_time: 45m + alertname: ManagedResourceApplyFailing + exp_alerts: [] From d466b419dfa85eaaad823e088790b52f47d9b765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:57:51 +0200 Subject: [PATCH 3/9] feat(observability): alert on controller-runtime errors, panics, backlog, latency and leadership (#184) Add ControllerReconcileErrors, ControllerReconcilePanics, ControllerWorkqueueBacklog, ControllerReconcileLatencyHigh and OperatorLeaderMissing on the controller-runtime metrics every operator exposes, with thresholds expressed as ratios or quantiles so they hold at any scale. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../alerts/controller_runtime.tpl.yaml | 106 +++++++++++ .../alerts/tests/controller_runtime_test.yaml | 174 ++++++++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 observability/alerts/controller_runtime.tpl.yaml create mode 100644 observability/alerts/tests/controller_runtime_test.yaml diff --git a/observability/alerts/controller_runtime.tpl.yaml b/observability/alerts/controller_runtime.tpl.yaml new file mode 100644 index 00000000..99d0e472 --- /dev/null +++ b/observability/alerts/controller_runtime.tpl.yaml @@ -0,0 +1,106 @@ +# Alerting rules for the controller-runtime metrics every operator exposes: +# reconcile results and latency, the workqueue, leader election. +# +# Thresholds are deliberately conservative and expressed as ratios or +# quantiles, so that they hold whether the operator manages three owners or +# three thousand. See docs/observability.md for the thresholds to tune. +groups: + - name: controller-runtime + rules: + - alert: ControllerReconcileErrors + for: 15m + expr: > + ( + sum by (controller) (rate(controller_runtime_reconcile_total{result="error"}[10m])) + / + sum by (controller) (rate(controller_runtime_reconcile_total[10m])) + ) > 0.25 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` fails {{ $value | humanizePercentage }} of its reconciles + description: | + Over the last 10 minutes, {{ $value | humanizePercentage }} of the reconciles of controller `{{ $labels.controller }}` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep {{ $labels.controller }} + ``` + + - alert: ControllerReconcilePanics + expr: > + sum by (controller) (increase(controller_runtime_reconcile_panics_total[10m])) > 0 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` panicked during reconcile + description: | + Controller `{{ $labels.controller }}` recovered from {{ $value | printf "%.0f" }} panic(s) in the last 10 minutes. The reconcile that panicked was requeued, so the same owner will likely panic again on its next turn. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -A 20 "Observed a panic" + ``` + + # Queue wait time rather than queue depth: depth has no threshold that is + # right for every operator, whereas items waiting minutes for a worker is + # wrong at any scale. + - alert: ControllerWorkqueueBacklog + for: 15m + expr: > + histogram_quantile( + 0.99, + sum by (controller, le) (rate(workqueue_queue_duration_seconds_bucket[10m])) + ) > 300 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` cannot keep up with its workqueue + description: | + Items in the workqueue of controller `{{ $labels.controller }}` waited {{ $value | humanizeDuration }} (p99 over 10 minutes) before a worker picked them up. The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. + + Quick check with: + ``` + kubectl top pod -l control-plane=controller-manager + ``` + + - alert: ControllerReconcileLatencyHigh + for: 15m + expr: > + histogram_quantile( + 0.99, + sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket[10m])) + ) > 60 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` reconciles are slow + description: | + The p99 reconcile time of controller `{{ $labels.controller }}` over the last 10 minutes is {{ $value | humanizeDuration }}. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep {{ $labels.controller }} | tail -50 + ``` + + # Silent when leader election is off: the gauge is not exported then. + - alert: OperatorLeaderMissing + for: 5m + expr: > + max by (name) (leader_election_master_status) == 0 + labels: + severity: warning + annotations: + summary: >- + Operator `{{ $labels.name }}` has no leader + description: | + No replica of operator `{{ $labels.name }}` has held the leader election lease for 5 minutes, so none of its controllers is reconciling. + + Quick check with: + ``` + kubectl get lease {{ $labels.name }} -A + ``` diff --git a/observability/alerts/tests/controller_runtime_test.yaml b/observability/alerts/tests/controller_runtime_test.yaml new file mode 100644 index 00000000..488b16bb --- /dev/null +++ b/observability/alerts/tests/controller_runtime_test.yaml @@ -0,0 +1,174 @@ +# Unit tests for the controller-runtime alerting rules. Run with `make test-alerts`. +rule_files: + - ../controller_runtime.yaml + +evaluation_interval: 1m + +tests: + - name: TestControllerReconcileErrors + interval: 1m + input_series: + # database: 3 of 10 reconciles a minute fail + - series: 'controller_runtime_reconcile_total{controller="database",result="error"}' + values: '0+3x60' + - series: 'controller_runtime_reconcile_total{controller="database",result="success"}' + values: '0+7x60' + # webapp: 1 of 10 fails, under the threshold + - series: 'controller_runtime_reconcile_total{controller="webapp",result="error"}' + values: '0+1x60' + - series: 'controller_runtime_reconcile_total{controller="webapp",result="success"}' + values: '0+9x60' + alert_rule_test: + - eval_time: 10m + alertname: ControllerReconcileErrors + exp_alerts: [] + - eval_time: 26m + alertname: ControllerReconcileErrors + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` fails 30% of its reconciles + description: | + Over the last 10 minutes, 30% of the reconciles of controller `database` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep database + ``` + + - name: TestControllerReconcilePanics + interval: 1m + input_series: + - series: 'controller_runtime_reconcile_panics_total{controller="database"}' + values: '0 0 0 1+0x30' + alert_rule_test: + - eval_time: 2m + alertname: ControllerReconcilePanics + exp_alerts: [] + - eval_time: 5m + alertname: ControllerReconcilePanics + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` panicked during reconcile + description: | + Controller `database` recovered from 1 panic(s) in the last 10 minutes. The reconcile that panicked was requeued, so the same owner will likely panic again on its next turn. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -A 20 "Observed a panic" + ``` + # Out of the 10m window again + - eval_time: 20m + alertname: ControllerReconcilePanics + exp_alerts: [] + + - name: TestControllerWorkqueueBacklog + interval: 1m + input_series: + # database: items wait between 100s and 1000s before being picked up + - series: 'workqueue_queue_duration_seconds_bucket{controller="database",name="database",le="100"}' + values: '0+0x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="database",name="database",le="1000"}' + values: '0+10x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="database",name="database",le="+Inf"}' + values: '0+10x60' + # webapp: everything under 100s + - series: 'workqueue_queue_duration_seconds_bucket{controller="webapp",name="webapp",le="100"}' + values: '0+10x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="webapp",name="webapp",le="1000"}' + values: '0+10x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="webapp",name="webapp",le="+Inf"}' + values: '0+10x60' + alert_rule_test: + - eval_time: 10m + alertname: ControllerWorkqueueBacklog + exp_alerts: [] + - eval_time: 26m + alertname: ControllerWorkqueueBacklog + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` cannot keep up with its workqueue + description: | + Items in the workqueue of controller `database` waited 16m 31s (p99 over 10 minutes) before a worker picked them up. The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. + + Quick check with: + ``` + kubectl top pod -l control-plane=controller-manager + ``` + + - name: TestControllerReconcileLatencyHigh + interval: 1m + input_series: + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="60"}' + values: '0+0x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="120"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="+Inf"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="60"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="120"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="+Inf"}' + values: '0+10x60' + alert_rule_test: + - eval_time: 10m + alertname: ControllerReconcileLatencyHigh + exp_alerts: [] + - eval_time: 26m + alertname: ControllerReconcileLatencyHigh + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` reconciles are slow + description: | + The p99 reconcile time of controller `database` over the last 10 minutes is 1m 59s. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep database | tail -50 + ``` + + - name: TestOperatorLeaderMissing + interval: 1m + input_series: + # the leader steps down at 10m and nobody takes over + - series: 'leader_election_master_status{name="demo-operator"}' + values: '1+0x10 0+0x50' + alert_rule_test: + - eval_time: 5m + alertname: OperatorLeaderMissing + exp_alerts: [] + - eval_time: 12m + alertname: OperatorLeaderMissing + exp_alerts: [] + - eval_time: 16m + alertname: OperatorLeaderMissing + exp_alerts: + - exp_labels: + name: demo-operator + severity: warning + exp_annotations: + summary: >- + Operator `demo-operator` has no leader + description: | + No replica of operator `demo-operator` has held the leader election lease for 5 minutes, so none of its controllers is reconciling. + + Quick check with: + ``` + kubectl get lease demo-operator -A + ``` From 70ddc921ba2796a8544526cdc3531683982eaca1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 20:58:12 +0200 Subject: [PATCH 4/9] ci: run the alert unit tests with promtool (#184) Add an observability job to the test workflow that installs a pinned promtool and runs make test-alerts on every push and pull request. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .github/workflows/test.yml | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d6b2950..39ecf56d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,4 +25,29 @@ jobs: make test - name: Running Scaffold Gate - run: make test-scaffold \ No newline at end of file + run: make test-scaffold + observability: + name: Alerts and dashboards + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + # promtool is not part of `make all` so that contributors without it can + # still run everything else. Bump PROMETHEUS_VERSION by hand. + - name: Install promtool + env: + PROMETHEUS_VERSION: "3.14.0" + run: | + mkdir -p "$HOME/.local/bin" + curl -sSfL "https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz" \ + | tar -xz --strip-components=1 -C "$HOME/.local/bin" "prometheus-$PROMETHEUS_VERSION.linux-amd64/promtool" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Test alerts + run: make test-alerts From f5e976be140eaa2901594a755be7de9756d4bea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:09:21 +0200 Subject: [PATCH 5/9] docs(observability): explain why the workqueue rule aggregates by controller (#184) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- observability/alerts/controller_runtime.tpl.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/observability/alerts/controller_runtime.tpl.yaml b/observability/alerts/controller_runtime.tpl.yaml index 99d0e472..d2c60978 100644 --- a/observability/alerts/controller_runtime.tpl.yaml +++ b/observability/alerts/controller_runtime.tpl.yaml @@ -46,7 +46,11 @@ groups: # Queue wait time rather than queue depth: depth has no threshold that is # right for every operator, whereas items waiting minutes for a worker is - # wrong at any scale. + # wrong at any scale. controller-runtime labels every workqueue series + # with both `name` and `controller` (same value) in every version the + # framework supports (v0.22 and later), so aggregating by `controller` + # keeps one label across the reconcile rules, the workqueue rules and the + # dashboards. - alert: ControllerWorkqueueBacklog for: 15m expr: > From aa7f4d57a9372a60f2294e150202fb2e980e6e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:33:32 +0200 Subject: [PATCH 6/9] fix(observability): make the latency and backlog thresholds reachable and alert per install (#184) controller_runtime_reconcile_time_seconds has 60 as its largest finite bucket and histogram_quantile caps there, so a threshold of 60 could never fire; lower it to 30, a bucket bound. The workqueue histogram has one bucket per decade, so move the backlog threshold to 100 seconds and say in the description that the p99 value is interpolated. Add the operator's namespace to every aggregation so two installs of one operator in a cluster are alerted on separately. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../alerts/controller_runtime.tpl.yaml | 31 +++++++----- observability/alerts/ocf_resources.tpl.yaml | 20 ++++---- .../alerts/tests/controller_runtime_test.yaml | 47 ++++++++++++++++--- 3 files changed, 71 insertions(+), 27 deletions(-) diff --git a/observability/alerts/controller_runtime.tpl.yaml b/observability/alerts/controller_runtime.tpl.yaml index d2c60978..b1e74a11 100644 --- a/observability/alerts/controller_runtime.tpl.yaml +++ b/observability/alerts/controller_runtime.tpl.yaml @@ -4,6 +4,10 @@ # Thresholds are deliberately conservative and expressed as ratios or # quantiles, so that they hold whether the operator manages three owners or # three thousand. See docs/observability.md for the thresholds to tune. +# +# `namespace` is the operator's own namespace as stamped by the scrape job +# (absent when scraping outside a cluster, which is harmless), so two installs +# of one operator in a cluster are alerted on separately. groups: - name: controller-runtime rules: @@ -11,9 +15,9 @@ groups: for: 15m expr: > ( - sum by (controller) (rate(controller_runtime_reconcile_total{result="error"}[10m])) + sum by (namespace, controller) (rate(controller_runtime_reconcile_total{result="error"}[10m])) / - sum by (controller) (rate(controller_runtime_reconcile_total[10m])) + sum by (namespace, controller) (rate(controller_runtime_reconcile_total[10m])) ) > 0.25 labels: severity: warning @@ -30,7 +34,7 @@ groups: - alert: ControllerReconcilePanics expr: > - sum by (controller) (increase(controller_runtime_reconcile_panics_total[10m])) > 0 + sum by (namespace, controller) (increase(controller_runtime_reconcile_panics_total[10m])) > 0 labels: severity: warning annotations: @@ -50,21 +54,24 @@ groups: # with both `name` and `controller` (same value) in every version the # framework supports (v0.22 and later), so aggregating by `controller` # keeps one label across the reconcile rules, the workqueue rules and the - # dashboards. + # dashboards. The workqueue histogram has one bucket per decade, so the + # threshold sits on a bucket bound: a p99 above 100 seconds means more + # than one percent of items waited longer than 100 seconds. The value in + # the description is interpolated within that bucket. - alert: ControllerWorkqueueBacklog for: 15m expr: > histogram_quantile( 0.99, - sum by (controller, le) (rate(workqueue_queue_duration_seconds_bucket[10m])) - ) > 300 + sum by (namespace, controller, le) (rate(workqueue_queue_duration_seconds_bucket[10m])) + ) > 100 labels: severity: warning annotations: summary: >- Controller `{{ $labels.controller }}` cannot keep up with its workqueue description: | - Items in the workqueue of controller `{{ $labels.controller }}` waited {{ $value | humanizeDuration }} (p99 over 10 minutes) before a worker picked them up. The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. + More than one percent of the items in the workqueue of controller `{{ $labels.controller }}` waited longer than 100 seconds before a worker picked them up (p99 wait {{ $value | humanizeDuration }}, interpolated within the histogram bucket). The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. Quick check with: ``` @@ -76,15 +83,15 @@ groups: expr: > histogram_quantile( 0.99, - sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket[10m])) - ) > 60 + sum by (namespace, controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket[10m])) + ) > 30 labels: severity: warning annotations: summary: >- - Controller `{{ $labels.controller }}` reconciles are slow + Controller `{{ $labels.controller }}` reconciles are slow, p99 above 30 seconds description: | - The p99 reconcile time of controller `{{ $labels.controller }}` over the last 10 minutes is {{ $value | humanizeDuration }}. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. + The p99 reconcile time of controller `{{ $labels.controller }}` over the last 10 minutes is {{ $value | humanizeDuration }}, above the 30 second threshold. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. Quick check with: ``` @@ -95,7 +102,7 @@ groups: - alert: OperatorLeaderMissing for: 5m expr: > - max by (name) (leader_election_master_status) == 0 + max by (namespace, name) (leader_election_master_status) == 0 labels: severity: warning annotations: diff --git a/observability/alerts/ocf_resources.tpl.yaml b/observability/alerts/ocf_resources.tpl.yaml index fd3a9673..6fb19d28 100644 --- a/observability/alerts/ocf_resources.tpl.yaml +++ b/observability/alerts/ocf_resources.tpl.yaml @@ -11,6 +11,10 @@ # `updated` rate above zero indefinitely; what distinguishes a resource that # never converges is that more than half of its applies rewrite it, at a rate # no human produces. See docs/observability.md for the thresholds to tune. +# +# `namespace` is the operator's own namespace as stamped by the scrape job +# (absent when scraping outside a cluster, which is harmless), so two installs +# of one operator in a cluster are alerted on separately. groups: - name: ocf-resources rules: @@ -26,16 +30,16 @@ groups: for: 15m expr: > ( - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( rate(ocf_resource_apply_total{operation="updated"}[15m]) ) / - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( rate(ocf_resource_apply_total[15m]) ) ) > 0.5 and - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( increase(ocf_resource_apply_total{operation="updated"}[15m]) ) > 15 labels: @@ -62,28 +66,28 @@ groups: for: 15m expr: > ( - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( rate(ocf_resource_apply_errors_total[15m]) ) / ( - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( rate(ocf_resource_apply_errors_total[15m]) ) + ( - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( rate(ocf_resource_apply_total[15m]) ) or - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( rate(ocf_resource_apply_errors_total[15m]) ) * 0 ) ) ) > 0.5 and - sum by (controller, owner_kind, component, resource, kind) ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( increase(ocf_resource_apply_errors_total[15m]) ) > 5 labels: diff --git a/observability/alerts/tests/controller_runtime_test.yaml b/observability/alerts/tests/controller_runtime_test.yaml index 488b16bb..68de1bbb 100644 --- a/observability/alerts/tests/controller_runtime_test.yaml +++ b/observability/alerts/tests/controller_runtime_test.yaml @@ -39,6 +39,39 @@ tests: kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep database ``` + # Two installs of the same operator in one cluster share controller names. + # The scrape job stamps each with its own namespace, so the broken install + # must fire on its own and the healthy one must not be dragged along. + - name: TestControllerReconcileErrorsPerInstall + interval: 1m + input_series: + - series: 'controller_runtime_reconcile_total{namespace="team-a",controller="database",result="error"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_total{namespace="team-a",controller="database",result="success"}' + values: '0+0x60' + - series: 'controller_runtime_reconcile_total{namespace="team-b",controller="database",result="error"}' + values: '0+0x60' + - series: 'controller_runtime_reconcile_total{namespace="team-b",controller="database",result="success"}' + values: '0+40x60' + alert_rule_test: + - eval_time: 26m + alertname: ControllerReconcileErrors + exp_alerts: + - exp_labels: + namespace: team-a + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` fails 100% of its reconciles + description: | + Over the last 10 minutes, 100% of the reconciles of controller `database` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep database + ``` + - name: TestControllerReconcilePanics interval: 1m input_series: @@ -100,7 +133,7 @@ tests: summary: >- Controller `database` cannot keep up with its workqueue description: | - Items in the workqueue of controller `database` waited 16m 31s (p99 over 10 minutes) before a worker picked them up. The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. + More than one percent of the items in the workqueue of controller `database` waited longer than 100 seconds before a worker picked them up (p99 wait 16m 31s, interpolated within the histogram bucket). The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. Quick check with: ``` @@ -110,15 +143,15 @@ tests: - name: TestControllerReconcileLatencyHigh interval: 1m input_series: - - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="60"}' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="30"}' values: '0+0x60' - - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="120"}' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="60"}' values: '0+10x60' - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="+Inf"}' values: '0+10x60' - - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="60"}' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="30"}' values: '0+10x60' - - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="120"}' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="60"}' values: '0+10x60' - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="+Inf"}' values: '0+10x60' @@ -134,9 +167,9 @@ tests: severity: warning exp_annotations: summary: >- - Controller `database` reconciles are slow + Controller `database` reconciles are slow, p99 above 30 seconds description: | - The p99 reconcile time of controller `database` over the last 10 minutes is 1m 59s. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. + The p99 reconcile time of controller `database` over the last 10 minutes is 59.7s, above the 30 second threshold. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. Quick check with: ``` From 51ed6a1dde17f3ca450d5b6ea01aa39978f29b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:34:36 +0200 Subject: [PATCH 7/9] fix(observability): correct the leader and apply failure quick checks and simplify the failure ratio (#184) A lease name cannot be combined with the all-namespaces flag, so the leader quick check uses a field selector. The framework records no event for a failed apply, so the apply failure quick check lists the owners' Ready conditions instead. Write the failure ratio as errors over (errors + applies) or errors, and say that OperatorLeaderMissing is also silent when no replica is alive to export the gauge. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../alerts/controller_runtime.tpl.yaml | 7 +++-- observability/alerts/ocf_resources.tpl.yaml | 26 +++++++++---------- .../alerts/tests/controller_runtime_test.yaml | 2 +- .../alerts/tests/ocf_resources_test.yaml | 8 +++--- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/observability/alerts/controller_runtime.tpl.yaml b/observability/alerts/controller_runtime.tpl.yaml index b1e74a11..d3b34734 100644 --- a/observability/alerts/controller_runtime.tpl.yaml +++ b/observability/alerts/controller_runtime.tpl.yaml @@ -98,7 +98,10 @@ groups: kubectl logs -l control-plane=controller-manager --all-containers | grep {{ $labels.controller }} | tail -50 ``` - # Silent when leader election is off: the gauge is not exported then. + # Silent when leader election is off (the gauge is not exported) and when + # no replica is alive to export it, for example a crash loop before the + # elector starts; pair it with your platform's target-down alert for the + # operator job. - alert: OperatorLeaderMissing for: 5m expr: > @@ -113,5 +116,5 @@ groups: Quick check with: ``` - kubectl get lease {{ $labels.name }} -A + kubectl get lease -A --field-selector metadata.name={{ $labels.name }} ``` diff --git a/observability/alerts/ocf_resources.tpl.yaml b/observability/alerts/ocf_resources.tpl.yaml index 6fb19d28..e1b30843 100644 --- a/observability/alerts/ocf_resources.tpl.yaml +++ b/observability/alerts/ocf_resources.tpl.yaml @@ -59,9 +59,9 @@ groups: # Applies of one resource type mostly failing. Transient conflicts among # successful applies stay under the threshold, and a resource failing for - # a single owner among many is the condition rules' job. The `or … * 0` - # keeps the ratio defined for a resource that has never applied - # successfully, where no success series exists to add. + # a single owner among many is the condition rules' job. The `or` keeps + # the ratio defined for a resource that has never applied successfully, + # where no success series exists to add. - alert: ManagedResourceApplyFailing for: 15m expr: > @@ -71,18 +71,18 @@ groups: ) / ( - sum by (namespace, controller, owner_kind, component, resource, kind) ( - rate(ocf_resource_apply_errors_total[15m]) - ) - + ( sum by (namespace, controller, owner_kind, component, resource, kind) ( - rate(ocf_resource_apply_total[15m]) + rate(ocf_resource_apply_errors_total[15m]) ) - or + + sum by (namespace, controller, owner_kind, component, resource, kind) ( - rate(ocf_resource_apply_errors_total[15m]) - ) * 0 + rate(ocf_resource_apply_total[15m]) + ) + ) + or + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) ) ) ) > 0.5 @@ -98,9 +98,9 @@ groups: description: | Over the last 15 minutes, {{ $value | humanizePercentage }} of the apply attempts of the `{{ $labels.kind }}` resource `{{ $labels.resource }}` in component `{{ $labels.component }}` of owner kind `{{ $labels.owner_kind }}` (controller `{{ $labels.controller }}`) failed. - The counter covers every failure of an attempt: building the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition; this rule says the failure is systematic for the resource type rather than one owner's bad day. + The counter covers every failure of an attempt: mutating the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition, which is where the failure message lives; this rule says the failure is systematic for the resource type rather than one owner's bad day. Quick check with: ``` - kubectl get events -A --field-selector type=Warning | grep {{ $labels.kind }} + kubectl get {{ $labels.owner_kind }} -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,REASON:.status.conditions[?(@.type=="Ready")].reason,MESSAGE:.status.conditions[?(@.type=="Ready")].message' ``` diff --git a/observability/alerts/tests/controller_runtime_test.yaml b/observability/alerts/tests/controller_runtime_test.yaml index 68de1bbb..54d7b740 100644 --- a/observability/alerts/tests/controller_runtime_test.yaml +++ b/observability/alerts/tests/controller_runtime_test.yaml @@ -203,5 +203,5 @@ tests: Quick check with: ``` - kubectl get lease demo-operator -A + kubectl get lease -A --field-selector metadata.name=demo-operator ``` diff --git a/observability/alerts/tests/ocf_resources_test.yaml b/observability/alerts/tests/ocf_resources_test.yaml index 0efa4593..a70a158c 100644 --- a/observability/alerts/tests/ocf_resources_test.yaml +++ b/observability/alerts/tests/ocf_resources_test.yaml @@ -106,11 +106,11 @@ tests: description: | Over the last 15 minutes, 75% of the apply attempts of the `PersistentVolumeClaim` resource `pvc` in component `storage` of owner kind `Database` (controller `database`) failed. - The counter covers every failure of an attempt: building the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition; this rule says the failure is systematic for the resource type rather than one owner's bad day. + The counter covers every failure of an attempt: mutating the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition, which is where the failure message lives; this rule says the failure is systematic for the resource type rather than one owner's bad day. Quick check with: ``` - kubectl get events -A --field-selector type=Warning | grep PersistentVolumeClaim + kubectl get Database -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,REASON:.status.conditions[?(@.type=="Ready")].reason,MESSAGE:.status.conditions[?(@.type=="Ready")].message' ``` # A resource that has only ever failed: no success series exists at all. The @@ -137,11 +137,11 @@ tests: description: | Over the last 15 minutes, 100% of the apply attempts of the `PersistentVolumeClaim` resource `pvc` in component `storage` of owner kind `Database` (controller `database`) failed. - The counter covers every failure of an attempt: building the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition; this rule says the failure is systematic for the resource type rather than one owner's bad day. + The counter covers every failure of an attempt: mutating the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition, which is where the failure message lives; this rule says the failure is systematic for the resource type rather than one owner's bad day. Quick check with: ``` - kubectl get events -A --field-selector type=Warning | grep PersistentVolumeClaim + kubectl get Database -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,REASON:.status.conditions[?(@.type=="Ready")].reason,MESSAGE:.status.conditions[?(@.type=="Ready")].message' ``` # Sporadic conflicts among many successful applies must not fire. From 66b33a6195b24dfdcae331fe5dff95072e49ab1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:36:23 +0200 Subject: [PATCH 8/9] feat(observability): install the shared rules once per cluster and set PrometheusRule metadata (#184) The controller-runtime and managed resource rules carry no metric namespace placeholder, so rendering them per operator installed a duplicate copy per operator. Keep per-operator templates as .tpl.yaml, named -, and ship the shared rules as plain .yaml files, copied through unchanged and named ocf-. Rename ocf_resources to managed_resources (group managed-resources) so the object is ocf-managed-resources. Add PROMETHEUSRULE_NAMESPACE and PROMETHEUSRULE_LABELS for ruleSelectors such as kube-prometheus-stack's release label. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- Makefile | 45 +++++++++++++++---- observability/README.md | 8 +++- ...ntime.tpl.yaml => controller_runtime.yaml} | 4 ++ ...ources.tpl.yaml => managed_resources.yaml} | 6 ++- ..._test.yaml => managed_resources_test.yaml} | 2 +- 5 files changed, 52 insertions(+), 13 deletions(-) rename observability/alerts/{controller_runtime.tpl.yaml => controller_runtime.yaml} (97%) rename observability/alerts/{ocf_resources.tpl.yaml => managed_resources.yaml} (96%) rename observability/alerts/tests/{ocf_resources_test.yaml => managed_resources_test.yaml} (99%) diff --git a/Makefile b/Makefile index 12ac0982..28390dcd 100644 --- a/Makefile +++ b/Makefile @@ -218,8 +218,13 @@ METRIC_NAMESPACE ?= unset # `exported_namespace`; override with NAMESPACE_LABEL=namespace if yours does not. NAMESPACE_LABEL ?= exported_namespace # Shape of the rendered alert files: prometheusrule (one PrometheusRule object -# per template) or rules (plain files for prometheus' rule_files). +# per rule file) or rules (plain files for prometheus' rule_files). ALERT_FORMAT ?= prometheusrule +# Optional metadata for the PrometheusRule objects: the namespace to create them +# in, and comma-separated key=value labels, for example the release label a +# kube-prometheus-stack ruleSelector matches on (PROMETHEUSRULE_LABELS=release=kps). +PROMETHEUSRULE_NAMESPACE ?= +PROMETHEUSRULE_LABELS ?= # Metric namespace the alert unit tests are written against. ALERT_TEST_NAMESPACE := test_operator @@ -254,19 +259,34 @@ alerts: ## Render the Prometheus alert rules for METRIC_NAMESPACE into observabi $(call require_metric_namespace,alerts) @echo "Rendering alerts for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL), format: $(ALERT_FORMAT))..." @mkdir -p $(OBS_OUT)/alerts - @for file in $(OBS_DIR)/alerts/*.tpl.yaml; do \ - name=$$(basename "$$file" .tpl.yaml); \ + @for file in $(OBS_DIR)/alerts/*.yaml; do \ + [ -e "$$file" ] || continue; \ + case "$$file" in \ + *.tpl.yaml) \ + name=$$(basename "$$file" .tpl.yaml); \ + rule_name="$(METRIC_NAMESPACE)-$$name" ;; \ + *) \ + name=$$(basename "$$file" .yaml); \ + rule_name="ocf-$$name" ;; \ + esac; \ + rule_name=$$(echo "$$rule_name" | tr '[:upper:]' '[:lower:]' | tr '_:' '--'); \ out="$(OBS_OUT)/alerts/$$name.yaml"; \ case "$(ALERT_FORMAT)" in \ rules) \ $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$$out" ;; \ prometheusrule) \ - rule_name=$$(echo "$(METRIC_NAMESPACE)-$$name" | tr '[:upper:]' '[:lower:]' | tr '_:' '--'); \ { \ echo "apiVersion: monitoring.coreos.com/v1"; \ echo "kind: PrometheusRule"; \ echo "metadata:"; \ echo " name: $$rule_name"; \ + [ -z "$(PROMETHEUSRULE_NAMESPACE)" ] || echo " namespace: $(PROMETHEUSRULE_NAMESPACE)"; \ + if [ -n "$(PROMETHEUSRULE_LABELS)" ]; then \ + echo " labels:"; \ + for kv in $$(echo "$(PROMETHEUSRULE_LABELS)" | tr ',' ' '); do \ + echo " $${kv%%=*}: $${kv#*=}"; \ + done; \ + fi; \ echo "spec:"; \ $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) \ | sed -e 's/^/ /' -e 's/[[:space:]]*$$//'; \ @@ -277,7 +297,7 @@ alerts: ## Render the Prometheus alert rules for METRIC_NAMESPACE into observabi done .PHONY: test-alerts -test-alerts: ## Lint and unit test the alert rule templates with promtool. +test-alerts: ## Lint and unit test the alert rules with promtool. @command -v promtool >/dev/null 2>&1 || { \ echo "Error: promtool is required to test the alert rules."; \ echo "It ships with prometheus: https://prometheus.io/download/"; \ @@ -287,10 +307,17 @@ test-alerts: ## Lint and unit test the alert rule templates with promtool. tmpdir=$$(mktemp -d "$${TMPDIR:-/tmp}/ocf-alerts.XXXXXX"); \ trap 'rm -rf "$$tmpdir"' EXIT; \ mkdir -p "$$tmpdir/tests" "$$tmpdir/namespace-label"; \ - for file in $(OBS_DIR)/alerts/*.tpl.yaml; do \ - name=$$(basename "$$file" .tpl.yaml); \ - $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),exported_namespace) > "$$tmpdir/$$name.yaml"; \ - $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),namespace) > "$$tmpdir/namespace-label/$$name.yaml"; \ + for file in $(OBS_DIR)/alerts/*.yaml; do \ + [ -e "$$file" ] || continue; \ + case "$$file" in \ + *.tpl.yaml) \ + name=$$(basename "$$file" .tpl.yaml); \ + $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),exported_namespace) > "$$tmpdir/$$name.yaml"; \ + $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),namespace) > "$$tmpdir/namespace-label/$$name.yaml" ;; \ + *) \ + cp "$$file" "$$tmpdir/"; \ + cp "$$file" "$$tmpdir/namespace-label/" ;; \ + esac; \ done; \ cp $(OBS_DIR)/alerts/tests/*.yaml "$$tmpdir/tests/"; \ echo "Linting rules..."; \ diff --git a/observability/README.md b/observability/README.md index 40929deb..f3b627f2 100644 --- a/observability/README.md +++ b/observability/README.md @@ -8,8 +8,12 @@ Render for your operator, where `METRIC_NAMESPACE` is the argument you gave `ocm make dashboards METRIC_NAMESPACE=myoperator make alerts METRIC_NAMESPACE=myoperator -Output lands in `generated/`. Add `NAMESPACE_LABEL=namespace` if your scrape keeps the exported `namespace` label, and -`ALERT_FORMAT=rules` for plain rule files instead of `PrometheusRule` objects. +Output lands in `generated/`. `generated/alerts/` contains the per-operator condition rules, named after the metric +namespace, plus the shared `ocf-*` rules for controller-runtime and the managed resource counters, which are installed +once per cluster. Add `NAMESPACE_LABEL=namespace` if your scrape keeps the exported `namespace` label, and +`ALERT_FORMAT=rules` for plain rule files instead of `PrometheusRule` objects. `PROMETHEUSRULE_NAMESPACE` and +`PROMETHEUSRULE_LABELS` set the metadata of the `PrometheusRule` objects; for kube-prometheus-stack pass +`PROMETHEUSRULE_LABELS=release=`. Run the alert unit tests with `make test-alerts` (needs `promtool`), and bring up Prometheus and Grafana with the simulator behind them with `make observability-up`. diff --git a/observability/alerts/controller_runtime.tpl.yaml b/observability/alerts/controller_runtime.yaml similarity index 97% rename from observability/alerts/controller_runtime.tpl.yaml rename to observability/alerts/controller_runtime.yaml index d3b34734..617188e6 100644 --- a/observability/alerts/controller_runtime.tpl.yaml +++ b/observability/alerts/controller_runtime.yaml @@ -1,6 +1,10 @@ # Alerting rules for the controller-runtime metrics every operator exposes: # reconcile results and latency, the workqueue, leader election. # +# These rules carry no metric namespace placeholder: they are shared by every +# operator in the cluster and installed once, as the PrometheusRule +# `ocf-controller-runtime`. +# # Thresholds are deliberately conservative and expressed as ratios or # quantiles, so that they hold whether the operator manages three owners or # three thousand. See docs/observability.md for the thresholds to tune. diff --git a/observability/alerts/ocf_resources.tpl.yaml b/observability/alerts/managed_resources.yaml similarity index 96% rename from observability/alerts/ocf_resources.tpl.yaml rename to observability/alerts/managed_resources.yaml index e1b30843..cf3fec9e 100644 --- a/observability/alerts/ocf_resources.tpl.yaml +++ b/observability/alerts/managed_resources.yaml @@ -1,6 +1,10 @@ # Alerting rules for the resource apply counters recorded by # github.com/sourcehawk/operator-component-framework/pkg/metrics. # +# These rules carry no metric namespace placeholder: they are shared by every +# operator in the cluster and installed once, as the PrometheusRule +# `ocf-managed-resources`. +# # Neither counter carries an owner name or namespace, only the operator's static # topology (controller, owner kind, component, resource identifier, kind), so # these rules fire per resource *type* rather than per owner. Per-owner signal @@ -16,7 +20,7 @@ # (absent when scraping outside a cluster, which is harmless), so two installs # of one operator in a cluster are alerted on separately. groups: - - name: ocf-resources + - name: managed-resources rules: # A managed resource rewritten on every reconcile. A converged resource # applies as `none`; one whose applies keep coming back `updated` is in a diff --git a/observability/alerts/tests/ocf_resources_test.yaml b/observability/alerts/tests/managed_resources_test.yaml similarity index 99% rename from observability/alerts/tests/ocf_resources_test.yaml rename to observability/alerts/tests/managed_resources_test.yaml index a70a158c..820df39a 100644 --- a/observability/alerts/tests/ocf_resources_test.yaml +++ b/observability/alerts/tests/managed_resources_test.yaml @@ -4,7 +4,7 @@ # series below is one resource *type* of one component, aggregated over all of # its owners. Time series values are cumulative counter samples one minute apart. rule_files: - - ../ocf_resources.yaml + - ../managed_resources.yaml evaluation_interval: 1m From e00dee74bde8b567ba3cf8dc9ba136a0230076aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:42:27 +0200 Subject: [PATCH 9/9] fix(observability): quote PrometheusRule label values (#184) Kubernetes label values must be strings, so emit them quoted; otherwise a value such as release=1 would be parsed as an integer. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 28390dcd..74443a00 100644 --- a/Makefile +++ b/Makefile @@ -284,7 +284,7 @@ alerts: ## Render the Prometheus alert rules for METRIC_NAMESPACE into observabi if [ -n "$(PROMETHEUSRULE_LABELS)" ]; then \ echo " labels:"; \ for kv in $$(echo "$(PROMETHEUSRULE_LABELS)" | tr ',' ' '); do \ - echo " $${kv%%=*}: $${kv#*=}"; \ + echo " $${kv%%=*}: \"$${kv#*=}\""; \ done; \ fi; \ echo "spec:"; \