diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d6b2950..39ecf56d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,4 +25,29 @@ jobs: make test - name: Running Scaffold Gate - run: make test-scaffold \ No newline at end of file + run: make test-scaffold + observability: + name: Alerts and dashboards + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@v7 + + - name: Setup Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + + # promtool is not part of `make all` so that contributors without it can + # still run everything else. Bump PROMETHEUS_VERSION by hand. + - name: Install promtool + env: + PROMETHEUS_VERSION: "3.14.0" + run: | + mkdir -p "$HOME/.local/bin" + curl -sSfL "https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz" \ + | tar -xz --strip-components=1 -C "$HOME/.local/bin" "prometheus-$PROMETHEUS_VERSION.linux-amd64/promtool" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Test alerts + run: make test-alerts diff --git a/.gitignore b/.gitignore index f2d6818b..a1d509ba 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,6 @@ docs/superpowers/ # MkDocs build output site/ + +# Rendered observability artifacts +observability/generated/ diff --git a/Makefile b/Makefile index 9e486268..74443a00 100644 --- a/Makefile +++ b/Makefile @@ -205,6 +205,128 @@ e2e-component: ginkgo kind-create kind-set-context ## Run component E2E tests on .PHONY: e2e-full e2e-full: kind-create kind-set-context e2e kind-delete ## Full E2E lifecycle: create cluster, test, teardown. +##@ Observability + +OBS_DIR := observability +# Render output directory. The dev stack overrides it to keep its render apart. +OBS_OUT ?= $(OBS_DIR)/generated +# Prometheus metric namespace the condition gauge was created with +# (ocm.NewOperatorConditionsGauge("")). Required for rendering. +METRIC_NAMESPACE ?= unset +# Label carrying the namespace of the custom resource. The pod scraping the +# operator usually owns `namespace`, so the exported label arrives as +# `exported_namespace`; override with NAMESPACE_LABEL=namespace if yours does not. +NAMESPACE_LABEL ?= exported_namespace +# Shape of the rendered alert files: prometheusrule (one PrometheusRule object +# per rule file) or rules (plain files for prometheus' rule_files). +ALERT_FORMAT ?= prometheusrule +# Optional metadata for the PrometheusRule objects: the namespace to create them +# in, and comma-separated key=value labels, for example the release label a +# kube-prometheus-stack ruleSelector matches on (PROMETHEUSRULE_LABELS=release=kps). +PROMETHEUSRULE_NAMESPACE ?= +PROMETHEUSRULE_LABELS ?= +# Metric namespace the alert unit tests are written against. +ALERT_TEST_NAMESPACE := test_operator + +# Fail unless METRIC_NAMESPACE was given. $(1) is the target name for the hint. +define require_metric_namespace +@[ "$(METRIC_NAMESPACE)" != "unset" ] && [ -n "$(METRIC_NAMESPACE)" ] || { \ + echo "Error: METRIC_NAMESPACE is required."; \ + echo "Usage: make $(1) METRIC_NAMESPACE=my_operator"; \ + exit 1; \ +} +endef + +# Render a template to stdout. $(1) template path, $(2) metric namespace, +# $(3) namespace label. +define render_template +sed -e 's/{{operator_namespace}}/$(2)_/g' -e 's/{{namespace_label}}/$(3)/g' $(1) +endef + +.PHONY: dashboards +dashboards: ## Render the Grafana dashboards for METRIC_NAMESPACE into observability/generated/dashboards. + $(call require_metric_namespace,dashboards) + @echo "Rendering dashboards for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL))..." + @mkdir -p $(OBS_OUT)/dashboards + @for file in $(OBS_DIR)/dashboards/*.tpl.json; do \ + [ -e "$$file" ] || continue; \ + name=$$(basename "$$file" .tpl.json); \ + $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$(OBS_OUT)/dashboards/$$name.json"; \ + done + +.PHONY: alerts +alerts: ## Render the Prometheus alert rules for METRIC_NAMESPACE into observability/generated/alerts. + $(call require_metric_namespace,alerts) + @echo "Rendering alerts for $(METRIC_NAMESPACE) (namespace label: $(NAMESPACE_LABEL), format: $(ALERT_FORMAT))..." + @mkdir -p $(OBS_OUT)/alerts + @for file in $(OBS_DIR)/alerts/*.yaml; do \ + [ -e "$$file" ] || continue; \ + case "$$file" in \ + *.tpl.yaml) \ + name=$$(basename "$$file" .tpl.yaml); \ + rule_name="$(METRIC_NAMESPACE)-$$name" ;; \ + *) \ + name=$$(basename "$$file" .yaml); \ + rule_name="ocf-$$name" ;; \ + esac; \ + rule_name=$$(echo "$$rule_name" | tr '[:upper:]' '[:lower:]' | tr '_:' '--'); \ + out="$(OBS_OUT)/alerts/$$name.yaml"; \ + case "$(ALERT_FORMAT)" in \ + rules) \ + $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) > "$$out" ;; \ + prometheusrule) \ + { \ + echo "apiVersion: monitoring.coreos.com/v1"; \ + echo "kind: PrometheusRule"; \ + echo "metadata:"; \ + echo " name: $$rule_name"; \ + [ -z "$(PROMETHEUSRULE_NAMESPACE)" ] || echo " namespace: $(PROMETHEUSRULE_NAMESPACE)"; \ + if [ -n "$(PROMETHEUSRULE_LABELS)" ]; then \ + echo " labels:"; \ + for kv in $$(echo "$(PROMETHEUSRULE_LABELS)" | tr ',' ' '); do \ + echo " $${kv%%=*}: \"$${kv#*=}\""; \ + done; \ + fi; \ + echo "spec:"; \ + $(call render_template,"$$file",$(METRIC_NAMESPACE),$(NAMESPACE_LABEL)) \ + | sed -e 's/^/ /' -e 's/[[:space:]]*$$//'; \ + } > "$$out" ;; \ + *) \ + echo "Error: ALERT_FORMAT must be prometheusrule or rules, got '$(ALERT_FORMAT)'."; exit 1 ;; \ + esac; \ + done + +.PHONY: test-alerts +test-alerts: ## Lint and unit test the alert rules with promtool. + @command -v promtool >/dev/null 2>&1 || { \ + echo "Error: promtool is required to test the alert rules."; \ + echo "It ships with prometheus: https://prometheus.io/download/"; \ + exit 1; \ + } + @set -e; \ + tmpdir=$$(mktemp -d "$${TMPDIR:-/tmp}/ocf-alerts.XXXXXX"); \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + mkdir -p "$$tmpdir/tests" "$$tmpdir/namespace-label"; \ + for file in $(OBS_DIR)/alerts/*.yaml; do \ + [ -e "$$file" ] || continue; \ + case "$$file" in \ + *.tpl.yaml) \ + name=$$(basename "$$file" .tpl.yaml); \ + $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),exported_namespace) > "$$tmpdir/$$name.yaml"; \ + $(call render_template,"$$file",$(ALERT_TEST_NAMESPACE),namespace) > "$$tmpdir/namespace-label/$$name.yaml" ;; \ + *) \ + cp "$$file" "$$tmpdir/"; \ + cp "$$file" "$$tmpdir/namespace-label/" ;; \ + esac; \ + done; \ + cp $(OBS_DIR)/alerts/tests/*.yaml "$$tmpdir/tests/"; \ + echo "Linting rules..."; \ + promtool check rules --lint=all --lint-fatal "$$tmpdir"/*.yaml; \ + echo "Linting rules with NAMESPACE_LABEL=namespace..."; \ + promtool check rules --lint=all --lint-fatal "$$tmpdir"/namespace-label/*.yaml; \ + echo "Running unit tests..."; \ + promtool test rules --diff "$$tmpdir"/tests/*.yaml + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary diff --git a/observability/README.md b/observability/README.md new file mode 100644 index 00000000..f3b627f2 --- /dev/null +++ b/observability/README.md @@ -0,0 +1,19 @@ +# Observability + +Grafana dashboards and Prometheus alert rules for operators built on the framework, plus a local stack to look at them. +Full documentation: [docs/observability.md](../docs/observability.md). + +Render for your operator, where `METRIC_NAMESPACE` is the argument you gave `ocm.NewOperatorConditionsGauge`: + + make dashboards METRIC_NAMESPACE=myoperator + make alerts METRIC_NAMESPACE=myoperator + +Output lands in `generated/`. `generated/alerts/` contains the per-operator condition rules, named after the metric +namespace, plus the shared `ocf-*` rules for controller-runtime and the managed resource counters, which are installed +once per cluster. Add `NAMESPACE_LABEL=namespace` if your scrape keeps the exported `namespace` label, and +`ALERT_FORMAT=rules` for plain rule files instead of `PrometheusRule` objects. `PROMETHEUSRULE_NAMESPACE` and +`PROMETHEUSRULE_LABELS` set the metadata of the `PrometheusRule` objects; for kube-prometheus-stack pass +`PROMETHEUSRULE_LABELS=release=`. + +Run the alert unit tests with `make test-alerts` (needs `promtool`), and bring up Prometheus and Grafana with the +simulator behind them with `make observability-up`. diff --git a/observability/alerts/controller_runtime.yaml b/observability/alerts/controller_runtime.yaml new file mode 100644 index 00000000..617188e6 --- /dev/null +++ b/observability/alerts/controller_runtime.yaml @@ -0,0 +1,124 @@ +# Alerting rules for the controller-runtime metrics every operator exposes: +# reconcile results and latency, the workqueue, leader election. +# +# These rules carry no metric namespace placeholder: they are shared by every +# operator in the cluster and installed once, as the PrometheusRule +# `ocf-controller-runtime`. +# +# Thresholds are deliberately conservative and expressed as ratios or +# quantiles, so that they hold whether the operator manages three owners or +# three thousand. See docs/observability.md for the thresholds to tune. +# +# `namespace` is the operator's own namespace as stamped by the scrape job +# (absent when scraping outside a cluster, which is harmless), so two installs +# of one operator in a cluster are alerted on separately. +groups: + - name: controller-runtime + rules: + - alert: ControllerReconcileErrors + for: 15m + expr: > + ( + sum by (namespace, controller) (rate(controller_runtime_reconcile_total{result="error"}[10m])) + / + sum by (namespace, controller) (rate(controller_runtime_reconcile_total[10m])) + ) > 0.25 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` fails {{ $value | humanizePercentage }} of its reconciles + description: | + Over the last 10 minutes, {{ $value | humanizePercentage }} of the reconciles of controller `{{ $labels.controller }}` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep {{ $labels.controller }} + ``` + + - alert: ControllerReconcilePanics + expr: > + sum by (namespace, controller) (increase(controller_runtime_reconcile_panics_total[10m])) > 0 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` panicked during reconcile + description: | + Controller `{{ $labels.controller }}` recovered from {{ $value | printf "%.0f" }} panic(s) in the last 10 minutes. The reconcile that panicked was requeued, so the same owner will likely panic again on its next turn. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -A 20 "Observed a panic" + ``` + + # Queue wait time rather than queue depth: depth has no threshold that is + # right for every operator, whereas items waiting minutes for a worker is + # wrong at any scale. controller-runtime labels every workqueue series + # with both `name` and `controller` (same value) in every version the + # framework supports (v0.22 and later), so aggregating by `controller` + # keeps one label across the reconcile rules, the workqueue rules and the + # dashboards. The workqueue histogram has one bucket per decade, so the + # threshold sits on a bucket bound: a p99 above 100 seconds means more + # than one percent of items waited longer than 100 seconds. The value in + # the description is interpolated within that bucket. + - alert: ControllerWorkqueueBacklog + for: 15m + expr: > + histogram_quantile( + 0.99, + sum by (namespace, controller, le) (rate(workqueue_queue_duration_seconds_bucket[10m])) + ) > 100 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` cannot keep up with its workqueue + description: | + More than one percent of the items in the workqueue of controller `{{ $labels.controller }}` waited longer than 100 seconds before a worker picked them up (p99 wait {{ $value | humanizeDuration }}, interpolated within the histogram bucket). The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. + + Quick check with: + ``` + kubectl top pod -l control-plane=controller-manager + ``` + + - alert: ControllerReconcileLatencyHigh + for: 15m + expr: > + histogram_quantile( + 0.99, + sum by (namespace, controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket[10m])) + ) > 30 + labels: + severity: warning + annotations: + summary: >- + Controller `{{ $labels.controller }}` reconciles are slow, p99 above 30 seconds + description: | + The p99 reconcile time of controller `{{ $labels.controller }}` over the last 10 minutes is {{ $value | humanizeDuration }}, above the 30 second threshold. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep {{ $labels.controller }} | tail -50 + ``` + + # Silent when leader election is off (the gauge is not exported) and when + # no replica is alive to export it, for example a crash loop before the + # elector starts; pair it with your platform's target-down alert for the + # operator job. + - alert: OperatorLeaderMissing + for: 5m + expr: > + max by (namespace, name) (leader_election_master_status) == 0 + labels: + severity: warning + annotations: + summary: >- + Operator `{{ $labels.name }}` has no leader + description: | + No replica of operator `{{ $labels.name }}` has held the leader election lease for 5 minutes, so none of its controllers is reconciling. + + Quick check with: + ``` + kubectl get lease -A --field-selector metadata.name={{ $labels.name }} + ``` diff --git a/observability/alerts/crd_conditions.tpl.yaml b/observability/alerts/crd_conditions.tpl.yaml new file mode 100644 index 00000000..ed559f5b --- /dev/null +++ b/observability/alerts/crd_conditions.tpl.yaml @@ -0,0 +1,166 @@ +# Alerting rules for the status condition metrics the framework records through +# pkg/metrics (github.com/sourcehawk/go-crd-condition-metrics under the hood). +# +# See docs/observability.md for how these rules are rendered and which +# thresholds to tune. +# +# Every rule aggregates the metric instead of matching series directly. That is +# deliberate and load bearing: +# +# * it drops the reason, status and id labels, so a controller that keeps +# changing the reason while a resource stays unhealthy does not restart the +# `for:` clock every time, +# * max() keeps the value a lastTransitionTime, the freshest one, so a former +# leader pod still exporting a stale series cannot skew it the way sum() +# would. +groups: + - name: crd-conditions + rules: + # A resource whose Ready condition has been False for too long. + # + # Scoped to Ready on purpose. Matching `status="False"` across every + # condition would fire forever on negative polarity conditions such as + # Degraded or FailedToProvision, where False is the healthy state. Add + # your own positive polarity conditions to the matcher below. + - alert: CustomResourceNotReady + for: 30m + expr: > + max ( + {{operator_namespace}}controller_condition{condition="Ready", status="False"} + ) by (controller, kind, name, {{namespace_label}}) + labels: + severity: warning + annotations: + summary: >- + {{- if $labels.{{namespace_label}} -}} + {{ $labels.kind }} CR `{{ $labels.{{namespace_label}} }}/{{ $labels.name }}` has not been ready for 30 minutes + {{- else -}} + {{ $labels.kind }} CR `{{ $labels.name }}` has not been ready for 30 minutes + {{- end -}} + description: | + {{- if $labels.{{namespace_label}} -}} + Custom Resource of kind `{{ $labels.kind }}` named `{{ $labels.name }}` in namespace + `{{ $labels.{{namespace_label}} }}` has not been ready for more than 30 minutes. + {{- else -}} + Custom Resource of kind `{{ $labels.kind }}` named `{{ $labels.name }}` (cluster scoped + resource) has not been ready for more than 30 minutes. + {{- end }} + + The Ready condition last transitioned at {{ $value | humanizeTimestamp }}. + + Quick check with: + ``` + {{- if $labels.{{namespace_label}} }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} -n {{ $labels.{{namespace_label}} }} + {{- else }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} + {{- end }} + ``` + # Deep link into the CRD Conditions Browser dashboard shipped in this + # repository. The id label is "/", which for a cluster + # scoped resource is "/", so the empty namespace renders correctly + # here without a conditional. + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind + }}&var-condition=Ready&var-status=False&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} + + # A condition the controller cannot determine the state of. Unlike the two + # rules around it this one is not scoped to a single condition: Unknown is + # bad whatever the condition's polarity is. + - alert: CustomResourceConditionUnknown + for: 30m + expr: > + max ( + {{operator_namespace}}controller_condition{status="Unknown"} + ) by (controller, kind, name, condition, {{namespace_label}}) + labels: + severity: warning + annotations: + summary: >- + {{- if $labels.{{namespace_label}} -}} + {{ $labels.kind }} CR `{{ $labels.{{namespace_label}} }}/{{ $labels.name }}` has had an unknown {{ $labels.condition }} condition for 30 minutes + {{- else -}} + {{ $labels.kind }} CR `{{ $labels.name }}` has had an unknown {{ $labels.condition }} condition for 30 minutes + {{- end -}} + description: | + {{- if $labels.{{namespace_label}} -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` in + namespace `{{ $labels.{{namespace_label}} }}` has had status Unknown for more than 30 minutes. + {{- else -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` + (cluster scoped resource) has had status Unknown for more than 30 minutes. + {{- end }} + + The controller is failing to determine the state of this resource, so neither a healthy + nor an unhealthy signal can be trusted for it. The condition last transitioned at + {{ $value | humanizeTimestamp }}. + + Quick check with: + ``` + {{- if $labels.{{namespace_label}} }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} -n {{ $labels.{{namespace_label}} }} + {{- else }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} + {{- end }} + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + $labels.condition }}&var-status=Unknown&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} + + # A resource that has sat in a non-ready state for hours. + # + # This rule is the reason the metric value is a lastTransitionTime. It needs + # no `for:` clause because the expression is itself a duration comparison, + # and that difference is the point: a `for:` clause measures how long the + # *alert* has been true, which is bounded by how long the series has been + # continuously present, whereas this measures how long the *resource* has + # actually been in this state according to its own status. It therefore + # survives scrape gaps, operator restarts and ruler restarts, any of which + # silently restart a `for:` clock, and reports the real age of the state. + # It also covers status Unknown, which CustomResourceNotReady does not. + # + # Scoped to Ready for the same polarity reason as CustomResourceNotReady. + # The condition label is kept in the aggregation so widening the matcher to + # your own conditions stays a one line change. + - alert: CustomResourceConditionStuck + expr: > + ( + time() + - + max ( + {{operator_namespace}}controller_condition{condition="Ready", status!="True"} + ) by (controller, kind, name, condition, {{namespace_label}}) + ) > 21600 + labels: + severity: warning + annotations: + summary: >- + {{- if $labels.{{namespace_label}} -}} + {{ $labels.kind }} CR `{{ $labels.{{namespace_label}} }}/{{ $labels.name }}` has been stuck not ready for {{ $value | humanizeDuration }} + {{- else -}} + {{ $labels.kind }} CR `{{ $labels.name }}` has been stuck not ready for {{ $value | humanizeDuration }} + {{- end -}} + description: | + {{- if $labels.{{namespace_label}} -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` in + namespace `{{ $labels.{{namespace_label}} }}` has not been True for + {{ $value | humanizeDuration }}. + {{- else -}} + The `{{ $labels.condition }}` condition of `{{ $labels.kind }}` `{{ $labels.name }}` + (cluster scoped resource) has not been True for {{ $value | humanizeDuration }}. + {{- end }} + + Nothing has moved for hours, so the controller is either wedged or waiting on something + that is never going to arrive. Reconciliation retries alone will not clear this. + + Quick check with: + ``` + {{- if $labels.{{namespace_label}} }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} -n {{ $labels.{{namespace_label}} }} + {{- else }} + kubectl describe {{ $labels.kind }}/{{ $labels.name }} + {{- end }} + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + $labels.condition }}&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} diff --git a/observability/alerts/managed_resources.yaml b/observability/alerts/managed_resources.yaml new file mode 100644 index 00000000..cf3fec9e --- /dev/null +++ b/observability/alerts/managed_resources.yaml @@ -0,0 +1,110 @@ +# Alerting rules for the resource apply counters recorded by +# github.com/sourcehawk/operator-component-framework/pkg/metrics. +# +# These rules carry no metric namespace placeholder: they are shared by every +# operator in the cluster and installed once, as the PrometheusRule +# `ocf-managed-resources`. +# +# Neither counter carries an owner name or namespace, only the operator's static +# topology (controller, owner kind, component, resource identifier, kind), so +# these rules fire per resource *type* rather than per owner. Per-owner signal +# comes from the condition rules in crd_conditions.tpl.yaml. +# +# Both rules are ratios of the resource's own applies rather than absolute +# rates. At scale, legitimate spec changes across many owners keep the absolute +# `updated` rate above zero indefinitely; what distinguishes a resource that +# never converges is that more than half of its applies rewrite it, at a rate +# no human produces. See docs/observability.md for the thresholds to tune. +# +# `namespace` is the operator's own namespace as stamped by the scrape job +# (absent when scraping outside a cluster, which is harmless), so two installs +# of one operator in a cluster are alerted on separately. +groups: + - name: managed-resources + rules: + # A managed resource rewritten on every reconcile. A converged resource + # applies as `none`; one whose applies keep coming back `updated` is in a + # hot loop (apply updates the object, the watch event requeues the owner, + # the next apply updates it again) or in a fight with a webhook or another + # controller. Legitimate churn is followed by a `none` apply on the next + # reconcile, which keeps its ratio at or below one half. The floor keeps a + # single edit on an idle resource from producing a ratio of one over a + # handful of samples. + - alert: ManagedResourceNotConverging + for: 15m + expr: > + ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_total{operation="updated"}[15m]) + ) + / + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_total[15m]) + ) + ) > 0.5 + and + sum by (namespace, controller, owner_kind, component, resource, kind) ( + increase(ocf_resource_apply_total{operation="updated"}[15m]) + ) > 15 + labels: + severity: warning + annotations: + summary: >- + {{ $labels.kind }} `{{ $labels.resource }}` of component `{{ $labels.component }}` ({{ $labels.owner_kind }}) is rewritten on every reconcile + description: | + Over the last 15 minutes, {{ $value | humanizePercentage }} of the applies of the `{{ $labels.kind }}` resource `{{ $labels.resource }}` in component `{{ $labels.component }}` of owner kind `{{ $labels.owner_kind }}` (controller `{{ $labels.controller }}`) updated the object. In steady state a converged resource applies as `none` on every pass, so a resource whose every apply is an update is being rewritten on every reconcile even though nothing changed. + + Usual causes: a mutation that is not idempotent (a timestamp, a random value, an unsorted list), a defaulting webhook or another controller fighting the operator, or a field the API server normalises. Events report the same thing, but client-go's spam filter truncates them within seconds under exactly these conditions. + + Quick check with: + ``` + kubectl get events -A --field-selector reason=Updated{{ $labels.kind }} + ``` + + # Applies of one resource type mostly failing. Transient conflicts among + # successful applies stay under the threshold, and a resource failing for + # a single owner among many is the condition rules' job. The `or` keeps + # the ratio defined for a resource that has never applied successfully, + # where no success series exists to add. + - alert: ManagedResourceApplyFailing + for: 15m + expr: > + ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) + ) + / + ( + ( + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) + ) + + + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_total[15m]) + ) + ) + or + sum by (namespace, controller, owner_kind, component, resource, kind) ( + rate(ocf_resource_apply_errors_total[15m]) + ) + ) + ) > 0.5 + and + sum by (namespace, controller, owner_kind, component, resource, kind) ( + increase(ocf_resource_apply_errors_total[15m]) + ) > 5 + labels: + severity: warning + annotations: + summary: >- + Applies of {{ $labels.kind }} `{{ $labels.resource }}` of component `{{ $labels.component }}` ({{ $labels.owner_kind }}) are failing + description: | + Over the last 15 minutes, {{ $value | humanizePercentage }} of the apply attempts of the `{{ $labels.kind }}` resource `{{ $labels.resource }}` in component `{{ $labels.component }}` of owner kind `{{ $labels.owner_kind }}` (controller `{{ $labels.controller }}`) failed. + + The counter covers every failure of an attempt: mutating the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition, which is where the failure message lives; this rule says the failure is systematic for the resource type rather than one owner's bad day. + + Quick check with: + ``` + kubectl get {{ $labels.owner_kind }} -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,REASON:.status.conditions[?(@.type=="Ready")].reason,MESSAGE:.status.conditions[?(@.type=="Ready")].message' + ``` diff --git a/observability/alerts/tests/controller_runtime_test.yaml b/observability/alerts/tests/controller_runtime_test.yaml new file mode 100644 index 00000000..54d7b740 --- /dev/null +++ b/observability/alerts/tests/controller_runtime_test.yaml @@ -0,0 +1,207 @@ +# Unit tests for the controller-runtime alerting rules. Run with `make test-alerts`. +rule_files: + - ../controller_runtime.yaml + +evaluation_interval: 1m + +tests: + - name: TestControllerReconcileErrors + interval: 1m + input_series: + # database: 3 of 10 reconciles a minute fail + - series: 'controller_runtime_reconcile_total{controller="database",result="error"}' + values: '0+3x60' + - series: 'controller_runtime_reconcile_total{controller="database",result="success"}' + values: '0+7x60' + # webapp: 1 of 10 fails, under the threshold + - series: 'controller_runtime_reconcile_total{controller="webapp",result="error"}' + values: '0+1x60' + - series: 'controller_runtime_reconcile_total{controller="webapp",result="success"}' + values: '0+9x60' + alert_rule_test: + - eval_time: 10m + alertname: ControllerReconcileErrors + exp_alerts: [] + - eval_time: 26m + alertname: ControllerReconcileErrors + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` fails 30% of its reconciles + description: | + Over the last 10 minutes, 30% of the reconciles of controller `database` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep database + ``` + + # Two installs of the same operator in one cluster share controller names. + # The scrape job stamps each with its own namespace, so the broken install + # must fire on its own and the healthy one must not be dragged along. + - name: TestControllerReconcileErrorsPerInstall + interval: 1m + input_series: + - series: 'controller_runtime_reconcile_total{namespace="team-a",controller="database",result="error"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_total{namespace="team-a",controller="database",result="success"}' + values: '0+0x60' + - series: 'controller_runtime_reconcile_total{namespace="team-b",controller="database",result="error"}' + values: '0+0x60' + - series: 'controller_runtime_reconcile_total{namespace="team-b",controller="database",result="success"}' + values: '0+40x60' + alert_rule_test: + - eval_time: 26m + alertname: ControllerReconcileErrors + exp_alerts: + - exp_labels: + namespace: team-a + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` fails 100% of its reconciles + description: | + Over the last 10 minutes, 100% of the reconciles of controller `database` returned an error. Errors are requeued with backoff, so a sustained ratio this high means the controller is stuck on something rather than riding out a blip. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -i "error" | grep database + ``` + + - name: TestControllerReconcilePanics + interval: 1m + input_series: + - series: 'controller_runtime_reconcile_panics_total{controller="database"}' + values: '0 0 0 1+0x30' + alert_rule_test: + - eval_time: 2m + alertname: ControllerReconcilePanics + exp_alerts: [] + - eval_time: 5m + alertname: ControllerReconcilePanics + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` panicked during reconcile + description: | + Controller `database` recovered from 1 panic(s) in the last 10 minutes. The reconcile that panicked was requeued, so the same owner will likely panic again on its next turn. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep -A 20 "Observed a panic" + ``` + # Out of the 10m window again + - eval_time: 20m + alertname: ControllerReconcilePanics + exp_alerts: [] + + - name: TestControllerWorkqueueBacklog + interval: 1m + input_series: + # database: items wait between 100s and 1000s before being picked up + - series: 'workqueue_queue_duration_seconds_bucket{controller="database",name="database",le="100"}' + values: '0+0x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="database",name="database",le="1000"}' + values: '0+10x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="database",name="database",le="+Inf"}' + values: '0+10x60' + # webapp: everything under 100s + - series: 'workqueue_queue_duration_seconds_bucket{controller="webapp",name="webapp",le="100"}' + values: '0+10x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="webapp",name="webapp",le="1000"}' + values: '0+10x60' + - series: 'workqueue_queue_duration_seconds_bucket{controller="webapp",name="webapp",le="+Inf"}' + values: '0+10x60' + alert_rule_test: + - eval_time: 10m + alertname: ControllerWorkqueueBacklog + exp_alerts: [] + - eval_time: 26m + alertname: ControllerWorkqueueBacklog + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` cannot keep up with its workqueue + description: | + More than one percent of the items in the workqueue of controller `database` waited longer than 100 seconds before a worker picked them up (p99 wait 16m 31s, interpolated within the histogram bucket). The queue grows faster than the workers drain it: too few concurrent reconciles, reconciles that take too long, or a burst of events the controller cannot absorb. + + Quick check with: + ``` + kubectl top pod -l control-plane=controller-manager + ``` + + - name: TestControllerReconcileLatencyHigh + interval: 1m + input_series: + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="30"}' + values: '0+0x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="60"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="database",le="+Inf"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="30"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="60"}' + values: '0+10x60' + - series: 'controller_runtime_reconcile_time_seconds_bucket{controller="webapp",le="+Inf"}' + values: '0+10x60' + alert_rule_test: + - eval_time: 10m + alertname: ControllerReconcileLatencyHigh + exp_alerts: [] + - eval_time: 26m + alertname: ControllerReconcileLatencyHigh + exp_alerts: + - exp_labels: + controller: database + severity: warning + exp_annotations: + summary: >- + Controller `database` reconciles are slow, p99 above 30 seconds + description: | + The p99 reconcile time of controller `database` over the last 10 minutes is 59.7s, above the 30 second threshold. Every slow reconcile holds a worker, so sustained latency this high turns into a workqueue backlog. + + Quick check with: + ``` + kubectl logs -l control-plane=controller-manager --all-containers | grep database | tail -50 + ``` + + - name: TestOperatorLeaderMissing + interval: 1m + input_series: + # the leader steps down at 10m and nobody takes over + - series: 'leader_election_master_status{name="demo-operator"}' + values: '1+0x10 0+0x50' + alert_rule_test: + - eval_time: 5m + alertname: OperatorLeaderMissing + exp_alerts: [] + - eval_time: 12m + alertname: OperatorLeaderMissing + exp_alerts: [] + - eval_time: 16m + alertname: OperatorLeaderMissing + exp_alerts: + - exp_labels: + name: demo-operator + severity: warning + exp_annotations: + summary: >- + Operator `demo-operator` has no leader + description: | + No replica of operator `demo-operator` has held the leader election lease for 5 minutes, so none of its controllers is reconciling. + + Quick check with: + ``` + kubectl get lease -A --field-selector metadata.name=demo-operator + ``` diff --git a/observability/alerts/tests/crd_conditions_test.yaml b/observability/alerts/tests/crd_conditions_test.yaml new file mode 100644 index 00000000..4e72524e --- /dev/null +++ b/observability/alerts/tests/crd_conditions_test.yaml @@ -0,0 +1,250 @@ +# Unit tests for the generic CRD condition alerting rules. +# +# Run with `make test-alerts`, which renders ../crd_conditions.tpl.yaml with the +# metric namespace pinned to `test_operator` and the namespace label left at its +# default before handing both files to promtool. +rule_files: + - ../crd_conditions.yaml + +evaluation_interval: 1m + +tests: + # A reason change while a resource stays not ready must not restart the 30m + # clock, and a resource with no namespace label (cluster scoped) must produce + # the alternate wording. + - name: TestCustomResourceNotReady + interval: 1m + input_series: + # Backup Ready=False reason=Error1 [0m-19m] + - series: 'test_operator_controller_condition{controller="backup",kind="Backup",name="nightly-abc123",exported_namespace="backups",condition="Ready",status="False",reason="Error1",id="backups/nightly-abc123"}' + values: '1760534009+0x19' + # Backup Ready=False reason=Error2 [20m-44m]. The Error1 series is still + # within the 5m lookback window until 24m, so both are present for a while + # and max() has to pick the freshest lastTransitionTime. + - series: 'test_operator_controller_condition{controller="backup",kind="Backup",name="nightly-abc123",exported_namespace="backups",condition="Ready",status="False",reason="Error2",id="backups/nightly-abc123"}' + values: '_x20 1760534348+0x24' + # Backup Ready=True [45m-60m] + - series: 'test_operator_controller_condition{controller="backup",kind="Backup",name="nightly-abc123",exported_namespace="backups",condition="Ready",status="True",reason="Ready",id="backups/nightly-abc123"}' + values: '_x45 1760534420+0x15' + # RemoteStorage Ready=False [15m-60m], cluster scoped so no namespace label + - series: 'test_operator_controller_condition{controller="remotestorage",kind="RemoteStorage",name="document-store",condition="Ready",status="False",reason="Error",id="/document-store"}' + values: '_x15 1760582000+0x45' + + alert_rule_test: + # Nothing fires immediately (for: 30m) + - eval_time: 0m + alertname: CustomResourceNotReady + exp_alerts: [] + + # Still pending at 25m: under the threshold, and the Error1 -> Error2 + # change at 20m must not have restarted the clock. + - eval_time: 25m + alertname: CustomResourceNotReady + exp_alerts: [] + + # Backup has been not ready for 30m. RemoteStorage only started at 15m so + # it is still pending. + - eval_time: 30m + alertname: CustomResourceNotReady + exp_alerts: + - exp_labels: + controller: backup + kind: Backup + name: nightly-abc123 + exported_namespace: backups + severity: warning + exp_annotations: + summary: >- + Backup CR `backups/nightly-abc123` has not been ready for 30 minutes + description: | + Custom Resource of kind `Backup` named `nightly-abc123` in namespace + `backups` has not been ready for more than 30 minutes. + + The Ready condition last transitioned at 2025-10-15 13:19:08 +0000 UTC. + + Quick check with: + ``` + kubectl describe Backup/nightly-abc123 -n backups + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=Backup&var-condition=Ready&var-status=False&var-resource_id=backups%2Fnightly-abc123 + + # Backup recovered at 45m, so it has resolved. RemoteStorage has now been + # not ready for 30m and is the only alert left. + - eval_time: 50m + alertname: CustomResourceNotReady + exp_alerts: + - exp_labels: + controller: remotestorage + kind: RemoteStorage + name: document-store + severity: warning + exp_annotations: + summary: >- + RemoteStorage CR `document-store` has not been ready for 30 minutes + description: | + Custom Resource of kind `RemoteStorage` named `document-store` (cluster scoped + resource) has not been ready for more than 30 minutes. + + The Ready condition last transitioned at 2025-10-16 02:33:20 +0000 UTC. + + Quick check with: + ``` + kubectl describe RemoteStorage/document-store + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=RemoteStorage&var-condition=Ready&var-status=False&var-resource_id=%2Fdocument-store + + # Unknown is bad whatever the condition's polarity is, so this rule is not + # scoped to a single condition: both fixtures below must fire. + - name: TestCustomResourceConditionUnknown + interval: 1m + input_series: + - series: 'test_operator_controller_condition{controller="app",kind="App",name="checkout",exported_namespace="shop",condition="Synced",status="Unknown",reason="ProbeFailed",id="shop/checkout"}' + values: '1760600000+0x60' + - series: 'test_operator_controller_condition{controller="app",kind="App",name="checkout",exported_namespace="shop",condition="Ready",status="Unknown",reason="ProbeFailed",id="shop/checkout"}' + values: '1760600000+0x60' + + alert_rule_test: + - eval_time: 25m + alertname: CustomResourceConditionUnknown + exp_alerts: [] + + - eval_time: 30m + alertname: CustomResourceConditionUnknown + exp_alerts: + - exp_labels: + controller: app + kind: App + name: checkout + condition: Ready + exported_namespace: shop + severity: warning + exp_annotations: + summary: >- + App CR `shop/checkout` has had an unknown Ready condition for 30 minutes + description: | + The `Ready` condition of `App` `checkout` in + namespace `shop` has had status Unknown for more than 30 minutes. + + The controller is failing to determine the state of this resource, so neither a healthy + nor an unhealthy signal can be trusted for it. The condition last transitioned at + 2025-10-16 07:33:20 +0000 UTC. + + Quick check with: + ``` + kubectl describe App/checkout -n shop + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Ready&var-status=Unknown&var-resource_id=shop%2Fcheckout + - exp_labels: + controller: app + kind: App + name: checkout + condition: Synced + exported_namespace: shop + severity: warning + exp_annotations: + summary: >- + App CR `shop/checkout` has had an unknown Synced condition for 30 minutes + description: | + The `Synced` condition of `App` `checkout` in + namespace `shop` has had status Unknown for more than 30 minutes. + + The controller is failing to determine the state of this resource, so neither a healthy + nor an unhealthy signal can be trusted for it. The condition last transitioned at + 2025-10-16 07:33:20 +0000 UTC. + + Quick check with: + ``` + kubectl describe App/checkout -n shop + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Synced&var-status=Unknown&var-resource_id=shop%2Fcheckout + + # The stuck rule compares against the metric value, so the fixture timestamps + # have to live on promtool's clock, which starts at the unix epoch. A Ready + # condition that transitioned at t=60s is 6h59m old at an eval_time of 7h. + - name: TestCustomResourceConditionStuck + interval: 1m + input_series: + - series: 'test_operator_controller_condition{controller="cluster",kind="Cluster",name="prod",exported_namespace="clusters",condition="Ready",status="False",reason="Provisioning",id="clusters/prod"}' + values: '60+0x480' + + alert_rule_test: + # 5h since the transition, under the 6h threshold + - eval_time: 5h + alertname: CustomResourceConditionStuck + exp_alerts: [] + + - eval_time: 7h + alertname: CustomResourceConditionStuck + exp_alerts: + - exp_labels: + controller: cluster + kind: Cluster + name: prod + condition: Ready + exported_namespace: clusters + severity: warning + exp_annotations: + summary: >- + Cluster CR `clusters/prod` has been stuck not ready for 6h 59m 0s + description: | + The `Ready` condition of `Cluster` `prod` in + namespace `clusters` has not been True for + 6h 59m 0s. + + Nothing has moved for hours, so the controller is either wedged or waiting on something + that is never going to arrive. Reconciliation retries alone will not clear this. + + Quick check with: + ``` + kubectl describe Cluster/prod -n clusters + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fprod + + # Why CustomResourceConditionStuck exists alongside CustomResourceNotReady: + # the series disappears between 1h and 6h (operator down, scrape gap), which + # restarts the `for: 30m` clock on its return. 20m after it comes back the + # `for:` based rule is still pending, while the value based rule correctly + # reports the resource as stuck for 6h19m. + - name: TestConditionStuckSurvivesSeriesGaps + interval: 1m + input_series: + - series: 'test_operator_controller_condition{controller="cluster",kind="Cluster",name="staging",exported_namespace="clusters",condition="Ready",status="False",reason="Provisioning",id="clusters/staging"}' + values: '60+0x60 _x300 60+0x60' + + alert_rule_test: + - eval_time: 6h20m + alertname: CustomResourceNotReady + exp_alerts: [] + + - eval_time: 6h20m + alertname: CustomResourceConditionStuck + exp_alerts: + - exp_labels: + controller: cluster + kind: Cluster + name: staging + condition: Ready + exported_namespace: clusters + severity: warning + exp_annotations: + summary: >- + Cluster CR `clusters/staging` has been stuck not ready for 6h 19m 0s + description: | + The `Ready` condition of `Cluster` `staging` in + namespace `clusters` has not been True for + 6h 19m 0s. + + Nothing has moved for hours, so the controller is either wedged or waiting on something + that is never going to arrive. Reconciliation retries alone will not clear this. + + Quick check with: + ``` + kubectl describe Cluster/staging -n clusters + ``` + dashboard_url: >- + /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fstaging diff --git a/observability/alerts/tests/managed_resources_test.yaml b/observability/alerts/tests/managed_resources_test.yaml new file mode 100644 index 00000000..820df39a --- /dev/null +++ b/observability/alerts/tests/managed_resources_test.yaml @@ -0,0 +1,158 @@ +# Unit tests for the managed resource alerting rules. +# +# Run with `make test-alerts`. The counters carry no owner identity, so every +# series below is one resource *type* of one component, aggregated over all of +# its owners. Time series values are cumulative counter samples one minute apart. +rule_files: + - ../managed_resources.yaml + +evaluation_interval: 1m + +tests: + # A hot loop: every apply of the configmap rewrites it, 4 updates a second. + # The other resources of the component converge with `none`, which must not + # mask the configmap's ratio. + - name: TestManagedResourceNotConvergingHotLoop + interval: 1m + input_series: + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="updated"}' + values: '0+240x60' + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="none"}' + values: '0+1x60' + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="deployment",kind="Deployment",operation="none"}' + values: '0+240x60' + alert_rule_test: + - eval_time: 10m + alertname: ManagedResourceNotConverging + exp_alerts: [] + # ratio 240/241 over the window, floor satisfied, for: 15m elapsed + - eval_time: 31m + alertname: ManagedResourceNotConverging + exp_alerts: + - exp_labels: + controller: webapp + owner_kind: WebApp + component: server + resource: configmap + kind: ConfigMap + severity: warning + exp_annotations: + summary: >- + ConfigMap `configmap` of component `server` (WebApp) is rewritten on every reconcile + description: | + Over the last 15 minutes, 99.59% of the applies of the `ConfigMap` resource `configmap` in component `server` of owner kind `WebApp` (controller `webapp`) updated the object. In steady state a converged resource applies as `none` on every pass, so a resource whose every apply is an update is being rewritten on every reconcile even though nothing changed. + + Usual causes: a mutation that is not idempotent (a timestamp, a random value, an unsorted list), a defaulting webhook or another controller fighting the operator, or a field the API server normalises. Events report the same thing, but client-go's spam filter truncates them within seconds under exactly these conditions. + + Quick check with: + ``` + kubectl get events -A --field-selector reason=UpdatedConfigMap + ``` + + # Legitimate churn at scale: many owners change spec, every `updated` apply is + # followed by a `none` apply on the reconcile the status update triggers. The + # absolute rate is high (2 updates a minute, well over the floor) but the ratio + # sits at 0.5, so the rule must stay silent. + - name: TestManagedResourceNotConvergingChurnDoesNotFire + interval: 1m + input_series: + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="updated"}' + values: '0+2x60' + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="none"}' + values: '0+2x60' + alert_rule_test: + - eval_time: 45m + alertname: ManagedResourceNotConverging + exp_alerts: [] + + # One edit on an otherwise idle resource: ratio is 1 but only one update in + # the window, under the floor. + - name: TestManagedResourceNotConvergingSingleEditDoesNotFire + interval: 1m + input_series: + - series: 'ocf_resource_apply_total{controller="webapp",owner_kind="WebApp",component="server",resource="configmap",kind="ConfigMap",operation="updated"}' + values: '0 0 0 1+0x57' + alert_rule_test: + - eval_time: 30m + alertname: ManagedResourceNotConverging + exp_alerts: [] + + # A PVC whose applies fail three times out of four, on a resource that also + # has the occasional successful apply. + - name: TestManagedResourceApplyFailing + interval: 1m + input_series: + - series: 'ocf_resource_apply_errors_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim"}' + values: '0+3x60' + - series: 'ocf_resource_apply_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim",operation="none"}' + values: '0+1x60' + alert_rule_test: + - eval_time: 10m + alertname: ManagedResourceApplyFailing + exp_alerts: [] + - eval_time: 31m + alertname: ManagedResourceApplyFailing + exp_alerts: + - exp_labels: + controller: database + owner_kind: Database + component: storage + resource: pvc + kind: PersistentVolumeClaim + severity: warning + exp_annotations: + summary: >- + Applies of PersistentVolumeClaim `pvc` of component `storage` (Database) are failing + description: | + Over the last 15 minutes, 75% of the apply attempts of the `PersistentVolumeClaim` resource `pvc` in component `storage` of owner kind `Database` (controller `database`) failed. + + The counter covers every failure of an attempt: mutating the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition, which is where the failure message lives; this rule says the failure is systematic for the resource type rather than one owner's bad day. + + Quick check with: + ``` + kubectl get Database -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,REASON:.status.conditions[?(@.type=="Ready")].reason,MESSAGE:.status.conditions[?(@.type=="Ready")].message' + ``` + + # A resource that has only ever failed: no success series exists at all. The + # ratio must still evaluate (to 1) rather than vanish on a missing vector. + - name: TestManagedResourceApplyFailingWithoutSuccesses + interval: 1m + input_series: + - series: 'ocf_resource_apply_errors_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim"}' + values: '0+2x60' + alert_rule_test: + - eval_time: 31m + alertname: ManagedResourceApplyFailing + exp_alerts: + - exp_labels: + controller: database + owner_kind: Database + component: storage + resource: pvc + kind: PersistentVolumeClaim + severity: warning + exp_annotations: + summary: >- + Applies of PersistentVolumeClaim `pvc` of component `storage` (Database) are failing + description: | + Over the last 15 minutes, 100% of the apply attempts of the `PersistentVolumeClaim` resource `pvc` in component `storage` of owner kind `Database` (controller `database`) failed. + + The counter covers every failure of an attempt: mutating the desired object, the server-side apply patch, and the classification that follows it. The owners whose resource fails report the error on their Ready condition, which is where the failure message lives; this rule says the failure is systematic for the resource type rather than one owner's bad day. + + Quick check with: + ``` + kubectl get Database -A -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,REASON:.status.conditions[?(@.type=="Ready")].reason,MESSAGE:.status.conditions[?(@.type=="Ready")].message' + ``` + + # Sporadic conflicts among many successful applies must not fire. + - name: TestManagedResourceApplyFailingSporadicDoesNotFire + interval: 1m + input_series: + - series: 'ocf_resource_apply_errors_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim"}' + values: '0+1x60' + - series: 'ocf_resource_apply_total{controller="database",owner_kind="Database",component="storage",resource="pvc",kind="PersistentVolumeClaim",operation="none"}' + values: '0+9x60' + alert_rule_test: + - eval_time: 45m + alertname: ManagedResourceApplyFailing + exp_alerts: []