From 49a8001b4d644b00a1a04b87cf49beca6af2859d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:12:10 +0200 Subject: [PATCH 1/9] feat(observability): add the conditions browser dashboard with stale-series protection (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../crd_conditions_browser.tpl.json | 750 ++++++++++++++++++ 1 file changed, 750 insertions(+) create mode 100644 observability/dashboards/crd_conditions_browser.tpl.json diff --git a/observability/dashboards/crd_conditions_browser.tpl.json b/observability/dashboards/crd_conditions_browser.tpl.json new file mode 100644 index 00000000..dc191f61 --- /dev/null +++ b/observability/dashboards/crd_conditions_browser.tpl.json @@ -0,0 +1,750 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard to monitor operator custom resource conditions", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [ + { + "title": "Operator overview", + "type": "link", + "url": "/d/ocf_operator/ocf-operator", + "keepTime": true + } + ], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "Operator Conditions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Conditions matching the given filter criteria ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "legendFormat": "Total", + "range": true, + "refId": "A" + } + ], + "title": "Total Conditions Matching", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Table of $kind conditions of type $condition", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "applyToRow": false, + "mode": "basic", + "type": "color-background", + "wrapText": true + }, + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 397 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeFromNow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "False": { + "color": "red", + "index": 1, + "text": "False" + }, + "True": { + "color": "semi-dark-green", + "index": 0, + "text": "True" + }, + "Unknown": { + "color": "semi-dark-yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n {{operator_namespace}}controller_condition{\n kind=\"$kind\",\n condition=~\"$condition\",\n status=~\"$status\",\n reason=~\"$reason\",\n id=~\"$resource_id\",\n {{namespace_label}}=~\"$namespace\"\n }\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n) by (name, {{namespace_label}}, condition, status, reason)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "$kind Conditions", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "value_ms", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "Value" + } + }, + "operator": "*", + "right": { + "fixed": "1000" + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "value_ms" + } + ], + "fields": {} + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "{{namespace_label}}": 1, + "name": 2, + "condition": 3, + "status": 4, + "reason": 5, + "value_ms": 6, + "Value": 7 + }, + "renameByName": { + "Value": "", + "condition": "Condition", + "{{namespace_label}}": "Namespace", + "name": "Name", + "reason": "Reason", + "status": "Status", + "value_ms": "Since" + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 9, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of $kind $condition Conditions in a False state", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 7, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "repeat": "condition", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"False\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "legendFormat": "Total", + "range": true, + "refId": "A" + } + ], + "title": "$condition=False", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of $kind $condition Conditions in a Unknown state", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-yellow", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 11, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "repeat": "condition", + "repeatDirection": "h", + "targets": [ + { + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"Unknown\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "legendFormat": "Total", + "range": true, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + } + } + ], + "title": "$condition=Unknown", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of $kind $condition Conditions in a True state", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 8, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "repeat": "condition", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"True\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "legendFormat": "Total", + "range": true, + "refId": "A" + } + ], + "title": "$condition=True", + "type": "stat" + } + ], + "title": "Status Counts Per Condition", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [ + "ocf" + ], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "description": "The data source to get the metrics from", + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition,kind)", + "description": "The resource kind to view", + "label": "Kind", + "name": "kind", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition,kind)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\"},condition)", + "description": "The condition type to display", + "includeAll": true, + "label": "Condition", + "multi": true, + "name": "condition", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\"},condition)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"},status)", + "description": "Filter for a specific status", + "includeAll": true, + "label": "Status", + "multi": true, + "name": "status", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"},status)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},reason)", + "description": "Filter for condition reason", + "includeAll": true, + "label": "Reason", + "multi": true, + "name": "reason", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},reason)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "query_result(sum ({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"} > 0) by ({{namespace_label}}))", + "description": "The namespaces in which to list the resources", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(sum ({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"} > 0) by ({{namespace_label}}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "^.*{{namespace_label}}=\"([^\"]+)\".*$", + "sort": 1, + "type": "query" + }, + { + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},id)", + "description": "Namespace and resource name combination", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "resource_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "label": "Resource" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "CRD Conditions Browser", + "uid": "crd_conditions_browser", + "version": 1 +} From f4573fd8692decf3d767651facb8b2f3b526ba87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:12:10 +0200 Subject: [PATCH 2/9] feat(observability): add the operator overview dashboard (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../dashboards/ocf_operator.tpl.json | 3332 +++++++++++++++++ 1 file changed, 3332 insertions(+) create mode 100644 observability/dashboards/ocf_operator.tpl.json diff --git a/observability/dashboards/ocf_operator.tpl.json b/observability/dashboards/ocf_operator.tpl.json new file mode 100644 index 00000000..52cc41ec --- /dev/null +++ b/observability/dashboards/ocf_operator.tpl.json @@ -0,0 +1,3332 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Health of an operator built on the operator-component-framework: reconciliation, workqueue, managed resource applies, owner conditions, API client and process.", + "editable": true, + "graphTooltip": 1, + "links": [ + { + "title": "CRD Conditions Browser", + "type": "link", + "url": "/d/crd_conditions_browser/crd-conditions-browser", + "keepTime": true + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "range": true + } + ], + "title": "Reconciles/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\", result=\"error\"}[$__rate_interval]))\n/\nsum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "range": true + } + ], + "title": "Reconcile error ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "range": true + } + ], + "title": "p99 reconcile time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 30 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "range": true + } + ], + "title": "p99 queue wait", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(controller_runtime_active_workers{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "active", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(controller_runtime_max_concurrent_reconciles{job=\"$job\", controller=~\"$controller\"})", + "refId": "B", + "legendFormat": "max", + "range": true + } + ], + "title": "Active / max workers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Standby" + }, + "1": { + "color": "green", + "index": 0, + "text": "Leader" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "n/a" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(leader_election_master_status{job=\"$job\"})", + "refId": "A", + "range": true + } + ], + "title": "Leader", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"True\"}\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", + "refId": "A", + "range": true + } + ], + "title": "Owners Ready", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n ({{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"False\"} < (time() - 120))\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", + "refId": "A", + "range": true + } + ], + "title": "Owners not Ready > 2m", + "type": "stat", + "description": "Owners whose Ready condition has been False for more than two minutes, debounced on lastTransitionTime." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"Unknown\"}\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", + "refId": "A", + "range": true + } + ], + "title": "Owners Unknown", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 12, + "panels": [], + "title": "Reconciliation", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller, result) (rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}} {{result}}", + "range": true + } + ], + "title": "Reconcile rate by result", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.25 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\", result=\"error\"}[$__rate_interval]))\n/\nsum by (controller) (rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Reconcile error ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "p50 {{controller}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "B", + "legendFormat": "p90 {{controller}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "C", + "legendFormat": "p99 {{controller}}", + "range": true + } + ], + "title": "Reconcile latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (increase(controller_runtime_reconcile_panics_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Reconcile panics", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by (controller) (workqueue_longest_running_processor_seconds{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Max in-progress reconcile age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^max .*/" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (controller_runtime_active_workers{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "active {{controller}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (controller_runtime_max_concurrent_reconciles{job=\"$job\", controller=~\"$controller\"})", + "refId": "B", + "legendFormat": "max {{controller}}", + "range": true + } + ], + "title": "Workers", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 19, + "panels": [], + "title": "Workqueue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (workqueue_depth{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(workqueue_adds_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Adds/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (controller, le) (rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Queue wait p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (controller, le) (rate(workqueue_work_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Work duration p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(workqueue_retries_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Retries/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (workqueue_unfinished_work_seconds{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Unfinished work", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 26, + "panels": [], + "title": "Managed resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "created" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "updated" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "none" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{operation}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "error", + "range": true + } + ], + "title": "Apply rate by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Managed resources sorted by how often an apply found a diff and updated them. A steadily high rate means the resource never converges.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Updated/s" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.05 + }, + { + "color": "red", + "value": 0.5 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Updated/s" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sort_desc(sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\", operation=\"updated\"}[$__rate_interval])))", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + } + ], + "title": "Updated rate per resource", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "controller": 1, + "owner_kind": 2, + "component": 3, + "resource": 4, + "kind": 5, + "Value": 6 + }, + "renameByName": { + "Value": "Updated/s", + "component": "Component", + "controller": "Controller", + "kind": "Kind", + "owner_kind": "Owner kind", + "resource": "Resource" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller, component, resource) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\", operation=\"updated\"}[15m]))\n/\nsum by (controller, component, resource) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[15m]))", + "refId": "A", + "legendFormat": "{{controller}}/{{component}}/{{resource}}", + "range": true + } + ], + "title": "Not-converging ratio", + "type": "timeseries", + "description": "Share of applies over 15m that found a diff and updated the resource. The ManagedResourceNotConverging alert uses the same expression with a request-rate floor." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller, component, resource) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n/\n(\n sum by (controller, component, resource) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n +\n (sum by (controller, component, resource) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) or sum by (controller, component, resource) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) * 0)\n)", + "refId": "A", + "legendFormat": "{{controller}}/{{component}}/{{resource}}", + "range": true + } + ], + "title": "Apply error ratio per resource", + "type": "timeseries", + "description": "Failed applies as a share of all apply attempts per resource. A resource whose every apply fails records no ocf_resource_apply_total sample, so the denominator falls back to the error rate alone and the ratio reads 1." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 31, + "panels": [], + "title": "Conditions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Owner conditions by type and status, one bar per combination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/ True$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ False$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ Unknown$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 59 + }, + "id": 32, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count by (condition, status) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"}\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"})\n)", + "refId": "A", + "legendFormat": "{{condition}} {{status}}", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Owners by condition and status", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Owners whose Ready condition is not True. The Name links into the conditions browser filtered to that owner.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "False": { + "color": "red", + "index": 1, + "text": "False" + }, + "True": { + "color": "semi-dark-green", + "index": 0, + "text": "True" + }, + "Unknown": { + "color": "semi-dark-yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeFromNow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Name" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open in the conditions browser", + "url": "/d/crd_conditions_browser/crd-conditions-browser?var-kind=${__data.fields.Kind}&var-condition=Ready&var-resource_id=${__data.fields.id:percentencode}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 8, + "y": 59 + }, + "id": 33, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (kind, {{namespace_label}}, name, reason, status, id) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status!=\"True\"}\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + } + ], + "title": "Owners not Ready", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "since_ms", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "Value" + } + }, + "operator": "*", + "right": { + "fixed": "1000" + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "since_ms" + } + ], + "fields": {} + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "kind": 1, + "{{namespace_label}}": 2, + "name": 3, + "status": 4, + "reason": 5, + "since_ms": 6, + "id": 7, + "Value": 8 + }, + "renameByName": { + "kind": "Kind", + "name": "Name", + "reason": "Reason", + "since_ms": "Since", + "status": "Status", + "{{namespace_label}}": "Namespace" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 34, + "panels": [], + "title": "API client", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (method) (rate(rest_client_requests_total{job=\"$job\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{method}}", + "range": true + } + ], + "title": "Requests/s by method", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(rest_client_requests_total{job=\"$job\", code!~\"2..\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{code}}", + "range": true + } + ], + "title": "Non-2xx responses/s by code", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 40, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=\"$job\"}[$__rate_interval])", + "refId": "A", + "legendFormat": "{{instance}}", + "range": true + } + ], + "title": "CPU", + "type": "timeseries", + "description": "CPU usage of the operator process; 1 is one full core." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=\"$job\"}", + "refId": "A", + "legendFormat": "{{instance}}", + "range": true + } + ], + "title": "Memory RSS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "go_goroutines{job=\"$job\"}", + "refId": "A", + "legendFormat": "{{instance}}", + "range": true + } + ], + "title": "Goroutines", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 41, + "tags": [ + "ocf" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(controller_runtime_reconcile_total,job)", + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(controller_runtime_reconcile_total,job)", + "refId": "job" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(controller_runtime_reconcile_total{job=\"$job\"},controller)", + "includeAll": true, + "label": "Controller", + "multi": true, + "name": "controller", + "options": [], + "query": { + "query": "label_values(controller_runtime_reconcile_total{job=\"$job\"},controller)", + "refId": "controller" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "1h" + ] + }, + "timezone": "browser", + "title": "OCF Operator", + "uid": "ocf_operator", + "version": 1 +} From 70dd0472ce3d80d1ea089434ef4456513ef741a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:12:10 +0200 Subject: [PATCH 3/9] test(observability): check rendered dashboards and alerts reference real metrics (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .github/workflows/test.yml | 3 + Makefile | 4 + observability/observability_test.go | 174 ++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 observability/observability_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39ecf56d..85cf7c6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,3 +51,6 @@ jobs: - name: Test alerts run: make test-alerts + + - name: Lint dashboards + run: make lint-dashboards diff --git a/Makefile b/Makefile index 9fddc49c..bce71e65 100644 --- a/Makefile +++ b/Makefile @@ -327,6 +327,10 @@ test-alerts: ## Lint and unit test the alert rules with promtool. echo "Running unit tests..."; \ promtool test rules --diff "$$tmpdir"/tests/*.yaml +.PHONY: lint-dashboards +lint-dashboards: ## Check the dashboard and alert templates render to valid files that reference real metrics. + go test ./$(OBS_DIR)/ + OBS_DEV_NAMESPACE := demo OBS_DEV_OUT := $(OBS_DIR)/generated/dev # Extra flags for the simulator, e.g. SIMULATOR_ARGS="-leader=false". diff --git a/observability/observability_test.go b/observability/observability_test.go new file mode 100644 index 00000000..8ebdcb0f --- /dev/null +++ b/observability/observability_test.go @@ -0,0 +1,174 @@ +// Package observability_test checks the dashboard and alert templates: that +// they render to valid JSON/YAML, keep their uids, leave no placeholder behind +// and reference only metric names that actually exist. +package observability_test + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + ocm "github.com/sourcehawk/go-crd-condition-metrics/pkg/crd-condition-metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" + + "github.com/sourcehawk/operator-component-framework/pkg/metrics" +) + +const ( + lintNamespace = "lint_operator" + nsLabel = "exported_namespace" +) + +// render mirrors the Makefile's render_template. +func render(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + require.NoError(t, err) + s := strings.ReplaceAll(string(b), "{{operator_namespace}}", lintNamespace+"_") + return strings.ReplaceAll(s, "{{namespace_label}}", nsLabel) +} + +// knownMetrics is every metric name the templates may reference: the +// framework's own, the condition gauge, and the controller-runtime, workqueue, +// client-go, leader election, process and Go runtime families. +func knownMetrics(t *testing.T) map[string]bool { + t.Helper() + known := map[string]bool{} + descs := make(chan *prometheus.Desc, 64) + go func() { + metrics.NewCollectors().Describe(descs) + ocm.NewOperatorConditionsGauge(lintNamespace).Describe(descs) + close(descs) + }() + fq := regexp.MustCompile(`fqName: "([^"]+)"`) + for d := range descs { + m := fq.FindStringSubmatch(d.String()) + require.Len(t, m, 2, d.String()) + known[m[1]] = true + } + for _, n := range []string{ + "controller_runtime_reconcile_total", "controller_runtime_reconcile_errors_total", + "controller_runtime_reconcile_panics_total", "controller_runtime_reconcile_time_seconds", + "controller_runtime_max_concurrent_reconciles", "controller_runtime_active_workers", + "workqueue_depth", "workqueue_adds_total", "workqueue_queue_duration_seconds", + "workqueue_work_duration_seconds", "workqueue_unfinished_work_seconds", + "workqueue_longest_running_processor_seconds", "workqueue_retries_total", + "rest_client_requests_total", "leader_election_master_status", + "process_cpu_seconds_total", "process_resident_memory_bytes", "go_goroutines", + } { + known[n] = true + } + return known +} + +// metricNameRe pulls every identifier that looks like one of the metric +// families we care about out of a PromQL expression, including the +// `_bucket`/`_sum`/`_count` suffixes of histograms. +var metricNameRe = regexp.MustCompile(`\b((?:ocf|controller_runtime|workqueue|rest_client|leader_election|process|go)_[a-z0-9_]+|` + lintNamespace + `_controller_condition)\b`) + +// baseName strips the histogram sample suffixes off a series name so that it +// can be looked up as a metric family. +func baseName(n string) string { + for _, s := range []string{"_bucket", "_sum", "_count"} { + n = strings.TrimSuffix(n, s) + } + return n +} + +// exprsFromJSON collects every PromQL-carrying string (panel targets, variable +// queries and definitions) from a decoded dashboard. +func exprsFromJSON(v any, out *[]string) { + switch x := v.(type) { + case map[string]any: + for k, val := range x { + if k == "expr" || k == "query" || k == "definition" { + if s, ok := val.(string); ok { + *out = append(*out, s) + } + } + exprsFromJSON(val, out) + } + case []any: + for _, val := range x { + exprsFromJSON(val, out) + } + } +} + +func TestDashboards(t *testing.T) { + files, err := filepath.Glob("dashboards/*.tpl.json") + require.NoError(t, err) + require.NotEmpty(t, files) + known := knownMetrics(t) + for _, f := range files { + t.Run(filepath.Base(f), func(t *testing.T) { + rendered := render(t, f) + assert.NotContains(t, rendered, "{{operator_namespace}}") + assert.NotContains(t, rendered, "{{namespace_label}}") + var d map[string]any + require.NoError(t, json.Unmarshal([]byte(rendered), &d), "valid JSON") + want := strings.TrimSuffix(filepath.Base(f), ".tpl.json") + assert.Equal(t, want, d["uid"], "uid matches the file name") + assert.Equal(t, "", d["refresh"], "auto refresh is off by default") + var exprs []string + exprsFromJSON(d, &exprs) + require.NotEmpty(t, exprs) + for _, e := range exprs { + for _, m := range metricNameRe.FindAllString(e, -1) { + assert.True(t, known[baseName(m)], "unknown metric %q in %q", m, e) + } + } + }) + } +} + +func TestAlerts(t *testing.T) { + // The glob matches the shared files and, because `.tpl.yaml` also ends in + // `.yaml`, the templated ones too; render handles both. + files, err := filepath.Glob("alerts/*.yaml") + require.NoError(t, err) + require.NotEmpty(t, files) + known := knownMetrics(t) + for _, f := range files { + t.Run(filepath.Base(f), func(t *testing.T) { + rendered := render(t, f) + assert.NotContains(t, rendered, "{{operator_namespace}}") + assert.NotContains(t, rendered, "{{namespace_label}}") + var doc struct { + Groups []struct { + Name string `json:"name"` + Rules []struct { + Alert string `json:"alert"` + Expr string `json:"expr"` + Labels map[string]string `json:"labels"` + } `json:"rules"` + } `json:"groups"` + } + require.NoError(t, yaml.Unmarshal([]byte(rendered), &doc)) + require.NotEmpty(t, doc.Groups) + for _, g := range doc.Groups { + for _, r := range g.Rules { + assert.Equal(t, map[string]string{"severity": "warning"}, r.Labels, "%s labels", r.Alert) + for _, m := range metricNameRe.FindAllString(r.Expr, -1) { + assert.True(t, known[baseName(m)], "unknown metric %q in %s", m, r.Alert) + } + } + } + // Every alert in the file has a promtool unit test. + base := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(f), ".yaml"), ".tpl") + tests, err := os.ReadFile(filepath.Join("alerts", "tests", base+"_test.yaml")) + require.NoError(t, err, "every rule file has a promtool test file") + for _, g := range doc.Groups { + for _, r := range g.Rules { + assert.Contains(t, string(tests), "alertname: "+r.Alert, "%s has a unit test", r.Alert) + } + } + }) + } +} From 2ecc7ba9376367cd55e8716c8a4ad3401a38ec9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:31:38 +0200 Subject: [PATCH 4/9] fix(observability): make dashboard filters and freshness keys consistent (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../dashboards/crd_conditions_browser.tpl.json | 12 ++++++------ observability/dashboards/ocf_operator.tpl.json | 18 +++++++++--------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/observability/dashboards/crd_conditions_browser.tpl.json b/observability/dashboards/crd_conditions_browser.tpl.json index dc191f61..a26ce759 100644 --- a/observability/dashboards/crd_conditions_browser.tpl.json +++ b/observability/dashboards/crd_conditions_browser.tpl.json @@ -104,7 +104,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", "range": true, "refId": "A" @@ -386,7 +386,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"False\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"False\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", "range": true, "refId": "A" @@ -455,7 +455,7 @@ "targets": [ { "editorMode": "code", - "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"Unknown\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"Unknown\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", "range": true, "refId": "A", @@ -528,7 +528,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"True\", reason=~\"$reason\", id=~\"$resource_id\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"})\n)", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"True\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", "range": true, "refId": "A" @@ -707,7 +707,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},id)", + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", {{namespace_label}}=~\"$namespace\"},id)", "description": "Namespace and resource name combination", "hide": 0, "includeAll": true, @@ -716,7 +716,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},id)", + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", {{namespace_label}}=~\"$namespace\"},id)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/observability/dashboards/ocf_operator.tpl.json b/observability/dashboards/ocf_operator.tpl.json index 52cc41ec..396a813d 100644 --- a/observability/dashboards/ocf_operator.tpl.json +++ b/observability/dashboards/ocf_operator.tpl.json @@ -536,7 +536,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"True\"}\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", + "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"True\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", "refId": "A", "range": true } @@ -603,7 +603,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(\n ({{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"False\"} < (time() - 120))\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", + "expr": "count(\n ({{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"False\"} < (time() - 120))\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", "refId": "A", "range": true } @@ -671,7 +671,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"Unknown\"}\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", + "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"Unknown\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", "refId": "A", "range": true } @@ -2295,9 +2295,9 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (controller, component, resource) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\", operation=\"updated\"}[15m]))\n/\nsum by (controller, component, resource) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[15m]))", + "expr": "sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\", operation=\"updated\"}[15m]))\n/\nsum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[15m]))", "refId": "A", - "legendFormat": "{{controller}}/{{component}}/{{resource}}", + "legendFormat": "{{controller}}/{{component}}/{{resource}} ({{kind}})", "range": true } ], @@ -2389,9 +2389,9 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (controller, component, resource) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n/\n(\n sum by (controller, component, resource) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n +\n (sum by (controller, component, resource) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) or sum by (controller, component, resource) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) * 0)\n)", + "expr": "sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n/\n(\n sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n +\n (sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) or sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) * 0)\n)", "refId": "A", - "legendFormat": "{{controller}}/{{component}}/{{resource}}", + "legendFormat": "{{controller}}/{{component}}/{{resource}} ({{kind}})", "range": true } ], @@ -2521,7 +2521,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count by (condition, status) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"}\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"})\n)", + "expr": "count by (condition, status) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"}\n and\n topk by (kind, id, condition) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"})\n)", "refId": "A", "legendFormat": "{{condition}} {{status}}", "exemplar": false, @@ -2672,7 +2672,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (kind, {{namespace_label}}, name, reason, status, id) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status!=\"True\"}\n and\n topk by (id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", + "expr": "sum by (kind, {{namespace_label}}, name, reason, status, id) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status!=\"True\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", "refId": "A", "exemplar": false, "instant": true, From 23c69d2dac118287687c51abcd323422a856dcad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:36:43 +0200 Subject: [PATCH 5/9] feat(observability): template the dashboard uids per metric namespace (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- observability/alerts/crd_conditions.tpl.yaml | 6 +++--- observability/alerts/tests/crd_conditions_test.yaml | 12 ++++++------ .../dashboards/crd_conditions_browser.tpl.json | 4 ++-- observability/dashboards/ocf_operator.tpl.json | 6 +++--- observability/observability_test.go | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/observability/alerts/crd_conditions.tpl.yaml b/observability/alerts/crd_conditions.tpl.yaml index ed559f5b..88200ed0 100644 --- a/observability/alerts/crd_conditions.tpl.yaml +++ b/observability/alerts/crd_conditions.tpl.yaml @@ -61,7 +61,7 @@ groups: # scoped resource is "/", so the empty namespace renders correctly # here without a conditional. dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind + /d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition=Ready&var-status=False&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} # A condition the controller cannot determine the state of. Unlike the two @@ -104,7 +104,7 @@ groups: {{- end }} ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + /d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ $labels.condition }}&var-status=Unknown&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} # A resource that has sat in a non-ready state for hours. @@ -162,5 +162,5 @@ groups: {{- end }} ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + /d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ $labels.condition }}&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} diff --git a/observability/alerts/tests/crd_conditions_test.yaml b/observability/alerts/tests/crd_conditions_test.yaml index 4e72524e..18be0ccd 100644 --- a/observability/alerts/tests/crd_conditions_test.yaml +++ b/observability/alerts/tests/crd_conditions_test.yaml @@ -67,7 +67,7 @@ tests: kubectl describe Backup/nightly-abc123 -n backups ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=Backup&var-condition=Ready&var-status=False&var-resource_id=backups%2Fnightly-abc123 + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=Backup&var-condition=Ready&var-status=False&var-resource_id=backups%2Fnightly-abc123 # Backup recovered at 45m, so it has resolved. RemoteStorage has now been # not ready for 30m and is the only alert left. @@ -93,7 +93,7 @@ tests: kubectl describe RemoteStorage/document-store ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=RemoteStorage&var-condition=Ready&var-status=False&var-resource_id=%2Fdocument-store + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=RemoteStorage&var-condition=Ready&var-status=False&var-resource_id=%2Fdocument-store # Unknown is bad whatever the condition's polarity is, so this rule is not # scoped to a single condition: both fixtures below must fire. @@ -136,7 +136,7 @@ tests: kubectl describe App/checkout -n shop ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Ready&var-status=Unknown&var-resource_id=shop%2Fcheckout + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Ready&var-status=Unknown&var-resource_id=shop%2Fcheckout - exp_labels: controller: app kind: App @@ -160,7 +160,7 @@ tests: kubectl describe App/checkout -n shop ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Synced&var-status=Unknown&var-resource_id=shop%2Fcheckout + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Synced&var-status=Unknown&var-resource_id=shop%2Fcheckout # The stuck rule compares against the metric value, so the fixture timestamps # have to live on promtool's clock, which starts at the unix epoch. A Ready @@ -203,7 +203,7 @@ tests: kubectl describe Cluster/prod -n clusters ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fprod + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fprod # Why CustomResourceConditionStuck exists alongside CustomResourceNotReady: # the series disappears between 1h and 6h (operator down, scrape gap), which @@ -247,4 +247,4 @@ tests: kubectl describe Cluster/staging -n clusters ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fstaging + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fstaging diff --git a/observability/dashboards/crd_conditions_browser.tpl.json b/observability/dashboards/crd_conditions_browser.tpl.json index a26ce759..858b12fb 100644 --- a/observability/dashboards/crd_conditions_browser.tpl.json +++ b/observability/dashboards/crd_conditions_browser.tpl.json @@ -23,7 +23,7 @@ { "title": "Operator overview", "type": "link", - "url": "/d/ocf_operator/ocf-operator", + "url": "/d/{{operator_namespace}}ocf_operator/ocf-operator", "keepTime": true } ], @@ -745,6 +745,6 @@ }, "timezone": "browser", "title": "CRD Conditions Browser", - "uid": "crd_conditions_browser", + "uid": "{{operator_namespace}}crd_conditions_browser", "version": 1 } diff --git a/observability/dashboards/ocf_operator.tpl.json b/observability/dashboards/ocf_operator.tpl.json index 396a813d..f365001f 100644 --- a/observability/dashboards/ocf_operator.tpl.json +++ b/observability/dashboards/ocf_operator.tpl.json @@ -22,7 +22,7 @@ { "title": "CRD Conditions Browser", "type": "link", - "url": "/d/crd_conditions_browser/crd-conditions-browser", + "url": "/d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser", "keepTime": true } ], @@ -2625,7 +2625,7 @@ "value": [ { "title": "Open in the conditions browser", - "url": "/d/crd_conditions_browser/crd-conditions-browser?var-kind=${__data.fields.Kind}&var-condition=Ready&var-resource_id=${__data.fields.id:percentencode}&${__url_time_range}" + "url": "/d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind=${__data.fields.Kind}&var-condition=Ready&var-resource_id=${__data.fields.id:percentencode}&${__url_time_range}" } ] } @@ -3327,6 +3327,6 @@ }, "timezone": "browser", "title": "OCF Operator", - "uid": "ocf_operator", + "uid": "{{operator_namespace}}ocf_operator", "version": 1 } diff --git a/observability/observability_test.go b/observability/observability_test.go index 8ebdcb0f..b104f410 100644 --- a/observability/observability_test.go +++ b/observability/observability_test.go @@ -114,7 +114,7 @@ func TestDashboards(t *testing.T) { var d map[string]any require.NoError(t, json.Unmarshal([]byte(rendered), &d), "valid JSON") want := strings.TrimSuffix(filepath.Base(f), ".tpl.json") - assert.Equal(t, want, d["uid"], "uid matches the file name") + assert.Equal(t, lintNamespace+"_"+want, d["uid"], "uid is the file name prefixed with the metric namespace") assert.Equal(t, "", d["refresh"], "auto refresh is off by default") var exprs []string exprsFromJSON(d, &exprs) From 915896a6e1d35a2bd592af40b6fae91c03ecf5ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:37:24 +0200 Subject: [PATCH 6/9] fix(observability): browser namespace handling for cluster-scoped owners and instant stats (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- .../crd_conditions_browser.tpl.json | 47 +++++++++++----- .../dashboards/ocf_operator.tpl.json | 56 ++++++++++++++----- 2 files changed, 75 insertions(+), 28 deletions(-) diff --git a/observability/dashboards/crd_conditions_browser.tpl.json b/observability/dashboards/crd_conditions_browser.tpl.json index 858b12fb..b43dae96 100644 --- a/observability/dashboards/crd_conditions_browser.tpl.json +++ b/observability/dashboards/crd_conditions_browser.tpl.json @@ -21,10 +21,18 @@ "graphTooltip": 0, "links": [ { - "title": "Operator overview", - "type": "link", - "url": "/d/{{operator_namespace}}ocf_operator/ocf-operator", - "keepTime": true + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "ocf" + ], + "targetBlank": false, + "title": "OCF dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" } ], "liveNow": true, @@ -388,8 +396,10 @@ "editorMode": "code", "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"False\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", - "range": true, - "refId": "A" + "range": false, + "refId": "A", + "exemplar": false, + "instant": true } ], "title": "$condition=False", @@ -457,12 +467,14 @@ "editorMode": "code", "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"Unknown\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", - "range": true, + "range": false, "refId": "A", "datasource": { "type": "prometheus", "uid": "${datasource}" - } + }, + "exemplar": false, + "instant": true } ], "title": "$condition=Unknown", @@ -530,8 +542,10 @@ "editorMode": "code", "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"True\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", "legendFormat": "Total", - "range": true, - "refId": "A" + "range": false, + "refId": "A", + "exemplar": false, + "instant": true } ], "title": "$condition=True", @@ -612,7 +626,8 @@ }, "refresh": 2, "regex": "", - "type": "query" + "type": "query", + "allValue": ".*" }, { "allowCustomValue": true, @@ -640,7 +655,8 @@ }, "refresh": 1, "regex": "", - "type": "query" + "type": "query", + "allValue": ".*" }, { "current": { @@ -667,7 +683,8 @@ }, "refresh": 1, "regex": "", - "type": "query" + "type": "query", + "allValue": ".*" }, { "current": { @@ -695,7 +712,8 @@ "refresh": 2, "regex": "^.*{{namespace_label}}=\"([^\"]+)\".*$", "sort": 1, - "type": "query" + "type": "query", + "allValue": ".*" }, { "allowCustomValue": false, @@ -722,6 +740,7 @@ "refresh": 1, "regex": "", "type": "query", + "allValue": ".*", "label": "Resource" }, { diff --git a/observability/dashboards/ocf_operator.tpl.json b/observability/dashboards/ocf_operator.tpl.json index f365001f..5a15411a 100644 --- a/observability/dashboards/ocf_operator.tpl.json +++ b/observability/dashboards/ocf_operator.tpl.json @@ -20,10 +20,18 @@ "graphTooltip": 1, "links": [ { - "title": "CRD Conditions Browser", - "type": "link", - "url": "/d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser", - "keepTime": true + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "ocf" + ], + "targetBlank": false, + "title": "OCF dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" } ], "panels": [ @@ -97,7 +105,9 @@ "editorMode": "code", "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Reconciles/s", @@ -168,7 +178,9 @@ "editorMode": "code", "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\", result=\"error\"}[$__rate_interval]))\n/\nsum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Reconcile error ratio", @@ -239,7 +251,9 @@ "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "p99 reconcile time", @@ -310,7 +324,9 @@ "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le) (rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "p99 queue wait", @@ -374,7 +390,9 @@ "expr": "sum(controller_runtime_active_workers{job=\"$job\", controller=~\"$controller\"})", "refId": "A", "legendFormat": "active", - "range": true + "range": false, + "exemplar": false, + "instant": true }, { "datasource": { @@ -385,7 +403,9 @@ "expr": "sum(controller_runtime_max_concurrent_reconciles{job=\"$job\", controller=~\"$controller\"})", "refId": "B", "legendFormat": "max", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Active / max workers", @@ -474,7 +494,9 @@ "editorMode": "code", "expr": "max(leader_election_master_status{job=\"$job\"})", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Leader", @@ -538,7 +560,9 @@ "editorMode": "code", "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"True\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Owners Ready", @@ -605,7 +629,9 @@ "editorMode": "code", "expr": "count(\n ({{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"False\"} < (time() - 120))\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Owners not Ready > 2m", @@ -673,7 +699,9 @@ "editorMode": "code", "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"Unknown\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", "refId": "A", - "range": true + "range": false, + "exemplar": false, + "instant": true } ], "title": "Owners Unknown", From 8b00091028c05c1955749c0a30196b21472ad936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:38:38 +0200 Subject: [PATCH 7/9] test(observability): tighten the template lint (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- observability/observability_test.go | 48 ++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/observability/observability_test.go b/observability/observability_test.go index b104f410..712d03e7 100644 --- a/observability/observability_test.go +++ b/observability/observability_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "strconv" "strings" "testing" @@ -72,6 +73,40 @@ func knownMetrics(t *testing.T) map[string]bool { // `_bucket`/`_sum`/`_count` suffixes of histograms. var metricNameRe = regexp.MustCompile(`\b((?:ocf|controller_runtime|workqueue|rest_client|leader_election|process|go)_[a-z0-9_]+|` + lintNamespace + `_controller_condition)\b`) +// selectorRe pulls the metric name out of every selector-shaped token +// (`name{...}`). It catches typos and unknown metric families that the prefix +// whitelist of metricNameRe skips over. +var selectorRe = regexp.MustCompile(`([a-zA-Z_:][a-zA-Z0-9_:]*)\s*\{`) + +// promqlKeywords are identifiers that may legally precede `{` in PromQL +// without naming a metric. +var promqlKeywords = map[string]bool{ + "by": true, "on": true, "ignoring": true, + "group_left": true, "group_right": true, "without": true, +} + +// placeholderRe matches a `{{name}}` template placeholder. After rendering, +// any hit in a PromQL expression is a mis-spelled placeholder the render left +// behind. +var placeholderRe = regexp.MustCompile(`\{\{ *[a-zA-Z_]+ *\}\}`) + +// assertExprClean checks a rendered PromQL expression: every metric-family +// token and every selector names a known metric, and no template placeholder +// survived rendering. +func assertExprClean(t *testing.T, known map[string]bool, expr, context string) { + t.Helper() + for _, m := range metricNameRe.FindAllString(expr, -1) { + assert.True(t, known[baseName(m)], "unknown metric %q in %s", m, context) + } + for _, m := range selectorRe.FindAllStringSubmatch(expr, -1) { + if promqlKeywords[m[1]] { + continue + } + assert.True(t, known[baseName(m[1])], "unknown selector metric %q in %s", m[1], context) + } + assert.NotRegexp(t, placeholderRe, expr, "unrendered placeholder in %s", context) +} + // baseName strips the histogram sample suffixes off a series name so that it // can be looked up as a metric family. func baseName(n string) string { @@ -120,9 +155,7 @@ func TestDashboards(t *testing.T) { exprsFromJSON(d, &exprs) require.NotEmpty(t, exprs) for _, e := range exprs { - for _, m := range metricNameRe.FindAllString(e, -1) { - assert.True(t, known[baseName(m)], "unknown metric %q in %q", m, e) - } + assertExprClean(t, known, e, strconv.Quote(e)) } }) } @@ -154,10 +187,9 @@ func TestAlerts(t *testing.T) { require.NotEmpty(t, doc.Groups) for _, g := range doc.Groups { for _, r := range g.Rules { - assert.Equal(t, map[string]string{"severity": "warning"}, r.Labels, "%s labels", r.Alert) - for _, m := range metricNameRe.FindAllString(r.Expr, -1) { - assert.True(t, known[baseName(m)], "unknown metric %q in %s", m, r.Alert) - } + assert.Len(t, r.Labels, 1, "%s carries only the severity label", r.Alert) + assert.Contains(t, []string{"warning", "critical"}, r.Labels["severity"], "%s severity", r.Alert) + assertExprClean(t, known, r.Expr, r.Alert) } } // Every alert in the file has a promtool unit test. @@ -166,7 +198,7 @@ func TestAlerts(t *testing.T) { require.NoError(t, err, "every rule file has a promtool test file") for _, g := range doc.Groups { for _, r := range g.Rules { - assert.Contains(t, string(tests), "alertname: "+r.Alert, "%s has a unit test", r.Alert) + assert.Contains(t, string(tests), "alertname: "+r.Alert+"\n", "%s has a unit test", r.Alert) } } }) From 133a6cca9946149fd079c9948ae89e863c1f858a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:44:14 +0200 Subject: [PATCH 8/9] test(observability): exercise a cluster-scoped owner in the simulator world (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- observability/dev/simulator/world.go | 13 +++++++++---- observability/dev/simulator/world_test.go | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/observability/dev/simulator/world.go b/observability/dev/simulator/world.go index 2fe2e5a4..2dd0db19 100644 --- a/observability/dev/simulator/world.go +++ b/observability/dev/simulator/world.go @@ -193,17 +193,20 @@ func (w *world) hotLoop(ctx context.Context) { } } -// databases: five owners. orders-db has been Ready=False (Failing) for eight +// databases: six owners. orders-db has been Ready=False (Failing) for eight // hours; users-db is Ready=Unknown; reports-db stays Ready=False while its -// reason flips between Failing and Creating; the rest are healthy. PVC applies -// fail three times out of four, thirty percent of reconciles error, and -// reconcile durations are slow enough for the p99 to cross a minute. +// reason flips between Failing and Creating; the rest are healthy, among them +// the cluster-scoped shared-gateway, whose empty namespace makes the condition +// gauge export series without a namespace label. PVC applies fail three times +// out of four, thirty percent of reconciles error, and reconcile durations are +// slow enough for the p99 to cross a minute. func (w *world) databases(ctx context.Context) { c := w.database orders := owner{namespaceShop, "orders-db"} users := owner{namespaceShop, "users-db"} reports := owner{"analytics", "reports-db"} healthy := []owner{{namespaceShop, "carts-db"}, {"analytics", "events-db"}} + clusterOwner := owner{namespace: "", name: "shared-gateway"} stuckSince := w.start.Add(-8 * time.Hour) unknownSince := w.start.Add(-1 * time.Hour) flipSince := w.start @@ -251,6 +254,8 @@ func (w *world) databases(ctx context.Context) { c.condition(o, "BackupReady", "True", reasonHealthy, w.start) c.condition(o, "Ready", "True", reasonHealthy, w.start) } + c.condition(clusterOwner, "StorageReady", "True", reasonHealthy, w.start) + c.condition(clusterOwner, "Ready", "True", reasonHealthy, w.start) } tick() ticker := time.NewTicker(3 * time.Second) diff --git a/observability/dev/simulator/world_test.go b/observability/dev/simulator/world_test.go index fa8a8b20..094e2063 100644 --- a/observability/dev/simulator/world_test.go +++ b/observability/dev/simulator/world_test.go @@ -51,8 +51,9 @@ func seriesValue(mfs []*dto.MetricFamily, name string, want map[string]string) ( // world and checks that every scenario's series appear with the shapes the // dev stack's alerts key on: the stuck Ready condition with its eight hour // old transition, the failing PVC applies, the hot loop's rising updated -// counter, both controllers reconciling and the leader gauge at one. It then -// cancels the context and requires run to return. +// counter, the cluster-scoped owner without a namespace label, both +// controllers reconciling and the leader gauge at one. It then cancels the +// context and requires run to return. func TestWorldScriptedScenarios(t *testing.T) { reg := prometheus.NewRegistry() conditions := ocm.NewOperatorConditionsGauge("demo") @@ -97,6 +98,15 @@ func TestWorldScriptedScenarios(t *testing.T) { if updated <= firstUpdated { return false } + // The cluster-scoped owner: client_golang exports the declared + // namespace label with an empty value, which Prometheus ingests as no + // namespace label at all, and the id keeps its "/" shape. + if _, ok := seriesValue(mfs, "demo_controller_condition", map[string]string{ + "kind": "Database", "name": "shared-gateway", "namespace": "", + "id": "/shared-gateway", "condition": "Ready", "status": "True", + }); !ok { + return false + } for _, controller := range []string{"webapp", "database"} { if _, ok := seriesValue(mfs, "controller_runtime_reconcile_total", map[string]string{"controller": controller}); !ok { return false From 3256589b363883953664b8fb59684f2a31573eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=86gir=20M=C3=A1ni=20Hauksson?= <54936225+sourcehawk@users.noreply.github.com> Date: Mon, 24 Aug 2026 00:50:59 +0200 Subject: [PATCH 9/9] fix(observability): correct the status stat descriptions in the browser (#186) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JPihvXVfS997iGmGabTGsd --- observability/dashboards/crd_conditions_browser.tpl.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/observability/dashboards/crd_conditions_browser.tpl.json b/observability/dashboards/crd_conditions_browser.tpl.json index b43dae96..542a9d73 100644 --- a/observability/dashboards/crd_conditions_browser.tpl.json +++ b/observability/dashboards/crd_conditions_browser.tpl.json @@ -335,7 +335,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Number of $kind $condition Conditions in a False state", + "description": "Number of $kind $condition Conditions with status False", "fieldConfig": { "defaults": { "color": { @@ -410,7 +410,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Number of $kind $condition Conditions in a Unknown state", + "description": "Number of $kind $condition Conditions with status Unknown", "fieldConfig": { "defaults": { "color": { @@ -485,7 +485,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "description": "Number of $kind $condition Conditions in a True state", + "description": "Number of $kind $condition Conditions with status True", "fieldConfig": { "defaults": { "color": {