diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 39ecf56d..85cf7c6c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,3 +51,6 @@ jobs: - name: Test alerts run: make test-alerts + + - name: Lint dashboards + run: make lint-dashboards diff --git a/Makefile b/Makefile index 9fddc49c..bce71e65 100644 --- a/Makefile +++ b/Makefile @@ -327,6 +327,10 @@ test-alerts: ## Lint and unit test the alert rules with promtool. echo "Running unit tests..."; \ promtool test rules --diff "$$tmpdir"/tests/*.yaml +.PHONY: lint-dashboards +lint-dashboards: ## Check the dashboard and alert templates render to valid files that reference real metrics. + go test ./$(OBS_DIR)/ + OBS_DEV_NAMESPACE := demo OBS_DEV_OUT := $(OBS_DIR)/generated/dev # Extra flags for the simulator, e.g. SIMULATOR_ARGS="-leader=false". diff --git a/observability/alerts/crd_conditions.tpl.yaml b/observability/alerts/crd_conditions.tpl.yaml index ed559f5b..88200ed0 100644 --- a/observability/alerts/crd_conditions.tpl.yaml +++ b/observability/alerts/crd_conditions.tpl.yaml @@ -61,7 +61,7 @@ groups: # scoped resource is "/", so the empty namespace renders correctly # here without a conditional. dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind + /d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition=Ready&var-status=False&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} # A condition the controller cannot determine the state of. Unlike the two @@ -104,7 +104,7 @@ groups: {{- end }} ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + /d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ $labels.condition }}&var-status=Unknown&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} # A resource that has sat in a non-ready state for hours. @@ -162,5 +162,5 @@ groups: {{- end }} ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ + /d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind={{ $labels.kind }}&var-condition={{ $labels.condition }}&var-resource_id={{ $labels.{{namespace_label}} }}%2F{{ $labels.name }} diff --git a/observability/alerts/tests/crd_conditions_test.yaml b/observability/alerts/tests/crd_conditions_test.yaml index 4e72524e..18be0ccd 100644 --- a/observability/alerts/tests/crd_conditions_test.yaml +++ b/observability/alerts/tests/crd_conditions_test.yaml @@ -67,7 +67,7 @@ tests: kubectl describe Backup/nightly-abc123 -n backups ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=Backup&var-condition=Ready&var-status=False&var-resource_id=backups%2Fnightly-abc123 + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=Backup&var-condition=Ready&var-status=False&var-resource_id=backups%2Fnightly-abc123 # Backup recovered at 45m, so it has resolved. RemoteStorage has now been # not ready for 30m and is the only alert left. @@ -93,7 +93,7 @@ tests: kubectl describe RemoteStorage/document-store ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=RemoteStorage&var-condition=Ready&var-status=False&var-resource_id=%2Fdocument-store + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=RemoteStorage&var-condition=Ready&var-status=False&var-resource_id=%2Fdocument-store # Unknown is bad whatever the condition's polarity is, so this rule is not # scoped to a single condition: both fixtures below must fire. @@ -136,7 +136,7 @@ tests: kubectl describe App/checkout -n shop ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Ready&var-status=Unknown&var-resource_id=shop%2Fcheckout + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Ready&var-status=Unknown&var-resource_id=shop%2Fcheckout - exp_labels: controller: app kind: App @@ -160,7 +160,7 @@ tests: kubectl describe App/checkout -n shop ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Synced&var-status=Unknown&var-resource_id=shop%2Fcheckout + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=App&var-condition=Synced&var-status=Unknown&var-resource_id=shop%2Fcheckout # The stuck rule compares against the metric value, so the fixture timestamps # have to live on promtool's clock, which starts at the unix epoch. A Ready @@ -203,7 +203,7 @@ tests: kubectl describe Cluster/prod -n clusters ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fprod + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fprod # Why CustomResourceConditionStuck exists alongside CustomResourceNotReady: # the series disappears between 1h and 6h (operator down, scrape gap), which @@ -247,4 +247,4 @@ tests: kubectl describe Cluster/staging -n clusters ``` dashboard_url: >- - /d/crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fstaging + /d/test_operator_crd_conditions_browser/crd-conditions-browser?var-kind=Cluster&var-condition=Ready&var-resource_id=clusters%2Fstaging diff --git a/observability/dashboards/crd_conditions_browser.tpl.json b/observability/dashboards/crd_conditions_browser.tpl.json new file mode 100644 index 00000000..542a9d73 --- /dev/null +++ b/observability/dashboards/crd_conditions_browser.tpl.json @@ -0,0 +1,769 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard to monitor operator custom resource conditions", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "ocf" + ], + "targetBlank": false, + "title": "OCF dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "panels": [], + "title": "Operator Conditions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Conditions matching the given filter criteria ", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "none", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", + "legendFormat": "Total", + "range": true, + "refId": "A" + } + ], + "title": "Total Conditions Matching", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Table of $kind conditions of type $condition", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "transparent", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "applyToRow": false, + "mode": "basic", + "type": "color-background", + "wrapText": true + }, + "filterable": true, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": 0 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Namespace" + }, + "properties": [ + { + "id": "custom.width", + "value": 397 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeFromNow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "False": { + "color": "red", + "index": 1, + "text": "False" + }, + "True": { + "color": "semi-dark-green", + "index": 0, + "text": "True" + }, + "Unknown": { + "color": "semi-dark-yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "12.1.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(\n {{operator_namespace}}controller_condition{\n kind=\"$kind\",\n condition=~\"$condition\",\n status=~\"$status\",\n reason=~\"$reason\",\n id=~\"$resource_id\",\n {{namespace_label}}=~\"$namespace\"\n }\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n) by (name, {{namespace_label}}, condition, status, reason)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "$kind Conditions", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "value_ms", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "Value" + } + }, + "operator": "*", + "right": { + "fixed": "1000" + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "value_ms" + } + ], + "fields": {} + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "{{namespace_label}}": 1, + "name": 2, + "condition": 3, + "status": 4, + "reason": 5, + "value_ms": 6, + "Value": 7 + }, + "renameByName": { + "Value": "", + "condition": "Condition", + "{{namespace_label}}": "Namespace", + "name": "Name", + "reason": "Reason", + "status": "Status", + "value_ms": "Since" + } + } + } + ], + "type": "table" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 9, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of $kind $condition Conditions with status False", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 7, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "repeat": "condition", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"False\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", + "legendFormat": "Total", + "range": false, + "refId": "A", + "exemplar": false, + "instant": true + } + ], + "title": "$condition=False", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of $kind $condition Conditions with status Unknown", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "semi-dark-yellow", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 11, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "repeat": "condition", + "repeatDirection": "h", + "targets": [ + { + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"Unknown\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", + "legendFormat": "Total", + "range": false, + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "exemplar": false, + "instant": true + } + ], + "title": "$condition=Unknown", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Number of $kind $condition Conditions with status True", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "semi-dark-red", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 8, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "12.1.1", + "repeat": "condition", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=\"True\", reason=~\"$reason\", id=~\"$resource_id\", {{namespace_label}}=~\"$namespace\"} > 0\n and\n topk by (id, condition) (1, {{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", {{namespace_label}}=~\"$namespace\"})\n)", + "legendFormat": "Total", + "range": false, + "refId": "A", + "exemplar": false, + "instant": true + } + ], + "title": "$condition=True", + "type": "stat" + } + ], + "title": "Status Counts Per Condition", + "type": "row" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [ + "ocf" + ], + "templating": { + "list": [ + { + "current": { + "text": "", + "value": "" + }, + "description": "The data source to get the metrics from", + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "text": "", + "value": "" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition,kind)", + "description": "The resource kind to view", + "label": "Kind", + "name": "kind", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition,kind)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\"},condition)", + "description": "The condition type to display", + "includeAll": true, + "label": "Condition", + "multi": true, + "name": "condition", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\"},condition)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "type": "query", + "allValue": ".*" + }, + { + "allowCustomValue": true, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"},status)", + "description": "Filter for a specific status", + "includeAll": true, + "label": "Status", + "multi": true, + "name": "status", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"},status)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "allValue": ".*" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},reason)", + "description": "Filter for condition reason", + "includeAll": true, + "label": "Reason", + "multi": true, + "name": "reason", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\"},reason)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "allValue": ".*" + }, + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "query_result(sum ({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"} > 0) by ({{namespace_label}}))", + "description": "The namespaces in which to list the resources", + "includeAll": true, + "label": "Namespace", + "multi": true, + "name": "namespace", + "options": [], + "query": { + "qryType": 3, + "query": "query_result(sum ({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\"} > 0) by ({{namespace_label}}))", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "^.*{{namespace_label}}=\"([^\"]+)\".*$", + "sort": 1, + "type": "query", + "allValue": ".*" + }, + { + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", {{namespace_label}}=~\"$namespace\"},id)", + "description": "Namespace and resource name combination", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "resource_id", + "options": [], + "query": { + "qryType": 1, + "query": "label_values({{operator_namespace}}controller_condition{kind=\"$kind\", condition=~\"$condition\", status=~\"$status\", reason=~\"$reason\", {{namespace_label}}=~\"$namespace\"},id)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query", + "allValue": ".*", + "label": "Resource" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "name": "Filters", + "type": "adhoc" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "hidden": true + }, + "timezone": "browser", + "title": "CRD Conditions Browser", + "uid": "{{operator_namespace}}crd_conditions_browser", + "version": 1 +} diff --git a/observability/dashboards/ocf_operator.tpl.json b/observability/dashboards/ocf_operator.tpl.json new file mode 100644 index 00000000..5a15411a --- /dev/null +++ b/observability/dashboards/ocf_operator.tpl.json @@ -0,0 +1,3360 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Health of an operator built on the operator-component-framework: reconciliation, workqueue, managed resource applies, owner conditions, API client and process.", + "editable": true, + "graphTooltip": 1, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "ocf" + ], + "targetBlank": false, + "title": "OCF dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Reconciles/s", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.05 + }, + { + "color": "red", + "value": 0.25 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\", result=\"error\"}[$__rate_interval]))\n/\nsum(rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Reconcile error ratio", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "p99 reconcile time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 30 + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "p99 queue wait", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(controller_runtime_active_workers{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "active", + "range": false, + "exemplar": false, + "instant": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(controller_runtime_max_concurrent_reconciles{job=\"$job\", controller=~\"$controller\"})", + "refId": "B", + "legendFormat": "max", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Active / max workers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Standby" + }, + "1": { + "color": "green", + "index": 0, + "text": "Leader" + } + }, + "type": "value" + }, + { + "options": { + "match": "null", + "result": { + "color": "text", + "index": 2, + "text": "n/a" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(leader_election_master_status{job=\"$job\"})", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Leader", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"True\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Owners Ready", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n ({{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"False\"} < (time() - 120))\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Owners not Ready > 2m", + "type": "stat", + "description": "Owners whose Ready condition has been False for more than two minutes, debounced on lastTransitionTime." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "same_as_value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status=\"Unknown\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n) or vector(0)", + "refId": "A", + "range": false, + "exemplar": false, + "instant": true + } + ], + "title": "Owners Unknown", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 12, + "panels": [], + "title": "Reconciliation", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller, result) (rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}} {{result}}", + "range": true + } + ], + "title": "Reconcile rate by result", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.25 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\", result=\"error\"}[$__rate_interval]))\n/\nsum by (controller) (rate(controller_runtime_reconcile_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Reconcile error ratio", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "p50 {{controller}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "B", + "legendFormat": "p90 {{controller}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (controller, le) (rate(controller_runtime_reconcile_time_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "C", + "legendFormat": "p99 {{controller}}", + "range": true + } + ], + "title": "Reconcile latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (increase(controller_runtime_reconcile_panics_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Reconcile panics", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max by (controller) (workqueue_longest_running_processor_seconds{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Max in-progress reconcile age", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/^max .*/" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (controller_runtime_active_workers{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "active {{controller}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (controller_runtime_max_concurrent_reconciles{job=\"$job\", controller=~\"$controller\"})", + "refId": "B", + "legendFormat": "max {{controller}}", + "range": true + } + ], + "title": "Workers", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 19, + "panels": [], + "title": "Workqueue", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (workqueue_depth{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Depth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(workqueue_adds_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Adds/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 300 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (controller, le) (rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Queue wait p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (controller, le) (rate(workqueue_work_duration_seconds_bucket{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Work duration p99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(workqueue_retries_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Retries/s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller) (workqueue_unfinished_work_seconds{job=\"$job\", controller=~\"$controller\"})", + "refId": "A", + "legendFormat": "{{controller}}", + "range": true + } + ], + "title": "Unfinished work", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 26, + "panels": [], + "title": "Managed resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "created" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "updated" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "none" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (operation) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{operation}}", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))", + "refId": "B", + "legendFormat": "error", + "range": true + } + ], + "title": "Apply rate by operation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Managed resources sorted by how often an apply found a diff and updated them. A steadily high rate means the resource never converges.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Updated/s" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.05 + }, + { + "color": "red", + "value": 0.5 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 28, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Updated/s" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sort_desc(sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\", operation=\"updated\"}[$__rate_interval])))", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + } + ], + "title": "Updated rate per resource", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "controller": 1, + "owner_kind": 2, + "component": 3, + "resource": 4, + "kind": 5, + "Value": 6 + }, + "renameByName": { + "Value": "Updated/s", + "component": "Component", + "controller": "Controller", + "kind": "Kind", + "owner_kind": "Owner kind", + "resource": "Resource" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\", operation=\"updated\"}[15m]))\n/\nsum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[15m]))", + "refId": "A", + "legendFormat": "{{controller}}/{{component}}/{{resource}} ({{kind}})", + "range": true + } + ], + "title": "Not-converging ratio", + "type": "timeseries", + "description": "Share of applies over 15m that found a diff and updated the resource. The ManagedResourceNotConverging alert uses the same expression with a request-rate floor." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n/\n(\n sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval]))\n +\n (sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) or sum by (controller, owner_kind, component, resource, kind) (rate(ocf_resource_apply_errors_total{job=\"$job\", controller=~\"$controller\"}[$__rate_interval])) * 0)\n)", + "refId": "A", + "legendFormat": "{{controller}}/{{component}}/{{resource}} ({{kind}})", + "range": true + } + ], + "title": "Apply error ratio per resource", + "type": "timeseries", + "description": "Failed applies as a share of all apply attempts per resource. A resource whose every apply fails records no ocf_resource_apply_total sample, so the denominator falls back to the error rate alone and the ratio reads 1." + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 31, + "panels": [], + "title": "Conditions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Owner conditions by type and status, one bar per combination.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/ True$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ False$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/ Unknown$/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 59 + }, + "id": 32, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count by (condition, status) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"}\n and\n topk by (kind, id, condition) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\"})\n)", + "refId": "A", + "legendFormat": "{{condition}} {{status}}", + "exemplar": false, + "instant": true, + "range": false + } + ], + "title": "Owners by condition and status", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Owners whose Ready condition is not True. The Name links into the conditions browser filtered to that owner.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Status" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "False": { + "color": "red", + "index": 1, + "text": "False" + }, + "True": { + "color": "semi-dark-green", + "index": 0, + "text": "True" + }, + "Unknown": { + "color": "semi-dark-yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Since" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeFromNow" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Name" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Open in the conditions browser", + "url": "/d/{{operator_namespace}}crd_conditions_browser/crd-conditions-browser?var-kind=${__data.fields.Kind}&var-condition=Ready&var-resource_id=${__data.fields.id:percentencode}&${__url_time_range}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 16, + "x": 8, + "y": 59 + }, + "id": 33, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (kind, {{namespace_label}}, name, reason, status, id) (\n {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\", status!=\"True\"}\n and\n topk by (kind, id) (1, {{operator_namespace}}controller_condition{job=\"$job\", controller=~\"$controller\", condition=\"Ready\"})\n)", + "refId": "A", + "exemplar": false, + "instant": true, + "range": false, + "format": "table" + } + ], + "title": "Owners not Ready", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "since_ms", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "Value" + } + }, + "operator": "*", + "right": { + "fixed": "1000" + } + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "time", + "targetField": "since_ms" + } + ], + "fields": {} + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "includeByName": {}, + "indexByName": { + "Time": 0, + "kind": 1, + "{{namespace_label}}": 2, + "name": 3, + "status": 4, + "reason": 5, + "since_ms": 6, + "id": 7, + "Value": 8 + }, + "renameByName": { + "kind": "Kind", + "name": "Name", + "reason": "Reason", + "since_ms": "Since", + "status": "Status", + "{{namespace_label}}": "Namespace" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 34, + "panels": [], + "title": "API client", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (method) (rate(rest_client_requests_total{job=\"$job\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{method}}", + "range": true + } + ], + "title": "Requests/s by method", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(rest_client_requests_total{job=\"$job\", code!~\"2..\"}[$__rate_interval]))", + "refId": "A", + "legendFormat": "{{code}}", + "range": true + } + ], + "title": "Non-2xx responses/s by code", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 40, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=\"$job\"}[$__rate_interval])", + "refId": "A", + "legendFormat": "{{instance}}", + "range": true + } + ], + "title": "CPU", + "type": "timeseries", + "description": "CPU usage of the operator process; 1 is one full core." + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=\"$job\"}", + "refId": "A", + "legendFormat": "{{instance}}", + "range": true + } + ], + "title": "Memory RSS", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 39, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "go_goroutines{job=\"$job\"}", + "refId": "A", + "legendFormat": "{{instance}}", + "range": true + } + ], + "title": "Goroutines", + "type": "timeseries" + } + ], + "title": "Process", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 41, + "tags": [ + "ocf" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(controller_runtime_reconcile_total,job)", + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(controller_runtime_reconcile_total,job)", + "refId": "job" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(controller_runtime_reconcile_total{job=\"$job\"},controller)", + "includeAll": true, + "label": "Controller", + "multi": true, + "name": "controller", + "options": [], + "query": { + "query": "label_values(controller_runtime_reconcile_total{job=\"$job\"},controller)", + "refId": "controller" + }, + "refresh": 2, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "1h" + ] + }, + "timezone": "browser", + "title": "OCF Operator", + "uid": "{{operator_namespace}}ocf_operator", + "version": 1 +} diff --git a/observability/dev/simulator/world.go b/observability/dev/simulator/world.go index 2fe2e5a4..2dd0db19 100644 --- a/observability/dev/simulator/world.go +++ b/observability/dev/simulator/world.go @@ -193,17 +193,20 @@ func (w *world) hotLoop(ctx context.Context) { } } -// databases: five owners. orders-db has been Ready=False (Failing) for eight +// databases: six owners. orders-db has been Ready=False (Failing) for eight // hours; users-db is Ready=Unknown; reports-db stays Ready=False while its -// reason flips between Failing and Creating; the rest are healthy. PVC applies -// fail three times out of four, thirty percent of reconciles error, and -// reconcile durations are slow enough for the p99 to cross a minute. +// reason flips between Failing and Creating; the rest are healthy, among them +// the cluster-scoped shared-gateway, whose empty namespace makes the condition +// gauge export series without a namespace label. PVC applies fail three times +// out of four, thirty percent of reconciles error, and reconcile durations are +// slow enough for the p99 to cross a minute. func (w *world) databases(ctx context.Context) { c := w.database orders := owner{namespaceShop, "orders-db"} users := owner{namespaceShop, "users-db"} reports := owner{"analytics", "reports-db"} healthy := []owner{{namespaceShop, "carts-db"}, {"analytics", "events-db"}} + clusterOwner := owner{namespace: "", name: "shared-gateway"} stuckSince := w.start.Add(-8 * time.Hour) unknownSince := w.start.Add(-1 * time.Hour) flipSince := w.start @@ -251,6 +254,8 @@ func (w *world) databases(ctx context.Context) { c.condition(o, "BackupReady", "True", reasonHealthy, w.start) c.condition(o, "Ready", "True", reasonHealthy, w.start) } + c.condition(clusterOwner, "StorageReady", "True", reasonHealthy, w.start) + c.condition(clusterOwner, "Ready", "True", reasonHealthy, w.start) } tick() ticker := time.NewTicker(3 * time.Second) diff --git a/observability/dev/simulator/world_test.go b/observability/dev/simulator/world_test.go index fa8a8b20..094e2063 100644 --- a/observability/dev/simulator/world_test.go +++ b/observability/dev/simulator/world_test.go @@ -51,8 +51,9 @@ func seriesValue(mfs []*dto.MetricFamily, name string, want map[string]string) ( // world and checks that every scenario's series appear with the shapes the // dev stack's alerts key on: the stuck Ready condition with its eight hour // old transition, the failing PVC applies, the hot loop's rising updated -// counter, both controllers reconciling and the leader gauge at one. It then -// cancels the context and requires run to return. +// counter, the cluster-scoped owner without a namespace label, both +// controllers reconciling and the leader gauge at one. It then cancels the +// context and requires run to return. func TestWorldScriptedScenarios(t *testing.T) { reg := prometheus.NewRegistry() conditions := ocm.NewOperatorConditionsGauge("demo") @@ -97,6 +98,15 @@ func TestWorldScriptedScenarios(t *testing.T) { if updated <= firstUpdated { return false } + // The cluster-scoped owner: client_golang exports the declared + // namespace label with an empty value, which Prometheus ingests as no + // namespace label at all, and the id keeps its "/" shape. + if _, ok := seriesValue(mfs, "demo_controller_condition", map[string]string{ + "kind": "Database", "name": "shared-gateway", "namespace": "", + "id": "/shared-gateway", "condition": "Ready", "status": "True", + }); !ok { + return false + } for _, controller := range []string{"webapp", "database"} { if _, ok := seriesValue(mfs, "controller_runtime_reconcile_total", map[string]string{"controller": controller}); !ok { return false diff --git a/observability/observability_test.go b/observability/observability_test.go new file mode 100644 index 00000000..712d03e7 --- /dev/null +++ b/observability/observability_test.go @@ -0,0 +1,206 @@ +// Package observability_test checks the dashboard and alert templates: that +// they render to valid JSON/YAML, keep their uids, leave no placeholder behind +// and reference only metric names that actually exist. +package observability_test + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + ocm "github.com/sourcehawk/go-crd-condition-metrics/pkg/crd-condition-metrics" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" + + "github.com/sourcehawk/operator-component-framework/pkg/metrics" +) + +const ( + lintNamespace = "lint_operator" + nsLabel = "exported_namespace" +) + +// render mirrors the Makefile's render_template. +func render(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + require.NoError(t, err) + s := strings.ReplaceAll(string(b), "{{operator_namespace}}", lintNamespace+"_") + return strings.ReplaceAll(s, "{{namespace_label}}", nsLabel) +} + +// knownMetrics is every metric name the templates may reference: the +// framework's own, the condition gauge, and the controller-runtime, workqueue, +// client-go, leader election, process and Go runtime families. +func knownMetrics(t *testing.T) map[string]bool { + t.Helper() + known := map[string]bool{} + descs := make(chan *prometheus.Desc, 64) + go func() { + metrics.NewCollectors().Describe(descs) + ocm.NewOperatorConditionsGauge(lintNamespace).Describe(descs) + close(descs) + }() + fq := regexp.MustCompile(`fqName: "([^"]+)"`) + for d := range descs { + m := fq.FindStringSubmatch(d.String()) + require.Len(t, m, 2, d.String()) + known[m[1]] = true + } + for _, n := range []string{ + "controller_runtime_reconcile_total", "controller_runtime_reconcile_errors_total", + "controller_runtime_reconcile_panics_total", "controller_runtime_reconcile_time_seconds", + "controller_runtime_max_concurrent_reconciles", "controller_runtime_active_workers", + "workqueue_depth", "workqueue_adds_total", "workqueue_queue_duration_seconds", + "workqueue_work_duration_seconds", "workqueue_unfinished_work_seconds", + "workqueue_longest_running_processor_seconds", "workqueue_retries_total", + "rest_client_requests_total", "leader_election_master_status", + "process_cpu_seconds_total", "process_resident_memory_bytes", "go_goroutines", + } { + known[n] = true + } + return known +} + +// metricNameRe pulls every identifier that looks like one of the metric +// families we care about out of a PromQL expression, including the +// `_bucket`/`_sum`/`_count` suffixes of histograms. +var metricNameRe = regexp.MustCompile(`\b((?:ocf|controller_runtime|workqueue|rest_client|leader_election|process|go)_[a-z0-9_]+|` + lintNamespace + `_controller_condition)\b`) + +// selectorRe pulls the metric name out of every selector-shaped token +// (`name{...}`). It catches typos and unknown metric families that the prefix +// whitelist of metricNameRe skips over. +var selectorRe = regexp.MustCompile(`([a-zA-Z_:][a-zA-Z0-9_:]*)\s*\{`) + +// promqlKeywords are identifiers that may legally precede `{` in PromQL +// without naming a metric. +var promqlKeywords = map[string]bool{ + "by": true, "on": true, "ignoring": true, + "group_left": true, "group_right": true, "without": true, +} + +// placeholderRe matches a `{{name}}` template placeholder. After rendering, +// any hit in a PromQL expression is a mis-spelled placeholder the render left +// behind. +var placeholderRe = regexp.MustCompile(`\{\{ *[a-zA-Z_]+ *\}\}`) + +// assertExprClean checks a rendered PromQL expression: every metric-family +// token and every selector names a known metric, and no template placeholder +// survived rendering. +func assertExprClean(t *testing.T, known map[string]bool, expr, context string) { + t.Helper() + for _, m := range metricNameRe.FindAllString(expr, -1) { + assert.True(t, known[baseName(m)], "unknown metric %q in %s", m, context) + } + for _, m := range selectorRe.FindAllStringSubmatch(expr, -1) { + if promqlKeywords[m[1]] { + continue + } + assert.True(t, known[baseName(m[1])], "unknown selector metric %q in %s", m[1], context) + } + assert.NotRegexp(t, placeholderRe, expr, "unrendered placeholder in %s", context) +} + +// baseName strips the histogram sample suffixes off a series name so that it +// can be looked up as a metric family. +func baseName(n string) string { + for _, s := range []string{"_bucket", "_sum", "_count"} { + n = strings.TrimSuffix(n, s) + } + return n +} + +// exprsFromJSON collects every PromQL-carrying string (panel targets, variable +// queries and definitions) from a decoded dashboard. +func exprsFromJSON(v any, out *[]string) { + switch x := v.(type) { + case map[string]any: + for k, val := range x { + if k == "expr" || k == "query" || k == "definition" { + if s, ok := val.(string); ok { + *out = append(*out, s) + } + } + exprsFromJSON(val, out) + } + case []any: + for _, val := range x { + exprsFromJSON(val, out) + } + } +} + +func TestDashboards(t *testing.T) { + files, err := filepath.Glob("dashboards/*.tpl.json") + require.NoError(t, err) + require.NotEmpty(t, files) + known := knownMetrics(t) + for _, f := range files { + t.Run(filepath.Base(f), func(t *testing.T) { + rendered := render(t, f) + assert.NotContains(t, rendered, "{{operator_namespace}}") + assert.NotContains(t, rendered, "{{namespace_label}}") + var d map[string]any + require.NoError(t, json.Unmarshal([]byte(rendered), &d), "valid JSON") + want := strings.TrimSuffix(filepath.Base(f), ".tpl.json") + assert.Equal(t, lintNamespace+"_"+want, d["uid"], "uid is the file name prefixed with the metric namespace") + assert.Equal(t, "", d["refresh"], "auto refresh is off by default") + var exprs []string + exprsFromJSON(d, &exprs) + require.NotEmpty(t, exprs) + for _, e := range exprs { + assertExprClean(t, known, e, strconv.Quote(e)) + } + }) + } +} + +func TestAlerts(t *testing.T) { + // The glob matches the shared files and, because `.tpl.yaml` also ends in + // `.yaml`, the templated ones too; render handles both. + files, err := filepath.Glob("alerts/*.yaml") + require.NoError(t, err) + require.NotEmpty(t, files) + known := knownMetrics(t) + for _, f := range files { + t.Run(filepath.Base(f), func(t *testing.T) { + rendered := render(t, f) + assert.NotContains(t, rendered, "{{operator_namespace}}") + assert.NotContains(t, rendered, "{{namespace_label}}") + var doc struct { + Groups []struct { + Name string `json:"name"` + Rules []struct { + Alert string `json:"alert"` + Expr string `json:"expr"` + Labels map[string]string `json:"labels"` + } `json:"rules"` + } `json:"groups"` + } + require.NoError(t, yaml.Unmarshal([]byte(rendered), &doc)) + require.NotEmpty(t, doc.Groups) + for _, g := range doc.Groups { + for _, r := range g.Rules { + assert.Len(t, r.Labels, 1, "%s carries only the severity label", r.Alert) + assert.Contains(t, []string{"warning", "critical"}, r.Labels["severity"], "%s severity", r.Alert) + assertExprClean(t, known, r.Expr, r.Alert) + } + } + // Every alert in the file has a promtool unit test. + base := strings.TrimSuffix(strings.TrimSuffix(filepath.Base(f), ".yaml"), ".tpl") + tests, err := os.ReadFile(filepath.Join("alerts", "tests", base+"_test.yaml")) + require.NoError(t, err, "every rule file has a promtool test file") + for _, g := range doc.Groups { + for _, r := range g.Rules { + assert.Contains(t, string(tests), "alertname: "+r.Alert+"\n", "%s has a unit test", r.Alert) + } + } + }) + } +}