From b75413e458854c5de0d706bcb3449a8605b757a6 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Sun, 13 Sep 2026 22:01:15 +0100 Subject: [PATCH 1/3] feat(helm): add portable Imp dashboards --- charts/imp/dashboards/imp-admin.json | 27 +++++++++++++++++++ charts/imp/dashboards/imp-github-runners.json | 26 ++++++++++++++++++ charts/imp/dashboards/imp-workloads.json | 26 ++++++++++++++++++ charts/imp/templates/grafana/dashboard.yaml | 6 +++++ charts/imp/tests/grafana_test.yaml | 6 +++++ 5 files changed, 91 insertions(+) create mode 100644 charts/imp/dashboards/imp-admin.json create mode 100644 charts/imp/dashboards/imp-github-runners.json create mode 100644 charts/imp/dashboards/imp-workloads.json diff --git a/charts/imp/dashboards/imp-admin.json b/charts/imp/dashboards/imp-admin.json new file mode 100644 index 0000000..7401f21 --- /dev/null +++ b/charts/imp/dashboards/imp-admin.json @@ -0,0 +1,27 @@ +{ + "annotations": {"list": []}, + "description": "Imp administrator view. Uses only Prometheus metrics emitted by Imp and works with any Prometheus-compatible Grafana datasource.", + "editable": true, + "refresh": "30s", + "schemaVersion": 38, + "tags": ["imp", "administrator", "microvm"], + "time": {"from": "now-1h", "to": "now"}, + "timezone": "browser", + "title": "Imp administrator", + "uid": "imp-admin-v1", + "version": 1, + "templating": {"list": [ + {"name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "refresh": 1, "current": {}}, + {"name": "namespace", "label": "Namespace", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state, namespace)", "query": {"query": "label_values(imp_vm_state, namespace)", "refId": "NamespaceVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}} + ]}, + "panels": [ + {"id": 1, "type": "stat", "title": "Runtime ready", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, "targets": [{"refId": "A", "expr": "min(imp_runtime_ready) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}}, + {"id": 2, "type": "stat", "title": "Active VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{namespace=~\"$namespace\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 3, "type": "stat", "title": "Running VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{namespace=~\"$namespace\",state=\"Running\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 4, "type": "stat", "title": "Reconcile errors / second", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, "targets": [{"refId": "A", "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"impvm\"}[5m])) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 5, "type": "timeseries", "title": "VM state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, "targets": [{"refId": "A", "expr": "sum by (state) (imp_vm_state{namespace=~\"$namespace\"})", "legendFormat": "{{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 6, "type": "timeseries", "title": "Scheduling latency p95", "description": "All namespaces. Histograms currently have no namespace label.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_scheduling_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 7, "type": "timeseries", "title": "Boot latency p95", "description": "All namespaces. Histograms currently have no namespace label.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 8, "type": "timeseries", "title": "Reconcile activity", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, "targets": [{"refId": "A", "expr": "sum(rate(controller_runtime_reconcile_total{controller=\"impvm\"}[5m])) by (result)", "legendFormat": "{{result}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}} + ] +} diff --git a/charts/imp/dashboards/imp-github-runners.json b/charts/imp/dashboards/imp-github-runners.json new file mode 100644 index 0000000..39a8bc0 --- /dev/null +++ b/charts/imp/dashboards/imp-github-runners.json @@ -0,0 +1,26 @@ +{ + "annotations": {"list": []}, + "description": "GitHub runner VM view. It uses Imp VM state and naming only; GitHub queue and job-assignment metrics are intentionally not inferred when they are not exported.", + "editable": true, + "refresh": "30s", + "schemaVersion": 38, + "tags": ["imp", "github", "runner", "microvm"], + "time": {"from": "now-1h", "to": "now"}, + "timezone": "browser", + "title": "Imp GitHub runners", + "uid": "imp-github-runners-v1", + "version": 1, + "templating": {"list": [ + {"name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "refresh": 1, "current": {}}, + {"name": "runner", "label": "Runner VM", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state{impvm=~\".*github-runner.*\"}, impvm)", "query": {"query": "label_values(imp_vm_state{impvm=~\".*github-runner.*\"}, impvm)", "refId": "RunnerVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}} + ]}, + "panels": [ + {"id": 1, "type": "stat", "title": "Runner VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{impvm=~\".*github-runner.*\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 2, "type": "stat", "title": "Running runner VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{impvm=~\".*github-runner.*\",state=\"Running\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 3, "type": "stat", "title": "Runtime ready", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, "targets": [{"refId": "A", "expr": "min(imp_runtime_ready) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}}, + {"id": 4, "type": "text", "title": "Scope", "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, "options": {"mode": "markdown", "content": "This dashboard updates from Imp VM state. GitHub job queue, registration, and assignment data appear only when Imp exports dedicated metrics; this dashboard does not guess them."}}, + {"id": 5, "type": "timeseries", "title": "Runner VM state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 9, "w": 12, "x": 0, "y": 6}, "targets": [{"refId": "A", "expr": "imp_vm_state{impvm=~\"$runner\"}", "legendFormat": "{{impvm}} · {{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 6, "type": "timeseries", "title": "Runner VMs by state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 9, "w": 12, "x": 12, "y": 6}, "targets": [{"refId": "A", "expr": "sum by (state) (imp_vm_state{impvm=~\".*github-runner.*\"})", "legendFormat": "{{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 7, "type": "timeseries", "title": "Boot latency p95", "description": "Fleet-wide lifecycle metric; runner-specific latency labels are not exported yet.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 24, "x": 0, "y": 15}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}} + ] +} diff --git a/charts/imp/dashboards/imp-workloads.json b/charts/imp/dashboards/imp-workloads.json new file mode 100644 index 0000000..640c3ef --- /dev/null +++ b/charts/imp/dashboards/imp-workloads.json @@ -0,0 +1,26 @@ +{ + "annotations": {"list": []}, + "description": "Imp workload view. Select a namespace and VM to see the capacity and lifecycle signals that Imp currently publishes.", + "editable": true, + "refresh": "30s", + "schemaVersion": 38, + "tags": ["imp", "workload", "microvm"], + "time": {"from": "now-1h", "to": "now"}, + "timezone": "browser", + "title": "Imp workloads", + "uid": "imp-workloads-v1", + "version": 1, + "templating": {"list": [ + {"name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "refresh": 1, "current": {}}, + {"name": "namespace", "label": "Namespace", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state, namespace)", "query": {"query": "label_values(imp_vm_state, namespace)", "refId": "NamespaceVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}}, + {"name": "vm", "label": "VM", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state{namespace=~\"$namespace\"}, impvm)", "query": {"query": "label_values(imp_vm_state{namespace=~\"$namespace\"}, impvm)", "refId": "VMVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}} + ]}, + "panels": [ + {"id": 1, "type": "stat", "title": "Selected VM state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 8, "x": 0, "y": 0}, "targets": [{"refId": "A", "expr": "sum(imp_vm_state{namespace=~\"$namespace\",impvm=~\"$vm\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 2, "type": "stat", "title": "Runtime ready", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 8, "x": 8, "y": 0}, "targets": [{"refId": "A", "expr": "min(imp_runtime_ready) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}}, + {"id": 3, "type": "stat", "title": "VMs running", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 8, "x": 16, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{namespace=~\"$namespace\",state=\"Running\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, + {"id": 4, "type": "timeseries", "title": "VM state history", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, "targets": [{"refId": "A", "expr": "imp_vm_state{namespace=~\"$namespace\",impvm=~\"$vm\"}", "legendFormat": "{{impvm}} · {{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 5, "type": "timeseries", "title": "Boot latency p95", "description": "Current Imp boot histograms are fleet-wide, not per-workload.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, + {"id": 6, "type": "timeseries", "title": "Scheduling latency p95", "description": "Current Imp scheduling histograms are fleet-wide, not per-workload.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 24, "x": 0, "y": 14}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_scheduling_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}} + ] +} diff --git a/charts/imp/templates/grafana/dashboard.yaml b/charts/imp/templates/grafana/dashboard.yaml index 32781d1..eab1696 100644 --- a/charts/imp/templates/grafana/dashboard.yaml +++ b/charts/imp/templates/grafana/dashboard.yaml @@ -10,6 +10,12 @@ metadata: data: imp-overview.json: |- {{ .Files.Get "dashboards/imp-overview.json" | indent 4 }} + imp-admin.json: |- +{{ .Files.Get "dashboards/imp-admin.json" | indent 4 }} + imp-workloads.json: |- +{{ .Files.Get "dashboards/imp-workloads.json" | indent 4 }} + imp-github-runners.json: |- +{{ .Files.Get "dashboards/imp-github-runners.json" | indent 4 }} imp-traces.json: |- {{ .Files.Get "dashboards/imp-traces.json" | indent 4 }} {{- end }} diff --git a/charts/imp/tests/grafana_test.yaml b/charts/imp/tests/grafana_test.yaml index 4525765..588e0ea 100644 --- a/charts/imp/tests/grafana_test.yaml +++ b/charts/imp/tests/grafana_test.yaml @@ -14,6 +14,12 @@ tests: value: "1" - isNotEmpty: path: data["imp-overview.json"] + - isNotEmpty: + path: data["imp-admin.json"] + - isNotEmpty: + path: data["imp-workloads.json"] + - isNotEmpty: + path: data["imp-github-runners.json"] - it: renders no resources when grafana.enabled=false template: templates/grafana/dashboard.yaml From 09d2c27e804fe08ba9ce19a108035dbd0c5b3d25 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Sun, 13 Sep 2026 22:06:07 +0100 Subject: [PATCH 2/3] fix(dashboards): preserve telemetry uncertainty --- charts/imp/dashboards/imp-admin.json | 356 +++++++++++++++++- charts/imp/dashboards/imp-github-runners.json | 156 +++++++- charts/imp/dashboards/imp-workloads.json | 201 +++++++++- 3 files changed, 667 insertions(+), 46 deletions(-) diff --git a/charts/imp/dashboards/imp-admin.json b/charts/imp/dashboards/imp-admin.json index 7401f21..7402484 100644 --- a/charts/imp/dashboards/imp-admin.json +++ b/charts/imp/dashboards/imp-admin.json @@ -1,27 +1,351 @@ { - "annotations": {"list": []}, - "description": "Imp administrator view. Uses only Prometheus metrics emitted by Imp and works with any Prometheus-compatible Grafana datasource.", + "annotations": { + "list": [] + }, + "description": "Technical health. Missing data means unknown or missing telemetry, not healthy.", "editable": true, "refresh": "30s", "schemaVersion": 38, - "tags": ["imp", "administrator", "microvm"], - "time": {"from": "now-1h", "to": "now"}, + "tags": [ + "imp", + "administrator", + "microvm" + ], + "time": { + "from": "now-1h", + "to": "now" + }, "timezone": "browser", "title": "Imp administrator", "uid": "imp-admin-v1", "version": 1, - "templating": {"list": [ - {"name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "refresh": 1, "current": {}}, - {"name": "namespace", "label": "Namespace", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state, namespace)", "query": {"query": "label_values(imp_vm_state, namespace)", "refId": "NamespaceVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}} - ]}, + "templating": { + "list": [ + { + "name": "datasource", + "label": "Data source", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "current": {} + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(imp_vm_state, namespace)", + "query": { + "query": "label_values(imp_vm_state, namespace)", + "refId": "NamespaceVariableQuery" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 2, + "current": {} + }, + { + "name": "vm", + "label": "VM", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(imp_vm_state{namespace=~\"${namespace:regex}\"}, impvm)", + "query": { + "query": "label_values(imp_vm_state{namespace=~\"${namespace:regex}\"}, impvm)", + "refId": "VMVariableQuery" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 2, + "current": {} + } + ] + }, "panels": [ - {"id": 1, "type": "stat", "title": "Runtime ready", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, "targets": [{"refId": "A", "expr": "min(imp_runtime_ready) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}}, - {"id": 2, "type": "stat", "title": "Active VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{namespace=~\"$namespace\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 3, "type": "stat", "title": "Running VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{namespace=~\"$namespace\",state=\"Running\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 4, "type": "stat", "title": "Reconcile errors / second", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, "targets": [{"refId": "A", "expr": "sum(rate(controller_runtime_reconcile_errors_total{controller=\"impvm\"}[5m])) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 5, "type": "timeseries", "title": "VM state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, "targets": [{"refId": "A", "expr": "sum by (state) (imp_vm_state{namespace=~\"$namespace\"})", "legendFormat": "{{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 6, "type": "timeseries", "title": "Scheduling latency p95", "description": "All namespaces. Histograms currently have no namespace label.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_scheduling_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 7, "type": "timeseries", "title": "Boot latency p95", "description": "All namespaces. Histograms currently have no namespace label.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 14}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 8, "type": "timeseries", "title": "Reconcile activity", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 14}, "targets": [{"refId": "A", "expr": "sum(rate(controller_runtime_reconcile_total{controller=\"impvm\"}[5m])) by (result)", "legendFormat": "{{result}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}} + { + "id": 1, + "type": "stat", + "title": "Runtime ready", + "description": "Missing series means unknown or missing telemetry.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "imp_runtime_ready" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Observed VMs by state", + "description": "Missing series means no observed VMs or missing telemetry.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (state) (imp_vm_state{namespace=~\"${namespace:regex}\",impvm=~\"${vm:regex}\"} == 1)", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Observed VMs by node", + "description": "Missing series means no observed VMs or missing telemetry.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (node) (imp_vm_state{namespace=~\"${namespace:regex}\",impvm=~\"${vm:regex}\"} == 1)", + "legendFormat": "{{node}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 4, + "type": "timeseries", + "title": "Provisioning latency p95", + "description": "Cluster-wide; namespace and VM filters do not apply.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_scheduling_latency_seconds_bucket[$__rate_interval])))", + "legendFormat": "scheduling p95" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Boot latency p95", + "description": "Cluster-wide; namespace and VM filters do not apply.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[$__rate_interval])))", + "legendFormat": "boot p95" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 6, + "type": "stat", + "title": "Recorded scheduling transitions", + "description": "Observed histogram events; not unique VM counts.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(increase(imp_vm_scheduling_latency_seconds_count[$__range]))" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 7, + "type": "stat", + "title": "Recorded boot transitions", + "description": "Observed histogram events; not unique VM counts.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 16 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(increase(imp_vm_boot_latency_seconds_count[$__range]))" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } ] } diff --git a/charts/imp/dashboards/imp-github-runners.json b/charts/imp/dashboards/imp-github-runners.json index 39a8bc0..9304e60 100644 --- a/charts/imp/dashboards/imp-github-runners.json +++ b/charts/imp/dashboards/imp-github-runners.json @@ -1,26 +1,152 @@ { - "annotations": {"list": []}, - "description": "GitHub runner VM view. It uses Imp VM state and naming only; GitHub queue and job-assignment metrics are intentionally not inferred when they are not exported.", + "annotations": { + "list": [] + }, + "description": "Runner VM infrastructure only. Select independently confirmed runner VM names. VM state does not prove GitHub registration, idle/busy state, job execution, or cleanup.", "editable": true, "refresh": "30s", "schemaVersion": 38, - "tags": ["imp", "github", "runner", "microvm"], - "time": {"from": "now-1h", "to": "now"}, + "tags": [ + "imp", + "github", + "runner", + "microvm" + ], + "time": { + "from": "now-1h", + "to": "now" + }, "timezone": "browser", "title": "Imp GitHub runners", "uid": "imp-github-runners-v1", "version": 1, - "templating": {"list": [ - {"name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "refresh": 1, "current": {}}, - {"name": "runner", "label": "Runner VM", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state{impvm=~\".*github-runner.*\"}, impvm)", "query": {"query": "label_values(imp_vm_state{impvm=~\".*github-runner.*\"}, impvm)", "refId": "RunnerVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}} - ]}, + "templating": { + "list": [ + { + "name": "datasource", + "label": "Data source", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "current": {} + }, + { + "name": "runner_vm", + "label": "Runner VM", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(imp_vm_state, impvm)", + "query": { + "query": "label_values(imp_vm_state, impvm)", + "refId": "RunnerVMVariableQuery" + }, + "includeAll": false, + "multi": true, + "refresh": 2, + "current": {} + } + ] + }, "panels": [ - {"id": 1, "type": "stat", "title": "Runner VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 0, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{impvm=~\".*github-runner.*\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 2, "type": "stat", "title": "Running runner VMs", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 6, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{impvm=~\".*github-runner.*\",state=\"Running\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 3, "type": "stat", "title": "Runtime ready", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 6, "x": 12, "y": 0}, "targets": [{"refId": "A", "expr": "min(imp_runtime_ready) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}}, - {"id": 4, "type": "text", "title": "Scope", "gridPos": {"h": 6, "w": 6, "x": 18, "y": 0}, "options": {"mode": "markdown", "content": "This dashboard updates from Imp VM state. GitHub job queue, registration, and assignment data appear only when Imp exports dedicated metrics; this dashboard does not guess them."}}, - {"id": 5, "type": "timeseries", "title": "Runner VM state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 9, "w": 12, "x": 0, "y": 6}, "targets": [{"refId": "A", "expr": "imp_vm_state{impvm=~\"$runner\"}", "legendFormat": "{{impvm}} · {{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 6, "type": "timeseries", "title": "Runner VMs by state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 9, "w": 12, "x": 12, "y": 6}, "targets": [{"refId": "A", "expr": "sum by (state) (imp_vm_state{impvm=~\".*github-runner.*\"})", "legendFormat": "{{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 7, "type": "timeseries", "title": "Boot latency p95", "description": "Fleet-wide lifecycle metric; runner-specific latency labels are not exported yet.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 24, "x": 0, "y": 15}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}} + { + "id": 1, + "type": "text", + "title": "Scope", + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "options": { + "mode": "markdown", + "content": "**Runner VM infrastructure only.** Select independently confirmed runner VM names. VM state does not prove GitHub registration, idle/busy state, job execution, or cleanup." + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Selected runner VM observations", + "description": "No selected runner VM telemetry.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "expr": "imp_vm_state{impvm=~\"${runner_vm:regex}\"} == 1", + "legendFormat": "{{impvm}} \u00b7 {{namespace}} \u00b7 {{node}} \u00b7 {{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Selected runner VMs by state", + "description": "No selected runner VM telemetry.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (state) (imp_vm_state{impvm=~\"${runner_vm:regex}\"} == 1)", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + } ] } diff --git a/charts/imp/dashboards/imp-workloads.json b/charts/imp/dashboards/imp-workloads.json index 640c3ef..2a0a4d6 100644 --- a/charts/imp/dashboards/imp-workloads.json +++ b/charts/imp/dashboards/imp-workloads.json @@ -1,26 +1,197 @@ { - "annotations": {"list": []}, - "description": "Imp workload view. Select a namespace and VM to see the capacity and lifecycle signals that Imp currently publishes.", + "annotations": { + "list": [] + }, + "description": "Lifecycle observations for workload owners. Running does not prove application readiness.", "editable": true, "refresh": "30s", "schemaVersion": 38, - "tags": ["imp", "workload", "microvm"], - "time": {"from": "now-1h", "to": "now"}, + "tags": [ + "imp", + "workload", + "microvm" + ], + "time": { + "from": "now-1h", + "to": "now" + }, "timezone": "browser", "title": "Imp workloads", "uid": "imp-workloads-v1", "version": 1, - "templating": {"list": [ - {"name": "datasource", "label": "Data source", "type": "datasource", "query": "prometheus", "refresh": 1, "current": {}}, - {"name": "namespace", "label": "Namespace", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state, namespace)", "query": {"query": "label_values(imp_vm_state, namespace)", "refId": "NamespaceVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}}, - {"name": "vm", "label": "VM", "type": "query", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "definition": "label_values(imp_vm_state{namespace=~\"$namespace\"}, impvm)", "query": {"query": "label_values(imp_vm_state{namespace=~\"$namespace\"}, impvm)", "refId": "VMVariableQuery"}, "includeAll": true, "allValue": ".*", "multi": false, "refresh": 2, "current": {}} - ]}, + "templating": { + "list": [ + { + "name": "datasource", + "label": "Data source", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "current": {} + }, + { + "name": "namespace", + "label": "Namespace", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(imp_vm_state, namespace)", + "query": { + "query": "label_values(imp_vm_state, namespace)", + "refId": "NamespaceVariableQuery" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 2, + "current": {} + }, + { + "name": "vm", + "label": "VM", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(imp_vm_state{namespace=~\"${namespace:regex}\"}, impvm)", + "query": { + "query": "label_values(imp_vm_state{namespace=~\"${namespace:regex}\"}, impvm)", + "refId": "VMVariableQuery" + }, + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 2, + "current": {} + } + ] + }, "panels": [ - {"id": 1, "type": "stat", "title": "Selected VM state", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 8, "x": 0, "y": 0}, "targets": [{"refId": "A", "expr": "sum(imp_vm_state{namespace=~\"$namespace\",impvm=~\"$vm\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 2, "type": "stat", "title": "Runtime ready", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 8, "x": 8, "y": 0}, "targets": [{"refId": "A", "expr": "min(imp_runtime_ready) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {"thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}}, "overrides": []}}, - {"id": 3, "type": "stat", "title": "VMs running", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 6, "w": 8, "x": 16, "y": 0}, "targets": [{"refId": "A", "expr": "count(imp_vm_state{namespace=~\"$namespace\",state=\"Running\"} == 1) or vector(0)"}], "options": {"reduceOptions": {"calcs": ["lastNotNull"]}, "orientation": "auto", "textMode": "auto", "colorMode": "value", "graphMode": "none"}, "fieldConfig": {"defaults": {}, "overrides": []}}, - {"id": 4, "type": "timeseries", "title": "VM state history", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 0, "y": 6}, "targets": [{"refId": "A", "expr": "imp_vm_state{namespace=~\"$namespace\",impvm=~\"$vm\"}", "legendFormat": "{{impvm}} · {{state}}"}], "fieldConfig": {"defaults": {"custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "table", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 5, "type": "timeseries", "title": "Boot latency p95", "description": "Current Imp boot histograms are fleet-wide, not per-workload.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 12, "x": 12, "y": 6}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_boot_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}}, - {"id": 6, "type": "timeseries", "title": "Scheduling latency p95", "description": "Current Imp scheduling histograms are fleet-wide, not per-workload.", "datasource": {"type": "prometheus", "uid": "${datasource}"}, "gridPos": {"h": 8, "w": 24, "x": 0, "y": 14}, "targets": [{"refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(imp_vm_scheduling_latency_seconds_bucket[5m])))", "legendFormat": "p95"}], "fieldConfig": {"defaults": {"unit": "s", "custom": {"lineWidth": 1, "showPoints": "never"}}, "overrides": []}, "options": {"legend": {"displayMode": "list", "placement": "bottom"}, "tooltip": {"mode": "multi"}}} + { + "id": 1, + "type": "timeseries", + "title": "Current observed VM inventory", + "description": "No matching VM observations can mean filters are wrong or telemetry is missing.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "targets": [ + { + "refId": "A", + "expr": "imp_vm_state{namespace=~\"${namespace:regex}\",impvm=~\"${vm:regex}\"} == 1", + "legendFormat": "{{namespace}} \u00b7 {{impvm}} \u00b7 {{node}} \u00b7 {{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 2, + "type": "timeseries", + "title": "Selected VMs by state", + "description": "No matching VM observations can mean filters are wrong or telemetry is missing.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (state) (imp_vm_state{namespace=~\"${namespace:regex}\",impvm=~\"${vm:regex}\"} == 1)", + "legendFormat": "{{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 3, + "type": "timeseries", + "title": "Observed lifecycle history", + "description": "Gaps are missing samples. The chart does not infer transitions between scrapes.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "targets": [ + { + "refId": "A", + "expr": "imp_vm_state{namespace=~\"${namespace:regex}\",impvm=~\"${vm:regex}\"} == 1", + "legendFormat": "{{impvm}} \u00b7 {{state}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "never" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + } ] } From 1ee3872a6def5fd68e8943c2d5b2a9fb92702f09 Mon Sep 17 00:00:00 2001 From: Giovanni Ferri Date: Sun, 13 Sep 2026 22:26:09 +0100 Subject: [PATCH 3/3] feat(runners): expose handoff activity metrics --- .../workflows/imp-dashboard-metrics-demo.yml | 35 ++++++++ charts/imp/dashboards/imp-github-runners.json | 86 ++++++++++++++++++- internal/agent/metrics.go | 37 ++++++++ internal/agent/metrics_test.go | 17 ++++ internal/agent/reconciler.go | 13 +++ internal/agent/runnerlaunch/launcher.go | 14 ++- 6 files changed, 198 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/imp-dashboard-metrics-demo.yml diff --git a/.github/workflows/imp-dashboard-metrics-demo.yml b/.github/workflows/imp-dashboard-metrics-demo.yml new file mode 100644 index 0000000..aed9859 --- /dev/null +++ b/.github/workflows/imp-dashboard-metrics-demo.yml @@ -0,0 +1,35 @@ +name: Imp dashboard metrics demo + +on: + workflow_dispatch: + inputs: + hold_seconds: + description: Seconds to keep the runner handoff active for dashboard inspection + required: true + default: "120" + type: choice + options: ["60", "120", "300"] + +permissions: + contents: read + +jobs: + dashboard-demo: + name: Create observable runner activity + runs-on: + group: omni-runner + labels: [self-hosted, imp] + timeout-minutes: 10 + steps: + - name: Report the dashboard correlation fields + run: | + printf '%s\n' \ + "github_repository=$GITHUB_REPOSITORY" \ + "github_run_id=$GITHUB_RUN_ID" \ + "github_run_attempt=$GITHUB_RUN_ATTEMPT" \ + "github_job=$GITHUB_JOB" \ + "runner_name=$RUNNER_NAME" + - name: Hold the runner handoff open + run: | + echo "holding runner activity for ${{ inputs.hold_seconds }} seconds" + sleep "${{ inputs.hold_seconds }}" diff --git a/charts/imp/dashboards/imp-github-runners.json b/charts/imp/dashboards/imp-github-runners.json index 9304e60..f7f2848 100644 --- a/charts/imp/dashboards/imp-github-runners.json +++ b/charts/imp/dashboards/imp-github-runners.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "Runner VM infrastructure only. Select independently confirmed runner VM names. VM state does not prove GitHub registration, idle/busy state, job execution, or cleanup.", + "description": "Runner VM infrastructure and guest runner handoffs. GitHub registration, job assignment, and cleanup need separate GitHub telemetry.", "editable": true, "refresh": "30s", "schemaVersion": 38, @@ -19,7 +19,7 @@ "timezone": "browser", "title": "Imp GitHub runners", "uid": "imp-github-runners-v1", - "version": 1, + "version": 2, "templating": { "list": [ { @@ -63,7 +63,7 @@ }, "options": { "mode": "markdown", - "content": "**Runner VM infrastructure only.** Select independently confirmed runner VM names. VM state does not prove GitHub registration, idle/busy state, job execution, or cleanup." + "content": "**Runner VM infrastructure and handoff activity.** The handoff panels show Imp delivering a one-time runner configuration to a guest. They do not prove GitHub registration, job assignment, or cleanup." } }, { @@ -147,6 +147,86 @@ "mode": "multi" } } + }, + { + "id": 4, + "type": "stat", + "title": "Active runner handoffs", + "description": "A nonzero value means Imp is waiting for a guest runner process to exit.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "targets": [ + { + "refId": "A", + "expr": "sum(imp_runner_handoffs_active{impvm=~\"${runner_vm:regex}\"})" + } + ], + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "orientation": "auto", + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "decimals": 0 + }, + "overrides": [] + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Completed runner handoffs", + "description": "Successful and failed handoffs observed by this agent.", + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (outcome) (increase(imp_runner_handoffs_total{impvm=~\"${runner_vm:regex}\"}[$__rate_interval]))", + "legendFormat": "{{outcome}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "showPoints": "always" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } } ] } diff --git a/internal/agent/metrics.go b/internal/agent/metrics.go index d06b9ec..b9128aa 100644 --- a/internal/agent/metrics.go +++ b/internal/agent/metrics.go @@ -36,6 +36,8 @@ type VMMetricsCollector struct { vmStates map[string]vmStateEntry // "ns/name" → {state, node} guestMetrics map[string]*guestData // "ns/name" → data gatherer prometheus.Gatherer + runnerActive metric.Int64UpDownCounter + runnerTotal metric.Int64Counter } // NewVMMetricsCollector creates a new collector using the provided OTel meter. @@ -47,6 +49,14 @@ func NewVMMetricsCollector(meter metric.Meter, gatherer prometheus.Gatherer) *VM guestMetrics: make(map[string]*guestData), gatherer: gatherer, } + c.runnerActive, _ = meter.Int64UpDownCounter( + "imp_runner_handoffs_active", + metric.WithDescription("Runner handoffs currently executing in guest VMs."), + ) + c.runnerTotal, _ = meter.Int64Counter( + "imp_runner_handoffs", + metric.WithDescription("Completed runner handoffs by outcome."), + ) _, _ = meter.Float64ObservableGauge( "imp_vm_state", @@ -175,6 +185,33 @@ func (c *VMMetricsCollector) ClearVM(key string) { delete(c.guestMetrics, key) } +// RecordRunnerHandoffStarted records a guest runner handoff that has begun. +func (c *VMMetricsCollector) RecordRunnerHandoffStarted(key, node string) { + ns, name := splitKey(key) + c.runnerActive.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("namespace", ns), + attribute.String("impvm", name), + attribute.String("node", node), + )) +} + +// RecordRunnerHandoffFinished records a completed guest runner handoff. +func (c *VMMetricsCollector) RecordRunnerHandoffFinished(key, node, outcome string) { + ns, name := splitKey(key) + attrs := metric.WithAttributes( + attribute.String("namespace", ns), + attribute.String("impvm", name), + attribute.String("node", node), + ) + c.runnerActive.Add(context.Background(), -1, attrs) + c.runnerTotal.Add(context.Background(), 1, metric.WithAttributes( + attribute.String("namespace", ns), + attribute.String("impvm", name), + attribute.String("node", node), + attribute.String("outcome", outcome), + )) +} + // NewMetricsHandlerWithCollector returns an HTTP handler for the collector's Prometheus registry. func NewMetricsHandlerWithCollector(c *VMMetricsCollector) http.Handler { return promhttp.HandlerFor(c.gatherer, promhttp.HandlerOpts{}) diff --git a/internal/agent/metrics_test.go b/internal/agent/metrics_test.go index bfc051b..c0e6027 100644 --- a/internal/agent/metrics_test.go +++ b/internal/agent/metrics_test.go @@ -81,3 +81,20 @@ func TestVMMetrics_guestMetricsAppear(t *testing.T) { } } } + +func TestVMMetrics_runnerHandoffMetricsAppear(t *testing.T) { + mc, h := newTestCollector(t) + mc.RecordRunnerHandoffStarted("default/test-vm", "test-node") + mc.RecordRunnerHandoffFinished("default/test-vm", "test-node", "success") + + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + w := httptest.NewRecorder() + h.ServeHTTP(w, req) + + body := w.Body.String() + for _, metric := range []string{"imp_runner_handoffs_active", "imp_runner_handoffs_total"} { + if !strings.Contains(body, metric) { + t.Errorf("expected %s in output, got:\n%s", metric, body) + } + } +} diff --git a/internal/agent/reconciler.go b/internal/agent/reconciler.go index d57cb56..121f9c3 100644 --- a/internal/agent/reconciler.go +++ b/internal/agent/reconciler.go @@ -264,6 +264,19 @@ func (r *ImpVMReconciler) maybeLaunchRunner(ctx context.Context, vm *impdevv1alp Reader: r.APIReader, Sock: sockPath, } + if r.Metrics != nil { + key := vm.Namespace + "/" + vm.Name + launch.OnStarted = func() { + r.Metrics.RecordRunnerHandoffStarted(key, r.NodeName) + } + launch.OnFinished = func(res runnerlaunch.Result) { + outcome := "success" + if res.Err != nil || res.ExitCode != 0 { + outcome = "failed" + } + r.Metrics.RecordRunnerHandoffFinished(key, r.NodeName, outcome) + } + } go func() { launchRunner := r.runnerLaunch if launchRunner == nil { diff --git a/internal/agent/runnerlaunch/launcher.go b/internal/agent/runnerlaunch/launcher.go index d6cd63e..afeb18f 100644 --- a/internal/agent/runnerlaunch/launcher.go +++ b/internal/agent/runnerlaunch/launcher.go @@ -125,12 +125,16 @@ type Launcher struct { Reader client.Reader // uncached reader; avoids a cluster-wide Secret informer Sock string // path to the VM's VSOCK unix socket proxy Log *slog.Logger + // OnStarted and OnFinished observe a real, de-duplicated handoff attempt. + // They must not read or log the JIT payload. + OnStarted func() + OnFinished func(Result) } // Run performs the handoff and blocks until the runner exits (or fails). // It is safe to call in a goroutine. Cancel ctx to abandon (the guest keeps // running; Firecracker exit handling stays with the reconciler). -func (l *Launcher) Run(ctx context.Context, vm *impv1alpha1.ImpVM) Result { +func (l *Launcher) Run(ctx context.Context, vm *impv1alpha1.ImpVM) (result Result) { log := l.Log if log == nil { log = slog.Default() @@ -144,6 +148,14 @@ func (l *Launcher) Run(ctx context.Context, vm *impv1alpha1.ImpVM) Result { return Result{} } defer launches.Delete(launchKey) + if l.OnStarted != nil { + l.OnStarted() + } + defer func() { + if l.OnFinished != nil { + l.OnFinished(result) + } + }() // The accepted marker is written before Exec because the unary RPC has no // acknowledgement boundary: it returns only when the guest process exits.