Route every severity a rule claims to a contact point that exists

.github/workflows/ci.yml

@@ -118,6 +118,16 @@ jobs:
       - name: Excluded alert states are somebody's job
         run: ./scripts/check-alert-coverage.py
 
+      # A rule labelled to page is asking for a human to be woken. Grafana keeps
+      # that promise by matching the label against a notification policy; with
+      # no matching route the alert falls to the tree's root, and with no tree at
+      # all to the workspace default policy and its empty default contact point.
+      # The rule and the policy never name each other, so nothing else in the
+      # tree can see the gap — and the signal arrives at the moment nobody is
+      # watching a dashboard.
+      - name: Every alert severity routes to a declared contact point
+        run: ./scripts/check-alert-severity-routes.py
+
   # ── Kyverno policy unit tests ────────────────────────────────────────
   # `kyverno test` proves each policy rule PASSES a compliant resource and FAILS
   # a violating one (policies/kyverno/tests). Without this, a policy could be

Taskfile.yaml

@@ -121,6 +121,11 @@ tasks:
     cmds:
       - ./scripts/tests/controls.py
 
+  validate:alert-severity-routes:
+    desc: "Alert routing gate — every severity a rule claims resolves to a declared contact point"
+    cmds:
+      - ./scripts/check-alert-severity-routes.py
+
   validate:empty-corpus:
     desc: "Vacuity gate — no gate reports success over a corpus that is not there"
     cmds:
@@ -217,6 +222,7 @@ tasks:
       - validate:policy-validity
       - validate:gate-tests
       - validate:gate-controls
+      - validate:alert-severity-routes
       - validate:empty-corpus
       - validate:reverify-tests
       - validate:named-things

applicationsets/dashboards.yaml

@@ -85,6 +85,13 @@ spec:
                 - op: replace
                   path: /spec/dataFrom/0/extract/key
                   value: '{{ index .metadata.labels "cluster_name" }}-grafana-token'
+            - target:
+                kind: ExternalSecret
+                name: alert-destinations
+              patch: |-
+                - op: replace
+                  path: /spec/dataFrom/0/extract/key
+                  value: '{{ index .metadata.labels "cluster_name" }}-alert-destinations'
             - target:
                 kind: CronJob
                 name: grafana-token-rotator

dashboards/base/alerting/alert-destinations.yaml

@@ -0,0 +1,35 @@
+# The credentials the alert destinations authenticate with, pulled from AWS
+# Secrets Manager through the same aws-secrets-manager ClusterSecretStore the
+# Grafana service-account token uses.
+#
+# The key committed here is a placeholder. The real secret is cluster-scoped —
+# `<cluster>-alert-destinations` — so co-located sibling clusters in one account
+# do not collide, and the dashboards ApplicationSet patches the name per cluster
+# from the cluster_name label, as it does for the Grafana token and the
+# monitoring endpoints.
+#
+# The secret itself is created and seeded by the landing-zone managed-monitoring
+# component, as `<cluster>-grafana-token` is — it is not hand-made here. Until it
+# carries a value the contact points reconcile with no integration key and
+# Grafana delivers nothing, which is the state this file exists to end rather
+# than one it introduces.
+#
+# extract, not a data list, for the reason monitoring-endpoints.yaml records:
+# the payload's field names are already the key names the contact points read,
+# so enumerating them here would restate the producer's shape and name one
+# cluster-scoped secret once per field.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: alert-destinations
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: aws-secrets-manager
+    kind: ClusterSecretStore
+  target:
+    name: alert-destinations
+    creationPolicy: Owner
+  dataFrom:
+    - extract:
+        key: eks-alert-destinations

dashboards/base/alerting/contact-points.yaml

@@ -0,0 +1,60 @@
+# Where a firing alert is delivered. One contact point per urgency, because the
+# urgency is what the rules label themselves with and a route can only send to a
+# receiver that exists.
+#
+# The integration credentials are never written here. Each receiver takes its
+# secret through valuesFrom, from the Secret alert-destinations.yaml
+# materialises — so the value lives in Secrets Manager and this file names the
+# key rather than carrying it.
+#
+# Neither is marked optional. An optional missing secret produces a contact
+# point with no integration key: Grafana accepts it, the route resolves, and
+# nothing is delivered — which is indistinguishable from a working destination
+# by every signal except an incident nobody answered. Required, the contact
+# point reports NotReady and says so.
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaContactPoint
+metadata:
+  name: platform-page
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: external
+  name: platform-page
+  receivers:
+    - uid: platform-page-pagerduty
+      type: pagerduty
+      settings:
+        severity: critical
+        # The alert's own labels, so the incident names the system it is about
+        # rather than only the rule that fired.
+        class: '{{ .CommonLabels.service }}'
+        component: '{{ .CommonLabels.component }}'
+      valuesFrom:
+        - targetPath: integrationKey
+          valueFrom:
+            secretKeyRef:
+              name: alert-destinations
+              key: PAGERDUTY_INTEGRATION_KEY
+---
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaContactPoint
+metadata:
+  name: platform-ticket
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: external
+  name: platform-ticket
+  receivers:
+    - uid: platform-ticket-slack
+      type: slack
+      settings:
+        title: '{{ .CommonLabels.alertname }} — {{ .CommonLabels.service }}'
+        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ "\n" }}{{ end }}'
+      valuesFrom:
+        - targetPath: url
+          valueFrom:
+            secretKeyRef:
+              name: alert-destinations
+              key: SLACK_WEBHOOK_URL

dashboards/base/alerting/notification-policy.yaml

@@ -0,0 +1,53 @@
+# The routing tree every alert rule under this directory is written against.
+#
+# A rule labels itself with the urgency it is asking for. Grafana matches those
+# labels against this tree and delivers to the receiver the matching route
+# names; with no tree the workspace default applies, and the workspace default
+# contact point is empty. A `severity: page` label with no route is a promise
+# that a human is woken, kept by nobody, in the one moment it matters.
+#
+# The root receiver is the low-urgency one rather than a dedicated fallback. An
+# alert whose severity matches no route is a rule this tree has not been taught
+# about, and the two answers to that are "deliver it somewhere a person reads"
+# and "drop it silently". scripts/check-alert-severity-routes.py makes the case
+# impossible to arrive at unnoticed; this makes it survivable if it does.
+apiVersion: grafana.integreatly.org/v1beta1
+kind: GrafanaNotificationPolicy
+metadata:
+  name: platform-severity-routes
+spec:
+  instanceSelector:
+    matchLabels:
+      dashboards: external
+  route:
+    receiver: platform-ticket
+    # Grouped by the system rather than by the rule, so a fleet-wide failure
+    # raising six rules against one service is one notification and not six.
+    group_by:
+      - alertname
+      - service
+      - component
+    routes:
+      # Wakes a human. Repeated hourly while firing, because an unanswered page
+      # is the failure this whole tree exists to prevent.
+      - receiver: platform-page
+        matchers:
+          - name: severity
+            value: page
+            isEqual: true
+            isRegex: false
+        group_wait: 30s
+        group_interval: 5m
+        repeat_interval: 1h
+      # Read in working hours. Grouped and repeated slowly on purpose: a ticket
+      # that re-notifies like a page teaches people to mute the channel both
+      # arrive in.
+      - receiver: platform-ticket
+        matchers:
+          - name: severity
+            value: ticket
+            isEqual: true
+            isRegex: false
+        group_wait: 5m
+        group_interval: 30m
+        repeat_interval: 12h

dashboards/base/kustomization.yaml

@@ -22,6 +22,12 @@ resources:
   # Grafana-managed SLO / burn-rate alert rules (folder + per-system rule groups),
   # reconciled onto the external Amazon Managed Grafana.
   - alerting/folder.yaml
+  # Where a firing rule is delivered. Without these the rules' severity labels
+  # select the workspace default policy and its empty default contact point,
+  # so every alert changes state in the Grafana list and reaches no human.
+  - alerting/alert-destinations.yaml
+  - alerting/contact-points.yaml
+  - alerting/notification-policy.yaml
   - alerting/portal.yaml
   - alerting/agent-operator.yaml
   - alerting/fleet-vend.yaml