Busiii-adetiba · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026 · Oct 19, 2026
diff --git a/.env.example b/.env.example
@@ -123,4 +123,28 @@ GRAFANA_PORT=3001
 
 # Grafana admin login for the local monitoring stack.
 GRAFANA_ADMIN_USER=admin
-GRAFANA_ADMIN_PASSWORD=admin
+GRAFANA_ADMIN_PASSWORD=admin
+
+
+# -----------------------------------------------------------------------------
+# Alertmanager receivers
+# -----------------------------------------------------------------------------
+# These values are injected into monitoring/alertmanager.yml at container
+# start (see the alertmanager service in docker-compose.yml). Leave them empty
+# to disable the corresponding receiver. Never commit real secrets.
+
+# Slack incoming webhook for warning-level alerts.
+ALERTMANAGER_SLACK_WEBHOOK_URL=
+# Slack channel to post warning alerts to (e.g. #alerts-warning).
+ALERTMANAGER_SLACK_CHANNEL=#alerts-warning
+
+# PagerDuty integration key (Events API v2) for critical alerts.
+ALERTMANAGER_PAGERDUTY_SERVICE_KEY=
+
+# SMTP settings for the email receiver (critical alerts).
+ALERTMANAGER_SMTP_SMARTHOST=localhost:587
+ALERTMANAGER_SMTP_FROM=alerts@example.com
+ALERTMANAGER_SMTP_AUTH_USERNAME=
+ALERTMANAGER_SMTP_AUTH_PASSWORD=
+# Comma-separated list of recipients for email alerts.
+ALERTMANAGER_EMAIL_TO=oncall@example.com
diff --git a/docs/MONITORING.md b/docs/MONITORING.md
@@ -20,6 +20,7 @@ All configuration referenced here lives in the [`monitoring/`](../monitoring) di
   - [Email](#email)
   - [Slack](#slack)
 - [6. Verify the setup](#6-verify-the-setup)
+- [Runbooks](#runbooks)
 - [Required metrics](#required-metrics)
 - [Troubleshooting](#troubleshooting)
 
@@ -207,6 +208,7 @@ Alert rules are defined in [`monitoring/alert-rules.yml`](../monitoring/alert-ru
 
 ## 5. Configure Alertmanager notification channels
 
+Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your real credentials and keep secrets out of version control.
 Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your own values and keep secrets out of version control.
 
 ### Email
@@ -215,6 +217,11 @@ Alertmanager routing and receivers are configured in [`monitoring/alertmanager.y
 receivers:
   - name: email
     email_configs:
+      - to: oncall@example.com
+        from: alerts@example.com
+        smarthost: smtp.example.com:587
+        auth_username: alerts@example.com
+        auth_password_file: /etc/alertmanager/smtp_password
       - to: alerts@example.com
         from: alertmanager@example.com
         smarthost: smtp.example.com:587
@@ -231,6 +238,31 @@ receivers:
   - name: slack
     slack_configs:
       - api_url_file: /etc/alertmanager/slack_webhook_url
+        channel: '#alerts'
+        send_resolved: true
+```
+
+Mount the referenced secret files into the Alertmanager container and keep them out of version control.
+
+## 6. Verify the setup
+
+1. Confirm Prometheus targets are up: **Prometheus → Status → Targets**.
+2. Confirm rules are loaded: **Prometheus → Status → Rules**.
+3. Trigger a test alert (e.g. temporarily lower a threshold) and confirm it reaches Alertmanager and the configured notification channel.
+4. Confirm the alert resolves and a resolved notification is delivered (if `send_resolved` is enabled).
+
+## Runbooks
+
+Every alert in [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) links to an operational runbook via its `runbook_url` annotation. Runbooks live in [`docs/runbooks/`](./runbooks) — see the [runbook index](./runbooks/README.md) for the full list.
+
+When you add or rename an alert, add a matching runbook under `docs/runbooks/` and point the alert's `runbook_url` at it.
+
+## Troubleshooting
+
+- **No metrics in Prometheus:** check the scrape target, network reachability, and that the backend `/metrics` endpoint responds.
+- **Rules not loading:** validate the YAML and confirm the rule file is mounted at the path in `rule_files`.
+- **Alerts not delivered:** check Alertmanager logs and confirm receiver credentials/secret files are correct.
+- **Dashboard empty:** confirm the Grafana data source points at the correct Prometheus instance.
         channel: "#alerts"
         send_resolved: true
 ```

diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml
@@ -10,7 +10,7 @@ groups:
         annotations:
           summary: "Backend instance {{ $labels.instance }} is down"
           description: "Backend target {{ $labels.instance }} (job {{ $labels.job }}) has been unreachable for more than 1 minute."
-          runbook_url: "https://docs.example.com/runbooks/backend-down"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md"
 
       - alert: BackendTargetMissing
         expr: absent(up{job="backend"})
@@ -21,7 +21,7 @@ groups:
         annotations:
           summary: "No backend targets are being scraped"
           description: "Prometheus has not scraped any backend targets for more than 5 minutes. The service may be removed from service discovery."
-          runbook_url: "https://docs.example.com/runbooks/backend-down"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md"
 
   - name: http-errors
     rules:
@@ -39,7 +39,7 @@ groups:
         annotations:
           summary: "High HTTP 5xx error rate ({{ $value | humanizePercentage }})"
           description: "More than 5% of backend HTTP requests have returned 5xx responses over the last 5 minutes."
-          runbook_url: "https://docs.example.com/runbooks/http-5xx"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-5xx.md"
 
       - alert: ElevatedHttp5xxErrorRate
         expr: |
@@ -55,7 +55,7 @@ groups:
         annotations:
           summary: "Elevated HTTP 5xx error rate ({{ $value | humanizePercentage }})"
           description: "More than 1% of backend HTTP requests have returned 5xx responses over the last 10 minutes."
-          runbook_url: "https://docs.example.com/runbooks/http-5xx"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-5xx.md"
 
       - alert: HighHttp4xxErrorRate
         expr: |
@@ -71,7 +71,7 @@ groups:
         annotations:
           summary: "High HTTP 4xx error rate ({{ $value | humanizePercentage }})"
           description: "More than 25% of backend HTTP requests have returned 4xx responses over the last 15 minutes."
-          runbook_url: "https://docs.example.com/runbooks/http-4xx"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-4xx.md"
 
   - name: http-latency
     rules:
@@ -88,7 +88,7 @@ groups:
         annotations:
           summary: "High HTTP p95 latency ({{ $value | humanizeDuration }})"
           description: "The 95th percentile backend HTTP request latency has exceeded 1s for more than 10 minutes."
-          runbook_url: "https://docs.example.com/runbooks/http-latency"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-latency.md"
 
       - alert: CriticalHttpLatencyP99
         expr: |
@@ -103,7 +103,7 @@ groups:
         annotations:
           summary: "Critical HTTP p99 latency ({{ $value | humanizeDuration }})"
           description: "The 99th percentile backend HTTP request latency has exceeded 3s for more than 10 minutes."
-          runbook_url: "https://docs.example.com/runbooks/http-latency"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-latency.md"
 
   - name: database
     rules:
@@ -117,7 +117,7 @@ groups:
         annotations:
           summary: "Database connection failures detected"
           description: "The backend is experiencing database connection errors at a rate of {{ $value | humanize }}/s over the last 5 minutes."
-          runbook_url: "https://docs.example.com/runbooks/database-connection"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-connection.md"
 
       - alert: DatabaseConnectionPoolExhausted
         expr: |
@@ -133,19 +133,31 @@ groups:
         annotations:
           summary: "Database connection pool nearly exhausted ({{ $value | humanizePercentage }})"
           description: "More than 90% of the backend database connection pool has been in use for over 5 minutes."
-          runbook_url: "https://docs.example.com/runbooks/database-connection"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-connection.md"
 
       - alert: DatabaseDown
+        expr: |
+          pg_up{job="database"} == 0
+        for: 1m
+        labels:
+          severity: critical
+          team: backend
+        annotations:
+          summary: "Postgres instance {{ $labels.instance }} is down"
+          description: "Postgres target {{ $labels.instance }} reports pg_up == 0 for more than 1 minute, indicating the database itself is unreachable even though the exporter is running."
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md"
+
+      - alert: DatabaseExporterDown
         expr: |
           up{job="database"} == 0
         for: 1m
         labels:
           severity: critical
           team: backend
         annotations:
-          summary: "Database instance {{ $labels.instance }} is down"
-          description: "Database target {{ $labels.instance }} has been unreachable for more than 1 minute."
-          runbook_url: "https://docs.example.com/runbooks/database-down"
+          summary: "Database exporter {{ $labels.instance }} is down"
+          description: "The postgres-exporter target {{ $labels.instance }} has been unreachable for more than 1 minute, so database health cannot be observed."
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md"
 
   - name: backups
     rules:
@@ -159,7 +171,7 @@ groups:
         annotations:
           summary: "Backup job failed for {{ $labels.instance }}"
           description: "The most recent backup run for {{ $labels.instance }} did not succeed (soroban_backup_success == 0, written by scripts/backup.sh) for more than 5 minutes."
-          runbook_url: "https://docs.example.com/runbooks/backup-failed"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backup-failed.md"
 
       - alert: BackupStale
         expr: |
@@ -171,7 +183,7 @@ groups:
         annotations:
           summary: "Backup is stale for {{ $labels.instance }}"
           description: "The last successful backup for {{ $labels.instance }} was more than 26 hours ago, exceeding the expected backup interval."
-          runbook_url: "https://docs.example.com/runbooks/backup-stale"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backup-stale.md"
 
   - name: stellar-soroban-rpc
     rules:
@@ -185,7 +197,7 @@ groups:
         annotations:
           summary: "Stellar RPC errors detected"
           description: "The backend is experiencing Stellar RPC errors at a rate of {{ $value | humanize }}/s over the last 5 minutes."
-          runbook_url: "https://docs.example.com/runbooks/stellar-rpc"
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/stellar-rpc.md"
 
       - alert: StellarRpcHighErrorRate
         expr: |
@@ -196,11 +208,12 @@ groups:
           ) > 0.05
         for: 5m
         labels:
-          severity: critical
+          severity: warning
           team: backend
         annotations:
           summary: "High Stellar RPC error rate ({{ $value | humanizePercentage }})"
           description: "More than 5% of Stellar RPC requests have failed over the last 5 minutes."
+          runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/stellar-rpc.md"
           runbook_url: "https://docs.example.com/runbooks/stellar-rpc"
 
       - alert: SorobanRpcErrors

diff --git a/monitoring/alertmanager.yml b/monitoring/alertmanager.yml
@@ -1,31 +1,79 @@
-# Minimal Alertmanager config. Alerts are grouped and routed to a single
-# receiver; replace the webhook URL (or add Slack/PagerDuty/email receivers)
-# for real notifications. See https://prometheus.io/docs/alerting/latest/configuration/
+# =============================================================================
+# Alertmanager configuration
+# =============================================================================
+# Receivers are configured from environment variables / secret files so that
+# no credentials are committed to the repository. The alertmanager service in
+# docker-compose.yml renders this file with `envsubst` before starting, using
+# the ALERTMANAGER_* variables documented in .env.example.
+#
+# Routing:
+#   - critical  -> PagerDuty + email (paging path)
+#   - warning   -> Slack (non-paging path)
+#   - default   -> Slack
+# =============================================================================
+
 global:
   resolve_timeout: 5m
+  smtp_smarthost: '${ALERTMANAGER_SMTP_SMARTHOST}'
+  smtp_from: '${ALERTMANAGER_SMTP_FROM}'
+  smtp_auth_username: '${ALERTMANAGER_SMTP_AUTH_USERNAME}'
+  smtp_auth_password: '${ALERTMANAGER_SMTP_AUTH_PASSWORD}'
+  smtp_require_tls: true
 
+# The root route dispatches by severity. Critical alerts page via PagerDuty
+# and email; warnings go to Slack only.
 route:
-  receiver: default
-  group_by: ['alertname', 'job']
+  receiver: slack-warning
+  group_by: ['alertname', 'severity']
   group_wait: 30s
   group_interval: 5m
   repeat_interval: 4h
   routes:
     - matchers:
         - severity="critical"
-      receiver: default
+      receiver: pagerduty-critical
+      group_wait: 10s
       repeat_interval: 1h
+      continue: true
+    - matchers:
+        - severity="critical"
+      receiver: email-critical
+      group_wait: 10s
+      repeat_interval: 1h
+    - matchers:
+        - severity="warning"
+      receiver: slack-warning
+      repeat_interval: 4h
 
 receivers:
-  - name: default
-    webhook_configs:
-      - url: 'http://localhost:9095/alerts'
+  - name: slack-warning
+    slack_configs:
+      - api_url: '${ALERTMANAGER_SLACK_WEBHOOK_URL}'
+        channel: '${ALERTMANAGER_SLACK_CHANNEL}'
+        send_resolved: true
+        title: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'
+        text: >-
+          {{ range .Alerts }}{{ .Annotations.summary }}
+          {{ .Annotations.description }}
+          {{ end }}
+
+  - name: pagerduty-critical
+    pagerduty_configs:
+      - service_key: '${ALERTMANAGER_PAGERDUTY_SERVICE_KEY}'
+        send_resolved: true
+        description: '{{ .CommonLabels.alertname }}: {{ .CommonAnnotations.summary }}'
+
+  - name: email-critical
+    email_configs:
+      - to: '${ALERTMANAGER_EMAIL_TO}'
         send_resolved: true
+        headers:
+          Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}'
 
 inhibit_rules:
-  # Don't page for warnings when the matching critical alert is already firing.
+  # A firing critical alert suppresses the matching warning for the same alert.
   - source_matchers:
       - severity="critical"
     target_matchers:
       - severity="warning"
-    equal: ['alertname', 'job']
+    equal: ['alertname']