diff --git a/.env.example b/.env.example index f672595..b7fd375 100644 --- a/.env.example +++ b/.env.example @@ -123,4 +123,28 @@ GRAFANA_PORT=3001 # Grafana admin login for the local monitoring stack. GRAFANA_ADMIN_USER=admin -GRAFANA_ADMIN_PASSWORD=admin \ No newline at end of file +GRAFANA_ADMIN_PASSWORD=admin + + +# ----------------------------------------------------------------------------- +# Alertmanager receivers +# ----------------------------------------------------------------------------- +# These values are injected into monitoring/alertmanager.yml at container +# start (see the alertmanager service in docker-compose.yml). Leave them empty +# to disable the corresponding receiver. Never commit real secrets. + +# Slack incoming webhook for warning-level alerts. +ALERTMANAGER_SLACK_WEBHOOK_URL= +# Slack channel to post warning alerts to (e.g. #alerts-warning). +ALERTMANAGER_SLACK_CHANNEL=#alerts-warning + +# PagerDuty integration key (Events API v2) for critical alerts. +ALERTMANAGER_PAGERDUTY_SERVICE_KEY= + +# SMTP settings for the email receiver (critical alerts). +ALERTMANAGER_SMTP_SMARTHOST=localhost:587 +ALERTMANAGER_SMTP_FROM=alerts@example.com +ALERTMANAGER_SMTP_AUTH_USERNAME= +ALERTMANAGER_SMTP_AUTH_PASSWORD= +# Comma-separated list of recipients for email alerts. +ALERTMANAGER_EMAIL_TO=oncall@example.com \ No newline at end of file diff --git a/docs/MONITORING.md b/docs/MONITORING.md index 3740833..7178adb 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -20,6 +20,7 @@ All configuration referenced here lives in the [`monitoring/`](../monitoring) di - [Email](#email) - [Slack](#slack) - [6. Verify the setup](#6-verify-the-setup) +- [Runbooks](#runbooks) - [Required metrics](#required-metrics) - [Troubleshooting](#troubleshooting) @@ -207,6 +208,7 @@ Alert rules are defined in [`monitoring/alert-rules.yml`](../monitoring/alert-ru ## 5. Configure Alertmanager notification channels +Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your real credentials and keep secrets out of version control. Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your own values and keep secrets out of version control. ### Email @@ -215,6 +217,11 @@ Alertmanager routing and receivers are configured in [`monitoring/alertmanager.y receivers: - name: email email_configs: + - to: oncall@example.com + from: alerts@example.com + smarthost: smtp.example.com:587 + auth_username: alerts@example.com + auth_password_file: /etc/alertmanager/smtp_password - to: alerts@example.com from: alertmanager@example.com smarthost: smtp.example.com:587 @@ -231,6 +238,31 @@ receivers: - name: slack slack_configs: - api_url_file: /etc/alertmanager/slack_webhook_url + channel: '#alerts' + send_resolved: true +``` + +Mount the referenced secret files into the Alertmanager container and keep them out of version control. + +## 6. Verify the setup + +1. Confirm Prometheus targets are up: **Prometheus → Status → Targets**. +2. Confirm rules are loaded: **Prometheus → Status → Rules**. +3. Trigger a test alert (e.g. temporarily lower a threshold) and confirm it reaches Alertmanager and the configured notification channel. +4. Confirm the alert resolves and a resolved notification is delivered (if `send_resolved` is enabled). + +## Runbooks + +Every alert in [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) links to an operational runbook via its `runbook_url` annotation. Runbooks live in [`docs/runbooks/`](./runbooks) — see the [runbook index](./runbooks/README.md) for the full list. + +When you add or rename an alert, add a matching runbook under `docs/runbooks/` and point the alert's `runbook_url` at it. + +## Troubleshooting + +- **No metrics in Prometheus:** check the scrape target, network reachability, and that the backend `/metrics` endpoint responds. +- **Rules not loading:** validate the YAML and confirm the rule file is mounted at the path in `rule_files`. +- **Alerts not delivered:** check Alertmanager logs and confirm receiver credentials/secret files are correct. +- **Dashboard empty:** confirm the Grafana data source points at the correct Prometheus instance. channel: "#alerts" send_resolved: true ``` diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml index 1391ef5..489bc30 100644 --- a/monitoring/alert-rules.yml +++ b/monitoring/alert-rules.yml @@ -10,7 +10,7 @@ groups: annotations: summary: "Backend instance {{ $labels.instance }} is down" description: "Backend target {{ $labels.instance }} (job {{ $labels.job }}) has been unreachable for more than 1 minute." - runbook_url: "https://docs.example.com/runbooks/backend-down" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md" - alert: BackendTargetMissing expr: absent(up{job="backend"}) @@ -21,7 +21,7 @@ groups: annotations: summary: "No backend targets are being scraped" description: "Prometheus has not scraped any backend targets for more than 5 minutes. The service may be removed from service discovery." - runbook_url: "https://docs.example.com/runbooks/backend-down" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md" - name: http-errors rules: @@ -39,7 +39,7 @@ groups: annotations: summary: "High HTTP 5xx error rate ({{ $value | humanizePercentage }})" description: "More than 5% of backend HTTP requests have returned 5xx responses over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/http-5xx" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-5xx.md" - alert: ElevatedHttp5xxErrorRate expr: | @@ -55,7 +55,7 @@ groups: annotations: summary: "Elevated HTTP 5xx error rate ({{ $value | humanizePercentage }})" description: "More than 1% of backend HTTP requests have returned 5xx responses over the last 10 minutes." - runbook_url: "https://docs.example.com/runbooks/http-5xx" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-5xx.md" - alert: HighHttp4xxErrorRate expr: | @@ -71,7 +71,7 @@ groups: annotations: summary: "High HTTP 4xx error rate ({{ $value | humanizePercentage }})" description: "More than 25% of backend HTTP requests have returned 4xx responses over the last 15 minutes." - runbook_url: "https://docs.example.com/runbooks/http-4xx" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-4xx.md" - name: http-latency rules: @@ -88,7 +88,7 @@ groups: annotations: summary: "High HTTP p95 latency ({{ $value | humanizeDuration }})" description: "The 95th percentile backend HTTP request latency has exceeded 1s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/http-latency" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-latency.md" - alert: CriticalHttpLatencyP99 expr: | @@ -103,7 +103,7 @@ groups: annotations: summary: "Critical HTTP p99 latency ({{ $value | humanizeDuration }})" description: "The 99th percentile backend HTTP request latency has exceeded 3s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/http-latency" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-latency.md" - name: database rules: @@ -117,7 +117,7 @@ groups: annotations: summary: "Database connection failures detected" description: "The backend is experiencing database connection errors at a rate of {{ $value | humanize }}/s over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/database-connection" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-connection.md" - alert: DatabaseConnectionPoolExhausted expr: | @@ -133,9 +133,21 @@ groups: annotations: summary: "Database connection pool nearly exhausted ({{ $value | humanizePercentage }})" description: "More than 90% of the backend database connection pool has been in use for over 5 minutes." - runbook_url: "https://docs.example.com/runbooks/database-connection" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-connection.md" - alert: DatabaseDown + expr: | + pg_up{job="database"} == 0 + for: 1m + labels: + severity: critical + team: backend + annotations: + summary: "Postgres instance {{ $labels.instance }} is down" + description: "Postgres target {{ $labels.instance }} reports pg_up == 0 for more than 1 minute, indicating the database itself is unreachable even though the exporter is running." + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md" + + - alert: DatabaseExporterDown expr: | up{job="database"} == 0 for: 1m @@ -143,9 +155,9 @@ groups: severity: critical team: backend annotations: - summary: "Database instance {{ $labels.instance }} is down" - description: "Database target {{ $labels.instance }} has been unreachable for more than 1 minute." - runbook_url: "https://docs.example.com/runbooks/database-down" + summary: "Database exporter {{ $labels.instance }} is down" + description: "The postgres-exporter target {{ $labels.instance }} has been unreachable for more than 1 minute, so database health cannot be observed." + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md" - name: backups rules: @@ -159,7 +171,7 @@ groups: annotations: summary: "Backup job failed for {{ $labels.instance }}" description: "The most recent backup run for {{ $labels.instance }} did not succeed (soroban_backup_success == 0, written by scripts/backup.sh) for more than 5 minutes." - runbook_url: "https://docs.example.com/runbooks/backup-failed" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backup-failed.md" - alert: BackupStale expr: | @@ -171,7 +183,7 @@ groups: annotations: summary: "Backup is stale for {{ $labels.instance }}" description: "The last successful backup for {{ $labels.instance }} was more than 26 hours ago, exceeding the expected backup interval." - runbook_url: "https://docs.example.com/runbooks/backup-stale" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backup-stale.md" - name: stellar-soroban-rpc rules: @@ -185,7 +197,7 @@ groups: annotations: summary: "Stellar RPC errors detected" description: "The backend is experiencing Stellar RPC errors at a rate of {{ $value | humanize }}/s over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/stellar-rpc" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/stellar-rpc.md" - alert: StellarRpcHighErrorRate expr: | @@ -196,11 +208,12 @@ groups: ) > 0.05 for: 5m labels: - severity: critical + severity: warning team: backend annotations: summary: "High Stellar RPC error rate ({{ $value | humanizePercentage }})" description: "More than 5% of Stellar RPC requests have failed over the last 5 minutes." + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/stellar-rpc.md" runbook_url: "https://docs.example.com/runbooks/stellar-rpc" - alert: SorobanRpcErrors diff --git a/monitoring/alertmanager.yml b/monitoring/alertmanager.yml index 1f31151..df5b38a 100644 --- a/monitoring/alertmanager.yml +++ b/monitoring/alertmanager.yml @@ -1,31 +1,79 @@ -# Minimal Alertmanager config. Alerts are grouped and routed to a single -# receiver; replace the webhook URL (or add Slack/PagerDuty/email receivers) -# for real notifications. See https://prometheus.io/docs/alerting/latest/configuration/ +# ============================================================================= +# Alertmanager configuration +# ============================================================================= +# Receivers are configured from environment variables / secret files so that +# no credentials are committed to the repository. The alertmanager service in +# docker-compose.yml renders this file with `envsubst` before starting, using +# the ALERTMANAGER_* variables documented in .env.example. +# +# Routing: +# - critical -> PagerDuty + email (paging path) +# - warning -> Slack (non-paging path) +# - default -> Slack +# ============================================================================= + global: resolve_timeout: 5m + smtp_smarthost: '${ALERTMANAGER_SMTP_SMARTHOST}' + smtp_from: '${ALERTMANAGER_SMTP_FROM}' + smtp_auth_username: '${ALERTMANAGER_SMTP_AUTH_USERNAME}' + smtp_auth_password: '${ALERTMANAGER_SMTP_AUTH_PASSWORD}' + smtp_require_tls: true +# The root route dispatches by severity. Critical alerts page via PagerDuty +# and email; warnings go to Slack only. route: - receiver: default - group_by: ['alertname', 'job'] + receiver: slack-warning + group_by: ['alertname', 'severity'] group_wait: 30s group_interval: 5m repeat_interval: 4h routes: - matchers: - severity="critical" - receiver: default + receiver: pagerduty-critical + group_wait: 10s repeat_interval: 1h + continue: true + - matchers: + - severity="critical" + receiver: email-critical + group_wait: 10s + repeat_interval: 1h + - matchers: + - severity="warning" + receiver: slack-warning + repeat_interval: 4h receivers: - - name: default - webhook_configs: - - url: 'http://localhost:9095/alerts' + - name: slack-warning + slack_configs: + - api_url: '${ALERTMANAGER_SLACK_WEBHOOK_URL}' + channel: '${ALERTMANAGER_SLACK_CHANNEL}' + send_resolved: true + title: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}' + text: >- + {{ range .Alerts }}{{ .Annotations.summary }} + {{ .Annotations.description }} + {{ end }} + + - name: pagerduty-critical + pagerduty_configs: + - service_key: '${ALERTMANAGER_PAGERDUTY_SERVICE_KEY}' + send_resolved: true + description: '{{ .CommonLabels.alertname }}: {{ .CommonAnnotations.summary }}' + + - name: email-critical + email_configs: + - to: '${ALERTMANAGER_EMAIL_TO}' send_resolved: true + headers: + Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}' inhibit_rules: - # Don't page for warnings when the matching critical alert is already firing. + # A firing critical alert suppresses the matching warning for the same alert. - source_matchers: - severity="critical" target_matchers: - severity="warning" - equal: ['alertname', 'job'] + equal: ['alertname']