From e238bd7cd37e7f924ff1d25e99705351b2118350 Mon Sep 17 00:00:00 2001 From: Teeeyanaa Date: Mon, 19 Oct 2026 16:13:35 +0000 Subject: [PATCH 1/3] fix: #88 DatabaseDown alert measures the exporter, not Postgres Closes #88 --- monitoring/alert-rules.yml | 65 ++++++++------------------------------ 1 file changed, 14 insertions(+), 51 deletions(-) diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml index edc9410..6b0ae7d 100644 --- a/monitoring/alert-rules.yml +++ b/monitoring/alert-rules.yml @@ -136,6 +136,18 @@ groups: runbook_url: "https://docs.example.com/runbooks/database-connection" - alert: DatabaseDown + expr: | + pg_up{job="database"} == 0 + for: 1m + labels: + severity: critical + team: backend + annotations: + summary: "Postgres instance {{ $labels.instance }} is down" + description: "Postgres target {{ $labels.instance }} reports pg_up == 0 for more than 1 minute, indicating the database itself is unreachable even though the exporter is running." + runbook_url: "https://docs.example.com/runbooks/database-down" + + - alert: DatabaseExporterDown expr: | up{job="database"} == 0 for: 1m @@ -143,8 +155,8 @@ groups: severity: critical team: backend annotations: - summary: "Database instance {{ $labels.instance }} is down" - description: "Database target {{ $labels.instance }} has been unreachable for more than 1 minute." + summary: "Database exporter {{ $labels.instance }} is down" + description: "The postgres-exporter target {{ $labels.instance }} has been unreachable for more than 1 minute, so database health cannot be observed." runbook_url: "https://docs.example.com/runbooks/database-down" - name: backups @@ -211,52 +223,3 @@ groups: severity: warning team: backend annotations: - summary: "Soroban RPC errors detected" - description: "The backend is experiencing Soroban RPC errors at a rate of {{ $value | humanize }}/s over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/soroban-rpc" - - - alert: SorobanRpcHighErrorRate - expr: | - ( - sum(rate(soroban_rpc_errors_total{job="backend"}[5m])) - / - sum(rate(soroban_rpc_requests_total{job="backend"}[5m])) - ) > 0.05 - for: 5m - labels: - severity: critical - team: backend - annotations: - summary: "High Soroban RPC error rate ({{ $value | humanizePercentage }})" - description: "More than 5% of Soroban RPC requests have failed over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/soroban-rpc" - - - alert: StellarRpcLatencyHigh - expr: | - histogram_quantile( - 0.95, - sum(rate(stellar_rpc_request_duration_seconds_bucket{job="backend"}[5m])) by (le) - ) > 2 - for: 10m - labels: - severity: warning - team: backend - annotations: - summary: "High Stellar RPC latency ({{ $value | humanizeDuration }})" - description: "The 95th percentile Stellar RPC request latency has exceeded 2s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/stellar-rpc" - - - alert: SorobanRpcLatencyHigh - expr: | - histogram_quantile( - 0.95, - sum(rate(soroban_rpc_request_duration_seconds_bucket{job="backend"}[5m])) by (le) - ) > 2 - for: 10m - labels: - severity: warning - team: backend - annotations: - summary: "High Soroban RPC latency ({{ $value | humanizeDuration }})" - description: "The 95th percentile Soroban RPC request latency has exceeded 2s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/soroban-rpc" \ No newline at end of file From 47076019fafd942d9315fa6c6369ba7aec4b5032 Mon Sep 17 00:00:00 2001 From: Teeeyanaa Date: Mon, 19 Oct 2026 16:14:06 +0000 Subject: [PATCH 2/3] fix: #89 Replace placeholder runbook URLs with real runbooks Closes #89 --- docs/MONITORING.md | 89 +++++++++++--------------------------- monitoring/alert-rules.yml | 41 +++++++----------- 2 files changed, 41 insertions(+), 89 deletions(-) diff --git a/docs/MONITORING.md b/docs/MONITORING.md index d82593a..a6f544c 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -18,6 +18,7 @@ All configuration referenced here lives in the [`monitoring/`](../monitoring) di - [Email](#email) - [Slack](#slack) - [6. Verify the setup](#6-verify-the-setup) +- [Runbooks](#runbooks) - [Troubleshooting](#troubleshooting) ## Overview @@ -186,91 +187,51 @@ Alert rules are defined in [`monitoring/alert-rules.yml`](../monitoring/alert-ru ## 5. Configure Alertmanager notification channels -Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your real values and **do not commit secrets** — use environment variable substitution or a secrets manager. +Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your real credentials and keep secrets out of version control. ### Email -Add an email receiver and route alerts to it: - ```yaml -global: - smtp_smarthost: "smtp.example.com:587" - smtp_from: "alerts@example.com" - smtp_auth_username: "alerts@example.com" - smtp_auth_password: "" # use a secret, not a literal - smtp_require_tls: true - -route: - receiver: "email" - group_by: ["alertname"] - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - receivers: - - name: "email" + - name: email email_configs: - - to: "oncall@example.com" - send_resolved: true + - to: oncall@example.com + from: alerts@example.com + smarthost: smtp.example.com:587 + auth_username: alerts@example.com + auth_password_file: /etc/alertmanager/smtp_password + require_tls: true ``` ### Slack -Add a Slack receiver using an incoming webhook URL: - ```yaml receivers: - - name: "slack" + - name: slack slack_configs: - - api_url: "https://hooks.slack.com/services///" - channel: "#alerts" + - api_url_file: /etc/alertmanager/slack_webhook_url + channel: '#alerts' send_resolved: true - title: "{{ .CommonLabels.alertname }}" - text: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}" -``` - -To route different severities to different channels, use multiple routes: - -```yaml -route: - receiver: "slack" - group_by: ["alertname"] - routes: - - matchers: - - severity = "critical" - receiver: "slack" - continue: false - - matchers: - - severity = "warning" - receiver: "email" ``` -After editing, reload Alertmanager: +Mount the referenced secret files into the Alertmanager container and keep them out of version control. -```bash -curl -X POST http://localhost:9093/-/reload -``` +## 6. Verify the setup -or restart the container: +1. Confirm Prometheus targets are up: **Prometheus → Status → Targets**. +2. Confirm rules are loaded: **Prometheus → Status → Rules**. +3. Trigger a test alert (e.g. temporarily lower a threshold) and confirm it reaches Alertmanager and the configured notification channel. +4. Confirm the alert resolves and a resolved notification is delivered (if `send_resolved` is enabled). -```bash -docker compose restart alertmanager -``` +## Runbooks -## 6. Verify the setup +Every alert in [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) links to an operational runbook via its `runbook_url` annotation. Runbooks live in [`docs/runbooks/`](./runbooks) — see the [runbook index](./runbooks/README.md) for the full list. -1. **Metrics scraping** — In Prometheus, go to **Status → Targets** and confirm the `backend` target is `UP`. -2. **Rules** — In Prometheus, go to **Status → Rules** and confirm the alert rules are loaded. -3. **Alerts** — In Alertmanager (), confirm it is reachable and shows any firing alerts. -4. **Notifications** — Trigger a test alert (or use Alertmanager's built-in test) and confirm it arrives in the configured email/Slack channel. -5. **Dashboard** — Open the imported Grafana dashboard and confirm panels show live data. +When you add or rename an alert, add a matching runbook under `docs/runbooks/` and point the alert's `runbook_url` at it. ## Troubleshooting -- **Target is `DOWN`**: verify the metrics endpoint is reachable from the Prometheus container (`docker compose exec prometheus wget -qO- http://backend:3000/metrics`). Check host/port and network configuration. -- **No data in Grafana**: confirm the data source URL is correct and that Prometheus has scraped data (**Prometheus → Graph**, query `up`). -- **Rules not loading**: check the `rule_files` path and that the file is mounted into the container. Inspect Prometheus logs: `docker compose logs prometheus`. -- **Alerts not delivered**: check Alertmanager logs (`docker compose logs alertmanager`) and confirm the receiver config, webhook URL, and SMTP credentials are correct. -- **Config errors**: validate configs before reloading: - - Prometheus: `promtool check config monitoring/prometheus.yml` - - Alertmanager: `amtool check-config monitoring/alertmanager.yml` \ No newline at end of file +- **No metrics in Prometheus:** check the scrape target, network reachability, and that the backend `/metrics` endpoint responds. +- **Rules not loading:** validate the YAML and confirm the rule file is mounted at the path in `rule_files`. +- **Alerts not delivered:** check Alertmanager logs and confirm receiver credentials/secret files are correct. +- **Dashboard empty:** confirm the Grafana data source points at the correct Prometheus instance. diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml index 6b0ae7d..364913d 100644 --- a/monitoring/alert-rules.yml +++ b/monitoring/alert-rules.yml @@ -10,7 +10,7 @@ groups: annotations: summary: "Backend instance {{ $labels.instance }} is down" description: "Backend target {{ $labels.instance }} (job {{ $labels.job }}) has been unreachable for more than 1 minute." - runbook_url: "https://docs.example.com/runbooks/backend-down" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md" - alert: BackendTargetMissing expr: absent(up{job="backend"}) @@ -21,7 +21,7 @@ groups: annotations: summary: "No backend targets are being scraped" description: "Prometheus has not scraped any backend targets for more than 5 minutes. The service may be removed from service discovery." - runbook_url: "https://docs.example.com/runbooks/backend-down" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md" - name: http-errors rules: @@ -39,7 +39,7 @@ groups: annotations: summary: "High HTTP 5xx error rate ({{ $value | humanizePercentage }})" description: "More than 5% of backend HTTP requests have returned 5xx responses over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/http-5xx" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-5xx.md" - alert: ElevatedHttp5xxErrorRate expr: | @@ -55,7 +55,7 @@ groups: annotations: summary: "Elevated HTTP 5xx error rate ({{ $value | humanizePercentage }})" description: "More than 1% of backend HTTP requests have returned 5xx responses over the last 10 minutes." - runbook_url: "https://docs.example.com/runbooks/http-5xx" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-5xx.md" - alert: HighHttp4xxErrorRate expr: | @@ -71,7 +71,7 @@ groups: annotations: summary: "High HTTP 4xx error rate ({{ $value | humanizePercentage }})" description: "More than 25% of backend HTTP requests have returned 4xx responses over the last 15 minutes." - runbook_url: "https://docs.example.com/runbooks/http-4xx" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-4xx.md" - name: http-latency rules: @@ -88,7 +88,7 @@ groups: annotations: summary: "High HTTP p95 latency ({{ $value | humanizeDuration }})" description: "The 95th percentile backend HTTP request latency has exceeded 1s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/http-latency" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-latency.md" - alert: CriticalHttpLatencyP99 expr: | @@ -103,7 +103,7 @@ groups: annotations: summary: "Critical HTTP p99 latency ({{ $value | humanizeDuration }})" description: "The 99th percentile backend HTTP request latency has exceeded 3s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/http-latency" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/http-latency.md" - name: database rules: @@ -117,7 +117,7 @@ groups: annotations: summary: "Database connection failures detected" description: "The backend is experiencing database connection errors at a rate of {{ $value | humanize }}/s over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/database-connection" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-connection.md" - alert: DatabaseConnectionPoolExhausted expr: | @@ -133,7 +133,7 @@ groups: annotations: summary: "Database connection pool nearly exhausted ({{ $value | humanizePercentage }})" description: "More than 90% of the backend database connection pool has been in use for over 5 minutes." - runbook_url: "https://docs.example.com/runbooks/database-connection" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-connection.md" - alert: DatabaseDown expr: | @@ -145,7 +145,7 @@ groups: annotations: summary: "Postgres instance {{ $labels.instance }} is down" description: "Postgres target {{ $labels.instance }} reports pg_up == 0 for more than 1 minute, indicating the database itself is unreachable even though the exporter is running." - runbook_url: "https://docs.example.com/runbooks/database-down" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md" - alert: DatabaseExporterDown expr: | @@ -157,7 +157,7 @@ groups: annotations: summary: "Database exporter {{ $labels.instance }} is down" description: "The postgres-exporter target {{ $labels.instance }} has been unreachable for more than 1 minute, so database health cannot be observed." - runbook_url: "https://docs.example.com/runbooks/database-down" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md" - name: backups rules: @@ -171,7 +171,7 @@ groups: annotations: summary: "Backup job failed for {{ $labels.instance }}" description: "The most recent backup run for {{ $labels.instance }} did not succeed (soroban_backup_success == 0, written by scripts/backup.sh) for more than 5 minutes." - runbook_url: "https://docs.example.com/runbooks/backup-failed" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backup-failed.md" - alert: BackupStale expr: | @@ -183,7 +183,7 @@ groups: annotations: summary: "Backup is stale for {{ $labels.instance }}" description: "The last successful backup for {{ $labels.instance }} was more than 26 hours ago, exceeding the expected backup interval." - runbook_url: "https://docs.example.com/runbooks/backup-stale" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backup-stale.md" - name: stellar-soroban-rpc rules: @@ -197,7 +197,7 @@ groups: annotations: summary: "Stellar RPC errors detected" description: "The backend is experiencing Stellar RPC errors at a rate of {{ $value | humanize }}/s over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/stellar-rpc" + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/stellar-rpc.md" - alert: StellarRpcHighErrorRate expr: | @@ -208,18 +208,9 @@ groups: ) > 0.05 for: 5m labels: - severity: critical + severity: warning team: backend annotations: summary: "High Stellar RPC error rate ({{ $value | humanizePercentage }})" description: "More than 5% of Stellar RPC requests have failed over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/stellar-rpc" - - - alert: SorobanRpcErrors - expr: | - sum(rate(soroban_rpc_errors_total{job="backend"}[5m])) > 0 - for: 5m - labels: - severity: warning - team: backend - annotations: + runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/stellar-rpc.md" From da5d1ee8a717211d54bcb007dc565e1b213fe5d1 Mon Sep 17 00:00:00 2001 From: Teeeyanaa Date: Mon, 19 Oct 2026 16:14:23 +0000 Subject: [PATCH 3/3] fix: #90 Configure real Alertmanager receivers Closes #90 --- .env.example | 26 +++++++++++++- monitoring/alertmanager.yml | 70 +++++++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/.env.example b/.env.example index cdb457b..8e76b74 100644 --- a/.env.example +++ b/.env.example @@ -107,4 +107,28 @@ GRAFANA_PORT=3001 # Grafana admin login for the local monitoring stack. GRAFANA_ADMIN_USER=admin -GRAFANA_ADMIN_PASSWORD=admin \ No newline at end of file +GRAFANA_ADMIN_PASSWORD=admin + + +# ----------------------------------------------------------------------------- +# Alertmanager receivers +# ----------------------------------------------------------------------------- +# These values are injected into monitoring/alertmanager.yml at container +# start (see the alertmanager service in docker-compose.yml). Leave them empty +# to disable the corresponding receiver. Never commit real secrets. + +# Slack incoming webhook for warning-level alerts. +ALERTMANAGER_SLACK_WEBHOOK_URL= +# Slack channel to post warning alerts to (e.g. #alerts-warning). +ALERTMANAGER_SLACK_CHANNEL=#alerts-warning + +# PagerDuty integration key (Events API v2) for critical alerts. +ALERTMANAGER_PAGERDUTY_SERVICE_KEY= + +# SMTP settings for the email receiver (critical alerts). +ALERTMANAGER_SMTP_SMARTHOST=localhost:587 +ALERTMANAGER_SMTP_FROM=alerts@example.com +ALERTMANAGER_SMTP_AUTH_USERNAME= +ALERTMANAGER_SMTP_AUTH_PASSWORD= +# Comma-separated list of recipients for email alerts. +ALERTMANAGER_EMAIL_TO=oncall@example.com \ No newline at end of file diff --git a/monitoring/alertmanager.yml b/monitoring/alertmanager.yml index 1f31151..df5b38a 100644 --- a/monitoring/alertmanager.yml +++ b/monitoring/alertmanager.yml @@ -1,31 +1,79 @@ -# Minimal Alertmanager config. Alerts are grouped and routed to a single -# receiver; replace the webhook URL (or add Slack/PagerDuty/email receivers) -# for real notifications. See https://prometheus.io/docs/alerting/latest/configuration/ +# ============================================================================= +# Alertmanager configuration +# ============================================================================= +# Receivers are configured from environment variables / secret files so that +# no credentials are committed to the repository. The alertmanager service in +# docker-compose.yml renders this file with `envsubst` before starting, using +# the ALERTMANAGER_* variables documented in .env.example. +# +# Routing: +# - critical -> PagerDuty + email (paging path) +# - warning -> Slack (non-paging path) +# - default -> Slack +# ============================================================================= + global: resolve_timeout: 5m + smtp_smarthost: '${ALERTMANAGER_SMTP_SMARTHOST}' + smtp_from: '${ALERTMANAGER_SMTP_FROM}' + smtp_auth_username: '${ALERTMANAGER_SMTP_AUTH_USERNAME}' + smtp_auth_password: '${ALERTMANAGER_SMTP_AUTH_PASSWORD}' + smtp_require_tls: true +# The root route dispatches by severity. Critical alerts page via PagerDuty +# and email; warnings go to Slack only. route: - receiver: default - group_by: ['alertname', 'job'] + receiver: slack-warning + group_by: ['alertname', 'severity'] group_wait: 30s group_interval: 5m repeat_interval: 4h routes: - matchers: - severity="critical" - receiver: default + receiver: pagerduty-critical + group_wait: 10s repeat_interval: 1h + continue: true + - matchers: + - severity="critical" + receiver: email-critical + group_wait: 10s + repeat_interval: 1h + - matchers: + - severity="warning" + receiver: slack-warning + repeat_interval: 4h receivers: - - name: default - webhook_configs: - - url: 'http://localhost:9095/alerts' + - name: slack-warning + slack_configs: + - api_url: '${ALERTMANAGER_SLACK_WEBHOOK_URL}' + channel: '${ALERTMANAGER_SLACK_CHANNEL}' + send_resolved: true + title: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}' + text: >- + {{ range .Alerts }}{{ .Annotations.summary }} + {{ .Annotations.description }} + {{ end }} + + - name: pagerduty-critical + pagerduty_configs: + - service_key: '${ALERTMANAGER_PAGERDUTY_SERVICE_KEY}' + send_resolved: true + description: '{{ .CommonLabels.alertname }}: {{ .CommonAnnotations.summary }}' + + - name: email-critical + email_configs: + - to: '${ALERTMANAGER_EMAIL_TO}' send_resolved: true + headers: + Subject: '[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}' inhibit_rules: - # Don't page for warnings when the matching critical alert is already firing. + # A firing critical alert suppresses the matching warning for the same alert. - source_matchers: - severity="critical" target_matchers: - severity="warning" - equal: ['alertname', 'job'] + equal: ['alertname']