diff --git a/docs/MONITORING.md b/docs/MONITORING.md index e30a968..3740833 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -20,6 +20,7 @@ All configuration referenced here lives in the [`monitoring/`](../monitoring) di - [Email](#email) - [Slack](#slack) - [6. Verify the setup](#6-verify-the-setup) +- [Required metrics](#required-metrics) - [Troubleshooting](#troubleshooting) ## Overview @@ -218,6 +219,7 @@ receivers: from: alertmanager@example.com smarthost: smtp.example.com:587 auth_username: alertmanager@example.com + auth_password_file: /etc/alertmanager/smtp_password auth_password: require_tls: true ``` @@ -228,6 +230,49 @@ receivers: receivers: - name: slack slack_configs: + - api_url_file: /etc/alertmanager/slack_webhook_url + channel: "#alerts" + send_resolved: true +``` + +Route alerts to the receivers in the `route` block, then reload Alertmanager: + +```bash +curl -X POST http://localhost:9093/-/reload +``` + +## 6. Verify the setup + +1. Confirm Prometheus is scraping the backend: **Prometheus → Status → Targets** should show the `backend` job as `UP`. +2. Confirm the required metrics are present (see [Required metrics](#required-metrics)). +3. Trigger a test alert (e.g. stop the backend) and confirm it reaches your notification channel. + +## Required metrics + +The alert rules in [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) depend on the backend exporting the following metric families. If any of these are missing from the `/metrics` output, the corresponding alerts will silently never fire. + +| Metric family | Used for | +| --- | --- | +| `db_connection_pool_*` | Database connection pool saturation and exhaustion alerts | +| `stellar_rpc_*` | Stellar RPC latency, error rate, and availability alerts | +| `soroban_rpc_*` | Soroban RPC latency, error rate, and availability alerts | + +Verify each family is exported before relying on the alerts: + +```bash +curl -s http://localhost:3000/metrics \ + | grep -E '^(db_connection_pool_|stellar_rpc_|soroban_rpc_)' \ + | awk -F'{' '{print $1}' | awk '{print $1}' | sort -u +``` + +If a family is absent, the backend is missing instrumentation for it. Open an issue in the backend repository (`SynapsVault/backend`) to add the missing metrics. The alert rules include `absent()` checks that fire when a required metric family is not exported, so missing instrumentation surfaces as an alert rather than failing silently. + +## Troubleshooting + +- **Targets show `DOWN`**: check the scrape target host/port and that the backend is reachable from Prometheus. +- **No metrics in Grafana**: confirm the data source URL and that the dashboard uses the correct data source. +- **Alerts never fire**: verify the required metrics are exported (see [Required metrics](#required-metrics)) and that the rule file is loaded. +- **Reload fails**: ensure Prometheus was started with `--web.enable-lifecycle`. - api_url: https://hooks.slack.com/services/// channel: '#alerts' send_resolved: true diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml index edc9410..1391ef5 100644 --- a/monitoring/alert-rules.yml +++ b/monitoring/alert-rules.yml @@ -231,32 +231,46 @@ groups: description: "More than 5% of Soroban RPC requests have failed over the last 5 minutes." runbook_url: "https://docs.example.com/runbooks/soroban-rpc" - - alert: StellarRpcLatencyHigh + - name: metrics-instrumentation + rules: + - alert: DatabaseConnectionPoolMetricsMissing expr: | - histogram_quantile( - 0.95, - sum(rate(stellar_rpc_request_duration_seconds_bucket{job="backend"}[5m])) by (le) - ) > 2 - for: 10m + absent(db_connection_pool_in_use{job="backend"}) + or + absent(db_connection_pool_max{job="backend"}) + for: 15m labels: severity: warning team: backend annotations: - summary: "High Stellar RPC latency ({{ $value | humanizeDuration }})" - description: "The 95th percentile Stellar RPC request latency has exceeded 2s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/stellar-rpc" + summary: "Database connection pool metrics are not being exported" + description: "The backend is not exporting db_connection_pool_in_use / db_connection_pool_max, so DatabaseConnectionPoolExhausted can never fire. Add the instrumentation in SynapsVault/backend." + runbook_url: "https://docs.example.com/runbooks/missing-metrics" - - alert: SorobanRpcLatencyHigh + - alert: StellarRpcMetricsMissing expr: | - histogram_quantile( - 0.95, - sum(rate(soroban_rpc_request_duration_seconds_bucket{job="backend"}[5m])) by (le) - ) > 2 - for: 10m + absent(stellar_rpc_requests_total{job="backend"}) + or + absent(stellar_rpc_errors_total{job="backend"}) + for: 15m + labels: + severity: warning + team: backend + annotations: + summary: "Stellar RPC metrics are not being exported" + description: "The backend is not exporting stellar_rpc_requests_total / stellar_rpc_errors_total, so the Stellar RPC alerts can never fire. Add the instrumentation in SynapsVault/backend." + runbook_url: "https://docs.example.com/runbooks/missing-metrics" + + - alert: SorobanRpcMetricsMissing + expr: | + absent(soroban_rpc_requests_total{job="backend"}) + or + absent(soroban_rpc_errors_total{job="backend"}) + for: 15m labels: severity: warning team: backend annotations: - summary: "High Soroban RPC latency ({{ $value | humanizeDuration }})" - description: "The 95th percentile Soroban RPC request latency has exceeded 2s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/soroban-rpc" \ No newline at end of file + summary: "Soroban RPC metrics are not being exported" + description: "The backend is not exporting soroban_rpc_requests_total / soroban_rpc_errors_total, so the Soroban RPC alerts can never fire. Add the instrumentation in SynapsVault/backend." + runbook_url: "https://docs.example.com/runbooks/missing-metrics"