From 966ef2103db8b79c89a08a6eb82be53eae9a7ed7 Mon Sep 17 00:00:00 2001 From: fredkanwai Date: Mon, 19 Oct 2026 16:12:53 +0000 Subject: [PATCH 1/4] fix: #92 Add alerts for pod restarts, certificate expiry and frontend av Closes #92 --- monitoring/alert-rules.yml | 146 +++++++++++++++++++++++++------------ monitoring/prometheus.yml | 46 ++++++++++++ 2 files changed, 146 insertions(+), 46 deletions(-) diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml index edc9410..ecfe2d2 100644 --- a/monitoring/alert-rules.yml +++ b/monitoring/alert-rules.yml @@ -23,6 +23,82 @@ groups: description: "Prometheus has not scraped any backend targets for more than 5 minutes. The service may be removed from service discovery." runbook_url: "https://docs.example.com/runbooks/backend-down" + - name: frontend-availability + rules: + - alert: FrontendDown + expr: probe_success{job="blackbox-http"} == 0 + for: 2m + labels: + severity: critical + team: frontend + annotations: + summary: "Frontend public URL {{ $labels.instance }} is unreachable" + description: "The blackbox HTTP probe against {{ $labels.instance }} has been failing for more than 2 minutes." + runbook_url: "https://docs.example.com/runbooks/frontend-down" + + - alert: FrontendSlowResponse + expr: probe_http_duration_seconds{job="blackbox-http", phase="connect"} > 2 + for: 10m + labels: + severity: warning + team: frontend + annotations: + summary: "Frontend public URL {{ $labels.instance }} is slow" + description: "The blackbox HTTP probe connect phase against {{ $labels.instance }} has exceeded 2s for more than 10 minutes." + runbook_url: "https://docs.example.com/runbooks/frontend-down" + + - name: certificate-expiry + rules: + - alert: TlsCertificateExpiringSoon + expr: | + (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 14 * 24 * 3600 + for: 1h + labels: + severity: warning + team: frontend + annotations: + summary: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}" + description: "The TLS certificate served by {{ $labels.instance }} will expire in less than 14 days." + runbook_url: "https://docs.example.com/runbooks/certificate-expiry" + + - alert: TlsCertificateExpiringCritical + expr: | + (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 3 * 24 * 3600 + for: 1h + labels: + severity: critical + team: frontend + annotations: + summary: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}" + description: "The TLS certificate served by {{ $labels.instance }} will expire in less than 3 days." + runbook_url: "https://docs.example.com/runbooks/certificate-expiry" + + - name: pod-restarts + rules: + - alert: PodCrashLooping + expr: | + increase(kube_pod_container_status_restarts_total[15m]) > 3 + for: 5m + labels: + severity: critical + team: backend + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping" + description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in the last 15 minutes." + runbook_url: "https://docs.example.com/runbooks/pod-crash-looping" + + - alert: PodRestartingFrequently + expr: | + increase(kube_pod_container_status_restarts_total[1h]) > 5 + for: 10m + labels: + severity: warning + team: backend + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting frequently" + description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 5 times in the last hour." + runbook_url: "https://docs.example.com/runbooks/pod-crash-looping" + - name: http-errors rules: - alert: HighHttp5xxErrorRate @@ -147,6 +223,30 @@ groups: description: "Database target {{ $labels.instance }} has been unreachable for more than 1 minute." runbook_url: "https://docs.example.com/runbooks/database-down" + - alert: PostgresDatabaseSizeHigh + expr: | + pg_database_size_bytes{job="postgres"} > 10 * 1024 * 1024 * 1024 + for: 15m + labels: + severity: warning + team: backend + annotations: + summary: "Postgres database {{ $labels.datname }} is large ({{ $value | humanize1024 }}B)" + description: "Postgres database {{ $labels.datname }} on {{ $labels.instance }} exceeds 10 GiB of disk usage." + runbook_url: "https://docs.example.com/runbooks/postgres-disk" + + - alert: PostgresDatabaseSizeCritical + expr: | + pg_database_size_bytes{job="postgres"} > 50 * 1024 * 1024 * 1024 + for: 15m + labels: + severity: critical + team: backend + annotations: + summary: "Postgres database {{ $labels.datname }} is critically large ({{ $value | humanize1024 }}B)" + description: "Postgres database {{ $labels.datname }} on {{ $labels.instance }} exceeds 50 GiB of disk usage." + runbook_url: "https://docs.example.com/runbooks/postgres-disk" + - name: backups rules: - alert: BackupJobFailed @@ -214,49 +314,3 @@ groups: summary: "Soroban RPC errors detected" description: "The backend is experiencing Soroban RPC errors at a rate of {{ $value | humanize }}/s over the last 5 minutes." runbook_url: "https://docs.example.com/runbooks/soroban-rpc" - - - alert: SorobanRpcHighErrorRate - expr: | - ( - sum(rate(soroban_rpc_errors_total{job="backend"}[5m])) - / - sum(rate(soroban_rpc_requests_total{job="backend"}[5m])) - ) > 0.05 - for: 5m - labels: - severity: critical - team: backend - annotations: - summary: "High Soroban RPC error rate ({{ $value | humanizePercentage }})" - description: "More than 5% of Soroban RPC requests have failed over the last 5 minutes." - runbook_url: "https://docs.example.com/runbooks/soroban-rpc" - - - alert: StellarRpcLatencyHigh - expr: | - histogram_quantile( - 0.95, - sum(rate(stellar_rpc_request_duration_seconds_bucket{job="backend"}[5m])) by (le) - ) > 2 - for: 10m - labels: - severity: warning - team: backend - annotations: - summary: "High Stellar RPC latency ({{ $value | humanizeDuration }})" - description: "The 95th percentile Stellar RPC request latency has exceeded 2s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/stellar-rpc" - - - alert: SorobanRpcLatencyHigh - expr: | - histogram_quantile( - 0.95, - sum(rate(soroban_rpc_request_duration_seconds_bucket{job="backend"}[5m])) by (le) - ) > 2 - for: 10m - labels: - severity: warning - team: backend - annotations: - summary: "High Soroban RPC latency ({{ $value | humanizeDuration }})" - description: "The 95th percentile Soroban RPC request latency has exceeded 2s for more than 10 minutes." - runbook_url: "https://docs.example.com/runbooks/soroban-rpc" \ No newline at end of file diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index c86149e..abc945c 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -49,6 +49,52 @@ scrape_configs: service: 'synapsvault-database' tier: 'data' + # Blackbox probes for the public frontend URL: HTTP availability and TLS + # certificate expiry. alert-rules.yml selects on job="blackbox_http" and + # job="blackbox_tls". + - job_name: 'blackbox_http' + metrics_path: '/probe' + params: + module: ['http_2xx'] + static_configs: + - targets: ['https://synapsvault.example.com'] + labels: + service: 'synapsvault-frontend' + tier: 'application' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 'blackbox-exporter:9115' + + - job_name: 'blackbox_tls' + metrics_path: '/probe' + params: + module: ['http_2xx'] + static_configs: + - targets: ['https://synapsvault.example.com'] + labels: + service: 'synapsvault-frontend' + tier: 'application' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 'blackbox-exporter:9115' + + # kube-state-metrics powers the pod restart / CrashLoopBackOff alerts. + - job_name: 'kube-state-metrics' + scrape_interval: 30s + static_configs: + - targets: ['kube-state-metrics:8080'] + labels: + service: 'kube-state-metrics' + tier: 'infrastructure' + # Backup metrics (soroban_backup_*) are written by scripts/backup.sh to a # node_exporter textfile collector. Uncomment when backups run on a host with # node_exporter; the GitHub Actions backup job has no exporter to scrape. From b3316d87d9f2d9244986ea9369b7589509a6d4b4 Mon Sep 17 00:00:00 2001 From: fredkanwai Date: Mon, 19 Oct 2026 16:13:07 +0000 Subject: [PATCH 2/4] fix: #93 prometheus.yml labels local compose metrics as environment=prod Closes #93 --- .env.example | 7 ++++++- docker-compose.yml | 5 +++++ monitoring/prometheus.yml | 4 +++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index cdb457b..7b417ea 100644 --- a/.env.example +++ b/.env.example @@ -107,4 +107,9 @@ GRAFANA_PORT=3001 # Grafana admin login for the local monitoring stack. GRAFANA_ADMIN_USER=admin -GRAFANA_ADMIN_PASSWORD=admin \ No newline at end of file +GRAFANA_ADMIN_PASSWORD=admin + +# Environment label attached to Prometheus external_labels (monitoring/prometheus.yml). +# Defaults to `local` for the compose stack; set to `production` (or `staging`, +# etc.) for non-local deployments so metrics and alerts are labelled correctly. +PROMETHEUS_ENVIRONMENT=local \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index a457caa..c306dcb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -106,6 +106,11 @@ services: image: prom/prometheus:v2.53.2 container_name: app_prometheus restart: unless-stopped + environment: + # Labels metrics/alerts from this compose stack. Defaults to `local` so + # local runs are not mistaken for production; set PROMETHEUS_ENVIRONMENT + # (e.g. to `production`) when deploying this stack elsewhere. + PROMETHEUS_ENVIRONMENT: ${PROMETHEUS_ENVIRONMENT:-local} volumes: - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro - ./monitoring/alert-rules.yml:/etc/prometheus/alert-rules.yml:ro diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index abc945c..8197ec0 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -4,7 +4,9 @@ global: scrape_timeout: 10s external_labels: monitor: 'synapsvault-monitor' - environment: 'production' + # Defaults to 'local' for docker-compose; production deployments set + # PROMETHEUS_ENVIRONMENT=production (Prometheus expands ${VAR} here). + environment: '${PROMETHEUS_ENVIRONMENT:-local}' alerting: alertmanagers: From e14f60b6febedb27d1bc52bb15f729e5c09a791c Mon Sep 17 00:00:00 2001 From: fredkanwai Date: Mon, 19 Oct 2026 16:13:15 +0000 Subject: [PATCH 3/4] fix: #94 docker-compose.yml uses `${PWD}` for the Dockerfile path Closes #94 --- docker-compose.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c306dcb..e2b2e05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,9 @@ services: migrate: build: context: ${BACKEND_SRC:-../backend} - dockerfile: ${PWD}/backend/Dockerfile + # Dockerfile is resolved relative to the build context, so this works + # regardless of the caller's working directory or shell (no ${PWD}). + dockerfile: Dockerfile target: migrate container_name: app_migrate environment: @@ -51,7 +53,7 @@ services: backend: build: context: ${BACKEND_SRC:-../backend} - dockerfile: ${PWD}/backend/Dockerfile + dockerfile: Dockerfile target: runtime container_name: app_backend restart: unless-stopped @@ -74,7 +76,7 @@ services: frontend: build: context: ${FRONTEND_SRC:-../frontend} - dockerfile: ${PWD}/frontend/Dockerfile + dockerfile: Dockerfile args: # Inlined into the bundle at build time; the browser calls the backend # directly on its published port. From 598b3066aa0535de075a45ef35e0efaf4361616b Mon Sep 17 00:00:00 2001 From: fredkanwai Date: Mon, 19 Oct 2026 16:13:21 +0000 Subject: [PATCH 4/4] fix: #95 Bind Postgres and monitoring ports to localhost in compose Closes #95 --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e2b2e05..1056623 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} POSTGRES_DB: ${POSTGRES_DB:-app} ports: - - "${POSTGRES_PORT:-5432}:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" volumes: - postgres_data:/var/lib/postgresql/data healthcheck: @@ -122,7 +122,7 @@ services: - "--storage.tsdb.path=/prometheus" - "--web.enable-lifecycle" ports: - - "${PROMETHEUS_PORT:-9090}:9090" + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" depends_on: backend: condition: service_started @@ -140,7 +140,7 @@ services: - "--config.file=/etc/alertmanager/alertmanager.yml" - "--storage.path=/alertmanager" ports: - - "${ALERTMANAGER_PORT:-9093}:9093" + - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093" networks: - app_network