diff --git a/.env.example b/.env.example index b7fd375..df8aa5d 100644 --- a/.env.example +++ b/.env.example @@ -125,6 +125,10 @@ GRAFANA_PORT=3001 GRAFANA_ADMIN_USER=admin GRAFANA_ADMIN_PASSWORD=admin +# Environment label attached to Prometheus external_labels (monitoring/prometheus.yml). +# Defaults to `local` for the compose stack; set to `production` (or `staging`, +# etc.) for non-local deployments so metrics and alerts are labelled correctly. +PROMETHEUS_ENVIRONMENT=local # ----------------------------------------------------------------------------- # Alertmanager receivers @@ -147,4 +151,4 @@ ALERTMANAGER_SMTP_FROM=alerts@example.com ALERTMANAGER_SMTP_AUTH_USERNAME= ALERTMANAGER_SMTP_AUTH_PASSWORD= # Comma-separated list of recipients for email alerts. -ALERTMANAGER_EMAIL_TO=oncall@example.com \ No newline at end of file +ALERTMANAGER_EMAIL_TO=oncall@example.com diff --git a/docker-compose.yml b/docker-compose.yml index fef220d..067cd2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} POSTGRES_DB: ${POSTGRES_DB:-app} ports: - - "${POSTGRES_PORT:-5432}:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" volumes: - postgres_data:/var/lib/postgresql/data healthcheck: @@ -46,7 +46,9 @@ services: migrate: build: context: ${BACKEND_SRC:-../backend} - dockerfile: ${PWD}/backend/Dockerfile + # Dockerfile is resolved relative to the build context, so this works + # regardless of the caller's working directory or shell (no ${PWD}). + dockerfile: Dockerfile target: migrate container_name: app_migrate environment: @@ -60,7 +62,7 @@ services: backend: build: context: ${BACKEND_SRC:-../backend} - dockerfile: ${PWD}/backend/Dockerfile + dockerfile: Dockerfile target: runtime container_name: app_backend restart: unless-stopped @@ -89,7 +91,7 @@ services: frontend: build: context: ${FRONTEND_SRC:-../frontend} - dockerfile: ${PWD}/frontend/Dockerfile + dockerfile: Dockerfile args: # Inlined into the bundle at build time; the browser calls the backend # directly on its published port. @@ -129,6 +131,11 @@ services: image: prom/prometheus:v3.15.0 container_name: app_prometheus restart: unless-stopped + environment: + # Labels metrics/alerts from this compose stack. Defaults to `local` so + # local runs are not mistaken for production; set PROMETHEUS_ENVIRONMENT + # (e.g. to `production`) when deploying this stack elsewhere. + PROMETHEUS_ENVIRONMENT: ${PROMETHEUS_ENVIRONMENT:-local} profiles: - monitoring volumes: @@ -140,7 +147,7 @@ services: - "--storage.tsdb.path=/prometheus" - "--web.enable-lifecycle" ports: - - "${PROMETHEUS_PORT:-9090}:9090" + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" depends_on: backend: condition: service_healthy @@ -166,6 +173,7 @@ services: - "--config.file=/etc/alertmanager/alertmanager.yml" - "--storage.path=/alertmanager" ports: + - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093" - "${ALERTMANAGER_PORT:-9093}:9093" healthcheck: test: ["CMD-SHELL", "wget -qO- http://localhost:9093/-/healthy || exit 1"] diff --git a/monitoring/alert-rules.yml b/monitoring/alert-rules.yml index 489bc30..e80c150 100644 --- a/monitoring/alert-rules.yml +++ b/monitoring/alert-rules.yml @@ -23,6 +23,82 @@ groups: description: "Prometheus has not scraped any backend targets for more than 5 minutes. The service may be removed from service discovery." runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md" + - name: frontend-availability + rules: + - alert: FrontendDown + expr: probe_success{job="blackbox-http"} == 0 + for: 2m + labels: + severity: critical + team: frontend + annotations: + summary: "Frontend public URL {{ $labels.instance }} is unreachable" + description: "The blackbox HTTP probe against {{ $labels.instance }} has been failing for more than 2 minutes." + runbook_url: "https://docs.example.com/runbooks/frontend-down" + + - alert: FrontendSlowResponse + expr: probe_http_duration_seconds{job="blackbox-http", phase="connect"} > 2 + for: 10m + labels: + severity: warning + team: frontend + annotations: + summary: "Frontend public URL {{ $labels.instance }} is slow" + description: "The blackbox HTTP probe connect phase against {{ $labels.instance }} has exceeded 2s for more than 10 minutes." + runbook_url: "https://docs.example.com/runbooks/frontend-down" + + - name: certificate-expiry + rules: + - alert: TlsCertificateExpiringSoon + expr: | + (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 14 * 24 * 3600 + for: 1h + labels: + severity: warning + team: frontend + annotations: + summary: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}" + description: "The TLS certificate served by {{ $labels.instance }} will expire in less than 14 days." + runbook_url: "https://docs.example.com/runbooks/certificate-expiry" + + - alert: TlsCertificateExpiringCritical + expr: | + (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 3 * 24 * 3600 + for: 1h + labels: + severity: critical + team: frontend + annotations: + summary: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}" + description: "The TLS certificate served by {{ $labels.instance }} will expire in less than 3 days." + runbook_url: "https://docs.example.com/runbooks/certificate-expiry" + + - name: pod-restarts + rules: + - alert: PodCrashLooping + expr: | + increase(kube_pod_container_status_restarts_total[15m]) > 3 + for: 5m + labels: + severity: critical + team: backend + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping" + description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in the last 15 minutes." + runbook_url: "https://docs.example.com/runbooks/pod-crash-looping" + + - alert: PodRestartingFrequently + expr: | + increase(kube_pod_container_status_restarts_total[1h]) > 5 + for: 10m + labels: + severity: warning + team: backend + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting frequently" + description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 5 times in the last hour." + runbook_url: "https://docs.example.com/runbooks/pod-crash-looping" + - name: http-errors rules: - alert: HighHttp5xxErrorRate @@ -159,6 +235,30 @@ groups: description: "The postgres-exporter target {{ $labels.instance }} has been unreachable for more than 1 minute, so database health cannot be observed." runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md" + - alert: PostgresDatabaseSizeHigh + expr: | + pg_database_size_bytes{job="postgres"} > 10 * 1024 * 1024 * 1024 + for: 15m + labels: + severity: warning + team: backend + annotations: + summary: "Postgres database {{ $labels.datname }} is large ({{ $value | humanize1024 }}B)" + description: "Postgres database {{ $labels.datname }} on {{ $labels.instance }} exceeds 10 GiB of disk usage." + runbook_url: "https://docs.example.com/runbooks/postgres-disk" + + - alert: PostgresDatabaseSizeCritical + expr: | + pg_database_size_bytes{job="postgres"} > 50 * 1024 * 1024 * 1024 + for: 15m + labels: + severity: critical + team: backend + annotations: + summary: "Postgres database {{ $labels.datname }} is critically large ({{ $value | humanize1024 }}B)" + description: "Postgres database {{ $labels.datname }} on {{ $labels.instance }} exceeds 50 GiB of disk usage." + runbook_url: "https://docs.example.com/runbooks/postgres-disk" + - name: backups rules: - alert: BackupJobFailed diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml index c86149e..8197ec0 100644 --- a/monitoring/prometheus.yml +++ b/monitoring/prometheus.yml @@ -4,7 +4,9 @@ global: scrape_timeout: 10s external_labels: monitor: 'synapsvault-monitor' - environment: 'production' + # Defaults to 'local' for docker-compose; production deployments set + # PROMETHEUS_ENVIRONMENT=production (Prometheus expands ${VAR} here). + environment: '${PROMETHEUS_ENVIRONMENT:-local}' alerting: alertmanagers: @@ -49,6 +51,52 @@ scrape_configs: service: 'synapsvault-database' tier: 'data' + # Blackbox probes for the public frontend URL: HTTP availability and TLS + # certificate expiry. alert-rules.yml selects on job="blackbox_http" and + # job="blackbox_tls". + - job_name: 'blackbox_http' + metrics_path: '/probe' + params: + module: ['http_2xx'] + static_configs: + - targets: ['https://synapsvault.example.com'] + labels: + service: 'synapsvault-frontend' + tier: 'application' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 'blackbox-exporter:9115' + + - job_name: 'blackbox_tls' + metrics_path: '/probe' + params: + module: ['http_2xx'] + static_configs: + - targets: ['https://synapsvault.example.com'] + labels: + service: 'synapsvault-frontend' + tier: 'application' + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: 'blackbox-exporter:9115' + + # kube-state-metrics powers the pod restart / CrashLoopBackOff alerts. + - job_name: 'kube-state-metrics' + scrape_interval: 30s + static_configs: + - targets: ['kube-state-metrics:8080'] + labels: + service: 'kube-state-metrics' + tier: 'infrastructure' + # Backup metrics (soroban_backup_*) are written by scripts/backup.sh to a # node_exporter textfile collector. Uncomment when backups run on a host with # node_exporter; the GitHub Actions backup job has no exporter to scrape.