Improve monitoring coverage and harden docker-compose configuration

.env.example

@@ -125,6 +125,10 @@ GRAFANA_PORT=3001
 GRAFANA_ADMIN_USER=admin
 GRAFANA_ADMIN_PASSWORD=admin
 
+# Environment label attached to Prometheus external_labels (monitoring/prometheus.yml).
+# Defaults to `local` for the compose stack; set to `production` (or `staging`,
+# etc.) for non-local deployments so metrics and alerts are labelled correctly.
+PROMETHEUS_ENVIRONMENT=local
 
 # -----------------------------------------------------------------------------
 # Alertmanager receivers
@@ -147,4 +151,4 @@ ALERTMANAGER_SMTP_FROM=alerts@example.com
 ALERTMANAGER_SMTP_AUTH_USERNAME=
 ALERTMANAGER_SMTP_AUTH_PASSWORD=
 # Comma-separated list of recipients for email alerts.
-ALERTMANAGER_EMAIL_TO=oncall@example.com
+ALERTMANAGER_EMAIL_TO=oncall@example.com

docker-compose.yml

@@ -30,7 +30,7 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
       POSTGRES_DB: ${POSTGRES_DB:-app}
     ports:
-      - "${POSTGRES_PORT:-5432}:5432"
+      - "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
     volumes:
       - postgres_data:/var/lib/postgresql/data
     healthcheck:
@@ -46,7 +46,9 @@ services:
   migrate:
     build:
       context: ${BACKEND_SRC:-../backend}
-      dockerfile: ${PWD}/backend/Dockerfile
+      # Dockerfile is resolved relative to the build context, so this works
+      # regardless of the caller's working directory or shell (no ${PWD}).
+      dockerfile: Dockerfile
       target: migrate
     container_name: app_migrate
     environment:
@@ -60,7 +62,7 @@ services:
   backend:
     build:
       context: ${BACKEND_SRC:-../backend}
-      dockerfile: ${PWD}/backend/Dockerfile
+      dockerfile: Dockerfile
       target: runtime
     container_name: app_backend
     restart: unless-stopped
@@ -89,7 +91,7 @@ services:
   frontend:
     build:
       context: ${FRONTEND_SRC:-../frontend}
-      dockerfile: ${PWD}/frontend/Dockerfile
+      dockerfile: Dockerfile
       args:
         # Inlined into the bundle at build time; the browser calls the backend
         # directly on its published port.
@@ -129,6 +131,11 @@ services:
     image: prom/prometheus:v3.15.0
     container_name: app_prometheus
     restart: unless-stopped
+    environment:
+      # Labels metrics/alerts from this compose stack. Defaults to `local` so
+      # local runs are not mistaken for production; set PROMETHEUS_ENVIRONMENT
+      # (e.g. to `production`) when deploying this stack elsewhere.
+      PROMETHEUS_ENVIRONMENT: ${PROMETHEUS_ENVIRONMENT:-local}
     profiles:
       - monitoring
     volumes:
@@ -140,7 +147,7 @@ services:
       - "--storage.tsdb.path=/prometheus"
       - "--web.enable-lifecycle"
     ports:
-      - "${PROMETHEUS_PORT:-9090}:9090"
+      - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090"
     depends_on:
       backend:
         condition: service_healthy
@@ -166,6 +173,7 @@ services:
       - "--config.file=/etc/alertmanager/alertmanager.yml"
       - "--storage.path=/alertmanager"
     ports:
+      - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093"
       - "${ALERTMANAGER_PORT:-9093}:9093"
     healthcheck:
       test: ["CMD-SHELL", "wget -qO- http://localhost:9093/-/healthy || exit 1"]

monitoring/alert-rules.yml

@@ -23,6 +23,82 @@ groups:
           description: "Prometheus has not scraped any backend targets for more than 5 minutes. The service may be removed from service discovery."
           runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/backend-down.md"
 
+  - name: frontend-availability
+    rules:
+      - alert: FrontendDown
+        expr: probe_success{job="blackbox-http"} == 0
+        for: 2m
+        labels:
+          severity: critical
+          team: frontend
+        annotations:
+          summary: "Frontend public URL {{ $labels.instance }} is unreachable"
+          description: "The blackbox HTTP probe against {{ $labels.instance }} has been failing for more than 2 minutes."
+          runbook_url: "https://docs.example.com/runbooks/frontend-down"
+
+      - alert: FrontendSlowResponse
+        expr: probe_http_duration_seconds{job="blackbox-http", phase="connect"} > 2
+        for: 10m
+        labels:
+          severity: warning
+          team: frontend
+        annotations:
+          summary: "Frontend public URL {{ $labels.instance }} is slow"
+          description: "The blackbox HTTP probe connect phase against {{ $labels.instance }} has exceeded 2s for more than 10 minutes."
+          runbook_url: "https://docs.example.com/runbooks/frontend-down"
+
+  - name: certificate-expiry
+    rules:
+      - alert: TlsCertificateExpiringSoon
+        expr: |
+          (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 14 * 24 * 3600
+        for: 1h
+        labels:
+          severity: warning
+          team: frontend
+        annotations:
+          summary: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}"
+          description: "The TLS certificate served by {{ $labels.instance }} will expire in less than 14 days."
+          runbook_url: "https://docs.example.com/runbooks/certificate-expiry"
+
+      - alert: TlsCertificateExpiringCritical
+        expr: |
+          (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 3 * 24 * 3600
+        for: 1h
+        labels:
+          severity: critical
+          team: frontend
+        annotations:
+          summary: "TLS certificate for {{ $labels.instance }} expires in {{ $value | humanizeDuration }}"
+          description: "The TLS certificate served by {{ $labels.instance }} will expire in less than 3 days."
+          runbook_url: "https://docs.example.com/runbooks/certificate-expiry"
+
+  - name: pod-restarts
+    rules:
+      - alert: PodCrashLooping
+        expr: |
+          increase(kube_pod_container_status_restarts_total[15m]) > 3
+        for: 5m
+        labels:
+          severity: critical
+          team: backend
+        annotations:
+          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping"
+          description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 3 times in the last 15 minutes."
+          runbook_url: "https://docs.example.com/runbooks/pod-crash-looping"
+
+      - alert: PodRestartingFrequently
+        expr: |
+          increase(kube_pod_container_status_restarts_total[1h]) > 5
+        for: 10m
+        labels:
+          severity: warning
+          team: backend
+        annotations:
+          summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting frequently"
+          description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has restarted more than 5 times in the last hour."
+          runbook_url: "https://docs.example.com/runbooks/pod-crash-looping"
+
   - name: http-errors
     rules:
       - alert: HighHttp5xxErrorRate
@@ -159,6 +235,30 @@ groups:
           description: "The postgres-exporter target {{ $labels.instance }} has been unreachable for more than 1 minute, so database health cannot be observed."
           runbook_url: "https://github.com/example/soroban-backend/blob/main/docs/runbooks/database-down.md"
 
+      - alert: PostgresDatabaseSizeHigh
+        expr: |
+          pg_database_size_bytes{job="postgres"} > 10 * 1024 * 1024 * 1024
+        for: 15m
+        labels:
+          severity: warning
+          team: backend
+        annotations:
+          summary: "Postgres database {{ $labels.datname }} is large ({{ $value | humanize1024 }}B)"
+          description: "Postgres database {{ $labels.datname }} on {{ $labels.instance }} exceeds 10 GiB of disk usage."
+          runbook_url: "https://docs.example.com/runbooks/postgres-disk"
+
+      - alert: PostgresDatabaseSizeCritical
+        expr: |
+          pg_database_size_bytes{job="postgres"} > 50 * 1024 * 1024 * 1024
+        for: 15m
+        labels:
+          severity: critical
+          team: backend
+        annotations:
+          summary: "Postgres database {{ $labels.datname }} is critically large ({{ $value | humanize1024 }}B)"
+          description: "Postgres database {{ $labels.datname }} on {{ $labels.instance }} exceeds 50 GiB of disk usage."
+          runbook_url: "https://docs.example.com/runbooks/postgres-disk"
+
   - name: backups
     rules:
       - alert: BackupJobFailed

monitoring/prometheus.yml

@@ -4,7 +4,9 @@ global:
   scrape_timeout: 10s
   external_labels:
     monitor: 'synapsvault-monitor'
-    environment: 'production'
+    # Defaults to 'local' for docker-compose; production deployments set
+    # PROMETHEUS_ENVIRONMENT=production (Prometheus expands ${VAR} here).
+    environment: '${PROMETHEUS_ENVIRONMENT:-local}'
 
 alerting:
   alertmanagers:
@@ -49,6 +51,52 @@ scrape_configs:
           service: 'synapsvault-database'
           tier: 'data'
 
+  # Blackbox probes for the public frontend URL: HTTP availability and TLS
+  # certificate expiry. alert-rules.yml selects on job="blackbox_http" and
+  # job="blackbox_tls".
+  - job_name: 'blackbox_http'
+    metrics_path: '/probe'
+    params:
+      module: ['http_2xx']
+    static_configs:
+      - targets: ['https://synapsvault.example.com']
+        labels:
+          service: 'synapsvault-frontend'
+          tier: 'application'
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 'blackbox-exporter:9115'
+
+  - job_name: 'blackbox_tls'
+    metrics_path: '/probe'
+    params:
+      module: ['http_2xx']
+    static_configs:
+      - targets: ['https://synapsvault.example.com']
+        labels:
+          service: 'synapsvault-frontend'
+          tier: 'application'
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 'blackbox-exporter:9115'
+
+  # kube-state-metrics powers the pod restart / CrashLoopBackOff alerts.
+  - job_name: 'kube-state-metrics'
+    scrape_interval: 30s
+    static_configs:
+      - targets: ['kube-state-metrics:8080']
+        labels:
+          service: 'kube-state-metrics'
+          tier: 'infrastructure'
+
   # Backup metrics (soroban_backup_*) are written by scripts/backup.sh to a
   # node_exporter textfile collector. Uncomment when backups run on a host with
   # node_exporter; the GitHub Actions backup job has no exporter to scrape.