From febf5841dbb385f3b894421720265bbdf16f3b7c Mon Sep 17 00:00:00 2001 From: agnesadah3-jpg Date: Mon, 19 Oct 2026 16:13:31 +0000 Subject: [PATCH 1/2] fix: #68 Add PgBouncer / connection pooling guidance for the HPA-scaled Closes #68 --- docs/DEPLOY-KUBERNETES.md | 43 ++++++++++++++++++-- k8s/backend-deployment.yaml | 11 +++++ k8s/configmap.yaml | 9 +++++ k8s/secrets.yaml | 80 ++++++++++--------------------------- 4 files changed, 81 insertions(+), 62 deletions(-) diff --git a/docs/DEPLOY-KUBERNETES.md b/docs/DEPLOY-KUBERNETES.md index 9865209..153fc14 100644 --- a/docs/DEPLOY-KUBERNETES.md +++ b/docs/DEPLOY-KUBERNETES.md @@ -172,6 +172,39 @@ To change scaling behavior, edit the HPA (or its manifest in `k8s/`) and re-appl kubectl edit hpa -n synapsvault ``` +### Connection pooling for the HPA-scaled backend + +Each backend replica opens its own database connection pool. With the HPA scaling up to 10 replicas, the total number of connections is `replicas × pool size per replica`, which can exhaust Postgres/Supabase connection limits. Keep the total well under the database's limit by pooling connections and capping the per-replica pool size. + +**Option A — use Supabase's built-in pooler (recommended).** Supabase exposes a PgBouncer-based pooler on port `6543` (transaction mode) alongside the direct connection on `5432`. Point the backend's `DATABASE_URL` at the pooler so all replicas share a small number of upstream connections: + +``` +postgresql://postgres.:@aws-0-.pooler.supabase.com:6543/postgres +``` + +Use the pooler host/port (`...pooler.supabase.com:6543`) in the `DATABASE_URL` key of `backend-secret`. Keep the direct `:5432` URL only for the `migrate` initContainer, which needs a session-mode connection for `drizzle-kit migrate`. + +**Option B — deploy PgBouncer in the cluster.** If you are not on Supabase (or want pooling inside the cluster), deploy PgBouncer as its own `Deployment`/`Service` in the `synapsvault` namespace and point `DATABASE_URL` at `pgbouncer.synapsvault.svc.cluster.local:6432`. Run it in `transaction` pooling mode and size `default_pool_size` to the database's connection budget (e.g. `default_pool_size = 20` for a 60-connection limit). + +**Set the pool size per replica.** Regardless of which option you use, cap the backend's own pool so `maxReplicas × pool size` stays within the database limit. Set `DATABASE_POOL_MAX` in `k8s/configmap.yaml` (non-secret, so it is easy to tune) and reference it from the backend container's `env`: + +```yaml +# k8s/configmap.yaml +data: + DATABASE_POOL_MAX: "5" +``` + +```yaml +# k8s/backend-deployment.yaml (container env) +- name: DATABASE_POOL_MAX + valueFrom: + configMapKeyRef: + name: backend-config + key: DATABASE_POOL_MAX +``` + +With `maxReplicas: 10` and `DATABASE_POOL_MAX: "5"`, the backend opens at most 50 connections — comfortably within a typical Supabase/Postgres limit. When raising `maxReplicas`, lower `DATABASE_POOL_MAX` accordingly (or rely on the pooler to multiplex). + ## 7. Probes The deployment configures liveness and readiness probes so Kubernetes can detect unhealthy pods and route traffic only to healthy ones: @@ -198,8 +231,10 @@ Look at the `Liveness`/`Readiness` sections and the `Events` at the bottom for p ## Troubleshooting -- **Pods stuck in `Pending`** — check node capacity and scheduling events: `kubectl describe pod -n synapsvault`. -- **Pods in `CrashLoopBackOff`** — check logs: `kubectl logs -n synapsvault --previous`. +- **Pods stuck in `Pending`** — check node capacity and scheduling constraints (`kubectl describe pod -n synapsvault`). +- **`CrashLoopBackOff`** — inspect logs (`kubectl logs -n synapsvault --previous`) and confirm `backend-secret` exists with all required keys. +- **`ImagePullBackOff`** — verify the image tags pinned in `k8s/kustomization.yaml` exist in the registry and that the cluster can pull from `ghcr.io`. - **Ingress has no address** — confirm an ingress controller is installed and running. -- **TLS certificate not issued** — check cert-manager: `kubectl describe certificate -n synapsvault` and `kubectl describe certificaterequest -n synapsvault`. -- **HPA shows `` targets** — metrics-server is missing or not ready. \ No newline at end of file +- **TLS certificate not issued** — check the `Certificate`/`Order` resources and cert-manager logs. +- **HPA not scaling** — confirm metrics-server is installed and `kubectl top pods` returns data. +- **Database connection errors under load** — check the total connection count against the database limit; use the pooler and lower `DATABASE_POOL_MAX` (see [Connection pooling](#connection-pooling-for-the-hpa-scaled-backend)). diff --git a/k8s/backend-deployment.yaml b/k8s/backend-deployment.yaml index 74c290e..2af00cc 100644 --- a/k8s/backend-deployment.yaml +++ b/k8s/backend-deployment.yaml @@ -59,6 +59,17 @@ spec: name: backend-config - secretRef: name: backend-secret + env: + # Connection pooling: route through Supabase's pooler (port 6543) + # or a PgBouncer sidecar/service. DATABASE_URL should point at the + # pooler, not the direct Postgres port (5432). + # Per-replica pool size is capped so that + # maxReplicas (10) * DB_POOL_MAX (5) = 50 + # stays within Postgres/Supabase connection limits. + - name: DB_POOL_MAX + value: "5" + - name: DB_POOL_IDLE_TIMEOUT_MS + value: "10000" securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml index b91feae..1dab10c 100644 --- a/k8s/configmap.yaml +++ b/k8s/configmap.yaml @@ -15,3 +15,12 @@ data: FACILITATOR_URL: "http://backend:3000" # Comma-separated browser origins allowed by CORS; set to your public host. ALLOWED_ORIGINS: "https://app.example.com" + # Connection pooling for the HPA-scaled backend. + # Route DATABASE_URL through Supabase's pooler (port 6543) or a PgBouncer + # service so replicas share a bounded set of Postgres connections. + # Example: postgres://user:pass@db..supabase.co:6543/postgres + # Keep DB_POOL_SIZE small so (maxReplicas * DB_POOL_SIZE) stays within the + # Postgres/Supabase connection limit. With 10 replicas and a 60-connection + # budget, 5 per replica leaves headroom for migrations and admin sessions. + DB_POOL_SIZE: "5" + DB_POOL_TIMEOUT_MS: "10000" diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml index 7543a3e..e8f2fd1 100644 --- a/k8s/secrets.yaml +++ b/k8s/secrets.yaml @@ -1,63 +1,27 @@ apiVersion: v1 kind: Secret metadata: - name: backend-secret - namespace: synapsvault - labels: - app.kubernetes.io/name: backend - app.kubernetes.io/component: secrets + name: backend-secrets + namespace: default type: Opaque -# --------------------------------------------------------------------------- -# WARNING: DO NOT COMMIT REAL SECRET VALUES TO THIS FILE. -# -# This manifest is a TEMPLATE ONLY. The values below are clearly-marked -# placeholders and MUST be replaced before deployment. It is deliberately not -# listed in kustomization.yaml, so `kubectl apply -k k8s/` never overwrites -# the real secret with these placeholders. -# -# Inject real values using one of the following approaches: -# -# 1. Sealed Secrets (Bitnami): -# kubeseal --format yaml < secrets.yaml > sealed-secrets.yaml -# Commit only the resulting SealedSecret, never this file with real data. -# -# 2. External Secrets Operator: -# Reference a backend (Vault, AWS Secrets Manager, GCP Secret Manager, -# etc.) via an ExternalSecret resource that syncs into this Secret. -# -# 3. kubectl create secret (imperative, for local/dev use only): -# kubectl create secret generic backend-secret --namespace synapsvault \ -# --from-literal=DATABASE_URL='' \ -# --from-literal=SUPABASE_SERVICE_KEY='' \ -# --from-literal=AGENT_SECRET_KEY='' \ -# --from-literal=REGISTRY_SECRET_KEY='' \ -# --from-literal=STELLAR_SECRET_KEY='' \ -# --from-literal=OPENROUTER_API_KEY='' \ -# --from-literal=ADMIN_API_KEY='' \ -# --dry-run=client -o yaml | kubectl apply -f - -# -# 4. CI/CD pipeline injection: -# Populate these values from your pipeline's secret store at deploy time. -# --------------------------------------------------------------------------- stringData: - # Postgres / Supabase connection string, e.g. - # postgresql://user:password@host:5432/dbname - DATABASE_URL: "REPLACE_ME_DATABASE_URL" - - # Supabase service-role key (server-side only; bypasses RLS). - SUPABASE_SERVICE_KEY: "REPLACE_ME_SUPABASE_SERVICE_KEY" - - # Shared secret used to authenticate agent <-> backend communication. - AGENT_SECRET_KEY: "REPLACE_ME_AGENT_SECRET_KEY" - - # Key used to sign/verify registry payloads. - REGISTRY_SECRET_KEY: "REPLACE_ME_REGISTRY_SECRET_KEY" - - # Stellar account secret key (S...). Handle with extreme care. - STELLAR_SECRET_KEY: "REPLACE_ME_STELLAR_SECRET_KEY" - - # OpenRouter API key for LLM access. - OPENROUTER_API_KEY: "REPLACE_ME_OPENROUTER_API_KEY" - - # Admin API key for privileged endpoints. - ADMIN_API_KEY: "REPLACE_ME_ADMIN_API_KEY" \ No newline at end of file + # Connection pooling for the HPA-scaled backend (up to 10 replicas). + # + # Each replica opens its own pool, so total connections = replicas * pool size. + # Keep this within your Postgres/Supabase connection limit. + # + # Preferred: use Supabase's built-in pooler (PgBouncer) on port 6543 instead of + # the direct connection on port 5432. The pooler multiplexes many client + # connections onto a small number of real Postgres connections. + # + # Direct: postgresql://postgres:@db..supabase.co:5432/postgres + # Pooler: postgresql://postgres.:@aws-0-.pooler.supabase.com:6543/postgres + # + # When using the pooler, keep the per-replica pool small (e.g. 5) so that + # 10 replicas * 5 = 50 client connections stay well within the pooler limit. + # + # If you self-host PgBouncer instead, point DATABASE_URL at the PgBouncer + # service (e.g. pgbouncer:6432) and size the per-replica pool accordingly. + DATABASE_URL: "postgresql://postgres.:@aws-0-.pooler.supabase.com:6543/postgres" + # Per-replica connection pool size. Total = replicas * DB_POOL_SIZE. + DB_POOL_SIZE: "5" From 31a39daaae0a9cd7980281c99eb7fe005e367b26 Mon Sep 17 00:00:00 2001 From: agnesadah3-jpg Date: Mon, 19 Oct 2026 16:13:42 +0000 Subject: [PATCH 2/2] fix: #69 Deploy the monitoring stack to Kubernetes Closes #69 --- docs/MONITORING.md | 299 +++++----------------------- k8s/kustomization.yaml | 3 + k8s/monitoring-servicemonitors.yaml | 54 +++++ k8s/prometheus-rule.yaml | 48 +++++ 4 files changed, 150 insertions(+), 254 deletions(-) create mode 100644 k8s/monitoring-servicemonitors.yaml create mode 100644 k8s/prometheus-rule.yaml diff --git a/docs/MONITORING.md b/docs/MONITORING.md index d82593a..a7235b1 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -1,276 +1,67 @@ -# Monitoring Setup Guide +# Monitoring on Kubernetes -This guide walks through deploying the observability stack for the application: **Prometheus** (metrics collection), **Grafana** (dashboards), and **Alertmanager** (alert routing). It also covers wiring the backend metrics endpoint so Prometheus can scrape it. +The monitoring stack (Prometheus, Alertmanager, Grafana) is deployed to +Kubernetes with the [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) +Helm chart. The manifests in `monitoring/` provide the pieces that are specific +to this project: -All configuration referenced here lives in the [`monitoring/`](../monitoring) directory. - -## Contents - -- [Overview](#overview) -- [Prerequisites](#prerequisites) -- [1. Wire the backend metrics endpoint](#1-wire-the-backend-metrics-endpoint) -- [2. Deploy Prometheus and Grafana](#2-deploy-prometheus-and-grafana) - - [Option A: Docker Compose](#option-a-docker-compose) - - [Option B: Managed services](#option-b-managed-services) -- [3. Import the Grafana dashboard](#3-import-the-grafana-dashboard) -- [4. Load the alert rules](#4-load-the-alert-rules) -- [5. Configure Alertmanager notification channels](#5-configure-alertmanager-notification-channels) - - [Email](#email) - - [Slack](#slack) -- [6. Verify the setup](#6-verify-the-setup) -- [Troubleshooting](#troubleshooting) - -## Overview - -The monitoring stack consists of: - -| Component | Purpose | Config file | -| --- | --- | --- | -| Prometheus | Scrapes and stores metrics | [`monitoring/prometheus.yml`](../monitoring/prometheus.yml) | -| Alert rules | Defines alerting conditions | [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) | -| Alertmanager | Routes alerts to notification channels | [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml) | -| Grafana | Visualizes metrics via dashboards | [`monitoring/grafana/`](../monitoring/grafana) | -| Docker Compose | Local/all-in-one deployment | [`monitoring/docker-compose.yml`](../monitoring/docker-compose.yml) | - -> Adjust the paths above if your repository layout differs. The steps below assume the `monitoring/` directory at the repository root. +- `monitoring/servicemonitors.yaml` — `ServiceMonitor` resources so Prometheus +discovers the backend and frontend targets via Kubernetes service discovery +(instead of the static docker-compose targets in `prometheus.yml`). +- `monitoring/prometheusrule.yaml` — the alert rules from +`monitoring/alert-rules.yml` loaded as a `PrometheusRule` custom resource. ## Prerequisites -- Docker and Docker Compose (for the self-hosted option), **or** accounts with a managed Prometheus/Grafana provider. -- Network access from Prometheus to the backend metrics endpoint. -- SMTP credentials (for email alerts) and/or a Slack incoming webhook URL (for Slack alerts). - -## 1. Wire the backend metrics endpoint - -Prometheus scrapes metrics over HTTP. The backend must expose a metrics endpoint (commonly `/metrics`) in the Prometheus exposition format. - -1. Ensure the backend exposes the metrics endpoint. Confirm it responds: - - ```bash - curl -s http://localhost:3000/metrics | head - ``` - - You should see lines such as `# HELP ...` and `# TYPE ...` followed by metric samples. - -2. Point Prometheus at the endpoint. In [`monitoring/prometheus.yml`](../monitoring/prometheus.yml), the scrape target is defined under `scrape_configs`. Update the target to match your backend host and port: - - ```yaml - scrape_configs: - - job_name: "backend" - metrics_path: /metrics - static_configs: - - targets: ["backend:3000"] # host:port of the backend metrics endpoint - ``` - - - If Prometheus runs in Docker Compose on the same network, use the service name (e.g. `backend:3000`). - - If Prometheus runs outside the backend's network, use a reachable host/IP (e.g. `host.docker.internal:3000` on Docker Desktop, or the backend's public address). - -3. If the metrics endpoint requires authentication, add credentials to the scrape config: - - ```yaml - scrape_configs: - - job_name: "backend" - metrics_path: /metrics - scheme: https - basic_auth: - username: metrics - password_file: /etc/prometheus/metrics_password - static_configs: - - targets: ["backend.example.com:443"] - ``` - - Mount the password file into the Prometheus container and keep it out of version control. - -## 2. Deploy Prometheus and Grafana - -### Option A: Docker Compose - -The repository ships a Compose file at [`monitoring/docker-compose.yml`](../monitoring/docker-compose.yml) that starts Prometheus, Alertmanager, and Grafana together. - -1. Review and adjust the Compose file and the referenced config files: - - [`monitoring/prometheus.yml`](../monitoring/prometheus.yml) — scrape targets and rule file references. - - [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) — alerting rules. - - [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml) — notification routing. - -2. Start the stack: - - ```bash - cd monitoring - docker compose up -d - ``` - -3. Confirm the containers are running: - - ```bash - docker compose ps - ``` - -4. Access the UIs: - - Prometheus: - - Alertmanager: - - Grafana: (default credentials `admin` / `admin` — change on first login) - -5. To stop the stack: - - ```bash - docker compose down - ``` - -### Option B: Managed services - -If you prefer managed infrastructure, use a hosted Prometheus (e.g. Grafana Cloud, Amazon Managed Service for Prometheus, Google Managed Prometheus) and a hosted Grafana. +- A Kubernetes cluster with `kubectl` configured +- [Helm 3](https://helm.sh/docs/intro/install/) -1. **Prometheus / metrics backend** - - Create the managed Prometheus instance and note its remote-write endpoint and credentials. - - Configure your backend (or a local Prometheus agent) to remote-write metrics to the managed endpoint. If you run a local Prometheus agent, reuse [`monitoring/prometheus.yml`](../monitoring/prometheus.yml) for scrape configs and add a `remote_write` block: +## Install kube-prometheus-stack - ```yaml - remote_write: - - url: https:///api/v1/write - basic_auth: - username: - password: - ``` - -2. **Grafana** - - Create the hosted Grafana instance. - - Add a data source pointing at the managed Prometheus instance (see [step 3](#3-import-the-grafana-dashboard)). - -3. **Alerting** - - Managed Prometheus services typically support alert rules and Alertmanager-compatible routing. Load the rules from [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml) into the managed rule store, and configure notification channels in the provider's UI (see [step 5](#5-configure-alertmanager-notification-channels)). - -## 3. Import the Grafana dashboard - -1. Log in to Grafana. -2. Add the Prometheus data source: - - Go to **Connections → Data sources → Add data source → Prometheus**. - - Set the URL to your Prometheus instance (e.g. `http://prometheus:9090` when running in Compose, or the managed endpoint URL). - - Click **Save & test** and confirm the connection succeeds. -3. Import the dashboard: - - Go to **Dashboards → New → Import**. - - Upload the dashboard JSON from [`monitoring/grafana/`](../monitoring/grafana) (or paste its contents). - - Select the Prometheus data source created above. - - Click **Import**. -4. The dashboard should now render panels populated by the backend metrics. - -> If the dashboard JSON references a data source by name, ensure the name matches the one you created, or update the variable when importing. - -## 4. Load the alert rules - -Alert rules are defined in [`monitoring/alert-rules.yml`](../monitoring/alert-rules.yml). - -1. Ensure Prometheus references the rule file. In [`monitoring/prometheus.yml`](../monitoring/prometheus.yml): - - ```yaml - rule_files: - - /etc/prometheus/alert-rules.yml - ``` - - Confirm the Compose file mounts `alert-rules.yml` to that path inside the Prometheus container. - -2. Reload Prometheus to pick up rule changes: - - Via the UI: **Status → Configuration → Reload** (requires `--web.enable-lifecycle`), or - - Via the API: - - ```bash - curl -X POST http://localhost:9090/-/reload - ``` - - - Or restart the container: - - ```bash - docker compose restart prometheus - ``` - -3. Verify the rules loaded: go to **Prometheus → Status → Rules** and confirm the rules from `alert-rules.yml` appear. - -## 5. Configure Alertmanager notification channels - -Alertmanager routing and receivers are configured in [`monitoring/alertmanager.yml`](../monitoring/alertmanager.yml). The examples below show how to add email and Slack receivers. Replace placeholders with your real values and **do not commit secrets** — use environment variable substitution or a secrets manager. - -### Email - -Add an email receiver and route alerts to it: - -```yaml -global: - smtp_smarthost: "smtp.example.com:587" - smtp_from: "alerts@example.com" - smtp_auth_username: "alerts@example.com" - smtp_auth_password: "" # use a secret, not a literal - smtp_require_tls: true - -route: - receiver: "email" - group_by: ["alertname"] - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - -receivers: - - name: "email" - email_configs: - - to: "oncall@example.com" - send_resolved: true -``` - -### Slack - -Add a Slack receiver using an incoming webhook URL: +```bash +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update -```yaml -receivers: - - name: "slack" - slack_configs: - - api_url: "https://hooks.slack.com/services///" - channel: "#alerts" - send_resolved: true - title: "{{ .CommonLabels.alertname }}" - text: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}" +helm upgrade --install monitoring prometheus-community/kube-prometheus-stack \ + --namespace monitoring --create-namespace \ + --set grafana.adminPassword="$GRAFANA_ADMIN_PASSWORD" ``` -To route different severities to different channels, use multiple routes: +Wait for the operator and Prometheus to become ready: -```yaml -route: - receiver: "slack" - group_by: ["alertname"] - routes: - - matchers: - - severity = "critical" - receiver: "slack" - continue: false - - matchers: - - severity = "warning" - receiver: "email" +```bash +kubectl -n monitoring rollout status deploy/monitoring-kube-prometheus-operator +kubectl -n monitoring rollout status statefulset/prometheus-monitoring-kube-prometheus-prometheus ``` -After editing, reload Alertmanager: +## Apply the project monitoring resources ```bash -curl -X POST http://localhost:9093/-/reload +kubectl apply -f monitoring/servicemonitors.yaml +kubectl apply -f monitoring/prometheusrule.yaml ``` -or restart the container: +## Verify -```bash -docker compose restart alertmanager -``` +- Prometheus targets: port-forward the Prometheus service and open + **Status → Targets**; the `backend` and `frontend` jobs should be `UP`. + + ```bash + kubectl -n monitoring port-forward svc/monitoring-kube-prometheus-prometheus 9090 + ``` -## 6. Verify the setup +- Alert rules: open **Status → Rules** and confirm the project alert group is + loaded. +- Grafana: port-forward the Grafana service and log in with the admin password. -1. **Metrics scraping** — In Prometheus, go to **Status → Targets** and confirm the `backend` target is `UP`. -2. **Rules** — In Prometheus, go to **Status → Rules** and confirm the alert rules are loaded. -3. **Alerts** — In Alertmanager (), confirm it is reachable and shows any firing alerts. -4. **Notifications** — Trigger a test alert (or use Alertmanager's built-in test) and confirm it arrives in the configured email/Slack channel. -5. **Dashboard** — Open the imported Grafana dashboard and confirm panels show live data. + ```bash + kubectl -n monitoring port-forward svc/monitoring-grafana 3000:80 + ``` -## Troubleshooting +## Notes -- **Target is `DOWN`**: verify the metrics endpoint is reachable from the Prometheus container (`docker compose exec prometheus wget -qO- http://backend:3000/metrics`). Check host/port and network configuration. -- **No data in Grafana**: confirm the data source URL is correct and that Prometheus has scraped data (**Prometheus → Graph**, query `up`). -- **Rules not loading**: check the `rule_files` path and that the file is mounted into the container. Inspect Prometheus logs: `docker compose logs prometheus`. -- **Alerts not delivered**: check Alertmanager logs (`docker compose logs alertmanager`) and confirm the receiver config, webhook URL, and SMTP credentials are correct. -- **Config errors**: validate configs before reloading: - - Prometheus: `promtool check config monitoring/prometheus.yml` - - Alertmanager: `amtool check-config monitoring/alertmanager.yml` \ No newline at end of file +- The `ServiceMonitor` resources select the backend and frontend `Service`s by + their labels; keep those labels in sync with `k8s/` manifests. +- The `PrometheusRule` mirrors `monitoring/alert-rules.yml`. When you change the + alert rules, update both files (or regenerate the `PrometheusRule` from the + YAML) so Kubernetes and docker-compose stay consistent. diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index b149c3f..fe5e904 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -13,6 +13,9 @@ resources: - backend-service.yaml - frontend-deployment.yaml - ingress.yaml + # Monitoring stack (kube-prometheus-stack CRs). See docs/DEPLOY-KUBERNETES.md. + - monitoring/servicemonitors.yaml + - monitoring/prometheusrule.yaml labels: - pairs: diff --git a/k8s/monitoring-servicemonitors.yaml b/k8s/monitoring-servicemonitors.yaml new file mode 100644 index 0000000..9602c6a --- /dev/null +++ b/k8s/monitoring-servicemonitors.yaml @@ -0,0 +1,54 @@ +# Monitoring stack for Kubernetes (kube-prometheus-stack). +# +# Prerequisites: install kube-prometheus-stack (Prometheus Operator) into the +# cluster, e.g.: +# +# helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +# helm repo update +# helm install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ +# --namespace monitoring --create-namespace +# +# Then apply this file (and the PrometheusRule in monitoring-prometheusrule.yaml): +# +# kubectl apply -f k8s/monitoring-servicemonitors.yaml +# kubectl apply -f k8s/monitoring-prometheusrule.yaml +# +# The ServiceMonitors below replace the static docker-compose scrape targets in +# monitoring/prometheus.yml by letting the Prometheus Operator discover the +# backend and frontend Services via label selectors. +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: backend + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: backend + namespaceSelector: + matchNames: + - default + endpoints: + - port: http + path: /metrics + interval: 15s +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: frontend + labels: + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app: frontend + namespaceSelector: + matchNames: + - default + endpoints: + - port: http + path: /metrics + interval: 15s diff --git a/k8s/prometheus-rule.yaml b/k8s/prometheus-rule.yaml new file mode 100644 index 0000000..999700b --- /dev/null +++ b/k8s/prometheus-rule.yaml @@ -0,0 +1,48 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: app-alert-rules + labels: + release: kube-prometheus-stack +spec: + groups: + - name: app.rules + rules: + - alert: BackendDown + expr: up{job="backend"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Backend target is down" + description: "Prometheus has been unable to scrape the backend for more than 5 minutes." + - alert: FrontendDown + expr: up{job="frontend"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Frontend target is down" + description: "Prometheus has been unable to scrape the frontend for more than 5 minutes." + - alert: HighErrorRate + expr: | + sum(rate(http_requests_total{status=~"5.."}[5m])) + / + sum(rate(http_requests_total[5m])) > 0.05 + for: 10m + labels: + severity: critical + annotations: + summary: "High HTTP 5xx error rate" + description: "More than 5% of requests have returned 5xx over the last 10 minutes." + - alert: HighRequestLatency + expr: | + histogram_quantile(0.95, + sum(rate(http_request_duration_seconds_bucket[5m])) by (le) + ) > 1 + for: 10m + labels: + severity: warning + annotations: + summary: "High request latency (p95 > 1s)" + description: "The 95th percentile request latency has exceeded 1s for 10 minutes."