From b52220ea1d6be65af0520a1006f2e9c3937fe996 Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Sat, 17 Oct 2026 16:29:04 +0100 Subject: [PATCH] feat(monitoring): SLO alert rules, alertmanager routing, and metrics for alerts (#1622) - Prometheus rules: API 5xx rate, p95 latency, DB pool utilisation, RPC probe failure rate - Alertmanager: PagerDuty for critical, webhook for warnings; secrets read from files - Runbook per alert, linked from alert annotations - Backend: cumulative request-duration histogram, probe counters, DB pool gauge --- backend/src/db/deepReadiness.ts | 3 + backend/src/db/healthMonitor.ts | 2 + backend/src/metrics/MetricsCollector.ts | 71 +++++++++++++++++ backend/src/metrics/MetricsExporter.ts | 66 +++++++++++++++- backend/src/metrics/dbPoolMetrics.ts | 23 ++++++ backend/tests/metricsSlo.test.ts | 101 ++++++++++++++++++++++++ monitoring/alertmanager/config.yml | 44 +++++++++++ monitoring/prometheus/alert.rules.yml | 85 ++++++++++++++++++++ monitoring/runbooks.md | 39 +++++++++ 9 files changed, 432 insertions(+), 2 deletions(-) create mode 100644 backend/src/metrics/dbPoolMetrics.ts create mode 100644 backend/tests/metricsSlo.test.ts create mode 100644 monitoring/alertmanager/config.yml create mode 100644 monitoring/prometheus/alert.rules.yml create mode 100644 monitoring/runbooks.md diff --git a/backend/src/db/deepReadiness.ts b/backend/src/db/deepReadiness.ts index 425bde661..f26b91232 100644 --- a/backend/src/db/deepReadiness.ts +++ b/backend/src/db/deepReadiness.ts @@ -19,6 +19,7 @@ import { Horizon } from '@stellar/stellar-sdk'; import prisma from '../db/index.js'; import redisClient from '../cache/RedisClient.js'; import { HORIZON_URL, SOROBAN_RPC_URL } from '../config/rpcConfig.js'; +import metricsCollector from '../metrics/MetricsCollector.js'; import logger from '../utils/logger.js'; const DEFAULT_PROBE_TIMEOUT_MS = 3000; @@ -128,12 +129,14 @@ async function probeSorobanRpc(timeout: number): Promise { timeout ); const latest = ledgers.records?.[0]?.sequence ?? null; + metricsCollector.recordProbe('horizon_ledger', latest != null); return { status: latest != null ? 'ready' : 'unavailable', latencyMs: Date.now() - start, detail: { latestLedgerHeight: latest }, }; } catch (error) { + metricsCollector.recordProbe('horizon_ledger', false); logger.error('Deep readiness: Soroban/Horizon probe failed', error); return { status: 'unavailable', diff --git a/backend/src/db/healthMonitor.ts b/backend/src/db/healthMonitor.ts index c5199e618..f70c2229c 100644 --- a/backend/src/db/healthMonitor.ts +++ b/backend/src/db/healthMonitor.ts @@ -1,5 +1,6 @@ import prisma from '../db/index.js'; import logger from '../utils/logger.js'; +import { setDbPoolUtilizationPct } from '../metrics/dbPoolMetrics.js'; export interface DbHealthStatus { status: 'healthy' | 'degraded' | 'unhealthy'; @@ -60,6 +61,7 @@ export async function checkDbHealth(): Promise { total: poolMax, utilizationPct, }; + setDbPoolUtilizationPct(utilizationPct); if (latencyMs > LATENCY_CRITICAL_MS) { alerts.push(`Critical: DB query latency ${latencyMs}ms exceeds ${LATENCY_CRITICAL_MS}ms`); diff --git a/backend/src/metrics/MetricsCollector.ts b/backend/src/metrics/MetricsCollector.ts index e2c59ceb4..18efb400f 100644 --- a/backend/src/metrics/MetricsCollector.ts +++ b/backend/src/metrics/MetricsCollector.ts @@ -17,6 +17,27 @@ import logger from '../utils/logger.js'; +// ─── Histogram + probe types ────────────────────────────────────────────────── + +/** + * Upper bounds (ms) for the cumulative request-duration histogram. Chosen so + * the 1500 ms SLO threshold is an exact bucket boundary, which keeps the p95 + * alert accurate. + */ +export const DURATION_BUCKETS_MS: readonly number[] = [50, 100, 250, 500, 1000, 1500, 2500, 5000]; + +export interface DurationHistogramSnapshot { + /** Cumulative counts: each bucket counts requests with duration <= le. */ + buckets: Array<{ le: number; count: number }>; + count: number; + sumMs: number; +} + +export interface ProbeCounts { + ok: number; + failed: number; +} + // ─── Types ──────────────────────────────────────────────────────────────────── export interface PerformanceMetric { @@ -86,6 +107,13 @@ export class MetricsCollector { private errorMetrics: ErrorMetric[] = []; private businessMetrics: BusinessMetric[] = []; + // Cumulative counters that survive ring-buffer trimming, so histogram and + // probe rates stay correct over long-running processes. + private durationBucketCounts: number[] = DURATION_BUCKETS_MS.map(() => 0); + private durationCount = 0; + private durationSumMs = 0; + private probeCounts = new Map(); + /** * Maximum number of raw metric entries kept in memory per category. * Older entries are dropped (ring-buffer style) to bound memory usage. @@ -119,6 +147,14 @@ export class MetricsCollector { this.performanceMetrics.push(metric); this.trim(this.performanceMetrics); + if (Number.isFinite(durationMs) && durationMs >= 0) { + this.durationCount += 1; + this.durationSumMs += durationMs; + DURATION_BUCKETS_MS.forEach((le, i) => { + if (durationMs <= le) this.durationBucketCounts[i] = (this.durationBucketCounts[i] ?? 0) + 1; + }); + } + // Log slow requests (>1 s) so they surface in existing log pipelines. if (durationMs > 1000) { logger.warn(`Slow request: ${method} ${route} took ${durationMs}ms (status ${statusCode})`); @@ -259,10 +295,45 @@ export class MetricsCollector { /** * Reset all collected metrics (useful in tests or after a scheduled flush). */ + /** + * Cumulative request-duration histogram since process start (or last reset). + */ + getDurationHistogram(): DurationHistogramSnapshot { + return { + buckets: DURATION_BUCKETS_MS.map((le, i) => ({ le, count: this.durationBucketCounts[i] ?? 0 })), + count: this.durationCount, + sumMs: this.durationSumMs, + }; + } + + /** + * Record the outcome of a periodic dependency probe (e.g. the deep readiness + * ledger check). `probe` is a low-cardinality label such as "horizon_ledger". + */ + recordProbe(probe: string, ok: boolean): void { + const counts = this.probeCounts.get(probe) ?? { ok: 0, failed: 0 }; + if (ok) counts.ok += 1; + else counts.failed += 1; + this.probeCounts.set(probe, counts); + } + + /** Cumulative probe outcomes keyed by probe name. */ + getProbeCounts(): Record { + const out: Record = {}; + this.probeCounts.forEach((value, key) => { + out[key] = { ok: value.ok, failed: value.failed }; + }); + return out; + } + reset(): void { this.performanceMetrics = []; this.errorMetrics = []; this.businessMetrics = []; + this.durationBucketCounts = DURATION_BUCKETS_MS.map(() => 0); + this.durationCount = 0; + this.durationSumMs = 0; + this.probeCounts = new Map(); logger.info('[Metrics] All metrics reset'); } diff --git a/backend/src/metrics/MetricsExporter.ts b/backend/src/metrics/MetricsExporter.ts index 12f93dbb5..16d5f5fbe 100644 --- a/backend/src/metrics/MetricsExporter.ts +++ b/backend/src/metrics/MetricsExporter.ts @@ -11,10 +11,15 @@ * stays bounded. */ -import metricsCollector, { type MetricsSummary } from './MetricsCollector.js'; +import metricsCollector, { + type DurationHistogramSnapshot, + type MetricsSummary, + type ProbeCounts, +} from './MetricsCollector.js'; import workerRegistry, { type WorkerStatus } from './WorkerRegistry.js'; import cacheService from '../cache/CacheService.js'; import redisClient from '../cache/RedisClient.js'; +import { getDbPoolUtilizationPct } from './dbPoolMetrics.js'; /** Snapshot schema version. Bump on a breaking field change. */ export const METRICS_SCHEMA_VERSION = '1'; @@ -56,6 +61,12 @@ export interface MetricsSnapshot { business: { eventsTotal: number; eventsByName: Record }; workers: WorkerStatus[]; system: { uptimeSeconds: number; memoryResidentBytes: number; cpuUserSeconds: number }; + /** Cumulative request-duration histogram (optional for hand-built snapshots). */ + durationHistogram?: DurationHistogramSnapshot; + /** Dependency probe outcomes keyed by probe name. */ + probes?: Record; + /** Last recorded DB pool utilisation (0-100), or null if never measured. */ + dbPoolUtilizationPct?: number | null; } /** Segments that look like identifiers rather than route names. */ @@ -147,9 +158,28 @@ export function buildMetricsSnapshot( memoryResidentBytes: Math.round(summary.system.memoryUsageMB * 1024 * 1024), cpuUserSeconds: Number((summary.system.cpuUserMs / 1000).toFixed(3)), }, + durationHistogram: metricsCollector.getDurationHistogram(), + probes: metricsCollector.getProbeCounts(), + dbPoolUtilizationPct: getDbPoolUtilizationPct(), }; } +/** + * Render a Prometheus histogram (cumulative `_bucket` lines plus `_sum` and + * `_count`). The generic renderFamily helper only supports counters and + * gauges, so histograms are rendered here. + */ +function renderHistogram(name: string, help: string, h: DurationHistogramSnapshot): string { + const lines = [`# HELP ${name} ${help}`, `# TYPE ${name} histogram`]; + for (const bucket of h.buckets) { + lines.push(`${name}_bucket{le="${bucket.le}"} ${bucket.count}`); + } + lines.push(`${name}_bucket{le="+Inf"} ${h.count}`); + lines.push(`${name}_sum ${h.sumMs}`); + lines.push(`${name}_count ${h.count}`); + return lines.join('\n'); +} + // ─── Prometheus text exposition ─────────────────────────────────────────────── /** Escape a label value per the Prometheus exposition format. */ @@ -304,7 +334,39 @@ export function renderPrometheus(snapshot: MetricsSnapshot = buildMetricsSnapsho }, ]; - return `${families.map(renderFamily).join('\n')}\n`; + if (snapshot.dbPoolUtilizationPct !== undefined && snapshot.dbPoolUtilizationPct !== null) { + families.push({ + name: `${PREFIX}_db_pool_utilization_percent`, + help: 'Database connection pool utilisation from the last health check, 0-100.', + type: 'gauge', + samples: [{ value: snapshot.dbPoolUtilizationPct }], + }); + } + + if (snapshot.probes && Object.keys(snapshot.probes).length > 0) { + families.push({ + name: `${PREFIX}_external_probe_total`, + help: 'Dependency probe outcomes since process start, by probe and outcome.', + type: 'counter', + samples: Object.entries(snapshot.probes).flatMap(([probe, counts]) => [ + { labels: { probe, outcome: 'ok' }, value: counts.ok }, + { labels: { probe, outcome: 'failed' }, value: counts.failed }, + ]), + }); + } + + const rendered = families.map(renderFamily); + if (snapshot.durationHistogram) { + rendered.push( + renderHistogram( + `${PREFIX}_http_request_duration_milliseconds`, + 'HTTP request duration histogram in milliseconds, cumulative since process start.', + snapshot.durationHistogram + ) + ); + } + + return `${rendered.join('\n')}\n`; } /** Content type expected by Prometheus-compatible scrapers. */ diff --git a/backend/src/metrics/dbPoolMetrics.ts b/backend/src/metrics/dbPoolMetrics.ts new file mode 100644 index 000000000..ef4c96b54 --- /dev/null +++ b/backend/src/metrics/dbPoolMetrics.ts @@ -0,0 +1,23 @@ +/** + * Last-known database connection pool utilisation, written by the DB health + * check and read by the Prometheus exporter. Kept in its own module so the + * exporter does not import the health check (avoids a circular dependency). + */ + +let poolUtilizationPct: number | null = null; + +/** Record the latest pool utilisation (0-100). Non-finite values are ignored. */ +export function setDbPoolUtilizationPct(value: number): void { + if (!Number.isFinite(value)) return; + poolUtilizationPct = Math.min(100, Math.max(0, value)); +} + +/** Latest recorded pool utilisation, or null if no health check has run yet. */ +export function getDbPoolUtilizationPct(): number | null { + return poolUtilizationPct; +} + +/** Test helper: forget the recorded value. */ +export function resetDbPoolUtilizationPct(): void { + poolUtilizationPct = null; +} diff --git a/backend/tests/metricsSlo.test.ts b/backend/tests/metricsSlo.test.ts new file mode 100644 index 000000000..aaf9ae9b0 --- /dev/null +++ b/backend/tests/metricsSlo.test.ts @@ -0,0 +1,101 @@ +/** + * SLO metric exposure: request-duration histogram (for p95 alerts), dependency + * probe counters (for the RPC failure-rate alert) and DB pool utilisation + * (for the pool alert). These feed monitoring/prometheus/alert.rules.yml. + */ + +import metricsCollector, { DURATION_BUCKETS_MS } from '../src/metrics/MetricsCollector.js'; +import { renderPrometheus } from '../src/metrics/MetricsExporter.js'; +import { + getDbPoolUtilizationPct, + resetDbPoolUtilizationPct, + setDbPoolUtilizationPct, +} from '../src/metrics/dbPoolMetrics.js'; + +describe('request duration histogram', () => { + beforeEach(() => { + metricsCollector.reset(); + }); + + it('counts requests cumulatively per bucket upper bound', () => { + metricsCollector.recordRequest('GET', '/a', 40, 200); // <= 50 + metricsCollector.recordRequest('GET', '/a', 600, 200); // <= 1000 + metricsCollector.recordRequest('GET', '/a', 1600, 200); // <= 2500 + + const h = metricsCollector.getDurationHistogram(); + const byLe = Object.fromEntries(h.buckets.map((b) => [b.le, b.count])); + + expect(h.count).toBe(3); + expect(h.sumMs).toBe(2240); + expect(byLe[50]).toBe(1); + expect(byLe[500]).toBe(1); + expect(byLe[1000]).toBe(2); + expect(byLe[1500]).toBe(2); + expect(byLe[2500]).toBe(3); + expect(h.buckets.map((b) => b.le)).toEqual([...DURATION_BUCKETS_MS]); + }); + + it('keeps the 1500 ms SLO threshold as an exact bucket boundary', () => { + expect(DURATION_BUCKETS_MS).toContain(1500); + }); + + it('renders histogram lines with +Inf, _sum and _count', () => { + metricsCollector.recordRequest('GET', '/a', 120, 200); + + const text = renderPrometheus(); + + expect(text).toContain('# TYPE w3sl_http_request_duration_milliseconds histogram'); + expect(text).toContain('w3sl_http_request_duration_milliseconds_bucket{le="250"} 1'); + expect(text).toContain('w3sl_http_request_duration_milliseconds_bucket{le="+Inf"} 1'); + expect(text).toContain('w3sl_http_request_duration_milliseconds_sum 120'); + expect(text).toContain('w3sl_http_request_duration_milliseconds_count 1'); + }); + + it('is cleared by reset()', () => { + metricsCollector.recordRequest('GET', '/a', 10, 200); + metricsCollector.reset(); + expect(metricsCollector.getDurationHistogram().count).toBe(0); + }); +}); + +describe('dependency probe counters', () => { + beforeEach(() => { + metricsCollector.reset(); + }); + + it('counts ok and failed probe outcomes per probe', () => { + metricsCollector.recordProbe('horizon_ledger', true); + metricsCollector.recordProbe('horizon_ledger', true); + metricsCollector.recordProbe('horizon_ledger', false); + + expect(metricsCollector.getProbeCounts()).toEqual({ + horizon_ledger: { ok: 2, failed: 1 }, + }); + + const text = renderPrometheus(); + expect(text).toContain('w3sl_external_probe_total{probe="horizon_ledger",outcome="ok"} 2'); + expect(text).toContain('w3sl_external_probe_total{probe="horizon_ledger",outcome="failed"} 1'); + }); +}); + +describe('DB pool utilisation gauge', () => { + afterEach(() => { + resetDbPoolUtilizationPct(); + }); + + it('is omitted until a health check has recorded a value', () => { + expect(getDbPoolUtilizationPct()).toBeNull(); + expect(renderPrometheus()).not.toContain('w3sl_db_pool_utilization_percent'); + }); + + it('renders the last recorded utilisation and clamps to 0-100', () => { + setDbPoolUtilizationPct(87); + expect(renderPrometheus()).toContain('w3sl_db_pool_utilization_percent 87'); + + setDbPoolUtilizationPct(250); + expect(getDbPoolUtilizationPct()).toBe(100); + + setDbPoolUtilizationPct(Number.NaN); + expect(getDbPoolUtilizationPct()).toBe(100); + }); +}); diff --git a/monitoring/alertmanager/config.yml b/monitoring/alertmanager/config.yml new file mode 100644 index 000000000..325fb140a --- /dev/null +++ b/monitoring/alertmanager/config.yml @@ -0,0 +1,44 @@ +# Alertmanager routing for Web3 Student Lab (BE-050, #1622). +# +# Secrets are read from files so they are never committed. Mount them into the +# Alertmanager container at /etc/alertmanager/secrets/ (for example from a +# Kubernetes Secret). Requires Alertmanager >= 0.27 for *_file options. +# +# Critical alerts page PagerDuty; warnings go to a webhook (for example a +# Discord or Slack relay). Both receivers get a resolved notification. + +global: + resolve_timeout: 5m + +route: + receiver: warnings-webhook + group_by: ["alertname", "service"] + group_wait: 10s + group_interval: 1m + repeat_interval: 4h + routes: + - matchers: + - severity="critical" + receiver: pagerduty-critical + group_wait: 10s + repeat_interval: 1h + - matchers: + - severity="warning" + receiver: warnings-webhook + +receivers: + - name: pagerduty-critical + pagerduty_configs: + - routing_key_file: /etc/alertmanager/secrets/pagerduty_routing_key + severity: "{{ .CommonLabels.severity }}" + description: "{{ .CommonAnnotations.summary }}" + send_resolved: true + details: + runbook: "{{ .CommonAnnotations.runbook_url }}" + diagnostic_query: "{{ .CommonAnnotations.diagnostic_query }}" + + - name: warnings-webhook + webhook_configs: + - url_file: /etc/alertmanager/secrets/warning_webhook_url + send_resolved: true + max_alerts: 20 diff --git a/monitoring/prometheus/alert.rules.yml b/monitoring/prometheus/alert.rules.yml new file mode 100644 index 000000000..f1315def4 --- /dev/null +++ b/monitoring/prometheus/alert.rules.yml @@ -0,0 +1,85 @@ +# Prometheus alerting rules for Web3 Student Lab SLOs (BE-050, #1622). +# +# Metric sources (backend/src/metrics): +# w3sl_http_responses_total{status_class} MetricsExporter +# w3sl_http_request_duration_milliseconds_bucket{le} MetricsCollector histogram +# w3sl_db_pool_utilization_percent dbPoolMetrics (set by DB health check) +# w3sl_external_probe_total{probe,outcome} deepReadiness probe outcomes +# +# Noise control: each ratio alert also requires a minimum request rate so a +# quiet service does not page on one failed request, and a `for:` duration so a +# single bad evaluation does not fire. Evaluation interval is 15s, so an alert +# fires roughly 45-60s after a sustained breach. + +groups: + - name: web3-student-lab-slo + interval: 15s + rules: + - alert: ApiErrorRateHigh + expr: | + ( + sum(rate(w3sl_http_responses_total{status_class="5xx"}[5m])) + / + clamp_min(sum(rate(w3sl_http_responses_total[5m])), 1e-9) + ) > 0.01 + and sum(rate(w3sl_http_responses_total[5m])) > 0.1 + for: 30s + labels: + severity: critical + service: backend + slo: api-error-rate + annotations: + summary: "API 5xx error rate above 1% for 30s" + description: "5xx responses are {{ $value | humanizePercentage }} of all responses over the last 5m." + runbook_url: "https://github.com/StellarDevHub/Web3-Student-Lab/blob/main/monitoring/runbooks.md#apierrorratehigh" + diagnostic_query: 'sum by (status_class) (rate(w3sl_http_responses_total[5m]))' + + - alert: ApiP95LatencyHigh + expr: | + histogram_quantile( + 0.95, + sum by (le) (rate(w3sl_http_request_duration_milliseconds_bucket[5m])) + ) > 1500 + and sum(rate(w3sl_http_request_duration_milliseconds_count[5m])) > 0.1 + for: 1m + labels: + severity: warning + service: backend + slo: api-p95-latency + annotations: + summary: "API p95 response time above 1500 ms" + description: "p95 latency is {{ $value | humanize }} ms over the last 5m." + runbook_url: "https://github.com/StellarDevHub/Web3-Student-Lab/blob/main/monitoring/runbooks.md#apip95latencyhigh" + diagnostic_query: 'histogram_quantile(0.95, sum by (le) (rate(w3sl_http_request_duration_milliseconds_bucket[5m])))' + + - alert: DbPoolUtilizationHigh + expr: w3sl_db_pool_utilization_percent > 85 + for: 30s + labels: + severity: critical + service: database + slo: db-pool-utilization + annotations: + summary: "Database connection pool above 85% utilisation" + description: "Pool utilisation is {{ $value }}% of DB_POOL_MAX (last health check)." + runbook_url: "https://github.com/StellarDevHub/Web3-Student-Lab/blob/main/monitoring/runbooks.md#dbpoolutilizationhigh" + diagnostic_query: 'w3sl_db_pool_utilization_percent' + + - alert: RpcProbeFailureRateHigh + expr: | + ( + sum(rate(w3sl_external_probe_total{probe="horizon_ledger",outcome="failed"}[5m])) + / + clamp_min(sum(rate(w3sl_external_probe_total{probe="horizon_ledger"}[5m])), 1e-9) + ) > 0.05 + and sum(rate(w3sl_external_probe_total{probe="horizon_ledger"}[5m])) > 0 + for: 1m + labels: + severity: warning + service: stellar-rpc + slo: rpc-failure-rate + annotations: + summary: "Stellar RPC readiness probe failing more than 5% of the time" + description: "Probe failure ratio is {{ $value | humanizePercentage }} over the last 5m." + runbook_url: "https://github.com/StellarDevHub/Web3-Student-Lab/blob/main/monitoring/runbooks.md#rpcprobefailureratehigh" + diagnostic_query: 'sum by (outcome) (rate(w3sl_external_probe_total{probe="horizon_ledger"}[5m]))' diff --git a/monitoring/runbooks.md b/monitoring/runbooks.md new file mode 100644 index 000000000..fec47e2dd --- /dev/null +++ b/monitoring/runbooks.md @@ -0,0 +1,39 @@ +# Monitoring runbooks + +One section per alert in `prometheus/alert.rules.yml`. Each alert's +`runbook_url` links to its heading here. + +## ApiErrorRateHigh + +5xx responses exceed 1% of all responses for 30 seconds (with at least 0.1 req/s traffic). + +1. Check the 5xx breakdown: `sum by (status_class) (rate(w3sl_http_responses_total[5m]))`. +2. Check worker and error types: `GET /api/v1/metrics/prometheus` and review `w3sl_errors_total` by `type`. +3. Check backend logs for `Unhandled error` entries and recent deploys. +4. If a dependency is down (see DB and RPC alerts), resolve that first. + +## ApiP95LatencyHigh + +p95 request duration exceeds 1500 ms over 5 minutes. + +1. Confirm with `histogram_quantile(0.95, sum by (le) (rate(w3sl_http_request_duration_milliseconds_bucket[5m])))`. +2. Compare with `w3sl_db_pool_utilization_percent`. Pool saturation is the usual cause. +3. Check slow routes in logs (requests over 1 s are logged as `Slow request`). + +## DbPoolUtilizationHigh + +Database pool utilisation is above 85% of `DB_POOL_MAX`. + +1. The value updates on each DB health check, so confirm checks are running. +2. Look for long-running queries: `SELECT pid, state, now() - query_start AS age, left(query, 120) FROM pg_stat_activity WHERE datname = current_database() ORDER BY age DESC;` +3. Check for a traffic burst, and raise `DB_POOL_MAX` only after confirming the database has headroom. + +## RpcProbeFailureRateHigh + +The deep-readiness ledger probe (`probe="horizon_ledger"`) fails more than 5% of the time. + +Note: this probe currently calls Horizon's ledgers endpoint, not Soroban RPC. + +1. Check the Horizon status page and `HORIZON_URL` in the environment. +2. Check outbound connectivity from the backend pod. +3. Check the probe failure count: `sum by (outcome) (rate(w3sl_external_probe_total{probe="horizon_ledger"}[5m]))`.