Close #1622, #1624, #1625, #1626: SLO alerts, SBT certificates, DAO timelock, quadratic funding hardening

backend/src/db/deepReadiness.ts

@@ -19,6 +19,7 @@ import { Horizon } from '@stellar/stellar-sdk';
 import prisma from '../db/index.js';
 import redisClient from '../cache/RedisClient.js';
 import { HORIZON_URL, SOROBAN_RPC_URL } from '../config/rpcConfig.js';
+import metricsCollector from '../metrics/MetricsCollector.js';
 import logger from '../utils/logger.js';
 
 const DEFAULT_PROBE_TIMEOUT_MS = 3000;
@@ -128,12 +129,14 @@ async function probeSorobanRpc(timeout: number): Promise<DeepProbeResult> {
       timeout
     );
     const latest = ledgers.records?.[0]?.sequence ?? null;
+    metricsCollector.recordProbe('horizon_ledger', latest != null);
     return {
       status: latest != null ? 'ready' : 'unavailable',
       latencyMs: Date.now() - start,
       detail: { latestLedgerHeight: latest },
     };
   } catch (error) {
+    metricsCollector.recordProbe('horizon_ledger', false);
     logger.error('Deep readiness: Soroban/Horizon probe failed', error);
     return {
       status: 'unavailable',

backend/src/db/healthMonitor.ts

@@ -1,5 +1,6 @@
 import prisma from '../db/index.js';
 import logger from '../utils/logger.js';
+import { setDbPoolUtilizationPct } from '../metrics/dbPoolMetrics.js';
 
 export interface DbHealthStatus {
   status: 'healthy' | 'degraded' | 'unhealthy';
@@ -60,6 +61,7 @@ export async function checkDbHealth(): Promise<DbHealthStatus> {
       total: poolMax,
       utilizationPct,
     };
+    setDbPoolUtilizationPct(utilizationPct);
 
     if (latencyMs > LATENCY_CRITICAL_MS) {
       alerts.push(`Critical: DB query latency ${latencyMs}ms exceeds ${LATENCY_CRITICAL_MS}ms`);

backend/src/metrics/MetricsCollector.ts

@@ -17,6 +17,27 @@
 
 import logger from '../utils/logger.js';
 
+// ─── Histogram + probe types ──────────────────────────────────────────────────
+
+/**
+ * Upper bounds (ms) for the cumulative request-duration histogram. Chosen so
+ * the 1500 ms SLO threshold is an exact bucket boundary, which keeps the p95
+ * alert accurate.
+ */
+export const DURATION_BUCKETS_MS: readonly number[] = [50, 100, 250, 500, 1000, 1500, 2500, 5000];
+
+export interface DurationHistogramSnapshot {
+  /** Cumulative counts: each bucket counts requests with duration <= le. */
+  buckets: Array<{ le: number; count: number }>;
+  count: number;
+  sumMs: number;
+}
+
+export interface ProbeCounts {
+  ok: number;
+  failed: number;
+}
+
 // ─── Types ────────────────────────────────────────────────────────────────────
 
 export interface PerformanceMetric {
@@ -86,6 +107,13 @@ export class MetricsCollector {
   private errorMetrics: ErrorMetric[] = [];
   private businessMetrics: BusinessMetric[] = [];
 
+  // Cumulative counters that survive ring-buffer trimming, so histogram and
+  // probe rates stay correct over long-running processes.
+  private durationBucketCounts: number[] = DURATION_BUCKETS_MS.map(() => 0);
+  private durationCount = 0;
+  private durationSumMs = 0;
+  private probeCounts = new Map<string, ProbeCounts>();
+
   /**
    * Maximum number of raw metric entries kept in memory per category.
    * Older entries are dropped (ring-buffer style) to bound memory usage.
@@ -119,6 +147,14 @@ export class MetricsCollector {
     this.performanceMetrics.push(metric);
     this.trim(this.performanceMetrics);
 
+    if (Number.isFinite(durationMs) && durationMs >= 0) {
+      this.durationCount += 1;
+      this.durationSumMs += durationMs;
+      DURATION_BUCKETS_MS.forEach((le, i) => {
+        if (durationMs <= le) this.durationBucketCounts[i] = (this.durationBucketCounts[i] ?? 0) + 1;
+      });
+    }
+
     // Log slow requests (>1 s) so they surface in existing log pipelines.
     if (durationMs > 1000) {
       logger.warn(`Slow request: ${method} ${route} took ${durationMs}ms (status ${statusCode})`);
@@ -259,10 +295,45 @@ export class MetricsCollector {
   /**
    * Reset all collected metrics (useful in tests or after a scheduled flush).
    */
+  /**
+   * Cumulative request-duration histogram since process start (or last reset).
+   */
+  getDurationHistogram(): DurationHistogramSnapshot {
+    return {
+      buckets: DURATION_BUCKETS_MS.map((le, i) => ({ le, count: this.durationBucketCounts[i] ?? 0 })),
+      count: this.durationCount,
+      sumMs: this.durationSumMs,
+    };
+  }
+
+  /**
+   * Record the outcome of a periodic dependency probe (e.g. the deep readiness
+   * ledger check). `probe` is a low-cardinality label such as "horizon_ledger".
+   */
+  recordProbe(probe: string, ok: boolean): void {
+    const counts = this.probeCounts.get(probe) ?? { ok: 0, failed: 0 };
+    if (ok) counts.ok += 1;
+    else counts.failed += 1;
+    this.probeCounts.set(probe, counts);
+  }
+
+  /** Cumulative probe outcomes keyed by probe name. */
+  getProbeCounts(): Record<string, ProbeCounts> {
+    const out: Record<string, ProbeCounts> = {};
+    this.probeCounts.forEach((value, key) => {
+      out[key] = { ok: value.ok, failed: value.failed };
+    });
+    return out;
+  }
+
   reset(): void {
     this.performanceMetrics = [];
     this.errorMetrics = [];
     this.businessMetrics = [];
+    this.durationBucketCounts = DURATION_BUCKETS_MS.map(() => 0);
+    this.durationCount = 0;
+    this.durationSumMs = 0;
+    this.probeCounts = new Map<string, ProbeCounts>();
     logger.info('[Metrics] All metrics reset');
   }
 

backend/src/metrics/MetricsExporter.ts

@@ -11,10 +11,15 @@
  *     stays bounded.
  */
 
-import metricsCollector, { type MetricsSummary } from './MetricsCollector.js';
+import metricsCollector, {
+  type DurationHistogramSnapshot,
+  type MetricsSummary,
+  type ProbeCounts,
+} from './MetricsCollector.js';
 import workerRegistry, { type WorkerStatus } from './WorkerRegistry.js';
 import cacheService from '../cache/CacheService.js';
 import redisClient from '../cache/RedisClient.js';
+import { getDbPoolUtilizationPct } from './dbPoolMetrics.js';
 
 /** Snapshot schema version. Bump on a breaking field change. */
 export const METRICS_SCHEMA_VERSION = '1';
@@ -56,6 +61,12 @@ export interface MetricsSnapshot {
   business: { eventsTotal: number; eventsByName: Record<string, number> };
   workers: WorkerStatus[];
   system: { uptimeSeconds: number; memoryResidentBytes: number; cpuUserSeconds: number };
+  /** Cumulative request-duration histogram (optional for hand-built snapshots). */
+  durationHistogram?: DurationHistogramSnapshot;
+  /** Dependency probe outcomes keyed by probe name. */
+  probes?: Record<string, ProbeCounts>;
+  /** Last recorded DB pool utilisation (0-100), or null if never measured. */
+  dbPoolUtilizationPct?: number | null;
 }
 
 /** Segments that look like identifiers rather than route names. */
@@ -147,9 +158,28 @@ export function buildMetricsSnapshot(
       memoryResidentBytes: Math.round(summary.system.memoryUsageMB * 1024 * 1024),
       cpuUserSeconds: Number((summary.system.cpuUserMs / 1000).toFixed(3)),
     },
+    durationHistogram: metricsCollector.getDurationHistogram(),
+    probes: metricsCollector.getProbeCounts(),
+    dbPoolUtilizationPct: getDbPoolUtilizationPct(),
   };
 }
 
+/**
+ * Render a Prometheus histogram (cumulative `_bucket` lines plus `_sum` and
+ * `_count`). The generic renderFamily helper only supports counters and
+ * gauges, so histograms are rendered here.
+ */
+function renderHistogram(name: string, help: string, h: DurationHistogramSnapshot): string {
+  const lines = [`# HELP ${name} ${help}`, `# TYPE ${name} histogram`];
+  for (const bucket of h.buckets) {
+    lines.push(`${name}_bucket{le="${bucket.le}"} ${bucket.count}`);
+  }
+  lines.push(`${name}_bucket{le="+Inf"} ${h.count}`);
+  lines.push(`${name}_sum ${h.sumMs}`);
+  lines.push(`${name}_count ${h.count}`);
+  return lines.join('\n');
+}
+
 // ─── Prometheus text exposition ───────────────────────────────────────────────
 
 /** Escape a label value per the Prometheus exposition format. */
@@ -304,7 +334,39 @@ export function renderPrometheus(snapshot: MetricsSnapshot = buildMetricsSnapsho
     },
   ];
 
-  return `${families.map(renderFamily).join('\n')}\n`;
+  if (snapshot.dbPoolUtilizationPct !== undefined && snapshot.dbPoolUtilizationPct !== null) {
+    families.push({
+      name: `${PREFIX}_db_pool_utilization_percent`,
+      help: 'Database connection pool utilisation from the last health check, 0-100.',
+      type: 'gauge',
+      samples: [{ value: snapshot.dbPoolUtilizationPct }],
+    });
+  }
+
+  if (snapshot.probes && Object.keys(snapshot.probes).length > 0) {
+    families.push({
+      name: `${PREFIX}_external_probe_total`,
+      help: 'Dependency probe outcomes since process start, by probe and outcome.',
+      type: 'counter',
+      samples: Object.entries(snapshot.probes).flatMap(([probe, counts]) => [
+        { labels: { probe, outcome: 'ok' }, value: counts.ok },
+        { labels: { probe, outcome: 'failed' }, value: counts.failed },
+      ]),
+    });
+  }
+
+  const rendered = families.map(renderFamily);
+  if (snapshot.durationHistogram) {
+    rendered.push(
+      renderHistogram(
+        `${PREFIX}_http_request_duration_milliseconds`,
+        'HTTP request duration histogram in milliseconds, cumulative since process start.',
+        snapshot.durationHistogram
+      )
+    );
+  }
+
+  return `${rendered.join('\n')}\n`;
 }
 
 /** Content type expected by Prometheus-compatible scrapers. */

backend/src/metrics/dbPoolMetrics.ts

@@ -0,0 +1,23 @@
+/**
+ * Last-known database connection pool utilisation, written by the DB health
+ * check and read by the Prometheus exporter. Kept in its own module so the
+ * exporter does not import the health check (avoids a circular dependency).
+ */
+
+let poolUtilizationPct: number | null = null;
+
+/** Record the latest pool utilisation (0-100). Non-finite values are ignored. */
+export function setDbPoolUtilizationPct(value: number): void {
+  if (!Number.isFinite(value)) return;
+  poolUtilizationPct = Math.min(100, Math.max(0, value));
+}
+
+/** Latest recorded pool utilisation, or null if no health check has run yet. */
+export function getDbPoolUtilizationPct(): number | null {
+  return poolUtilizationPct;
+}
+
+/** Test helper: forget the recorded value. */
+export function resetDbPoolUtilizationPct(): void {
+  poolUtilizationPct = null;
+}

backend/tests/metricsSlo.test.ts

@@ -0,0 +1,101 @@
+/**
+ * SLO metric exposure: request-duration histogram (for p95 alerts), dependency
+ * probe counters (for the RPC failure-rate alert) and DB pool utilisation
+ * (for the pool alert). These feed monitoring/prometheus/alert.rules.yml.
+ */
+
+import metricsCollector, { DURATION_BUCKETS_MS } from '../src/metrics/MetricsCollector.js';
+import { renderPrometheus } from '../src/metrics/MetricsExporter.js';
+import {
+  getDbPoolUtilizationPct,
+  resetDbPoolUtilizationPct,
+  setDbPoolUtilizationPct,
+} from '../src/metrics/dbPoolMetrics.js';
+
+describe('request duration histogram', () => {
+  beforeEach(() => {
+    metricsCollector.reset();
+  });
+
+  it('counts requests cumulatively per bucket upper bound', () => {
+    metricsCollector.recordRequest('GET', '/a', 40, 200); // <= 50
+    metricsCollector.recordRequest('GET', '/a', 600, 200); // <= 1000
+    metricsCollector.recordRequest('GET', '/a', 1600, 200); // <= 2500
+
+    const h = metricsCollector.getDurationHistogram();
+    const byLe = Object.fromEntries(h.buckets.map((b) => [b.le, b.count]));
+
+    expect(h.count).toBe(3);
+    expect(h.sumMs).toBe(2240);
+    expect(byLe[50]).toBe(1);
+    expect(byLe[500]).toBe(1);
+    expect(byLe[1000]).toBe(2);
+    expect(byLe[1500]).toBe(2);
+    expect(byLe[2500]).toBe(3);
+    expect(h.buckets.map((b) => b.le)).toEqual([...DURATION_BUCKETS_MS]);
+  });
+
+  it('keeps the 1500 ms SLO threshold as an exact bucket boundary', () => {
+    expect(DURATION_BUCKETS_MS).toContain(1500);
+  });
+
+  it('renders histogram lines with +Inf, _sum and _count', () => {
+    metricsCollector.recordRequest('GET', '/a', 120, 200);
+
+    const text = renderPrometheus();
+
+    expect(text).toContain('# TYPE w3sl_http_request_duration_milliseconds histogram');
+    expect(text).toContain('w3sl_http_request_duration_milliseconds_bucket{le="250"} 1');
+    expect(text).toContain('w3sl_http_request_duration_milliseconds_bucket{le="+Inf"} 1');
+    expect(text).toContain('w3sl_http_request_duration_milliseconds_sum 120');
+    expect(text).toContain('w3sl_http_request_duration_milliseconds_count 1');
+  });
+
+  it('is cleared by reset()', () => {
+    metricsCollector.recordRequest('GET', '/a', 10, 200);
+    metricsCollector.reset();
+    expect(metricsCollector.getDurationHistogram().count).toBe(0);
+  });
+});
+
+describe('dependency probe counters', () => {
+  beforeEach(() => {
+    metricsCollector.reset();
+  });
+
+  it('counts ok and failed probe outcomes per probe', () => {
+    metricsCollector.recordProbe('horizon_ledger', true);
+    metricsCollector.recordProbe('horizon_ledger', true);
+    metricsCollector.recordProbe('horizon_ledger', false);
+
+    expect(metricsCollector.getProbeCounts()).toEqual({
+      horizon_ledger: { ok: 2, failed: 1 },
+    });
+
+    const text = renderPrometheus();
+    expect(text).toContain('w3sl_external_probe_total{probe="horizon_ledger",outcome="ok"} 2');
+    expect(text).toContain('w3sl_external_probe_total{probe="horizon_ledger",outcome="failed"} 1');
+  });
+});
+
+describe('DB pool utilisation gauge', () => {
+  afterEach(() => {
+    resetDbPoolUtilizationPct();
+  });
+
+  it('is omitted until a health check has recorded a value', () => {
+    expect(getDbPoolUtilizationPct()).toBeNull();
+    expect(renderPrometheus()).not.toContain('w3sl_db_pool_utilization_percent');
+  });
+
+  it('renders the last recorded utilisation and clamps to 0-100', () => {
+    setDbPoolUtilizationPct(87);
+    expect(renderPrometheus()).toContain('w3sl_db_pool_utilization_percent 87');
+
+    setDbPoolUtilizationPct(250);
+    expect(getDbPoolUtilizationPct()).toBe(100);
+
+    setDbPoolUtilizationPct(Number.NaN);
+    expect(getDbPoolUtilizationPct()).toBe(100);
+  });
+});

contracts/certificate/src/errors.rs

@@ -17,6 +17,8 @@ pub enum CertificateError {
     Unauthorized = 101,
     /// 106 — Soulbound certificate cannot be transferred or delegated.
     SoulboundNonTransferable = 106,
+    /// 107 - Certificate transfer or delegation attempted; certificates are non-transferable.
+    NonTransferable = 107,
     /// 201 — Certificate record was not found in persistent storage.
     NotFound = 201,
     /// 202 — Certificate has already been revoked.