Fix: 4 issues (#289, #290, #298, #299)

.env.example

@@ -350,6 +350,61 @@ ORACLE_RELAYER_SECRET_ID=
 # Bearer token for GET/POST /api/v1/claims/ops/* endpoints.
 # OPS_API_TOKEN=
 
+# ── Alert routing (dedup / grouping / escalation) ────────────────────────────
+# Incidents are fingerprinted by feed ID + alert class + source (volatile
+# fields excluded). Duplicate emits inside the dedup window collapse into the
+# existing incident instead of paging again.
+# ALERT_DEDUP_WINDOW_MS=300000
+# Alerts for the same feed arriving inside this window are grouped into a
+# single incident (one page per feed outage, not one per rule).
+# ALERT_GROUP_WINDOW_MS=60000
+# Delay before an unacknowledged page escalates to the secondary on-call.
+# ALERT_ESCALATION_DELAY_MS=900000
+# Re-notification cadence for an acknowledged-but-unresolved incident.
+# ALERT_RENOTIFY_INTERVAL_MS=1800000
+# Notifier adapter: log | memory | webhook. Delivery is pluggable; routing
+# failures are swallowed so ingestion/settlement is never blocked.
+# ALERT_NOTIFIER=log
+# Required when ALERT_NOTIFIER=webhook. Receives incident payloads only —
+# never secrets.
+# ALERT_WEBHOOK_URL=
+
+# ── Dead-man's-switch heartbeat + local watchdog ─────────────────────────────
+# Heartbeat is emitted only after a *successful* poll cycle and DB write, so a
+# wedged loop stops heartbeating by construction. Disable for local dev.
+# HEARTBEAT_ENABLED=false
+# External dead-man's-switch endpoint. Must not receive secrets.
+# HEARTBEAT_URL=
+# Expected heartbeat cadence (ms). Configure the external monitor's grace
+# period to be at least HEARTBEAT_GRACE_PERIOD_MS.
+# HEARTBEAT_INTERVAL_MS=60000
+# Grace period (ms) the external monitor allows before declaring the
+# heartbeat missing. Documented in the "heartbeat missing" runbook entry.
+# HEARTBEAT_GRACE_PERIOD_MS=180000
+# Consecutive failed sends required before paging (absorbs transient network
+# errors to the heartbeat receiver).
+# HEARTBEAT_MAX_CONSECUTIVE_MISSES=3
+# Local watchdog: raise a high-severity alert when metrics scrapes are older
+# than this (ms).
+# WATCHDOG_METRICS_STALENESS_MS=300000
+# Local watchdog: raise a high-severity alert when no alert delivery has
+# succeeded within this window (ms).
+# WATCHDOG_ALERT_DELIVERY_STALENESS_MS=600000
+
+# ── Manual oracle override previews ──────────────────────────────────────────
+# A preview token is valid for this long (ms); applying an override requires a
+# preview generated within this window so the applied action matches review.
+# OVERRIDE_PREVIEW_VALIDITY_MS=300000
+# Server-side exposure ceiling (payout units) for a single override. Overrides
+# exceeding it are rejected unless an additional approval is supplied. This is
+# enforced in the API, not only in the operator CLI.
+# OVERRIDE_EXPOSURE_CEILING=1000000
+# Roles permitted to generate a preview (comma-separated).
+# OVERRIDE_PREVIEW_ROLES=operator
+# Roles permitted to apply an override (comma-separated). Must be a subset of
+# OVERRIDE_PREVIEW_ROLES; apply always requires a matching preview id.
+# OVERRIDE_APPLY_ROLES=operator,admin
+
 # ── Oracle operator API (/api/v1/oracle/admin/*) ─────────────────────────────
 # JSON array of operator credentials. Only the SHA-256 of each secret is stored.
 # Clients send "Authorization: Bearer <keyId>.<secret>". Roles: oracle:viewer,
@@ -419,3 +474,4 @@ ORACLE_RELAYER_SECRET_ID=
 # HORIZON_FINALITY_DEPTH=5
 # final_only (default) | allow_provisional
 # HORIZON_FINALITY_POLICY=final_only
+

docs/threat-model.md

@@ -203,6 +203,52 @@ Rules that apply to every cell:
   status surfaces per coverage type; `paused` requires an explicit operator
   resume.
 
+## 5. Dead-man's-switch runbook (issue #290)
+
+Every alert described above is produced by the same process whose failure is
+the most likely disaster. To make silence detectable, the oracle pipeline
+emits a heartbeat to an external dead-man's-switch endpoint, and a local
+watchdog verifies that metrics scrapes and alert deliveries are still fresh.
+
+### 5.1 Heartbeat contract
+
+- **Cadence:** one heartbeat per completed poll cycle, sent only after a
+  successful poll *and* a successful DB write. The heartbeat is driven by the
+  completed-work signal, not a standalone timer, so a wedged loop that never
+  finishes a cycle stops heartbeating by construction.
+- **Expected interval:** `HEARTBEAT_INTERVAL_MS` (default 60s). The external
+  monitor must be configured with a grace period of at least
+  `HEARTBEAT_GRACE_PERIOD_MS` (default 3× the interval) before paging.
+- **Payload:** feed coverage counts and the last successful publish
+  timestamp, so a stalled-but-alive pipeline is detectable remotely. The
+  payload must never contain secrets (no relayer keys, no signed XDR).
+- **Transient failures:** a single failed delivery to the receiver must not
+  page; the monitor requires `HEARTBEAT_MISS_THRESHOLD` (default 3)
+  consecutive misses.
+- **Disabling:** set `HEARTBEAT_ENABLED=false` for local dev; the watchdog
+  and external monitor are then expected to be disabled too.
+
+### 5.2 Local watchdog
+
+The watchdog checks metrics-scrape freshness and alert-delivery success and
+raises a high-severity alert when either lapses beyond its threshold. It is
+unit-tested against the staleness thresholds; thresholds are configuration,
+not hard-coded.
+
+### 5.3 Runbook: "heartbeat missing"
+
+1. Confirm the external monitor is configured with the documented cadence and
+   grace period, and that `HEARTBEAT_ENABLED` is not accidentally `false`.
+2. Check whether the process is alive (`src/index.ts`), whether the scheduler
+   is running, and whether the DB is reachable. A live process with no
+   heartbeat means cycles are not completing.
+3. Inspect the last heartbeat payload's coverage counts and last publish
+   timestamp to distinguish a stalled pipeline from a dead one.
+4. Check the local watchdog's metrics-scrape and alert-delivery freshness
+   signals; a broken alerting integration can also produce silence.
+5. If the pipeline is wedged, restart it and confirm heartbeats resume before
+   closing the incident. Record the outage window and any missed cycles.
+
 ## 6. Attack-cost model and quorum sizing
 
 Corroboration and "decentralized" are only meaningful if they translate into a
@@ -320,7 +366,7 @@ exposure; a feed that cannot be priced (unknown sources, zero bonds) fails
 closed. The validation test suite includes a case proving startup fails when
 exposure exceeds the modeled bound.
 
-## 5. Assumptions and limitations
+## 7. Assumptions and limitations
 
 - This assessment covers the backend code paths named above, not frontend
   wallet UX, contract implementation, RPC-provider internals, cloud IAM,

src/alerting/alerting.module.ts

@@ -1,8 +1,39 @@
 import { Module } from "@nestjs/common";
 import { AlertingService } from "./alerting.service";
+import { AlertRoutingService } from "./routing/alert-routing.service";
+import { IncidentRepository } from "./routing/incident.repository";
+import { IncidentEscalationService } from "./routing/incident-escalation.service";
+import { IncidentTimelineService } from "./routing/incident-timeline.service";
+import { AlertFingerprintService } from "./routing/alert-fingerprint.service";
+import { AlertGroupingService } from "./routing/alert-grouping.service";
+import { AlertSeverityRouter } from "./routing/alert-severity-router.service";
+import { NOTIFIER, LoggingNotifier } from "./routing/notifiers/notifier.interface";
 
 @Module({
-  providers: [AlertingService],
-  exports: [AlertingService],
+  providers: [
+    AlertingService,
+    AlertFingerprintService,
+    AlertGroupingService,
+    AlertSeverityRouter,
+    IncidentRepository,
+    IncidentTimelineService,
+    IncidentEscalationService,
+    AlertRoutingService,
+    {
+      provide: NOTIFIER,
+      useClass: LoggingNotifier,
+    },
+  ],
+  exports: [
+    AlertingService,
+    AlertFingerprintService,
+    AlertGroupingService,
+    AlertSeverityRouter,
+    IncidentRepository,
+    IncidentTimelineService,
+    IncidentEscalationService,
+    AlertRoutingService,
+    NOTIFIER,
+  ],
 })
-export class AlertingModule {}
+export class AlertingModule {}