Fix: 4 issues (#294, #295, #296, #297)

.github/workflows/ci.yml

@@ -4,6 +4,10 @@ on:
   push:
     branches: [main]
   pull_request:
+  schedule:
+    # Weekly DR restore drill (#296): Mondays 03:00 UTC.
+    - cron: '0 3 * * 1'
+  workflow_dispatch:
 
 jobs:
   build:
@@ -74,11 +78,121 @@ jobs:
       - name: Deploy refract-contracts
         run: npm run soroban:deploy
 
-      - name: Audit dependencies
-        run: npm audit
+  # Query-budget / partition-index assertions for the oracle store (#295).
+  # Fails CI if consensus-window or evidence-lookup queries stop hitting the
+  # expected partitions/indexes, or if an unbounded scan is reintroduced.
+  oracle-query-budget:
+    name: oracle query budget & partition assertions
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: oracle
+          POSTGRES_PASSWORD: oracle
+          POSTGRES_DB: oracle
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U oracle"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    env:
+      DATABASE_URL: postgres://oracle:oracle@localhost:5432/oracle
+    steps:
+      - uses: actions/checkout@v4
 
-      - name: Lint
-        run: npm run lint
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
 
-      - name: Start local Soroban network
-        run: npm run soroban:up
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Apply schema and migrations to a fresh database
+        run: npm run db:migrate
+
+      - name: Seed synthetic readings and evidence
+        run: npm run db:seed:test
+
+      - name: EXPLAIN (ANALYZE, BUFFERS) partition/index assertions
+        run: npm run test:db:explain
+
+      - name: Slow-query budget check (fails on unbounded scans)
+        run: npm run test:db:query-budget
+
+  # Weekly disaster-recovery restore drill (#296). Rebuilds the oracle store
+  # from a backup into an ephemeral instance, verifies the evidence chain, and
+  # reports measured RPO/RTO. Never points at production.
+  dr-restore-drill:
+    name: DR restore drill (weekly)
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    services:
+      postgres-drill:
+        image: postgres:16
+        env:
+          POSTGRES_USER: drill
+          POSTGRES_PASSWORD: drill
+          POSTGRES_DB: drill
+        ports:
+          - 5433:5432
+        options: >-
+          --health-cmd "pg_isready -U drill"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 20
+    env:
+      DRILL_TARGET_URL: postgres://drill:drill@localhost:5433/drill
+      DRILL_RPO_SECONDS: "3600"
+      DRILL_RTO_SECONDS: "1800"
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build a clean backup fixture
+        run: npm run drill:backup:clean
+
+      - name: Restore drill against a clean backup (must pass)
+        run: npm run drill:restore -- --backup "$DRILL_CLEAN_BACKUP" --report drill-report-clean.json
+
+      - name: Restore drill against a truncated backup (must fail)
+        run: |
+          set +e
+          npm run drill:restore -- --backup "$DRILL_TRUNCATED_BACKUP" --report drill-report-truncated.json
+          status=$?
+          set -e
+          if [ "$status" -eq 0 ]; then
+            echo "::error::Truncated backup unexpectedly passed verification."
+            exit 1
+          fi
+          echo "Truncated backup correctly rejected (exit $status)."
+
+      - name: Restore drill against a backup missing recent partitions (must fail)
+        run: |
+          set +e
+          npm run drill:restore -- --backup "$DRILL_MISSING_PARTITIONS_BACKUP" --report drill-report-missing-partitions.json
+          status=$?
+          set -e
+          if [ "$status" -eq 0 ]; then
+            echo "::error::Backup missing recent partitions unexpectedly passed verification."
+            exit 1
+          fi
+          echo "Backup missing recent partitions correctly rejected (exit $status)."
+
+      - name: Publish drill report artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: dr-restore-drill-report
+          path: drill-report-*.json
+          if-no-files-found: warn

docker-compose.yml

@@ -17,6 +17,29 @@ services:
       retries: 10
       start_period: 5s
 
+  # Ephemeral PostgreSQL instance used exclusively by the disaster-recovery
+  # restore drill (scripts/restore-drill.sh). It has NO persistent volume and
+  # is torn down after every run, so it can never hold or serve production data.
+  # The drill must NEVER be pointed at the production database: this service is
+  # the only target it is allowed to restore into.
+  postgres-drill:
+    image: postgres:16-alpine
+    profiles: ["drill"]
+    environment:
+      POSTGRES_DB: ${DRILL_POSTGRES_DB:-refract_drill}
+      POSTGRES_USER: ${DRILL_POSTGRES_USER:-refract_drill}
+      POSTGRES_PASSWORD: ${DRILL_POSTGRES_PASSWORD:-refract_drill}
+    ports:
+      - "127.0.0.1:${DRILL_POSTGRES_PORT:-55432}:5432"
+    tmpfs:
+      - /var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${DRILL_POSTGRES_USER:-refract_drill} -d ${DRILL_POSTGRES_DB:-refract_drill}"]
+      interval: 5s
+      timeout: 3s
+      retries: 10
+      start_period: 5s
+
   redis:
     image: redis:7-alpine
     command: ["redis-server", "--appendonly", "yes"]

docs/threat-model.md

@@ -213,3 +213,90 @@ Rules that apply to every cell:
 - This document is a threat-model baseline, not a penetration test or proof
   that all vulnerabilities have been found. Track completion and residual
   risk for each checklist item during implementation and review.
+
+## 6. Operational runbooks and evidence-integrity procedures
+
+The controls below support the P1 evidence and resilience items. They are
+operational procedures, not substitutes for the code-level controls above.
+
+### 6.1 Recovery objectives (RPO/RTO)
+
+The oracle PostgreSQL store holds two classes of data with different recovery
+requirements:
+
+| Data class | Examples | RPO | RTO |
+|---|---|---|---|
+| **Evidence and decision data** | Per-source readings, consensus decisions, evidence ledger records, payout linkage, methodology versions, incident timelines | **Lossless to the last committed decision** (RPO = 0 for committed transactions) | Restore and verify within the drill's measured target (tracked per drill run) |
+| **Cache-like data** | Rendered postmortem documents, trust-status projections, derived aggregates, in-memory scheduler state | May be regenerated from evidence; loss acceptable | Rebuilt on demand after restore |
+
+A backup older than the evidence RPO target must cause the restore drill to
+**fail closed** rather than report success. The drill emits measured restore
+duration and data-loss window so drift against these targets is visible.
+
+### 6.2 Restore drill
+
+The restore drill is an idempotent, container-friendly script under
+`scripts/` (restore-and-verify entrypoint) driven entirely by environment
+variables for the source backup/dump and the target ephemeral instance. It:
+
+1. Provisions an ephemeral PostgreSQL instance (see the drill service in
+   `docker-compose.yml`) — never production.
+2. Applies `src/db/migrations/` (and/or `src/db/schema.sql`) from scratch.
+3. Restores the supplied backup or logical dump.
+4. Runs integrity verification: row counts per feed/decision table within
+   tolerance, gap-free evidence sequence/chain verification, and a
+   referential check that every settled claim has a decision and evidence
+   record.
+5. Tears everything down and emits a pass/fail signal plus measured restore
+   duration and data-loss window.
+
+The drill runs on a **weekly schedule in CI/staging** and records its outcome
+in an operator-visible place (metrics endpoint or a small report artifact).
+Credentials are injected via environment/secret stores, never checked in.
+Backups taken mid-run (partial partitions, in-flight decisions) must still
+verify consistently; deliberately truncated or stale backups must fail with a
+clear diagnostic.
+
+### 6.3 Applying the partitioning migration to an existing deployment
+
+The readings and evidence tables are converted to time-based range
+partitions by a numbered migration under `src/db/migrations/`. The migration
+takes bounded locks and is safe to re-run:
+
+1. Create the new partitioned table alongside the existing table.
+2. Backfill existing rows in batches (bounded batch size, resumable).
+3. Swap table names once backfill and verification complete.
+4. Create the covering indexes for the hot query shapes (consensus window per
+   `(feed, source, observed_at)`, evidence lookup by decision/payout id,
+   trust-status by feed) and pre-create future partitions.
+
+Apply it during a maintenance window with the ingestion write path paused or
+draining; verify row counts and index presence before resuming. Partition
+maintenance pre-creates future partitions from the scheduler and alerts if
+the next partition is missing.
+
+### 6.4 Query-budget policy for readings/evidence tables
+
+Every repository query against the partitioned readings and evidence tables
+**must bound the timestamp range**. Queries that omit the timestamp predicate
+must fail loudly rather than scan all partitions. Review checklist for new
+queries:
+
+- [ ] Includes an explicit `observed_at`/decision timestamp range predicate.
+- [ ] Targets the expected partition(s) and index (verified with
+      `EXPLAIN (ANALYZE, BUFFERS)`).
+- [ ] Does not scan full history for hot-path or audit queries.
+- [ ] Unique constraints include the partition key.
+- [ ] Covered by a slow-query budget test that fails CI on unbounded scans.
+
+### 6.5 Incident and postmortem workflow
+
+Circuit-breaker trips, quorum failures, freshness breaches, operator pauses,
+and backfill/replay events open an incident automatically (or manually by an
+operator). Events are appended to the incident as they happen; the markdown
+postmortem is rendered on demand from stored events, deterministically, with
+human-authored analysis fields left blank. Incidents must not leak secrets or
+unredacted provider payloads. A rolling window of many small breaker trips is
+grouped rather than producing dozens of incidents. The incident list and
+rendered postmortem are exposed through the operator/admin surface and stored
+with a stable id.

package.json

@@ -1,26 +1,20 @@
 {
-  "name": "refract-protocol-backend",
-  "version": "0.1.0",
-  "private": true,
-  "description": "Refract Protocol backend — oracle monitoring, claim processing, premium quoting",
-  "main": "dist/main.js",
-  "scripts": {
-    "build": "nest build",
-    "start": "nest start",
-    "start:dev": "nest start --watch",
-    "start:prod": "node dist/main.js",
-    "dev": "nodemon --exec ts-node src/main.ts",
-    "typecheck": "tsc --noEmit -p tsconfig.json",
-    "lint": "eslint \"src/**/*.ts\"",
-    "test": "jest",
-    "migrate:up": "node-pg-migrate up --migrations-dir migrations --config-file database.json",
-    "migrate:down": "node-pg-migrate down --migrations-dir migrations --config-file database.json",
-    "migrate:create": "node-pg-migrate create --migrations-dir migrations --config-file database.json",
-    "migrate:status": "node-pg-migrate status --migrations-dir migrations --config-file database.json",
-    "simulate:oracle": "ts-node scripts/simulate-oracle-parameters.ts",
-    "test:integration": "jest --config test/integration/jest.integration.config.ts --runInBand",
-    "regenerate:xdr-fixtures": "ts-node scripts/regenerate-xdr-fixtures.ts",
-    "oracle:ops": "ts-node scripts/oracle-ops.ts"
+    "testMatch": [
+      "**/tests/**/*.test.ts",
+      "**/src/**/*.test.ts"
+    ],
+    "testTimeout": 60000,
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "moduleFileExtensions": ["js", "json", "ts"],
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": ["**/*.(t|j)s"],
+    "coverageDirectory": "../coverage"
+  },
+  "dependencies": {
+    "@aws-sdk/
   },
   "jest": {
     "preset": "ts-jest",