Darkvader-ship-it · Oct 20, 2026 · Oct 20, 2026
diff --git a/.env.local.example b/.env.local.example
@@ -221,3 +221,17 @@ NEXT_PUBLIC_CLASSIC_LIQ_ISSUER=
 # PHASE_PROD_CRON_SECRET_SHA256=
 # PHASE_PROD_CONTRACT_IDS=C...,C...
 # PHASE_PREVIEW_CRON_ENABLED=1
+
+# ─── Spikes 616–619 (phase-178..181) — default off ────────────────────────
+# phase-178 leaderboard integrity: anti-farm scoring, decay, sybil discount
+# NEXT_PUBLIC_FEATURE_PHASE_178=1
+# FEATURE_PHASE_178=1
+# phase-179 retention + GDPR erasure: TTL sweeps, wallet deletion, receipt log
+# NEXT_PUBLIC_FEATURE_PHASE_179=1
+# FEATURE_PHASE_179=1
+# phase-180 analytics warehouse export (GET /api/analytics/export?format=ndjson|parquet)
+# NEXT_PUBLIC_FEATURE_PHASE_180=1
+# FEATURE_PHASE_180=1
+# phase-181 anomaly detection cron (volume, counter deltas, balance drops)
+# NEXT_PUBLIC_FEATURE_PHASE_181=1
+# FEATURE_PHASE_181=1
diff --git a/PROJECT_ARCHITECTURE.md b/PROJECT_ARCHITECTURE.md
@@ -216,6 +216,10 @@ is a fixed-size digest of the canonical payload, keeping it under wallet
 | `phase-171` | `NEXT_PUBLIC_FEATURE_PHASE_171` / `FEATURE_PHASE_171` | Backup and DR snapshots (`lib/backup-snapshot.ts`): lock-consistent sidecar set + `VACUUM INTO` SQLite image, sha256/row-count manifest, `integrity_check`/`foreign_key_check` verify, tmp+rename restore with a restore marker that makes `/api/cron/faucet-health` return 503, and an AES-256-GCM sealed secrets envelope. `scripts/backup-drill.ts` runs snapshot→verify→restore→parity. Spike 605 (issue #469) — partial slice: no off-box shipping or scheduled snapshots | off | Unset var, restart — entry points throw `FLAG_DISABLED`, restore marker ignored; snapshots on disk are inert |
 | `phase-172` | `NEXT_PUBLIC_FEATURE_PHASE_172` / `FEATURE_PHASE_172` | SQLite→Postgres rehearsal (`lib/db-parity.ts`): dialect helpers (`?`→`$n`, type map, explicit upsert), typed per-table parity (value, `type_coercion`, checksum), per-test schema names, and a `DualWriter` (primary first, bounded/timeout-guarded secondary, drift log, `primary-only` rollback). Needs `PHASE_DATABASE_URL` to enter `dual` mode. Spike 604 (issue #468) — partial slice: no Postgres driver, async CAS re-plumbing or pool storm test | off | Unset var, restart — `resolveDualWriteMode` returns `primary-only`; SQLite stays authoritative, nothing in production paths calls the module |
 | `phase-173` | `NEXT_PUBLIC_FEATURE_PHASE_173` / `FEATURE_PHASE_173` | Vercel preview isolation (`lib/preview-isolation.ts`): per-preview data root (`<root>/previews/<pr-N or br-slug-hash>`), `validatePhaseEnv` fails a preview that shares the production issuer, cron secret, contract IDs, or runs mainnet (declared via `PHASE_PROD_ISSUER`, `PHASE_PROD_CRON_SECRET_SHA256`, `PHASE_PROD_CONTRACT_IDS`), preview cron skipped unless `PHASE_PREVIEW_CRON_ENABLED=1`, and `x-vercel-cron` no longer trusted in previews. Spike 603 (issue #467) — partial slice: no per-PR contract deploy, preview CI job, or branch-DB seeding | off | Unset var, restart — data root, cron auth and env validation revert; per-preview directories stay inert |
+| `phase-178` | `NEXT_PUBLIC_FEATURE_PHASE_178` / `FEATURE_PHASE_178` | Leaderboard integrity (`lib/leaderboard-integrity.ts`): multiplicative scoring `decayed × farmPenalty × sybilDiscount` — exponential half-life decay (7d), anti-farm rules (`event_burst`, `counterparty_concentration`, `point_velocity`) floored at 0.15, and sybil band discounts (trusted 1 / caution 0.7 / suspect 0.35, missing band = caution). `GET /api/leaderboard` ranks; admin `POST /api/leaderboard` scores before persisting to `leaderboard-entries.json`. Spike 616 (issue #479) — partial slice: no on-chain sybil fetch in-route; callers supply the band | off | Unset var, restart — routes 404, `leaderboard-entries.json` stays inert |
+| `phase-179` | `NEXT_PUBLIC_FEATURE_PHASE_179` / `FEATURE_PHASE_179` | Retention and GDPR erasure (`lib/retention-erasure.ts`): TTL sweeps (expired SEP-10 sessions, consumed challenges, 90-day notifications — fail-safe on missing timestamps) via `GET /api/cron/retention-sweep` (daily 03:15, preview + restore fenced); `POST /api/user/erasure` deletes wallet-owned records across five sidecar domains behind `assertNoLegalHold` (423 on hold) and appends fingerprint-only receipts to `gdpr-erasure-log.json`. Spike 617 (issue #480) — partial slice: SQLite tables not walked; consent ledger deliberately retained per Art. 17(3)(b) | off | Unset var, restart — routes 404, receipts stay inert; nothing is deleted while off |
+| `phase-180` | `NEXT_PUBLIC_FEATURE_PHASE_180` / `FEATURE_PHASE_180` | Analytics warehouse export (`lib/analytics-warehouse.ts`): flat row model (`counter`/`health`/`volume` tables) with an unconditional redaction pass (wallets fingerprinted, secrets stripped), schema-versioned manifest (`phase-warehouse-export` 1.0.0, rows-only sha256 checksum) emitted as NDJSON header line or Parquet `key_value_metadata` (`hyparquet-writer`). Admin-keyed `GET /api/analytics/export?format=ndjson\|parquet` streams downloads with `X-Warehouse-Schema-Version`/`X-Warehouse-Checksum`. Spike 618 (issue #481) — partial slice: three source tables only | off | Unset var, restart — route 404; no stored warehouse state, collectors read live sources |
+| `phase-181` | `NEXT_PUBLIC_FEATURE_PHASE_181` / `FEATURE_PHASE_181` | Anomaly detection (`lib/anomaly-detection.ts`): robust z-score (median+MAD, σ floor) on wash-excluded volume and per-run counter deltas (own counters excluded), deterministic balance drop tiers (≥15% warning, ≥35% critical), detection-before-push baselines in `anomaly-baselines.json` (capped ring). `GET /api/cron/anomaly-detect` (hourly, minute 5, preview + restore fenced) routes findings to webhooks with distinct kind+metric titles. Spike 619 (issue #482) — partial slice: no per-channel thresholds or suppressions | off | Unset var, restart — cron 404s, baselines stay inert, no alerts routed |
 
 
 

diff --git a/app/api/analytics/export/route.ts b/app/api/analytics/export/route.ts
@@ -0,0 +1,73 @@
+import { NextRequest, NextResponse } from "next/server"
+import {
+  isWarehouseExportEnabled,
+  collectWarehouseRows,
+  redactWarehouseRows,
+  formatWarehouseNdjson,
+  rowsToParquetBuffer,
+  WarehouseExportError,
+  WAREHOUSE_SCHEMA_VERSION,
+} from "@/lib/analytics-warehouse"
+import { checkAdminKey } from "@/lib/admin-key"
+import { incSecurityCounter } from "@/lib/security-counters"
+
+export const dynamic = "force-dynamic"
+
+export async function GET(request: NextRequest) {
+  if (!isWarehouseExportEnabled()) {
+    return NextResponse.json(
+      { error: "Analytics warehouse export is disabled (set NEXT_PUBLIC_FEATURE_PHASE_180=1)" },
+      { status: 404 },
+    )
+  }
+
+  if (checkAdminKey(request) !== "ok") {
+    return NextResponse.json(
+      { error: "Unauthorized: valid x-admin-key required" },
+      { status: 401 },
+    )
+  }
+
+  const formatParam = request.nextUrl.searchParams.get("format")?.toLowerCase() ?? "ndjson"
+  if (formatParam !== "ndjson" && formatParam !== "parquet") {
+    return NextResponse.json({ error: "format must be ndjson or parquet" }, { status: 400 })
+  }
+
+  try {
+    const rows = redactWarehouseRows(await collectWarehouseRows())
+    incSecurityCounter("warehouse_records_redacted", rows.length)
+    incSecurityCounter("warehouse_export_runs")
+    const generatedAt = Date.now()
+
+    if (formatParam === "parquet") {
+      const { manifest, buffer } = rowsToParquetBuffer(rows, { generatedAt })
+      return new NextResponse(Buffer.from(buffer), {
+        status: 200,
+        headers: {
+          "Content-Type": "application/vnd.apache.parquet",
+          "Content-Disposition": `attachment; filename="phase-warehouse-${generatedAt}.parquet"`,
+          "X-Warehouse-Schema-Version": WAREHOUSE_SCHEMA_VERSION,
+          "X-Warehouse-Checksum": manifest.checksum,
+        },
+      })
+    }
+
+    const { manifest, body } = formatWarehouseNdjson(rows, { generatedAt })
+    return new NextResponse(body, {
+      status: 200,
+      headers: {
+        "Content-Type": "application/x-ndjson; charset=utf-8",
+        "Content-Disposition": `attachment; filename="phase-warehouse-${generatedAt}.ndjson"`,
+        "X-Warehouse-Schema-Version": WAREHOUSE_SCHEMA_VERSION,
+        "X-Warehouse-Checksum": manifest.checksum,
+      },
+    })
+  } catch (error) {
+    incSecurityCounter("warehouse_export_failed")
+    if (error instanceof WarehouseExportError) {
+      const status = error.code === "FLAG_DISABLED" ? 404 : error.code === "UNAUTHORIZED" ? 401 : 500
+      return NextResponse.json({ error: error.message, code: error.code }, { status })
+    }
+    throw error
+  }
+}
diff --git a/app/api/cron/anomaly-detect/route.ts b/app/api/cron/anomaly-detect/route.ts
@@ -0,0 +1,78 @@
+import { NextRequest, NextResponse } from "next/server"
+import {
+  isAnomalyDetectionEnabled,
+  collectAnomalySample,
+  detectAnomalies,
+  normalizeBaselineStore,
+  routeAnomalyAlerts,
+} from "@/lib/anomaly-detection"
+import { isRestoreInProgress } from "@/lib/backup-snapshot"
+import { isCronBlockedInPreview, previewTrustsVercelCronHeader } from "@/lib/preview-isolation"
+import { readStore, updateStore } from "@/lib/json-store"
+import { incSecurityCounter } from "@/lib/security-counters"
+
+export const dynamic = "force-dynamic"
+
+function validateCronAuth(req: NextRequest): boolean {
+  const authHeader = req.headers.get("authorization")
+  const cronSecret = process.env.CRON_SECRET?.trim()
+
+  if (process.env.NODE_ENV === "development" && !cronSecret) {
+    return true
+  }
+
+  const vercelCronHeader = req.headers.get("x-vercel-cron")
+  if (vercelCronHeader && previewTrustsVercelCronHeader()) {
+    return true
+  }
+
+  if (!cronSecret) return false
+  if (!authHeader || !authHeader.startsWith("Bearer ")) return false
+  return authHeader.substring(7) === cronSecret
+}
+
+export async function GET(req: NextRequest) {
+  if (!validateCronAuth(req)) {
+    return NextResponse.json({ error: "Unauthorized - valid cron secret required" }, { status: 401 })
+  }
+
+  if (!isAnomalyDetectionEnabled()) {
+    return NextResponse.json(
+      { error: "Anomaly detection is disabled (set NEXT_PUBLIC_FEATURE_PHASE_181=1)" },
+      { status: 404 },
+    )
+  }
+
+  if (isCronBlockedInPreview()) {
+    incSecurityCounter("preview_cron_blocked")
+    return NextResponse.json({ ok: true, skipped: "preview_cron_disabled" })
+  }
+
+  // Baselines built on half-restored data would fire false alerts and poison history.
+  if (await isRestoreInProgress()) {
+    return NextResponse.json(
+      { error: "Restore in progress" },
+      { status: 503, headers: { "Retry-After": "60" } },
+    )
+  }
+
+  const stored = await readStore<object>("anomalyBaselines", () => ({}))
+  const baseline = normalizeBaselineStore(stored)
+  const sample = await collectAnomalySample()
+  const { findings, baseline: updated } = detectAnomalies(baseline, sample)
+
+  await updateStore<object, void>("anomalyBaselines", (data) => {
+    Object.assign(data, updated)
+  })
+
+  const { routed } = await routeAnomalyAlerts(findings)
+  incSecurityCounter("anomaly_checks_run")
+
+  return NextResponse.json({
+    ok: true,
+    checked_at: sample.ts,
+    findings,
+    alerts_routed: routed,
+    timestamp: new Date().toISOString(),
+  })
+}
diff --git a/app/api/cron/retention-sweep/route.ts b/app/api/cron/retention-sweep/route.ts
@@ -0,0 +1,66 @@
+import { NextRequest, NextResponse } from "next/server"
+import { isRetentionErasureEnabled, runRetentionSweeps, RetentionErasureError } from "@/lib/retention-erasure"
+import { isRestoreInProgress } from "@/lib/backup-snapshot"
+import { isCronBlockedInPreview, previewTrustsVercelCronHeader } from "@/lib/preview-isolation"
+import { incSecurityCounter } from "@/lib/security-counters"
+
+export const dynamic = "force-dynamic"
+
+function validateCronAuth(req: NextRequest): boolean {
+  const authHeader = req.headers.get("authorization")
+  const cronSecret = process.env.CRON_SECRET?.trim()
+
+  if (process.env.NODE_ENV === "development" && !cronSecret) {
+    return true
+  }
+
+  const vercelCronHeader = req.headers.get("x-vercel-cron")
+  if (vercelCronHeader && previewTrustsVercelCronHeader()) {
+    return true
+  }
+
+  if (!cronSecret) return false
+  if (!authHeader || !authHeader.startsWith("Bearer ")) return false
+  return authHeader.substring(7) === cronSecret
+}
+
+export async function GET(req: NextRequest) {
+  if (!validateCronAuth(req)) {
+    return NextResponse.json({ error: "Unauthorized - valid cron secret required" }, { status: 401 })
+  }
+
+  if (!isRetentionErasureEnabled()) {
+    return NextResponse.json(
+      { error: "Retention sweeps are disabled (set NEXT_PUBLIC_FEATURE_PHASE_179=1)" },
+      { status: 404 },
+    )
+  }
+
+  if (isCronBlockedInPreview()) {
+    incSecurityCounter("preview_cron_blocked")
+    return NextResponse.json({ ok: true, skipped: "preview_cron_disabled" })
+  }
+
+  // Sweeping prunes sidecars — running mid-restore would fight the restore write.
+  if (await isRestoreInProgress()) {
+    return NextResponse.json(
+      { error: "Restore in progress" },
+      { status: 503, headers: { "Retry-After": "60" } },
+    )
+  }
+
+  try {
+    const result = await runRetentionSweeps()
+    return NextResponse.json({
+      ok: true,
+      total_removed: result.total_removed,
+      sweeps: result.sweeps,
+      timestamp: new Date().toISOString(),
+    })
+  } catch (error) {
+    if (error instanceof RetentionErasureError) {
+      return NextResponse.json({ error: error.message, code: error.code }, { status: 400 })
+    }
+    throw error
+  }
+}
diff --git a/app/api/leaderboard/route.ts b/app/api/leaderboard/route.ts
@@ -0,0 +1,146 @@
+import { NextRequest, NextResponse } from "next/server"
+import {
+  isLeaderboardIntegrityEnabled,
+  rankLeaderboard,
+  scoreLeaderboardCandidate,
+  LeaderboardIntegrityError,
+  LEADERBOARD_SCHEMA_VERSION,
+  type LeaderboardCandidate,
+  type SybilBand,
+} from "@/lib/leaderboard-integrity"
+import { checkAdminKey } from "@/lib/admin-key"
+import { readStore, updateStore } from "@/lib/json-store"
+import { readBoundedBody } from "@/lib/bounded-body"
+import { incSecurityCounter } from "@/lib/security-counters"
+
+export const dynamic = "force-dynamic"
+
+type EntriesStore = {
+  schema?: string
+  version?: string
+  entries?: Record<string, LeaderboardCandidate>
+}
+
+const G_ADDRESS_RE = /^G[A-Z2-7]{55}$/
+const SYBIL_BANDS: SybilBand[] = ["trusted", "caution", "suspect"]
+const MAX_EVENTS_PER_ENTRY = 500
+const MAX_BODY_BYTES = 256 * 1024
+
+function normalizeStore(raw: unknown): EntriesStore {
+  const parsed = (raw ?? {}) as Partial<EntriesStore>
+  return {
+    schema: LEADERBOARD_SCHEMA_VERSION,
+    version: LEADERBOARD_SCHEMA_VERSION,
+    entries: parsed.entries && typeof parsed.entries === "object" ? parsed.entries : {},
+  }
+}
+
+function parseCandidate(body: unknown): LeaderboardCandidate {
+  if (typeof body !== "object" || body === null) {
+    throw new LeaderboardIntegrityError("VALIDATION_FAILED", "JSON object body required")
+  }
+  const raw = body as Record<string, unknown>
+  if (typeof raw.wallet !== "string" || !G_ADDRESS_RE.test(raw.wallet)) {
+    throw new LeaderboardIntegrityError("VALIDATION_FAILED", "wallet must be a Stellar G... address")
+  }
+  if (!Array.isArray(raw.events) || raw.events.length > MAX_EVENTS_PER_ENTRY) {
+    throw new LeaderboardIntegrityError(
+      "VALIDATION_FAILED",
+      `events must be an array of at most ${MAX_EVENTS_PER_ENTRY} entries`,
+    )
+  }
+  const events = raw.events.map((entry) => {
+    const e = (entry ?? {}) as Record<string, unknown>
+    if (!Number.isFinite(e.points as number) || !Number.isFinite(e.at as number)) {
+      throw new LeaderboardIntegrityError("VALIDATION_FAILED", "each event needs finite points and at (epoch ms)")
+    }
+    if (e.counterparty !== undefined && e.counterparty !== null && typeof e.counterparty !== "string") {
+      throw new LeaderboardIntegrityError("VALIDATION_FAILED", "counterparty must be a string when present")
+    }
+    return {
+      points: e.points as number,
+      at: e.at as number,
+      ...(typeof e.counterparty === "string" && e.counterparty.length > 0 ? { counterparty: e.counterparty } : {}),
+    }
+  })
+  const band =
+    raw.sybil_band === undefined || raw.sybil_band === null
+      ? undefined
+      : SYBIL_BANDS.includes(raw.sybil_band as SybilBand)
+        ? (raw.sybil_band as SybilBand)
+        : (() => {
+            throw new LeaderboardIntegrityError("VALIDATION_FAILED", "sybil_band must be trusted|caution|suspect")
+          })()
+  return { wallet: raw.wallet, events, ...(band ? { sybilBand: band } : {}) }
+}
+
+export async function GET(request: NextRequest) {
+  if (!isLeaderboardIntegrityEnabled()) {
+    return NextResponse.json(
+      { error: "Leaderboard integrity is disabled (set NEXT_PUBLIC_FEATURE_PHASE_178=1)" },
+      { status: 404 },
+    )
+  }
+
+  const limitParam = Number(request.nextUrl.searchParams.get("limit") ?? "50")
+  const limit = Number.isFinite(limitParam) ? Math.max(1, Math.min(Math.floor(limitParam), 200)) : 50
+
+  const store = await readStore<EntriesStore>("leaderboardEntries", () => ({}))
+  const entries = Object.values(normalizeStore(store).entries ?? {})
+  const ranked = rankLeaderboard(entries).slice(0, limit)
+
+  incSecurityCounter("leaderboard_scored")
+  return NextResponse.json({
+    schema: LEADERBOARD_SCHEMA_VERSION,
+    generated_at: Date.now(),
+    count: ranked.length,
+    entries: ranked,
+  })
+}
+
+export async function POST(request: NextRequest) {
+  if (!isLeaderboardIntegrityEnabled()) {
+    return NextResponse.json(
+      { error: "Leaderboard integrity is disabled (set NEXT_PUBLIC_FEATURE_PHASE_178=1)" },
+      { status: 404 },
+    )
+  }
+
+  if (checkAdminKey(request) !== "ok") {
+    return NextResponse.json({ error: "Unauthorized: valid x-admin-key required" }, { status: 401 })
+  }
+
+  let body: unknown
+  try {
+    const text = await readBoundedBody(request, MAX_BODY_BYTES)
+    body = JSON.parse(text || "{}")
+  } catch {
+    return NextResponse.json({ error: "Invalid JSON body" }, { status: 400 })
+  }
+
+  try {
+    const candidate = parseCandidate(body)
+    // Score before persisting so malformed events never reach the store.
+    const score = scoreLeaderboardCandidate(candidate)
+    await updateStore<EntriesStore, void>("leaderboardEntries", (data) => {
+      const store = normalizeStore(data)
+      store.entries = store.entries ?? {}
+      store.entries[candidate.wallet] = candidate
+      data.schema = store.schema
+      data.version = store.version
+      data.entries = store.entries
+    })
+    incSecurityCounter("leaderboard_entries_ingested")
+    if (score.farmPenalty < 1) incSecurityCounter("leaderboard_farm_penalized")
+    if (score.sybilDiscount < 1) incSecurityCounter("leaderboard_sybil_discounted")
+    return NextResponse.json({ ok: true, score })
+  } catch (error) {
+    if (error instanceof LeaderboardIntegrityError) {
+      return NextResponse.json(
+        { error: error.message, code: error.code },
+        { status: error.code === "FLAG_DISABLED" ? 404 : 400 },
+      )
+    }
+    throw error
+  }
+}