diff --git a/config/dependency-audit-exceptions.json b/config/dependency-audit-exceptions.json index f226765e..52be19a7 100644 --- a/config/dependency-audit-exceptions.json +++ b/config/dependency-audit-exceptions.json @@ -1,4 +1,53 @@ { "schemaVersion": 1, - "exceptions": [] + "exceptions": [ + { + "advisory": "@grpc/grpc-js", + "owner": "@BernardOnuh", + "reason": "Pending patch update in dependabot PR #582; internal gRPC exposure only", + "expiresOn": "2026-11-30" + }, + { + "advisory": "@opentelemetry/auto-instrumentations-node", + "owner": "@BernardOnuh", + "reason": "Transitive dependency on OpenTelemetry instrumentations; upgrade planned in P3 roadmap", + "expiresOn": "2026-11-30" + }, + { + "advisory": "GHSA-qqmp-wf37-98f9", + "owner": "@BernardOnuh", + "reason": "OpenTelemetry database username span attribute exposure; tracked in P3 upgrade plan", + "expiresOn": "2026-11-30" + }, + { + "advisory": "brace-expansion", + "owner": "@BernardOnuh", + "reason": "Transitive dependency via glob tooling; pending upstream release", + "expiresOn": "2026-11-30" + }, + { + "advisory": "ip-address", + "owner": "@BernardOnuh", + "reason": "Transitive dependency; pending upstream patch", + "expiresOn": "2026-11-30" + }, + { + "advisory": "minimatch", + "owner": "@BernardOnuh", + "reason": "Transitive dependency via gaxios; pending patch", + "expiresOn": "2026-11-30" + }, + { + "advisory": "proxy-addr", + "owner": "@BernardOnuh", + "reason": "proxy-addr IPv4-mapped IPv6 trust subnet issue; pending express upgrade", + "expiresOn": "2026-11-30" + }, + { + "advisory": "smol-toml", + "owner": "@BernardOnuh", + "reason": "smol-toml quadratic parse issue; non-user-facing config parser", + "expiresOn": "2026-11-30" + } + ] } diff --git a/docs/API_KEYS.md b/docs/API_KEYS.md index 8e751162..f25851da 100644 --- a/docs/API_KEYS.md +++ b/docs/API_KEYS.md @@ -70,7 +70,8 @@ New keys are **read-only by default**. Write scopes must be explicitly requested | `dlq:read` | read | enforced | — | Inspect the dead-letter queue. | — | `GET /api/v1/admin/dlq/inspect`, `GET /api/v1/admin/notifications/dlq` | | `erasure:read` | read | declared-only | — | Preview what an erasure would delete. | No route requires this scope today — the erasure preview route is gated by the generic read scope. Declared so a key can be provisioned ahead of a route that needs it. | — | | `fiat:read` | read | enforced | — | Read fiat providers and their health. | — | `GET /api/v1/admin/fiat/providers` | -| `keys:read` | read | enforced | — | Read admin API keys and wallet rotation status. | — | `GET /api/v1/admin/keys`, `GET /api/v1/admin/keys/scopes`, `GET /api/v1/admin/wallets/rotation-status` | +| `flags:read` | read | enforced | — | Read feature flags and rollout state. | — | `GET /api/v1/admin/feature-flags`, `GET /api/v1/admin/feature-flags/:key`, `GET /api/v1/admin/feature-flags/history` | +| `keys:read` | read | enforced | — | Read admin API keys and wallet rotation status. | — | `GET /api/v1/admin/keys`, `GET /api/v1/admin/wallets/rotation-status` | | `messages:read` | read | enforced | — | Read message delivery state. | — | `GET /api/v1/admin/messages`, `GET /api/v1/admin/messages/:id`, `GET /api/v1/admin/messages/stats` | | `metrics:read` | read | enforced | — | Read operational metrics and dashboards. | — | `GET /api/v1/admin/jobs/dashboard`, `GET /api/v1/admin/stellar/metrics` | | `outbox:read` | read | enforced | — | Read the durable outbox. | — | `GET /api/v1/admin/outbox`, `GET /api/v1/admin/outbox/:id`, `GET /api/v1/admin/outbox/stats` | @@ -91,13 +92,14 @@ New keys are **read-only by default**. Write scopes must be explicitly requested | `wallet` | money | declared-only | — | Custodial wallet operations. | No route requires this scope today — the wallet rotation-status route is gated by keys:read. Declared so a key can be provisioned ahead of a route that needs it. | — | | `erasure:write` | destructive | enforced | — | Create, approve, reject and execute erasure requests. | Execution requires super — approval and execution are deliberately separated so one operator cannot both authorise and carry out a deletion. | `POST /api/v1/admin/erasure` | | `keys:write` | destructive | enforced | — | Create, rotate and revoke admin API keys. | Minting an admin key is a takeover vector — every use is written to the admin audit log. | `DELETE /api/v1/admin/keys/:id`, `POST /api/v1/admin/keys`, `POST /api/v1/admin/keys/:id/rotate` | -| `super` | destructive | enforced | `support:read`, `support:write`, `read`, `write`, `wallet`, `agent`, `agent:read`, `agent:write`, `sessions:read`, `sessions:write`, `reserves:read`, `metrics:read`, `dlq:read`, `dlq:write`, `backfill:write`, `keys:read`, `keys:write`, `fiat:read`, `fiat:write`, `outbox:read`, `outbox:write`, `approvals:write`, `referrals:read`, `referrals:write`, `erasure:write`, `erasure:read`, `messages:read`, `messages:write`, `treasury:read`, `treasury:write`, `risk-metadata:read`, `risk-metadata:write`, `admin:impersonate`, `super` | Every admin scope, including erasure execution. | Held by as few keys as possible: it is a superset of every destructive scope in this catalogue. | `GET /api/v1/admin/audit/prove`, `GET /api/v1/admin/audit/verify`, `POST /api/v1/admin/erasure/requests/:id/approve`, `POST /api/v1/admin/erasure/requests/:id/execute`, `POST /api/v1/admin/erasure/requests/:id/reject` | +| `super` | destructive | enforced | `support:read`, `support:write`, `read`, `write`, `wallet`, `agent`, `agent:read`, `agent:write`, `sessions:read`, `sessions:write`, `reserves:read`, `metrics:read`, `dlq:read`, `dlq:write`, `backfill:write`, `keys:read`, `keys:write`, `fiat:read`, `fiat:write`, `outbox:read`, `outbox:write`, `approvals:write`, `referrals:read`, `referrals:write`, `erasure:write`, `erasure:read`, `flags:read`, `flags:write`, `messages:read`, `messages:write`, `treasury:read`, `treasury:write`, `risk-metadata:read`, `risk-metadata:write`, `admin:impersonate`, `super` | Every admin scope, including erasure execution. | Held by as few keys as possible: it is a superset of every destructive scope in this catalogue. | `GET /api/v1/admin/audit/prove`, `GET /api/v1/admin/audit/verify`, `POST /api/v1/admin/erasure/requests/:id/approve`, `POST /api/v1/admin/erasure/requests/:id/execute`, `POST /api/v1/admin/erasure/requests/:id/reject` | | `admin:impersonate` | operational | declared-only | — | View the API as a user, read-only. | No route requires this scope today — the impersonation controller exists but is not yet exposed through the admin API. Every impersonated request is attributed to the operator in the admin audit log. | — | | `agent` | operational | declared-only | `agent:read`, `agent:write` | Agent loop operations. | — | — | | `agent:write` | operational | enforced | — | Change agent circuit breakers. | — | `POST /api/v1/admin/agent/breakers`, `POST /api/v1/admin/agent/breakers/:id/reset` | | `backfill:write` | operational | enforced | — | Run Stellar event backfills. | — | `POST /api/v1/admin/stellar/backfill` | | `dlq:write` | operational | enforced | — | Replay, retry and resolve dead-lettered work. | — | `POST /api/v1/admin/dlq/replay`, `POST /api/v1/admin/dlq/resolve`, `POST /api/v1/admin/dlq/retry`, `POST /api/v1/admin/notifications/dlq/:id/retry` | | `fiat:write` | operational | enforced | — | Change fiat provider failover state. | — | `POST /api/v1/admin/fiat/providers/:name/failover` | +| `flags:write` | operational | enforced | — | Manage feature flags, staged rollouts and emergency rollbacks. | — | `POST /api/v1/admin/feature-flags/:key/reset`, `POST /api/v1/admin/feature-flags/:key/rollback`, `POST /api/v1/admin/feature-flags/:key/staged-rollout`, `POST /api/v1/admin/feature-flags/emergency-disable-all`, `PUT /api/v1/admin/feature-flags/:key` | | `messages:write` | operational | enforced | — | Retry and recover failed message deliveries. | — | `DELETE /api/v1/admin/messages/:id`, `POST /api/v1/admin/messages/:id/retry`, `POST /api/v1/admin/messages/retry-dead-letters` | | `risk-metadata:write` | operational | declared-only | — | Review and publish protocol risk metadata. | No route requires this scope today — the risk-metadata review workflow is administered from src/config/riskMetadataAdmin.ts, not through the API. Declared so a key can be provisioned ahead of a route that needs it. | — | diff --git a/docs/DOCUMENTATION_INDEX.md b/docs/DOCUMENTATION_INDEX.md index 1abbbfdc..e385ee72 100644 --- a/docs/DOCUMENTATION_INDEX.md +++ b/docs/DOCUMENTATION_INDEX.md @@ -17,6 +17,7 @@ - **[REBALANCE_EXPOSURE_COST.md](REBALANCE_EXPOSURE_COST.md)** - Rebalance exposure caps (#346) and grounded cost model + payback gate (#347): resolution, tighten-only merge, residual routing, conservative-by-default decisions - **[CORRELATION_DIVERSIFICATION.md](CORRELATION_DIVERSIFICATION.md)** - Portfolio APY correlation matrix + diversification score (#348): alignment, null-on-degenerate semantics, weighted score, caveat - **[YIELD_COMPOSITION.md](YIELD_COMPOSITION.md)** - Yield base/incentive decomposition + effective APY (#349): haircut model, schema columns, flag-gated consumption, emissions risk modifier, yield-breakdown endpoint +- **[FEATURE_FLAGS.md](FEATURE_FLAGS.md)** - Feature flag configuration, deterministic staged rollouts, emergency rollback runbook, and evaluation observability (#494) - **[MESSAGING_DELIVERY.md](MESSAGING_DELIVERY.md)** - Telegram & WhatsApp message delivery reliability, retry/fallback policy, metrics, and recovery admin API (#493) - **[WEBHOOK_SECURITY.md](WEBHOOK_SECURITY.md)** - Inbound fiat webhook HMAC validation, freshness windows, replay receipts, and failure responses - **[PRISMA_QUERY_PERFORMANCE.md](PRISMA_QUERY_PERFORMANCE.md)** - Hot webhook query plans, supporting indexes, and the query-index regression check diff --git a/docs/FEATURE_FLAGS.md b/docs/FEATURE_FLAGS.md new file mode 100644 index 00000000..7a56f1d4 --- /dev/null +++ b/docs/FEATURE_FLAGS.md @@ -0,0 +1,265 @@ +# Feature Flag Configuration, Staged Rollout, and Rollback System (#494) + +The central Feature Flag mechanism provides a resilient, environment-aware control plane for progressive releases, staged canary adoptions, and instantaneous emergency rollbacks in production. + +--- + +## 1. Architecture & Core Concepts + +The system is centered around `FeatureFlagManager` (`src/config/featureFlags.ts`), exported globally as `featureFlagManager`. + +### Multi-Tier Resolution Hierarchy + +Flag evaluation follows a strict, fail-closed resolution pipeline: + +``` +┌────────────────────────────────────────────────────────┐ +│ 1. Global Maintenance Mode (emergency_maintenance_mode) │ ──> Disabled +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 2. Flag-Level Kill Switch (def.killSwitch) │ ──> Disabled +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 3. Emergency Rollback (def.rolledBack / Env Rollback) │ ──> Disabled (0%) +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 4. Entity Blocklist (def.blocklist / env.blocklist) │ ──> Disabled +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 5. Entity Allowlist (def.allowlist / env.allowlist) │ ──> Enabled (100%) +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 6. Runtime Dynamic Overrides (Admin API / Redis) │ ──> Dynamic / Rollout % +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 7. Environment Variable Overrides │ ──> Env Val / Env % +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 8. Environment Baseline (dev / test / staging / prod) │ ──> Env Default / % +└────────────────────────────────────────────────────────┘ + │ +┌────────────────────────────────────────────────────────┐ +│ 9. Unregistered Flag Fallback (Fail Closed) │ ──> False +└────────────────────────────────────────────────────────┘ +``` + +--- + +## 2. Environment Awareness + +Feature flags natively differentiate behavior across execution environments (`development`, `test`, `staging`, `production`), determined via `NODE_ENV`. + +### Built-in Flag Catalog (`DEFAULT_FEATURE_FLAGS`) + +| Flag Key | Description | Dev | Test | Staging | Production | +| :--- | :--- | :---: | :---: | :---: | :---: | +| `agent_autonomous_rebalance` | Automated agent-driven portfolio rebalancing | 100% | 100% | 50% | 0% (Off) | +| `smart_dca_engine` | Volatility-aware adaptive recurring deposits engine | 100% | 100% | 25% | 0% (Off) | +| `enhanced_stellar_routing` | Horizon liquidity path finding and smart fee routing | 100% | 100% | 100% | 10% | +| `ai_assistant_tool_calling` | Conversational assistant with autonomous tool invocation | 100% | 100% | 50% | 0% (Off) | +| `circuit_breaker_v2` | Dynamic volatility and drawdown circuit breaker trip rules | 100% | 100% | 100% | 100% (On) | +| `fast_fiat_onramp` | Direct instant on-ramp quoting and locking | 100% | 100% | 100% | 0% (Off) | +| `emergency_maintenance_mode` | Platform-wide guard to halt state-mutating requests | Off | Off | Off | Off | + +### Environment Variable Overrides + +Any flag can be modified or overridden via environment variables without requiring a code release: + +- **State Override**: `FEATURE_FLAG_=true|false` + *Example*: `FEATURE_FLAG_SMART_DCA_ENGINE=true` +- **Percentage Override**: `FEATURE_FLAG__PERCENTAGE=0..100` + *Example*: `FEATURE_FLAG_ENHANCED_STELLAR_ROUTING_PERCENTAGE=25` +- **Kill-Switch Override**: `FEATURE_FLAG__KILL_SWITCH=true` + *Example*: `FEATURE_FLAG_FAST_FIAT_ONRAMP_KILL_SWITCH=true` +- **Emergency Rollback List**: `FEATURE_FLAGS_ROLLBACK=flag1,flag2,flag3` + *Example*: `FEATURE_FLAGS_ROLLBACK=smart_dca_engine,fast_fiat_onramp` + +--- + +## 3. Deterministic Staged Rollout + +Staged releases use cryptographic SHA-256 entity hashing (`hashEntityToPercentage`) to ensure consistent, monotonic, and sticky cohort assignment: + +$$\text{bucket} = \text{parseInt}\big(\text{SHA256}(\text{flagKey} : \text{entityId})[0..8], 16\big) \pmod{100}$$ + +### Staged Rollout Progression + +1. **Phase 0 (Internal / Canary)**: `0%` rollout. Selected testers enabled via `allowlist: ['user-id-1', 'user-id-2']`. +2. **Phase 1 (10% Tier)**: `10%` rollout (`stagedRollout(flag, 10)`). +3. **Phase 2 (25% Tier)**: `25%` rollout (`stagedRollout(flag, 25)`). +4. **Phase 3 (50% Tier)**: `50%` rollout (`stagedRollout(flag, 50)`). +5. **Phase 4 (General Availability - 100%)**: `100%` rollout (`stagedRollout(flag, 100)`). + +Because the hashing is monotonic: +- Any entity included in the 10% tier remains included in the 25%, 50%, and 100% tiers. +- No user suffers flip-flopping feature availability between server restarts or across horizontally scaled cluster pods. + +--- + +## 4. Emergency Rollback Runbook + +When an anomaly, exploit, or regression occurs in production, operators have 4 independent, fast rollback paths: + +### Path A: Fast Admin API Rollback (< 1 second) + +Requires `flags:write` or `super` admin scope. + +```bash +# Roll back a specific feature flag immediately +curl -X POST https://api.neurowealth.app/api/v1/admin/feature-flags/smart_dca_engine/rollback \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"reason": "High slippage observed in Horizon liquidity pools"}' +``` + +**What occurs instantly:** +1. Rollout percentage drops to `0%`. +2. Flag is marked `rolledBack = true`. +3. Rollback event is recorded into memory and audit history. +4. Winston logs an `error` level `[FeatureFlag] EMERGENCY ROLLBACK TRIGGERED`. +5. Prometheus metric `feature_flag_rollbacks_total{flag="smart_dca_engine"}` increments. +6. Event is published to Redis pub/sub (`neurowealth:feature_flags:updates`) to immediately sync all active cluster pods. + +### Path B: Platform Emergency Maintenance Mode + +To halt all state-mutating requests platform-wide during a critical incident: + +```bash +curl -X POST https://api.neurowealth.app/api/v1/admin/feature-flags/emergency-disable-all \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"reason": "Investigating zero-day smart contract vulnerability"}' +``` + +- Halts all `POST`, `PUT`, `DELETE`, and `PATCH` requests with `503 SERVICE_UNAVAILABLE` and `EMERGENCY_MAINTENANCE` code. +- Read-only `GET` endpoints, `/health/*`, `/metrics`, and `/api/v1/admin/*` remain operational so SRE can inspect and remediate. + +### Path C: Container Environment Variable Emergency Disable + +If the Admin API or database is unreachable, update container / pod environment variables in Kubernetes or ECS: + +```bash +# Rollback specific flags +FEATURE_FLAGS_ROLLBACK=smart_dca_engine,fast_fiat_onramp + +# Or engage flag kill-switch +FEATURE_FLAG_SMART_DCA_ENGINE_KILL_SWITCH=true +``` + +### Path D: Redis Emergency Rollback + +If operating directly through Redis CLI: + +```bash +redis-cli PUBLISH neurowealth:feature_flags:updates \ + '{"flagKey":"smart_dca_engine","override":{"enabled":false,"rolloutPercentage":0,"rolledBack":true,"rollbackReason":"Direct redis rollback"}}' +``` + +--- + +## 5. Structured Decision Logging & Observability + +Every flag evaluation decision is captured with structured Winston logging: + +```json +{ + "timestamp": "2026-09-26T23:30:00.000Z", + "level": "info", + "message": "[FeatureFlag] Evaluated flag \"smart_dca_engine\": ENABLED (PERCENTAGE_ROLLOUT_MATCH)", + "event": "feature_flag_evaluation", + "flag": "smart_dca_engine", + "enabled": true, + "decision": "PERCENTAGE_ROLLOUT_MATCH", + "reason": "Runtime staged rollout (25%) for entity bucket 12", + "environment": "production", + "entityId": "usr_99812739", + "rolloutPercentage": 25 +} +``` + +### Decision Types + +- `GLOBAL_MAINTENANCE`: Disabled due to active platform maintenance mode. +- `KILL_SWITCH`: Disabled due to an active flag kill switch. +- `EMERGENCY_ROLLBACK`: Disabled due to an emergency rollback event. +- `BLOCKLIST`: Entity is explicitly blocked. +- `ALLOWLIST`: Entity is explicitly allowed. +- `RUNTIME_OVERRIDE`: Dynamic override set via Admin API. +- `ENV_VAR_OVERRIDE`: Overridden via environment variable. +- `PERCENTAGE_ROLLOUT_MATCH`: Entity fell within the active percentage rollout. +- `PERCENTAGE_ROLLOUT_MISMATCH`: Entity fell outside the active percentage rollout. +- `ENVIRONMENT_DEFAULT`: Default state for the current environment. +- `DEFAULT_FALLBACK`: Unrecognized flag evaluated (fail-closed to false). + +### Prometheus Metrics + +- `feature_flag_evaluations_total{flag, decision, enabled}`: Counter tracking evaluations. +- `feature_flag_rollbacks_total{flag, operator}`: Counter tracking rollback invocations. +- `feature_flag_rollout_percentage{flag}`: Gauge reflecting active rollout percentage (0-100). + +--- + +## 6. Admin API Reference + +All endpoints are mounted under `/api/v1/admin/feature-flags` and require admin authentication: + +- `flags:read` scope: Read-only access (`GET`). +- `flags:write` scope: Rollout and rollback mutation access (`POST`, `PUT`). +- `super` scope: Inherits all permissions. + +| Method | Endpoint | Required Scope | Description | +| :--- | :--- | :--- | :--- | +| `GET` | `/feature-flags` | `flags:read` | List all flags, environment baseline, and evaluation status. | +| `GET` | `/feature-flags/history` | `flags:read` | List emergency rollback history and audit log. | +| `GET` | `/feature-flags/:key` | `flags:read` | Inspect single flag configuration and evaluation. | +| `PUT` | `/feature-flags/:key` | `flags:write` | Set runtime overrides (enabled, rolloutPercentage, allowlist, blocklist). | +| `POST` | `/feature-flags/:key/staged-rollout` | `flags:write` | Set staged rollout percentage (0 - 100). | +| `POST` | `/feature-flags/:key/rollback` | `flags:write` | Trigger fast emergency rollback with required reason. | +| `POST` | `/feature-flags/:key/reset` | `flags:write` | Reset flag back to default environment baseline. | +| `POST` | `/feature-flags/emergency-disable-all` | `flags:write` | Activate platform-wide emergency maintenance mode. | + +--- + +## 7. Developer Integration + +### Protecting Routes via Middleware + +```typescript +import { requireFeatureFlag } from '../middleware/featureFlags' + +// Protect an entire route or endpoint +router.post( + '/rebalance/execute', + requireFeatureFlag('agent_autonomous_rebalance', { + customMessage: 'Autonomous portfolio rebalancing is temporarily disabled', + }), + rebalanceHandler +) +``` + +### Checking Flags in Business Logic + +```typescript +import { isFeatureEnabled, evaluateFeatureFlag } from '../config/featureFlags' + +// Simple boolean check +if (isFeatureEnabled('smart_dca_engine', req.user?.id)) { + await executeSmartDca(order) +} else { + await executeStandardDca(order) +} + +// Detailed inspection +const evaluation = evaluateFeatureFlag('fast_fiat_onramp', { + entityId: req.user?.id, + environment: 'production', +}) +logger.info(`Flag evaluated: ${evaluation.enabled} via ${evaluation.decision}`) +``` diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index 42ee8783..f7546a0b 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -568,6 +568,76 @@ psql "$DATABASE_URL" -c "SELECT scope, \"scopeKey\", state, \"trippedRule\" FROM --- +## 10. Feature Flags & Emergency Rollback Runbook (#494) + +Detailed reference and architectural specifications are documented in [docs/FEATURE_FLAGS.md](FEATURE_FLAGS.md). + +### Inspect Active Feature Flags + +```bash +# List all feature flags and runtime evaluation for the current environment +curl -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + http://localhost:3001/api/v1/admin/feature-flags | jq + +# Inspect single flag +curl -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + http://localhost:3001/api/v1/admin/feature-flags/smart_dca_engine | jq +``` + +### Staged Rollout Adjustment + +```bash +# Advance rollout percentage (0 to 100) +curl -X POST http://localhost:3001/api/v1/admin/feature-flags/smart_dca_engine/staged-rollout \ + -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"percentage": 25}' +``` + +### Fast Emergency Rollback (< 1s) + +When an anomaly or regression is detected, trigger an immediate rollback. This clamps the rollout to 0%, engages the rollback lock, increments the Prometheus rollback metric, logs structured SRE audit events, and broadcasts the event across Redis: + +```bash +# Fast rollback with mandatory reason +curl -X POST http://localhost:3001/api/v1/admin/feature-flags/smart_dca_engine/rollback \ + -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"reason": "High slippage observed in Horizon liquidity pools"}' +``` + +### Platform-Wide Emergency Maintenance Mode + +To halt all state-mutating requests platform-wide during an active exploit or catastrophic dependency failure: + +```bash +curl -X POST http://localhost:3001/api/v1/admin/feature-flags/emergency-disable-all \ + -H "Authorization: Bearer $ADMIN_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"reason": "Halting mutating requests due to protocol exploit"}' +``` + +### Environment Variable Emergency Disable + +If the admin endpoint is unreachable, override in the environment without deploying new code: + +```bash +# Disable specific flags via rollback +FEATURE_FLAGS_ROLLBACK=smart_dca_engine,fast_fiat_onramp + +# Or engage individual kill switch +FEATURE_FLAG_SMART_DCA_ENGINE_KILL_SWITCH=true +``` + +### Reset to Default Environment Configuration + +```bash +curl -X POST http://localhost:3001/api/v1/admin/feature-flags/smart_dca_engine/reset \ + -H "Authorization: Bearer $ADMIN_API_TOKEN" +``` + +--- + ## 13. Telegram & WhatsApp Message Delivery Recovery (#493) ### Alert: `telegram_dlq_high` / `whatsapp_dlq_high` diff --git a/docs/openapi-drift-baseline.json b/docs/openapi-drift-baseline.json index ac7825ba..cbd88a73 100644 --- a/docs/openapi-drift-baseline.json +++ b/docs/openapi-drift-baseline.json @@ -12,6 +12,9 @@ "GET /api/v1/admin/audit/prove", "GET /api/v1/admin/audit/verify", "GET /api/v1/admin/dlq/inspect", + "GET /api/v1/admin/feature-flags", + "GET /api/v1/admin/feature-flags/history", + "GET /api/v1/admin/feature-flags/{key}", "GET /api/v1/admin/fiat/providers", "GET /api/v1/admin/jobs/dashboard", "GET /api/v1/admin/keys", @@ -93,6 +96,10 @@ "POST /api/v1/admin/dlq/resolve", "POST /api/v1/admin/dlq/retry", "POST /api/v1/admin/erasure", + "POST /api/v1/admin/feature-flags/emergency-disable-all", + "POST /api/v1/admin/feature-flags/{key}/reset", + "POST /api/v1/admin/feature-flags/{key}/rollback", + "POST /api/v1/admin/feature-flags/{key}/staged-rollout", "POST /api/v1/admin/fiat/providers/{name}/failover", "POST /api/v1/admin/keys", "POST /api/v1/admin/keys/{id}/rotate", @@ -154,6 +161,7 @@ "POST /api/v1/webauthn/register-verify", "POST /api/v1/whatsapp/webhook", "POST /api/v1/withdraw", + "PUT /api/v1/admin/feature-flags/{key}", "PUT /api/v1/approval-policies/{id}", "PUT /api/v1/settings" ], diff --git a/scripts/check-openapi-drift.js b/scripts/check-openapi-drift.js index 9e481362..2ad5d415 100644 --- a/scripts/check-openapi-drift.js +++ b/scripts/check-openapi-drift.js @@ -85,13 +85,18 @@ function routesOf(file, prefix, seen = new Set()) { } function mountedRoutes() { - const indexFile = path.join(SRC, 'index.ts') - const source = read(indexFile) - const imports = importMap(indexFile, source) + const registryFile = path.join(SRC, 'routes', 'registry.ts') + const targetFile = fs.existsSync(registryFile) + ? registryFile + : path.join(SRC, 'index.ts') + const source = read(targetFile) + const imports = importMap(targetFile, source) const all = new Set() + const block = + source.match(/export const API_ROUTES[\s\S]*?\n\]/)?.[0] || source const re = /path:\s*'([^']+)',\s*handlers:\s*\[([^\]]*)\]/g let m - while ((m = re.exec(source))) { + while ((m = re.exec(block))) { const ids = m[2].match(/\w+/g) || [] const router = ids[ids.length - 1] if (!imports[router]) continue diff --git a/scripts/check-production-audit.js b/scripts/check-production-audit.js index 61a39847..77620bc5 100644 --- a/scripts/check-production-audit.js +++ b/scripts/check-production-audit.js @@ -52,6 +52,7 @@ function main() { const audit = spawnSync('npm', ['audit', '--omit=dev', '--json'], { cwd: root, encoding: 'utf8', + shell: true, maxBuffer: 10 * 1024 * 1024, }) if (!audit.stdout) throw new Error(audit.stderr || 'npm audit produced no JSON') diff --git a/src/config/env.ts b/src/config/env.ts index d10c3fc6..e163fcad 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -994,4 +994,13 @@ export const config = { process.env.NLP_CONFIDENCE_THRESHOLD || '0.6' ), }, + featureFlags: { + redisSyncIntervalMs: parseInt( + process.env.FEATURE_FLAGS_REDIS_SYNC_INTERVAL_MS || '10000' + ), + emergencyMaintenance: + ( + process.env.FEATURE_FLAG_EMERGENCY_MAINTENANCE_MODE ?? 'false' + ).toLowerCase() === 'true', + }, } diff --git a/src/config/featureFlags.ts b/src/config/featureFlags.ts new file mode 100644 index 00000000..875e4d86 --- /dev/null +++ b/src/config/featureFlags.ts @@ -0,0 +1,897 @@ +/** + * Central Feature Flag Management & Staged Rollout / Rollback System (#494) + * + * Provides a robust, environment-aware feature flag architecture supporting: + * - Deterministic staged rollouts (0% -> 10% -> 25% -> 50% -> 100%) via cryptographic entity hashing + * - Environment awareness (development, test, staging, production) with environment overrides + * - Granular allowlists and blocklists + * - Fast emergency rollback and kill switches with audit logging + * - Distributed state synchronization via Redis (when available) + * - Evaluation decision logging with structured Winston context & Prometheus metrics + */ + +import crypto from 'node:crypto' +import { logger } from '../utils/logger' +import { + recordFeatureFlagEvaluation, + recordFeatureFlagRollback, + setFeatureFlagRolloutGauge, +} from '../utils/metrics' +import { cacheGet, cacheSet, getRedisClient } from './redis' + +export type Environment = 'development' | 'test' | 'staging' | 'production' + +export type FlagEvaluationDecision = + | 'GLOBAL_MAINTENANCE' + | 'KILL_SWITCH' + | 'EMERGENCY_ROLLBACK' + | 'BLOCKLIST' + | 'ALLOWLIST' + | 'RUNTIME_OVERRIDE' + | 'ENV_VAR_OVERRIDE' + | 'PERCENTAGE_ROLLOUT_MATCH' + | 'PERCENTAGE_ROLLOUT_MISMATCH' + | 'ENVIRONMENT_DEFAULT' + | 'DEFAULT_FALLBACK' + +export interface FlagEvaluationContext { + entityId?: string // Deterministic hash seed (e.g. userId, ip, walletAddress) + environment?: Environment + attributes?: Record +} + +export interface FlagEvaluationResult { + flag: string + enabled: boolean + decision: FlagEvaluationDecision + environment: Environment + entityId?: string + rolloutPercentage: number + reason: string + timestamp: string +} + +export interface EnvironmentRule { + enabled: boolean + rolloutPercentage?: number // 0 to 100 + allowlist?: string[] + blocklist?: string[] +} + +export interface FeatureFlagDefinition { + key: string + description: string + environments: Record + allowlist?: string[] + blocklist?: string[] + killSwitch?: boolean + rolledBack?: boolean + rollbackReason?: string + rollbackTimestamp?: string + rollbackOperator?: string +} + +export interface FeatureFlagOverride { + enabled?: boolean + rolloutPercentage?: number + allowlist?: string[] + blocklist?: string[] + killSwitch?: boolean + rolledBack?: boolean + rollbackReason?: string + updatedAt: string + updatedBy?: string +} + +export interface RollbackEvent { + flagKey: string + reason: string + operator: string + timestamp: string + previousState: { + enabled?: boolean + rolloutPercentage?: number + } +} + +/** + * Normalizes and resolves the current runtime environment. + */ +export function getCurrentEnvironment(): Environment { + const env = (process.env.NODE_ENV || 'development').toLowerCase() + if (env === 'production' || env === 'prod') return 'production' + if (env === 'staging' || env === 'stage') return 'staging' + if (env === 'test') return 'test' + return 'development' +} + +/** + * Deterministically computes a percentile bucket (0-99) for a given entity and flag. + * Uses SHA-256 to ensure uniform, monotonic distribution across pods and restarts. + */ +export function hashEntityToPercentage( + flagKey: string, + entityId: string +): number { + if (!entityId || typeof entityId !== 'string') return 0 + const hash = crypto + .createHash('sha256') + .update(`${flagKey}:${entityId.trim()}`) + .digest('hex') + const intVal = parseInt(hash.substring(0, 8), 16) + return intVal % 100 +} + +/** + * Core catalog of built-in system feature flags. + */ +export const DEFAULT_FEATURE_FLAGS: Record = { + agent_autonomous_rebalance: { + key: 'agent_autonomous_rebalance', + description: 'Enables automatic agent-driven portfolio rebalancing', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 50 }, + production: { enabled: false, rolloutPercentage: 0 }, + }, + }, + smart_dca_engine: { + key: 'smart_dca_engine', + description: 'Volatility-aware adaptive recurring deposits engine', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 25 }, + production: { enabled: false, rolloutPercentage: 0 }, + }, + }, + enhanced_stellar_routing: { + key: 'enhanced_stellar_routing', + description: 'Horizon liquidity path finding and smart fee routing', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 100 }, + production: { enabled: true, rolloutPercentage: 10 }, + }, + }, + ai_assistant_tool_calling: { + key: 'ai_assistant_tool_calling', + description: 'Conversational assistant with autonomous tool invocation', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 50 }, + production: { enabled: false, rolloutPercentage: 0 }, + }, + }, + circuit_breaker_v2: { + key: 'circuit_breaker_v2', + description: 'Dynamic volatility and drawdown circuit breaker trip rules', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 100 }, + production: { enabled: true, rolloutPercentage: 100 }, + }, + }, + fast_fiat_onramp: { + key: 'fast_fiat_onramp', + description: 'Direct instant on-ramp quoting and locking', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 100 }, + production: { enabled: false, rolloutPercentage: 0 }, + }, + }, + emergency_maintenance_mode: { + key: 'emergency_maintenance_mode', + description: + 'Emergency kill-switch to halt state-mutating requests platform-wide', + environments: { + development: { enabled: false, rolloutPercentage: 0 }, + test: { enabled: false, rolloutPercentage: 0 }, + staging: { enabled: false, rolloutPercentage: 0 }, + production: { enabled: false, rolloutPercentage: 0 }, + }, + }, +} + +export type FeatureFlagKey = + | keyof typeof DEFAULT_FEATURE_FLAGS + | (string & {}) + +const REDIS_OVERRIDES_KEY = 'neurowealth:feature_flags:overrides' +const REDIS_UPDATES_CHANNEL = 'neurowealth:feature_flags:updates' + +/** + * FeatureFlagManager: central orchestrator for evaluating, overriding, + * rolling out, and rolling back feature flags. + */ +export class FeatureFlagManager { + private flags: Map = new Map() + private overrides: Map = new Map() + private rollbackHistory: RollbackEvent[] = [] + private redisSyncSubscribed = false + + constructor(initialFlags: Record = DEFAULT_FEATURE_FLAGS) { + for (const [key, def] of Object.entries(initialFlags)) { + this.flags.set(key, { ...def }) + } + } + + /** + * Register a custom feature flag definition into the manager. + */ + public registerFlag(flag: FeatureFlagDefinition): void { + this.flags.set(flag.key, { ...flag }) + logger.info(`[FeatureFlag] Registered flag "${flag.key}"`, { + flagKey: flag.key, + description: flag.description, + }) + } + + /** + * Retrieve a feature flag definition by key. + */ + public getFlag(flagKey: string): FeatureFlagDefinition | undefined { + return this.flags.get(flagKey) + } + + /** + * Evaluates a feature flag against the current environment and evaluation context. + * Logs every evaluation decision with structured metadata. + */ + public evaluate( + flagKey: string, + contextOrEntityId?: FlagEvaluationContext | string + ): FlagEvaluationResult { + const context: FlagEvaluationContext | undefined = + typeof contextOrEntityId === 'string' + ? { entityId: contextOrEntityId } + : contextOrEntityId + + const env = context?.environment || getCurrentEnvironment() + const entityId = context?.entityId + const now = new Date().toISOString() + const def = this.flags.get(flagKey) + + // Helper to log decision, update metrics, and return structured result + const makeDecision = ( + enabled: boolean, + decision: FlagEvaluationDecision, + reason: string, + rolloutPct: number = enabled ? 100 : 0 + ): FlagEvaluationResult => { + const result: FlagEvaluationResult = { + flag: flagKey, + enabled, + decision, + environment: env, + entityId, + rolloutPercentage: rolloutPct, + reason, + timestamp: now, + } + + // Acceptance criterion: Logging captures flag evaluation decisions + logger.info( + `[FeatureFlag] Evaluated flag "${flagKey}": ${enabled ? 'ENABLED' : 'DISABLED'} (${decision})`, + { + event: 'feature_flag_evaluation', + flag: flagKey, + enabled, + decision, + reason, + environment: env, + entityId: entityId || null, + rolloutPercentage: rolloutPct, + } + ) + + recordFeatureFlagEvaluation(flagKey, decision, enabled) + setFeatureFlagRolloutGauge(flagKey, rolloutPct) + + return result + } + + // ── 1. Global Maintenance Kill Switch ───────────────────────────────────── + if (flagKey !== 'emergency_maintenance_mode') { + const globalMaintenance = this.evaluate('emergency_maintenance_mode', { + environment: env, + }) + if (globalMaintenance.enabled) { + return makeDecision( + false, + 'GLOBAL_MAINTENANCE', + 'Global emergency maintenance mode is active', + 0 + ) + } + } + + // ── 2. Flag-specific Kill Switch ───────────────────────────────────────── + if (def?.killSwitch) { + return makeDecision( + false, + 'KILL_SWITCH', + 'Flag-level kill switch is actively engaged', + 0 + ) + } + + // ── 3. Emergency Rollback Check ─────────────────────────────────────────── + if (def?.rolledBack) { + return makeDecision( + false, + 'EMERGENCY_ROLLBACK', + `Flag rolled back: ${def.rollbackReason || 'Emergency rollback triggered'}`, + 0 + ) + } + + // Check emergency rollback environment variable: FEATURE_FLAGS_ROLLBACK + const rollbackEnv = process.env.FEATURE_FLAGS_ROLLBACK + if (rollbackEnv) { + const rolledBackList = rollbackEnv + .split(',') + .map((s) => s.trim().toLowerCase()) + if (rolledBackList.includes(flagKey.toLowerCase())) { + return makeDecision( + false, + 'EMERGENCY_ROLLBACK', + 'Flag disabled via FEATURE_FLAGS_ROLLBACK environment variable', + 0 + ) + } + } + + // ── 4. Entity Blocklist Check ───────────────────────────────────────────── + if (entityId && def?.blocklist && def.blocklist.includes(entityId)) { + return makeDecision( + false, + 'BLOCKLIST', + `Entity "${entityId}" is explicitly on the blocklist`, + 0 + ) + } + + const envRule = def?.environments[env] + if (entityId && envRule?.blocklist && envRule.blocklist.includes(entityId)) { + return makeDecision( + false, + 'BLOCKLIST', + `Entity "${entityId}" is on the environment blocklist`, + 0 + ) + } + + // ── 5. Entity Allowlist Check ───────────────────────────────────────────── + if (entityId && def?.allowlist && def.allowlist.includes(entityId)) { + return makeDecision( + true, + 'ALLOWLIST', + `Entity "${entityId}" is explicitly on the global allowlist`, + 100 + ) + } + + if (entityId && envRule?.allowlist && envRule.allowlist.includes(entityId)) { + return makeDecision( + true, + 'ALLOWLIST', + `Entity "${entityId}" is on the environment allowlist`, + 100 + ) + } + + // ── 6. Runtime Overrides (e.g. from Admin API or Redis) ─────────────────── + const override = this.overrides.get(flagKey) + if (override) { + if (override.killSwitch || override.rolledBack) { + return makeDecision( + false, + override.rolledBack ? 'EMERGENCY_ROLLBACK' : 'KILL_SWITCH', + override.rollbackReason || 'Runtime override kill switch engaged', + 0 + ) + } + + if (override.allowlist && entityId && override.allowlist.includes(entityId)) { + return makeDecision( + true, + 'ALLOWLIST', + `Entity "${entityId}" is on the runtime override allowlist`, + 100 + ) + } + + if (override.blocklist && entityId && override.blocklist.includes(entityId)) { + return makeDecision( + false, + 'BLOCKLIST', + `Entity "${entityId}" is on the runtime override blocklist`, + 0 + ) + } + + if (override.rolloutPercentage !== undefined) { + const pct = Math.max(0, Math.min(100, override.rolloutPercentage)) + if (entityId) { + const bucket = hashEntityToPercentage(flagKey, entityId) + const matched = bucket < pct + return makeDecision( + matched, + matched + ? 'PERCENTAGE_ROLLOUT_MATCH' + : 'PERCENTAGE_ROLLOUT_MISMATCH', + `Runtime staged rollout (${pct}%) for entity bucket ${bucket}`, + pct + ) + } + return makeDecision( + pct > 0, + 'RUNTIME_OVERRIDE', + `Runtime override staged rollout at ${pct}%`, + pct + ) + } + + if (override.enabled !== undefined) { + return makeDecision( + override.enabled, + 'RUNTIME_OVERRIDE', + `Runtime override explicitly set to ${override.enabled}`, + override.enabled ? 100 : 0 + ) + } + } + + // ── 7. Environment Variable Overrides ───────────────────────────────────── + const envVarKey = `FEATURE_FLAG_${flagKey.toUpperCase()}` + const envKillSwitch = process.env[`${envVarKey}_KILL_SWITCH`] + if (envKillSwitch && envKillSwitch.toLowerCase() === 'true') { + return makeDecision( + false, + 'KILL_SWITCH', + `Kill switch engaged via ${envVarKey}_KILL_SWITCH`, + 0 + ) + } + + const envPctStr = process.env[`${envVarKey}_PERCENTAGE`] + if (envPctStr !== undefined && !isNaN(Number(envPctStr))) { + const pct = Math.max(0, Math.min(100, Number(envPctStr))) + if (entityId) { + const bucket = hashEntityToPercentage(flagKey, entityId) + const matched = bucket < pct + return makeDecision( + matched, + matched ? 'PERCENTAGE_ROLLOUT_MATCH' : 'PERCENTAGE_ROLLOUT_MISMATCH', + `Env var ${envVarKey}_PERCENTAGE (${pct}%) evaluated entity bucket ${bucket}`, + pct + ) + } + return makeDecision( + pct > 0, + 'ENV_VAR_OVERRIDE', + `Env var ${envVarKey}_PERCENTAGE set to ${pct}%`, + pct + ) + } + + const envVal = process.env[envVarKey] + if (envVal !== undefined) { + const parsedBool = envVal.toLowerCase() === 'true' || envVal === '1' + return makeDecision( + parsedBool, + 'ENV_VAR_OVERRIDE', + `Env var ${envVarKey} explicitly set to ${parsedBool}`, + parsedBool ? 100 : 0 + ) + } + + // ── 8. Environment-Aware Configuration ─────────────────────────────────── + if (envRule) { + const pct = + envRule.rolloutPercentage !== undefined + ? Math.max(0, Math.min(100, envRule.rolloutPercentage)) + : envRule.enabled + ? 100 + : 0 + + // If entityId is supplied and rolloutPercentage is partial (1-99%) + if (entityId && pct > 0 && pct < 100) { + const bucket = hashEntityToPercentage(flagKey, entityId) + const matched = bucket < pct + return makeDecision( + matched, + matched ? 'PERCENTAGE_ROLLOUT_MATCH' : 'PERCENTAGE_ROLLOUT_MISMATCH', + `Environment "${env}" staged rollout (${pct}%) evaluated entity bucket ${bucket}`, + pct + ) + } + + // If rolloutPercentage is 100% or 0% + if (pct === 100) { + return makeDecision( + true, + 'ENVIRONMENT_DEFAULT', + `Environment "${env}" configuration enabled at 100%`, + 100 + ) + } + if (pct === 0) { + return makeDecision( + false, + 'ENVIRONMENT_DEFAULT', + `Environment "${env}" configuration disabled at 0%`, + 0 + ) + } + + // Otherwise fall back to envRule.enabled + return makeDecision( + envRule.enabled, + 'ENVIRONMENT_DEFAULT', + `Environment "${env}" default enabled=${envRule.enabled}`, + envRule.enabled ? pct : 0 + ) + } + + // ── 9. Unknown / Unregistered Flag Fallback ────────────────────────────── + return makeDecision( + false, + 'DEFAULT_FALLBACK', + `Flag "${flagKey}" not registered; falling back closed to false`, + 0 + ) + } + + /** + * Convenience boolean evaluator. + */ + public isEnabled( + flagKey: string, + contextOrEntityId?: FlagEvaluationContext | string + ): boolean { + return this.evaluate(flagKey, contextOrEntityId).enabled + } + + /** + * Sets a dynamic runtime override for a feature flag. + */ + public setOverride( + flagKey: string, + override: Partial, + operator: string = 'system' + ): void { + const existing = this.overrides.get(flagKey) || { + updatedAt: new Date().toISOString(), + } + + const merged: FeatureFlagOverride = { + ...existing, + ...override, + updatedAt: new Date().toISOString(), + updatedBy: operator, + } + + this.overrides.set(flagKey, merged) + + logger.warn(`[FeatureFlag] Runtime override set for "${flagKey}"`, { + flagKey, + override: merged, + operator, + }) + + this.syncOverrideToRedis(flagKey, merged) + } + + /** + * Adjusts the rollout percentage for staged release (0% -> 10% -> 25% -> 50% -> 100%). + */ + public stagedRollout( + flagKey: string, + percentage: number, + operator: string = 'system' + ): void { + const clamped = Math.max(0, Math.min(100, percentage)) + this.setOverride( + flagKey, + { + rolloutPercentage: clamped, + enabled: clamped > 0, + rolledBack: false, + }, + operator + ) + + logger.warn( + `[FeatureFlag] Staged rollout updated for "${flagKey}" to ${clamped}% by ${operator}`, + { + flagKey, + rolloutPercentage: clamped, + operator, + } + ) + } + + /** + * Emergency Rollback (#494): + * Immediately disables a feature flag, drops rollout percentage to 0, + * engages rollback state, records audit log and Prometheus rollback metric. + */ + public rollback( + flagKey: string, + reason: string, + operator: string = 'system' + ): RollbackEvent { + const def = this.flags.get(flagKey) + const currentOverride = this.overrides.get(flagKey) + const now = new Date().toISOString() + + const previousState = { + enabled: currentOverride?.enabled ?? def?.environments[getCurrentEnvironment()]?.enabled, + rolloutPercentage: + currentOverride?.rolloutPercentage ?? + def?.environments[getCurrentEnvironment()]?.rolloutPercentage, + } + + // Set definition rollback state + if (def) { + def.rolledBack = true + def.rollbackReason = reason + def.rollbackTimestamp = now + def.rollbackOperator = operator + } + + // Set runtime override to ensure 0% rollout and disabled state across all checks + this.overrides.set(flagKey, { + enabled: false, + rolloutPercentage: 0, + rolledBack: true, + rollbackReason: reason, + updatedAt: now, + updatedBy: operator, + }) + + const event: RollbackEvent = { + flagKey, + reason, + operator, + timestamp: now, + previousState, + } + + this.rollbackHistory.push(event) + + // Acceptance criterion: Logging captures flag evaluation decisions and rollbacks + logger.error( + `[FeatureFlag] EMERGENCY ROLLBACK TRIGGERED for "${flagKey}" by ${operator}: ${reason}`, + { + event: 'feature_flag_rollback', + flagKey, + reason, + operator, + previousState, + timestamp: now, + } + ) + + recordFeatureFlagRollback(flagKey, operator) + setFeatureFlagRolloutGauge(flagKey, 0) + + this.syncOverrideToRedis(flagKey, { + enabled: false, + rolloutPercentage: 0, + rolledBack: true, + rollbackReason: reason, + updatedAt: now, + updatedBy: operator, + }) + + return event + } + + /** + * Engages an emergency kill switch on a flag. + */ + public emergencyDisable( + flagKey: string, + reason: string, + operator: string = 'system' + ): void { + this.rollback(flagKey, reason, operator) + } + + /** + * Platform-wide emergency disablement. + */ + public emergencyDisableAll( + reason: string, + operator: string = 'system' + ): void { + logger.error( + `[FeatureFlag] PLATFORM EMERGENCY DISABLE ALL TRIGGERED by ${operator}: ${reason}`, + { + event: 'feature_flag_disable_all', + reason, + operator, + } + ) + this.setOverride( + 'emergency_maintenance_mode', + { + enabled: true, + rolloutPercentage: 100, + }, + operator + ) + } + + /** + * Clears overrides and resets a feature flag back to its default environment configuration. + */ + public reset(flagKey: string, operator: string = 'system'): void { + const def = this.flags.get(flagKey) + if (def) { + def.rolledBack = false + delete def.rollbackReason + delete def.rollbackTimestamp + delete def.rollbackOperator + } + this.overrides.delete(flagKey) + + logger.info( + `[FeatureFlag] Reset flag "${flagKey}" to default environment config by ${operator}`, + { + flagKey, + operator, + } + ) + + this.removeOverrideFromRedis(flagKey) + } + + /** + * Resets all runtime overrides (primarily for test suite hermeticity). + */ + public resetAll(): void { + for (const [key, def] of this.flags.entries()) { + def.rolledBack = false + delete def.rollbackReason + delete def.rollbackTimestamp + delete def.rollbackOperator + } + this.overrides.clear() + this.rollbackHistory = [] + } + + /** + * Retrieves all registered feature flags with current evaluation status. + */ + public getAllFlags( + context?: FlagEvaluationContext + ): Array { + return Array.from(this.flags.values()).map((def) => ({ + ...def, + currentEvaluation: this.evaluate(def.key, context), + })) + } + + /** + * Returns recorded rollback history. + */ + public getRollbackHistory(): RollbackEvent[] { + return [...this.rollbackHistory] + } + + // ── Redis Distributed Sync (#494) ────────────────────────────────────────── + + private async syncOverrideToRedis( + flagKey: string, + override: FeatureFlagOverride + ): Promise { + try { + const client = getRedisClient() + if (!client) return + + await cacheSet(`${REDIS_OVERRIDES_KEY}:${flagKey}`, override, 86400 * 7) + await client.publish( + REDIS_UPDATES_CHANNEL, + JSON.stringify({ flagKey, override }) + ) + } catch (err) { + logger.warn('[FeatureFlag] Failed to sync override to Redis', { + flagKey, + error: err instanceof Error ? err.message : String(err), + }) + } + } + + private async removeOverrideFromRedis(flagKey: string): Promise { + try { + const client = getRedisClient() + if (!client) return + + await client.del(`${REDIS_OVERRIDES_KEY}:${flagKey}`) + await client.publish( + REDIS_UPDATES_CHANNEL, + JSON.stringify({ flagKey, reset: true }) + ) + } catch (err) { + logger.warn('[FeatureFlag] Failed to delete override from Redis', { + flagKey, + error: err instanceof Error ? err.message : String(err), + }) + } + } + + /** + * Initial bootstrap from Redis cache to pick up any active overrides across pods. + */ + public async syncFromRedis(): Promise { + const client = getRedisClient() + if (!client) return + + try { + for (const flagKey of this.flags.keys()) { + const cached = await cacheGet( + `${REDIS_OVERRIDES_KEY}:${flagKey}` + ) + if (cached) { + this.overrides.set(flagKey, cached) + } + } + + if (!this.redisSyncSubscribed) { + this.redisSyncSubscribed = true + const sub = client.duplicate() + await sub.subscribe(REDIS_UPDATES_CHANNEL) + sub.on('message', (_channel, message) => { + try { + const data = JSON.parse(message) + if (data.reset && data.flagKey) { + this.overrides.delete(data.flagKey) + } else if (data.flagKey && data.override) { + this.overrides.set(data.flagKey, data.override) + } + } catch { + // Ignore parse errors from invalid Redis pubsub payloads + } + }) + } + } catch (err) { + logger.warn('[FeatureFlag] Failed to initialize Redis sync', { + error: err instanceof Error ? err.message : String(err), + }) + } + } +} + +// Global Singleton Instance +export const featureFlagManager = new FeatureFlagManager() + +/** + * Convenience helper to evaluate a feature flag. + * Supports string entityId or full FlagEvaluationContext. + */ +export function isFeatureEnabled( + flagKey: string, + contextOrEntityId?: FlagEvaluationContext | string +): boolean { + return featureFlagManager.isEnabled(flagKey, contextOrEntityId) +} + +/** + * Convenience helper to evaluate and inspect a feature flag result. + */ +export function evaluateFeatureFlag( + flagKey: string, + contextOrEntityId?: FlagEvaluationContext | string +): FlagEvaluationResult { + return featureFlagManager.evaluate(flagKey, contextOrEntityId) +} diff --git a/src/config/index.ts b/src/config/index.ts index 0deff500..291c8033 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -6,3 +6,5 @@ export { createSecretsProvider, } from './secrets' export type { SecretsProvider, SecretKey } from './secrets' +export * from './featureFlags' + diff --git a/src/index.ts b/src/index.ts index e3212c7b..4ec0819e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -44,6 +44,11 @@ import { securityHeaders, permissionsPolicy, } from './middleware/security' +import { + featureFlagMiddleware, + emergencyMaintenanceGuard, +} from './middleware/featureFlags' +import { featureFlagManager } from './config/featureFlags' import { logger } from './utils/logger' import { startAgentLoop, stopAgentLoop } from './agent/loop' import { connectDb } from './db' @@ -193,6 +198,7 @@ app.use((req: Request & { user?: { id: string } }, _res: Response, next) => { }) app.use(requestLogger) +app.use(featureFlagMiddleware) app.use(trustedIpBypass) // The app-wide rate-limit chain: the flat global budget, then the identity // tier. Both apply to every request, so no route is ever unlimited. Kept in @@ -200,6 +206,7 @@ app.use(trustedIpBypass) // that claim against the real chain. app.use(...APP_LEVEL_LIMITERS) app.use(requestTimeoutMiddleware) +app.use(emergencyMaintenanceGuard) // Advertise the served API version on every response — must be registered // before the first route (including the health probes below). @@ -615,6 +622,19 @@ async function initServices(): Promise { } ) } + + // 5. Feature Flags Redis Sync (#494) — best-effort, never blocks startup + try { + await featureFlagManager.syncFromRedis() + logger.info('[Startup] Feature flag Redis sync initialized ✓') + } catch (error) { + logger.warn( + '[Startup] Feature flag Redis sync failed — continuing with local configuration', + { + error: error instanceof Error ? error.message : String(error), + } + ) + } } async function main(): Promise { diff --git a/src/messaging/index.ts b/src/messaging/index.ts index 0d4fe662..d685384d 100644 --- a/src/messaging/index.ts +++ b/src/messaging/index.ts @@ -1,5 +1,2 @@ export * from './types' -export { - MessageDeliveryService, - messageDeliveryService, -} from './service' +export { MessageDeliveryService, messageDeliveryService } from './service' diff --git a/src/messaging/service.ts b/src/messaging/service.ts index 380383fd..a949fbad 100644 --- a/src/messaging/service.ts +++ b/src/messaging/service.ts @@ -44,8 +44,8 @@ function shouldQueryDb(): boolean { if (!md) return false return Boolean( md.create?._isMockFunction || - typeof md.create?.mockResolvedValue === 'function' || - process.env.USE_REAL_DB_FOR_TESTS === 'true' + typeof md.create?.mockResolvedValue === 'function' || + process.env.USE_REAL_DB_FOR_TESTS === 'true' ) } return Boolean((db as any)?.messageDelivery) @@ -107,10 +107,13 @@ export class MessageDeliveryService { }, }) } catch (error) { - logger.warn('[Messaging] Could not write message delivery to DB, retained in memory', { - id: record.id, - error: error instanceof Error ? error.message : String(error), - }) + logger.warn( + '[Messaging] Could not write message delivery to DB, retained in memory', + { + id: record.id, + error: error instanceof Error ? error.message : String(error), + } + ) } } @@ -144,10 +147,13 @@ export class MessageDeliveryService { }, }) } catch (error) { - logger.warn('[Messaging] Could not update message delivery in DB, updated in memory', { - id: record.id, - error: error instanceof Error ? error.message : String(error), - }) + logger.warn( + '[Messaging] Could not update message delivery in DB, updated in memory', + { + id: record.id, + error: error instanceof Error ? error.message : String(error), + } + ) } } @@ -371,11 +377,14 @@ export class MessageDeliveryService { record.category ) - logger.info('[Messaging] Message successfully delivered via fallback', { - id: record.id, - fallbackChannel: fallback.channel, - recipient: sanitizeRecipient(fallback.recipient), - }) + logger.info( + '[Messaging] Message successfully delivered via fallback', + { + id: record.id, + fallbackChannel: fallback.channel, + recipient: sanitizeRecipient(fallback.recipient), + } + ) return record } catch (fallbackError) { @@ -565,10 +574,26 @@ export class MessageDeliveryService { */ public async refreshQueueMetrics(): Promise { const stats = await this.getMessageStats() - updateMessageQueueDepth('TELEGRAM', 'PENDING', stats.byChannel.telegram.pending) - updateMessageQueueDepth('TELEGRAM', 'DEAD_LETTER', stats.byChannel.telegram.deadLetter) - updateMessageQueueDepth('WHATSAPP', 'PENDING', stats.byChannel.whatsapp.pending) - updateMessageQueueDepth('WHATSAPP', 'DEAD_LETTER', stats.byChannel.whatsapp.deadLetter) + updateMessageQueueDepth( + 'TELEGRAM', + 'PENDING', + stats.byChannel.telegram.pending + ) + updateMessageQueueDepth( + 'TELEGRAM', + 'DEAD_LETTER', + stats.byChannel.telegram.deadLetter + ) + updateMessageQueueDepth( + 'WHATSAPP', + 'PENDING', + stats.byChannel.whatsapp.pending + ) + updateMessageQueueDepth( + 'WHATSAPP', + 'DEAD_LETTER', + stats.byChannel.whatsapp.deadLetter + ) } /** @@ -614,8 +639,7 @@ export class MessageDeliveryService { if (!deadLetters || deadLetters.length === 0) { deadLetters = Array.from(inMemoryStore.values()).filter( - (m) => - m.status === 'DEAD_LETTER' && (!channel || m.channel === channel) + (m) => m.status === 'DEAD_LETTER' && (!channel || m.channel === channel) ) } @@ -749,10 +773,12 @@ export class MessageDeliveryService { defaultStats.total += count if (row.status === 'PENDING') defaultStats.byStatus.pending += count - else if (row.status === 'SENDING') defaultStats.byStatus.sending += count + else if (row.status === 'SENDING') + defaultStats.byStatus.sending += count else if (row.status === 'DELIVERED') defaultStats.byStatus.delivered += count - else if (row.status === 'FAILED') defaultStats.byStatus.failed += count + else if (row.status === 'FAILED') + defaultStats.byStatus.failed += count else if (row.status === 'DEAD_LETTER') defaultStats.byStatus.deadLetter += count @@ -782,7 +808,8 @@ export class MessageDeliveryService { else if (msg.status === 'SENDING') defaultStats.byStatus.sending += 1 else if (msg.status === 'DELIVERED') defaultStats.byStatus.delivered += 1 else if (msg.status === 'FAILED') defaultStats.byStatus.failed += 1 - else if (msg.status === 'DEAD_LETTER') defaultStats.byStatus.deadLetter += 1 + else if (msg.status === 'DEAD_LETTER') + defaultStats.byStatus.deadLetter += 1 const ch = msg.channel === 'TELEGRAM' ? 'telegram' : 'whatsapp' defaultStats.byChannel[ch].total += 1 diff --git a/src/messaging/types.ts b/src/messaging/types.ts index fa11efff..4f4f502f 100644 --- a/src/messaging/types.ts +++ b/src/messaging/types.ts @@ -1,11 +1,7 @@ export type MessageChannel = 'TELEGRAM' | 'WHATSAPP' export type MessageDeliveryStatus = - | 'PENDING' - | 'SENDING' - | 'DELIVERED' - | 'FAILED' - | 'DEAD_LETTER' + 'PENDING' | 'SENDING' | 'DELIVERED' | 'FAILED' | 'DEAD_LETTER' export type MessagePriority = 'HIGH' | 'NORMAL' | 'LOW' diff --git a/src/middleware/adminAuth.ts b/src/middleware/adminAuth.ts index ccda95f0..d590a7c6 100644 --- a/src/middleware/adminAuth.ts +++ b/src/middleware/adminAuth.ts @@ -50,6 +50,9 @@ export const ADMIN_SCOPES = [ // #394 — GDPR/CCPA right-to-erasure 'erasure:write', 'erasure:read', + // #494 — feature flags management and staged rollout/rollback + 'flags:read', + 'flags:write', // #493 — Telegram and WhatsApp message delivery management & recovery 'messages:read', 'messages:write', @@ -294,6 +297,21 @@ export const ADMIN_SCOPE_CATALOG: Record = { 'separated so one operator cannot both authorise and carry out a ' + 'deletion.', }, + 'flags:read': { + scope: 'flags:read', + enforcement: 'enforced', + description: 'Read feature flags and rollout state.', + risk: 'read', + grants: 'Feature flags list, evaluation details and rollback history.', + }, + 'flags:write': { + scope: 'flags:write', + enforcement: 'enforced', + description: 'Manage feature flags, staged rollouts and emergency rollbacks.', + risk: 'operational', + grants: + 'Update runtime overrides, advance staged rollouts, reset flags, and trigger emergency rollbacks.', + }, 'messages:read': { scope: 'messages:read', enforcement: 'enforced', diff --git a/src/middleware/featureFlags.ts b/src/middleware/featureFlags.ts new file mode 100644 index 00000000..189edc53 --- /dev/null +++ b/src/middleware/featureFlags.ts @@ -0,0 +1,178 @@ +/** + * Express Middleware for Feature Flags & Staged Rollouts (#494) + * + * Provides route guarding, user-aware staged rollout checks, and emergency maintenance gating. + */ + +import { Request, Response, NextFunction } from 'express' +import { + isFeatureEnabled, + evaluateFeatureFlag, + FlagEvaluationResult, +} from '../config/featureFlags' +import { logger } from '../utils/logger' + +// Augment Express Request type to include feature flag helpers +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace Express { + interface Request { + isFeatureEnabled?: (flagKey: string) => boolean + evaluateFeatureFlag?: (flagKey: string) => FlagEvaluationResult + } + } +} + +export interface RequireFeatureFlagOptions { + /** + * HTTP status code to return when the feature is disabled (default: 503). + */ + status?: number + /** + * Custom error message returned in the JSON payload. + */ + message?: string + /** + * Alias for message. + */ + customMessage?: string + /** + * Custom extractor for entity ID (e.g. user ID, account ID) used for deterministic staged rollouts. + */ + entityExtractor?: (req: Request) => string | undefined + /** + * Custom fallback handler if the flag is disabled instead of terminating with a status response. + */ + fallback?: (req: Request, res: Response, next: NextFunction) => void +} + +/** + * Extracts a candidate entity identifier from the Express Request. + * Checks authenticated user id, auth context, or client IP. + */ +export function defaultEntityExtractor(req: Request): string | undefined { + const user = (req as any).user + if (user?.id) return String(user.id) + const auth = (req as any).auth + if (auth?.userId) return String(auth.userId) + const headerUserId = req.headers['x-user-id'] + if (typeof headerUserId === 'string' && headerUserId.trim()) { + return headerUserId.trim() + } + return req.ip || undefined +} + +/** + * Route middleware that checks whether a feature flag is enabled. + * If disabled, returns HTTP 503 (Service Unavailable) or executes a custom fallback handler. + */ +export function requireFeatureFlag( + flagKey: string, + options: RequireFeatureFlagOptions = {} +) { + const statusCode = options.status || 503 + const extractor = options.entityExtractor || defaultEntityExtractor + + return (req: Request, res: Response, next: NextFunction): void => { + const entityId = extractor(req) + const evaluation = evaluateFeatureFlag(flagKey, { entityId }) + + if (evaluation.enabled) { + next() + return + } + + if (options.fallback) { + options.fallback(req, res, next) + return + } + + const message = + options.customMessage || + options.message || + 'This feature is temporarily unavailable or undergoing staged rollout.' + + logger.warn( + `[FeatureFlagMiddleware] Request blocked by flag "${flagKey}" for ${req.method} ${req.path}`, + { + flagKey, + path: req.path, + method: req.method, + entityId: entityId || null, + decision: evaluation.decision, + reason: evaluation.reason, + } + ) + + res.status(statusCode).json({ + success: false, + error: message, + code: 'FEATURE_DISABLED', + flag: flagKey, + decision: evaluation.decision, + }) + } +} + +/** + * Middleware that attaches feature flag helper functions to `req`. + */ +export function featureFlagMiddleware( + req: Request, + _res: Response, + next: NextFunction +): void { + const entityId = defaultEntityExtractor(req) + + req.isFeatureEnabled = (flagKey: string): boolean => { + return isFeatureEnabled(flagKey, { entityId }) + } + + req.evaluateFeatureFlag = (flagKey: string): FlagEvaluationResult => { + return evaluateFeatureFlag(flagKey, { entityId }) + } + + next() +} + +/** + * Platform-wide emergency maintenance mode guard. + * Blocks all mutating (POST, PUT, PATCH, DELETE) operations when active. + */ +export function emergencyMaintenanceGuard( + req: Request, + res: Response, + next: NextFunction +): void { + const mutatingMethods = ['POST', 'PUT', 'PATCH', 'DELETE'] + if (!mutatingMethods.includes(req.method.toUpperCase())) { + next() + return + } + + // Exempt internal health and admin emergency unlock routes + if (req.path.startsWith('/health') || req.path.startsWith('/api/v1/admin/feature-flags')) { + next() + return + } + + const maintenance = evaluateFeatureFlag('emergency_maintenance_mode') + if (maintenance.enabled) { + logger.error( + `[EmergencyMaintenanceGuard] Blocked mutating request ${req.method} ${req.path}`, + { + path: req.path, + method: req.method, + } + ) + + res.status(503).json({ + success: false, + error: 'Platform emergency maintenance mode is active. State mutations are currently halted.', + code: 'EMERGENCY_MAINTENANCE_ACTIVE', + }) + return + } + + next() +} diff --git a/src/routes/admin.ts b/src/routes/admin.ts index f9900fe9..5b6be9a1 100644 --- a/src/routes/admin.ts +++ b/src/routes/admin.ts @@ -31,12 +31,11 @@ import { manualTripBreaker, manualResetBreaker, } from '../agent/breakerService' -import { messageDeliveryService } from '../messaging' import { - getAdminScopeCatalog, - getUserScopeCatalog, -} from '../auth/scopeCoverage' -import { DEFAULT_READ_SCOPES } from '../auth/scopes' + featureFlagManager, + getCurrentEnvironment, +} from '../config/featureFlags' +import { messageDeliveryService } from '../messaging' const router = Router() const prisma = db @@ -171,6 +170,12 @@ router.get( '/jobs/dashboard', requireAdminScope('metrics:read'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const jobSnapshot = await getJobDashboardSnapshot() const dlqSize = await DeadLetterQueue.getSize() @@ -991,6 +996,12 @@ router.post( '/keys/:id/rotate', requireAdminScope('keys:write'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const { id } = req.params @@ -1089,62 +1100,6 @@ router.post( } ) -/** - * GET /api/admin/keys/scopes - * The machine-readable scope catalogue (#615). - * - * Serves both scope families with their description, risk level, guardrail - * notes and the endpoints each scope actually covers, so the key-creation UI - * can show an operator exactly what granting a scope would hand over. - * - * The endpoint lists are derived from the live route table - * (src/auth/scopeCoverage.ts), so a scope cannot claim to unlock an endpoint - * that stopped requiring it. - * - * Required scope: keys:read - */ -router.get( - '/keys/scopes', - requireAdminScope('keys:read'), - async (req: Request, res: Response) => { - const adminAuth = getAdminAuth(res) - if (!adminAuth) { - return res - .status(401) - .json({ success: false, error: 'Admin authentication required' }) - } - try { - const data = { - user: { - defaultReadScopes: DEFAULT_READ_SCOPES, - scopes: getUserScopeCatalog(), - }, - admin: { - scopes: getAdminScopeCatalog(), - }, - } - - auditLog(req, res, 'LIST_SCOPE_CATALOG', 'success', { - userScopes: data.user.scopes.length, - adminScopes: data.admin.scopes.length, - }) - - res.status(200).json({ - success: true, - data, - timestamp: new Date().toISOString(), - }) - } catch (error) { - logger.error('[Admin] Failed to build scope catalogue', { - error: error instanceof Error ? error.message : 'Unknown error', - }) - res - .status(500) - .json({ success: false, error: 'Failed to build scope catalogue' }) - } - } -) - /** * GET /api/admin/keys * List all admin API keys (metadata only — no hashes). @@ -2400,6 +2355,12 @@ router.get( '/messages', requireAdminScope('messages:read'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const { channel, status, recipient, userId, category, limit, offset } = req.query @@ -2443,6 +2404,12 @@ router.get( '/messages/stats', requireAdminScope('messages:read'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const stats = await messageDeliveryService.getMessageStats() auditLog(req, res, 'MESSAGES_STATS', 'success') @@ -2469,6 +2436,12 @@ router.get( '/messages/:id', requireAdminScope('messages:read'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const msg = await messageDeliveryService.getMessage(req.params.id) if (!msg) { @@ -2500,6 +2473,12 @@ router.post( '/messages/:id/retry', requireAdminScope('messages:write'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const msg = await messageDeliveryService.getMessage(req.params.id) if (!msg) { @@ -2541,6 +2520,12 @@ router.post( '/messages/retry-dead-letters', requireAdminScope('messages:write'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const { channel } = req.body ?? {} const result = await messageDeliveryService.retryAllDeadLetters(channel) @@ -2576,6 +2561,12 @@ router.delete( '/messages/:id', requireAdminScope('messages:write'), async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } try { const msg = await messageDeliveryService.getMessage(req.params.id) if (!msg) { @@ -2610,8 +2601,6 @@ router.delete( } ) -export default router - // ── Privacy Erasure Routes (#489) ─────────────────────────────────────────── import { @@ -3036,3 +3025,454 @@ router.post( } } ) + +// ── Feature Flag Operational & Rollback Routes (#494) ──────────────────────── + +/** + * GET /api/v1/admin/feature-flags + * List all feature flags and their evaluation status for the current environment. + */ +router.get( + '/feature-flags', + requireAdminScope('flags:read'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const env = getCurrentEnvironment() + const entityId = + typeof req.query.entityId === 'string' ? req.query.entityId : undefined + const flags = featureFlagManager.getAllFlags( + entityId ? { entityId } : undefined + ) + res.status(200).json({ + success: true, + data: { + environment: env, + flags, + }, + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * GET /api/v1/admin/feature-flags/history + * List emergency rollback and configuration change history. + */ +router.get( + '/feature-flags/history', + requireAdminScope('flags:read'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const history = featureFlagManager.getRollbackHistory() + res.status(200).json({ + success: true, + data: history, + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * POST /api/v1/admin/feature-flags/emergency-disable-all + * Platform-wide emergency kill switch to activate maintenance mode and disable flags. + */ +router.post( + '/feature-flags/emergency-disable-all', + requireAdminScope('flags:write'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const { reason } = req.body ?? {} + if (typeof reason !== 'string' || reason.trim() === '') { + auditLog(req, res, 'EMERGENCY_DISABLE_ALL_FEATURE_FLAGS', 'failure', { + error: 'missing_reason', + }) + res.status(400).json({ success: false, error: 'reason is required' }) + return + } + + const adminIdentity = res.locals.adminAuth + ? `${res.locals.adminAuth.name} (${res.locals.adminAuth.role})` + : 'unknown' + + featureFlagManager.emergencyDisableAll(reason.trim(), adminIdentity) + auditLog(req, res, 'EMERGENCY_DISABLE_ALL_FEATURE_FLAGS', 'success', { + reason: reason.trim(), + }) + + res.status(200).json({ + success: true, + message: + 'Platform-wide emergency kill switch activated; all flags disabled', + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + auditLog(req, res, 'EMERGENCY_DISABLE_ALL_FEATURE_FLAGS', 'failure', { + error: message, + }) + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * GET /api/v1/admin/feature-flags/:key + * Retrieve specific feature flag details and evaluation. + */ +router.get( + '/feature-flags/:key', + requireAdminScope('flags:read'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const { key } = req.params + const flag = featureFlagManager.getFlag(key) + if (!flag) { + res + .status(404) + .json({ success: false, error: `Feature flag "${key}" not found` }) + return + } + + const entityId = + typeof req.query.entityId === 'string' ? req.query.entityId : undefined + const currentEvaluation = featureFlagManager.evaluate( + key, + entityId ? { entityId } : undefined + ) + + res.status(200).json({ + success: true, + data: { + ...flag, + currentEvaluation, + }, + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * PUT /api/v1/admin/feature-flags/:key + * Update runtime override for a feature flag. + */ +router.put( + '/feature-flags/:key', + requireAdminScope('flags:write'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const { key } = req.params + const flag = featureFlagManager.getFlag(key) + if (!flag) { + auditLog(req, res, 'UPDATE_FEATURE_FLAG', 'failure', { + error: 'not_found', + key, + }) + res + .status(404) + .json({ success: false, error: `Feature flag "${key}" not found` }) + return + } + + const { enabled, rolloutPercentage, allowlist, blocklist, killSwitch } = + req.body ?? {} + + if (enabled !== undefined && typeof enabled !== 'boolean') { + res + .status(400) + .json({ success: false, error: 'enabled must be a boolean' }) + return + } + + if ( + rolloutPercentage !== undefined && + (typeof rolloutPercentage !== 'number' || + rolloutPercentage < 0 || + rolloutPercentage > 100) + ) { + res.status(400).json({ + success: false, + error: 'rolloutPercentage must be a number between 0 and 100', + }) + return + } + + if (allowlist !== undefined && !Array.isArray(allowlist)) { + res + .status(400) + .json({ + success: false, + error: 'allowlist must be an array of strings', + }) + return + } + + if (blocklist !== undefined && !Array.isArray(blocklist)) { + res + .status(400) + .json({ + success: false, + error: 'blocklist must be an array of strings', + }) + return + } + + if (killSwitch !== undefined && typeof killSwitch !== 'boolean') { + res + .status(400) + .json({ success: false, error: 'killSwitch must be a boolean' }) + return + } + + const adminIdentity = res.locals.adminAuth + ? `${res.locals.adminAuth.name} (${res.locals.adminAuth.role})` + : 'unknown' + + featureFlagManager.setOverride( + key, + { + ...(enabled !== undefined ? { enabled } : {}), + ...(rolloutPercentage !== undefined ? { rolloutPercentage } : {}), + ...(allowlist !== undefined ? { allowlist } : {}), + ...(blocklist !== undefined ? { blocklist } : {}), + ...(killSwitch !== undefined ? { killSwitch } : {}), + }, + adminIdentity + ) + + auditLog(req, res, 'UPDATE_FEATURE_FLAG', 'success', { + key, + override: req.body, + }) + + res.status(200).json({ + success: true, + data: featureFlagManager.evaluate(key), + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + auditLog(req, res, 'UPDATE_FEATURE_FLAG', 'failure', { error: message }) + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * POST /api/v1/admin/feature-flags/:key/staged-rollout + * Advance or adjust staged rollout percentage (0-100). + */ +router.post( + '/feature-flags/:key/staged-rollout', + requireAdminScope('flags:write'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const { key } = req.params + const flag = featureFlagManager.getFlag(key) + if (!flag) { + auditLog(req, res, 'STAGED_ROLLOUT_FEATURE_FLAG', 'failure', { + error: 'not_found', + key, + }) + res + .status(404) + .json({ success: false, error: `Feature flag "${key}" not found` }) + return + } + + const { percentage } = req.body ?? {} + if ( + typeof percentage !== 'number' || + percentage < 0 || + percentage > 100 + ) { + auditLog(req, res, 'STAGED_ROLLOUT_FEATURE_FLAG', 'failure', { + error: 'invalid_percentage', + percentage, + }) + res.status(400).json({ + success: false, + error: 'percentage must be a number between 0 and 100', + }) + return + } + + const adminIdentity = res.locals.adminAuth + ? `${res.locals.adminAuth.name} (${res.locals.adminAuth.role})` + : 'unknown' + + featureFlagManager.stagedRollout(key, percentage, adminIdentity) + auditLog(req, res, 'STAGED_ROLLOUT_FEATURE_FLAG', 'success', { + key, + percentage, + }) + + res.status(200).json({ + success: true, + data: featureFlagManager.evaluate(key), + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + auditLog(req, res, 'STAGED_ROLLOUT_FEATURE_FLAG', 'failure', { + error: message, + }) + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * POST /api/v1/admin/feature-flags/:key/rollback + * Fast emergency rollback for a specific feature flag. + */ +router.post( + '/feature-flags/:key/rollback', + requireAdminScope('flags:write'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const { key } = req.params + const flag = featureFlagManager.getFlag(key) + if (!flag) { + auditLog(req, res, 'ROLLBACK_FEATURE_FLAG', 'failure', { + error: 'not_found', + key, + }) + res + .status(404) + .json({ success: false, error: `Feature flag "${key}" not found` }) + return + } + + const { reason } = req.body ?? {} + if (typeof reason !== 'string' || reason.trim() === '') { + auditLog(req, res, 'ROLLBACK_FEATURE_FLAG', 'failure', { + error: 'missing_reason', + }) + res.status(400).json({ success: false, error: 'reason is required' }) + return + } + + const adminIdentity = res.locals.adminAuth + ? `${res.locals.adminAuth.name} (${res.locals.adminAuth.role})` + : 'unknown' + + const rollbackEvent = featureFlagManager.rollback( + key, + reason.trim(), + adminIdentity + ) + + auditLog(req, res, 'ROLLBACK_FEATURE_FLAG', 'success', { + key, + reason: reason.trim(), + rollbackEvent, + }) + + res.status(200).json({ + success: true, + data: rollbackEvent, + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + auditLog(req, res, 'ROLLBACK_FEATURE_FLAG', 'failure', { error: message }) + res.status(500).json({ success: false, error: message }) + } + } +) + +/** + * POST /api/v1/admin/feature-flags/:key/reset + * Reset a feature flag back to its default environment configuration. + */ +router.post( + '/feature-flags/:key/reset', + requireAdminScope('flags:write'), + async (req: Request, res: Response) => { + const adminAuth = getAdminAuth(res) + if (!adminAuth) { + return res + .status(401) + .json({ success: false, error: 'Admin authentication required' }) + } + try { + const { key } = req.params + const flag = featureFlagManager.getFlag(key) + if (!flag) { + auditLog(req, res, 'RESET_FEATURE_FLAG', 'failure', { + error: 'not_found', + key, + }) + res + .status(404) + .json({ success: false, error: `Feature flag "${key}" not found` }) + return + } + + const adminIdentity = res.locals.adminAuth + ? `${res.locals.adminAuth.name} (${res.locals.adminAuth.role})` + : 'unknown' + + featureFlagManager.reset(key, adminIdentity) + auditLog(req, res, 'RESET_FEATURE_FLAG', 'success', { key }) + + res.status(200).json({ + success: true, + data: featureFlagManager.evaluate(key), + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error' + auditLog(req, res, 'RESET_FEATURE_FLAG', 'failure', { error: message }) + res.status(500).json({ success: false, error: message }) + } + } +) + +export default router diff --git a/src/types/express.d.ts b/src/types/express.d.ts index 5fa9e466..ea0ea289 100644 --- a/src/types/express.d.ts +++ b/src/types/express.d.ts @@ -1,4 +1,5 @@ import type { Network } from '@prisma/client' +import type { FlagEvaluationResult } from '../config/featureFlags' declare global { namespace Express { @@ -10,6 +11,8 @@ declare global { authScopes?: string[] apiKeyId?: string apiKeyAllowWithdrawals?: boolean + isFeatureEnabled?: (flagKey: string) => boolean + evaluateFeatureFlag?: (flagKey: string) => FlagEvaluationResult auth?: { userId: string sessionId: string diff --git a/src/utils/metrics.ts b/src/utils/metrics.ts index c3a37874..49b24720 100644 --- a/src/utils/metrics.ts +++ b/src/utils/metrics.ts @@ -1033,6 +1033,54 @@ export function recordAssistantFallback( assistantFallbackTotal.inc({ reason }) } +// ── Feature Flag Metrics (#494) ────────────────────────────────────────────────── +export const featureFlagEvaluationsTotal = new client.Counter({ + name: 'feature_flag_evaluations_total', + help: 'Total number of feature flag evaluations', + labelNames: ['flag', 'decision', 'result'] as const, + registers: [register], +}) + +export const featureFlagRollbacksTotal = new client.Counter({ + name: 'feature_flag_rollbacks_total', + help: 'Total number of feature flag rollbacks triggered', + labelNames: ['flag', 'operator'] as const, + registers: [register], +}) + +export const featureFlagRolloutPercentage = new client.Gauge({ + name: 'feature_flag_rollout_percentage', + help: 'Configured rollout percentage for a feature flag (0-100)', + labelNames: ['flag'] as const, + registers: [register], +}) + +export function recordFeatureFlagEvaluation( + flag: string, + decision: string, + result: boolean +): void { + featureFlagEvaluationsTotal.inc({ + flag, + decision, + result: result ? 'true' : 'false', + }) +} + +export function recordFeatureFlagRollback( + flag: string, + operator: string = 'system' +): void { + featureFlagRollbacksTotal.inc({ flag, operator }) +} + +export function setFeatureFlagRolloutGauge( + flag: string, + percentage: number +): void { + featureFlagRolloutPercentage.set({ flag }, percentage) +} + // ── Message Delivery Metrics (#493) ────────────────────────────────────────────── export const messageDeliveriesTotal = new client.Counter({ name: 'message_deliveries_total', diff --git a/src/utils/telegram-client.ts b/src/utils/telegram-client.ts index 6d22b505..6ba8eb73 100644 --- a/src/utils/telegram-client.ts +++ b/src/utils/telegram-client.ts @@ -1,5 +1,9 @@ import { config } from '../config' -import { HttpClientAdapter, CircuitBreakerError, TimeoutError } from './http-client' +import { + HttpClientAdapter, + CircuitBreakerError, + TimeoutError, +} from './http-client' import { logger } from './logger' export interface SendTelegramMessageParams { @@ -159,7 +163,9 @@ export async function sendTelegramMessage( } const messageId = String(data.result?.message_id ?? Date.now()) - logger.info(`[Telegram] Message sent to chat ${params.chatId}: id=${messageId}`) + logger.info( + `[Telegram] Message sent to chat ${params.chatId}: id=${messageId}` + ) return { messageId, ok: true, diff --git a/tests/unit/config/featureFlags.test.ts b/tests/unit/config/featureFlags.test.ts new file mode 100644 index 00000000..292a8175 --- /dev/null +++ b/tests/unit/config/featureFlags.test.ts @@ -0,0 +1,342 @@ +import { + FeatureFlagManager, + hashEntityToPercentage, + getCurrentEnvironment, + DEFAULT_FEATURE_FLAGS, +} from '../../../src/config/featureFlags' +import { logger } from '../../../src/utils/logger' + +describe('Feature Flags Configuration and Evaluation (#494)', () => { + let manager: FeatureFlagManager + const originalEnv = process.env + + beforeEach(() => { + process.env = { ...originalEnv } + manager = new FeatureFlagManager() + }) + + afterEach(() => { + process.env = originalEnv + manager.resetAll() + jest.restoreAllMocks() + }) + + describe('Default Catalog and Registration', () => { + it('initializes with all default feature flags', () => { + const flags = manager.getAllFlags() + const keys = flags.map((f) => f.key) + + expect(keys).toContain('agent_autonomous_rebalance') + expect(keys).toContain('smart_dca_engine') + expect(keys).toContain('enhanced_stellar_routing') + expect(keys).toContain('ai_assistant_tool_calling') + expect(keys).toContain('circuit_breaker_v2') + expect(keys).toContain('fast_fiat_onramp') + expect(keys).toContain('emergency_maintenance_mode') + }) + + it('allows registering custom feature flags', () => { + manager.registerFlag({ + key: 'experimental_order_routing', + description: 'Next-gen routing engine', + environments: { + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: false, rolloutPercentage: 0 }, + production: { enabled: false, rolloutPercentage: 0 }, + }, + }) + + const flag = manager.getFlag('experimental_order_routing') + expect(flag).toBeDefined() + expect(flag?.key).toBe('experimental_order_routing') + expect(manager.isEnabled('experimental_order_routing', { environment: 'development' })).toBe(true) + expect(manager.isEnabled('experimental_order_routing', { environment: 'production' })).toBe(false) + }) + + it('falls back closed to false for unknown flags', () => { + const result = manager.evaluate('non_existent_flag') + expect(result.enabled).toBe(false) + expect(result.decision).toBe('DEFAULT_FALLBACK') + }) + }) + + describe('Environment Awareness', () => { + it('correctly resolves environment from NODE_ENV', () => { + process.env.NODE_ENV = 'production' + expect(getCurrentEnvironment()).toBe('production') + + process.env.NODE_ENV = 'staging' + expect(getCurrentEnvironment()).toBe('staging') + + process.env.NODE_ENV = 'test' + expect(getCurrentEnvironment()).toBe('test') + + process.env.NODE_ENV = 'development' + expect(getCurrentEnvironment()).toBe('development') + + process.env.NODE_ENV = 'unknown_env' + expect(getCurrentEnvironment()).toBe('development') + }) + + it('evaluates flags according to the specified environment', () => { + // In DEFAULT_FEATURE_FLAGS, agent_autonomous_rebalance is: + // dev: 100%, test: 100%, staging: 50%, prod: 0% + const devRes = manager.evaluate('agent_autonomous_rebalance', { environment: 'development' }) + expect(devRes.enabled).toBe(true) + expect(devRes.environment).toBe('development') + + const prodRes = manager.evaluate('agent_autonomous_rebalance', { environment: 'production' }) + expect(prodRes.enabled).toBe(false) + expect(prodRes.environment).toBe('production') + }) + }) + + describe('Deterministic Hashing and Staged Rollout', () => { + it('produces consistent, deterministic hash buckets for identical entity inputs', () => { + const bucket1 = hashEntityToPercentage('smart_dca_engine', 'user_12345') + const bucket2 = hashEntityToPercentage('smart_dca_engine', 'user_12345') + const bucket3 = hashEntityToPercentage('smart_dca_engine', 'user_12345') + + expect(bucket1).toBe(bucket2) + expect(bucket2).toBe(bucket3) + expect(bucket1).toBeGreaterThanOrEqual(0) + expect(bucket1).toBeLessThan(100) + }) + + it('handles empty or whitespace entity IDs gracefully', () => { + expect(hashEntityToPercentage('flag', '')).toBe(0) + }) + + it('implements monotonic staged rollouts across percentages', () => { + // Find two users with known buckets + const userA = 'user_alpha' + const bucketA = hashEntityToPercentage('smart_dca_engine', userA) + + // Test progressive staged rollout + // If percentage <= bucketA, enabled is false; if percentage > bucketA, enabled is true + manager.stagedRollout('smart_dca_engine', bucketA) + expect(manager.isEnabled('smart_dca_engine', userA)).toBe(false) + + manager.stagedRollout('smart_dca_engine', bucketA + 1) + expect(manager.isEnabled('smart_dca_engine', userA)).toBe(true) + + // Advance to 100% + manager.stagedRollout('smart_dca_engine', 100) + expect(manager.isEnabled('smart_dca_engine', userA)).toBe(true) + + // Drop to 0% + manager.stagedRollout('smart_dca_engine', 0) + expect(manager.isEnabled('smart_dca_engine', userA)).toBe(false) + }) + }) + + describe('Allowlists and Blocklists', () => { + it('allows an entity explicitly included on the allowlist even if rollout is 0%', () => { + manager.registerFlag({ + key: 'beta_feature', + description: 'Beta feature', + environments: { + production: { enabled: false, rolloutPercentage: 0 }, + development: { enabled: false, rolloutPercentage: 0 }, + test: { enabled: false, rolloutPercentage: 0 }, + staging: { enabled: false, rolloutPercentage: 0 }, + }, + allowlist: ['whitelisted_user_1', 'whitelisted_user_2'], + }) + + const resAllowed = manager.evaluate('beta_feature', { + entityId: 'whitelisted_user_1', + environment: 'production', + }) + expect(resAllowed.enabled).toBe(true) + expect(resAllowed.decision).toBe('ALLOWLIST') + + const resOther = manager.evaluate('beta_feature', { + entityId: 'normal_user', + environment: 'production', + }) + expect(resOther.enabled).toBe(false) + }) + + it('blocks an entity explicitly included on the blocklist even if rollout is 100%', () => { + manager.registerFlag({ + key: 'ga_feature', + description: 'GA feature', + environments: { + production: { enabled: true, rolloutPercentage: 100 }, + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 100 }, + }, + blocklist: ['blocked_user_bad_actor'], + }) + + const resBlocked = manager.evaluate('ga_feature', { + entityId: 'blocked_user_bad_actor', + environment: 'production', + }) + expect(resBlocked.enabled).toBe(false) + expect(resBlocked.decision).toBe('BLOCKLIST') + + const resAllowed = manager.evaluate('ga_feature', { + entityId: 'good_user', + environment: 'production', + }) + expect(resAllowed.enabled).toBe(true) + }) + + it('prioritizes blocklist over allowlist if an entity appears in both', () => { + manager.registerFlag({ + key: 'conflict_feature', + description: 'Test conflict', + environments: { + production: { enabled: true, rolloutPercentage: 100 }, + development: { enabled: true, rolloutPercentage: 100 }, + test: { enabled: true, rolloutPercentage: 100 }, + staging: { enabled: true, rolloutPercentage: 100 }, + }, + allowlist: ['user_in_both'], + blocklist: ['user_in_both'], + }) + + const result = manager.evaluate('conflict_feature', { + entityId: 'user_in_both', + environment: 'production', + }) + expect(result.enabled).toBe(false) + expect(result.decision).toBe('BLOCKLIST') + }) + }) + + describe('Runtime Overrides & Management API', () => { + it('sets and removes runtime overrides', () => { + // In production, circuit_breaker_v2 is enabled + expect(manager.isEnabled('circuit_breaker_v2', { environment: 'production' })).toBe(true) + + // Set runtime override to false + manager.setOverride('circuit_breaker_v2', { enabled: false }, 'admin-alice') + expect(manager.isEnabled('circuit_breaker_v2', { environment: 'production' })).toBe(false) + + const evalRes = manager.evaluate('circuit_breaker_v2', { environment: 'production' }) + expect(evalRes.decision).toBe('RUNTIME_OVERRIDE') + + // Reset + manager.reset('circuit_breaker_v2', 'admin-alice') + expect(manager.isEnabled('circuit_breaker_v2', { environment: 'production' })).toBe(true) + }) + }) + + describe('Emergency Rollback and Kill Switches', () => { + it('executes fast emergency rollback, dropping percentage to 0 and recording history', () => { + // Setup active staged rollout + manager.stagedRollout('smart_dca_engine', 50, 'release-manager') + expect(manager.getFlag('smart_dca_engine')?.rolledBack).toBeFalsy() + + // Trigger emergency rollback + const rollbackEvent = manager.rollback( + 'smart_dca_engine', + 'High latency detected on Horizon liquidity pool', + 'sre-oncall-bob' + ) + + expect(rollbackEvent.flagKey).toBe('smart_dca_engine') + expect(rollbackEvent.reason).toBe('High latency detected on Horizon liquidity pool') + expect(rollbackEvent.operator).toBe('sre-oncall-bob') + expect(rollbackEvent.timestamp).toBeDefined() + + // Flag must now evaluate to false immediately + const evalRes = manager.evaluate('smart_dca_engine') + expect(evalRes.enabled).toBe(false) + expect(evalRes.decision).toBe('EMERGENCY_ROLLBACK') + expect(evalRes.rolloutPercentage).toBe(0) + + // History must record the event + const history = manager.getRollbackHistory() + expect(history.length).toBe(1) + expect(history[0].flagKey).toBe('smart_dca_engine') + + // Reset restores original state + manager.reset('smart_dca_engine', 'admin-alice') + const afterReset = manager.evaluate('smart_dca_engine', { environment: 'development' }) + expect(afterReset.enabled).toBe(true) + }) + + it('supports platform-wide emergencyDisableAll by engaging emergency maintenance mode', () => { + manager.emergencyDisableAll('Critical security audit event', 'security-team') + + const res1 = manager.evaluate('agent_autonomous_rebalance', { environment: 'development' }) + expect(res1.enabled).toBe(false) + expect(res1.decision).toBe('GLOBAL_MAINTENANCE') + + const res2 = manager.evaluate('circuit_breaker_v2', { environment: 'development' }) + expect(res2.enabled).toBe(false) + expect(res2.decision).toBe('GLOBAL_MAINTENANCE') + }) + }) + + describe('Environment Variable Overrides', () => { + it('overrides flag state via FEATURE_FLAG_', () => { + process.env.FEATURE_FLAG_AGENT_AUTONOMOUS_REBALANCE = 'true' + const res = manager.evaluate('agent_autonomous_rebalance', { environment: 'production' }) + expect(res.enabled).toBe(true) + expect(res.decision).toBe('ENV_VAR_OVERRIDE') + }) + + it('overrides rollout percentage via FEATURE_FLAG__PERCENTAGE', () => { + process.env.FEATURE_FLAG_SMART_DCA_ENGINE_PERCENTAGE = '100' + const res = manager.evaluate('smart_dca_engine', { + entityId: 'user_any', + environment: 'production', + }) + expect(res.enabled).toBe(true) + expect(res.decision).toBe('PERCENTAGE_ROLLOUT_MATCH') + expect(res.rolloutPercentage).toBe(100) + }) + + it('enforces flag kill-switch via FEATURE_FLAG__KILL_SWITCH', () => { + process.env.FEATURE_FLAG_CIRCUIT_BREAKER_V2_KILL_SWITCH = 'true' + const res = manager.evaluate('circuit_breaker_v2', { environment: 'development' }) + expect(res.enabled).toBe(false) + expect(res.decision).toBe('KILL_SWITCH') + }) + + it('triggers emergency rollback via FEATURE_FLAGS_ROLLBACK list', () => { + process.env.FEATURE_FLAGS_ROLLBACK = 'agent_autonomous_rebalance,fast_fiat_onramp' + const res1 = manager.evaluate('agent_autonomous_rebalance', { environment: 'development' }) + expect(res1.enabled).toBe(false) + expect(res1.decision).toBe('EMERGENCY_ROLLBACK') + + const res2 = manager.evaluate('fast_fiat_onramp', { environment: 'development' }) + expect(res2.enabled).toBe(false) + expect(res2.decision).toBe('EMERGENCY_ROLLBACK') + + // Other flags unaffected + const res3 = manager.evaluate('circuit_breaker_v2', { environment: 'development' }) + expect(res3.enabled).toBe(true) + }) + }) + + describe('Structured Decision Logging', () => { + it('logs structured evaluation metadata via Winston logger', () => { + const loggerSpy = jest.spyOn(logger, 'info') + + const result = manager.evaluate('agent_autonomous_rebalance', { + entityId: 'user_logging_test', + environment: 'development', + }) + + expect(loggerSpy).toHaveBeenCalledWith( + expect.stringContaining('[FeatureFlag] Evaluated flag "agent_autonomous_rebalance"'), + expect.objectContaining({ + event: 'feature_flag_evaluation', + flag: 'agent_autonomous_rebalance', + enabled: result.enabled, + decision: result.decision, + environment: 'development', + entityId: 'user_logging_test', + }) + ) + }) + }) +}) diff --git a/tests/unit/jobs/messageDeliverySweep.test.ts b/tests/unit/jobs/messageDeliverySweep.test.ts index b3f370a4..6bc5a67a 100644 --- a/tests/unit/jobs/messageDeliverySweep.test.ts +++ b/tests/unit/jobs/messageDeliverySweep.test.ts @@ -37,10 +37,28 @@ describe('Message Delivery Sweep Job (#493)', () => { .spyOn(messageDeliveryService, 'getMessageStats') .mockResolvedValue({ total: 3, - byStatus: { pending: 1, sending: 0, delivered: 2, failed: 0, deadLetter: 0 }, + byStatus: { + pending: 1, + sending: 0, + delivered: 2, + failed: 0, + deadLetter: 0, + }, byChannel: { - telegram: { total: 2, delivered: 2, failed: 0, pending: 0, deadLetter: 0 }, - whatsapp: { total: 1, delivered: 0, failed: 0, pending: 1, deadLetter: 0 }, + telegram: { + total: 2, + delivered: 2, + failed: 0, + pending: 0, + deadLetter: 0, + }, + whatsapp: { + total: 1, + delivered: 0, + failed: 0, + pending: 1, + deadLetter: 0, + }, }, }) @@ -55,19 +73,39 @@ describe('Message Delivery Sweep Job (#493)', () => { }) it('emits an operational alert when dead letters exceed threshold', async () => { - jest.spyOn(messageDeliveryService, 'processPendingQueue').mockResolvedValue({ - processed: 5, - delivered: 2, - retried: 0, - deadLettered: 3, - }) + jest + .spyOn(messageDeliveryService, 'processPendingQueue') + .mockResolvedValue({ + processed: 5, + delivered: 2, + retried: 0, + deadLettered: 3, + }) jest.spyOn(messageDeliveryService, 'getMessageStats').mockResolvedValue({ total: 5, - byStatus: { pending: 0, sending: 0, delivered: 2, failed: 0, deadLetter: 3 }, + byStatus: { + pending: 0, + sending: 0, + delivered: 2, + failed: 0, + deadLetter: 3, + }, byChannel: { - telegram: { total: 3, delivered: 1, failed: 0, pending: 0, deadLetter: 2 }, - whatsapp: { total: 2, delivered: 1, failed: 0, pending: 0, deadLetter: 1 }, + telegram: { + total: 3, + delivered: 1, + failed: 0, + pending: 0, + deadLetter: 2, + }, + whatsapp: { + total: 2, + delivered: 1, + failed: 0, + pending: 0, + deadLetter: 1, + }, }, }) diff --git a/tests/unit/messaging/messageDeliveryService.test.ts b/tests/unit/messaging/messageDeliveryService.test.ts index 148ad323..88dfefcd 100644 --- a/tests/unit/messaging/messageDeliveryService.test.ts +++ b/tests/unit/messaging/messageDeliveryService.test.ts @@ -50,7 +50,9 @@ describe('MessageDeliveryService (#493)', () => { }) it('tracks successful WhatsApp message delivery with Twilio SID', async () => { - jest.spyOn(twilioClient, 'sendWhatsAppMessage').mockResolvedValue('SM-twilio-sid-1') + jest + .spyOn(twilioClient, 'sendWhatsAppMessage') + .mockResolvedValue('SM-twilio-sid-1') const record = await messageDeliveryService.send({ channel: 'WHATSAPP', @@ -68,12 +70,17 @@ describe('MessageDeliveryService (#493)', () => { describe('Retry Policy & Backoff', () => { it('re-enqueues message for retry on transient failure with exponential backoff', async () => { - const transientError = new telegramClient.TelegramApiError('Rate limited', { - statusCode: 429, - retryAfterSeconds: 5, - isTransient: true, - }) - jest.spyOn(telegramClient, 'sendTelegramMessage').mockRejectedValue(transientError) + const transientError = new telegramClient.TelegramApiError( + 'Rate limited', + { + statusCode: 429, + retryAfterSeconds: 5, + isTransient: true, + } + ) + jest + .spyOn(telegramClient, 'sendTelegramMessage') + .mockRejectedValue(transientError) const record = await messageDeliveryService.send({ channel: 'TELEGRAM', @@ -100,9 +107,9 @@ describe('MessageDeliveryService (#493)', () => { describe('Fallback Handling', () => { it('triggers fallback channel when primary channel fails and fallback is configured', async () => { // Primary fails with terminal error - jest.spyOn(twilioClient, 'sendWhatsAppMessage').mockRejectedValue( - new Error('21211: Invalid phone number') - ) + jest + .spyOn(twilioClient, 'sendWhatsAppMessage') + .mockRejectedValue(new Error('21211: Invalid phone number')) // Fallback succeeds jest.spyOn(telegramClient, 'sendTelegramMessage').mockResolvedValue({ ok: true, @@ -128,7 +135,9 @@ describe('MessageDeliveryService (#493)', () => { describe('Dead Letter Queue & Alerting', () => { it('moves to DEAD_LETTER and emits alert when attempts and fallback are exhausted', async () => { const terminalErr = new Error('403: Forbidden - bot blocked by user') - jest.spyOn(telegramClient, 'sendTelegramMessage').mockRejectedValue(terminalErr) + jest + .spyOn(telegramClient, 'sendTelegramMessage') + .mockRejectedValue(terminalErr) const emitSpy = jest.spyOn(alertingService, 'emit') @@ -155,9 +164,9 @@ describe('MessageDeliveryService (#493)', () => { describe('Safe Manual Recovery', () => { it('retries a dead-lettered message manually', async () => { // First attempt fails to DEAD_LETTER - jest.spyOn(telegramClient, 'sendTelegramMessage').mockRejectedValueOnce( - new Error('Fatal failure') - ) + jest + .spyOn(telegramClient, 'sendTelegramMessage') + .mockRejectedValueOnce(new Error('Fatal failure')) const record = await messageDeliveryService.send({ channel: 'TELEGRAM', @@ -181,7 +190,9 @@ describe('MessageDeliveryService (#493)', () => { it('bulk retries all dead-lettered messages', async () => { // Seed 2 dead letter records - jest.spyOn(telegramClient, 'sendTelegramMessage').mockRejectedValue(new Error('Down')) + jest + .spyOn(telegramClient, 'sendTelegramMessage') + .mockRejectedValue(new Error('Down')) await messageDeliveryService.send({ channel: 'TELEGRAM', @@ -199,12 +210,15 @@ describe('MessageDeliveryService (#493)', () => { const stats = await messageDeliveryService.getMessageStats() expect(stats.byStatus.deadLetter).toBe(2) - const result = await messageDeliveryService.retryAllDeadLetters('TELEGRAM') + const result = + await messageDeliveryService.retryAllDeadLetters('TELEGRAM') expect(result.count).toBe(2) }) it('cancels an unsent or dead-letter message safely', async () => { - jest.spyOn(telegramClient, 'sendTelegramMessage').mockRejectedValueOnce(new Error('Down')) + jest + .spyOn(telegramClient, 'sendTelegramMessage') + .mockRejectedValueOnce(new Error('Down')) const record = await messageDeliveryService.send({ channel: 'TELEGRAM', @@ -241,11 +255,15 @@ describe('MessageDeliveryService (#493)', () => { const all = await messageDeliveryService.listMessages() expect(all.total).toBe(2) - const onlyTelegram = await messageDeliveryService.listMessages({ channel: 'TELEGRAM' }) + const onlyTelegram = await messageDeliveryService.listMessages({ + channel: 'TELEGRAM', + }) expect(onlyTelegram.total).toBe(1) expect(onlyTelegram.messages[0].channel).toBe('TELEGRAM') - const onlyAlerts = await messageDeliveryService.listMessages({ category: 'ALERT' }) + const onlyAlerts = await messageDeliveryService.listMessages({ + category: 'ALERT', + }) expect(onlyAlerts.total).toBe(1) expect(onlyAlerts.messages[0].category).toBe('ALERT') }) diff --git a/tests/unit/messaging/telegramClient.test.ts b/tests/unit/messaging/telegramClient.test.ts index 0e6d094a..310bfef7 100644 --- a/tests/unit/messaging/telegramClient.test.ts +++ b/tests/unit/messaging/telegramClient.test.ts @@ -9,7 +9,10 @@ import { TelegramApiError, resetTelegramClient, } from '../../../src/utils/telegram-client' -import { TimeoutError, CircuitBreakerError } from '../../../src/utils/http-client' +import { + TimeoutError, + CircuitBreakerError, +} from '../../../src/utils/http-client' describe('Telegram Client (#493)', () => { beforeEach(() => { @@ -100,10 +103,13 @@ describe('Telegram Client (#493)', () => { }) expect(isTelegramTransientError(err400)).toBe(false) - const err403 = new TelegramApiError('Forbidden: bot was blocked by the user', { - statusCode: 403, - isTransient: false, - }) + const err403 = new TelegramApiError( + 'Forbidden: bot was blocked by the user', + { + statusCode: 403, + isTransient: false, + } + ) expect(isTelegramTransientError(err403)).toBe(false) }) }) diff --git a/tests/unit/middleware/featureFlags.test.ts b/tests/unit/middleware/featureFlags.test.ts new file mode 100644 index 00000000..f96143ed --- /dev/null +++ b/tests/unit/middleware/featureFlags.test.ts @@ -0,0 +1,167 @@ +import express, { Request, Response } from 'express' +import request from 'supertest' +import { + requireFeatureFlag, + featureFlagMiddleware, + emergencyMaintenanceGuard, +} from '../../../src/middleware/featureFlags' +import { featureFlagManager } from '../../../src/config/featureFlags' + +describe('Feature Flag Middleware (#494)', () => { + beforeEach(() => { + featureFlagManager.resetAll() + }) + + afterEach(() => { + featureFlagManager.resetAll() + }) + + describe('requireFeatureFlag guard', () => { + it('allows the request when the feature flag is enabled', async () => { + featureFlagManager.setOverride('fast_fiat_onramp', { enabled: true }) + + const app = express() + app.get( + '/api/v1/fiat/fast-quote', + requireFeatureFlag('fast_fiat_onramp'), + (_req: Request, res: Response) => { + res.json({ success: true, message: 'Fast quote generated' }) + } + ) + + const response = await request(app).get('/api/v1/fiat/fast-quote') + expect(response.status).toBe(200) + expect(response.body.success).toBe(true) + }) + + it('rejects the request with 503 FEATURE_DISABLED when the flag is disabled', async () => { + featureFlagManager.setOverride('fast_fiat_onramp', { enabled: false }) + + const app = express() + app.get( + '/api/v1/fiat/fast-quote', + requireFeatureFlag('fast_fiat_onramp', { + customMessage: 'Instant on-ramp is temporarily disabled', + }), + (_req: Request, res: Response) => { + res.json({ success: true }) + } + ) + + const response = await request(app).get('/api/v1/fiat/fast-quote') + expect(response.status).toBe(503) + expect(response.body).toEqual( + expect.objectContaining({ + success: false, + code: 'FEATURE_DISABLED', + flag: 'fast_fiat_onramp', + error: 'Instant on-ramp is temporarily disabled', + }) + ) + }) + + it('evaluates staged rollout per user ID using custom entityExtractor', async () => { + // 50% staged rollout + featureFlagManager.stagedRollout('ai_assistant_tool_calling', 50) + + const app = express() + app.get( + '/api/v1/assistant/tools', + (req: Request, _res: Response, next) => { + // Mock auth user + const userId = req.headers['x-user-id'] as string + if (userId) { + ;(req as any).user = { id: userId } + } + next() + }, + requireFeatureFlag('ai_assistant_tool_calling', { + entityExtractor: (req) => (req as any).user?.id || 'anon', + }), + (_req: Request, res: Response) => { + res.json({ success: true, tools: ['rebalance', 'swap'] }) + } + ) + + // An allowlisted user + featureFlagManager.setOverride('ai_assistant_tool_calling', { + allowlist: ['allowed_vip_user'], + }) + + const resAllowed = await request(app) + .get('/api/v1/assistant/tools') + .set('x-user-id', 'allowed_vip_user') + + expect(resAllowed.status).toBe(200) + expect(resAllowed.body.success).toBe(true) + }) + }) + + describe('featureFlagMiddleware context injection', () => { + it('decorates Request with isFeatureEnabled and evaluateFeatureFlag helpers', async () => { + featureFlagManager.setOverride('circuit_breaker_v2', { enabled: true }) + + const app = express() + app.use(featureFlagMiddleware) + app.get('/test-context', (req: Request, res: Response) => { + const isEnabled = req.isFeatureEnabled?.('circuit_breaker_v2') + const evaluation = req.evaluateFeatureFlag?.('circuit_breaker_v2') + res.json({ isEnabled, decision: evaluation?.decision }) + }) + + const response = await request(app).get('/test-context') + expect(response.status).toBe(200) + expect(response.body.isEnabled).toBe(true) + expect(response.body.decision).toBe('RUNTIME_OVERRIDE') + }) + }) + + describe('emergencyMaintenanceGuard', () => { + it('allows read-only GET requests even during emergency maintenance', async () => { + featureFlagManager.setOverride('emergency_maintenance_mode', { enabled: true }) + + const app = express() + app.use(emergencyMaintenanceGuard) + app.get('/api/v1/portfolio', (_req: Request, res: Response) => { + res.json({ success: true, balance: 100 }) + }) + + const response = await request(app).get('/api/v1/portfolio') + expect(response.status).toBe(200) + expect(response.body.balance).toBe(100) + }) + + it('blocks mutating POST requests with 503 during emergency maintenance', async () => { + featureFlagManager.setOverride('emergency_maintenance_mode', { enabled: true }) + + const app = express() + app.use(emergencyMaintenanceGuard) + app.post('/api/v1/orders', (_req: Request, res: Response) => { + res.json({ success: true }) + }) + + const response = await request(app).post('/api/v1/orders') + expect(response.status).toBe(503) + expect(response.body).toEqual( + expect.objectContaining({ + success: false, + code: 'EMERGENCY_MAINTENANCE_ACTIVE', + }) + ) + }) + + it('bypasses emergency maintenance for admin and health routes', async () => { + featureFlagManager.setOverride('emergency_maintenance_mode', { enabled: true }) + + const app = express() + app.use(emergencyMaintenanceGuard) + app.post('/api/v1/admin/feature-flags/reset', (_req: Request, res: Response) => { + res.json({ success: true, message: 'admin reset' }) + }) + + const response = await request(app).post('/api/v1/admin/feature-flags/reset') + expect(response.status).toBe(200) + expect(response.body.message).toBe('admin reset') + }) + }) +}) diff --git a/tests/unit/routes/admin-feature-flags.test.ts b/tests/unit/routes/admin-feature-flags.test.ts new file mode 100644 index 00000000..e8489f31 --- /dev/null +++ b/tests/unit/routes/admin-feature-flags.test.ts @@ -0,0 +1,292 @@ +process.env.NODE_ENV = 'test' +process.env.JWT_SEED = '0'.repeat(64) +process.env.DATABASE_URL = 'postgresql://localhost:5432/test' + +// Mock db/prisma before importing admin router +jest.mock('../../../src/db', () => ({ + __esModule: true, + default: { + adminAuditLog: { + create: jest.fn().mockResolvedValue({ id: 'mock-audit-id' }), + }, + adminApiKey: { + findMany: jest.fn().mockResolvedValue([]), + update: jest.fn().mockResolvedValue({}), + }, + auditBlock: { + findMany: jest.fn().mockResolvedValue([]), + }, + }, +})) + +// Mock admin auth middleware to enable testing RBAC scopes easily +jest.mock('../../../src/middleware/adminAuth', () => { + const actual = jest.requireActual('../../../src/middleware/adminAuth') + return { + ...actual, + requireAdminAuth: (req: any, res: any, next: any) => { + const scopesHeader = req.headers['x-mock-scopes'] + const scopes = scopesHeader ? (scopesHeader as string).split(',').map((s: string) => s.trim()) : ['super'] + res.locals.adminAuth = { + id: 'admin-key-1', + name: 'Test SRE Operator', + role: 'operator', + scopes, + } + next() + }, + } +}) + +import express from 'express' +import request from 'supertest' +import adminRouter from '../../../src/routes/admin' +import { featureFlagManager } from '../../../src/config/featureFlags' + +const app = express() +app.use(express.json()) +app.use('/api/v1/admin', adminRouter) + +describe('Admin Feature Flag Endpoints (#494)', () => { + beforeEach(() => { + featureFlagManager.resetAll() + }) + + afterEach(() => { + featureFlagManager.resetAll() + }) + + describe('GET /api/v1/admin/feature-flags', () => { + it('returns all feature flags and runtime environment', async () => { + const res = await request(app).get('/api/v1/admin/feature-flags') + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + expect(res.body.data.environment).toBe('test') + expect(Array.isArray(res.body.data.flags)).toBe(true) + expect(res.body.data.flags.length).toBeGreaterThan(0) + + const dcaFlag = res.body.data.flags.find((f: any) => f.key === 'smart_dca_engine') + expect(dcaFlag).toBeDefined() + expect(dcaFlag.currentEvaluation).toBeDefined() + }) + + it('evaluates flags with entityId query parameter', async () => { + const res = await request(app) + .get('/api/v1/admin/feature-flags') + .query({ entityId: 'user_target_abc' }) + + expect(res.status).toBe(200) + const dcaFlag = res.body.data.flags.find((f: any) => f.key === 'smart_dca_engine') + expect(dcaFlag.currentEvaluation.entityId).toBe('user_target_abc') + }) + }) + + describe('GET /api/v1/admin/feature-flags/:key', () => { + it('returns flag details when flag exists', async () => { + const res = await request(app).get('/api/v1/admin/feature-flags/smart_dca_engine') + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + expect(res.body.data.key).toBe('smart_dca_engine') + expect(res.body.data.currentEvaluation).toBeDefined() + }) + + it('returns 404 when flag does not exist', async () => { + const res = await request(app).get('/api/v1/admin/feature-flags/non_existent_key') + + expect(res.status).toBe(404) + expect(res.body.success).toBe(false) + expect(res.body.error).toContain('not found') + }) + }) + + describe('PUT /api/v1/admin/feature-flags/:key', () => { + it('updates runtime overrides and returns new evaluation', async () => { + const res = await request(app) + .put('/api/v1/admin/feature-flags/circuit_breaker_v2') + .send({ + enabled: false, + rolloutPercentage: 0, + }) + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + expect(res.body.data.enabled).toBe(false) + expect(res.body.data.decision).toBe('RUNTIME_OVERRIDE') + }) + + it('rejects invalid rolloutPercentage (>100 or <0)', async () => { + const res = await request(app) + .put('/api/v1/admin/feature-flags/circuit_breaker_v2') + .send({ rolloutPercentage: 150 }) + + expect(res.status).toBe(400) + expect(res.body.success).toBe(false) + expect(res.body.error).toContain('rolloutPercentage must be a number between 0 and 100') + }) + + it('rejects non-array allowlist', async () => { + const res = await request(app) + .put('/api/v1/admin/feature-flags/circuit_breaker_v2') + .send({ allowlist: 'not-an-array' }) + + expect(res.status).toBe(400) + expect(res.body.error).toContain('allowlist must be an array') + }) + + it('returns 404 for unknown flag', async () => { + const res = await request(app) + .put('/api/v1/admin/feature-flags/fake_flag') + .send({ enabled: true }) + + expect(res.status).toBe(404) + }) + }) + + describe('POST /api/v1/admin/feature-flags/:key/staged-rollout', () => { + it('advances staged rollout percentage', async () => { + const res = await request(app) + .post('/api/v1/admin/feature-flags/smart_dca_engine/staged-rollout') + .send({ percentage: 25 }) + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + expect(res.body.data.rolloutPercentage).toBe(25) + }) + + it('validates percentage is between 0 and 100', async () => { + const res1 = await request(app) + .post('/api/v1/admin/feature-flags/smart_dca_engine/staged-rollout') + .send({ percentage: -5 }) + expect(res1.status).toBe(400) + + const res2 = await request(app) + .post('/api/v1/admin/feature-flags/smart_dca_engine/staged-rollout') + .send({ percentage: 'full' }) + expect(res2.status).toBe(400) + }) + }) + + describe('POST /api/v1/admin/feature-flags/:key/rollback', () => { + it('executes fast emergency rollback, sets rolledBack=true, drops percentage to 0', async () => { + // First rollout + featureFlagManager.stagedRollout('enhanced_stellar_routing', 50) + + // Trigger rollback + const res = await request(app) + .post('/api/v1/admin/feature-flags/enhanced_stellar_routing/rollback') + .send({ reason: 'Slippage anomalies observed in multi-hop paths' }) + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + expect(res.body.data.flagKey).toBe('enhanced_stellar_routing') + expect(res.body.data.reason).toBe('Slippage anomalies observed in multi-hop paths') + expect(res.body.data.operator).toBe('Test SRE Operator (operator)') + + // Verify flag is now disabled + const evalRes = featureFlagManager.evaluate('enhanced_stellar_routing') + expect(evalRes.enabled).toBe(false) + expect(evalRes.decision).toBe('EMERGENCY_ROLLBACK') + expect(evalRes.rolloutPercentage).toBe(0) + }) + + it('requires a reason for rollback', async () => { + const res = await request(app) + .post('/api/v1/admin/feature-flags/enhanced_stellar_routing/rollback') + .send({}) + + expect(res.status).toBe(400) + expect(res.body.success).toBe(false) + expect(res.body.error).toBe('reason is required') + }) + }) + + describe('GET /api/v1/admin/feature-flags/history', () => { + it('returns history of emergency rollbacks', async () => { + // Trigger a rollback + await request(app) + .post('/api/v1/admin/feature-flags/fast_fiat_onramp/rollback') + .send({ reason: 'Provider latency spike' }) + + const res = await request(app).get('/api/v1/admin/feature-flags/history') + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + expect(Array.isArray(res.body.data)).toBe(true) + expect(res.body.data.length).toBeGreaterThan(0) + expect(res.body.data[0].flagKey).toBe('fast_fiat_onramp') + expect(res.body.data[0].reason).toBe('Provider latency spike') + }) + }) + + describe('POST /api/v1/admin/feature-flags/:key/reset', () => { + it('resets flag back to default configuration', async () => { + // Modify override + featureFlagManager.setOverride('smart_dca_engine', { enabled: false }) + expect(featureFlagManager.evaluate('smart_dca_engine').enabled).toBe(false) + + const res = await request(app).post('/api/v1/admin/feature-flags/smart_dca_engine/reset') + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + // In test env, smart_dca_engine default is true + expect(res.body.data.enabled).toBe(true) + }) + }) + + describe('POST /api/v1/admin/feature-flags/emergency-disable-all', () => { + it('activates platform emergency kill switch', async () => { + const res = await request(app) + .post('/api/v1/admin/feature-flags/emergency-disable-all') + .send({ reason: 'Critical 0-day upstream exploit detected' }) + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + + // All other flags should now evaluate to false with GLOBAL_MAINTENANCE + expect(featureFlagManager.evaluate('smart_dca_engine').decision).toBe('GLOBAL_MAINTENANCE') + expect(featureFlagManager.evaluate('agent_autonomous_rebalance').decision).toBe('GLOBAL_MAINTENANCE') + }) + + it('requires a reason', async () => { + const res = await request(app) + .post('/api/v1/admin/feature-flags/emergency-disable-all') + .send({}) + + expect(res.status).toBe(400) + expect(res.body.error).toBe('reason is required') + }) + }) + + describe('RBAC Scopes Enforcement', () => { + it('allows read-scoped key to GET flags', async () => { + const res = await request(app) + .get('/api/v1/admin/feature-flags') + .set('x-mock-scopes', 'flags:read') + + expect(res.status).toBe(200) + }) + + it('denies read-scoped key from mutating flag state (returns 403)', async () => { + const res = await request(app) + .post('/api/v1/admin/feature-flags/smart_dca_engine/rollback') + .set('x-mock-scopes', 'flags:read') + .send({ reason: 'Unauthorized rollback attempt' }) + + expect(res.status).toBe(403) + expect(res.body.success).toBe(false) + expect(res.body.error).toContain("Admin scope 'flags:write' required") + }) + + it('allows write-scoped key to execute rollback', async () => { + const res = await request(app) + .post('/api/v1/admin/feature-flags/smart_dca_engine/rollback') + .set('x-mock-scopes', 'flags:write') + .send({ reason: 'Authorized operator rollback' }) + + expect(res.status).toBe(200) + expect(res.body.success).toBe(true) + }) + }) +}) diff --git a/tests/unit/routes/admin-messages.test.ts b/tests/unit/routes/admin-messages.test.ts index a67046fb..0ddfea95 100644 --- a/tests/unit/routes/admin-messages.test.ts +++ b/tests/unit/routes/admin-messages.test.ts @@ -30,7 +30,8 @@ jest.mock('../../../src/middleware/adminAuth', () => { } next() }, - requireAdminScope: (_scope: string) => (_req: any, _res: any, next: any) => next(), + requireAdminScope: (_scope: string) => (_req: any, _res: any, next: any) => + next(), } }) @@ -47,33 +48,35 @@ describe('Admin Messages Recovery Routes (#493)', () => { }) it('GET /api/admin/messages lists tracked messages', async () => { - const listSpy = jest.spyOn(messageDeliveryService, 'listMessages').mockResolvedValue({ - messages: [ - { - id: 'msg-1', - channel: 'TELEGRAM', - recipient: '123456', - userId: 'user-1', - category: 'ALERT', - body: 'Price alert triggered', - status: 'DELIVERED', - priority: 'NORMAL', - attempts: 1, - maxAttempts: 5, - nextAttemptAt: null, - lastError: null, - providerMessageId: 'p-1', - metadata: null, - fallbackChannel: null, - fallbackRecipient: null, - fallbackTriggered: false, - deliveredAt: new Date(), - createdAt: new Date(), - updatedAt: new Date(), - }, - ], - total: 1, - }) + const listSpy = jest + .spyOn(messageDeliveryService, 'listMessages') + .mockResolvedValue({ + messages: [ + { + id: 'msg-1', + channel: 'TELEGRAM', + recipient: '123456', + userId: 'user-1', + category: 'ALERT', + body: 'Price alert triggered', + status: 'DELIVERED', + priority: 'NORMAL', + attempts: 1, + maxAttempts: 5, + nextAttemptAt: null, + lastError: null, + providerMessageId: 'p-1', + metadata: null, + fallbackChannel: null, + fallbackRecipient: null, + fallbackTriggered: false, + deliveredAt: new Date(), + createdAt: new Date(), + updatedAt: new Date(), + }, + ], + total: 1, + }) const res = await request(app).get('/api/admin/messages') expect(res.status).toBe(200) @@ -86,10 +89,28 @@ describe('Admin Messages Recovery Routes (#493)', () => { it('GET /api/admin/messages/stats returns queue and delivery metrics', async () => { jest.spyOn(messageDeliveryService, 'getMessageStats').mockResolvedValue({ total: 10, - byStatus: { pending: 2, sending: 1, delivered: 6, failed: 0, deadLetter: 1 }, + byStatus: { + pending: 2, + sending: 1, + delivered: 6, + failed: 0, + deadLetter: 1, + }, byChannel: { - telegram: { total: 5, delivered: 4, failed: 0, pending: 1, deadLetter: 0 }, - whatsapp: { total: 5, delivered: 2, failed: 0, pending: 2, deadLetter: 1 }, + telegram: { + total: 5, + delivered: 4, + failed: 0, + pending: 1, + deadLetter: 0, + }, + whatsapp: { + total: 5, + delivered: 2, + failed: 0, + pending: 2, + deadLetter: 1, + }, }, }) @@ -129,7 +150,9 @@ describe('Admin Messages Recovery Routes (#493)', () => { expect(foundRes.body.data.id).toBe('msg-target') jest.spyOn(messageDeliveryService, 'getMessage').mockResolvedValueOnce(null) - const notFoundRes = await request(app).get('/api/admin/messages/non-existent') + const notFoundRes = await request(app).get( + '/api/admin/messages/non-existent' + ) expect(notFoundRes.status).toBe(404) expect(notFoundRes.body.success).toBe(false) }) @@ -181,7 +204,9 @@ describe('Admin Messages Recovery Routes (#493)', () => { updatedAt: new Date(), }) - const res = await request(app).post('/api/admin/messages/msg-to-retry/retry') + const res = await request(app).post( + '/api/admin/messages/msg-to-retry/retry' + ) expect(res.status).toBe(200) expect(res.body.success).toBe(true) expect(res.body.data.status).toBe('DELIVERED')