fix(staging): enable LiteLLM so native-runtime agents (Scout) can answer

backend/__tests__/unit/helm/litellmStagingEnabled.test.js

@@ -0,0 +1,63 @@
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+// Regression guard for the 2026-10-31 staging-walk incident: Scout's onboarding
+// REPLY turns failed ("native runtime failed: turn failed") because the staging
+// backend points native-runtime LLM calls at the in-cluster `litellm` service
+// (LITELLM_BASE_URL=http://litellm:4000), but values-staging carried no `litellm:`
+// stanza and so inherited the base default `enabled:false` — no litellm in the
+// namespace. The scripted welcome still posted, which hid it. This test fails if
+// anyone points staging's backend at a litellm that isn't deployed in the same ns.
+const loadValues = (name) => yaml.load(fs.readFileSync(path.join(__dirname,
+  `../../../../k8s/helm/commonly/${name}`), 'utf8'));
+
+const staging = loadValues('values-staging.yaml');
+const dev = loadValues('values-dev.yaml');
+
+describe('staging LiteLLM is enabled so native-runtime agents (Scout) can answer', () => {
+  test('values-staging enables litellm', () => {
+    // The staging backend always drives the native runtime at http://litellm:4000,
+    // so litellm MUST be enabled in the staging namespace or every agent turn fails.
+    expect(staging.litellm).toBeDefined();
+    expect(staging.litellm.enabled).toBe(true);
+  });
+
+  test('prod (values-dev) also has litellm enabled — same invariant', () => {
+    expect(dev.litellm.enabled).toBe(true);
+  });
+
+  test('staging is kept small: master key only, no budgeted virtual keys to mint', () => {
+    // serviceKeys.fromSecret:true would require commonly-staging-litellm-backend-key /
+    // -commonly-bot-key to exist in Secret Manager or ExternalSecrets rejects the whole
+    // api-keys bundle. Staging intentionally uses the master key, so this stays false.
+    expect(staging.litellm.serviceKeys && staging.litellm.serviceKeys.fromSecret).toBe(false);
+  });
+
+  test('staging keeps chatgpt-auth storage as emptyDir (no PVC) — nothing to persist without Codex', () => {
+    // CodeRabbit #3077: a silent flip to persistence.enabled:true would make Helm render a
+    // LiteLLM auth PVC in the reckless, resettable staging namespace. Guard the effective
+    // false value at this test's altitude (values); the rendered emptyDir/PVC-absence is
+    // exercised by the Chart Lint CI tier, which templates the chart.
+    expect(staging.litellm.chatgptAuth
+      && staging.litellm.chatgptAuth.persistence
+      && staging.litellm.chatgptAuth.persistence.enabled).toBe(false);
+  });
+
+  test('staging runs no Codex rotator — it carries no Codex traffic', () => {
+    // Base default is true (prod rotates Codex accounts); staging must override to false
+    // so the rotator sidecar is not rendered. Scout runs on deepseek-v4-flash.
+    expect(staging.litellm.codexAuthRotator.enabled).toBe(false);
+  });
+
+  test('staging litellm schedules on the untainted default-pool — stable (not spot), isolated from prod dev-pool', () => {
+    // It must NOT be on spot (Scout would be reclaimed mid-walk) and must NOT be pinned
+    // to prod's single dev-pool node (which runs prod litellm/clawdbot/cloud-codex — the
+    // reckless staging lane must not pressure it). Empty tolerations keep it off both
+    // tainted pools; pool=default selects the untainted one. The pod requests only
+    // 120m CPU / 576Mi, which fits default-pool's measured headroom.
+    expect(staging.litellm.nodeSelector).toEqual({ pool: 'default' });
+    const tol = staging.litellm.tolerations;
+    expect(tol == null || (Array.isArray(tol) && tol.length === 0)).toBe(true);
+  });
+});

k8s/helm/commonly/values-staging.yaml

@@ -106,6 +106,58 @@ agents:
       workspace:
         enabled: false
 
+# LiteLLM — ENABLED for staging (SRE 2026-10-31). Staging native agents answer through
+# the native runtime -> LiteLLM, exactly as prod does; the values-staging comment above
+# ("Hosted agents answer through litellm") assumed this but the stanza was missing, so
+# litellm inherited the base default (enabled:false). The staging backend still pointed
+# native-runtime turns at http://litellm:4000 — nonexistent in this namespace — so every
+# Scout ONBOARDING-REPLY turn failed ("native runtime failed: turn failed", model
+# undefined, no cost) while the scripted welcome still posted. That was the 2026-10-31
+# staging-walk incident (Sam: "is scout not working? I saw it not responding in staging").
+#
+# Kept deliberately small (spend approved by Cofounder for Sam's checks + dev:live
+# screenshots):
+#   - MASTER KEY ONLY (serviceKeys.fromSecret:false): no budgeted virtual keys to mint
+#     in Secret Manager. The cost ceiling is the shared litellm-config's deepseek-v4-flash
+#     group max_budget ($60/1mo), metered in THIS instance's router cache, independent of
+#     prod's litellm instance.
+#   - CODEX OFF (codexAuthRotator.enabled:false): staging carries no Codex traffic. Scout
+#     runs on deepseek-v4-flash (backend LITELLM_CHAT_MODEL), the cheapest route it works
+#     on. The codex-cli sidecar + codex-auth-seed init still render (ungated in the
+#     template) but idle harmlessly — the seed writes an empty auth.json and exits 0.
+#   - NO chatgpt-auth PVC (emptyDir): nothing to persist without Codex.
+#   - default-pool (untainted, NOT prod's dev-pool, NOT spot): Scout must stay up for
+#     Sam's checks, so it must not be reclaimed mid-walk the way the spot-pool (which the
+#     rest of staging rides) can be. It is NOT pinned to the shared cluster's dev-pool
+#     either: that is a SINGLE node already running prod litellm + clawdbot + cloud-codex,
+#     and pinning the reckless staging lane there would let it pressure the one node
+#     serving PROD agent runtimes (the staging->prod blast-radius this whole file avoids).
+#     The default-pool is two untainted nodes with headroom for this small pod (requests
+#     120m CPU / 576Mi; a default-pool node had ~377m CPU free when measured), isolated
+#     from prod and stable. (The codex-cli sidecar + codex-auth-seed init still render
+#     ungated in the shared template but do not gate litellm readiness or block Scout;
+#     gating them on codex-enabled is a follow-up — TASK filed.)
+#
+# No new secret and no new database: every key litellm needs already exists in the
+# staging api-keys ring (deepseek-api-key, cloudflare-account-id/-workers-ai-token,
+# gemini/openrouter/openai-api-key, litellm-master-key — all unconditional in the
+# ExternalSecret), and DATABASE_URL reuses staging's own Aiven Postgres via
+# ?schema=litellm. Guard test: backend/__tests__/unit/helm/litellmStagingEnabled.test.js.
+litellm:
+  enabled: true
+  serviceKeys:
+    fromSecret: false
+  codexAuthRotator:
+    enabled: false
+  chatgptAuth:
+    persistence:
+      enabled: false
+  # No tolerations + pool=default => the untainted default-pool ONLY (both dev-pool and
+  # spot-pool are tainted, so an untolerated pod cannot land on either). See the
+  # default-pool rationale in the block comment above.
+  nodeSelector:
+    pool: default
+
 backend:
   env:
     agentComputerEnabled: "true"