diff --git a/backend/__tests__/unit/helm/litellmStagingEnabled.test.js b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js new file mode 100644 index 0000000000..4b03a8de3e --- /dev/null +++ b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js @@ -0,0 +1,63 @@ +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); + +// Regression guard for the 2026-10-31 staging-walk incident: Scout's onboarding +// REPLY turns failed ("native runtime failed: turn failed") because the staging +// backend points native-runtime LLM calls at the in-cluster `litellm` service +// (LITELLM_BASE_URL=http://litellm:4000), but values-staging carried no `litellm:` +// stanza and so inherited the base default `enabled:false` — no litellm in the +// namespace. The scripted welcome still posted, which hid it. This test fails if +// anyone points staging's backend at a litellm that isn't deployed in the same ns. +const loadValues = (name) => yaml.load(fs.readFileSync(path.join(__dirname, + `../../../../k8s/helm/commonly/${name}`), 'utf8')); + +const staging = loadValues('values-staging.yaml'); +const dev = loadValues('values-dev.yaml'); + +describe('staging LiteLLM is enabled so native-runtime agents (Scout) can answer', () => { + test('values-staging enables litellm', () => { + // The staging backend always drives the native runtime at http://litellm:4000, + // so litellm MUST be enabled in the staging namespace or every agent turn fails. + expect(staging.litellm).toBeDefined(); + expect(staging.litellm.enabled).toBe(true); + }); + + test('prod (values-dev) also has litellm enabled — same invariant', () => { + expect(dev.litellm.enabled).toBe(true); + }); + + test('staging is kept small: master key only, no budgeted virtual keys to mint', () => { + // serviceKeys.fromSecret:true would require commonly-staging-litellm-backend-key / + // -commonly-bot-key to exist in Secret Manager or ExternalSecrets rejects the whole + // api-keys bundle. Staging intentionally uses the master key, so this stays false. + expect(staging.litellm.serviceKeys && staging.litellm.serviceKeys.fromSecret).toBe(false); + }); + + test('staging keeps chatgpt-auth storage as emptyDir (no PVC) — nothing to persist without Codex', () => { + // CodeRabbit #3077: a silent flip to persistence.enabled:true would make Helm render a + // LiteLLM auth PVC in the reckless, resettable staging namespace. Guard the effective + // false value at this test's altitude (values); the rendered emptyDir/PVC-absence is + // exercised by the Chart Lint CI tier, which templates the chart. + expect(staging.litellm.chatgptAuth + && staging.litellm.chatgptAuth.persistence + && staging.litellm.chatgptAuth.persistence.enabled).toBe(false); + }); + + test('staging runs no Codex rotator — it carries no Codex traffic', () => { + // Base default is true (prod rotates Codex accounts); staging must override to false + // so the rotator sidecar is not rendered. Scout runs on deepseek-v4-flash. + expect(staging.litellm.codexAuthRotator.enabled).toBe(false); + }); + + test('staging litellm schedules on the untainted default-pool — stable (not spot), isolated from prod dev-pool', () => { + // It must NOT be on spot (Scout would be reclaimed mid-walk) and must NOT be pinned + // to prod's single dev-pool node (which runs prod litellm/clawdbot/cloud-codex — the + // reckless staging lane must not pressure it). Empty tolerations keep it off both + // tainted pools; pool=default selects the untainted one. The pod requests only + // 120m CPU / 576Mi, which fits default-pool's measured headroom. + expect(staging.litellm.nodeSelector).toEqual({ pool: 'default' }); + const tol = staging.litellm.tolerations; + expect(tol == null || (Array.isArray(tol) && tol.length === 0)).toBe(true); + }); +}); diff --git a/k8s/helm/commonly/values-staging.yaml b/k8s/helm/commonly/values-staging.yaml index bf86061483..83cd90dad6 100644 --- a/k8s/helm/commonly/values-staging.yaml +++ b/k8s/helm/commonly/values-staging.yaml @@ -106,6 +106,58 @@ agents: workspace: enabled: false +# LiteLLM — ENABLED for staging (SRE 2026-10-31). Staging native agents answer through +# the native runtime -> LiteLLM, exactly as prod does; the values-staging comment above +# ("Hosted agents answer through litellm") assumed this but the stanza was missing, so +# litellm inherited the base default (enabled:false). The staging backend still pointed +# native-runtime turns at http://litellm:4000 — nonexistent in this namespace — so every +# Scout ONBOARDING-REPLY turn failed ("native runtime failed: turn failed", model +# undefined, no cost) while the scripted welcome still posted. That was the 2026-10-31 +# staging-walk incident (Sam: "is scout not working? I saw it not responding in staging"). +# +# Kept deliberately small (spend approved by Cofounder for Sam's checks + dev:live +# screenshots): +# - MASTER KEY ONLY (serviceKeys.fromSecret:false): no budgeted virtual keys to mint +# in Secret Manager. The cost ceiling is the shared litellm-config's deepseek-v4-flash +# group max_budget ($60/1mo), metered in THIS instance's router cache, independent of +# prod's litellm instance. +# - CODEX OFF (codexAuthRotator.enabled:false): staging carries no Codex traffic. Scout +# runs on deepseek-v4-flash (backend LITELLM_CHAT_MODEL), the cheapest route it works +# on. The codex-cli sidecar + codex-auth-seed init still render (ungated in the +# template) but idle harmlessly — the seed writes an empty auth.json and exits 0. +# - NO chatgpt-auth PVC (emptyDir): nothing to persist without Codex. +# - default-pool (untainted, NOT prod's dev-pool, NOT spot): Scout must stay up for +# Sam's checks, so it must not be reclaimed mid-walk the way the spot-pool (which the +# rest of staging rides) can be. It is NOT pinned to the shared cluster's dev-pool +# either: that is a SINGLE node already running prod litellm + clawdbot + cloud-codex, +# and pinning the reckless staging lane there would let it pressure the one node +# serving PROD agent runtimes (the staging->prod blast-radius this whole file avoids). +# The default-pool is two untainted nodes with headroom for this small pod (requests +# 120m CPU / 576Mi; a default-pool node had ~377m CPU free when measured), isolated +# from prod and stable. (The codex-cli sidecar + codex-auth-seed init still render +# ungated in the shared template but do not gate litellm readiness or block Scout; +# gating them on codex-enabled is a follow-up — TASK filed.) +# +# No new secret and no new database: every key litellm needs already exists in the +# staging api-keys ring (deepseek-api-key, cloudflare-account-id/-workers-ai-token, +# gemini/openrouter/openai-api-key, litellm-master-key — all unconditional in the +# ExternalSecret), and DATABASE_URL reuses staging's own Aiven Postgres via +# ?schema=litellm. Guard test: backend/__tests__/unit/helm/litellmStagingEnabled.test.js. +litellm: + enabled: true + serviceKeys: + fromSecret: false + codexAuthRotator: + enabled: false + chatgptAuth: + persistence: + enabled: false + # No tolerations + pool=default => the untainted default-pool ONLY (both dev-pool and + # spot-pool are tainted, so an untolerated pod cannot land on either). See the + # default-pool rationale in the block comment above. + nodeSelector: + pool: default + backend: env: agentComputerEnabled: "true"