From 536532bed9925508ea8ac82f1c75ed7a02d4f2a6 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:46:30 -0700 Subject: [PATCH 1/3] fix(staging): enable LiteLLM so native-runtime agents (Scout) can answer (D20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit values-staging had no `litellm:` stanza, so it inherited the base default `enabled:false` — no litellm in commonly-staging. The staging backend still points native-runtime turns at http://litellm:4000, so every Scout onboarding REPLY turn failed ("native runtime failed: turn failed", model undefined, no cost) while the scripted welcome still posted. 2026-10-31 staging-walk incident (Sam: "is scout not working? I saw it not responding in staging"). Enable it, kept small (spend approved for Sam's checks + dev:live screenshots): - master key only (serviceKeys.fromSecret:false) — no budgeted virtual keys to mint - Codex rotator off (staging has no Codex traffic; Scout runs on deepseek-v4-flash, the cheapest route it works on). codex-cli/seed still render but idle harmlessly - no chatgpt-auth PVC (emptyDir); dev-pool so Sam-check turns aren't reclaimed mid-walk No new secret or DB: every key already exists in the staging api-keys ring, and DATABASE_URL reuses staging's own Aiven via ?schema=litellm. Guard test asserts staging (and prod) keep litellm enabled and the small config holds. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01N4VWLErW29t4LnhRfUy1NH --- .../unit/helm/litellmStagingEnabled.test.js | 49 +++++++++++++++++++ k8s/helm/commonly/values-staging.yaml | 46 +++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 backend/__tests__/unit/helm/litellmStagingEnabled.test.js diff --git a/backend/__tests__/unit/helm/litellmStagingEnabled.test.js b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js new file mode 100644 index 0000000000..65994b360d --- /dev/null +++ b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js @@ -0,0 +1,49 @@ +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); + +// Regression guard for the 2026-10-31 staging-walk incident: Scout's onboarding +// REPLY turns failed ("native runtime failed: turn failed") because the staging +// backend points native-runtime LLM calls at the in-cluster `litellm` service +// (LITELLM_BASE_URL=http://litellm:4000), but values-staging carried no `litellm:` +// stanza and so inherited the base default `enabled:false` — no litellm in the +// namespace. The scripted welcome still posted, which hid it. This test fails if +// anyone points staging's backend at a litellm that isn't deployed in the same ns. +const loadValues = (name) => yaml.load(fs.readFileSync(path.join(__dirname, + `../../../../k8s/helm/commonly/${name}`), 'utf8')); + +const staging = loadValues('values-staging.yaml'); +const dev = loadValues('values-dev.yaml'); + +describe('staging LiteLLM is enabled so native-runtime agents (Scout) can answer', () => { + test('values-staging enables litellm', () => { + // The staging backend always drives the native runtime at http://litellm:4000, + // so litellm MUST be enabled in the staging namespace or every agent turn fails. + expect(staging.litellm).toBeDefined(); + expect(staging.litellm.enabled).toBe(true); + }); + + test('prod (values-dev) also has litellm enabled — same invariant', () => { + expect(dev.litellm.enabled).toBe(true); + }); + + test('staging is kept small: master key only, no budgeted virtual keys to mint', () => { + // serviceKeys.fromSecret:true would require commonly-staging-litellm-backend-key / + // -commonly-bot-key to exist in Secret Manager or ExternalSecrets rejects the whole + // api-keys bundle. Staging intentionally uses the master key, so this stays false. + expect(staging.litellm.serviceKeys && staging.litellm.serviceKeys.fromSecret).toBe(false); + }); + + test('staging runs no Codex rotator — it carries no Codex traffic', () => { + // Base default is true (prod rotates Codex accounts); staging must override to false + // so the rotator sidecar is not rendered. Scout runs on deepseek-v4-flash. + expect(staging.litellm.codexAuthRotator.enabled).toBe(false); + }); + + test('staging litellm schedules on dev-pool so Sam-check turns are not reclaimed mid-walk', () => { + expect(staging.litellm.nodeSelector).toEqual({ pool: 'dev' }); + expect(staging.litellm.tolerations).toEqual([ + { key: 'pool', operator: 'Equal', value: 'dev', effect: 'NoSchedule' }, + ]); + }); +}); diff --git a/k8s/helm/commonly/values-staging.yaml b/k8s/helm/commonly/values-staging.yaml index bf86061483..1740977fb5 100644 --- a/k8s/helm/commonly/values-staging.yaml +++ b/k8s/helm/commonly/values-staging.yaml @@ -106,6 +106,52 @@ agents: workspace: enabled: false +# LiteLLM — ENABLED for staging (SRE 2026-10-31). Staging native agents answer through +# the native runtime -> LiteLLM, exactly as prod does; the values-staging comment above +# ("Hosted agents answer through litellm") assumed this but the stanza was missing, so +# litellm inherited the base default (enabled:false). The staging backend still pointed +# native-runtime turns at http://litellm:4000 — nonexistent in this namespace — so every +# Scout ONBOARDING-REPLY turn failed ("native runtime failed: turn failed", model +# undefined, no cost) while the scripted welcome still posted. That was the 2026-10-31 +# staging-walk incident (Sam: "is scout not working? I saw it not responding in staging"). +# +# Kept deliberately small (spend approved by Cofounder for Sam's checks + dev:live +# screenshots): +# - MASTER KEY ONLY (serviceKeys.fromSecret:false): no budgeted virtual keys to mint +# in Secret Manager. The cost ceiling is the shared litellm-config's deepseek-v4-flash +# group max_budget ($60/1mo), metered in THIS instance's router cache, independent of +# prod's litellm instance. +# - CODEX OFF (codexAuthRotator.enabled:false): staging carries no Codex traffic. Scout +# runs on deepseek-v4-flash (backend LITELLM_CHAT_MODEL), the cheapest route it works +# on. The codex-cli sidecar + codex-auth-seed init still render (ungated in the +# template) but idle harmlessly — the seed writes an empty auth.json and exits 0. +# - NO chatgpt-auth PVC (emptyDir): nothing to persist without Codex. +# - dev-pool (not spot): Scout must stay up for Sam's checks, so it should not be +# reclaimed mid-walk the way a spot node can be. +# +# No new secret and no new database: every key litellm needs already exists in the +# staging api-keys ring (deepseek-api-key, cloudflare-account-id/-workers-ai-token, +# gemini/openrouter/openai-api-key, litellm-master-key — all unconditional in the +# ExternalSecret), and DATABASE_URL reuses staging's own Aiven Postgres via +# ?schema=litellm. Guard test: backend/__tests__/unit/helm/litellmStagingEnabled.test.js. +litellm: + enabled: true + serviceKeys: + fromSecret: false + codexBypassLitellm: false + codexAuthRotator: + enabled: false + chatgptAuth: + persistence: + enabled: false + nodeSelector: + pool: dev + tolerations: + - key: "pool" + operator: "Equal" + value: "dev" + effect: "NoSchedule" + backend: env: agentComputerEnabled: "true" From 20c5969b682413745e1b18f6521da8340ef46ccd Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:58:22 -0700 Subject: [PATCH 2/3] fix(staging): schedule litellm on untainted default-pool, drop dead codexBypassLitellm (review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Writer's subagent review (D218) caught a scheduling risk and a dead-config line. - nodeSelector pool:dev -> pool:default (drop the toleration). The reviewer's STATED reason ("no pool=dev node in staging") is a false positive — staging and prod are two namespaces in ONE cluster, node pools are cluster-scoped, and prod litellm runs on the dev-pool node right now with this exact selector. But the reviewer's TARGET is right for a better reason: dev-pool is a SINGLE node already running prod litellm + clawdbot + cloud-codex, so pinning the reckless staging lane there lets it pressure the one node serving PROD agent runtimes. The untainted default-pool is two nodes, not spot (Scout is never reclaimed mid-walk), isolated from prod, and schedulable: the pod requests only 120m CPU / 576Mi and a default-pool node had ~377m CPU free when measured. Empty tolerations + pool=default keep it off both tainted pools (dev, spot). Prod (values-dev) is unchanged — still dev-pool + toleration. - Remove `codexBypassLitellm: false`: dead config restating values.yaml's base default; the backend reads CODEX_BYPASS_LITELLM from backend.env, not this chart field. codex-cli sidecar + codex-auth-seed render ungated in the shared template but do not gate litellm readiness or block Scout; gating them on codex-enabled is a tracked follow-up. Guard test updated to assert default-pool + empty tolerations. helm template verified: staging litellm -> nodeSelector pool:default, no tolerations; prod render unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01N4VWLErW29t4LnhRfUy1NH --- .../unit/helm/litellmStagingEnabled.test.js | 14 +++++++---- k8s/helm/commonly/values-staging.yaml | 24 ++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/backend/__tests__/unit/helm/litellmStagingEnabled.test.js b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js index 65994b360d..4d9b91b1be 100644 --- a/backend/__tests__/unit/helm/litellmStagingEnabled.test.js +++ b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js @@ -40,10 +40,14 @@ describe('staging LiteLLM is enabled so native-runtime agents (Scout) can answer expect(staging.litellm.codexAuthRotator.enabled).toBe(false); }); - test('staging litellm schedules on dev-pool so Sam-check turns are not reclaimed mid-walk', () => { - expect(staging.litellm.nodeSelector).toEqual({ pool: 'dev' }); - expect(staging.litellm.tolerations).toEqual([ - { key: 'pool', operator: 'Equal', value: 'dev', effect: 'NoSchedule' }, - ]); + test('staging litellm schedules on the untainted default-pool — stable (not spot), isolated from prod dev-pool', () => { + // It must NOT be on spot (Scout would be reclaimed mid-walk) and must NOT be pinned + // to prod's single dev-pool node (which runs prod litellm/clawdbot/cloud-codex — the + // reckless staging lane must not pressure it). Empty tolerations keep it off both + // tainted pools; pool=default selects the untainted one. The pod requests only + // 120m CPU / 576Mi, which fits default-pool's measured headroom. + expect(staging.litellm.nodeSelector).toEqual({ pool: 'default' }); + const tol = staging.litellm.tolerations; + expect(tol == null || (Array.isArray(tol) && tol.length === 0)).toBe(true); }); }); diff --git a/k8s/helm/commonly/values-staging.yaml b/k8s/helm/commonly/values-staging.yaml index 1740977fb5..83cd90dad6 100644 --- a/k8s/helm/commonly/values-staging.yaml +++ b/k8s/helm/commonly/values-staging.yaml @@ -126,8 +126,17 @@ agents: # on. The codex-cli sidecar + codex-auth-seed init still render (ungated in the # template) but idle harmlessly — the seed writes an empty auth.json and exits 0. # - NO chatgpt-auth PVC (emptyDir): nothing to persist without Codex. -# - dev-pool (not spot): Scout must stay up for Sam's checks, so it should not be -# reclaimed mid-walk the way a spot node can be. +# - default-pool (untainted, NOT prod's dev-pool, NOT spot): Scout must stay up for +# Sam's checks, so it must not be reclaimed mid-walk the way the spot-pool (which the +# rest of staging rides) can be. It is NOT pinned to the shared cluster's dev-pool +# either: that is a SINGLE node already running prod litellm + clawdbot + cloud-codex, +# and pinning the reckless staging lane there would let it pressure the one node +# serving PROD agent runtimes (the staging->prod blast-radius this whole file avoids). +# The default-pool is two untainted nodes with headroom for this small pod (requests +# 120m CPU / 576Mi; a default-pool node had ~377m CPU free when measured), isolated +# from prod and stable. (The codex-cli sidecar + codex-auth-seed init still render +# ungated in the shared template but do not gate litellm readiness or block Scout; +# gating them on codex-enabled is a follow-up — TASK filed.) # # No new secret and no new database: every key litellm needs already exists in the # staging api-keys ring (deepseek-api-key, cloudflare-account-id/-workers-ai-token, @@ -138,19 +147,16 @@ litellm: enabled: true serviceKeys: fromSecret: false - codexBypassLitellm: false codexAuthRotator: enabled: false chatgptAuth: persistence: enabled: false + # No tolerations + pool=default => the untainted default-pool ONLY (both dev-pool and + # spot-pool are tainted, so an untolerated pod cannot land on either). See the + # default-pool rationale in the block comment above. nodeSelector: - pool: dev - tolerations: - - key: "pool" - operator: "Equal" - value: "dev" - effect: "NoSchedule" + pool: default backend: env: From b10cfceac9688917db20e11f7605c13e4e138f3a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 05:06:36 -0700 Subject: [PATCH 3/3] test(staging): assert litellm chatgptAuth persistence stays emptyDir (CodeRabbit #3077) Guard against a silent flip of litellm.chatgptAuth.persistence.enabled to true, which would render a LiteLLM auth PVC in the reckless, resettable staging namespace. Asserted at the values altitude (the test's level); the rendered emptyDir / PVC-absence is exercised by the Chart Lint CI tier that templates the chart. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01N4VWLErW29t4LnhRfUy1NH --- .../__tests__/unit/helm/litellmStagingEnabled.test.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/__tests__/unit/helm/litellmStagingEnabled.test.js b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js index 4d9b91b1be..4b03a8de3e 100644 --- a/backend/__tests__/unit/helm/litellmStagingEnabled.test.js +++ b/backend/__tests__/unit/helm/litellmStagingEnabled.test.js @@ -34,6 +34,16 @@ describe('staging LiteLLM is enabled so native-runtime agents (Scout) can answer expect(staging.litellm.serviceKeys && staging.litellm.serviceKeys.fromSecret).toBe(false); }); + test('staging keeps chatgpt-auth storage as emptyDir (no PVC) — nothing to persist without Codex', () => { + // CodeRabbit #3077: a silent flip to persistence.enabled:true would make Helm render a + // LiteLLM auth PVC in the reckless, resettable staging namespace. Guard the effective + // false value at this test's altitude (values); the rendered emptyDir/PVC-absence is + // exercised by the Chart Lint CI tier, which templates the chart. + expect(staging.litellm.chatgptAuth + && staging.litellm.chatgptAuth.persistence + && staging.litellm.chatgptAuth.persistence.enabled).toBe(false); + }); + test('staging runs no Codex rotator — it carries no Codex traffic', () => { // Base default is true (prod rotates Codex accounts); staging must override to false // so the rotator sidecar is not rendered. Scout runs on deepseek-v4-flash.