From 4a2512c0fb7be7f42e37e31bccf929e2b2e4da9e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:33:38 -0700 Subject: [PATCH 1/2] fix(litellm): size per-agent key budget, add key_alias, tighten proxy config Blast-radius control for the public /v1 endpoint (BYO goose/pi callers). A leaked virtual key spends from any network, so bound what one key can do: - Raise per-key max_budget 30 -> 150 USD, kept paired with budget_duration 30d. The flat $30 (D111) sat below the heaviest agent's measured monthly spend (theo ~$88/mo; 2026-10-28 key audit), so a fresh key would 429 within one 30d window. $150/30d clears it with headroom while still bounding a leaked key to a finite monthly spend instead of unlimited. - Stamp a unique, attributable key_alias on both mint paths (the audit found 73 keys with no alias). Timestamp suffix avoids a LiteLLM key_alias uniqueness 409 on delete-then-remint for the same agent. - litellm config: set allow_client_side_credentials:false explicitly and tighten ui_access_mode "all" -> "admin_only" (defense-in-depth; the UI is not publicly routed, only /v1). Expiry + provisioner renewal (re-mint under 7d left, no gap) and a backfill of the 44 existing unbounded keys are separate follow-up rows. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01MDs9DS64Df3HsKsCHiKBWN --- .../agentProvisionerServiceK8s.test.js | 28 +++++++++++++++++-- .../services/agentProvisionerServiceK8s.ts | 26 +++++++++++++---- .../templates/configmaps/litellm-config.yaml | 11 ++++++-- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js b/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js index afa0d9badd..61ab83f8f1 100644 --- a/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js +++ b/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js @@ -570,7 +570,9 @@ describe('issueLiteLLMVirtualKey never sends the master key to LiteLLM', () => { const [url, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); expect(url).toBe('http://litellm:4000/key/generate'); - expect(body.max_budget).toBe(30); + // Sized above the heaviest agent's measured monthly (~$88/mo) with headroom; + // a bare max_budget is lifetime in LiteLLM, so budget_duration is required. + expect(body.max_budget).toBe(150); expect(body.budget_duration).toBe('30d'); }); @@ -582,9 +584,31 @@ describe('issueLiteLLMVirtualKey never sends the master key to LiteLLM', () => { const [url, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); expect(url).toBe('http://litellm:4000/key/generate'); expect(body.user_id).toBe('community-agent'); - expect(body.max_budget).toBe(30); + expect(body.max_budget).toBe(150); expect(body.budget_duration).toBe('30d'); }); + + // 2026-10-28 key audit found 73 keys with no alias — attribution had to lean on + // user_id. Both mint paths now stamp a unique, human-readable key_alias so a + // key is traceable to its agent, and the uniqueness suffix avoids a 409 on + // delete-then-remint for the same agent (which would silently fail the seat). + it('stamps a unique, attributable key_alias on the virtual key', async () => { + k8s.__mock.execStdout = ''; + + await issueLiteLLMVirtualKey('theo'); + + const [, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); + expect(body.key_alias).toMatch(/^agent-theo-\d+$/); + }); + + it('stamps a unique, attributable key_alias on the community/openrouter key', async () => { + k8s.__mock.execStdout = ''; + + await issueLiteLLMOpenRouterKey('community-agent'); + + const [, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); + expect(body.key_alias).toMatch(/^agent-community-agent-openrouter-\d+$/); + }); }); diff --git a/backend/services/agentProvisionerServiceK8s.ts b/backend/services/agentProvisionerServiceK8s.ts index 2a7ce0523c..38ee68452e 100644 --- a/backend/services/agentProvisionerServiceK8s.ts +++ b/backend/services/agentProvisionerServiceK8s.ts @@ -54,13 +54,21 @@ const AGENT_TOLERATIONS = (() => { // misbehaving agent can't monopolize the single-replica LiteLLM proxy. const LITELLM_KEY_MAX_PARALLEL_REQUESTS = 4; const LITELLM_KEY_RPM_LIMIT = 120; -// Monthly spend cap per issued key (D111, 2026-10-27). Unlike the breakers above -// this IS a usage quota: since the GPT aliases were repointed at DeepSeek, and dead -// routes fall back to it, calls through these keys are paid, while the -// deepseek-v4-flash group budget sits on that model name, which alias callers never -// touch. The whole proxy runs ~$15/mo, so $30 per agent only trips on a runaway. +// Rolling spend cap per issued key. Unlike the breakers above this IS a usage +// quota: since the GPT aliases were repointed at DeepSeek, and dead routes fall +// back to it, calls through these keys are paid, while the deepseek-v4-flash +// group budget sits on that model name, which alias callers never touch. +// +// Sized from measured monthly spend (2026-10-28 key audit, lifetime/age): the +// heaviest agent, theo, runs ~$75-88/mo; nova ~$37/mo; others <$30/mo. The +// first cut was a flat $30 (D111, 2026-10-27), which would 429 theo and nova +// within one 30d window — a self-inflicted silent-seat outage. $150/30d clears +// the heaviest legit agent with ~70% headroom while still bounding a leaked key +// to a finite monthly spend instead of unlimited. budget_duration is REQUIRED: +// a bare max_budget is lifetime in LiteLLM, so it would silence any key already +// past the cap the moment it applied. Tighten per-agent later if needed. // A reused key keeps whatever budget it already carries; this stamps new ones. -const LITELLM_KEY_MAX_BUDGET_USD = 30; +const LITELLM_KEY_MAX_BUDGET_USD = 150; const LITELLM_KEY_BUDGET_DURATION = '30d'; /** @@ -1660,6 +1668,11 @@ const issueLiteLLMVirtualKey = async (agentId: any) => { rpm_limit: LITELLM_KEY_RPM_LIMIT, max_budget: LITELLM_KEY_MAX_BUDGET_USD, budget_duration: LITELLM_KEY_BUDGET_DURATION, + // Human-readable attribution on the key (the 2026-10-28 audit found 73 keys + // with no alias). Suffixed with a mint timestamp because LiteLLM enforces + // key_alias uniqueness: a bare `agent-` would 409 when a stale key is + // deleted and re-minted for the same agent, silently failing the seat. + key_alias: `agent-${agentId}-${Date.now()}`, metadata: { agent_id: agentId, provisioned_at: new Date().toISOString() }, }, { headers }, @@ -1741,6 +1754,7 @@ const issueLiteLLMOpenRouterKey = async (agentId: any) => { rpm_limit: LITELLM_KEY_RPM_LIMIT, max_budget: LITELLM_KEY_MAX_BUDGET_USD, budget_duration: LITELLM_KEY_BUDGET_DURATION, + key_alias: `agent-${agentId}-openrouter-${Date.now()}`, metadata: { agent_id: agentId, key_type: 'openrouter', provisioned_at: new Date().toISOString() }, }, { headers }, diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index 0739d438e3..06651939b8 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -344,8 +344,15 @@ data: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL store_model_in_db: true - # Allow UI access - ui_access_mode: "all" + # Restrict the admin UI to admin users. The UI is not publicly routed + # (the nginx ingress maps only litellm.commonly.me/v1), so this is + # defense-in-depth against a future ingress/tunnel change exposing it. + ui_access_mode: "admin_only" + # Reject client-supplied api_base / api_key on requests. The /v1 endpoint + # is public for BYO goose/pi callers and is key-gated, so a leaked virtual + # key must NOT be usable to proxy to arbitrary upstreams on our spend. + # Default is already false; set explicitly so it can't drift (2026-10-28). + allow_client_side_credentials: false # Spend log retention — without these, LiteLLM_SpendLogs grows unbounded # (prod dev observed 5+ GB on the 8 GiB Cloud SQL tier before this cap). # STORE_PROMPTS_IN_SPEND_LOGS=false keeps raw prompt and response text From f52b7349996610f1d984b5af3144bfed1e14d14e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Wed, 28 Oct 2026 05:45:16 -0700 Subject: [PATCH 2/2] fix(provisioner): collision-free key_alias + correct the client-creds comment (#2961 gate HOLD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the gate HOLD on #2961: - key_alias: append crypto.randomBytes(6) to the Date.now() suffix on both mint paths. Two concurrent mints for the same agent can read the same millisecond; LiteLLM then 409s the duplicate alias and silently fails the seat. The random suffix makes the alias collision-free regardless of clock. - litellm-config.yaml: the allow_client_side_credentials comment overclaimed. On LiteLLM v1.106.2 _BANNED_REQUEST_BODY_PARAMS covers api_base/base_url and the provider endpoint/credential-override fields but NOT api_key (verified against the v1.106.2 source). Comment now states what the setting actually blocks; the stated threat — a leaked key redirecting to arbitrary upstreams on our spend — is still fully covered by the api_base/base_url ban. - test: key_alias assertions tightened to require the random hex suffix. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01MDs9DS64Df3HsKsCHiKBWN --- .../services/agentProvisionerServiceK8s.test.js | 4 ++-- backend/services/agentProvisionerServiceK8s.ts | 13 ++++++++----- .../templates/configmaps/litellm-config.yaml | 11 ++++++++--- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js b/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js index 61ab83f8f1..1738be5382 100644 --- a/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js +++ b/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js @@ -598,7 +598,7 @@ describe('issueLiteLLMVirtualKey never sends the master key to LiteLLM', () => { await issueLiteLLMVirtualKey('theo'); const [, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); - expect(body.key_alias).toMatch(/^agent-theo-\d+$/); + expect(body.key_alias).toMatch(/^agent-theo-\d+-[0-9a-f]{12}$/); }); it('stamps a unique, attributable key_alias on the community/openrouter key', async () => { @@ -607,7 +607,7 @@ describe('issueLiteLLMVirtualKey never sends the master key to LiteLLM', () => { await issueLiteLLMOpenRouterKey('community-agent'); const [, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); - expect(body.key_alias).toMatch(/^agent-community-agent-openrouter-\d+$/); + expect(body.key_alias).toMatch(/^agent-community-agent-openrouter-\d+-[0-9a-f]{12}$/); }); }); diff --git a/backend/services/agentProvisionerServiceK8s.ts b/backend/services/agentProvisionerServiceK8s.ts index 38ee68452e..0e39e3d49f 100644 --- a/backend/services/agentProvisionerServiceK8s.ts +++ b/backend/services/agentProvisionerServiceK8s.ts @@ -11,6 +11,7 @@ const stream = require('stream'); const fs = require('fs'); const os = require('os'); const path = require('path'); +const crypto = require('crypto'); const PodAsset = require('../models/PodAsset'); const PodAssetService = require('./podAssetService'); const GlobalModelConfigService = require('./globalModelConfigService'); @@ -1669,10 +1670,12 @@ const issueLiteLLMVirtualKey = async (agentId: any) => { max_budget: LITELLM_KEY_MAX_BUDGET_USD, budget_duration: LITELLM_KEY_BUDGET_DURATION, // Human-readable attribution on the key (the 2026-10-28 audit found 73 keys - // with no alias). Suffixed with a mint timestamp because LiteLLM enforces - // key_alias uniqueness: a bare `agent-` would 409 when a stale key is - // deleted and re-minted for the same agent, silently failing the seat. - key_alias: `agent-${agentId}-${Date.now()}`, + // with no alias). Suffixed with a mint timestamp AND 6 random bytes because + // LiteLLM enforces key_alias uniqueness: a bare `agent-` would 409 when a + // stale key is deleted and re-minted for the same agent, and two concurrent + // mints can read the same Date.now() millisecond — either silently fails the + // seat. The random suffix makes the alias collision-free regardless of clock. + key_alias: `agent-${agentId}-${Date.now()}-${crypto.randomBytes(6).toString('hex')}`, metadata: { agent_id: agentId, provisioned_at: new Date().toISOString() }, }, { headers }, @@ -1754,7 +1757,7 @@ const issueLiteLLMOpenRouterKey = async (agentId: any) => { rpm_limit: LITELLM_KEY_RPM_LIMIT, max_budget: LITELLM_KEY_MAX_BUDGET_USD, budget_duration: LITELLM_KEY_BUDGET_DURATION, - key_alias: `agent-${agentId}-openrouter-${Date.now()}`, + key_alias: `agent-${agentId}-openrouter-${Date.now()}-${crypto.randomBytes(6).toString('hex')}`, metadata: { agent_id: agentId, key_type: 'openrouter', provisioned_at: new Date().toISOString() }, }, { headers }, diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index 06651939b8..0ded542327 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -348,9 +348,14 @@ data: # (the nginx ingress maps only litellm.commonly.me/v1), so this is # defense-in-depth against a future ingress/tunnel change exposing it. ui_access_mode: "admin_only" - # Reject client-supplied api_base / api_key on requests. The /v1 endpoint - # is public for BYO goose/pi callers and is key-gated, so a leaked virtual - # key must NOT be usable to proxy to arbitrary upstreams on our spend. + # Reject client-supplied upstream overrides on requests: api_base / base_url + # and the provider endpoint + credential-override fields in LiteLLM's + # _BANNED_REQUEST_BODY_PARAMS. The /v1 endpoint is public for BYO goose/pi + # callers and is key-gated, so a leaked virtual key must NOT be usable to + # redirect to arbitrary upstreams on our spend — which this blocks. + # NOTE: on v1.106.2 this does NOT ban a request-body `api_key` (it is not in + # _BANNED_REQUEST_BODY_PARAMS); that path would bill the caller's own key, + # not our spend, so it is out of scope for this setting. # Default is already false; set explicitly so it can't drift (2026-10-28). allow_client_side_credentials: false # Spend log retention — without these, LiteLLM_SpendLogs grows unbounded