diff --git a/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js b/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js index afa0d9badd..1738be5382 100644 --- a/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js +++ b/backend/__tests__/unit/services/agentProvisionerServiceK8s.test.js @@ -570,7 +570,9 @@ describe('issueLiteLLMVirtualKey never sends the master key to LiteLLM', () => { const [url, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); expect(url).toBe('http://litellm:4000/key/generate'); - expect(body.max_budget).toBe(30); + // Sized above the heaviest agent's measured monthly (~$88/mo) with headroom; + // a bare max_budget is lifetime in LiteLLM, so budget_duration is required. + expect(body.max_budget).toBe(150); expect(body.budget_duration).toBe('30d'); }); @@ -582,9 +584,31 @@ describe('issueLiteLLMVirtualKey never sends the master key to LiteLLM', () => { const [url, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); expect(url).toBe('http://litellm:4000/key/generate'); expect(body.user_id).toBe('community-agent'); - expect(body.max_budget).toBe(30); + expect(body.max_budget).toBe(150); expect(body.budget_duration).toBe('30d'); }); + + // 2026-10-28 key audit found 73 keys with no alias — attribution had to lean on + // user_id. Both mint paths now stamp a unique, human-readable key_alias so a + // key is traceable to its agent, and the uniqueness suffix avoids a 409 on + // delete-then-remint for the same agent (which would silently fail the seat). + it('stamps a unique, attributable key_alias on the virtual key', async () => { + k8s.__mock.execStdout = ''; + + await issueLiteLLMVirtualKey('theo'); + + const [, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); + expect(body.key_alias).toMatch(/^agent-theo-\d+-[0-9a-f]{12}$/); + }); + + it('stamps a unique, attributable key_alias on the community/openrouter key', async () => { + k8s.__mock.execStdout = ''; + + await issueLiteLLMOpenRouterKey('community-agent'); + + const [, body] = axios.post.mock.calls.find(([u]) => u.endsWith('/key/generate')); + expect(body.key_alias).toMatch(/^agent-community-agent-openrouter-\d+-[0-9a-f]{12}$/); + }); }); diff --git a/backend/services/agentProvisionerServiceK8s.ts b/backend/services/agentProvisionerServiceK8s.ts index 2a7ce0523c..0e39e3d49f 100644 --- a/backend/services/agentProvisionerServiceK8s.ts +++ b/backend/services/agentProvisionerServiceK8s.ts @@ -11,6 +11,7 @@ const stream = require('stream'); const fs = require('fs'); const os = require('os'); const path = require('path'); +const crypto = require('crypto'); const PodAsset = require('../models/PodAsset'); const PodAssetService = require('./podAssetService'); const GlobalModelConfigService = require('./globalModelConfigService'); @@ -54,13 +55,21 @@ const AGENT_TOLERATIONS = (() => { // misbehaving agent can't monopolize the single-replica LiteLLM proxy. const LITELLM_KEY_MAX_PARALLEL_REQUESTS = 4; const LITELLM_KEY_RPM_LIMIT = 120; -// Monthly spend cap per issued key (D111, 2026-10-27). Unlike the breakers above -// this IS a usage quota: since the GPT aliases were repointed at DeepSeek, and dead -// routes fall back to it, calls through these keys are paid, while the -// deepseek-v4-flash group budget sits on that model name, which alias callers never -// touch. The whole proxy runs ~$15/mo, so $30 per agent only trips on a runaway. +// Rolling spend cap per issued key. Unlike the breakers above this IS a usage +// quota: since the GPT aliases were repointed at DeepSeek, and dead routes fall +// back to it, calls through these keys are paid, while the deepseek-v4-flash +// group budget sits on that model name, which alias callers never touch. +// +// Sized from measured monthly spend (2026-10-28 key audit, lifetime/age): the +// heaviest agent, theo, runs ~$75-88/mo; nova ~$37/mo; others <$30/mo. The +// first cut was a flat $30 (D111, 2026-10-27), which would 429 theo and nova +// within one 30d window — a self-inflicted silent-seat outage. $150/30d clears +// the heaviest legit agent with ~70% headroom while still bounding a leaked key +// to a finite monthly spend instead of unlimited. budget_duration is REQUIRED: +// a bare max_budget is lifetime in LiteLLM, so it would silence any key already +// past the cap the moment it applied. Tighten per-agent later if needed. // A reused key keeps whatever budget it already carries; this stamps new ones. -const LITELLM_KEY_MAX_BUDGET_USD = 30; +const LITELLM_KEY_MAX_BUDGET_USD = 150; const LITELLM_KEY_BUDGET_DURATION = '30d'; /** @@ -1660,6 +1669,13 @@ const issueLiteLLMVirtualKey = async (agentId: any) => { rpm_limit: LITELLM_KEY_RPM_LIMIT, max_budget: LITELLM_KEY_MAX_BUDGET_USD, budget_duration: LITELLM_KEY_BUDGET_DURATION, + // Human-readable attribution on the key (the 2026-10-28 audit found 73 keys + // with no alias). Suffixed with a mint timestamp AND 6 random bytes because + // LiteLLM enforces key_alias uniqueness: a bare `agent-` would 409 when a + // stale key is deleted and re-minted for the same agent, and two concurrent + // mints can read the same Date.now() millisecond — either silently fails the + // seat. The random suffix makes the alias collision-free regardless of clock. + key_alias: `agent-${agentId}-${Date.now()}-${crypto.randomBytes(6).toString('hex')}`, metadata: { agent_id: agentId, provisioned_at: new Date().toISOString() }, }, { headers }, @@ -1741,6 +1757,7 @@ const issueLiteLLMOpenRouterKey = async (agentId: any) => { rpm_limit: LITELLM_KEY_RPM_LIMIT, max_budget: LITELLM_KEY_MAX_BUDGET_USD, budget_duration: LITELLM_KEY_BUDGET_DURATION, + key_alias: `agent-${agentId}-openrouter-${Date.now()}-${crypto.randomBytes(6).toString('hex')}`, metadata: { agent_id: agentId, key_type: 'openrouter', provisioned_at: new Date().toISOString() }, }, { headers }, diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index 0739d438e3..0ded542327 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -344,8 +344,20 @@ data: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL store_model_in_db: true - # Allow UI access - ui_access_mode: "all" + # Restrict the admin UI to admin users. The UI is not publicly routed + # (the nginx ingress maps only litellm.commonly.me/v1), so this is + # defense-in-depth against a future ingress/tunnel change exposing it. + ui_access_mode: "admin_only" + # Reject client-supplied upstream overrides on requests: api_base / base_url + # and the provider endpoint + credential-override fields in LiteLLM's + # _BANNED_REQUEST_BODY_PARAMS. The /v1 endpoint is public for BYO goose/pi + # callers and is key-gated, so a leaked virtual key must NOT be usable to + # redirect to arbitrary upstreams on our spend — which this blocks. + # NOTE: on v1.106.2 this does NOT ban a request-body `api_key` (it is not in + # _BANNED_REQUEST_BODY_PARAMS); that path would bill the caller's own key, + # not our spend, so it is out of scope for this setting. + # Default is already false; set explicitly so it can't drift (2026-10-28). + allow_client_side_credentials: false # Spend log retention — without these, LiteLLM_SpendLogs grows unbounded # (prod dev observed 5+ GB on the 8 GiB Cloud SQL tier before this cap). # STORE_PROMPTS_IN_SPEND_LOGS=false keeps raw prompt and response text