diff --git a/backend/__tests__/unit/services/litellmBudgetAlarmService.test.js b/backend/__tests__/unit/services/litellmBudgetAlarmService.test.js new file mode 100644 index 0000000000..0b875551bd --- /dev/null +++ b/backend/__tests__/unit/services/litellmBudgetAlarmService.test.js @@ -0,0 +1,315 @@ +/** + * TASK-555: the LiteLLM key budget alarm lists every virtual key with the + * master key, and at 80% of a key's max_budget logs one `alarm: + * litellm_budget` line per key per budget period and pages SRE. It reports + * and never blocks. + */ +jest.mock('axios', () => ({ get: jest.fn() })); +const mockPageSre = jest.fn(); +jest.mock('../../../services/spendAlarmService', () => ({ pageSre: (...args) => mockPageSre(...args) })); + +const axios = require('axios'); +const budget = require('../../../services/litellmBudgetAlarmService'); + +const ENV_KEYS = ['LITELLM_BASE_URL', 'LITELLM_MASTER_KEY', 'LITELLM_API_KEY', 'SPEND_ALARM_MENTION']; +const savedEnv = {}; + +const key = (overrides) => ({ + token: 'aaaabbbbccccdddd', + key_alias: 'svc-backend', + spend: 0, + max_budget: 40, + budget_duration: '30d', + budget_reset_at: '2026-11-24T00:00:00Z', + ...overrides, +}); + +const page = (keys, totalPages = 1) => ({ data: { keys, total_pages: totalPages } }); + +let warn; + +beforeEach(() => { + ENV_KEYS.forEach((k) => { savedEnv[k] = process.env[k]; }); + process.env.LITELLM_BASE_URL = 'http://litellm:4000/'; + process.env.LITELLM_MASTER_KEY = 'sk-master-test'; + delete process.env.SPEND_ALARM_MENTION; + delete process.env.LITELLM_API_KEY; + axios.get.mockReset(); + mockPageSre.mockReset().mockResolvedValue(undefined); + budget.resetBudgetAlarmMemory(); + warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); +}); + +afterEach(() => { + ENV_KEYS.forEach((k) => { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + }); + warn.mockRestore(); +}); + +const alarmLines = () => warn.mock.calls + .map(([first]) => first) + .filter((first) => typeof first === 'string' && first.startsWith('{')) + .map((line) => JSON.parse(line)) + .filter((line) => line.alarm === 'litellm_budget'); + +describe('keysNearBudget', () => { + test('flags a key at or past 80% of its budget and nothing below', () => { + const alarms = budget.keysNearBudget([ + key({ key_alias: 'at-line', spend: 32 }), + key({ key_alias: 'below', spend: 31.99 }), + key({ key_alias: 'over', spend: 45 }), + ]); + expect(alarms.map((a) => a.keyAlias)).toEqual(['at-line', 'over']); + expect(alarms[0]).toEqual({ + keyAlias: 'at-line', + spendUsd: 32, + maxBudgetUsd: 40, + percent: 80, + budgetDuration: '30d', + resetsAt: '2026-11-24T00:00:00Z', + }); + }); + + test('ignores keys with no budget, a zero budget or no spend figure', () => { + expect(budget.keysNearBudget([ + key({ max_budget: null, spend: 1000 }), + key({ max_budget: 0, spend: 1000 }), + key({ spend: null }), + ])).toEqual([]); + }); + + test('names a key without an alias by its stored hash prefix, and strips the alias to an identifier', () => { + const [noAlias, odd] = budget.keysNearBudget([ + key({ key_alias: null, spend: 39 }), + key({ key_alias: 'svc bot\n@everyone ', spend: 39 }), + ]); + expect(noAlias.keyAlias).toBe('key:aaaabbbb'); + expect(odd.keyAlias).toBe('svcboteveryonex'); + }); +}); + +describe('listLiteLLMKeys', () => { + const full = (prefix) => Array.from({ length: 100 }, (_, i) => key({ key_alias: `${prefix}${i}`, token: `${prefix}${i}` })); + + test('keeps paging when total_pages is missing and the page is full', async () => { + axios.get + .mockResolvedValueOnce({ data: { keys: full('a') } }) + .mockResolvedValueOnce({ data: { keys: [key({ key_alias: 'last' })] } }); + const { rows, incomplete } = await budget.listLiteLLMKeys(); + expect(axios.get).toHaveBeenCalledTimes(2); + expect(rows).toHaveLength(101); + expect(incomplete).toBeNull(); + }); + + test('a total_pages of 0 with a full page is not read as the end', async () => { + axios.get + .mockResolvedValueOnce({ data: { keys: full('a'), total_pages: 0 } }) + .mockResolvedValueOnce({ data: { keys: [], total_pages: 0 } }); + const { rows, incomplete } = await budget.listLiteLLMKeys(); + expect(axios.get).toHaveBeenCalledTimes(2); + expect(rows).toHaveLength(100); + expect(incomplete).toBeNull(); + }); + + test('without total_pages, a run of full pages stops at the bound as truncated', async () => { + axios.get.mockResolvedValue({ data: { keys: full('a') } }); + const { incomplete } = await budget.listLiteLLMKeys(); + expect(axios.get).toHaveBeenCalledTimes(20); + expect(incomplete).toEqual({ reason: 'truncated', pagesRead: 20, totalPages: null }); + }); + + test('returns null without the master key, and makes no call', async () => { + delete process.env.LITELLM_MASTER_KEY; + expect(await budget.listLiteLLMKeys()).toBeNull(); + expect(axios.get).not.toHaveBeenCalled(); + }); + + test('pages through /key/list with the master key until total_pages', async () => { + axios.get + .mockResolvedValueOnce(page([key({ key_alias: 'a' })], 2)) + .mockResolvedValueOnce(page([key({ key_alias: 'b' })], 2)); + const { rows, incomplete } = await budget.listLiteLLMKeys(); + expect(rows.map((r) => r.key_alias)).toEqual(['a', 'b']); + expect(incomplete).toBeNull(); + expect(axios.get).toHaveBeenCalledTimes(2); + const [url, opts] = axios.get.mock.calls[0]; + expect(url).toBe('http://litellm:4000/key/list'); + expect(opts.headers.Authorization).toBe('Bearer sk-master-test'); + expect(opts.params).toEqual({ return_full_object: true, page: 1, size: 100 }); + expect(axios.get.mock.calls[1][1].params.page).toBe(2); + }); + + test('stops at the page bound and reports the scan as truncated', async () => { + axios.get.mockResolvedValue(page([key()], 9999)); + const { incomplete } = await budget.listLiteLLMKeys(); + expect(axios.get).toHaveBeenCalledTimes(20); + expect(incomplete).toEqual({ reason: 'truncated', pagesRead: 20, totalPages: 9999 }); + }); +}); + +describe('checkLiteLLMBudgets', () => { + test('logs and pages once per key per budget period', async () => { + process.env.SPEND_ALARM_MENTION = 'sre-ops'; + axios.get.mockResolvedValue(page([key({ spend: 36 }), key({ key_alias: 'svc-commonly-bot', max_budget: 20, spend: 3 })])); + + const first = await budget.checkLiteLLMBudgets(); + expect(first.map((a) => a.keyAlias)).toEqual(['svc-backend']); + expect(alarmLines()).toEqual([expect.objectContaining({ + alarm: 'litellm_budget', thresholdFraction: 0.8, keyAlias: 'svc-backend', spendUsd: 36, maxBudgetUsd: 40, percent: 90, + })]); + expect(mockPageSre).toHaveBeenCalledTimes(1); + expect(mockPageSre.mock.calls[0][0]).toBe( + '[budget alarm] LiteLLM key svc-backend has spent $36.00 of its $40 budget (90%). ' + + 'At 100% its calls fail. It resets 2026-11-24T00:00:00Z. @sre-ops', + ); + + expect(await budget.checkLiteLLMBudgets()).toEqual([]); + expect(mockPageSre).toHaveBeenCalledTimes(1); + + // A new budget period is a new alarm. + axios.get.mockResolvedValue(page([key({ spend: 36, budget_reset_at: '2026-12-24T00:00:00Z' })])); + expect((await budget.checkLiteLLMBudgets()).map((a) => a.resetsAt)).toEqual(['2026-12-24T00:00:00Z']); + expect(mockPageSre).toHaveBeenCalledTimes(2); + }); + + test('two keys whose labels sanitise alike each alarm', async () => { + axios.get.mockResolvedValue(page([ + key({ token: 'aaaa1111', key_alias: 'svc bot', spend: 39 }), + key({ token: 'bbbb2222', key_alias: 'svcbot', spend: 39 }), + ])); + expect((await budget.checkLiteLLMBudgets()).map((a) => a.keyAlias)).toEqual(['svcbot', 'svcbot']); + expect(mockPageSre).toHaveBeenCalledTimes(2); + }); + + test('a failed page keeps the log line and is retried next sweep without a second line', async () => { + mockPageSre.mockRejectedValueOnce(new Error('pod gone')); + axios.get.mockResolvedValue(page([key({ spend: 40 })])); + expect((await budget.checkLiteLLMBudgets()).length).toBe(1); + expect(alarmLines().length).toBe(1); + expect(mockPageSre).toHaveBeenCalledTimes(1); + + expect(await budget.checkLiteLLMBudgets()).toEqual([]); + expect(mockPageSre).toHaveBeenCalledTimes(2); + expect(alarmLines().length).toBe(1); + + await budget.checkLiteLLMBudgets(); + expect(mockPageSre).toHaveBeenCalledTimes(2); + }); + + test('a truncated scan alarms once, still checks the keys it read', async () => { + axios.get.mockResolvedValue(page([key({ spend: 39 })], 9999)); + await budget.checkLiteLLMBudgets(); + expect(alarmLines()).toEqual([ + { alarm: 'litellm_budget', scanIncomplete: 'truncated', pagesRead: 20, totalPages: 9999 }, + expect.objectContaining({ keyAlias: 'svc-backend' }), + ]); + expect(mockPageSre.mock.calls[0][0]).toMatch(/stopped at 20 of 9999 pages/); + await budget.checkLiteLLMBudgets(); + expect(alarmLines().filter((l) => l.scanIncomplete).length).toBe(1); + }); + + test('a failed later page keeps the keys already read and alarms the incomplete scan', async () => { + axios.get + .mockResolvedValueOnce(page([key({ spend: 39 })], 3)) + .mockRejectedValueOnce(new Error('timeout of 15000ms exceeded')); + const fresh = await budget.checkLiteLLMBudgets(); + expect(fresh.map((a) => a.keyAlias)).toEqual(['svc-backend']); + expect(alarmLines()[0]).toEqual({ alarm: 'litellm_budget', scanIncomplete: 'failed', pagesRead: 1, totalPages: 3 }); + expect(mockPageSre.mock.calls[0][0]).toMatch(/failed after 1 page,/); + expect(mockPageSre).toHaveBeenCalledTimes(2); + }); + + test.each([ + ['{}', {}], + ['{keys:null}', { keys: null }], + ])('a 200 page with no keys array (%s) is a failed scan, logged and paged', async (_label, data) => { + axios.get.mockResolvedValueOnce({ data }); + expect(await budget.checkLiteLLMBudgets()).toEqual([]); + expect(alarmLines()).toEqual([{ alarm: 'litellm_budget', scanIncomplete: 'failed', pagesRead: 0, totalPages: null }]); + expect(mockPageSre).toHaveBeenCalledTimes(1); + expect(mockPageSre.mock.calls[0][0]).toMatch(/failed after 0 pages/); + }); + + test('an explicit empty keys array is a complete, empty scan', async () => { + axios.get.mockResolvedValueOnce({ data: { keys: [] } }); + expect(await budget.checkLiteLLMBudgets()).toEqual([]); + expect(alarmLines()).toEqual([]); + expect(mockPageSre).not.toHaveBeenCalled(); + }); + + test('an empty page that reports more pages is an incomplete scan', async () => { + axios.get + .mockResolvedValueOnce(page([key({ key_alias: 'a' })], 3)) + .mockResolvedValueOnce(page([], 3)); + const { rows, incomplete } = await budget.listLiteLLMKeys(); + expect(rows.length).toBe(1); + expect(incomplete).toEqual({ reason: 'failed', pagesRead: 2, totalPages: 3 }); + }); + + test('a failed incomplete-scan page is retried until it is delivered, logged once', async () => { + mockPageSre.mockRejectedValueOnce(new Error('pod gone')); + axios.get.mockRejectedValue(new Error('down')); + await budget.checkLiteLLMBudgets(); + await budget.checkLiteLLMBudgets(); + expect(mockPageSre).toHaveBeenCalledTimes(2); + expect(mockPageSre.mock.calls[1][0]).toMatch(/key scan failed after 0 pages/); + await budget.checkLiteLLMBudgets(); + expect(mockPageSre).toHaveBeenCalledTimes(2); + expect(alarmLines().filter((l) => l.scanIncomplete).length).toBe(1); + }); + + test('an incomplete scan alarms again only after a complete one in between', async () => { + axios.get.mockRejectedValueOnce(new Error('down')); + await budget.checkLiteLLMBudgets(); + axios.get.mockRejectedValueOnce(new Error('down')); + await budget.checkLiteLLMBudgets(); + expect(alarmLines().filter((l) => l.scanIncomplete).length).toBe(1); + axios.get.mockResolvedValueOnce(page([])); + await budget.checkLiteLLMBudgets(); + axios.get.mockRejectedValueOnce(new Error('down')); + await budget.checkLiteLLMBudgets(); + expect(alarmLines().filter((l) => l.scanIncomplete).length).toBe(2); + }); + + test('alarms once, and pages, when LiteLLM is in use but the master key is missing', async () => { + delete process.env.LITELLM_MASTER_KEY; + expect(await budget.checkLiteLLMBudgets()).toEqual([]); + expect(alarmLines()).toEqual([{ alarm: 'litellm_budget', scanUnconfigured: ['LITELLM_MASTER_KEY'] }]); + expect(mockPageSre).toHaveBeenCalledTimes(1); + expect(mockPageSre.mock.calls[0][0]).toMatch(/cannot run because LITELLM_MASTER_KEY is not set/); + + await budget.checkLiteLLMBudgets(); + expect(alarmLines()).toHaveLength(1); + expect(mockPageSre).toHaveBeenCalledTimes(1); + expect(axios.get).not.toHaveBeenCalled(); + }); + + test('a service key alone also counts as LiteLLM in use', async () => { + delete process.env.LITELLM_MASTER_KEY; + delete process.env.LITELLM_BASE_URL; + process.env.LITELLM_API_KEY = 'sk-svc-test'; + await budget.checkLiteLLMBudgets(); + expect(alarmLines()).toEqual([{ alarm: 'litellm_budget', scanUnconfigured: ['LITELLM_BASE_URL', 'LITELLM_MASTER_KEY'] }]); + }); + + test('stays silent when no LiteLLM setting is present at all', async () => { + delete process.env.LITELLM_MASTER_KEY; + delete process.env.LITELLM_BASE_URL; + expect(await budget.checkLiteLLMBudgets()).toEqual([]); + expect(alarmLines()).toEqual([]); + expect(mockPageSre).not.toHaveBeenCalled(); + }); + + test('an undelivered unconfigured page is retried until it goes out', async () => { + delete process.env.LITELLM_MASTER_KEY; + mockPageSre.mockRejectedValueOnce(new Error('SPEND_ALARM_POD_ID is not set')); + await budget.checkLiteLLMBudgets(); + await budget.checkLiteLLMBudgets(); + expect(mockPageSre).toHaveBeenCalledTimes(2); + await budget.checkLiteLLMBudgets(); + expect(mockPageSre).toHaveBeenCalledTimes(2); + expect(alarmLines()).toHaveLength(1); + }); +}); diff --git a/backend/__tests__/unit/services/nativeRuntimeService.litellmKey.test.js b/backend/__tests__/unit/services/nativeRuntimeService.litellmKey.test.js new file mode 100644 index 0000000000..d51b0f78d6 --- /dev/null +++ b/backend/__tests__/unit/services/nativeRuntimeService.litellmKey.test.js @@ -0,0 +1,57 @@ +/** + * TASK-555: hosted turns call LiteLLM on the backend's budgeted service key + * (LITELLM_API_KEY) and fall back to the master key only where no service key + * is deployed. The standalone guardrail check is the exception: LiteLLM serves + * /guardrails/apply_guardrail to proxy admins only, so it keeps the master key. + */ +jest.mock('axios', () => ({ __esModule: true, default: { post: jest.fn() } })); + +const { resolveLiteLLM, guardrailCheckKey } = require('../../../services/nativeRuntimeService'); + +const ENV_KEYS = ['LITELLM_BASE_URL', 'LITELLM_API_KEY', 'LITELLM_MASTER_KEY']; +const savedEnv = {}; + +beforeEach(() => { + ENV_KEYS.forEach((k) => { savedEnv[k] = process.env[k]; delete process.env[k]; }); + process.env.LITELLM_BASE_URL = 'http://litellm:4000/'; +}); + +afterEach(() => { + ENV_KEYS.forEach((k) => { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + }); +}); + +describe('resolveLiteLLM', () => { + test('prefers the service key over the master key', () => { + process.env.LITELLM_API_KEY = 'sk-svc-backend'; + process.env.LITELLM_MASTER_KEY = 'sk-master'; + expect(resolveLiteLLM()).toEqual({ baseUrl: 'http://litellm:4000', apiKey: 'sk-svc-backend' }); + }); + + test('falls back to the master key when no service key is set, or it is blank', () => { + process.env.LITELLM_MASTER_KEY = 'sk-master'; + expect(resolveLiteLLM().apiKey).toBe('sk-master'); + process.env.LITELLM_API_KEY = ' '; + expect(resolveLiteLLM().apiKey).toBe('sk-master'); + }); + + test('is unavailable with no key or no base URL', () => { + expect(resolveLiteLLM()).toBeNull(); + process.env.LITELLM_API_KEY = 'sk-svc-backend'; + delete process.env.LITELLM_BASE_URL; + expect(resolveLiteLLM()).toBeNull(); + }); +}); + +describe('guardrailCheckKey', () => { + test('uses the master key even when the turn runs on the service key', () => { + process.env.LITELLM_MASTER_KEY = 'sk-master'; + expect(guardrailCheckKey({ apiKey: 'sk-svc-backend' })).toBe('sk-master'); + }); + + test('falls back to the turn key when no master key is deployed', () => { + expect(guardrailCheckKey({ apiKey: 'sk-svc-backend' })).toBe('sk-svc-backend'); + }); +}); diff --git a/backend/__tests__/unit/services/spendAlarmService.test.js b/backend/__tests__/unit/services/spendAlarmService.test.js index 1b8304fbc2..9604f6d265 100644 --- a/backend/__tests__/unit/services/spendAlarmService.test.js +++ b/backend/__tests__/unit/services/spendAlarmService.test.js @@ -295,6 +295,11 @@ describe('paging SRE in a pod', () => { expect(text.startsWith('[spend alarm] u1 spent $3.00')).toBe(true); }); + test('pageSre throws when SPEND_ALARM_POD_ID is unset, so no caller counts the page as sent', async () => { + await expect(spend.pageSre('x')).rejects.toThrow('SPEND_ALARM_POD_ID is not set'); + expect(AgentMessageService._postToTarget).not.toHaveBeenCalled(); + }); + test('with SPEND_ALARM_POD_ID unset there is no post, and the log line stands', async () => { const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); await overTheLine(); diff --git a/backend/services/litellmBudgetAlarmService.ts b/backend/services/litellmBudgetAlarmService.ts new file mode 100644 index 0000000000..8439196bb6 --- /dev/null +++ b/backend/services/litellmBudgetAlarmService.ts @@ -0,0 +1,236 @@ +/** + * LiteLLM key budget alarm (TASK-555). + * + * The durable spend caps are LiteLLM virtual-key budgets (max_budget plus + * budget_duration). They live in LiteLLM's Postgres, so a LiteLLM restart + * keeps them. Backend and commonly-bot each call on their own budgeted key, + * and the dsh writers have one. Over a budget, LiteLLM answers 422 + * budget_exceeded and the caller fails. This sweep warns first: at + * BUDGET_ALARM_FRACTION of any budgeted key it logs one `alarm: + * litellm_budget` line per key per budget period, and pages SRE the same way + * the user spend alarm does. A log-based metric turns the line into an email + * (k8s/monitoring). + * + * Group caps in litellm-config.yaml (deployment max_budget) are not covered. + * LiteLLM keeps their month-to-date figure in the router cache and exposes no + * endpoint for it, so they are a restart-resettable backstop behind the keys. + * + * It reports and never blocks. The master key is needed only to list keys. + */ +// eslint-disable-next-line @typescript-eslint/no-require-imports +const axios = require('axios'); + +export const BUDGET_ALARM_FRACTION = 0.8; +const PAGE_SIZE = 100; +// A bound on pages, so a malformed total_pages cannot loop the sweep. +const MAX_PAGES = 20; + +export interface LiteLLMKeyRow { + token?: string; + key_alias?: string | null; + spend?: number | null; + max_budget?: number | null; + budget_duration?: string | null; + budget_reset_at?: string | null; +} + +export interface BudgetAlarm { + keyAlias: string; + spendUsd: number; + maxBudgetUsd: number; + percent: number; + budgetDuration: string | null; + resetsAt: string | null; +} + +// The alias names the key in the log and the page. A key without one is +// named by the first 8 characters of LiteLLM's stored hash, which is not the +// key itself. Either way the text is reduced to an identifier, because the +// page lands in an inbox an SRE model session reads. +const keyLabel = (row: LiteLLMKeyRow): string => { + const alias = String(row.key_alias || '').replace(/[^A-Za-z0-9._-]/g, '').slice(0, 60); + if (alias) return alias; + return `key:${String(row.token || '').replace(/[^A-Za-z0-9]/g, '').slice(0, 8) || 'unknown'}`; +}; + +// What remembers an alarm. The label is lossy (two aliases can sanitise to +// the same text), so dedup keys on LiteLLM's stored hash and falls back to +// the raw alias. It is never logged or posted. +const keyIdentity = (row: LiteLLMKeyRow): string => ( + row.token ? `token:${row.token}` : `alias:${String(row.key_alias || '')}` +); + +const nearBudget = (row: LiteLLMKeyRow, fraction: number): boolean => ( + Number(row.max_budget) > 0 && Number(row.spend || 0) >= fraction * Number(row.max_budget) +); + +export const keysNearBudget = ( + rows: LiteLLMKeyRow[], + fraction: number = BUDGET_ALARM_FRACTION, +): BudgetAlarm[] => rows + .filter((row) => nearBudget(row, fraction)) + .map((row) => { + const spendUsd = Number(row.spend || 0); + const maxBudgetUsd = Number(row.max_budget); + return { + keyAlias: keyLabel(row), + spendUsd: Number(spendUsd.toFixed(4)), + maxBudgetUsd, + percent: Math.round((spendUsd / maxBudgetUsd) * 100), + budgetDuration: row.budget_duration || null, + resetsAt: row.budget_reset_at || null, + }; + }); + +export interface KeyScan { + rows: LiteLLMKeyRow[]; + // Set when the scan did not cover every key: /key/list reported more pages + // than the bound allows, or a page request failed. Either way a budgeted + // key on a later page went unchecked. The rows already read are kept. + incomplete: { reason: 'truncated' | 'failed'; pagesRead: number; totalPages: number | null } | null; +} + +export const listLiteLLMKeys = async (): Promise => { + const baseUrl = (process.env.LITELLM_BASE_URL || '').trim().replace(/\/$/, ''); + const masterKey = (process.env.LITELLM_MASTER_KEY || '').trim(); + if (!baseUrl || !masterKey) return null; + const rows: LiteLLMKeyRow[] = []; + let totalPages: number | null = null; + for (let page = 1; page <= MAX_PAGES; page += 1) { + let resp; + try { + // eslint-disable-next-line no-await-in-loop + resp = await axios.get(`${baseUrl}/key/list`, { + params: { return_full_object: true, page, size: PAGE_SIZE }, + headers: { Authorization: `Bearer ${masterKey}` }, + timeout: 15000, + }); + } catch (error) { + console.warn(`[budget-alarm] /key/list page ${page} failed:`, (error as Error).message); + return { rows, incomplete: { reason: 'failed', pagesRead: page - 1, totalPages } }; + } + // A 200 without a keys array is a broken page, not an empty list; only + // an explicit [] counts as zero keys. + if (!Array.isArray(resp?.data?.keys)) { + console.warn(`[budget-alarm] /key/list page ${page} has no keys array`); + return { rows, incomplete: { reason: 'failed', pagesRead: page - 1, totalPages } }; + } + const keys: LiteLLMKeyRow[] = resp.data.keys; + rows.push(...keys); + const reported = Number(resp?.data?.total_pages); + if (Number.isInteger(reported) && reported > 0) { + totalPages = reported; + if (page >= totalPages) return { rows, incomplete: null }; + } else { + // No usable total_pages: a short page is the last one, and a full page + // means more may follow, so keep reading rather than call it complete. + totalPages = null; + if (keys.length < PAGE_SIZE) return { rows, incomplete: null }; + } + // An empty page that still reports more pages is not the end of the list. + if (keys.length === 0) return { rows, incomplete: { reason: 'failed', pagesRead: page, totalPages } }; + if (page === MAX_PAGES) return { rows, incomplete: { reason: 'truncated', pagesRead: page, totalPages } }; + } + return { rows, incomplete: null }; +}; + +export const composeBudgetPage = (alarm: BudgetAlarm, mention?: string): string => { + const handle = mention ? ` @${mention}` : ''; + const reset = alarm.resetsAt ? ` It resets ${alarm.resetsAt}.` : ''; + return `[budget alarm] LiteLLM key ${alarm.keyAlias} has spent $${alarm.spendUsd.toFixed(2)} of its ` + + `$${alarm.maxBudgetUsd} budget (${alarm.percent}%). At 100% its calls fail.${reset}${handle}`; +}; + +// Two memories, in process only like the user spend alarm (after a restart a +// key still over the line is logged once more). `alarmed` holds keys already +// logged this budget period, keyed on the key and its reset time. `unpaged` +// holds logged alarms whose SRE page has not gone out yet, so a failed post +// is retried on the next sweep instead of being lost. The log line is what +// the email alert counts, so it is never repeated for a retry. +const alarmed = new Set(); +const unpaged = new Map(); +// An incomplete scan logs once, and again only after a complete scan in +// between, so a lasting failure is one alert rather than one every sweep. +// Its SRE page is held until it is delivered, like the key pages. +let incompleteLogged = false; +let unconfiguredLogged = false; +let unpagedScan: string | null = null; + +// The scan needs both of these. LITELLM_BASE_URL or a service key being set +// means this deployment uses LiteLLM, so a missing piece is an alarm, not a +// quiet skip. With none of the three set there is no LiteLLM to watch. +const SCAN_ENV = ['LITELLM_BASE_URL', 'LITELLM_MASTER_KEY']; +const setEnv = (name: string): boolean => Boolean((process.env[name] || '').trim()); + +const sendPage = async (content: string): Promise => { + try { + // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports + const { pageSre } = require('./spendAlarmService'); + await pageSre(content); + return true; + } catch (error) { + console.warn('[budget-alarm] SRE page failed; it is retried next sweep:', (error as Error).message); + return false; + } +}; + +export const checkLiteLLMBudgets = async (): Promise => { + const scan = await listLiteLLMKeys(); + const mention = String(process.env.SPEND_ALARM_MENTION || '').trim() || undefined; + if (!scan) { + const missing = SCAN_ENV.filter((name) => !setEnv(name)); + const usesLiteLLM = [...SCAN_ENV, 'LITELLM_API_KEY'].some(setEnv); + if (usesLiteLLM && !unconfiguredLogged) { + unconfiguredLogged = true; + console.warn(JSON.stringify({ alarm: 'litellm_budget', scanUnconfigured: missing })); + const handle = mention ? ` @${mention}` : ''; + unpagedScan = `[budget alarm] The LiteLLM key scan cannot run because ${missing.join(' and ')} ` + + `${missing.length === 1 ? 'is' : 'are'} not set, so no key is checked against its budget.${handle}`; + } + if (unpagedScan && await sendPage(unpagedScan)) unpagedScan = null; + return []; + } + unconfiguredLogged = false; + + // An incomplete scan can hide a key near its budget, so it alarms too, on + // the same metric, toward over-delivery. The rows it did read are checked. + if (!scan.incomplete) { + incompleteLogged = false; + } else if (!incompleteLogged) { + incompleteLogged = true; + const { reason, pagesRead, totalPages } = scan.incomplete; + console.warn(JSON.stringify({ alarm: 'litellm_budget', scanIncomplete: reason, pagesRead, totalPages })); + const handle = mention ? ` @${mention}` : ''; + const what = reason === 'failed' + ? `failed after ${pagesRead} page${pagesRead === 1 ? '' : 's'}` + : `stopped at ${pagesRead} of ${totalPages} pages`; + unpagedScan = `[budget alarm] The LiteLLM key scan ${what}, so some keys were not checked against their budget.${handle}`; + } + if (unpagedScan && await sendPage(unpagedScan)) unpagedScan = null; + + const fresh: BudgetAlarm[] = []; + scan.rows.filter((row) => nearBudget(row, BUDGET_ALARM_FRACTION)).forEach((row) => { + const id = `${keyIdentity(row)}:${row.budget_reset_at || null}`; + if (alarmed.has(id)) return; + const [alarm] = keysNearBudget([row]); + alarmed.add(id); + unpaged.set(id, alarm); + fresh.push(alarm); + console.warn(JSON.stringify({ alarm: 'litellm_budget', thresholdFraction: BUDGET_ALARM_FRACTION, ...alarm })); + }); + + for (const [id, alarm] of [...unpaged.entries()]) { + // eslint-disable-next-line no-await-in-loop + if (await sendPage(composeBudgetPage(alarm, mention))) unpaged.delete(id); + } + return fresh; +}; + +/** Test seam: forget which keys were already logged and paged. */ +export const resetBudgetAlarmMemory = (): void => { + alarmed.clear(); + unpaged.clear(); + incompleteLogged = false; + unconfiguredLogged = false; + unpagedScan = null; +}; diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index 6fd8722ddf..3031496de7 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -138,17 +138,25 @@ function normalizeConfig(config: unknown): PlainConfig { return config as PlainConfig; } -function resolveLiteLLM(): { baseUrl: string; apiKey: string } | null { +export function resolveLiteLLM(): { baseUrl: string; apiKey: string } | null { const baseUrl = (process.env.LITELLM_BASE_URL || '').trim().replace(/\/$/, ''); - const apiKey = ( - process.env.LITELLM_MASTER_KEY - || process.env.LITELLM_API_KEY - || '' - ).trim(); + // The backend's budgeted service key first, like llmService (TASK-555); the + // master key only where no service key is deployed. A hosted turn on the + // master key would bypass the backend's budget. + const apiKey = (process.env.LITELLM_API_KEY || '').trim() + || (process.env.LITELLM_MASTER_KEY || '').trim(); if (!baseUrl || !apiKey) return null; return { baseUrl, apiKey }; } +// LiteLLM serves /guardrails/apply_guardrail to proxy admins only: a virtual +// key gets 401 "Only proxy admin can be used" (measured 2026-10-25, v1.106.2). +// So the standalone guard check keeps the master key; model calls, which carry +// their guardrails in the request body, accept the service key. +export function guardrailCheckKey(litellm: { apiKey: string }): string { + return (process.env.LITELLM_MASTER_KEY || '').trim() || litellm.apiKey; +} + export function isNativeRuntimeAvailable(): boolean { return resolveLiteLLM() !== null; } @@ -171,7 +179,7 @@ async function textsTheGuardStops( `${litellm.baseUrl}/guardrails/apply_guardrail`, { guardrail_name: guardrailName, text }, { - headers: { Authorization: `Bearer ${litellm.apiKey}`, 'Content-Type': 'application/json' }, + headers: { Authorization: `Bearer ${guardrailCheckKey(litellm)}`, 'Content-Type': 'application/json' }, timeout: GUARD_CHECK_TIMEOUT_MS, validateStatus: () => true, }, diff --git a/backend/services/schedulerService.ts b/backend/services/schedulerService.ts index 3342b88da1..8ac96da28f 100644 --- a/backend/services/schedulerService.ts +++ b/backend/services/schedulerService.ts @@ -600,6 +600,14 @@ class SchedulerService { } catch (error) { console.error('[spend-alarm] check failed:', error); } + // TASK-555: warn at 80% of any LiteLLM key budget (litellmBudgetAlarmService). + try { + // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports + const { checkLiteLLMBudgets } = require('./litellmBudgetAlarmService'); + await checkLiteLLMBudgets(); + } catch (error) { + console.error('[budget-alarm] check failed:', (error as Error).message); + } }, { scheduled: false, timezone: 'UTC' }, ); diff --git a/backend/services/spendAlarmService.ts b/backend/services/spendAlarmService.ts index dc5135726e..a7f58bd336 100644 --- a/backend/services/spendAlarmService.ts +++ b/backend/services/spendAlarmService.ts @@ -211,9 +211,11 @@ export const composeSpendPage = ( + `across ${spend.runs} run${spend.runs === 1 ? '' : 's'}.${handle}`; }; -const pageSre = async (content: string): Promise => { +export const pageSre = async (content: string): Promise => { const podId = String(process.env.SPEND_ALARM_POD_ID || '').trim(); - if (!podId) return; + // Throw rather than return: a caller that retries undelivered pages must + // not count a page with nowhere to go as sent. + if (!podId) throw new Error('SPEND_ALARM_POD_ID is not set'); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const AgentIdentityService = require('./agentIdentityService'); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports diff --git a/docs/runbooks/alerting.md b/docs/runbooks/alerting.md index 27666a251e..3ec561cefd 100644 --- a/docs/runbooks/alerting.md +++ b/docs/runbooks/alerting.md @@ -346,3 +346,52 @@ A model the price table does not know is charged at an Opus-class rate, so a new route over-alarms instead of hiding. If the alert is about a model that is not in the table, add its price, with the source, rather than raising the threshold. + +### LiteLLM key budget + +"A LiteLLM key passed 80% of its budget" (WARNING) fires when a LiteLLM +virtual key with a `max_budget` has spent 80% of it for the current +`budget_duration`. Every 15 minutes the backend lists the keys with the master +key (`GET /key/list`, `backend/services/litellmBudgetAlarmService.ts`) and logs +one `{"alarm":"litellm_budget", ...}` line per key per budget period, naming +the key alias, the spend, the budget and the reset time. The same text goes to +the "SRE alerts" pod as described under *User spend*; a failed post is retried +on the next sweep without a second log line. If the scan does not cover every +key, it logs one `scanIncomplete` line on the same metric, so the alert can +also mean "some keys were not checked". `truncated` means `/key/list` has more +pages than the sweep reads (20 pages of 100): raise the bound in the service. +`failed` means a page request failed: check LiteLLM's health first. Either +way the keys already read are still checked, and the line repeats only after +a complete scan in between. + +The keys that matter (TASK-555, minted 2026-10-25): + +| key alias | used by | budget | +|---|---|---| +| `svc-backend` | backend + ops-runner (`LITELLM_API_KEY`, SM `litellm-backend-key`) | $40 / 30 days | +| `svc-commonly-bot` | commonly-bot (`LITELLM_API_KEY`, SM `litellm-commonly-bot-key`) | $20 / 30 days | + +Both are gated by `litellm.serviceKeys.fromSecret`. Budgets and spend live in +LiteLLM's Postgres, so they survive a LiteLLM restart (verified 2026-10-25: a +key exhausted before a rollout restart still answered 422 after it). The +`max_budget` on the `deepseek-v4-flash` deployment in the LiteLLM config is a +router-cache backstop only and resets on every LiteLLM restart. + +**At 100%** LiteLLM answers `422 budget_exceeded` and every call on that key +fails: backend summaries, embeddings and native-runtime turns for +`svc-backend`, the bot's replies for `svc-commonly-bot`. The guardrail check in +`nativeRuntimeService` keeps using the master key (the route is admin-only), so +it is not affected. + +When it fires: + +1. Read the log line for the key and the spend. +2. Break the spend down by model and day: `GET /spend/logs` for that key, or + the LiteLLM UI. A single model or a single day carrying it is a loop or a + misroute; fix that, not the budget. +3. Steady growth is real use. Raise the budget with + `POST /key/update {"key": ..., "max_budget": ...}` using the master key, + and update the table above in the same PR. +4. If the key is already blocked and the cause is not fixed yet, unset + `litellm.serviceKeys.fromSecret` only as a last resort: the services fall + back to the master key, which has no budget. diff --git a/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml b/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml index 1df14c1d2b..b780f4f930 100644 --- a/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml +++ b/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml @@ -52,6 +52,15 @@ spec: name: api-keys key: litellm-master-key optional: true + {{- if .Values.litellm.serviceKeys.fromSecret }} + # Model calls use this budgeted key (TASK-555); litellm-client.js + # prefers LITELLM_API_KEY over the master key. + - name: LITELLM_API_KEY + valueFrom: + secretKeyRef: + name: api-keys + key: litellm-commonly-bot-key + {{- end }} - name: GEMINI_API_KEY valueFrom: secretKeyRef: diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index ee91fe21c5..1f12b75d86 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -31,6 +31,14 @@ data: litellm_params: model: deepseek/deepseek-v4-flash api_key: os.environ/DEEPSEEK_API_KEY + # $60 per month across every caller of this group (TASK-555). Over + # it, LiteLLM answers 429 "crossed budget". Like deepseek-flash-1017 + # below, the month-to-date figure is in the router cache and a + # LiteLLM restart resets it: this is a backstop. The durable caps + # are the per-service keys (backend $40, commonly-bot $20, in + # LiteLLM's Postgres), and the budget alarm watches those. + max_budget: 60 + budget_duration: 1mo # DeepSeek V4.1 Flash, 1017 build. DeepSeek publishes no dated snapshot # id: per api-docs.deepseek.com/updates, 2026-10-17, `deepseek-flash` is diff --git a/k8s/helm/commonly/templates/core/backend-deployment.yaml b/k8s/helm/commonly/templates/core/backend-deployment.yaml index 61517210b9..b4615d49ea 100644 --- a/k8s/helm/commonly/templates/core/backend-deployment.yaml +++ b/k8s/helm/commonly/templates/core/backend-deployment.yaml @@ -558,6 +558,15 @@ spec: name: api-keys key: litellm-master-key optional: true + {{- if .Values.litellm.serviceKeys.fromSecret }} + # Model calls use this budgeted key (TASK-555); the master key stays + # for key administration (agent key issuance, the budget alarm). + - name: LITELLM_API_KEY + valueFrom: + secretKeyRef: + name: api-keys + key: litellm-backend-key + {{- end }} - name: LITELLM_BASE_URL value: "http://litellm:4000" # injection-guard only (Sam, 2026-09-01): openai-moderation-enforce diff --git a/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml b/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml index 98b89cb749..94fd29e461 100644 --- a/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml +++ b/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml @@ -525,6 +525,14 @@ spec: name: api-keys key: litellm-master-key optional: true + {{- if .Values.litellm.serviceKeys.fromSecret }} + # Mirrors backend (TASK-555): model calls bill the backend's budgeted key. + - name: LITELLM_API_KEY + valueFrom: + secretKeyRef: + name: api-keys + key: litellm-backend-key + {{- end }} - name: LITELLM_BASE_URL value: "http://litellm:4000" # injection-guard only (Sam, 2026-09-01): openai-moderation-enforce diff --git a/k8s/helm/commonly/templates/secrets/api-keys.yaml b/k8s/helm/commonly/templates/secrets/api-keys.yaml index 986995c855..af28c220d0 100644 --- a/k8s/helm/commonly/templates/secrets/api-keys.yaml +++ b/k8s/helm/commonly/templates/secrets/api-keys.yaml @@ -262,6 +262,15 @@ spec: remoteRef: key: {{ $keyPrefix }}github-app-installation-id + {{- if .Values.litellm.serviceKeys.fromSecret }} + - secretKey: litellm-backend-key + remoteRef: + key: {{ $keyPrefix }}litellm-backend-key + - secretKey: litellm-commonly-bot-key + remoteRef: + key: {{ $keyPrefix }}litellm-commonly-bot-key + {{- end }} + {{- if .Values.githubGig.fromSecret }} - secretKey: GITHUB_GIG_WEBHOOK_SECRET remoteRef: diff --git a/k8s/helm/commonly/values-dev.yaml b/k8s/helm/commonly/values-dev.yaml index 187e9ddfdb..e0fd4a4924 100644 --- a/k8s/helm/commonly/values-dev.yaml +++ b/k8s/helm/commonly/values-dev.yaml @@ -347,6 +347,10 @@ cloudflared: litellm: enabled: true + # TASK-555: budgeted service keys (backend $40/30d, commonly-bot $20/30d), + # minted 2026-10-25 and stored in Secret Manager. + serviceKeys: + fromSecret: true # The base image pin must match the shared boot script. v1.106.2 includes # spend-log tool-index retention and the headless device-login guard; # BerriAI/litellm#48153 tracks the remaining ChatGPT Responses input patch. diff --git a/k8s/helm/commonly/values.yaml b/k8s/helm/commonly/values.yaml index 0710666d97..a2b6cf18b6 100644 --- a/k8s/helm/commonly/values.yaml +++ b/k8s/helm/commonly/values.yaml @@ -561,6 +561,14 @@ cloudflared: # LiteLLM proxy (model gateway + request logging) litellm: enabled: false + # Per-service LiteLLM virtual keys (TASK-555). When true, backend and + # commonly-bot call LiteLLM with their own budgeted keys + # (litellm-backend-key, litellm-commonly-bot-key) as + # LITELLM_API_KEY instead of the master key. A key's budget lives in + # LiteLLM's Postgres, so it survives a LiteLLM restart; mint both keys in + # Secret Manager before enabling, or ExternalSecrets rejects the bundle. + serviceKeys: + fromSecret: false # When true, bypass LiteLLM for Codex calls. moltbot.json's openai-codex # provider points at chatgpt.com directly via OpenClaw's native # openai-codex-responses handler. Workaround for BerriAI/litellm#25429 diff --git a/k8s/monitoring/alert-policies.json b/k8s/monitoring/alert-policies.json index d6b92f71c1..83a52d0697 100644 --- a/k8s/monitoring/alert-policies.json +++ b/k8s/monitoring/alert-policies.json @@ -404,6 +404,34 @@ } ] }, + { + "displayName": "commonly-dev: a LiteLLM key passed 80% of its budget", + "severity": "WARNING", + "documentation": { + "mimeType": "text/markdown", + "content": "A LiteLLM virtual key (backend, commonly-bot, or any other key with a max_budget) has spent 80% of its budget for the period. At 100% LiteLLM answers 422 budget_exceeded and every call on that key fails, so this is the warning before a feature stops. The log line names the key, the spend, the budget and the reset time. Runbook: docs/runbooks/alerting.md#litellm-key-budget" + }, + "combiner": "OR", + "conditions": [ + { + "displayName": "a LiteLLM key crossed 80% of its budget", + "conditionThreshold": { + "filter": "metric.type=\"logging.googleapis.com/user/litellm_key_budget_near_limit\" AND resource.type=\"k8s_container\"", + "aggregations": [ + { + "alignmentPeriod": "300s", + "perSeriesAligner": "ALIGN_SUM", + "crossSeriesReducer": "REDUCE_SUM" + } + ], + "comparison": "COMPARISON_GT", + "thresholdValue": 0, + "duration": "0s", + "trigger": { "count": 1 } + } + } + ] + }, { "displayName": "commonly-dev: www.commonly.me no longer reaches the app", "severity": "WARNING", diff --git a/k8s/monitoring/log-metrics.json b/k8s/monitoring/log-metrics.json index b5dc3af22c..cd7d7f8c3c 100644 --- a/k8s/monitoring/log-metrics.json +++ b/k8s/monitoring/log-metrics.json @@ -40,5 +40,11 @@ "description": "A user's priced model spend for the UTC day crossed SPEND_ALARM_USD_PER_DAY (default $2). One line per user per day from backend/services/spendAlarmService.ts; the userId, the amount and the token split are in the same log line.", "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"backend\" AND jsonPayload.alarm=\"user_spend\"", "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } + }, + { + "name": "litellm_key_budget_near_limit", + "description": "A LiteLLM virtual key passed 80% of its max_budget (TASK-555). One line per key per budget period from backend/services/litellmBudgetAlarmService.ts; the key alias, spend, budget and reset time are in the same log line.", + "filter": "resource.type=\"k8s_container\" AND resource.labels.namespace_name=\"commonly-dev\" AND resource.labels.container_name=\"backend\" AND jsonPayload.alarm=\"litellm_budget\"", + "metricDescriptor": { "metricKind": "DELTA", "valueType": "INT64", "unit": "1" } } ]