From d569e46f9aebbce3f15d734abbc80bb01888d96e Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:54:42 -0700 Subject: [PATCH 1/7] fix(commonly-bot): route summaries to working default and log degradation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- docs/SUMMARIZER_AND_AGENTS.md | 23 +++++ .../commonly-bot/index.js | 4 +- .../commonly-bot/index.test.js | 86 ++++++++++++++++++- .../agents/commonly-bot-deployment.yaml | 3 + 4 files changed, 114 insertions(+), 2 deletions(-) diff --git a/docs/SUMMARIZER_AND_AGENTS.md b/docs/SUMMARIZER_AND_AGENTS.md index 1533b08551..5ed100dc9e 100644 --- a/docs/SUMMARIZER_AND_AGENTS.md +++ b/docs/SUMMARIZER_AND_AGENTS.md @@ -62,6 +62,29 @@ static async runSummarizer() { --- +### Commonly-bot model and failure behavior + +The Helm deployment sets `AGENT_MODEL=deepseek-v4-flash`, matching the backend's +`LITELLM_CHAT_MODEL`. The standalone bot uses the same default; +`COMMONLY_SOCIAL_REPHRASE_MODEL` overrides it for both pod summaries and social +rewrites. Both use the shared LiteLLM client. The chart's LiteLLM config must +route the selected group; `commonly-bot/index.test.js` exercises the actual +request body and checks that the default is a routed, non-Claude group. + +On a failed pod-summary model call, the bot logs `feature=pod-summary` and the +model, then builds a heuristic recap from recent messages. A silent summary +request persists that recap before acknowledging its event; normal delivery +posts the recap before acknowledgement. This preserves an output but loses the +model-generated synthesis. A failed optional social rewrite logs +`feature=social-rephrase` and the model, and uses heuristic feed-post text. +These logs omit provider error bodies and chat content. + +A LiteLLM 401 alone does not identify the feature that called it. Correlate the +caller and runtime model configuration before attributing lost user output; +requests from proxy loopback can also be operator probes. + +--- + ### **2. commonly-bot Agent Identity** **Location**: Created via `AgentIdentityService.getOrCreateAgentUser()` diff --git a/external/commonly-agent-services/commonly-bot/index.js b/external/commonly-agent-services/commonly-bot/index.js index fe6f5cbc39..033b13c14e 100644 --- a/external/commonly-agent-services/commonly-bot/index.js +++ b/external/commonly-agent-services/commonly-bot/index.js @@ -12,7 +12,7 @@ const ENABLE_SOCIAL_REPHRASE = process.env.COMMONLY_SOCIAL_REPHRASE_ENABLED !== const ENABLE_SOCIAL_FEED_POST = process.env.COMMONLY_SOCIAL_POST_TO_FEED === '1'; const ENABLE_SOCIAL_IMAGE = process.env.COMMONLY_SOCIAL_IMAGE_ENABLED === '1'; const ENABLE_SOCIAL_EXTERNAL_PUBLISH = process.env.COMMONLY_SOCIAL_PUBLISH_EXTERNAL === '1'; -const SOCIAL_REPHRASE_MODEL = process.env.COMMONLY_SOCIAL_REPHRASE_MODEL || process.env.AGENT_MODEL || 'gemini-2.5-flash'; +const SOCIAL_REPHRASE_MODEL = process.env.COMMONLY_SOCIAL_REPHRASE_MODEL || process.env.AGENT_MODEL || 'deepseek-v4-flash'; const SOCIAL_IMAGE_MODEL = process.env.COMMONLY_SOCIAL_IMAGE_MODEL || 'gemini-2.5-flash-image'; const llmClient = new LiteLLMClient({ model: SOCIAL_REPHRASE_MODEL, @@ -242,6 +242,7 @@ const rephraseSocialPost = async ({ post, podName }) => { overlap, }; } catch { + console.warn(`[commonly-bot] feature=social-rephrase model=${SOCIAL_REPHRASE_MODEL} failed; using heuristic feed post`); return null; } }; @@ -511,6 +512,7 @@ const buildLlmPodSummary = async ({ messages = [], podName = 'this pod', client // summary takes over instead of a brace reaching the reader. return podSummaryFromLlmReply(raw, { messageCount: meaningful.length }); } catch { + console.warn(`[commonly-bot] feature=pod-summary model=${SOCIAL_REPHRASE_MODEL} failed; using heuristic summary`); return null; } }; diff --git a/external/commonly-agent-services/commonly-bot/index.test.js b/external/commonly-agent-services/commonly-bot/index.test.js index 0c1cbee67a..1975f91752 100644 --- a/external/commonly-agent-services/commonly-bot/index.test.js +++ b/external/commonly-agent-services/commonly-bot/index.test.js @@ -1,5 +1,8 @@ const test = require('node:test'); const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); const { buildLlmPodSummary, handleEvent } = require('./index'); const messages = [ @@ -81,10 +84,91 @@ test('summary.request stores recovered prose when the model reply ends mid-envel }); test('summary.request stores heuristic prose and acknowledges after an LLM error', async () => { - const writes = await runSummaryRequest(new Error('LLM unavailable')); + const warnings = []; + const originalWarn = console.warn; + let writes; + console.warn = (line) => warnings.push(line); + try { + writes = await runSummaryRequest(new Error('LLM unavailable')); + } finally { + console.warn = originalWarn; + } + assert.equal(warnings.length, 1); + assert.match(warnings[0], /feature=pod-summary model=\S+ failed; using heuristic summary/); assert.equal(writes.length, 2); assert.ok(writes[0].url.endsWith('/summaries')); assert.match(writes[0].body.summary, /^A quick update from Sharpen:/); assert.match(writes[0].body.summary, /The deploy fix landed\./); assert.ok(writes[1].url.endsWith('/events/event-1/ack')); }); + + +test('deployed and standalone pod summaries request the routed non-Claude default', () => { + const chart = path.resolve(__dirname, '../../../k8s/helm/commonly/templates'); + const deployment = fs.readFileSync(path.join(chart, 'agents/commonly-bot-deployment.yaml'), 'utf8'); + const model = deployment.match(/- name: AGENT_MODEL\n\s+value: "([^"]+)"/)[1]; + const backend = fs.readFileSync(path.join(chart, 'core/backend-deployment.yaml'), 'utf8'); + const backendModel = backend.match(/- name: LITELLM_CHAT_MODEL[\s\S]*?value: "([^"]+)"/)[1]; + assert.equal(model, backendModel); + const router = fs.readFileSync(path.join(chart, 'configmaps/litellm-config.yaml'), 'utf8'); + const groups = [...router.matchAll(/- model_name: ([^\n]+)\n\s+litellm_params:\n\s+model: ([^\n]+)/g)]; + const route = groups.find((group) => group[1] === model); + assert.ok(route, `No route for ${model}`); + assert.doesNotMatch(`${model} ${route[2]}`, /claude|anthropic/i); + for (const deployed of [false, true]) { + const env = { ...process.env, LITELLM_BASE_URL: 'http://fake-llm' }; + delete env.COMMONLY_SOCIAL_REPHRASE_MODEL; + delete env.AGENT_MODEL; + if (deployed) env.AGENT_MODEL = model; + // Fresh process exercises actual client + request body, without singleton + // env leakage or a paid model call. + const result = execFileSync(process.execPath, ['-e', ` + const { buildLlmPodSummary } = require('./index'); + global.fetch = async (_url, options) => { + process.stdout.write(JSON.parse(options.body).model); + return { ok: true, json: async () => ({ choices: [{ message: { content: '{"summary":"Done"}' } }] }) }; + }; + buildLlmPodSummary({ messages: [{ username: 'Sam', content: 'Done' }] }).catch(() => process.exit(1)); + `], { cwd: __dirname, env, encoding: 'utf8' }); + assert.equal(result, model); + } +}); + + +test('failed social rewrite names its feature/model and still posts heuristic content', () => { + const env = { + ...process.env, + COMMONLY_SOCIAL_POST_TO_FEED: '1', + COMMONLY_SOCIAL_REPHRASE_ENABLED: '1', + COMMONLY_SOCIAL_REPHRASE_MODEL: 'deepseek-v4-flash', + COMMONLY_SOCIAL_IMAGE_ENABLED: '0', + COMMONLY_SOCIAL_PUBLISH_EXTERNAL: '0', + LITELLM_BASE_URL: 'http://fake-llm', + OPENROUTER_API_KEY: '', + GEMINI_API_KEY: '', + }; + const result = execFileSync(process.execPath, ['-e', ` + const assert = require('node:assert/strict'); + const { handleEvent } = require('./index'); + const calls = []; + const warnings = []; + console.warn = (line) => warnings.push(line); + global.fetch = async (url, options = {}) => { + calls.push({ url, body: options.body && JSON.parse(options.body) }); + let data = {}; + if (url.endsWith('/context')) data = { pod: { name: 'Testing' } }; + else if (url.endsWith('/api/posts?category=Social')) data = [{ content: 'Testing deploy finished', source: { provider: 'x' } }]; + else if (url.endsWith('/social-policy')) data = { policy: { socialMode: 'rewrite' } }; + else if (url.endsWith('/v1/chat/completions')) return { ok: false, status: 401, text: async () => 'bad key' }; + else if (url.includes('category=Social')) data = []; + return { ok: true, json: async () => data }; + }; + handleEvent({ runtimeToken: 'test', userToken: 'test' }, { _id: 'curate-1', type: 'curate', podId: 'pod-1' }).then(() => { + assert.deepEqual(warnings, ['[commonly-bot] feature=social-rephrase model=deepseek-v4-flash failed; using heuristic feed post']); + const feed = calls.find((call) => call.url.endsWith('/api/posts')); + assert.match(feed.body.content, /A notable update/); + assert.ok(calls.at(-1).url.endsWith('/curate-1/ack')); + }).catch((error) => { console.error(error); process.exit(1); }); + `], { cwd: __dirname, env, encoding: 'utf8' }); + assert.equal(result, ''); +}); diff --git a/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml b/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml index 1df14c1d2b..bf729689ce 100644 --- a/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml +++ b/k8s/helm/commonly/templates/agents/commonly-bot-deployment.yaml @@ -44,6 +44,9 @@ spec: value: /app/config/runtime.json - name: COMMONLY_AGENT_POLL_MS value: {{ .Values.agents.commonlyBot.env.pollIntervalMs | quote }} + # Match the backend default; avoid a dead Gemini primary before fallback. + - name: AGENT_MODEL + value: "deepseek-v4-flash" - name: LITELLM_BASE_URL value: http://litellm:4000 - name: LITELLM_MASTER_KEY From ea87caf6242948a8bc911524f288c1f097730dad Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:57:40 -0700 Subject: [PATCH 2/7] fix(runtime): remove direct Anthropic routes and inventory installs read-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D76 amendment: census only, no apply path or installation writes. Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- .../censusAnthropicInstallations.test.js | 65 +++++++++++++++++++ .../scripts/census-anthropic-installations.ts | 57 ++++++++++++++++ .../services/gatewayProvisionerServiceK8s.ts | 4 -- docs/SUMMARIZER_AND_AGENTS.md | 31 +++++++++ .../commonly-bot/index.test.js | 19 ++++++ .../templates/agents/clawdbot-deployment.yaml | 16 ----- .../templates/agents/litellm-deployment.yaml | 8 +-- .../templates/configmaps/litellm-config.yaml | 53 +-------------- .../templates/core/backend-deployment.yaml | 6 -- .../templates/ops/ops-runner-deployment.yaml | 6 -- .../templates/secrets/local-secrets.yaml | 1 - k8s/helm/commonly/values-local.yaml | 1 - 12 files changed, 175 insertions(+), 92 deletions(-) create mode 100644 backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js create mode 100644 backend/scripts/census-anthropic-installations.ts diff --git a/backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js b/backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js new file mode 100644 index 0000000000..d760e81e17 --- /dev/null +++ b/backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js @@ -0,0 +1,65 @@ +const mongoose = require('mongoose'); +const { censusAnthropicInstallations, main } = require('../../../scripts/census-anthropic-installations'); + +const row = (runtimeType, model, environmentModel) => ({ + config: { runtime: { runtimeType, model }, environment: { model: environmentModel } }, +}); + +describe('D76 dry-run-only Anthropic census', () => { + it('counts hosted tiers and excludes local CLI, webhooks and unknown tiers regardless of Claude model', () => { + const rows = [ + ...['native', 'hosted', 'internal', 'moltbot', 'managed-agents'].map((type) => row(type, 'claude-haiku-4-5')), + ...['claude-code', 'codex', 'webhook', undefined, ''].map((type) => row(type, undefined, 'claude-sonnet-4-6')), + row('native', 'deepseek-v4-flash'), + row('native', 'anthropic/claude-opus-4-7'), + row('hosted', 'openrouter/anthropic/claude-sonnet-4-6'), + row('NATIVE ', undefined, ' CLAUDE-haiku-4-5 '), + {}, + ]; + const before = JSON.stringify(rows); + const result = censusAnthropicInstallations(rows); + expect(result).toMatchObject({ dryRun: true, scanned: 15, hostedCandidates: 8, excludedByoOrUnknown: 5 }); + expect(result.byRuntime).toMatchObject({ native: 3, hosted: 2, unset: 2, 'claude-code': 1, codex: 1 }); + expect(JSON.stringify(rows)).toBe(before); + }); + + it.each(['--apply', '--dry-run', '--unknown'])('rejects %s before connecting or writing', async (arg) => { + const connect = jest.spyOn(mongoose, 'createConnection'); + try { + await expect(main([arg])).rejects.toThrow('Read-only census'); + expect(connect).not.toHaveBeenCalled(); + } finally { connect.mockRestore(); } + }); +}); + +it('reads real Mongo without changing BYO/hosted documents or indexes', async () => { + const { MongoMemoryServer } = require('mongodb-memory-server'); + const mongo = await MongoMemoryServer.create(); + const uri = mongo.getUri('d76-census'); + const connection = await mongoose.createConnection(uri).asPromise(); + const collection = connection.collection('agentinstallations'); + const oldUri = process.env.MONGO_URI; + const log = jest.spyOn(console, 'log').mockImplementation(() => {}); + try { + await collection.insertMany([ + row('native', 'claude-haiku-4-5'), + row('claude-code', undefined, 'claude-sonnet-4-6'), + row(undefined, undefined, 'claude-opus-4-7'), + ]); + const documents = await collection.find({}).toArray(); + const indexes = await collection.indexes(); + process.env.MONGO_URI = uri; + await main([]); + expect(JSON.parse(log.mock.calls[0][0])).toMatchObject({ + scanned: 3, hostedCandidates: 1, excludedByoOrUnknown: 2, dryRun: true, + }); + expect(await collection.find({}).toArray()).toEqual(documents); + expect(await collection.indexes()).toEqual(indexes); + } finally { + if (oldUri === undefined) delete process.env.MONGO_URI; + else process.env.MONGO_URI = oldUri; + log.mockRestore(); + await connection.close(); + await mongo.stop(); + } +}); diff --git a/backend/scripts/census-anthropic-installations.ts b/backend/scripts/census-anthropic-installations.ts new file mode 100644 index 0000000000..801070267c --- /dev/null +++ b/backend/scripts/census-anthropic-installations.ts @@ -0,0 +1,57 @@ +/** + * D76 read-only census. Do not migrate models until OpenRouter routing is ready. + * Run: npx ts-node scripts/census-anthropic-installations.ts + * Requires MONGO_URI. Every argument (including --apply) is refused before DB access. + * Uses a raw collection read: no model/index initialization, notices or updates. + */ +/* eslint-disable no-console */ +import mongoose from 'mongoose'; + +const HOSTED_RUNTIME_TYPES = new Set(['native', 'hosted', 'internal', 'moltbot', 'managed-agents']); +type Installation = { + config?: { + runtime?: { runtimeType?: unknown; model?: unknown }; + environment?: { model?: unknown }; + }; +}; + +const isClaudeModel = (value: unknown): boolean => typeof value === 'string' + && /^(?:(?:anthropic|openrouter)\/)?(?:anthropic\/)?claude-/i.test(value.trim()); + +export const censusAnthropicInstallations = (rows: Installation[]) => { + let hostedCandidates = 0; + let excluded = 0; + const byRuntime: Record = {}; + for (const row of rows) { + const runtime = row.config?.runtime; + if (!isClaudeModel(runtime?.model) && !isClaudeModel(row.config?.environment?.model)) continue; + const runtimeType = String(runtime?.runtimeType || '').trim().toLowerCase(); + byRuntime[runtimeType || 'unset'] = (byRuntime[runtimeType || 'unset'] || 0) + 1; + if (HOSTED_RUNTIME_TYPES.has(runtimeType)) hostedCandidates += 1; + else excluded += 1; // Local CLI, webhook and unknown tiers must never be reclassified by model name. + } + return { dryRun: true, scanned: rows.length, hostedCandidates, excludedByoOrUnknown: excluded, byRuntime }; +}; + +export const main = async (args = process.argv.slice(2)): Promise => { + if (args.length) throw new Error('Read-only census: no arguments accepted; --apply is unavailable (D76).'); + if (!process.env.MONGO_URI) throw new Error('MONGO_URI is required'); + const connection = mongoose.createConnection(process.env.MONGO_URI, { autoIndex: false, autoCreate: false }); + try { + await connection.asPromise(); + const rows = await connection.collection('agentinstallations').find({}, { + projection: { _id: 0, 'config.runtime.runtimeType': 1, 'config.runtime.model': 1, 'config.environment.model': 1 }, + }).toArray(); + console.log(JSON.stringify(censusAnthropicInstallations(rows as Installation[]), null, 2)); + console.log('DRY RUN ONLY: no installations changed and no owner notices sent. OpenRouter routing pending.'); + } finally { + await connection.close(); + } +}; + +if (require.main === module) { + main().catch((error) => { + console.error('Anthropic installation census failed:', error.message); + process.exitCode = 1; + }); +} diff --git a/backend/services/gatewayProvisionerServiceK8s.ts b/backend/services/gatewayProvisionerServiceK8s.ts index e76631ae22..4b9d73c1dd 100644 --- a/backend/services/gatewayProvisionerServiceK8s.ts +++ b/backend/services/gatewayProvisionerServiceK8s.ts @@ -305,10 +305,6 @@ const createOrUpdateDeployment = async ({ name: 'GEMINI_API_KEY', valueFrom: { secretKeyRef: { name: 'api-keys', key: 'gemini-api-key' } }, }, - { - name: 'ANTHROPIC_API_KEY', - valueFrom: { secretKeyRef: { name: 'api-keys', key: 'anthropic-api-key', optional: true } }, - }, { name: 'OPENAI_API_KEY', valueFrom: { secretKeyRef: { name: 'api-keys', key: 'openai-api-key', optional: true } }, diff --git a/docs/SUMMARIZER_AND_AGENTS.md b/docs/SUMMARIZER_AND_AGENTS.md index 5ed100dc9e..83906090bb 100644 --- a/docs/SUMMARIZER_AND_AGENTS.md +++ b/docs/SUMMARIZER_AND_AGENTS.md @@ -85,6 +85,37 @@ requests from proxy loopback can also be operator probes. --- +### Direct Anthropic removal (D76) + +Direct `anthropic/*` LiteLLM routes and cluster-key consumers are removed. +Claude on BYO agents still runs on the owner's machine. Hosted Claude routing +through OpenRouter is separate work; existing model choices must not be +rewritten while that decision is pending. + +The census is strictly read-only: + +```sh +cd backend +npx ts-node scripts/census-anthropic-installations.ts +``` + +It requires `MONGO_URI`, prints only aggregate counts, and refuses every +argument, including `--apply`, before connecting. It checks runtime and +environment model fields, but only explicitly hosted runtimes (`native`, +`hosted`, `internal`, `moltbot`, `managed-agents`) count as hosted candidates. +CLI, webhook and unset/unknown tiers are reported separately. No documents, +owner notices or indexes are written. Use read-only DB credentials for the +production run; SRE owns that census. + +Deployment sequencing: merge and deploy the runtime/chart removal first. +SRE then removes the existing Secret Manager value and ESO mapping from +`templates/secrets/api-keys.yaml`, verifies ESO reconciliation, and checks +remaining runtime/PVC credential copies. The ExternalSecret entry is retained +until that owned cleanup so it is not confused with removal already performed +on production. No secret value is handled by this census or its tests. + +--- + ### **2. commonly-bot Agent Identity** **Location**: Created via `AgentIdentityService.getOrCreateAgentUser()` diff --git a/external/commonly-agent-services/commonly-bot/index.test.js b/external/commonly-agent-services/commonly-bot/index.test.js index 1975f91752..785ff03974 100644 --- a/external/commonly-agent-services/commonly-bot/index.test.js +++ b/external/commonly-agent-services/commonly-bot/index.test.js @@ -172,3 +172,22 @@ test('failed social rewrite names its feature/model and still posts heuristic co `], { cwd: __dirname, env, encoding: 'utf8' }); assert.equal(result, ''); }); + +test('chart and dynamic gateways no longer consume a direct Anthropic key or route', () => { + const root = path.resolve(__dirname, '../../..'); + const chart = path.join(root, 'k8s/helm/commonly'); + const routes = fs.readFileSync(path.join(chart, 'templates/configmaps/litellm-config.yaml'), 'utf8'); + assert.doesNotMatch(routes, /model_name:\s*claude-|model:\s*anthropic\//i); + assert.doesNotMatch(routes, /ANTHROPIC_API_KEY|["']claude-[^"']*["']/); + for (const file of [ + 'templates/core/backend-deployment.yaml', + 'templates/ops/ops-runner-deployment.yaml', + 'templates/agents/litellm-deployment.yaml', + 'templates/agents/clawdbot-deployment.yaml', + 'templates/secrets/local-secrets.yaml', + 'values-local.yaml', + ]) { + assert.doesNotMatch(fs.readFileSync(path.join(chart, file), 'utf8'), /ANTHROPIC_API_KEY|anthropic-api-key|anthropicApiKey/); + } + assert.doesNotMatch(fs.readFileSync(path.join(root, 'backend/services/gatewayProvisionerServiceK8s.ts'), 'utf8'), /ANTHROPIC_API_KEY|anthropic-api-key/); +}); diff --git a/k8s/helm/commonly/templates/agents/clawdbot-deployment.yaml b/k8s/helm/commonly/templates/agents/clawdbot-deployment.yaml index 16c454cd65..49598466ba 100644 --- a/k8s/helm/commonly/templates/agents/clawdbot-deployment.yaml +++ b/k8s/helm/commonly/templates/agents/clawdbot-deployment.yaml @@ -47,7 +47,6 @@ spec: const geminiKey = (process.env.GEMINI_API_KEY || '').trim(); const geminiKey2 = (process.env.GEMINI_API_KEY_2 || '').trim(); const openrouterKey = (process.env.OPENROUTER_API_KEY || '').trim(); - const anthropicKey = (process.env.ANTHROPIC_API_KEY || '').trim(); const openaiKey = (process.env.OPENAI_API_KEY || '').trim(); const codexAccess = (process.env.OPENAI_CODEX_ACCESS_TOKEN || '').trim(); const codexRefresh = (process.env.OPENAI_CODEX_REFRESH_TOKEN || '').trim(); @@ -144,9 +143,6 @@ spec: if (openrouterKey && !store.profiles['openrouter:default']) { store.profiles['openrouter:default'] = { type: 'api_key', provider: 'openrouter', key: openrouterKey }; } - if (anthropicKey && !store.profiles['anthropic:default']) { - store.profiles['anthropic:default'] = { type: 'api_key', provider: 'anthropic', key: anthropicKey }; - } if (openaiKey && !store.profiles['openai:default']) { store.profiles['openai:default'] = { type: 'api_key', provider: 'openai', key: openaiKey }; } @@ -338,12 +334,6 @@ spec: name: api-keys key: gemini-api-key-2 optional: true - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: - name: api-keys - key: anthropic-api-key - optional: true - name: OPENROUTER_API_KEY valueFrom: secretKeyRef: @@ -638,12 +628,6 @@ spec: secretKeyRef: name: api-keys key: gemini-api-key - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: - name: api-keys - key: anthropic-api-key - optional: true - name: OPENAI_API_KEY valueFrom: secretKeyRef: diff --git a/k8s/helm/commonly/templates/agents/litellm-deployment.yaml b/k8s/helm/commonly/templates/agents/litellm-deployment.yaml index 9fd511543d..6a9eb38c91 100644 --- a/k8s/helm/commonly/templates/agents/litellm-deployment.yaml +++ b/k8s/helm/commonly/templates/agents/litellm-deployment.yaml @@ -179,7 +179,7 @@ spec: # The source secret was fixed, but a malformed key from ANY source must never be # able to break routing again — so sanitize here too. litellm inherits this env # via exec below. - for _k in OPENROUTER_API_KEY OPENAI_API_KEY ANTHROPIC_API_KEY GEMINI_API_KEY; do + for _k in OPENROUTER_API_KEY OPENAI_API_KEY GEMINI_API_KEY; do eval "_orig=\${$_k:-}" _clean=$(printf '%s' "$_orig" | tr -d '\r\n') [ "$_orig" != "$_clean" ] && echo "sanitized $_k (stripped CR/LF)" @@ -243,12 +243,6 @@ spec: name: api-keys key: openai-api-key optional: true - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: - name: api-keys - key: anthropic-api-key - optional: true # chatgpt/ provider reads auth.json from CHATGPT_TOKEN_DIR (written by the # codex-auth-rotator sidecar; source files seeded by operator via the codex-cli # sidecar). See ADR-014. diff --git a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml index ee91fe21c5..ef1739dc96 100644 --- a/k8s/helm/commonly/templates/configmaps/litellm-config.yaml +++ b/k8s/helm/commonly/templates/configmaps/litellm-config.yaml @@ -232,57 +232,8 @@ data: model: openai/gpt-image-1 api_key: os.environ/OPENAI_API_KEY - # --- Anthropic (cluster ANTHROPIC_API_KEY, standard BYOK pattern) --- - # Consumers will point at these entries: - # 1. cluster-side `cloud-claude-code` runtime pods (Deployment - # templated parallel to cloud-codex) — set - # ANTHROPIC_BASE_URL=http://litellm:4000 and - # ANTHROPIC_API_KEY= - # 2. openclaw moltbots that need Claude models (dev-only, gated - # the same way as openai-codex/* — see applyOpenClawModelDefaults) - # - # Auth model matches every other LiteLLM caller in the cluster - # (codex CLI, backend services, dev moltbots): caller sends its - # LiteLLM virtual key as Authorization; LiteLLM validates it - # against LiteLLM_VerificationTokenTable; on hit, LiteLLM calls - # api.anthropic.com using the cluster's own ANTHROPIC_API_KEY - # (which holds Anthropic's $200/mo prepaid credit, provisioned - # into anthropic-api-key in api-keys secret on 2026-06-15). - # - # Until the real key replaces `placeholder` in the api-keys secret - # (target: 2026-06-15), any call to a claude-* model will 401 - # upstream with `invalid x-api-key`. No current consumers, so this - # is a harmless dormant path until provisioning day. - # - # OAuth pass-through (PR #432, x-litellm-api-key + ANTHROPIC_ - # CUSTOM_HEADERS) was considered and rejected: TOS risk on Max - # subscription multiplexing, refresh sidecar complexity matching - # codex-auth-rotator (incident-prone), quota concentration on a - # single account. Predictable per-token billing against $200/mo - # API credit is the right trade. - # - # Per-key scoping (models[], rpm/tpm/spend limits, key_alias) is - # set at `/key/generate` time — see docs/runbooks/litellm-claude-code.md. - - model_name: claude-opus-4-7 - litellm_params: - model: anthropic/claude-opus-4-7 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-sonnet-4-6 - litellm_params: - model: anthropic/claude-sonnet-4-6 - api_key: os.environ/ANTHROPIC_API_KEY - - - model_name: claude-haiku-4-5 - litellm_params: - model: anthropic/claude-haiku-4-5-20251001 - api_key: os.environ/ANTHROPIC_API_KEY - - # Versioned alias — Claude Code sometimes sends date-suffixed IDs - - model_name: claude-haiku-4-5-20251001 - litellm_params: - model: anthropic/claude-haiku-4-5-20251001 - api_key: os.environ/ANTHROPIC_API_KEY + # Direct Anthropic routes removed under D76. Claude on hosted runtimes + # awaits OpenRouter routing; BYO Claude Code remains independent. router_settings: # The codex-auth-rotator sidecar rotates Codex accounts every 10 min on diff --git a/k8s/helm/commonly/templates/core/backend-deployment.yaml b/k8s/helm/commonly/templates/core/backend-deployment.yaml index 61517210b9..b2d7080f92 100644 --- a/k8s/helm/commonly/templates/core/backend-deployment.yaml +++ b/k8s/helm/commonly/templates/core/backend-deployment.yaml @@ -290,12 +290,6 @@ spec: name: api-keys key: openai-api-key optional: true - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: - name: api-keys - key: anthropic-api-key - optional: true {{- if .Values.discord.enabled }} diff --git a/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml b/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml index 98b89cb749..3ae1aa98cb 100644 --- a/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml +++ b/k8s/helm/commonly/templates/ops/ops-runner-deployment.yaml @@ -257,12 +257,6 @@ spec: name: api-keys key: openai-api-key optional: true - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: - name: api-keys - key: anthropic-api-key - optional: true {{- if .Values.discord.enabled }} diff --git a/k8s/helm/commonly/templates/secrets/local-secrets.yaml b/k8s/helm/commonly/templates/secrets/local-secrets.yaml index c45ce2bdde..c4e72e7a7c 100644 --- a/k8s/helm/commonly/templates/secrets/local-secrets.yaml +++ b/k8s/helm/commonly/templates/secrets/local-secrets.yaml @@ -20,7 +20,6 @@ stringData: gemini-api-key: {{ .Values.localSecrets.geminiApiKey | default "" | quote }} openrouter-api-key: {{ .Values.localSecrets.openrouterApiKey | default "" | quote }} openai-api-key: {{ .Values.localSecrets.openaiApiKey | default "" | quote }} - anthropic-api-key: {{ .Values.localSecrets.anthropicApiKey | default "" | quote }} litellm-master-key: {{ .Values.localSecrets.litellmMasterKey | default "" | quote }} # Discord — empty by default; set to enable Discord integration diff --git a/k8s/helm/commonly/values-local.yaml b/k8s/helm/commonly/values-local.yaml index dd2e5fd4b1..48809d9b2a 100644 --- a/k8s/helm/commonly/values-local.yaml +++ b/k8s/helm/commonly/values-local.yaml @@ -174,7 +174,6 @@ localSecrets: geminiApiKey: "" openrouterApiKey: "" openaiApiKey: "" - anthropicApiKey: "" litellmMasterKey: "" discordBotToken: "" discordClientId: "" From 2321e8c8cd68d78f7113f8c0214d87ae8a8d52ec Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:59:50 -0700 Subject: [PATCH 3/7] test(runtime): preserve BYO selections and drop unneeded migration census MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Latest CTO D76 ruling: zero hosted candidates; no migration script or apply path. Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- .../censusAnthropicInstallations.test.js | 65 ------------------- .../services/litellmAvailability.byo.test.js | 44 +++++++++++++ .../scripts/census-anthropic-installations.ts | 57 ---------------- docs/SUMMARIZER_AND_AGENTS.md | 21 ++---- 4 files changed, 50 insertions(+), 137 deletions(-) delete mode 100644 backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js create mode 100644 backend/__tests__/unit/services/litellmAvailability.byo.test.js delete mode 100644 backend/scripts/census-anthropic-installations.ts diff --git a/backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js b/backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js deleted file mode 100644 index d760e81e17..0000000000 --- a/backend/__tests__/unit/scripts/censusAnthropicInstallations.test.js +++ /dev/null @@ -1,65 +0,0 @@ -const mongoose = require('mongoose'); -const { censusAnthropicInstallations, main } = require('../../../scripts/census-anthropic-installations'); - -const row = (runtimeType, model, environmentModel) => ({ - config: { runtime: { runtimeType, model }, environment: { model: environmentModel } }, -}); - -describe('D76 dry-run-only Anthropic census', () => { - it('counts hosted tiers and excludes local CLI, webhooks and unknown tiers regardless of Claude model', () => { - const rows = [ - ...['native', 'hosted', 'internal', 'moltbot', 'managed-agents'].map((type) => row(type, 'claude-haiku-4-5')), - ...['claude-code', 'codex', 'webhook', undefined, ''].map((type) => row(type, undefined, 'claude-sonnet-4-6')), - row('native', 'deepseek-v4-flash'), - row('native', 'anthropic/claude-opus-4-7'), - row('hosted', 'openrouter/anthropic/claude-sonnet-4-6'), - row('NATIVE ', undefined, ' CLAUDE-haiku-4-5 '), - {}, - ]; - const before = JSON.stringify(rows); - const result = censusAnthropicInstallations(rows); - expect(result).toMatchObject({ dryRun: true, scanned: 15, hostedCandidates: 8, excludedByoOrUnknown: 5 }); - expect(result.byRuntime).toMatchObject({ native: 3, hosted: 2, unset: 2, 'claude-code': 1, codex: 1 }); - expect(JSON.stringify(rows)).toBe(before); - }); - - it.each(['--apply', '--dry-run', '--unknown'])('rejects %s before connecting or writing', async (arg) => { - const connect = jest.spyOn(mongoose, 'createConnection'); - try { - await expect(main([arg])).rejects.toThrow('Read-only census'); - expect(connect).not.toHaveBeenCalled(); - } finally { connect.mockRestore(); } - }); -}); - -it('reads real Mongo without changing BYO/hosted documents or indexes', async () => { - const { MongoMemoryServer } = require('mongodb-memory-server'); - const mongo = await MongoMemoryServer.create(); - const uri = mongo.getUri('d76-census'); - const connection = await mongoose.createConnection(uri).asPromise(); - const collection = connection.collection('agentinstallations'); - const oldUri = process.env.MONGO_URI; - const log = jest.spyOn(console, 'log').mockImplementation(() => {}); - try { - await collection.insertMany([ - row('native', 'claude-haiku-4-5'), - row('claude-code', undefined, 'claude-sonnet-4-6'), - row(undefined, undefined, 'claude-opus-4-7'), - ]); - const documents = await collection.find({}).toArray(); - const indexes = await collection.indexes(); - process.env.MONGO_URI = uri; - await main([]); - expect(JSON.parse(log.mock.calls[0][0])).toMatchObject({ - scanned: 3, hostedCandidates: 1, excludedByoOrUnknown: 2, dryRun: true, - }); - expect(await collection.find({}).toArray()).toEqual(documents); - expect(await collection.indexes()).toEqual(indexes); - } finally { - if (oldUri === undefined) delete process.env.MONGO_URI; - else process.env.MONGO_URI = oldUri; - log.mockRestore(); - await connection.close(); - await mongo.stop(); - } -}); diff --git a/backend/__tests__/unit/services/litellmAvailability.byo.test.js b/backend/__tests__/unit/services/litellmAvailability.byo.test.js new file mode 100644 index 0000000000..638b56beda --- /dev/null +++ b/backend/__tests__/unit/services/litellmAvailability.byo.test.js @@ -0,0 +1,44 @@ +const mongoose = require('mongoose'); +const { MongoMemoryServer } = require('mongodb-memory-server'); +const LiteLLMClient = require('../../../../external/commonly-agent-services/shared/litellm-client'); + +// D76: gateway model availability describes hosted routes. Local CLI model +// selections must survive both disappearance of Claude routes and gateway failure. +describe('LiteLLM availability preserves BYO installation models', () => { + let mongo; + let collection; + const originalFetch = global.fetch; + beforeAll(async () => { + mongo = await MongoMemoryServer.create(); + await mongoose.connect(mongo.getUri('d76-byo')); + collection = mongoose.connection.collection('agentinstallations'); + }); + afterAll(async () => { + global.fetch = originalFetch; + await mongoose.disconnect(); + await mongo.stop(); + }); + + it.each([200, 401, 503])('does not rewrite local or unknown tiers when model listing returns %s', async (status) => { + await collection.deleteMany({}); + await collection.insertMany(['claude-code', 'codex', 'webhook', 'wrapper', undefined].map((runtimeType) => ({ + config: { + runtime: { ...(runtimeType ? { runtimeType } : {}), model: 'claude-haiku-4-5' }, + environment: { model: 'claude-sonnet-4-6' }, + }, + }))); + const before = await collection.find({}).toArray(); + global.fetch = jest.fn().mockResolvedValue({ + ok: status === 200, + status, + json: async () => ({ data: [{ id: 'deepseek-v4-flash' }] }), + }); + const client = new LiteLLMClient({ baseUrl: 'http://fake-litellm', apiKey: 'test-key' }); + try { + if (status === 200) await expect(client.listModels()).resolves.toEqual([{ id: 'deepseek-v4-flash' }]); + else await expect(client.listModels()).rejects.toThrow(`Failed to list models: ${status}`); + expect(global.fetch).toHaveBeenCalledWith('http://fake-litellm/v1/models', expect.any(Object)); + expect(await collection.find({}).toArray()).toEqual(before); + } finally { global.fetch = originalFetch; } + }); +}); diff --git a/backend/scripts/census-anthropic-installations.ts b/backend/scripts/census-anthropic-installations.ts deleted file mode 100644 index 801070267c..0000000000 --- a/backend/scripts/census-anthropic-installations.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** - * D76 read-only census. Do not migrate models until OpenRouter routing is ready. - * Run: npx ts-node scripts/census-anthropic-installations.ts - * Requires MONGO_URI. Every argument (including --apply) is refused before DB access. - * Uses a raw collection read: no model/index initialization, notices or updates. - */ -/* eslint-disable no-console */ -import mongoose from 'mongoose'; - -const HOSTED_RUNTIME_TYPES = new Set(['native', 'hosted', 'internal', 'moltbot', 'managed-agents']); -type Installation = { - config?: { - runtime?: { runtimeType?: unknown; model?: unknown }; - environment?: { model?: unknown }; - }; -}; - -const isClaudeModel = (value: unknown): boolean => typeof value === 'string' - && /^(?:(?:anthropic|openrouter)\/)?(?:anthropic\/)?claude-/i.test(value.trim()); - -export const censusAnthropicInstallations = (rows: Installation[]) => { - let hostedCandidates = 0; - let excluded = 0; - const byRuntime: Record = {}; - for (const row of rows) { - const runtime = row.config?.runtime; - if (!isClaudeModel(runtime?.model) && !isClaudeModel(row.config?.environment?.model)) continue; - const runtimeType = String(runtime?.runtimeType || '').trim().toLowerCase(); - byRuntime[runtimeType || 'unset'] = (byRuntime[runtimeType || 'unset'] || 0) + 1; - if (HOSTED_RUNTIME_TYPES.has(runtimeType)) hostedCandidates += 1; - else excluded += 1; // Local CLI, webhook and unknown tiers must never be reclassified by model name. - } - return { dryRun: true, scanned: rows.length, hostedCandidates, excludedByoOrUnknown: excluded, byRuntime }; -}; - -export const main = async (args = process.argv.slice(2)): Promise => { - if (args.length) throw new Error('Read-only census: no arguments accepted; --apply is unavailable (D76).'); - if (!process.env.MONGO_URI) throw new Error('MONGO_URI is required'); - const connection = mongoose.createConnection(process.env.MONGO_URI, { autoIndex: false, autoCreate: false }); - try { - await connection.asPromise(); - const rows = await connection.collection('agentinstallations').find({}, { - projection: { _id: 0, 'config.runtime.runtimeType': 1, 'config.runtime.model': 1, 'config.environment.model': 1 }, - }).toArray(); - console.log(JSON.stringify(censusAnthropicInstallations(rows as Installation[]), null, 2)); - console.log('DRY RUN ONLY: no installations changed and no owner notices sent. OpenRouter routing pending.'); - } finally { - await connection.close(); - } -}; - -if (require.main === module) { - main().catch((error) => { - console.error('Anthropic installation census failed:', error.message); - process.exitCode = 1; - }); -} diff --git a/docs/SUMMARIZER_AND_AGENTS.md b/docs/SUMMARIZER_AND_AGENTS.md index 83906090bb..fb9075aab6 100644 --- a/docs/SUMMARIZER_AND_AGENTS.md +++ b/docs/SUMMARIZER_AND_AGENTS.md @@ -92,27 +92,18 @@ Claude on BYO agents still runs on the owner's machine. Hosted Claude routing through OpenRouter is separate work; existing model choices must not be rewritten while that decision is pending. -The census is strictly read-only: - -```sh -cd backend -npx ts-node scripts/census-anthropic-installations.ts -``` - -It requires `MONGO_URI`, prints only aggregate counts, and refuses every -argument, including `--apply`, before connecting. It checks runtime and -environment model fields, but only explicitly hosted runtimes (`native`, -`hosted`, `internal`, `moltbot`, `managed-agents`) count as hosted candidates. -CLI, webhook and unset/unknown tiers are reported separately. No documents, -owner notices or indexes are written. Use read-only DB credentials for the -production run; SRE owns that census. +SRE's read-only census found no hosted/native Claude selections; the 47 +Claude selections were BYO seats. There is no migration script and no apply +path. Removing a hosted route must not rewrite BYO or unknown-tier models. +`litellmAvailability.byo.test.js` checks persisted local selections against both +a model listing without Claude and failed gateway availability requests. Deployment sequencing: merge and deploy the runtime/chart removal first. SRE then removes the existing Secret Manager value and ESO mapping from `templates/secrets/api-keys.yaml`, verifies ESO reconciliation, and checks remaining runtime/PVC credential copies. The ExternalSecret entry is retained until that owned cleanup so it is not confused with removal already performed -on production. No secret value is handled by this census or its tests. +on production. No secret value is handled by these tests. --- From 308d222f60de8208d11e7cae2e1100e6f6982abe Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:02:04 -0700 Subject: [PATCH 4/7] fix(commonly-bot): log rejected social rewrites before heuristic delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- .../commonly-bot/index.js | 10 ++- .../commonly-bot/index.test.js | 87 +++++++++++-------- 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/external/commonly-agent-services/commonly-bot/index.js b/external/commonly-agent-services/commonly-bot/index.js index 033b13c14e..e8e996b8c7 100644 --- a/external/commonly-agent-services/commonly-bot/index.js +++ b/external/commonly-agent-services/commonly-bot/index.js @@ -232,9 +232,15 @@ const rephraseSocialPost = async ({ post, podName }) => { { model: SOCIAL_REPHRASE_MODEL, temperature: 0.5, maxTokens: 300 }, ); const parsed = parseJsonBlock(raw); - if (!parsed?.summary) return null; + if (typeof parsed?.summary !== 'string' || !parsed.summary.trim()) { + console.warn(`[commonly-bot] feature=social-rephrase model=${SOCIAL_REPHRASE_MODEL} rejected=missing-summary; using heuristic feed post`); + return null; + } const overlap = lexicalOverlap(sourceText, parsed.summary); - if (overlap > 0.7) return null; + if (overlap > 0.7) { + console.warn(`[commonly-bot] feature=social-rephrase model=${SOCIAL_REPHRASE_MODEL} rejected=source-overlap; using heuristic feed post`); + return null; + } return { summary: String(parsed.summary).trim(), why: String(parsed.why || '').trim(), diff --git a/external/commonly-agent-services/commonly-bot/index.test.js b/external/commonly-agent-services/commonly-bot/index.test.js index 785ff03974..f10318b98f 100644 --- a/external/commonly-agent-services/commonly-bot/index.test.js +++ b/external/commonly-agent-services/commonly-bot/index.test.js @@ -135,43 +135,58 @@ test('deployed and standalone pod summaries request the routed non-Claude defaul }); -test('failed social rewrite names its feature/model and still posts heuristic content', () => { - const env = { - ...process.env, - COMMONLY_SOCIAL_POST_TO_FEED: '1', - COMMONLY_SOCIAL_REPHRASE_ENABLED: '1', - COMMONLY_SOCIAL_REPHRASE_MODEL: 'deepseek-v4-flash', - COMMONLY_SOCIAL_IMAGE_ENABLED: '0', - COMMONLY_SOCIAL_PUBLISH_EXTERNAL: '0', - LITELLM_BASE_URL: 'http://fake-llm', - OPENROUTER_API_KEY: '', - GEMINI_API_KEY: '', - }; - const result = execFileSync(process.execPath, ['-e', ` - const assert = require('node:assert/strict'); - const { handleEvent } = require('./index'); - const calls = []; - const warnings = []; - console.warn = (line) => warnings.push(line); - global.fetch = async (url, options = {}) => { - calls.push({ url, body: options.body && JSON.parse(options.body) }); - let data = {}; - if (url.endsWith('/context')) data = { pod: { name: 'Testing' } }; - else if (url.endsWith('/api/posts?category=Social')) data = [{ content: 'Testing deploy finished', source: { provider: 'x' } }]; - else if (url.endsWith('/social-policy')) data = { policy: { socialMode: 'rewrite' } }; - else if (url.endsWith('/v1/chat/completions')) return { ok: false, status: 401, text: async () => 'bad key' }; - else if (url.includes('category=Social')) data = []; - return { ok: true, json: async () => data }; +for (const [mode, reason] of [ + ['http-error', 'failed'], + ['missing', 'rejected=missing-summary'], + ['blank', 'rejected=missing-summary'], + ['copied', 'rejected=source-overlap'], +]) { + test(`${mode} social rewrite names its feature/model and still posts heuristic content`, () => { + const env = { + ...process.env, + COMMONLY_TEST_REWRITE_MODE: mode, + COMMONLY_TEST_REWRITE_REASON: reason, + COMMONLY_SOCIAL_POST_TO_FEED: '1', + COMMONLY_SOCIAL_REPHRASE_ENABLED: '1', + COMMONLY_SOCIAL_REPHRASE_MODEL: 'deepseek-v4-flash', + COMMONLY_SOCIAL_IMAGE_ENABLED: '0', + COMMONLY_SOCIAL_PUBLISH_EXTERNAL: '0', + LITELLM_BASE_URL: 'http://fake-llm', + OPENROUTER_API_KEY: '', + GEMINI_API_KEY: '', }; - handleEvent({ runtimeToken: 'test', userToken: 'test' }, { _id: 'curate-1', type: 'curate', podId: 'pod-1' }).then(() => { - assert.deepEqual(warnings, ['[commonly-bot] feature=social-rephrase model=deepseek-v4-flash failed; using heuristic feed post']); - const feed = calls.find((call) => call.url.endsWith('/api/posts')); - assert.match(feed.body.content, /A notable update/); - assert.ok(calls.at(-1).url.endsWith('/curate-1/ack')); - }).catch((error) => { console.error(error); process.exit(1); }); - `], { cwd: __dirname, env, encoding: 'utf8' }); - assert.equal(result, ''); -}); + const result = execFileSync(process.execPath, ['-e', ` + const assert = require('node:assert/strict'); + const { handleEvent } = require('./index'); + const calls = []; + const warnings = []; + console.warn = (line) => warnings.push(line); + global.fetch = async (url, options = {}) => { + calls.push({ url, body: options.body && JSON.parse(options.body) }); + let data = {}; + if (url.endsWith('/context')) data = { pod: { name: 'Testing' } }; + else if (url.endsWith('/api/posts?category=Social')) data = [{ content: 'Testing deploy finished', source: { provider: 'x' } }]; + else if (url.endsWith('/social-policy')) data = { policy: { socialMode: 'rewrite' } }; + else if (url.endsWith('/v1/chat/completions')) { + if (process.env.COMMONLY_TEST_REWRITE_MODE === 'http-error') return { ok: false, status: 401, text: async () => 'bad key' }; + const reply = process.env.COMMONLY_TEST_REWRITE_MODE === 'missing' ? { why: 'No summary' } + : { summary: process.env.COMMONLY_TEST_REWRITE_MODE === 'blank' ? ' ' : 'Testing deploy finished' }; + data = { choices: [{ message: { content: JSON.stringify(reply) } }] }; + } + else if (url.includes('category=Social')) data = []; + return { ok: true, json: async () => data }; + }; + handleEvent({ runtimeToken: 'test', userToken: 'test' }, { _id: 'curate-1', type: 'curate', podId: 'pod-1' }).then(() => { + assert.deepEqual(warnings, ['[commonly-bot] feature=social-rephrase model=deepseek-v4-flash ' + process.env.COMMONLY_TEST_REWRITE_REASON + '; using heuristic feed post']); + const feed = calls.find((call) => call.url.endsWith('/api/posts')); + assert.match(feed.body.content, /A notable update/); + assert.ok(calls.at(-1).url.endsWith('/curate-1/ack')); + }).catch((error) => { console.error(error); process.exit(1); }); + `], { cwd: __dirname, env, encoding: 'utf8' }); + assert.equal(result, ''); + }); + +} test('chart and dynamic gateways no longer consume a direct Anthropic key or route', () => { const root = path.resolve(__dirname, '../../..'); From 7ee534b6fcda5b3092caa34fb598e8ca93650afe Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:02:47 -0700 Subject: [PATCH 5/7] docs(runtime): source D76 census and bound availability regression claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- docs/SUMMARIZER_AND_AGENTS.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/SUMMARIZER_AND_AGENTS.md b/docs/SUMMARIZER_AND_AGENTS.md index fb9075aab6..fac9d9f112 100644 --- a/docs/SUMMARIZER_AND_AGENTS.md +++ b/docs/SUMMARIZER_AND_AGENTS.md @@ -92,11 +92,13 @@ Claude on BYO agents still runs on the owner's machine. Hosted Claude routing through OpenRouter is separate work; existing model choices must not be rewritten while that decision is pending. -SRE's read-only census found no hosted/native Claude selections; the 47 -Claude selections were BYO seats. There is no migration script and no apply +[SRE's read-only census, relayed in TASK-001 on 2026-10-25](https://github.com/Team-Commonly/commonly/pull/2704#issuecomment-6396818571), +found no hosted/native Claude selections; the 47 Claude selections were BYO seats. There is no migration script and no apply path. Removing a hosted route must not rewrite BYO or unknown-tier models. -`litellmAvailability.byo.test.js` checks persisted local selections against both -a model listing without Claude and failed gateway availability requests. +`litellmAvailability.byo.test.js` checks that the shared client leaves persisted +local selections unchanged across a model listing without Claude and failed +gateway availability requests. There is currently no backend caller or rewrite +path for that listing; this test does not guard a future migration implementation. Deployment sequencing: merge and deploy the runtime/chart removal first. SRE then removes the existing Secret Manager value and ESO mapping from From 4577c4bec8577f43598831fa5074cb643d508d69 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:13:44 -0700 Subject: [PATCH 6/7] test(runtime): remove inert Mongo fixture and prove route removal patterns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- ...tellmAvailability.byo.test.js => litellmClient.models.test.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename backend/__tests__/unit/services/{litellmAvailability.byo.test.js => litellmClient.models.test.js} (100%) diff --git a/backend/__tests__/unit/services/litellmAvailability.byo.test.js b/backend/__tests__/unit/services/litellmClient.models.test.js similarity index 100% rename from backend/__tests__/unit/services/litellmAvailability.byo.test.js rename to backend/__tests__/unit/services/litellmClient.models.test.js From 6f926db487e334c79d9999ade207f1747c00ca90 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sun, 25 Oct 2026 04:13:54 -0700 Subject: [PATCH 7/7] test(runtime): finish model listing scope and positive route controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Written by Kai, a Commonly agent — https://commonly.me/v2/pods/6addd4c17f8fc798a6f66431 Co-authored-by: kai Agent-Model: codex/gpt-6.1-sol/medium --- .../services/litellmClient.models.test.js | 40 ++++--------------- docs/SUMMARIZER_AND_AGENTS.md | 9 +++-- .../commonly-bot/index.test.js | 10 ++++- 3 files changed, 21 insertions(+), 38 deletions(-) diff --git a/backend/__tests__/unit/services/litellmClient.models.test.js b/backend/__tests__/unit/services/litellmClient.models.test.js index 638b56beda..cdacdb47fd 100644 --- a/backend/__tests__/unit/services/litellmClient.models.test.js +++ b/backend/__tests__/unit/services/litellmClient.models.test.js @@ -1,44 +1,20 @@ -const mongoose = require('mongoose'); -const { MongoMemoryServer } = require('mongodb-memory-server'); const LiteLLMClient = require('../../../../external/commonly-agent-services/shared/litellm-client'); -// D76: gateway model availability describes hosted routes. Local CLI model -// selections must survive both disappearance of Claude routes and gateway failure. -describe('LiteLLM availability preserves BYO installation models', () => { - let mongo; - let collection; +// The client reads hosted gateway availability. It accepts no installations; +// there is currently no backend caller that rewrites local model selections. +describe('LiteLLM model listing', () => { const originalFetch = global.fetch; - beforeAll(async () => { - mongo = await MongoMemoryServer.create(); - await mongoose.connect(mongo.getUri('d76-byo')); - collection = mongoose.connection.collection('agentinstallations'); - }); - afterAll(async () => { - global.fetch = originalFetch; - await mongoose.disconnect(); - await mongo.stop(); - }); + afterEach(() => { global.fetch = originalFetch; }); - it.each([200, 401, 503])('does not rewrite local or unknown tiers when model listing returns %s', async (status) => { - await collection.deleteMany({}); - await collection.insertMany(['claude-code', 'codex', 'webhook', 'wrapper', undefined].map((runtimeType) => ({ - config: { - runtime: { ...(runtimeType ? { runtimeType } : {}), model: 'claude-haiku-4-5' }, - environment: { model: 'claude-sonnet-4-6' }, - }, - }))); - const before = await collection.find({}).toArray(); + it.each([200, 401, 503])('returns models or rejects gateway status %s', async (status) => { global.fetch = jest.fn().mockResolvedValue({ ok: status === 200, status, json: async () => ({ data: [{ id: 'deepseek-v4-flash' }] }), }); const client = new LiteLLMClient({ baseUrl: 'http://fake-litellm', apiKey: 'test-key' }); - try { - if (status === 200) await expect(client.listModels()).resolves.toEqual([{ id: 'deepseek-v4-flash' }]); - else await expect(client.listModels()).rejects.toThrow(`Failed to list models: ${status}`); - expect(global.fetch).toHaveBeenCalledWith('http://fake-litellm/v1/models', expect.any(Object)); - expect(await collection.find({}).toArray()).toEqual(before); - } finally { global.fetch = originalFetch; } + if (status === 200) await expect(client.listModels()).resolves.toEqual([{ id: 'deepseek-v4-flash' }]); + else await expect(client.listModels()).rejects.toThrow(`Failed to list models: ${status}`); + expect(global.fetch).toHaveBeenCalledWith('http://fake-litellm/v1/models', expect.any(Object)); }); }); diff --git a/docs/SUMMARIZER_AND_AGENTS.md b/docs/SUMMARIZER_AND_AGENTS.md index fac9d9f112..540f92a5fe 100644 --- a/docs/SUMMARIZER_AND_AGENTS.md +++ b/docs/SUMMARIZER_AND_AGENTS.md @@ -95,10 +95,11 @@ rewritten while that decision is pending. [SRE's read-only census, relayed in TASK-001 on 2026-10-25](https://github.com/Team-Commonly/commonly/pull/2704#issuecomment-6396818571), found no hosted/native Claude selections; the 47 Claude selections were BYO seats. There is no migration script and no apply path. Removing a hosted route must not rewrite BYO or unknown-tier models. -`litellmAvailability.byo.test.js` checks that the shared client leaves persisted -local selections unchanged across a model listing without Claude and failed -gateway availability requests. There is currently no backend caller or rewrite -path for that listing; this test does not guard a future migration implementation. +There is currently no backend caller or installation-rewrite path for the +shared client's model listing. `litellmClient.models.test.js` checks only the +listing response and gateway errors. It does not prove BYO persistence safety; +a future availability-driven rewrite requires its own runtime-tier regression. +This change includes no installation writes or migration code. Deployment sequencing: merge and deploy the runtime/chart removal first. SRE then removes the existing Secret Manager value and ESO mapping from diff --git a/external/commonly-agent-services/commonly-bot/index.test.js b/external/commonly-agent-services/commonly-bot/index.test.js index f10318b98f..45c4d44e7a 100644 --- a/external/commonly-agent-services/commonly-bot/index.test.js +++ b/external/commonly-agent-services/commonly-bot/index.test.js @@ -192,8 +192,14 @@ test('chart and dynamic gateways no longer consume a direct Anthropic key or rou const root = path.resolve(__dirname, '../../..'); const chart = path.join(root, 'k8s/helm/commonly'); const routes = fs.readFileSync(path.join(chart, 'templates/configmaps/litellm-config.yaml'), 'utf8'); - assert.doesNotMatch(routes, /model_name:\s*claude-|model:\s*anthropic\//i); - assert.doesNotMatch(routes, /ANTHROPIC_API_KEY|["']claude-[^"']*["']/); + const directRoute = /model_name:\s*claude-|model:\s*anthropic\//i; + const directKeyOrFallback = /ANTHROPIC_API_KEY|["']claude-[^"']*["']/; + assert.match(' - model_name: claude-haiku-4-5', directRoute); + assert.match(' model: anthropic/claude-haiku-4-5-20251001', directRoute); + assert.match(' api_key: os.environ/ANTHROPIC_API_KEY', directKeyOrFallback); + assert.match(' - {"claude-haiku-4-5": ["deepseek-v4-flash"]}', directKeyOrFallback); + assert.doesNotMatch(routes, directRoute); + assert.doesNotMatch(routes, directKeyOrFallback); for (const file of [ 'templates/core/backend-deployment.yaml', 'templates/ops/ops-runner-deployment.yaml',