diff --git a/.changeset/byok-cheap-small-model.md b/.changeset/byok-cheap-small-model.md new file mode 100644 index 000000000000..9df72ecb01f1 --- /dev/null +++ b/.changeset/byok-cheap-small-model.md @@ -0,0 +1,5 @@ +--- +"@kilocode/cli": patch +--- + +Prefer the session provider's cheapest chat-capable model for auxiliary tasks (session titles, prompt enhance, commit messages, branch names) instead of falling back to Kilo's auto small model. BYOK users no longer silently draw Kilo credits when their configured provider has no recognized small-model family. Local model servers keep the session's own model instead of picking a catalog entry the server may not have installed. diff --git a/packages/opencode/src/kilocode/provider/provider.ts b/packages/opencode/src/kilocode/provider/provider.ts index e0ae755451ed..44a0777dbc62 100644 --- a/packages/opencode/src/kilocode/provider/provider.ts +++ b/packages/opencode/src/kilocode/provider/provider.ts @@ -14,7 +14,7 @@ import { optionalOmitUndefined } from "@opencode-ai/core/schema" import { ProviderError } from "@/provider/error" import { Effect, Schema } from "effect" import type { LanguageModelV3 } from "@ai-sdk/provider" -import { mapValues, omit, pickBy } from "remeda" +import { mapValues, omit, pickBy, sortBy } from "remeda" import { reasoningSummary } from "./reasoning-summary" import type { Provider } from "@/provider/provider" import type { Auth } from "@/auth" @@ -299,6 +299,101 @@ export function kiloSmallModelPriority(providerID: string): string[] | undefined return undefined } +/** + * Picks the cheapest chat-capable model from the provider's own catalog as the + * small model for auxiliary tasks (session titles, prompt enhance, commit + * messages, branch names). Keeps those calls on the provider the user + * configured instead of falling through to Kilo's auto small model, which + * draws Kilo credits in BYOK setups. Chat-capable means text in and text out; + * tool calling is not required (e.g. Perplexity sonar and morph chat models + * advertise tool_call false). Models with no text output (image, audio, video) + * are skipped, and so are embedding and rerank models: they also take text in + * and return text scores, but requests to them cannot produce titles or commit + * messages. Missing cost data sorts as free; ties break by release date and + * id, matching the catalog ordering in getSmallModel. + * + * Non-generative exclusion is a case-insensitive id/family/api-id heuristic: + * besides the obvious embed/rerank substrings it covers common embedding + * families (bge, gte, e5, uae, instructor, colbert, voyage, + * sentence-similarity) whose ids carry no embed/rerank token (e.g. + * intfloat/e5-mistral-7b-instruct, bge-m3, Qwen/Qwen3-Reranker-0.6B), plus + * safety/classification models (guard, safeguard, moderation, shield, + * llama-guard, shieldgemma) whose text output is a label rather than + * generated prose. The api id is checked alongside the alias/family so an + * alias like `vectors` pointing at `nomic-embed-text` cannot bypass the + * exclusion. + */ +const NON_GENERATIVE_RE = + /embed|rerank|sentence[-_ ]similarity|text-embedding|bge|gte-?(?:large|base|small)?|e5-|-e5\b|\buae\b|instructor|colbert|voyage(?![a-z])|guard|safeguard|moderation|shield|llama[-_]?guard|shieldgemma|toxic|harm|pii|nsfw/i + +export function cheapestSmallModel(models: Provider.Model[]) { + return sortBy( + models.filter((model) => { + if (!model.capabilities.input.text || !model.capabilities.output.text) return false + return !NON_GENERATIVE_RE.test(`${model.family} ${model.id} ${model.api?.id ?? ""}`) + }), + [(model) => model.cost.input + model.cost.output, "asc"], + [(model) => model.release_date, "desc"], + [(model) => model.id, "desc"], + ).at(0) +} + +/** + * Provider ids that always front a local model server (LM Studio, Ollama, + * Atomic Chat, Anaconda Desktop). Their catalogs list every known model, but + * the server only serves installed ones — regardless of which hostname the + * user points them at (localhost, a LAN ip, host.docker.internal, ...). + */ +const LOCAL_SERVER_IDS = new Set(["lmstudio", "ollama", "atomic-chat", "anaconda-desktop"]) + +/** + * True when the provider routes to a local server (LM Studio, Ollama, ...). + * Local catalogs list every known model, but the server only serves installed + * ones — the cheapest catalog entry can name a model the user never installed + * (e.g. openai/gpt-oss-20b while the session uses Qwen). Callers fall back to + * the session's own model when getSmallModel returns undefined, so the cheap + * pick is skipped for local providers. A hostname check alone is not enough: + * a local server remains local when reached over the LAN + * (192.168.x.x, host.docker.internal, ollama.local), so known local-server + * ids always take the safe fallback. Only an unrelated remote endpoint keeps + * the cheap pick. + */ +export function isLoopbackProvider(provider: { + id?: unknown + options?: Record + models: Record +}) { + if (typeof provider.id === "string" && LOCAL_SERVER_IDS.has(provider.id)) return true + const raw = provider.options?.baseURL ?? provider.options?.baseUrl + if (typeof raw === "string" && raw !== "") return isLoopbackURL(raw) + const urls = Object.values(provider.models).map((model) => model.api?.url).filter(Boolean) + if (urls.length === 0) return false + return urls.every(isLoopbackURL) +} + +function isLoopbackURL(raw: string) { + try { + // URL keeps IPv6 brackets, so normalize before comparing hostnames. + const host = new URL(raw).hostname.toLowerCase().replace(/^\[|\]$/g, "") + if (host === "localhost" || host === "0.0.0.0" || host === "::" || host === "::1") return true + if (host === "host.docker.internal" || host.endsWith(".local") || host.endsWith(".internal")) return true + // Private, link-local, and loopback IPv4 ranges, plus IPv6 ULA (fc00::/7) + // and link-local (fe80::/10). Treating a company endpoint on these ranges + // as local only skips the cheap pick: callers keep the session model, so + // the conservative classification is safe. + return ( + /^127\./.test(host) || + /^10\./.test(host) || + /^192\.168\./.test(host) || + /^172\.(?:1[6-9]|2\d|3[01])\./.test(host) || + /^169\.254\./.test(host) || + /^(?:f[cd][0-9a-f]{2}|fe[89ab][0-9a-f]):/.test(host) + ) + } catch { + return false + } +} + /** * True when the user has kilo credentials: a KILO_API_KEY env var, a stored * auth entry, or an apiKey in the kilo provider config. Mirrors the hasKey diff --git a/packages/opencode/src/provider/provider.ts b/packages/opencode/src/provider/provider.ts index 73e63769e56b..2864563637f8 100644 --- a/packages/opencode/src/provider/provider.ts +++ b/packages/opencode/src/provider/provider.ts @@ -45,6 +45,8 @@ import { publicKiloProvider, kiloSmallModelPriority, hasKiloCredentials, + cheapestSmallModel, + isLoopbackProvider, buildTimeoutSignal, requestTimeout, wrapFirstByte, @@ -2141,6 +2143,16 @@ const layer = Layer.effect( if (candidates[0]) return candidates[0] } + // kilocode_change start - skip local-server providers: their catalog lists + // every known model but the server only serves installed ones, so the + // cheapest entry can name a model the user never installed. Return + // undefined before the Kilo fallback so auxiliary tasks use the session's + // own model; falling through would bill Kilo credits for a BYOK setup. + if (isLoopbackProvider(provider)) return undefined + const cheap = cheapestSmallModel(Object.values(provider.models)) + if (cheap) return cheap + // kilocode_change end + // kilocode_change start - fall back to kilo's auto small model only when the user actually has // kilo credentials. The kilo provider is always autoloaded (anonymous key), so checking it // unconditionally would route auxiliary tasks (session titles, commit messages, branch names) diff --git a/packages/opencode/test/kilocode/provider/provider.test.ts b/packages/opencode/test/kilocode/provider/provider.test.ts index 4d53ddb9fc9f..d3dbc92bdd7e 100644 --- a/packages/opencode/test/kilocode/provider/provider.test.ts +++ b/packages/opencode/test/kilocode/provider/provider.test.ts @@ -1,10 +1,11 @@ -import { afterEach, expect } from "bun:test" +import { afterEach, describe, expect, test } from "bun:test" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { ProviderV2 } from "@opencode-ai/core/provider" import { Effect } from "effect" import { Env } from "@/env" import { Plugin } from "@/plugin/index" import { Provider } from "@/provider/provider" +import { cheapestSmallModel, isLoopbackProvider } from "@/kilocode/provider/provider" import { disposeAllInstances } from "../../fixture/fixture" import { testEffect } from "../../lib/effect" @@ -33,13 +34,13 @@ afterEach(async () => { const it = testEffect(LayerNode.compile(LayerNode.group([Provider.node, Env.node, Plugin.node]))) it.instance( - "getSmallModel returns undefined without kilo credentials when model IDs lack family metadata", + "getSmallModel picks provider model when model IDs lack family metadata", Effect.gen(function* () { for (const key of ["KILO_API_KEY", "KILO_AUTH_CONTENT", "KILO_CONFIG_CONTENT"]) { yield* clearEnv(key) } const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("test-provider")) - expect(model).toBeUndefined() + expect(model).toMatchObject({ providerID: "test-provider", id: "gpt-5-nano" }) }), { config: { @@ -59,10 +60,10 @@ it.instance( ) it.instance( - "getSmallModel falls back to Kilo auto when the kilo provider is configured", + "getSmallModel picks provider model over kilo auto", Effect.gen(function* () { const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("test-provider")) - expect(model).toMatchObject({ providerID: "kilo", id: "kilo-auto/small" }) + expect(model).toMatchObject({ providerID: "test-provider", id: "gpt-5-nano" }) }), { config: { @@ -82,3 +83,304 @@ it.instance( }, }, ) + +it.instance( + "getSmallModel prefers family match over a cheaper unlisted model", + Effect.gen(function* () { + const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("test-provider")) + expect(model).toMatchObject({ providerID: "test-provider", id: "new-flash" }) + }), + { + config: { + provider: { + "test-provider": { + name: "Test Provider", + npm: "@ai-sdk/openai-compatible", + models: { + "new-flash": { family: "gemini-flash", release_date: "2026-01-01", cost: { input: 3, output: 15 } }, + "cheap-thing": { release_date: "2026-06-01", cost: { input: 0, output: 0 } }, + }, + options: { apiKey: "test-key" }, + }, + }, + }, + }, +) + +// The #13192 contract: without kilo credentials the cloud fallback stays +// unreachable, so auxiliary tasks fall back to the session's own model +// (getSmallModel returning undefined). +it.instance( + "getSmallModel returns undefined for loopback providers so callers keep the session model", + Effect.gen(function* () { + for (const key of ["KILO_API_KEY", "KILO_AUTH_CONTENT", "KILO_CONFIG_CONTENT"]) { + yield* clearEnv(key) + } + const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("lmstudio")) + expect(model).toBeUndefined() + }), + { + config: { + provider: { + lmstudio: { + name: "LM Studio", + npm: "@ai-sdk/openai-compatible", + models: { + // Catalog lists every known model, but only installed ones are + // served. The cheap pick must not name an uninstalled model. + "openai/gpt-oss-20b": { release_date: "2026-01-01" }, + "qwen/qwen3-30b-a3b-2507": { release_date: "2026-02-01" }, + }, + options: { baseURL: "http://localhost:1234/v1" }, + }, + kilo: null, + }, + }, + }, +) +it.instance( + "getSmallModel keeps the session model for loopback providers even with kilo credentials", + Effect.gen(function* () { + for (const key of ["KILO_API_KEY", "KILO_AUTH_CONTENT", "KILO_CONFIG_CONTENT"]) { + yield* clearEnv(key) + } + const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("lmstudio")) + expect(model).toBeUndefined() + }), + { + config: { + provider: { + lmstudio: { + name: "LM Studio", + npm: "@ai-sdk/openai-compatible", + models: { + "openai/gpt-oss-20b": { release_date: "2026-01-01" }, + "qwen/qwen3-30b-a3b-2507": { release_date: "2026-02-01" }, + }, + options: { baseURL: "http://localhost:1234/v1" }, + }, + kilo: { options: { apiKey: "kilo-key" } }, + }, + }, + }, +) +it.instance( + "getSmallModel returns undefined without kilo credentials when the provider has no chat-capable model", + Effect.gen(function* () { + for (const key of ["KILO_API_KEY", "KILO_AUTH_CONTENT", "KILO_CONFIG_CONTENT"]) { + yield* clearEnv(key) + } + const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("test-provider")) + expect(model).toBeUndefined() + }), + { + config: { + provider: { + "test-provider": { + name: "Test Provider", + npm: "@ai-sdk/openai-compatible", + models: { + "nomic-embed-text": { release_date: "2026-01-01", modalities: { input: ["text"], output: [] } }, + }, + options: { apiKey: "test-key" }, + }, + kilo: null, + }, + }, + }, +) + +it.instance( + "getSmallModel falls back to kilo-auto/small with kilo credentials when the provider has no chat-capable model", + Effect.gen(function* () { + const model = yield* Provider.use.getSmallModel(ProviderV2.ID.make("test-provider")) + expect(model).toMatchObject({ providerID: "kilo", id: "kilo-auto/small" }) + }), + { + config: { + provider: { + "test-provider": { + name: "Test Provider", + npm: "@ai-sdk/openai-compatible", + models: { + "nomic-embed-text": { release_date: "2026-01-01", modalities: { input: ["text"], output: [] } }, + }, + options: { apiKey: "test-key" }, + }, + kilo: { + options: { apiKey: "kilo-key" }, + }, + }, + }, + }, +) + +describe("cheapestSmallModel", () => { + const model = (over: { id: string } & Record): Provider.Model => + ({ + providerID: "test", + name: over.id, + family: "", + capabilities: { + toolcall: true, + attachment: false, + reasoning: false, + temperature: true, + input: { text: true, image: false, audio: false, video: false, pdf: false }, + output: { text: true, image: false, audio: false, video: false, pdf: false }, + }, + cost: { input: 1, output: 1, cache: { read: 0, write: 0 } }, + release_date: "2026-01-01", + options: {}, + headers: {}, + ...over, + }) as Provider.Model + + test("picks the cheapest text model", () => { + const picked = cheapestSmallModel([ + model({ id: "premium", cost: { input: 3, output: 15, cache: { read: 0, write: 0 } } }), + model({ id: "budget", cost: { input: 0.1, output: 0.2, cache: { read: 0, write: 0 } } }), + model({ id: "mid", cost: { input: 0.5, output: 1, cache: { read: 0, write: 0 } } }), + ]) + expect(picked).toMatchObject({ id: "budget" }) + }) + + test("breaks cost ties by release date, then id", () => { + const same = model({ id: "b", release_date: "2026-01-01" }) + const newer = model({ id: "a", release_date: "2026-06-01" }) + const sameAsNewer = model({ id: "z", release_date: "2026-06-01" }) + expect(cheapestSmallModel([same, newer])).toMatchObject({ id: "a" }) + expect(cheapestSmallModel([newer, sameAsNewer])).toMatchObject({ id: "z" }) + }) + + test("skips embedding and rerank models even when they are free", () => { + const free = { input: 0, output: 0, cache: { read: 0, write: 0 } } + const picked = cheapestSmallModel([ + model({ id: "nomic-embed-text", family: "nomic-embed", cost: free }), + model({ id: "jina-reranker-v3", cost: free }), + model({ id: "Qwen/Qwen3-Reranker-0.6B", cost: free }), + model({ id: "intfloat/e5-mistral-7b-instruct", cost: free }), + model({ id: "bge-m3", cost: free }), + model({ id: "gpt-oss-120b", cost: { input: 0.15, output: 0.6, cache: { read: 0, write: 0 } } }), + ]) + expect(picked).toMatchObject({ id: "gpt-oss-120b" }) + }) + + test("keeps similarly-named chat models", () => { + const picked = cheapestSmallModel([ + model({ id: "voyager-chat", cost: { input: 0.5, output: 0.5, cache: { read: 0, write: 0 } } }), + ]) + expect(picked).toMatchObject({ id: "voyager-chat" }) + }) + + test("keeps tool-less chat models like sonar", () => { + const picked = cheapestSmallModel([ + model({ + id: "sonar", + family: "sonar", + capabilities: { + toolcall: false, + attachment: false, + reasoning: false, + temperature: true, + input: { text: true, image: false, audio: false, video: false, pdf: false }, + output: { text: true, image: false, audio: false, video: false, pdf: false }, + }, + cost: { input: 1, output: 1, cache: { read: 0, write: 0 } }, + }), + ]) + expect(picked).toMatchObject({ id: "sonar" }) + }) + + test("skips models without text output", () => { + const picked = cheapestSmallModel([ + model({ + id: "image-gen", + capabilities: { + toolcall: true, + attachment: false, + reasoning: false, + temperature: true, + input: { text: true, image: false, audio: false, video: false, pdf: false }, + output: { text: false, image: true, audio: false, video: false, pdf: false }, + }, + cost: { input: 0, output: 0, cache: { read: 0, write: 0 } }, + }), + model({ id: "chat", cost: { input: 0.5, output: 0.5, cache: { read: 0, write: 0 } } }), + ]) + expect(picked).toMatchObject({ id: "chat" }) + }) + + test("skips safety and classification models even when they are free", () => { + const free = { input: 0, output: 0, cache: { read: 0, write: 0 } } + const picked = cheapestSmallModel([ + model({ id: "gpt-oss-safeguard-120b", cost: free }), + model({ id: "qwen3guard-gen-8b", cost: free }), + model({ id: "llama-guard-4-12b", cost: free }), + model({ id: "chat", cost: { input: 0.5, output: 0.5, cache: { read: 0, write: 0 } } }), + ]) + expect(picked).toMatchObject({ id: "chat" }) + }) + + test("checks the api id so an alias cannot bypass the embedding exclusion", () => { + const free = { input: 0, output: 0, cache: { read: 0, write: 0 } } + const picked = cheapestSmallModel([ + model({ id: "vectors", api: { id: "nomic-embed-text" }, cost: free }), + model({ id: "chat", api: { id: "chat" }, cost: { input: 0.5, output: 0.5, cache: { read: 0, write: 0 } } }), + ]) + expect(picked).toMatchObject({ id: "chat" }) + }) +}) + +describe("isLoopbackProvider", () => { + test("loopback baseURL marks a local provider", () => { + expect(isLoopbackProvider({ options: { baseURL: "http://localhost:1234/v1" }, models: {} })).toBe(true) + expect(isLoopbackProvider({ options: { baseURL: "http://127.0.0.1:11434/v1" }, models: {} })).toBe(true) + }) + + test("remote baseURL is not local even for a local-sounding provider", () => { + expect(isLoopbackProvider({ options: { baseURL: "https://api.openai.com/v1" }, models: {} })).toBe(false) + }) + + test("loopback model urls mark a local provider without a baseURL", () => { + const url = "http://127.0.0.1:1234/v1" + expect(isLoopbackProvider({ options: {}, models: { a: { api: { url } } as any } })).toBe(true) + expect(isLoopbackProvider({ options: {}, models: {} })).toBe(false) + }) + + test("known local-server ids stay local on LAN and custom hostnames", () => { + // A local server remains install-gated when reached over the LAN: the + // catalog still lists models the user never installed. + for (const id of ["lmstudio", "ollama", "atomic-chat", "anaconda-desktop"]) { + for (const baseURL of [ + "http://192.168.1.50:1234/v1", + "http://host.docker.internal:11434/v1", + "http://ollama.local:11434/v1", + ]) { + expect(isLoopbackProvider({ id, options: { baseURL }, models: {} })).toBe(true) + } + } + }) + + test("unknown ids on remote hosts keep the cheap catalog pick", () => { + expect(isLoopbackProvider({ id: "openai", options: { baseURL: "https://api.openai.com/v1" }, models: {} })).toBe( + false, + ) + }) + + test("unknown ids on private hosts are treated as local", () => { + for (const baseURL of [ + "http://192.168.1.50:1234/v1", + "http://10.0.0.12:1234/v1", + "http://172.16.4.2:1234/v1", + "http://host.docker.internal:11434/v1", + "http://ollama.local:11434/v1", + "http://0.0.0.0:1234/v1", + "http://[::]:1234/v1", + "http://[fd00::1]:1234/v1", + "http://[fe80::1]:1234/v1", + ]) { + expect(isLoopbackProvider({ id: "my-local-server", options: { baseURL }, models: {} })).toBe(true) + } + }) +})