From 621c1b7dcedae1adcdcaf1ea7b1e4cbed55af508 Mon Sep 17 00:00:00 2001 From: "Jorge Thomas (Akrista)" <23145794+Akrista@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:51:06 +0000 Subject: [PATCH] fix(ai): support OpenCode and OpenRouter gateways in OpenAI-compatible provider --- src/lib/ai/providers.test.ts | 44 ++++++++ src/lib/ai/providers/openai-compatible.ts | 126 +++++++++++++++++++--- src/lib/ai/structured-output.test.ts | 45 ++++++++ 3 files changed, 200 insertions(+), 15 deletions(-) diff --git a/src/lib/ai/providers.test.ts b/src/lib/ai/providers.test.ts index a7ef67e5..4671fdc3 100644 --- a/src/lib/ai/providers.test.ts +++ b/src/lib/ai/providers.test.ts @@ -163,6 +163,50 @@ describe("OpenAICompatibleProvider", () => { const provider = new OpenAICompatibleProvider("OPENAI", "https://api.example.com/v1", "sk", null, 60_000); expect(await provider.listModels()).toEqual(["a-model", "b-model"]); }); + + it("normalises OpenCode and OpenRouter addresses, sends OpenCode's session header, and omits dummy auth when no key is set", async () => { + const requests = stubFetch(({ url }) => + url.pathname.endsWith("/models") + ? json({ object: "list", data: [{ id: "glm-5.3-flash", object: "model" }] }) + : openaiChat("Hello!"), + ); + const go = new OpenAICompatibleProvider("OPENAI", "https://opencode.ai/inference/go/openai/v1", null, "glm-5.3-flash", 60_000); + expect(await go.listModels()).toEqual(["glm-5.3-flash"]); + expect(requests[0].url.href).toBe("https://opencode.ai/inference/go/v1/models"); + expect(requests[0].headers.get("authorization")).toBeNull(); + expect(requests[0].headers.get("x-opencode-session")).toMatch(/^ses_/); + expect(requests[0].headers.get("user-agent")).toMatch(/^Dossier\//); + + const goRoot = new OpenAICompatibleProvider("OPENAI", "https://opencode.ai/inference/go", "oc_sk_test", "glm-5.3-flash", 60_000); + expect(await goRoot.complete(ask)).toBe("Hello!"); + expect(requests[1].url.href).toBe("https://opencode.ai/inference/go/v1/chat/completions"); + expect(requests[1].headers.get("authorization")).toBe("Bearer oc_sk_test"); + expect(requests[1].headers.get("x-opencode-session")).toMatch(/^ses_/); + + const router = new OpenAICompatibleProvider("OPENAI", "https://openrouter.ai/v1", "sk-or-test", "openrouter/auto", 60_000); + expect(await router.complete(ask)).toBe("Hello!"); + expect(requests[2].url.href).toBe("https://openrouter.ai/api/v1/chat/completions"); + }); + + it("leaves room for thinking when a reasoning model gets cut off at the default budget, and remembers it", async () => { + const requests = stubFetch(({ body }) => { + const max = (body as { max_tokens?: number }).max_tokens ?? 0; + if (max <= 4096) { + return json({ + id: "c1", + object: "chat.completion", + created: 0, + model: "glm-5", + choices: [{ index: 0, message: { role: "assistant", content: "", reasoning_content: "Thinking through the resume..." }, finish_reason: "length" }], + }); + } + return openaiChat('{"name":"Jane","years":9}'); + }); + const provider = new OpenAICompatibleProvider("OPENAI", "https://thinker.example/v1", "sk", "glm-5", 60_000); + expect(await provider.complete({ ...ask, schema: Person })).toEqual({ name: "Jane", years: 9 }); + expect(await provider.complete({ ...ask, schema: Person })).toEqual({ name: "Jane", years: 9 }); + expect(requests.map((request) => (request.body as { max_tokens?: number }).max_tokens)).toEqual([4096, 4096 + 12_000, 4096 + 12_000]); + }); }); describe("AnthropicProvider", () => { diff --git a/src/lib/ai/providers/openai-compatible.ts b/src/lib/ai/providers/openai-compatible.ts index 3e66db75..47f374dd 100644 --- a/src/lib/ai/providers/openai-compatible.ts +++ b/src/lib/ai/providers/openai-compatible.ts @@ -1,9 +1,13 @@ import "server-only"; +import { randomUUID } from "node:crypto"; + import OpenAI from "openai"; import { zodResponseFormat } from "openai/helpers/zod"; import type { ReasoningEffort } from "openai/resources/shared"; +import { appVersion } from "@/lib/version"; + import { BaseProvider, withJsonSchema, type ChatAnswer, type ChatCall } from "../provider"; import { keyGuardedFetch } from "./guarded-fetch"; @@ -11,7 +15,10 @@ import { keyGuardedFetch } from "./guarded-fetch"; const JSON_MODES = ["schema", "object", "prompt"] as const; type JsonMode = (typeof JSON_MODES)[number]; -type Quirks = { noTemperature: boolean; maxCompletionTokens: boolean; noEffort: boolean; json: JsonMode }; +/** Room for a model that thinks before it writes; the thinking counts towards `max_tokens`. */ +const THINKING_TOKENS = 12_000; + +type Quirks = { noTemperature: boolean; maxCompletionTokens: boolean; noEffort: boolean; thinks: boolean; json: JsonMode }; // What each server and model turned out to accept, so it's only learnt once: // newer OpenAI models refuse `temperature` and `max_tokens`, many compatible @@ -19,6 +26,13 @@ type Quirks = { noTemperature: boolean; maxCompletionTokens: boolean; noEffort: // every model takes a reasoning effort. const quirks = new Map(); +// Schemas the API refused as too complex (union-typed and optional parameters, +// grammar size), by server, model and schema name: large ones, such as a whole +// profile, go in the prompt instead while simpler ones keep structured output (#230). +const promptSchemas = new Set(); +const SCHEMA_REFUSED = /grammar is too large|parameters with union types|optional parameters|schema is too complex/i; +const JSON_REFUSED = /response_format|json_schema|json_object|structured|invalid schema|support the provided parameters|provider returned error/i; + /** * The answer's text. Some servers, Mistral's among them, send a model that * thinks as a list of chunks: the text ones are the answer, the thinking @@ -32,56 +46,131 @@ export function answerText(content: unknown): string { .join(""); } +/** Whether a chat completion included reasoning tokens that counted towards `max_tokens`. */ +function hasReasoning(completion: OpenAI.Chat.Completions.ChatCompletion): boolean { + if ((completion.usage?.completion_tokens_details?.reasoning_tokens ?? 0) > 0) return true; + const message = completion.choices[0]?.message as unknown as Record | undefined; + if (!message) return false; + if (typeof message.reasoning_content === "string" && message.reasoning_content.length > 0) return true; + if (typeof message.reasoning === "string" && message.reasoning.length > 0) return true; + return Array.isArray(message.content) && message.content.some((chunk: { type?: unknown }) => chunk?.type === "thinking" || chunk?.type === "reasoning"); +} + +function hostname(baseURL: string): string { + return URL.canParse(baseURL) ? new URL(baseURL).hostname : ""; +} + /** * DeepSeek's API, by the server's host rather than the model's name (#189): * its models think before they answer unless told not to, and the thinking * counts towards `max_tokens`, so longer answers ran out of room. */ export function isDeepSeek(baseURL: string): boolean { - const host = URL.canParse(baseURL) ? new URL(baseURL).hostname : ""; + const host = hostname(baseURL); return host === "deepseek.com" || host.endsWith(".deepseek.com"); } +/** OpenCode's inference gateway, which requires a session header and serves `/models` at `/v1`. */ +export function isOpenCode(baseURL: string): boolean { + const host = hostname(baseURL); + return host === "opencode.ai" || host.endsWith(".opencode.ai"); +} + +/** OpenRouter's gateway, whose OpenAI-compatible root is `/api/v1` rather than `/v1`. */ +export function isOpenRouter(baseURL: string): boolean { + const host = hostname(baseURL); + return host === "openrouter.ai" || host.endsWith(".openrouter.ai"); +} + +/** Normalises known gateway addresses so both `/models` and `/chat/completions` resolve. */ +export function normalizeBaseURL(address: string): string { + const trimmed = address.trim().replace(/\/+$/, ""); + if (!URL.canParse(trimmed)) return trimmed; + const url = new URL(trimmed); + const path = url.pathname.replace(/\/+$/, ""); + if (isOpenRouter(trimmed) && (path === "" || path === "/v1" || path === "/api")) { + url.pathname = "/api/v1"; + return url.toString().replace(/\/+$/, ""); + } + if (isOpenCode(trimmed)) { + const prefix = path.replace(/\/(?:openai\/v1|openai|v1)$/, "") || "/inference"; + url.pathname = `${prefix}/v1`; + return url.toString().replace(/\/+$/, ""); + } + return trimmed; +} + /** Any server that speaks the OpenAI chat completions API, including llama.cpp's server. */ export class OpenAICompatibleProvider extends BaseProvider { protected readonly client: OpenAI; + protected readonly baseURL: string; private readonly noThinking: boolean; constructor( readonly kind: "OPENAI" | "LOCAL" | "MISTRAL" | "LMSTUDIO", - private readonly baseURL: string, + baseURL: string, apiKey: string | null, model: string | null, timeoutMs: number, ) { super(model, timeoutMs); - // Local servers often need no key, but the SDK insists on one. + this.baseURL = normalizeBaseURL(baseURL); + const openCodeHeaders = isOpenCode(this.baseURL) + ? { + "User-Agent": `Dossier/${appVersion} (+https://github.com/AienTech/dossier)`, + "x-opencode-session": `ses_${randomUUID()}`, + } + : {}; + // Local servers often need no key, but the SDK insists on one; omit the + // dummy bearer header so public model lists don't reject it as invalid. this.client = new OpenAI({ - baseURL, + baseURL: this.baseURL, apiKey: apiKey || "not-needed", timeout: timeoutMs, maxRetries: 1, + defaultHeaders: { + ...(apiKey ? {} : { Authorization: null }), + ...openCodeHeaders, + }, // With a key attached, no redirects to other hosts (E11-5). ...(apiKey ? { fetch: keyGuardedFetch() } : {}), }); - this.noThinking = isDeepSeek(baseURL); + this.noThinking = isDeepSeek(this.baseURL); } protected async chat(call: ChatCall): Promise { const key = `${this.baseURL} ${call.model}`; - let known: Quirks = quirks.get(key) ?? { noTemperature: false, maxCompletionTokens: false, noEffort: false, json: "schema" }; - // A 400 or 422 naming a parameter this model doesn't take: adapt and - // remember, once per parameter. + const schemaKey = call.json ? `${key} ${call.json.name}` : ""; + let known: Quirks = quirks.get(key) ?? { noTemperature: false, maxCompletionTokens: false, noEffort: false, thinks: false, json: "schema" }; + // A 400, 422 or gateway 404 naming a parameter this model doesn't take: + // adapt and remember, once per parameter. for (;;) { try { - return await this.send(call, known); + const effective: Quirks = promptSchemas.has(schemaKey) ? { ...known, json: "prompt" } : known; + const { answer, reasoned } = await this.send(call, effective); + if (reasoned && !known.thinks) { + known = { ...known, thinks: true }; + quirks.set(key, known); + if (answer.cutOff) continue; + } + return answer; } catch (error) { - const message = error instanceof OpenAI.BadRequestError || error instanceof OpenAI.UnprocessableEntityError ? error.message : ""; - const jsonRefused = call.json !== undefined && /response_format|json_schema|json_object|structured/i.test(message); + const status = error instanceof OpenAI.APIError ? error.status : undefined; + const message = error instanceof OpenAI.APIError && (status === 400 || status === 404 || status === 422) ? error.message : ""; + if (call.json !== undefined && (status === 400 || status === 422) && SCHEMA_REFUSED.test(message) && !promptSchemas.has(schemaKey)) { + promptSchemas.add(schemaKey); + continue; + } + const jsonRefused = + call.json !== undefined && + (status === 400 || status === 422 + ? JSON_REFUSED.test(message) + : status === 404 && /response_format|json_schema|json_object|structured|support the provided parameters/i.test(message)); const adapted: Quirks = { noTemperature: known.noTemperature || (call.temperature !== undefined && /temperature/i.test(message)), maxCompletionTokens: known.maxCompletionTokens || /max_tokens/.test(message), noEffort: known.noEffort || (this.reasoningEffort(call) !== undefined && /reasoning_effort/i.test(message)), + thinks: known.thinks, json: jsonRefused ? (JSON_MODES[JSON_MODES.indexOf(known.json) + 1] ?? known.json) : known.json, }; if ( @@ -104,8 +193,12 @@ export class OpenAICompatibleProvider extends BaseProvider { return undefined; } - private async send(call: ChatCall, { noTemperature, maxCompletionTokens, noEffort, json }: Quirks): Promise { + private async send( + call: ChatCall, + { noTemperature, maxCompletionTokens, noEffort, thinks, json }: Quirks, + ): Promise<{ answer: ChatAnswer; reasoned: boolean }> { const effort = noEffort ? undefined : this.reasoningEffort(call); + const maxTokens = call.maxTokens + (thinks ? THINKING_TOKENS : 0); // Without schema support, the schema goes into the instructions instead. const system = call.json && json !== "schema" ? withJsonSchema(call.system, call.json.schema) : call.system; const responseFormat = !call.json @@ -120,7 +213,7 @@ export class OpenAICompatibleProvider extends BaseProvider { model: call.model, messages: [{ role: "system", content: system }, ...call.messages], ...(noTemperature || call.temperature === undefined ? {} : { temperature: call.temperature }), - ...(maxCompletionTokens ? { max_completion_tokens: call.maxTokens } : { max_tokens: call.maxTokens }), + ...(maxCompletionTokens ? { max_completion_tokens: maxTokens } : { max_tokens: maxTokens }), ...(effort ? { reasoning_effort: effort } : {}), ...(this.noThinking ? { thinking: { type: "disabled" } } : {}), ...responseFormat, @@ -130,7 +223,10 @@ export class OpenAICompatibleProvider extends BaseProvider { const choice = completion.choices[0]; // Mistral says `model_length` when the context ran out instead. const finish: string | undefined = choice?.finish_reason; - return { text: answerText(choice?.message?.content), cutOff: finish === "length" || finish === "model_length" }; + return { + answer: { text: answerText(choice?.message?.content), cutOff: finish === "length" || finish === "model_length" }, + reasoned: hasReasoning(completion), + }; } protected async fetchModels(signal: AbortSignal): Promise { diff --git a/src/lib/ai/structured-output.test.ts b/src/lib/ai/structured-output.test.ts index 79893867..fe6eb018 100644 --- a/src/lib/ai/structured-output.test.ts +++ b/src/lib/ai/structured-output.test.ts @@ -142,4 +142,49 @@ describe("OpenAI-compatible servers without JSON schemas", () => { await provider.complete(ask); expect(requests.length - before).toBe(1); }); + + it("falls back when a gateway refuses response_format with a generic or 404 parameter error", async () => { + const requests = stubFetch(({ body }) => { + const format = (body as { response_format?: { type: string } }).response_format?.type; + if (format === "json_schema") { + return json({ error: { message: "No endpoints found that support the provided parameters" } }, { status: 404 }); + } + if (format === "json_object") { + return json({ error: { message: "Provider returned error" } }, { status: 400 }); + } + return json({ + id: "c", + object: "chat.completion", + created: 0, + model: "gateway-model", + choices: [{ index: 0, message: { role: "assistant", content: '{"company": "Ledgerly", "years": 5}' }, finish_reason: "stop" }], + }); + }); + const provider = new OpenAICompatibleProvider("OPENAI", "https://gateway.example/v1", "sk", "gateway-model", 60_000); + expect(await provider.complete(ask)).toEqual({ company: "Ledgerly", years: 5 }); + expect(requests).toHaveLength(3); + }); + + it("puts a schema refused as too complex in the prompt only for that schema, keeping structured output for others", async () => { + const City = z.object({ city: z.string() }); + const requests = stubFetch(({ body }) => { + const format = (body as { response_format?: unknown }).response_format; + if (JSON.stringify(format ?? {}).includes('"years"')) { + return json({ error: { message: "The compiled grammar is too large, which would cause performance issues." } }, { status: 400 }); + } + return json({ + id: "c", + object: "chat.completion", + created: 0, + model: "strict-model", + choices: [{ index: 0, message: { role: "assistant", content: format ? '{"city":"Utrecht"}' : '{"company":"Ledgerly","years":5}' }, finish_reason: "stop" }], + }); + }); + const provider = new OpenAICompatibleProvider("OPENAI", "https://complex.example/v1", "sk", "strict-model", 60_000); + expect(await provider.complete({ ...ask, schemaName: "posting" })).toEqual({ company: "Ledgerly", years: 5 }); + expect(await provider.complete({ ...ask, schema: City, schemaName: "city" })).toEqual({ city: "Utrecht" }); + expect(requests).toHaveLength(3); + expect(requests[1].body).not.toHaveProperty("response_format"); + expect(requests[2].body).toMatchObject({ response_format: { type: "json_schema" } }); + }); });