fix: put the schema in the prompt when Anthropic refuses it as too complex

src/lib/ai/providers.test.ts

@@ -200,6 +200,25 @@ describe("AnthropicProvider", () => {
     expect(requests[0].body).toMatchObject({ output_config: { format: { type: "json_schema" } } });
   });
 
+  it("puts the schema in the prompt when the API refuses it as too complex, and remembers that", async () => {
+    const baseUrl = `https://claude-${Math.random()}.example`;
+    const requests = stubFetch(({ body }) =>
+      (body as { output_config?: { format?: unknown } }).output_config?.format
+        ? json(
+            { type: "error", error: { type: "invalid_request_error", message: "The compiled grammar is too large, which would cause performance issues." } },
+            { status: 400 },
+          )
+        : anthropicMessage('{"name":"Jane","years":9}'),
+    );
+    const provider = new AnthropicProvider(baseUrl, "sk-ant", "claude-sonnet-5", 60_000);
+    expect(await provider.complete({ ...ask, schema: Person })).toEqual({ name: "Jane", years: 9 });
+    expect(await provider.complete({ ...ask, schema: Person })).toEqual({ name: "Jane", years: 9 });
+    expect(requests).toHaveLength(3);
+    expect(requests[1].body).not.toHaveProperty("output_config");
+    expect((requests[1].body as { system: string }).system).toContain("matches this JSON schema");
+    expect(requests[2].body).not.toHaveProperty("output_config");
+  });
+
   it("says when the model declines", async () => {
     stubFetch(() => anthropicMessage("", "refusal"));
     const provider = new AnthropicProvider("https://api.anthropic.com", "sk-ant", "claude-sonnet-5", 60_000);

src/lib/ai/providers/anthropic.ts

@@ -4,7 +4,7 @@ import Anthropic from "@anthropic-ai/sdk";
 import { zodOutputFormat } from "@anthropic-ai/sdk/helpers/zod";
 
 import { AiError } from "../errors";
-import { BaseProvider, type ChatAnswer, type ChatCall, type Effort } from "../provider";
+import { BaseProvider, withJsonSchema, type ChatAnswer, type ChatCall, type Effort } from "../provider";
 import { keyGuardedFetch } from "./guarded-fetch";
 
 /** Offered when the model list can't be fetched, e.g. before a key is saved. */
@@ -21,6 +21,12 @@ const THINKING_TOKENS = 12_000;
 // models, such as Claude Haiku 4.5, refuse the parameter.
 const effortLevels = new Map<string, ReadonlySet<Effort>>();
 
+// Schemas the API refused for structured output, by server, model and schema
+// name: it caps how complex one may be (union-typed parameters, grammar size),
+// so large ones, such as a whole profile, go in the prompt instead (E6-4).
+const promptSchemas = new Set<string>();
+const SCHEMA_REFUSED = /grammar is too large|union types|schema is too complex|too many (?:parameters|properties)/i;
+
 /** Claude through the Messages API with the user's own key. */
 export class AnthropicProvider extends BaseProvider {
   readonly kind = "ANTHROPIC" as const;
@@ -44,12 +50,26 @@ export class AnthropicProvider extends BaseProvider {
   protected async chat(call: ChatCall): Promise<ChatAnswer> {
     if (!this.client) throw AiError.config("Add your Anthropic API key in Settings → AI provider first.");
     const effort = call.effort && (await this.effortLevels(this.client, call.model, call.signal)).has(call.effort) ? call.effort : undefined;
-    const outputConfig = { ...(call.json ? { format: zodOutputFormat(call.json.schema) } : {}), ...(effort ? { effort } : {}) };
-    const message = await this.client.messages.create(
+    const schemaKey = call.json ? `${this.baseURL} ${call.model} ${call.json.name}` : "";
+    if (!call.json || promptSchemas.has(schemaKey)) return this.send(this.client, call, effort, false);
+    try {
+      return await this.send(this.client, call, effort, true);
+    } catch (error) {
+      if (!(error instanceof Anthropic.BadRequestError && SCHEMA_REFUSED.test(error.message))) throw error;
+      promptSchemas.add(schemaKey);
+      return this.send(this.client, call, effort, false);
+    }
+  }
+
+  /** One request; `structured` asks for `call.json` through output_config, otherwise the schema goes in the prompt. */
+  private async send(client: Anthropic, call: ChatCall, effort: Effort | undefined, structured: boolean): Promise<ChatAnswer> {
+    const format = call.json && structured ? { format: zodOutputFormat(call.json.schema) } : {};
+    const outputConfig = { ...format, ...(effort ? { effort } : {}) };
+    const message = await client.messages.create(
       {
         model: call.model,
         max_tokens: call.maxTokens + THINKING_TOKENS,
-        system: call.system,
+        system: call.json && !structured ? withJsonSchema(call.system, call.json.schema) : call.system,
         messages: call.messages,
         ...(Object.keys(outputConfig).length > 0 ? { output_config: outputConfig } : {}),
       },