diff --git a/.agents/skills/refresh-model-catalog/SKILL.md b/.agents/skills/refresh-model-catalog/SKILL.md index 25140ba..b5997a7 100644 --- a/.agents/skills/refresh-model-catalog/SKILL.md +++ b/.agents/skills/refresh-model-catalog/SKILL.md @@ -32,7 +32,7 @@ npm run sync:commandcode-catalog Regenerates `src/commandcode-catalog.ts` and bumps the documented CLI version in `README.md`. Review the diff; the catalog also lists reasoning models without selectable efforts. -Never add efforts to the generated file by hand. Manual effort policy for reasoning models that upstream ships without levels lives in `src/commandcode-catalog-overrides.ts` and is merged at load time. When the sync report lists a model from that file under "New effort metadata", remove its override; `tests/test-models.ts` fails until you do. +Never add efforts to the generated file by hand. Manual effort policy for reasoning models that upstream ships without levels lives in `src/commandcode-catalog-overrides.ts` and is merged at load time. The sync drops an override itself once upstream publishes its own levels. It parses only the `MODEL_EFFORT_OVERRIDES` object literal and preserves neighboring declarations. Keep shared rationale above the declaration; review entry-specific comments after pruning because they may describe removed entries. ### 3. Update display pricing (manual review) diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts index 543e8e0..5388cc4 100644 --- a/.github/scripts/check-commandcode-model-metadata.ts +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os" import { join, resolve } from "node:path" import { pathToFileURL } from "node:url" import { promisify } from "node:util" +import ts from "typescript" import { COMMAND_CODE_CLI_VERSION, @@ -40,6 +41,7 @@ function execNpmFileAsync( } const CATALOG_SOURCE_PATH = new URL("../../src/commandcode-catalog.ts", import.meta.url) const README_PATH = new URL("../../README.md", import.meta.url) +const OVERRIDES_SOURCE_PATH = new URL("../../src/commandcode-catalog-overrides.ts", import.meta.url) export interface CommandCodeModelMetadata { imageModelIds: readonly string[] @@ -379,6 +381,65 @@ function updateDocumentedCatalogVersion( return contents.replace(pattern, `command-code@${packageVersion}`) } +/** + * Drop manual effort overrides that upstream now publishes itself. + * + * `tests/test-models.ts` fails while an override duplicates upstream efforts, so + * leaving the removal to a human kept the scheduled workflow red and blocked its + * own pull request. The overrides file is hand-formatted, so this rewrites entries + * and leaves comments, ordering, and still-needed entries untouched. + */ +export function pruneObsoleteEffortOverrides( + contents: string, + upstreamEffortModelIds: readonly string[], +): { contents: string; removedModelIds: readonly string[] } { + const source = ts.createSourceFile("overrides.ts", contents, ts.ScriptTarget.Latest, true) + const upstreamModelIds = new Set(upstreamEffortModelIds) + for (const statement of source.statements) { + if (!ts.isVariableStatement(statement)) continue + for (const declaration of statement.declarationList.declarations) { + if (!ts.isIdentifier(declaration.name) || declaration.name.text !== "MODEL_EFFORT_OVERRIDES") + continue + const map = declaration.initializer + if (!map || !ts.isObjectLiteralExpression(map)) { + throw new Error("MODEL_EFFORT_OVERRIDES must be an object literal") + } + const obsolete = map.properties.filter( + (property) => + ts.isPropertyAssignment(property) && + ts.isStringLiteral(property.name) && + upstreamModelIds.has(property.name.text), + ) + const removedModelIds = obsolete.flatMap((property) => + ts.isPropertyAssignment(property) && ts.isStringLiteral(property.name) + ? [property.name.text] + : [], + ) + if (obsolete.length === 0) return { contents, removedModelIds } + if (obsolete.length === map.properties.length) { + return { + contents: contents.slice(0, map.getStart(source)) + "{}" + contents.slice(map.end), + removedModelIds: sorted(removedModelIds), + } + } + // Only remove syntax spans, preserving comments and all neighboring declarations. + let updated = contents + for (const property of [...obsolete].reverse()) { + const tokenStart = property.getStart(source) + const lineStart = contents.lastIndexOf("\n", tokenStart - 1) + 1 + const start = /^\s*$/.test(contents.slice(lineStart, tokenStart)) ? lineStart : tokenStart + const comma = /^\s*,/.exec(contents.slice(property.end, map.end)) + const tokenEnd = property.end + (comma?.[0].length ?? 0) + const newline = /^[ \t]*\r?\n/.exec(contents.slice(tokenEnd)) + const end = tokenEnd + (newline?.[0].length ?? 0) + updated = updated.slice(0, start) + updated.slice(end) + } + return { contents: updated, removedModelIds: sorted(removedModelIds) } + } + } + return { contents, removedModelIds: [] } +} + export function updateReadmeCatalogVersion(readme: string, packageVersion: string): string { return updateDocumentedCatalogVersion(readme, packageVersion, "README") } @@ -386,12 +447,19 @@ export function updateReadmeCatalogVersion(readme: string, packageVersion: strin async function writeSynchronizedCatalog( packageVersion: string, metadata: CommandCodeModelMetadata, -): Promise { +): Promise { const readme = await readFile(README_PATH, "utf-8") await Promise.all([ writeFile(CATALOG_SOURCE_PATH, renderCommandCodeCatalog(packageVersion, metadata), "utf-8"), writeFile(README_PATH, updateReadmeCatalogVersion(readme, packageVersion), "utf-8"), ]) + + const overrides = await readFile(OVERRIDES_SOURCE_PATH, "utf-8") + const pruned = pruneObsoleteEffortOverrides(overrides, Object.keys(metadata.reasoningEfforts)) + if (pruned.removedModelIds.length > 0) { + await writeFile(OVERRIDES_SOURCE_PATH, pruned.contents, "utf-8") + } + return pruned.removedModelIds } function metadataReport( @@ -510,8 +578,16 @@ async function main(): Promise { console.log(report) if (write) { - await writeSynchronizedCatalog(upstreamPackage.packageVersion, upstreamPackage.metadata) + const removedOverrides = await writeSynchronizedCatalog( + upstreamPackage.packageVersion, + upstreamPackage.metadata, + ) console.log(`Synchronized static metadata with command-code@${upstreamPackage.packageVersion}.`) + if (removedOverrides.length > 0) { + console.log( + `Removed ${removedOverrides.length} manual effort override(s) now published upstream: ${removedOverrides.join(", ")}.`, + ) + } return } diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index 88c9ca5..b9d39b1 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -6,10 +6,13 @@ on: paths: - ".github/scripts/check-commandcode-model-metadata.ts" - ".github/workflows/model-metadata.yml" + - "README.md" + - "src/commandcode-catalog-overrides.ts" - "src/commandcode-catalog.ts" - "src/core.ts" - "src/models.ts" - "tests/test-model-metadata-check.ts" + - "tests/test-models.ts" schedule: - cron: "17 6 * * *" workflow_dispatch: @@ -58,7 +61,7 @@ jobs: run: npm run sync:commandcode-catalog | tee commandcode-catalog-report.md - name: Format and verify synchronized files run: | - npm run format -- src/commandcode-catalog.ts README.md + npm run format -- src/commandcode-catalog.ts src/commandcode-catalog-overrides.ts README.md npm run typecheck npm run test:models npm run format:check @@ -87,4 +90,5 @@ jobs: assignees: patlux add-paths: | src/commandcode-catalog.ts + src/commandcode-catalog-overrides.ts README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 879148f..6010d8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +- Rebind a host's preselected built-in Command Code model to the extension's registered transport at session start, preserving configured endpoints and generate fallback on Oh My Pi. + +- Make the daily catalog sync self-healing: it now drops manual effort overrides that upstream has published itself, instead of leaving the removal to a human. The scheduled workflow previously failed on its own guard test, which skipped the pull-request step, so it could never propose the fix. The sync pull request also carries `src/commandcode-catalog-overrides.ts` now. +- Fix the pi end-to-end suite against pi 0.85 and newer, which streams Anthropic Messages through the SDK and appends `?beta=true` to `/v1/messages`. The mock matched the exact URL and answered 404, so every pull request failed while CI installs pi unpinned. + ## 0.6.4 - 2026-09-03 - Refresh the generated Command Code capability catalog from `command-code@1.40.1` to `command-code@1.44.0`, adding current image-input, reasoning, effort, and output-limit metadata for newly published models. diff --git a/README.md b/README.md index 909fdbc..0eb4693 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ Other extensions that stream with the active Command Code model, such as backgro ### Reasoning support -Reasoning capability and selectable effort levels follow the official CLI catalog independently. Models can therefore be marked as reasoning-capable even when Command Code chooses their depth automatically. Models with explicit effort support register a model-specific `thinkingLevelMap`, so pi and OMP expose only valid levels. For a few reasoning models the CLI catalog ships no effort levels although the endpoint accepts `reasoning_effort`; `src/commandcode-catalog-overrides.ts` adds a manual level set for those (currently `meta/muse-spark-1.1`, `meta/muse-spark-1.2`, and `meta/muse-spark-1.2-contributor`) on top of the generated catalog, and the tests fail once upstream publishes its own levels so the override gets removed. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. +Reasoning capability and selectable effort levels follow the official CLI catalog independently. Models can therefore be marked as reasoning-capable even when Command Code chooses their depth automatically. Models with explicit effort support register a model-specific `thinkingLevelMap`, so pi and OMP expose only valid levels. For a few reasoning models the CLI catalog ships no effort levels although the endpoint accepts `reasoning_effort`; `src/commandcode-catalog-overrides.ts` adds a manual level set for those (currently `meta/muse-spark-1.1`, `meta/muse-spark-1.2`, and `meta/muse-spark-1.2-contributor`) on top of the generated catalog. The catalog sync removes an override as soon as upstream publishes its own levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. List Command Code models from the terminal: @@ -139,7 +139,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.44.0`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions job synchronizes the CLI version, image capabilities, reasoning flags, reasoning efforts, and model-specific output limits with the latest published CLI package and opens or updates a reviewable pull request when they change. Pricing remains manually reviewed because temporary promotions and long-context tiers require explicit review. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.44.0`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions job synchronizes the CLI version, image capabilities, reasoning flags, reasoning efforts, and model-specific output limits with the latest published CLI package, also dropping manual effort overrides that upstream has published itself, and opens or updates a reviewable pull request when they change. Pricing remains manually reviewed because temporary promotions and long-context tiers require explicit review. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. diff --git a/index.ts b/index.ts index 3253d82..e7fb049 100644 --- a/index.ts +++ b/index.ts @@ -153,6 +153,15 @@ function legacyApiBase(providerApiBase: string): string { } export default async function (pi: ExtensionAPI) { + // Hosts may select their built-in Command Code model before loading extensions. + // Rebind that stale selection to our registered transport before the first turn. + pi.on("session_start", async (_event, ctx) => { + if (ctx.model?.provider !== "commandcode") return + const registered = ctx.modelRegistry.find("commandcode", ctx.model.id) + if (registered?.api === COMMAND_CODE_API && ctx.model.api !== COMMAND_CODE_API) { + await pi.setModel(registered) + } + }) const apiBase = process.env.COMMANDCODE_API_BASE ?? DEFAULT_PROVIDER_API_BASE const modelsUrl = process.env.COMMANDCODE_MODELS_URL ?? DEFAULT_MODELS_URL const modelsTimeoutMs = getModelsTimeoutMs() diff --git a/src/commandcode-catalog-overrides.ts b/src/commandcode-catalog-overrides.ts index cb4accd..3e92d75 100644 --- a/src/commandcode-catalog-overrides.ts +++ b/src/commandcode-catalog-overrides.ts @@ -6,11 +6,22 @@ import type { CommandCodeReasoningEffort } from "./commandcode-catalog.ts" * * `src/commandcode-catalog.ts` is generated from the CLI package and must stay * byte-identical to upstream so the daily drift check works. Entries here are - * merged over the generated catalog at load time and are not touched by - * `npm run sync:commandcode-catalog`. + * merged over the generated catalog at load time. `npm run sync:commandcode-catalog` + * deletes an entry as soon as upstream publishes its own levels, so nothing has to + * be removed by hand. + * + * An entry only takes effect for a model the generated catalog already marks as + * reasoning-capable: `src/core.ts` drops `reasoning_effort` when `model.reasoning` + * is false, and a model missing from `MODEL_REASONING` stays false. Upstream emits + * the flag whenever it emits efforts, so a sync that brings in new efforts brings + * the flag with it. + * + * Keep shared rationale above the declaration. The sync parses this object + * literal and preserves neighboring declarations; review entry-specific comments + * after pruning because they may describe removed entries. * * Add a model only when the effort parameter is known to be accepted by the - * Command Code endpoint; remove it once the CLI catalog ships its own efforts. + * Command Code endpoint. */ export const MODEL_EFFORT_OVERRIDES: Readonly< Record diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts index 97fe30e..00e9b47 100644 --- a/tests/test-model-metadata-check.ts +++ b/tests/test-model-metadata-check.ts @@ -9,11 +9,27 @@ import { parseKnownTextOnlyModelIds, parseModelsReference, parsePackageVersion, + pruneObsoleteEffortOverrides, renderCommandCodeCatalog, updateReadmeCatalogVersion, type CommandCodeModelMetadata, } from "../.github/scripts/check-commandcode-model-metadata.ts" +const OVERRIDES_SOURCE = `import type { CommandCodeReasoningEffort } from "./commandcode-catalog.ts" + +/** + * Manual reasoning-effort policy for models the official CLI marks as + * reasoning-capable without publishing selectable efforts. + */ +export const MODEL_EFFORT_OVERRIDES: Readonly< + Record +> = { + // Meta Muse Spark: the CLI ships no effort levels for these models. + "meta/muse-spark-1.1": ["minimal", "low", "medium", "high", "xhigh"], + "meta/muse-spark-1.2": ["minimal", "low", "medium", "high", "xhigh"], +} +` + const MODELS_REFERENCE = ` | Id (use EXACTLY this) | Name | Context | Efforts | $/1M in/out ยท cache read | Min plan | Best for | |---|---|---|---|---|---|---| @@ -167,4 +183,126 @@ export const MODEL_MAX_OUTPUT_TOKENS: Readonly> = { ) assert.throws(() => parseKnownTextOnlyModelIds("const unrelated = true"), /Could not find/) }) + + it("prunes only the overrides that upstream now publishes", () => { + const pruned = pruneObsoleteEffortOverrides(OVERRIDES_SOURCE, ["meta/muse-spark-1.1"]) + + assert.deepEqual(pruned.removedModelIds, ["meta/muse-spark-1.1"]) + assert.ok(!pruned.contents.includes("meta/muse-spark-1.1")) + assert.ok(pruned.contents.includes("meta/muse-spark-1.2")) + // Comments and the declaration must survive an entry removal. + assert.ok(pruned.contents.includes("Meta Muse Spark: the CLI ships no effort levels")) + assert.ok(pruned.contents.includes("export const MODEL_EFFORT_OVERRIDES")) + }) + + it("preserves neighboring declarations when pruning the effort map", () => { + const before = 'export const OTHER = {\n "meta/muse-spark-1.1": ["image"],\n}\n' + const after = "\nexport const AFTER = { nested: { value: true } }\n" + const source = before + OVERRIDES_SOURCE + after + const pruned = pruneObsoleteEffortOverrides(source, [ + "meta/muse-spark-1.1", + "meta/muse-spark-1.2", + ]) + assert.ok(pruned.contents.startsWith(before)) + assert.ok(pruned.contents.endsWith(after)) + assert.deepEqual(pruned.removedModelIds, ["meta/muse-spark-1.1", "meta/muse-spark-1.2"]) + assert.equal( + pruneObsoleteEffortOverrides(pruned.contents, ["meta/muse-spark-1.1"]).contents, + pruned.contents, + ) + }) + + it("ignores entries inside block comments", () => { + const source = OVERRIDES_SOURCE.replace( + ' "meta/muse-spark-1.1":', + ' /*\n "not/an-override": ["low"],\n */\n "meta/muse-spark-1.1":', + ) + assert.equal(pruneObsoleteEffortOverrides(source, ["not/an-override"]).contents, source) + }) + + it("leaves the overrides file untouched when nothing is obsolete", () => { + const pruned = pruneObsoleteEffortOverrides(OVERRIDES_SOURCE, ["some/other-model"]) + + assert.deepEqual(pruned.removedModelIds, []) + assert.equal(pruned.contents, OVERRIDES_SOURCE) + }) + + it("collapses the override map once every entry is obsolete", () => { + const pruned = pruneObsoleteEffortOverrides(OVERRIDES_SOURCE, [ + "meta/muse-spark-1.2", + "meta/muse-spark-1.1", + ]) + + assert.deepEqual(pruned.removedModelIds, ["meta/muse-spark-1.1", "meta/muse-spark-1.2"]) + // An empty map must render as `= {}` so the sync workflow's format check passes. + assert.ok(pruned.contents.includes("= {}")) + assert.ok(!pruned.contents.includes("\n}\n\n\n"), "no stray blank lines are left behind") + assert.ok(pruned.contents.endsWith("= {}\n")) + }) + + it("ignores commented-out override entries", () => { + const commented = OVERRIDES_SOURCE.replace( + ' "meta/muse-spark-1.1"', + ' // "meta/muse-spark-1.1"', + ) + const pruned = pruneObsoleteEffortOverrides(commented, ["meta/muse-spark-1.1"]) + + assert.deepEqual(pruned.removedModelIds, []) + assert.equal(pruned.contents, commented) + }) + + it("prunes entries written with single quotes or spaced colons", () => { + const styled = OVERRIDES_SOURCE.replace( + ' "meta/muse-spark-1.1": ["minimal", "low", "medium", "high", "xhigh"],', + " 'meta/muse-spark-1.1' : ['minimal', 'low', 'medium', 'high', 'xhigh'],", + ) + const pruned = pruneObsoleteEffortOverrides(styled, ["meta/muse-spark-1.1"]) + + assert.deepEqual(pruned.removedModelIds, ["meta/muse-spark-1.1"]) + assert.ok(!pruned.contents.includes("muse-spark-1.1")) + }) + + it("collapses the map even when a comment above it contains '= {'", () => { + const withComment = OVERRIDES_SOURCE.replace( + 'import type { CommandCodeReasoningEffort } from "./commandcode-catalog.ts"', + 'import type { CommandCodeReasoningEffort } from "./commandcode-catalog.ts"\n\n// Illustrative only: const other = { }', + ) + const pruned = pruneObsoleteEffortOverrides(withComment, [ + "meta/muse-spark-1.1", + "meta/muse-spark-1.2", + ]) + + assert.ok(pruned.contents.includes("const other = { }"), "the comment is preserved") + assert.ok(pruned.contents.includes("MODEL_EFFORT_OVERRIDES")) + assert.ok(pruned.contents.includes("= {}")) + }) + + it("removes a wrapped entry without leaving its array behind", () => { + // Prettier wraps an entry whose id exceeds the print width, so the effort + // array spans several lines. Only removing the first line would leave + // orphaned level lines and break the file's syntax. + const wrapped = `export const MODEL_EFFORT_OVERRIDES: Readonly< + Record +> = { + "vendor/a-very-long-model-identifier-that-exceeds-the-print-width": [ + "minimal", + "low", + ], + "short": ["low"], +} +` + const longId = "vendor/a-very-long-model-identifier-that-exceeds-the-print-width" + const pruned = pruneObsoleteEffortOverrides(wrapped, [longId]) + + assert.deepEqual(pruned.removedModelIds, [longId]) + assert.equal( + pruned.contents, + `export const MODEL_EFFORT_OVERRIDES: Readonly< + Record +> = { + "short": ["low"], +} +`, + ) + }) }) diff --git a/tests/test-models.ts b/tests/test-models.ts index 821a2e6..b221aa0 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -198,13 +198,14 @@ describe("commandCodeModelsFromApiResponse()", () => { it("merges manual effort overrides over the generated catalog", () => { const validEfforts = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]) - assert.ok(Object.keys(MODEL_EFFORT_OVERRIDES).length > 0) + // An empty override map is the healthy end state once upstream publishes every + // level, so asserting it is non-empty made that state unreachable. for (const [modelId, efforts] of Object.entries(MODEL_EFFORT_OVERRIDES)) { assert.equal(MODEL_REASONING[modelId], true, `${modelId} override needs a reasoning flag`) assert.equal( CATALOG_MODEL_EFFORTS[modelId], undefined, - `${modelId} now has upstream efforts; drop the manual override`, + `${modelId} now has upstream efforts; run npm run sync:commandcode-catalog to drop it`, ) assert.ok(efforts.length > 0) assert.ok(efforts.every((effort) => validEfforts.has(effort))) diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index aee3495..3c7813a 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -112,7 +112,12 @@ function modelCatalog() { } const server = createServer((req, res) => { - if (req.method === "GET" && req.url === "/provider/v1/models") { + // Match on the pathname only. Newer pi releases append a query string (for + // example `/provider/v1/messages?beta=true`) when the Anthropic SDK streams, + // and an exact URL comparison turned that into a 404 that failed every PR. + const pathname = new URL(req.url ?? "/", "http://localhost").pathname + + if (req.method === "GET" && pathname === "/provider/v1/models") { modelListRequestCount += 1 const respond = () => { if (res.destroyed) return @@ -124,8 +129,8 @@ const server = createServer((req, res) => { return } - const isOpenAIRequest = req.method === "POST" && req.url === "/provider/v1/chat/completions" - const isAnthropicRequest = req.method === "POST" && req.url === "/provider/v1/messages" + const isOpenAIRequest = req.method === "POST" && pathname === "/provider/v1/chat/completions" + const isAnthropicRequest = req.method === "POST" && pathname === "/provider/v1/messages" if (!isOpenAIRequest && !isAnthropicRequest) { res.writeHead(404) res.end("Not found")