From 6afe29fdf1c3fa5e4a13cef6f271a278f656adde Mon Sep 17 00:00:00 2001 From: robertn702 <8119609+robertn702@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:32:45 +0000 Subject: [PATCH 1/4] feat: add local Laya classifier --- package-lock.json | 245 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 3 + src/laya.ts | 157 +++++++++++++++++++++++++++++ test/laya.test.ts | 88 +++++++++++++++++ 4 files changed, 492 insertions(+), 1 deletion(-) create mode 100644 src/laya.ts create mode 100644 test/laya.test.ts diff --git a/package-lock.json b/package-lock.json index 126f2dd..07caedb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.6.2", "license": "MIT", "dependencies": { + "@receptron/laya": "^0.1.2", "@typesafe-ai/sdk": "^0.6.0" }, "bin": { @@ -23,6 +24,9 @@ }, "engines": { "node": ">=24.0.0 <25" + }, + "optionalDependencies": { + "@receptron/laya": "^0.1.2" } }, "node_modules/@ai-sdk/provider": { @@ -1047,6 +1051,13 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/@huggingface/tokenizers": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.2.0.tgz", + "integrity": "sha512-LidMHe1FpcSYH4vcSjXooda34pC0M8m1gmDOv7SQi6Iv+ib1zX7J3sHBCC9/3FPCkn2k1b55FneVqMRxrr99pg==", + "license": "Apache-2.0", + "optional": true + }, "node_modules/@isaacs/cliui": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", @@ -1818,6 +1829,20 @@ "node": ">=14" } }, + "node_modules/@receptron/laya": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@receptron/laya/-/laya-0.1.2.tgz", + "integrity": "sha512-cTqQC4DEY/wDAHY44EQhbW8wlrO7mZyLVWOy512783LUq55a7nLGGu16wXVi6oi4uNRfR0cLctKcQAxdXCXQfA==", + "license": "MIT", + "optional": true, + "dependencies": { + "@huggingface/tokenizers": "^0.2.0", + "onnxruntime-node": "^1.22.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@redis/bloom": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-6.2.1.tgz", @@ -2907,6 +2932,16 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/adm-zip": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.1.tgz", + "integrity": "sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -3293,6 +3328,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/define-lazy-prop": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", @@ -3306,6 +3359,24 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "license": "MIT", + "optional": true, + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -3361,6 +3432,26 @@ "node": ">=6" } }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-module-lexer": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", @@ -3410,6 +3501,19 @@ "@esbuild/win32-x64": "0.28.2" } }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -3641,6 +3745,39 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/global-agent": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", + "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", + "license": "BSD-3-Clause", + "optional": true, + "dependencies": { + "globalthis": "^1.0.2", + "matcher": "^4.0.0", + "semver": "^7.3.5", + "serialize-error": "^8.1.0" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/google-auth-library": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", @@ -3670,6 +3807,19 @@ "node": ">=18" } }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", @@ -3691,6 +3841,19 @@ "node": ">=18" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "license": "MIT", + "optional": true, + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/hosted-git-info": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-9.0.3.tgz", @@ -4303,6 +4466,22 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/matcher": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", + "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/mime": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/mime/-/mime-4.1.0.tgz", @@ -4781,6 +4960,16 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "license": "MIT", + "optional": true, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/obug": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/obug/-/obug-2.2.1.tgz", @@ -4795,6 +4984,31 @@ "node": ">=12.20.0" } }, + "node_modules/onnxruntime-common": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.30.0.tgz", + "integrity": "sha512-7fdVWjAID1dVhH/G8qK3APARunV4VkBFoCQAP7qp4Wkab0mrorvmc+sqiT+mKXOzDqdjN5j+/Z9nb4gzNPWcyA==", + "license": "MIT", + "optional": true + }, + "node_modules/onnxruntime-node": { + "version": "1.30.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.30.0.tgz", + "integrity": "sha512-twhs1C2C/BFkz1yc5OY0KIU2GUq6DURO7hD4bx5Q2Qy3nAMJwRXW8xU3NVczE29VA9lolLOYepoD8fjTGOfIqw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.6.0", + "global-agent": "^4.1.3", + "onnxruntime-common": "1.30.0" + } + }, "node_modules/open": { "version": "11.0.4", "resolved": "https://registry.npmjs.org/open/-/open-11.0.4.tgz", @@ -5249,7 +5463,7 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "dev": true, + "devOptional": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -5258,6 +5472,22 @@ "node": ">=10" } }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5618,6 +5848,19 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "license": "(MIT OR CC0-1.0)", + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", diff --git a/package.json b/package.json index 79a8cad..da63e47 100644 --- a/package.json +++ b/package.json @@ -53,5 +53,8 @@ "tsx": "^4.23.15", "typescript": "^7.0.2", "vitest": "^5.0.1" + }, + "optionalDependencies": { + "@receptron/laya": "^0.1.2" } } diff --git a/src/laya.ts b/src/laya.ts new file mode 100644 index 0000000..df78963 --- /dev/null +++ b/src/laya.ts @@ -0,0 +1,157 @@ +import type { LayaOptions } from "@receptron/laya"; + +import { classificationPolicy, ClassificationFailedError, type ClassificationPolicyOptions } from "./classification-policy.js"; +import { EffortCache } from "./effort-cache.js"; +import { supportsEffort, type Effort, type ModelProfile } from "./models.js"; +import { ClassificationCancelledError, type JevState } from "./jev.js"; +import type { EffortDecision, EffortSelector } from "./router.js"; +import { wireFor } from "./wire.js"; + +const DESCRIPTIONS: Record = { + none: "Mechanical work that does not benefit from reasoning.", + low: "Simple, mechanical, or well-understood work.", + medium: "Routine engineering work needing some reasoning.", + high: "Hard problems, debugging, or multi-step reasoning.", + xhigh: "Deeply complex or ambiguous work.", + max: "The hardest work where extra thinking clearly helps.", +}; + +type LayaAnswer = + | { type: "choice"; choice: unknown } + | { type: "score"; score: unknown } + | { type: "noul"; noul: unknown }; + +export interface LayaInstance { + systemOne(state: unknown, questions: Record): Promise; + close(): Promise; +} + +export interface LayaClassifierOptions extends ClassificationPolicyOptions { + timeoutMs: number; + cacheEntries?: number; + cacheTtlMs?: number; + modelDir?: string; + cacheDir?: string; + load?: () => Promise; +} + +export interface LayaClassifier { + select: EffortSelector; + close(): Promise; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function mapLayaAnswer(answer: unknown, model: ModelProfile): Effort | null { + if (!isRecord(answer) || typeof answer.type !== "string") return null; + const typed = answer as unknown as LayaAnswer; + if (typed.type === "choice") { + return supportsEffort(model, typed.choice) ? typed.choice : null; + } + let position: number; + if (typed.type === "score") { + if (typeof typed.score !== "number" || !Number.isFinite(typed.score)) return null; + position = typed.score; + } else if (typed.type === "noul") { + if (typeof typed.noul !== "number" || !Number.isFinite(typed.noul) || typed.noul < 0 || typed.noul > 1) return null; + position = typed.noul * (model.supportedEfforts.length - 1); + } else { + return null; + } + const index = Math.max(0, Math.min(model.supportedEfforts.length - 1, Math.round(position))); + return model.supportedEfforts[index] ?? null; +} + +function usableCacheKey(value: unknown): string | null { + return typeof value === "string" && value.trim().length > 0 ? value : null; +} + +async function defaultLoad(options: Pick): Promise { + const { Laya } = await import("@receptron/laya"); + const loadOptions: LayaOptions = { + ...(options.modelDir === undefined ? {} : { modelDir: options.modelDir }), + ...(options.cacheDir === undefined ? {} : { cacheDir: options.cacheDir }), + }; + return Laya.load(loadOptions); +} + +export function createLayaClassifier(options: LayaClassifierOptions): LayaClassifier { + if (!Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1) throw new Error("timeoutMs must be a positive integer"); + const policy = classificationPolicy(options); + const previousEfforts = new EffortCache(options.cacheEntries ?? 256, options.cacheTtlMs ?? 600_000); + let loaded: Promise | undefined; + const load = (): Promise => loaded ??= (options.load ? options.load() : defaultLoad(options)); + + const select: EffortSelector = async ({ body, signal, model, cacheScope, cacheKey: selectedCacheKey }) => { + if (signal.aborted) throw new ClassificationCancelledError(); + const startedAt = performance.now(); + const latency = (): number => Math.round(performance.now() - startedAt); + const key = selectedCacheKey !== undefined ? selectedCacheKey : usableCacheKey(body.prompt_cache_key); + const cacheKey = key === null ? null : JSON.stringify([cacheScope ?? "", model.id, key]); + const state: JevState & { model: string } = { ...wireFor(model.provider).jevState(body), model: model.id }; + const questions = { + effort: { + type: "score", + instructions: "Select the reasoning effort for the next model call.", + criteria: model.supportedEfforts.map((effort) => `${effort}: ${DESCRIPTIONS[effort]}`), + }, + } as const; + + let timer: ReturnType | undefined; + let cancelListener: (() => void) | undefined; + const timeout = new Promise<{ kind: "timeout" }>((resolve) => { + timer = setTimeout(() => resolve({ kind: "timeout" }), options.timeoutMs); + }); + const cancelled = new Promise<{ kind: "cancelled" }>((resolve) => { + cancelListener = () => resolve({ kind: "cancelled" }); + signal.addEventListener("abort", cancelListener, { once: true }); + }); + const inference = (async () => { + try { + const result = await (await load()).systemOne(state, questions); + return { kind: "result" as const, result }; + } catch { + return { kind: "error" as const }; + } + })(); + + try { + const outcome = await Promise.race([inference, timeout, cancelled]); + if (outcome.kind === "cancelled" || signal.aborted) throw new ClassificationCancelledError(); + const fallback = (code: "jev_timeout" | "jev_error" | "jev_invalid_output"): EffortDecision => { + if (policy.fallbackMode === "error") throw new ClassificationFailedError(code, 1, latency()); + const previous = policy.fallbackMode === "previous" && cacheKey ? previousEfforts.get(cacheKey) : undefined; + const fromPrevious = supportsEffort(model, previous); + return { + effort: fromPrevious ? previous : policy.fallbackEffort, + jevLatencyMs: latency(), + jevAttempts: 1, + fallbackSource: fromPrevious ? "previous" : "fixed", + fallback: code, + }; + }; + if (outcome.kind === "timeout") return fallback("jev_timeout"); + if (outcome.kind === "error") return { ...fallback("jev_error"), jevErrorCategory: "unknown" }; + const result = outcome.result; + const effort = isRecord(result) && isRecord(result.answers) ? mapLayaAnswer(result.answers.effort, model) : null; + if (effort === null) return fallback("jev_invalid_output"); + if (cacheKey !== null) previousEfforts.set(cacheKey, effort); + return { effort, jevLatencyMs: latency(), jevAttempts: 1, fallback: null }; + } finally { + if (timer !== undefined) clearTimeout(timer); + if (cancelListener !== undefined) signal.removeEventListener("abort", cancelListener); + } + }; + + return { + select, + async close() { + if (loaded !== undefined) { + const instance = await loaded.catch(() => undefined); + await instance?.close(); + } + }, + }; +} diff --git a/test/laya.test.ts b/test/laya.test.ts new file mode 100644 index 0000000..031d84d --- /dev/null +++ b/test/laya.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createLayaClassifier, mapLayaAnswer, type LayaInstance } from "../src/laya.js"; +import { findModel } from "../src/models.js"; + +const model = findModel("gpt-6-astra")!; +const body = { model: model.id, prompt_cache_key: "cache", input: [{ role: "user", content: "keep this local" }] }; +const args = (signal = new AbortController().signal) => ({ model, body, signal }); +const answer = (value: unknown) => ({ answers: { effort: value } }); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + return { promise, resolve, reject }; +} + +function instance(systemOne: LayaInstance["systemOne"]): LayaInstance { + return { systemOne, close: vi.fn(async () => undefined) }; +} + +describe("Laya effort mapping", () => { + it("maps choice, score, and noul typed answers to supported efforts", () => { + expect(mapLayaAnswer({ type: "choice", choice: "xhigh" }, model)).toBe("xhigh"); + expect(mapLayaAnswer({ type: "score", score: 2.6 }, model)).toBe("xhigh"); + expect(mapLayaAnswer({ type: "noul", noul: 0 }, model)).toBe("low"); + expect(mapLayaAnswer({ type: "noul", noul: 1 }, model)).toBe("max"); + expect(mapLayaAnswer({ type: "score", score: Number.NaN }, model)).toBeNull(); + expect(mapLayaAnswer({ type: "choice", choice: "none" }, model)).toBeNull(); + }); +}); + +describe("Laya classifier", () => { + it("loads once, classifies locally, and supports concurrent requests", async () => { + const firstRun = deferred(); + const secondRun = deferred(); + const pending = [firstRun, secondRun]; + const local = instance(vi.fn(() => pending.shift()!.promise)); + const load = vi.fn(async () => local); + const classifier = createLayaClassifier({ timeoutMs: 1_000, load }); + + const first = classifier.select(args()); + const second = classifier.select(args()); + await vi.waitFor(() => expect(local.systemOne).toHaveBeenCalledTimes(2)); + firstRun.resolve(answer({ type: "score", score: 0 })); + secondRun.resolve(answer({ type: "score", score: 4 })); + + await expect(Promise.all([first, second])).resolves.toMatchObject([ + { effort: "low", fallback: null }, + { effort: "max", fallback: null }, + ]); + expect(load).toHaveBeenCalledTimes(1); + expect(local.systemOne).toHaveBeenCalledWith(expect.objectContaining({ recent_user_text: "keep this local", model: model.id }), expect.objectContaining({ effort: expect.objectContaining({ type: "score" }) })); + }); + + it("falls back when initialization fails without retrying or leaking the error", async () => { + const load = vi.fn(async () => { throw new Error("private model path"); }); + const classifier = createLayaClassifier({ timeoutMs: 1_000, load, fallbackEffort: "high" }); + + await expect(classifier.select(args())).resolves.toMatchObject({ effort: "high", fallback: "jev_error", jevErrorCategory: "unknown" }); + expect(load).toHaveBeenCalledTimes(1); + }); + + it("times out a hanging local inference and ignores its late result", async () => { + const run = deferred(); + let calls = 0; + const local = instance(() => calls++ === 0 ? run.promise : new Promise(() => undefined)); + const classifier = createLayaClassifier({ timeoutMs: 20, load: async () => local, fallbackEffort: "medium" }); + + await expect(classifier.select(args())).resolves.toMatchObject({ effort: "medium", fallback: "jev_timeout" }); + run.resolve(answer({ type: "score", score: 4 })); + await new Promise((resolve) => setTimeout(resolve, 0)); + await expect(classifier.select(args())).resolves.toMatchObject({ effort: "medium", fallback: "jev_timeout" }); + }); + + it("cancels without fallback and closes a loaded model", async () => { + const run = deferred(); + const local = instance(() => run.promise); + const classifier = createLayaClassifier({ timeoutMs: 1_000, load: async () => local }); + const controller = new AbortController(); + const selected = classifier.select(args(controller.signal)); + controller.abort(); + + await expect(selected).rejects.toThrow("cancelled"); + await classifier.close(); + expect(local.close).toHaveBeenCalledTimes(1); + }); +}); From 3be9c3a81a78d66cfdf40d94ab78fd77e2e3bb6a Mon Sep 17 00:00:00 2001 From: robertn702 <8119609+robertn702@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:42:02 +0000 Subject: [PATCH 2/4] feat: select Laya in proxy and plugin --- src/classifier-backend.ts | 49 +++++++++++++++++++++++++++++++++ src/config.ts | 13 +++++++++ src/index.ts | 24 ++++++++++------ src/plugin-runtime.ts | 19 +++++++++---- test/classifier-backend.test.ts | 35 +++++++++++++++++++++++ test/environment.test.ts | 13 +++++++++ test/plugin-laya.test.ts | 46 +++++++++++++++++++++++++++++++ 7 files changed, 185 insertions(+), 14 deletions(-) create mode 100644 src/classifier-backend.ts create mode 100644 test/classifier-backend.test.ts create mode 100644 test/plugin-laya.test.ts diff --git a/src/classifier-backend.ts b/src/classifier-backend.ts new file mode 100644 index 0000000..78f802a --- /dev/null +++ b/src/classifier-backend.ts @@ -0,0 +1,49 @@ +import type { ClassificationPolicyOptions } from "./classification-policy.js"; +import type { JevConnection } from "./config.js"; +import { createJevClassifier, type JevClassifier, type JevClassifierOptions } from "./jev.js"; +import { createLayaClassifier, type LayaClassifier, type LayaInstance } from "./laya.js"; +import type { EffortSelector } from "./router.js"; + +interface CommonOptions extends ClassificationPolicyOptions { + timeoutMs: number; + cacheEntries?: number; + cacheTtlMs?: number; + createJev?: (options: JevClassifierOptions) => JevClassifier; +} + +type BackendOptions = CommonOptions & ( + | { backend: "jev"; jev: JevConnection } + | { backend: "laya"; modelDir?: string; cacheDir?: string; load?: () => Promise } +); + +export interface ClassifierBackend { + select: EffortSelector; + close(): Promise; +} + +export function createClassifierBackend(options: BackendOptions): ClassifierBackend { + if (options.backend === "jev") { + const classifier = (options.createJev ?? createJevClassifier)({ + ...options.jev, + timeoutMs: options.timeoutMs, + maxRetries: options.maxRetries, + fallbackMode: options.fallbackMode, + fallbackEffort: options.fallbackEffort, + cacheEntries: options.cacheEntries, + cacheTtlMs: options.cacheTtlMs, + }); + return { select: classifier.select, close: async () => undefined }; + } + const classifier: LayaClassifier = createLayaClassifier({ + timeoutMs: options.timeoutMs, + maxRetries: options.maxRetries, + fallbackMode: options.fallbackMode, + fallbackEffort: options.fallbackEffort, + cacheEntries: options.cacheEntries, + cacheTtlMs: options.cacheTtlMs, + modelDir: options.modelDir, + cacheDir: options.cacheDir, + load: options.load, + }); + return classifier; +} diff --git a/src/config.ts b/src/config.ts index 3773481..389df6f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,9 @@ const EFFORTS: readonly string[] = ["low", "medium", "high", "xhigh", "max"]; export interface AppConfig extends ClassificationPolicyOptions { port: number; + classifierBackend: "jev" | "laya"; + layaModelDir?: string; + layaCacheDir?: string; upstreamBaseUrl: string; upstreamAuth: UpstreamAuth; anthropicUpstream?: { baseUrl: string; auth: { policy: "forward" } | { policy: "key"; apiKey: string } }; @@ -92,6 +95,13 @@ function parseEffort(raw: string | undefined): Effort | undefined { export function loadConfig(env: Record): AppConfig { const jevPolicy = classificationPolicy({ maxRetries: env.JEV_ROUTER_MAX_RETRIES === undefined ? undefined : Number(env.JEV_ROUTER_MAX_RETRIES), fallbackMode: env.JEV_ROUTER_FALLBACK_MODE as ClassificationPolicyOptions["fallbackMode"], fallbackEffort: env.JEV_ROUTER_FALLBACK_EFFORT as Effort | undefined }); + const classifierBackend = env.JEV_ROUTER_CLASSIFIER_BACKEND ?? "jev"; + if (classifierBackend !== "jev" && classifierBackend !== "laya") { + throw new Error("JEV_ROUTER_CLASSIFIER_BACKEND must be jev or laya"); + } + for (const name of ["JEV_ROUTER_LAYA_MODEL_DIR", "JEV_ROUTER_LAYA_CACHE_DIR"] as const) { + if (env[name] !== undefined && !env[name]?.trim()) throw new Error(`${name} must not be empty`); + } for (const name of ["UPSTREAM_MODEL", "UPSTREAM_MODELS", "ALLOWED_MODELS"]) { if (env[name] !== undefined) throw new Error(`${name} is unsupported; select a registered model through request.model`); } @@ -171,6 +181,9 @@ export function loadConfig(env: Record): AppConfig { return { ...jevPolicy, port: parsePort(env.JEV_ROUTER_PORT), + classifierBackend, + layaModelDir: env.JEV_ROUTER_LAYA_MODEL_DIR, + layaCacheDir: env.JEV_ROUTER_LAYA_CACHE_DIR, upstreamBaseUrl, anthropicUpstream, upstreamAuth: policy === "bearer" diff --git a/src/index.ts b/src/index.ts index 728299d..9bf7723 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,20 +2,23 @@ import { existsSync } from "node:fs"; import { connect } from "node:net"; import { loadConfig, loadJevConnection, upstreamHostname } from "./config.js"; +import { createClassifierBackend } from "./classifier-backend.js"; import { createDecisionLogger } from "./decision-log.js"; import { formatEvidence } from "./evidence.js"; -import { createJevClassifier } from "./jev.js"; import { createAppServer, shutdownAppServer } from "./server.js"; if (process.argv.includes("--help") || process.argv.includes("-h")) { console.log(`Usage: opencode-jev-router [--help] -Start the local Jev-powered Responses API proxy. +Start the local adaptive-effort Responses API proxy. Environment: - JEV_API_KEY Required Jev classifier key (separate from upstream/client keys) + JEV_ROUTER_CLASSIFIER_BACKEND jev (default) or laya + JEV_API_KEY Required only for the Jev backend (separate from upstream/client keys) JEV_BASE_URL Jev API root (default: https://api.typesafe.ai) Vercel: https://ai-gateway.vercel.sh/typesafe + JEV_ROUTER_LAYA_MODEL_DIR Optional local ONNX bundle directory + JEV_ROUTER_LAYA_CACHE_DIR Optional Laya download cache directory JEV_ROUTER_PORT Listening port (default: 4320) JEV_ROUTER_UPSTREAM_BASE_URL Required Responses API-compatible base URL, e.g. https://api.openai.com/v1 JEV_ROUTER_UPSTREAM_AUTH forward (default, loopback only) or bearer @@ -48,24 +51,26 @@ if (existsSync(".env")) { } let config: ReturnType; -let jev: ReturnType; +let jev: ReturnType | undefined; try { config = loadConfig(process.env); - jev = loadJevConnection(process.env); + jev = config.classifierBackend === "jev" ? loadJevConnection(process.env) : undefined; } catch (error) { console.error(JSON.stringify({ event: "startup_failed", reason: "invalid_configuration", message: error instanceof Error ? error.message : "invalid configuration" })); process.exit(1); } -const classifier = createJevClassifier({ - ...jev, +const classifierOptions = { timeoutMs: config.jevTimeoutMs, maxRetries: config.maxRetries, fallbackMode: config.fallbackMode, fallbackEffort: config.fallbackEffort, cacheEntries: config.effortCacheEntries, cacheTtlMs: config.effortCacheTtlMs, -}); +}; +const classifier = config.classifierBackend === "jev" + ? createClassifierBackend({ ...classifierOptions, backend: "jev", jev: jev! }) + : createClassifierBackend({ ...classifierOptions, backend: "laya", modelDir: config.layaModelDir, cacheDir: config.layaCacheDir }); const logDecision = config.decisionsLogPath ? createDecisionLogger(config.decisionsLogPath) : undefined; const server = createAppServer({ @@ -104,7 +109,8 @@ const stop = (): void => { console.log(JSON.stringify({ event: "shutdown_started" })); void shutdownAppServer(server, config.shutdownGraceMs, () => { console.log(JSON.stringify({ event: "shutdown_deadline" })); - }).then(() => { + }).then(async () => { + await classifier.close(); console.log(JSON.stringify({ event: "shutdown_complete" })); }, () => { console.error(JSON.stringify({ event: "shutdown_failed" })); diff --git a/src/plugin-runtime.ts b/src/plugin-runtime.ts index adfaf49..3a9dc36 100644 --- a/src/plugin-runtime.ts +++ b/src/plugin-runtime.ts @@ -1,9 +1,9 @@ import { createHash } from "node:crypto"; import { resolveJevConnection, upstreamHostname } from "./config.js"; +import { createClassifierBackend, type ClassifierBackend } from "./classifier-backend.js"; import { createDecisionLogger } from "./decision-log.js"; import { buildPluginUpstreamRequestHeaders, pickFetchResponseHeaders } from "./headers.js"; -import { createJevClassifier } from "./jev.js"; import type { ClassificationPolicyOptions } from "./classification-policy.js"; import { MODELS, supportsEffort, type Effort, type Provider } from "./models.js"; import { ResponsesRouter, type PreparedRequest } from "./router.js"; @@ -12,7 +12,7 @@ import { resolveModel, validateRequest, wireFor, UnsupportedInputError } from ". import { anthropicVersion, mergeAnthropicBeta } from "./wire-anthropic.js"; /** Options for the OpenCode V2 plugin runtime. */ -export type PluginOptions = ClassificationPolicyOptions & { jevTimeoutMs?: number; jevApiKey?: string; jevBaseUrl?: string; jevModel?: string; baseEffort?: Effort; fixedEffort?: Effort; maxRequestBytes?: number; maxInFlight?: number; upstreamHeaderTimeoutMs?: number; upstreamIdleTimeoutMs?: number; decisionsLogPath?: string }; +export type PluginOptions = ClassificationPolicyOptions & { classifierBackend?: "jev" | "laya"; layaModelDir?: string; layaCacheDir?: string; jevTimeoutMs?: number; jevApiKey?: string; jevBaseUrl?: string; jevModel?: string; baseEffort?: Effort; fixedEffort?: Effort; maxRequestBytes?: number; maxInFlight?: number; upstreamHeaderTimeoutMs?: number; upstreamIdleTimeoutMs?: number; decisionsLogPath?: string }; /** A local rejection with an HTTP status for the plugin request hook. */ export class PluginRequestError extends Error { @@ -93,7 +93,11 @@ async function boundedBody(request: Request, maxBytes: number, signal: AbortSign /** Per-plugin-instance state, validation, Jev selection, rewrite, and usage observation. */ export function createPluginRuntime(options: PluginOptions): PluginRuntime { const env = process.env; - const connection = options.fixedEffort === undefined ? resolveJevConnection(options.jevApiKey ?? env.JEV_API_KEY ?? "", options.jevBaseUrl ?? env.JEV_BASE_URL) : undefined; + const backend = options.classifierBackend ?? "jev"; + if (backend !== "jev" && backend !== "laya") throw new Error("classifierBackend must be jev or laya"); + if (backend === "laya" && (options.jevApiKey !== undefined || options.jevBaseUrl !== undefined || options.jevModel !== undefined)) throw new Error("Jev options cannot be used with classifierBackend=laya"); + if (backend === "jev" && (options.layaModelDir !== undefined || options.layaCacheDir !== undefined)) throw new Error("Laya options require classifierBackend=laya"); + const connection = options.fixedEffort === undefined && backend === "jev" ? resolveJevConnection(options.jevApiKey ?? env.JEV_API_KEY ?? "", options.jevBaseUrl ?? env.JEV_BASE_URL) : undefined; if (options.jevModel !== undefined && options.jevModel !== connection?.model) throw new Error("jevModel must match the configured Jev endpoint"); if (options.baseEffort !== undefined && !["low", "medium", "high", "xhigh", "max"].includes(options.baseEffort)) throw new Error("baseEffort is unsupported"); if (options.fixedEffort !== undefined && !MODELS.every((model) => supportsEffort(model, options.fixedEffort))) throw new Error("fixedEffort must be supported by every model"); @@ -101,7 +105,12 @@ export function createPluginRuntime(options: PluginOptions): PluginRuntime { const maxInFlight = positive(options.maxInFlight, 32, "maxInFlight"); const headerTimeoutMs = positive(options.upstreamHeaderTimeoutMs, 10_000, "upstreamHeaderTimeoutMs"); const idleTimeoutMs = positive(options.upstreamIdleTimeoutMs, 60_000, "upstreamIdleTimeoutMs"); - const selectEffort = connection ? createJevClassifier({ ...connection, maxRetries: options.maxRetries, fallbackMode: options.fallbackMode, fallbackEffort: options.fallbackEffort, timeoutMs: options.jevTimeoutMs ?? 4_000 }).select : async () => ({ effort: options.fixedEffort!, jevLatencyMs: 0, fallback: null }); + let classifier: ClassifierBackend | undefined; + const selectEffort = options.fixedEffort !== undefined + ? async () => ({ effort: options.fixedEffort!, jevLatencyMs: 0, fallback: null }) + : (classifier = backend === "jev" + ? createClassifierBackend({ backend, jev: connection!, maxRetries: options.maxRetries, fallbackMode: options.fallbackMode, fallbackEffort: options.fallbackEffort, timeoutMs: options.jevTimeoutMs ?? 4_000 }) + : createClassifierBackend({ backend, modelDir: options.layaModelDir, cacheDir: options.layaCacheDir, maxRetries: options.maxRetries, fallbackMode: options.fallbackMode, fallbackEffort: options.fallbackEffort, timeoutMs: options.jevTimeoutMs ?? 4_000 })).select; const onEvidence = options.decisionsLogPath === undefined ? undefined : createDecisionLogger(options.decisionsLogPath); const router = new ResponsesRouter({ baseEffort: options.baseEffort, selectEffort, onEvidence }); // Each open exchange's controller maps to its incoming request. Undici follows a @@ -209,6 +218,6 @@ export function createPluginRuntime(options: PluginOptions): PluginRuntime { return { start, - dispose() { disposed = true; for (const controller of controllers.keys()) controller.abort(); for (const abandon of [...abandons]) abandon(); router.reset(); }, + dispose() { disposed = true; for (const controller of controllers.keys()) controller.abort(); for (const abandon of [...abandons]) abandon(); router.reset(); void classifier?.close(); }, }; } diff --git a/test/classifier-backend.test.ts b/test/classifier-backend.test.ts new file mode 100644 index 0000000..20f39b9 --- /dev/null +++ b/test/classifier-backend.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it, vi } from "vitest"; + +import { createClassifierBackend } from "../src/classifier-backend.js"; +import type { LayaInstance } from "../src/laya.js"; + +const common = { timeoutMs: 1_000, maxRetries: 0, fallbackMode: "fixed" as const, fallbackEffort: "high" as const }; + +describe("classifier backend selection", () => { + it("keeps hosted Jev as the default backend implementation", () => { + const hosted = { select: vi.fn(), client: {} }; + const createJev = vi.fn(() => hosted as never); + const backend = createClassifierBackend({ + ...common, + backend: "jev", + jev: { apiKey: "key", baseURL: "https://api.typesafe.ai", model: "jev-latest" }, + createJev, + }); + + expect(createJev).toHaveBeenCalledOnce(); + expect(backend.select).toBe(hosted.select); + }); + + it("does not construct or transmit to TypeSafe/Jev in local mode", async () => { + const createJev = vi.fn(() => { throw new Error("hosted backend must stay unused"); }); + const local: LayaInstance = { + systemOne: vi.fn(async () => ({ answers: { effort: { type: "score", score: 0 } } })), + close: vi.fn(async () => undefined), + }; + const backend = createClassifierBackend({ ...common, backend: "laya", load: async () => local, createJev }); + + expect(createJev).not.toHaveBeenCalled(); + await backend.close(); + expect(local.close).not.toHaveBeenCalled(); + }); +}); diff --git a/test/environment.test.ts b/test/environment.test.ts index 2aa2d7d..c1e63d0 100644 --- a/test/environment.test.ts +++ b/test/environment.test.ts @@ -1,6 +1,8 @@ import { it, expect } from "vitest"; import { loadConfig, loadJevConnection } from "../src/config.js"; +const upstream = { JEV_ROUTER_UPSTREAM_BASE_URL: "http://127.0.0.1:8080/v1" }; + it("requires the Jev credential names and ignores old router settings", () => { expect(() => loadJevConnection({ JEV_ROUTER_API_KEY: "old" })).toThrow("JEV_API_KEY"); expect(() => loadJevConnection({ JEV_API_KEY: "key", JEV_ROUTER_BASE_URL: "https://api.typesafe.ai" })).toThrow("JEV_BASE_URL"); @@ -11,3 +13,14 @@ it("keeps credential and endpoint selection explicit", () => { expect(loadJevConnection({ JEV_API_KEY: "gateway", JEV_BASE_URL: "https://ai-gateway.vercel.sh/typesafe" }).model).toBe("typesafe-ai/jev"); expect(loadConfig({ JEV_ROUTER_UPSTREAM_BASE_URL: "http://127.0.0.1:8080/v1", JEV_ROUTER_PORT: "4321", JEV_ROUTER_CLASSIFICATION_TIMEOUT_MS: "10000" })).toMatchObject({ port: 4321, jevTimeoutMs: 10000 }); }); + +it("selects the hosted backend by default and Laya only when explicit", () => { + expect(loadConfig(upstream)).toMatchObject({ classifierBackend: "jev", layaModelDir: undefined, layaCacheDir: undefined }); + expect(loadConfig({ + ...upstream, + JEV_ROUTER_CLASSIFIER_BACKEND: "laya", + JEV_ROUTER_LAYA_MODEL_DIR: "/models/laya", + JEV_ROUTER_LAYA_CACHE_DIR: "/cache/laya", + })).toMatchObject({ classifierBackend: "laya", layaModelDir: "/models/laya", layaCacheDir: "/cache/laya" }); + expect(() => loadConfig({ ...upstream, JEV_ROUTER_CLASSIFIER_BACKEND: "local" })).toThrow("JEV_ROUTER_CLASSIFIER_BACKEND"); +}); diff --git a/test/plugin-laya.test.ts b/test/plugin-laya.test.ts new file mode 100644 index 0000000..225c057 --- /dev/null +++ b/test/plugin-laya.test.ts @@ -0,0 +1,46 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const laya = vi.hoisted(() => { + const close = vi.fn(async () => undefined); + const systemOne = vi.fn(async () => ({ answers: { effort: { type: "score", score: 0 } } })); + const load = vi.fn(async () => ({ systemOne, close })); + return { close, systemOne, load }; +}); +vi.mock("@receptron/laya", () => ({ Laya: { load: laya.load } })); + +import { createPluginRuntime } from "../src/plugin-runtime.js"; + +const originalKey = process.env.JEV_API_KEY; +afterEach(() => { + if (originalKey === undefined) delete process.env.JEV_API_KEY; + else process.env.JEV_API_KEY = originalKey; +}); + +describe("plugin classifier backend selection", () => { + it("allows explicit local Laya mode without a hosted classifier key", () => { + delete process.env.JEV_API_KEY; + const runtime = createPluginRuntime({ classifierBackend: "laya" }); + runtime.dispose(); + }); + + it("rejects unknown classifier backends", () => { + expect(() => createPluginRuntime({ classifierBackend: "local" as "laya" })).toThrow("classifierBackend"); + }); + + it("routes one request through Laya without a hosted classifier request and closes the model", async () => { + delete process.env.JEV_API_KEY; + const runtime = createPluginRuntime({ classifierBackend: "laya", jevTimeoutMs: 1_000 }); + const request = new Request("https://api.openai.com/v1/responses", { + method: "POST", + headers: { authorization: "Bearer upstream-only" }, + body: JSON.stringify({ model: "gpt-6-astra", input: [{ role: "user", content: "private prompt" }] }), + }); + + const exchange = await runtime.start(request, { session: null, turnId: null }, "openai"); + expect(laya.load).toHaveBeenCalledOnce(); + expect(laya.systemOne).toHaveBeenCalledWith(expect.objectContaining({ recent_user_text: "private prompt" }), expect.anything()); + expect(JSON.parse(exchange.body).input).toContainEqual(expect.objectContaining({ reasoning: { effort: "low" } })); + runtime.dispose(); + await vi.waitFor(() => expect(laya.close).toHaveBeenCalledOnce()); + }); +}); From 5e67ff37cbd0fae4b42cc2f21c8f7f049a99e634 Mon Sep 17 00:00:00 2001 From: robertn702 <8119609+robertn702@users.noreply.github.com> Date: Wed, 30 Sep 2026 11:47:28 +0000 Subject: [PATCH 3/4] docs: document local Laya setup --- .env.example | 7 +++++ CHANGELOG.md | 6 +++++ README.md | 50 +++++++++++++++++++++++++---------- docs/behavior.md | 10 ++++--- docs/classification-policy.md | 14 +++++++--- docs/environment.md | 14 +++++++--- examples/opencode.jsonc | 4 +++ 7 files changed, 80 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index f8fa07a..b66c489 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,14 @@ JEV_ROUTER_PORT=4320 +# Hosted Jev is the default classification backend. +# JEV_ROUTER_CLASSIFIER_BACKEND=jev # TypeSafe key by default; Gateway requires its credential AND the URL below. JEV_API_KEY= # JEV_BASE_URL=https://ai-gateway.vercel.sh/typesafe +# For local classification, set the backend to laya and omit both Jev variables. +# The first request downloads ~1.7 GB to ~/.cache/receptron-laya and loads ~2 GB RAM. +# JEV_ROUTER_CLASSIFIER_BACKEND=laya +# JEV_ROUTER_LAYA_CACHE_DIR=/var/cache/receptron-laya +# JEV_ROUTER_LAYA_MODEL_DIR=/models/laya # Required: any Responses API-compatible base URL that serves GPT-6 models. JEV_ROUTER_UPSTREAM_BASE_URL=https://api.openai.com/v1 # bearer: the router sends JEV_ROUTER_UPSTREAM_API_KEY (HTTPS or loopback). diff --git a/CHANGELOG.md b/CHANGELOG.md index 5df60e4..a8322bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- Add optional local Laya classification for the plugin and standalone proxy, + with explicit backend selection, shared timeout/fallback behavior, local-only + conversation state, lazy model loading, and documented cache/resource needs. + ## 0.6.2 - Add GPT-6.1 Sol (`gpt-6.1-sol`) to the OpenAI profiles, with `low` through `max` efforts (`none` is not supported). diff --git a/README.md b/README.md index 3f6eec1..1a3c0ca 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,10 @@ may differ. [See the evaluation](eval/results/router-consolidated-2026-09-25.md) For each primary request to a wrapped model, the plugin: -1. Sends a bounded summary of the recent conversation to Jev, which picks a - reasoning effort (for example `low` for a rename, `high` for a failing test). +1. Sends a bounded summary of the recent conversation to the configured + classifier. Hosted Jev is the default; optional local Laya keeps the summary + on the machine. The classifier picks a reasoning effort (for example `low` + for a rename, `high` for a failing test). 2. Adds that choice to the request as an OpenAI [`configuration_update`](https://developers.openai.com/api/docs/guides/reasoning#change-reasoning-mid-conversation) item, or as an Anthropic effort-only system message; earlier updates stay in @@ -36,12 +38,15 @@ If Jev is slow or unavailable, the request continues at a fallback effort - **OpenCode V2 2.0.4 or newer** (tested with 2.0.4 and 2.0.18). V1 users can pin `@robertn702/opencode-jev-router@0.5` or use the standalone proxy. -- **A Jev key**, from either: - - [TypeSafe](https://typesafe.ai/) (direct), or - - [Vercel AI Gateway](https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe) - (a Gateway key, used with Vercel's TypeSafe-compatible endpoint). - - Each routed request makes one classification call, billed by that provider. +- **A classification backend:** + - Hosted `jev` is the default. Use a [TypeSafe](https://typesafe.ai/) key, or a + [Vercel AI Gateway](https://vercel.com/docs/ai-gateway/sdks-and-apis/typesafe) + key with Vercel's TypeSafe-compatible endpoint. Each routed request makes + one classification call, billed by that provider. + - Local `laya` needs no classifier key and sends no conversation excerpts to + TypeSafe or Vercel. It downloads about 1.7 GB of ONNX weights on first use, + caches them under `~/.cache/receptron-laya` by default, and needs roughly + 2 GB of RAM for the loaded model plus a few hundred MB per question batch. - **A Responses API-compatible endpoint** that serves GPT-6 Astra, Luna, or Sol and accepts `configuration_update` input items, plus its API key. The OpenAI API (`https://api.openai.com/v1`) works; so does any gateway that exposes the @@ -82,6 +87,11 @@ for all projects, or `opencode.json` in a project root: - **Using a direct TypeSafe key?** Delete the `jevBaseUrl` line. A key only works with its own endpoint. +- **Using local Laya instead?** Replace `jevApiKey` and `jevBaseUrl` with + `"classifierBackend": "laya"`. The first classified request downloads and + loads the model, so it can take several minutes. Set `layaCacheDir` to move + the cache or `layaModelDir` to use an already downloaded ONNX bundle. Laya + runs in this Node.js process; plan for the disk and memory listed above. - **Source auth:** `OPENAI_API_KEY` or an API key saved with `opencode auth login` supplies the wrapped OpenAI model. If your OpenCode catalog lacks the model (for example, offline or with 2.0.4's built-ins), declare it under @@ -186,10 +196,13 @@ host nevertheless selects websocket, rather than silently bypassing Jev. ## What is sent where -- **To Jev (TypeSafe or Vercel):** bounded excerpts of recent user and assistant - text, up to 8 recent tool results (with tool names and error flags), a short - failure summary, and the model ID. Hosted-tool and computer-use payloads are - not sent. +- **To hosted Jev (TypeSafe or Vercel, `classifierBackend: "jev"`):** bounded + excerpts of recent user and assistant text, up to 8 recent tool results (with + tool names and error flags), a short failure summary, and the model ID. + Hosted-tool and computer-use payloads are not sent. +- **To local Laya (`classifierBackend: "laya"`):** the same bounded classifier + state stays in-process. No classifier request is sent to TypeSafe, Vercel, or + another hosted Jev endpoint. - **To your endpoint:** the full OpenCode request, with the effort update added. - **Stored locally:** nothing by default. With `decisionsLogPath`, metadata only (IDs, model, effort, latency, token counts). Prompts, tool output, @@ -201,8 +214,11 @@ Plugin options go in `options` of a `plugins` entry. | Option | Default | Purpose | | --- | --- | --- | -| `jevApiKey` | `JEV_API_KEY` env | Jev classifier key. Required. | +| `classifierBackend` | `jev` | `jev` for hosted TypeSafe-compatible classification or `laya` for local ONNX inference. | +| `jevApiKey` | `JEV_API_KEY` env | Hosted Jev classifier key. Required only for `jev`. | | `jevBaseUrl` | `https://api.typesafe.ai` | Set to `https://ai-gateway.vercel.sh/typesafe` for a Vercel key. No other values are accepted. | +| `layaModelDir` | download/cache | Existing Laya ONNX bundle; skips download when set. | +| `layaCacheDir` | `~/.cache/receptron-laya` | First-run model download cache. | | `wrap` | none | Required nonempty object with `openai` and/or `anthropic` arrays of `provider/model` source refs. | | `decisionsLogPath` | off | Absolute path for metadata-only `JevDecision` JSONL. | | `baseEffort` | `medium` | Request-level effort reported by responses. | @@ -231,7 +247,8 @@ instead appears as `Model unavailable`. | Symptom | Fix | | --- | --- | | `fallback` is `jev_error` with `jev_error_category: "http_auth"` | The Jev key doesn't match the endpoint. Vercel keys need `jevBaseUrl`; TypeSafe keys must omit it. | -| `fallback` is `jev_timeout` | Jev was slow or unreachable; requests still ran at the fallback effort. Raise `jevTimeoutMs` if it happens often. | +| `fallback` is `jev_timeout` | The selected classifier was slow; requests still ran at the fallback effort. Raise `jevTimeoutMs` if it happens often. Laya's first load can exceed a short timeout while its model continues loading. | +| Laya first request falls back or uses substantial disk/RAM | Allow the initial ~1.7 GB download and model load to finish, verify cache permissions/free space, and budget roughly 2 GB RAM plus batch overhead. Reuse the cache on later starts. | | OpenCode log says `failed to load plugin` | Check `wrap` syntax and remove old plugin upstream options. | | 401/403/404 from the model | The endpoint's response is passed through unchanged. Check the source provider key and model. | | Local 400 mentioning `reasoning.mode`, `truncation`, or the model | The request uses an unsupported mode or model; see [Models](#models). | @@ -253,8 +270,13 @@ needs Node.js 24.x. Create a `.env` in the directory you'll run from (or export the variables): ```dotenv +# Hosted default: JEV_API_KEY=your-jev-key # JEV_BASE_URL=https://ai-gateway.vercel.sh/typesafe # Vercel keys only +# Or local classification (omit JEV_API_KEY/JEV_BASE_URL): +# JEV_ROUTER_CLASSIFIER_BACKEND=laya +# JEV_ROUTER_LAYA_CACHE_DIR=/var/cache/receptron-laya +# JEV_ROUTER_LAYA_MODEL_DIR=/models/laya # existing bundle; skips download JEV_ROUTER_UPSTREAM_BASE_URL=https://api.openai.com/v1 JEV_ROUTER_UPSTREAM_AUTH=bearer JEV_ROUTER_UPSTREAM_API_KEY=your-endpoint-key diff --git a/docs/behavior.md b/docs/behavior.md index fe3d9b7..a25a5c7 100644 --- a/docs/behavior.md +++ b/docs/behavior.md @@ -228,10 +228,12 @@ listening, configured, not draining, and the upstream TCP port is reachable. Otherwise it returns `503 {"status":"not_ready","reason":"..."}` with one of `starting`, `missing_configuration`, `draining`, or `dependency_unavailable`. The upstream probe is bounded to 500 ms and cached for two seconds; it sends no -model or Jev requests. The CLI validates configuration (including the required -Jev key) before listening, so missing configuration normally prevents startup -rather than serving an endpoint. The TCP check verifies connectivity, not -upstream authentication or model availability. +model or classifier requests. The CLI validates configuration (including the +required Jev key when `JEV_ROUTER_CLASSIFIER_BACKEND=jev`) before listening, so +missing configuration normally prevents startup rather than serving an +endpoint. Laya model download and initialization are lazy on the first +classified request. The TCP check verifies connectivity, not upstream +authentication, classifier availability, or model availability. For a container orchestrator, use `/health` for liveness and `/ready` for readiness, for example: diff --git a/docs/classification-policy.md b/docs/classification-policy.md index d30fabc..09ef353 100644 --- a/docs/classification-policy.md +++ b/docs/classification-policy.md @@ -1,12 +1,18 @@ # Classification retries and fallback -Normal use defaults to one additional retry, then fixed **high** effort. -The shared classifier retries connection failures, SDK timeouts, HTTP 429 and -HTTP 5xx. Authentication errors, other 4xx responses, and invalid classifier -output are not retried. SDK-level retries remain disabled to avoid multiplying +Hosted Jev defaults to one additional retry, then fixed **high** effort. The +hosted classifier retries connection failures, SDK timeouts, HTTP 429 and HTTP +5xx. Authentication errors, other 4xx responses, and invalid classifier output +are not retried. SDK-level retries remain disabled to avoid multiplying attempts. Exponential backoff with jitter starts at approximately 200 ms; Retry-After is respected within the total classification deadline. +Local Laya performs one in-process inference attempt. Model-load failure, +inference failure, invalid typed output, and timeout use the same fallback modes +and decision metadata as hosted Jev. Client cancellation never falls back. +Because the ONNX API cannot interrupt an inference already running, a timed-out +or cancelled call's late result is ignored. + | Plugin option | CLI environment | Default | | --- | --- | --- | | `maxRetries` | `JEV_ROUTER_MAX_RETRIES` | 1 (0–10 additional attempts) | diff --git a/docs/environment.md b/docs/environment.md index 9277c1b..96831ff 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -1,9 +1,17 @@ # Router environment namespace Use `JEV_ROUTER_*` variables for router configuration and `JEV_API_KEY` / -`JEV_BASE_URL` for the Jev classifier; `.env.example` lists the supported -names. `JEV_ROUTER_API_KEY` and `JEV_ROUTER_BASE_URL` are rejected. This beta migration -is breaking: old environment names are no longer supported. +`JEV_BASE_URL` for the default hosted Jev classifier; `.env.example` lists the +supported names. `JEV_ROUTER_API_KEY` and `JEV_ROUTER_BASE_URL` are rejected. +This beta migration is breaking: old environment names are no longer supported. + +`JEV_ROUTER_CLASSIFIER_BACKEND` is explicit and accepts `jev` (the default) or +`laya`. The local Laya backend does not read or send a Jev credential. Optional +`JEV_ROUTER_LAYA_MODEL_DIR` points to an existing ONNX bundle; +`JEV_ROUTER_LAYA_CACHE_DIR` changes the download cache. Without a model directory, +the first classified request downloads about 1.7 GB and caches it under +`~/.cache/receptron-laya`; the loaded model needs roughly 2 GB RAM plus a few +hundred MB per batch. `JEV_API_KEY` is a TypeSafe credential when the base URL is omitted (default `https://api.typesafe.ai`). A Vercel Gateway credential also requires diff --git a/examples/opencode.jsonc b/examples/opencode.jsonc index 75212f5..ac1a374 100644 --- a/examples/opencode.jsonc +++ b/examples/opencode.jsonc @@ -5,6 +5,10 @@ // Jev classifier key. For a direct TypeSafe key, delete jevBaseUrl. "jevApiKey": "{env:JEV_API_KEY}", "jevBaseUrl": "https://ai-gateway.vercel.sh/typesafe", + // Local alternative: remove both Jev lines and use this instead. + // "classifierBackend": "laya", + // Optional: "layaCacheDir": "/var/cache/receptron-laya", + // Optional existing bundle: "layaModelDir": "/models/laya", // Wrap existing OpenCode models with API-key credentials. "wrap": { "openai": ["openai/gpt-6-astra"] }, // Add "anthropic": ["anthropic/claude-opus-5-5"] for Messages. From 0d915f2e86b9f7daccce79be3f82c00676117550 Mon Sep 17 00:00:00 2001 From: robertn702 <8119609+robertn702@users.noreply.github.com> Date: Wed, 30 Sep 2026 12:03:26 +0000 Subject: [PATCH 4/4] fix: harden Laya lifecycle and retries --- src/index.ts | 2 +- src/laya.ts | 48 +++++++++++++++++---- src/plugin-runtime.ts | 2 +- test/classifier-backend.test.ts | 10 ++++- test/laya.test.ts | 76 ++++++++++++++++++++++++++++++--- 5 files changed, 121 insertions(+), 17 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9bf7723..a4f1f6d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -112,7 +112,7 @@ const stop = (): void => { }).then(async () => { await classifier.close(); console.log(JSON.stringify({ event: "shutdown_complete" })); - }, () => { + }).catch(() => { console.error(JSON.stringify({ event: "shutdown_failed" })); process.exitCode = 1; }); diff --git a/src/laya.ts b/src/laya.ts index df78963..76c09a2 100644 --- a/src/laya.ts +++ b/src/laya.ts @@ -52,7 +52,7 @@ export function mapLayaAnswer(answer: unknown, model: ModelProfile): Effort | nu } let position: number; if (typed.type === "score") { - if (typeof typed.score !== "number" || !Number.isFinite(typed.score)) return null; + if (typeof typed.score !== "number" || !Number.isFinite(typed.score) || typed.score < 0 || typed.score > model.supportedEfforts.length - 1) return null; position = typed.score; } else if (typed.type === "noul") { if (typeof typed.noul !== "number" || !Number.isFinite(typed.noul) || typed.noul < 0 || typed.noul > 1) return null; @@ -82,9 +82,22 @@ export function createLayaClassifier(options: LayaClassifierOptions): LayaClassi const policy = classificationPolicy(options); const previousEfforts = new EffortCache(options.cacheEntries ?? 256, options.cacheTtlMs ?? 600_000); let loaded: Promise | undefined; - const load = (): Promise => loaded ??= (options.load ? options.load() : defaultLoad(options)); + let closing: Promise | undefined; + let closed = false; + const active = new Set>(); + const load = (): Promise => { + if (closed) return Promise.reject(new Error("Laya classifier is closed")); + if (loaded === undefined) { + loaded = Promise.resolve().then(() => options.load ? options.load() : defaultLoad(options)).catch((error: unknown) => { + loaded = undefined; + throw error; + }); + } + return loaded; + }; const select: EffortSelector = async ({ body, signal, model, cacheScope, cacheKey: selectedCacheKey }) => { + if (closed) throw new Error("Laya classifier is closed"); if (signal.aborted) throw new ClassificationCancelledError(); const startedAt = performance.now(); const latency = (): number => Math.round(performance.now() - startedAt); @@ -100,9 +113,10 @@ export function createLayaClassifier(options: LayaClassifierOptions): LayaClassi } as const; let timer: ReturnType | undefined; + let expired = false; let cancelListener: (() => void) | undefined; const timeout = new Promise<{ kind: "timeout" }>((resolve) => { - timer = setTimeout(() => resolve({ kind: "timeout" }), options.timeoutMs); + timer = setTimeout(() => { expired = true; resolve({ kind: "timeout" }); }, options.timeoutMs); }); const cancelled = new Promise<{ kind: "cancelled" }>((resolve) => { cancelListener = () => resolve({ kind: "cancelled" }); @@ -110,8 +124,17 @@ export function createLayaClassifier(options: LayaClassifierOptions): LayaClassi }); const inference = (async () => { try { - const result = await (await load()).systemOne(state, questions); - return { kind: "result" as const, result }; + const instance = await load(); + // ONNX inference cannot be interrupted once started. Never launch it for + // a request that expired, disconnected, or was disposed during loading. + if (expired || signal.aborted || closed) return { kind: "stale" as const }; + const run = instance.systemOne(state, questions); + active.add(run); + try { + return { kind: "result" as const, result: await run }; + } finally { + active.delete(run); + } } catch { return { kind: "error" as const }; } @@ -132,7 +155,8 @@ export function createLayaClassifier(options: LayaClassifierOptions): LayaClassi fallback: code, }; }; - if (outcome.kind === "timeout") return fallback("jev_timeout"); + if (outcome.kind === "timeout" || expired) return fallback("jev_timeout"); + if (outcome.kind === "stale") throw new ClassificationCancelledError(); if (outcome.kind === "error") return { ...fallback("jev_error"), jevErrorCategory: "unknown" }; const result = outcome.result; const effort = isRecord(result) && isRecord(result.answers) ? mapLayaAnswer(result.answers.effort, model) : null; @@ -148,10 +172,16 @@ export function createLayaClassifier(options: LayaClassifierOptions): LayaClassi return { select, async close() { - if (loaded !== undefined) { - const instance = await loaded.catch(() => undefined); + if (closing !== undefined) return closing; + closed = true; + closing = (async () => { + // Loading is shared but inference is not abortable. Let it settle, then + // close the model only after every in-flight ONNX call has finished. + const instance = await loaded?.catch(() => undefined); + await Promise.allSettled([...active]); await instance?.close(); - } + })(); + return closing; }, }; } diff --git a/src/plugin-runtime.ts b/src/plugin-runtime.ts index 3a9dc36..0e56da8 100644 --- a/src/plugin-runtime.ts +++ b/src/plugin-runtime.ts @@ -218,6 +218,6 @@ export function createPluginRuntime(options: PluginOptions): PluginRuntime { return { start, - dispose() { disposed = true; for (const controller of controllers.keys()) controller.abort(); for (const abandon of [...abandons]) abandon(); router.reset(); void classifier?.close(); }, + dispose() { disposed = true; for (const controller of controllers.keys()) controller.abort(); for (const abandon of [...abandons]) abandon(); router.reset(); void classifier?.close().catch(() => undefined); }, }; } diff --git a/test/classifier-backend.test.ts b/test/classifier-backend.test.ts index 20f39b9..fc1d5cf 100644 --- a/test/classifier-backend.test.ts +++ b/test/classifier-backend.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest"; import { createClassifierBackend } from "../src/classifier-backend.js"; import type { LayaInstance } from "../src/laya.js"; +import { findModel } from "../src/models.js"; const common = { timeoutMs: 1_000, maxRetries: 0, fallbackMode: "fixed" as const, fallbackEffort: "high" as const }; +const model = findModel("gpt-6-astra")!; describe("classifier backend selection", () => { it("keeps hosted Jev as the default backend implementation", () => { @@ -28,8 +30,14 @@ describe("classifier backend selection", () => { }; const backend = createClassifierBackend({ ...common, backend: "laya", load: async () => local, createJev }); + await expect(backend.select({ + model, + body: { model: model.id, input: [{ role: "user", content: "local only" }] }, + signal: new AbortController().signal, + })).resolves.toMatchObject({ effort: "low", fallback: null }); + expect(local.systemOne).toHaveBeenCalledWith(expect.objectContaining({ recent_user_text: "local only" }), expect.anything()); expect(createJev).not.toHaveBeenCalled(); await backend.close(); - expect(local.close).not.toHaveBeenCalled(); + expect(local.close).toHaveBeenCalledOnce(); }); }); diff --git a/test/laya.test.ts b/test/laya.test.ts index 031d84d..239b2f2 100644 --- a/test/laya.test.ts +++ b/test/laya.test.ts @@ -26,6 +26,8 @@ describe("Laya effort mapping", () => { expect(mapLayaAnswer({ type: "noul", noul: 0 }, model)).toBe("low"); expect(mapLayaAnswer({ type: "noul", noul: 1 }, model)).toBe("max"); expect(mapLayaAnswer({ type: "score", score: Number.NaN }, model)).toBeNull(); + expect(mapLayaAnswer({ type: "score", score: -1 }, model)).toBeNull(); + expect(mapLayaAnswer({ type: "score", score: 5 }, model)).toBeNull(); expect(mapLayaAnswer({ type: "choice", choice: "none" }, model)).toBeNull(); }); }); @@ -53,12 +55,16 @@ describe("Laya classifier", () => { expect(local.systemOne).toHaveBeenCalledWith(expect.objectContaining({ recent_user_text: "keep this local", model: model.id }), expect.objectContaining({ effort: expect.objectContaining({ type: "score" }) })); }); - it("falls back when initialization fails without retrying or leaking the error", async () => { - const load = vi.fn(async () => { throw new Error("private model path"); }); + it("falls back when initialization fails and retries loading on the next request", async () => { + const local = instance(async () => answer({ type: "score", score: 2 })); + const load = vi.fn() + .mockRejectedValueOnce(new Error("private model path")) + .mockResolvedValue(local); const classifier = createLayaClassifier({ timeoutMs: 1_000, load, fallbackEffort: "high" }); await expect(classifier.select(args())).resolves.toMatchObject({ effort: "high", fallback: "jev_error", jevErrorCategory: "unknown" }); - expect(load).toHaveBeenCalledTimes(1); + await expect(classifier.select(args())).resolves.toMatchObject({ effort: "high", fallback: null }); + expect(load).toHaveBeenCalledTimes(2); }); it("times out a hanging local inference and ignores its late result", async () => { @@ -73,16 +79,76 @@ describe("Laya classifier", () => { await expect(classifier.select(args())).resolves.toMatchObject({ effort: "medium", fallback: "jev_timeout" }); }); - it("cancels without fallback and closes a loaded model", async () => { + it("does not start inference after timeout during lazy model loading", async () => { + const loading = deferred(); + const local = instance(vi.fn(async () => answer({ type: "score", score: 0 }))); + const classifier = createLayaClassifier({ timeoutMs: 15, load: () => loading.promise, fallbackEffort: "high" }); + await expect(classifier.select(args())).resolves.toMatchObject({ fallback: "jev_timeout" }); + loading.resolve(local); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(local.systemOne).not.toHaveBeenCalled(); + await classifier.close(); + }); + + it("does not start inference after cancellation during lazy model loading", async () => { + const loading = deferred(); + const local = instance(vi.fn(async () => answer({ type: "score", score: 0 }))); + const classifier = createLayaClassifier({ timeoutMs: 1_000, load: () => loading.promise }); + const controller = new AbortController(); + const selected = classifier.select(args(controller.signal)); + controller.abort(); + await expect(selected).rejects.toThrow("cancelled"); + loading.resolve(local); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(local.systemOne).not.toHaveBeenCalled(); + await classifier.close(); + }); + + it("recovers from a transient load failure on the next request", async () => { + const local = instance(async () => answer({ type: "score", score: 0 })); + const load = vi.fn().mockRejectedValueOnce(new Error("temporary disk problem")).mockResolvedValue(local); + const classifier = createLayaClassifier({ timeoutMs: 1_000, load }); + await expect(classifier.select(args())).resolves.toMatchObject({ fallback: "jev_error" }); + await expect(classifier.select(args())).resolves.toMatchObject({ effort: "low", fallback: null }); + expect(load).toHaveBeenCalledTimes(2); + await classifier.close(); + }); + + it("uses previous effort on failure only for the matching cache context", async () => { + let fails = false; + const local = instance(async () => fails ? answer({ type: "choice", choice: "bad" }) : answer({ type: "score", score: 0 })); + const classifier = createLayaClassifier({ timeoutMs: 1_000, load: async () => local, fallbackMode: "previous", fallbackEffort: "high" }); + await expect(classifier.select({ ...args(), cacheScope: "tenant-a" })).resolves.toMatchObject({ effort: "low", fallback: null }); + fails = true; + await expect(classifier.select({ ...args(), cacheScope: "tenant-a" })).resolves.toMatchObject({ effort: "low", fallback: "jev_invalid_output", fallbackSource: "previous" }); + await expect(classifier.select({ ...args(), cacheScope: "tenant-b" })).resolves.toMatchObject({ effort: "high", fallback: "jev_invalid_output", fallbackSource: "fixed" }); + await classifier.close(); + }); + + it("fails closed on classifier errors when fallbackMode is error", async () => { + const classifier = createLayaClassifier({ timeoutMs: 1_000, load: async () => { throw new Error("private path"); }, fallbackMode: "error" }); + await expect(classifier.select(args())).rejects.toThrow("jev_classification_failed"); + await classifier.close(); + }); + + it("cancels without fallback and waits for an active run before closing", async () => { const run = deferred(); - const local = instance(() => run.promise); + const local = instance(vi.fn(() => run.promise)); const classifier = createLayaClassifier({ timeoutMs: 1_000, load: async () => local }); const controller = new AbortController(); const selected = classifier.select(args(controller.signal)); + await vi.waitFor(() => expect(local.systemOne).toHaveBeenCalledOnce()); controller.abort(); await expect(selected).rejects.toThrow("cancelled"); + const closing = classifier.close(); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(local.close).not.toHaveBeenCalled(); + run.resolve(answer({ type: "score", score: 0 })); + await closing; + expect(local.close).toHaveBeenCalledTimes(1); await classifier.close(); expect(local.close).toHaveBeenCalledTimes(1); + await expect(classifier.select(args())).rejects.toThrow("closed"); }); });