diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f5c7da2..1aa7a3ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,5 +31,11 @@ jobs: - name: Run TypeScript build run: bun run ts:build + - name: Type-check SDK tests + run: bunx tsc -p packages/sdk/tsconfig.test.json --noEmit + - name: Run tests run: bun run test + + - name: Run release tooling tests + run: bun run test:release diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ad1fb33a..b7ccf651 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,6 +12,10 @@ permissions: id-token: write contents: read +concurrency: + group: publish-${{ github.repository }} + cancel-in-progress: false + jobs: publish: runs-on: ubuntu-latest @@ -21,6 +25,9 @@ jobs: with: fetch-depth: 0 + - name: Verify release ancestry + run: git merge-base --is-ancestor "${{ github.sha }}" origin/main + - uses: actions/setup-node@v4 with: node-version: '24' @@ -45,8 +52,22 @@ jobs: - name: TypeScript build run: bun run ts:build + - name: Type-check SDK tests + run: bunx tsc -p packages/sdk/tsconfig.test.json --noEmit + + - name: Run tests + run: bun run test + + - name: Run release tooling tests + run: bun run test:release + - name: Prepare package manifests - run: node ./scripts/npm-publish/prepare-packages.mjs "${{ github.ref_name }}" + run: node ./scripts/npm-publish/prepare-packages.mjs "$RELEASE_VERSION" + env: + RELEASE_VERSION: ${{ github.ref_name }} + + - name: Pack and validate published artifacts + run: node ./scripts/npm-publish/pack-and-validate.mjs - name: Publish NPM run: bash ./scripts/npm-publish/run.sh diff --git a/.gitignore b/.gitignore index 331586b4..8d8e9d55 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,4 @@ buzola.gen.ts # jj .jj/ .claude/agent-canvas/ +.smellhunt/ diff --git a/bun.lock b/bun.lock index 314bd741..902cb255 100644 --- a/bun.lock +++ b/bun.lock @@ -43,7 +43,6 @@ "zustand": "5.0.11", }, "devDependencies": { - "@roj-ai/sdk": "workspace:*", "@types/react": "19.2.14", "@types/react-dom": "19.2.3", "react": "19.2.4", @@ -51,9 +50,9 @@ "typescript": "5.9.3", }, "peerDependencies": { - "lucide-react": "*", - "react": "19.2.4", - "react-dom": "19.2.4", + "lucide-react": ">=0.400.0 <1", + "react": "^19.0.0", + "react-dom": "^19.0.0", }, }, "packages/debug": { @@ -61,12 +60,12 @@ "version": "0.1.0", "dependencies": { "@roj-ai/client": "workspace:*", + "@roj-ai/sdk": "workspace:*", "@roj-ai/shared": "workspace:*", "tokenx": "1.3.0", "zustand": "5.0.11", }, "devDependencies": { - "@roj-ai/sdk": "workspace:*", "@types/react": "19.2.14", "@types/react-dom": "19.2.3", "react": "19.2.4", @@ -74,9 +73,9 @@ "typescript": "5.9.3", }, "peerDependencies": { - "lucide-react": "*", - "react": "19.2.4", - "react-dom": "19.2.4", + "lucide-react": ">=0.400.0 <1", + "react": "^19.0.0", + "react-dom": "^19.0.0", }, }, "packages/demo": { @@ -106,9 +105,10 @@ "name": "@roj-ai/platform-cli", "version": "0.1.0", "bin": { - "roj": "./src/main.ts", + "roj": "./dist/main.js", }, "dependencies": { + "@roj-ai/client": "workspace:*", "@roj-ai/sandbox-runtime": "workspace:*", }, "devDependencies": { @@ -135,18 +135,15 @@ "name": "@roj-ai/sdk", "version": "0.1.0", "dependencies": { - "@hono/zod-validator": "0.7.6", "@roj-ai/transport": "workspace:*", "hono": "4.13.1", "ignore": "7.0.5", "tokenx": "^1.3.0", "uuidv7": "^1.1.0", - "ws": "^8.18.0", "zod": "4.3.6", }, "devDependencies": { "@types/bun": "latest", - "@types/ws": "^8.5.10", "typescript": "^5.7.2", }, }, @@ -154,10 +151,10 @@ "name": "@roj-ai/shared", "version": "0.1.0", "dependencies": { + "@roj-ai/sdk": "workspace:*", "zod": "catalog:libs", }, "devDependencies": { - "@roj-ai/sdk": "workspace:*", "typescript": "^5.7.2", }, }, @@ -165,14 +162,14 @@ "name": "@roj-ai/standalone-server", "version": "0.1.0", "bin": { - "roj-standalone": "./src/main.ts", + "roj-standalone": "./dist/main.js", }, "dependencies": { "@roj-ai/client": "workspace:*", "@roj-ai/sdk": "workspace:*", - "@roj-ai/shared": "workspace:*", "@roj-ai/transport": "workspace:*", "hono": "catalog:libs", + "zod": "catalog:libs", }, "devDependencies": { "@types/bun": "catalog:", @@ -313,8 +310,6 @@ "@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], - "@hono/zod-validator": ["@hono/zod-validator@0.7.6", "", { "peerDependencies": { "hono": ">=3.9.0", "zod": "^3.25.0 || ^4.0.0" } }, "sha512-Io1B6d011Gj1KknV4rXYz4le5+5EubcWEU/speUjuw9XMMIaP3n78yXLhjd2A3PXaXaUwEAluOiAyLqhBEJgsw=="], - "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], "@jridgewell/remapping": ["@jridgewell/remapping@2.3.5", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ=="], @@ -453,8 +448,6 @@ "@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="], - "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], - "@vitejs/plugin-react": ["@vitejs/plugin-react@4.7.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/plugin-transform-react-jsx-self": "^7.27.1", "@babel/plugin-transform-react-jsx-source": "^7.27.1", "@rolldown/pluginutils": "1.0.0-beta.27", "@types/babel__core": "^7.20.5", "react-refresh": "^0.17.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" } }, "sha512-gUu9hwfWvvEDBBmgtAowQCojwZmJ5mcLn3aufeCsitijs3+f2NsrPtlAWIR6OPiqljl96GVCUbLe0HyqIpVaoA=="], "anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="], @@ -651,8 +644,6 @@ "vite-tsconfig-paths": ["vite-tsconfig-paths@5.1.4", "", { "dependencies": { "debug": "^4.1.1", "globrex": "^0.1.2", "tsconfck": "^3.0.3" }, "peerDependencies": { "vite": "*" }, "optionalPeers": ["vite"] }, "sha512-cYj0LRuLV2c2sMqhqhGpaO3LretdtMn/BVX4cPLanIZuwwrkVl+lK84E/miEXkCHWXuq65rhNN4rXsBcOB3S4w=="], - "ws": ["ws@8.18.0", "", { "peerDependencies": { "bufferutil": "^4.0.1", "utf-8-validate": ">=5.0.2" }, "optionalPeers": ["bufferutil", "utf-8-validate"] }, "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw=="], - "yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="], "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], @@ -661,13 +652,13 @@ "@roj-ai/demo/@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], - "@roj-ai/platform-cli/@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + "@roj-ai/platform-cli/@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@roj-ai/sandbox-runtime/@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@roj-ai/sdk/@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], - "@roj-ai/standalone-server/@types/bun": ["@types/bun@1.3.13", "", { "dependencies": { "bun-types": "1.3.13" } }, "sha512-9fqXWk5YIHGGnUau9TEi+qdlTYDAnOj+xLCmSTwXfAIqXr2x4tytJb43E9uCvt09zJURKXwAtkoH4nLQfzeTXw=="], + "@roj-ai/standalone-server/@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], "@roj-ai/transport/@types/bun": ["@types/bun@1.3.14", "", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], @@ -693,13 +684,13 @@ "@roj-ai/demo/@types/bun/bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], - "@roj-ai/platform-cli/@types/bun/bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + "@roj-ai/platform-cli/@types/bun/bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "@roj-ai/sandbox-runtime/@types/bun/bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "@roj-ai/sdk/@types/bun/bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], - "@roj-ai/standalone-server/@types/bun/bun-types": ["bun-types@1.3.13", "", { "dependencies": { "@types/node": "*" } }, "sha512-QXKeHLlOLqQX9LgYaHJfzdBaV21T63HhFJnvuRCcjZiaUDpbs5ED1MgxbMra71CsryN/1dAoXuJJJwIv/2drVA=="], + "@roj-ai/standalone-server/@types/bun/bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], "@roj-ai/transport/@types/bun/bun-types": ["bun-types@1.3.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], } diff --git a/package.json b/package.json index b626cbb7..390ab2dc 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "ts:watch": "tsc --build --watch", "lint": "biome lint .", "lint:fix": "biome lint . --write", - "test": "bun test packages/*/src packages/*/tests" + "test": "bun test packages/*/src packages/*/tests", + "test:release": "bun test scripts/npm-publish" }, "devDependencies": { "@biomejs/biome": "catalog:build", diff --git a/packages/cli/package.json b/packages/cli/package.json index c4480e8e..f541c2ab 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -3,6 +3,9 @@ "version": "0.1.0", "type": "module", "main": "./dist/main.js", + "bin": { + "roj-cli": "./dist/main.js" + }, "files": [ "src", "dist" diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index b0264197..d0f6ed88 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -1,3 +1,4 @@ +#!/usr/bin/env bun import { SessionId } from '@roj-ai/shared' import { RpcError } from '@roj-ai/shared/rpc' import { createCliClient } from './client.js' diff --git a/packages/client-react/package.json b/packages/client-react/package.json index 1864cf86..98f08bee 100644 --- a/packages/client-react/package.json +++ b/packages/client-react/package.json @@ -48,7 +48,6 @@ "zustand": "5.0.11" }, "devDependencies": { - "@roj-ai/sdk": "workspace:*", "@types/react": "19.2.14", "@types/react-dom": "19.2.3", "react": "19.2.4", @@ -56,9 +55,9 @@ "typescript": "5.9.3" }, "peerDependencies": { - "lucide-react": "*", - "react": "19.2.4", - "react-dom": "19.2.4" + "lucide-react": ">=0.400.0 <1", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "types": "./dist/index.d.ts" } diff --git a/packages/client/src/platform/index.ts b/packages/client/src/platform/index.ts index 4612482b..b097a26f 100644 --- a/packages/client/src/platform/index.ts +++ b/packages/client/src/platform/index.ts @@ -33,5 +33,10 @@ export type { BuildPreviewUrlOptions, BuildWsUrlOptions } from './urls.js' export { createRojClient } from './rest-client.js' export type { RojClient, RojClientOptions, SessionRpcInput } from './rest-client.js' +// File upload wire contract — shared with @roj-ai/platform-cli, which posts to +// the same /api/v1/files/upload endpoint against a server outside this repo. +export { postFile, sha256Hex } from './rest-client.js' +export type { PostFileArgs, PostFileResult } from './rest-client.js' + // Errors export { RojApiError } from './errors.js' diff --git a/packages/client/src/platform/rest-client.ts b/packages/client/src/platform/rest-client.ts index 943264ee..770e4f6e 100644 --- a/packages/client/src/platform/rest-client.ts +++ b/packages/client/src/platform/rest-client.ts @@ -240,7 +240,7 @@ export function createRojClient(options: RojClientOptions): RojClient { } } -interface PostFileArgs { +export interface PostFileArgs { url: string apiKey: string contentHash: string @@ -249,7 +249,7 @@ interface PostFileArgs { body?: Blob } -interface PostFileResult { +export interface PostFileResult { status: number body: { ok?: boolean @@ -263,7 +263,7 @@ interface PostFileResult { } | null } -async function postFile(args: PostFileArgs): Promise { +export async function postFile(args: PostFileArgs): Promise { const formData = new FormData() formData.append('contentHash', args.contentHash) formData.append('filename', args.filename) @@ -280,7 +280,7 @@ async function postFile(args: PostFileArgs): Promise { return { status: response.status, body } } -async function sha256Hex(buf: ArrayBuffer): Promise { +export async function sha256Hex(buf: ArrayBuffer): Promise { const digest = await crypto.subtle.digest('SHA-256', buf) const bytes = new Uint8Array(digest) let hex = '' diff --git a/packages/debug/package.json b/packages/debug/package.json index 659d177e..1a57a76a 100644 --- a/packages/debug/package.json +++ b/packages/debug/package.json @@ -31,12 +31,12 @@ }, "dependencies": { "@roj-ai/client": "workspace:*", + "@roj-ai/sdk": "workspace:*", "@roj-ai/shared": "workspace:*", "tokenx": "1.3.0", "zustand": "5.0.11" }, "devDependencies": { - "@roj-ai/sdk": "workspace:*", "@types/react": "19.2.14", "@types/react-dom": "19.2.3", "react": "19.2.4", @@ -44,9 +44,9 @@ "typescript": "5.9.3" }, "peerDependencies": { - "lucide-react": "*", - "react": "19.2.4", - "react-dom": "19.2.4" + "lucide-react": ">=0.400.0 <1", + "react": "^19.0.0", + "react-dom": "^19.0.0" }, "types": "./dist/index.d.ts" } diff --git a/packages/debug/src/components/debug/ResumeAgentButton.tsx b/packages/debug/src/components/debug/ResumeAgentButton.tsx new file mode 100644 index 00000000..2e93e13a --- /dev/null +++ b/packages/debug/src/components/debug/ResumeAgentButton.tsx @@ -0,0 +1,48 @@ +import { api, unwrap } from "@roj-ai/client"; +import { AgentId } from "@roj-ai/shared"; +import { type FormEvent, useCallback, useState } from "react"; + +/** + * Resume a paused agent. Shared by DashboardPage and AgentDetailPage, which + * each carried a byte-identical copy. + */ +export function ResumeAgentButton({ + sessionId, + agentId, +}: { + sessionId: string; + agentId: string; +}) { + const [resuming, setResuming] = useState(false); + + const handleResume = useCallback( + async (e: FormEvent) => { + e.stopPropagation(); + setResuming(true); + try { + unwrap( + await api.call("agents.resume", { + sessionId, + agentId: AgentId(agentId), + }), + ); + } catch { + // Error is visible via state change (or lack thereof) + } finally { + setResuming(false); + } + }, + [sessionId, agentId], + ); + + return ( + + ); +} diff --git a/packages/debug/src/components/debug/pages/AgentDetailPage.tsx b/packages/debug/src/components/debug/pages/AgentDetailPage.tsx index d2a13e58..b94f07aa 100644 --- a/packages/debug/src/components/debug/pages/AgentDetailPage.tsx +++ b/packages/debug/src/components/debug/pages/AgentDetailPage.tsx @@ -4,6 +4,7 @@ import type { ToolCallView, ToolConversationMessageView, } from "@roj-ai/shared"; +import { ResumeAgentButton } from "../ResumeAgentButton.js"; import { AgentId } from "@roj-ai/shared"; import { type FormEvent, useCallback, useMemo, useState } from "react"; import { api, unwrap } from "@roj-ai/client"; @@ -1202,46 +1203,6 @@ function SendMessageForm({ // ResumeAgentButton // ============================================================================ -function ResumeAgentButton({ - sessionId, - agentId, -}: { - sessionId: string; - agentId: string; -}) { - const [resuming, setResuming] = useState(false); - - const handleResume = useCallback( - async (e: FormEvent) => { - e.stopPropagation(); - setResuming(true); - try { - unwrap( - await api.call("agents.resume", { - sessionId, - agentId: AgentId(agentId), - }), - ); - } catch { - // Error is visible via state change (or lack thereof) - } finally { - setResuming(false); - } - }, - [sessionId, agentId], - ); - - return ( - - ); -} // ============================================================================ // Badges diff --git a/packages/debug/src/components/debug/pages/DashboardPage.tsx b/packages/debug/src/components/debug/pages/DashboardPage.tsx index d7884faa..9ab1af9c 100644 --- a/packages/debug/src/components/debug/pages/DashboardPage.tsx +++ b/packages/debug/src/components/debug/pages/DashboardPage.tsx @@ -1,7 +1,6 @@ -import type { TimelineItem } from "@roj-ai/shared"; -import { AgentId } from "@roj-ai/shared"; -import { type FormEvent, useCallback, useMemo, useRef, useState } from "react"; -import { api, unwrap } from "@roj-ai/client"; +import { AgentId, type TimelineItem } from "@roj-ai/shared"; +import { ResumeAgentButton } from "../ResumeAgentButton.js"; +import { useCallback, useMemo, useRef, useState } from "react"; import { useEventStore, useMetrics, @@ -792,46 +791,6 @@ function SummaryStat({ ); } -function ResumeAgentButton({ - sessionId, - agentId, -}: { - sessionId: string; - agentId: string; -}) { - const [resuming, setResuming] = useState(false); - - const handleResume = useCallback( - async (e: FormEvent) => { - e.stopPropagation(); - setResuming(true); - try { - unwrap( - await api.call("agents.resume", { - sessionId, - agentId: AgentId(agentId), - }), - ); - } catch { - // Error is visible via state change (or lack thereof) - } finally { - setResuming(false); - } - }, - [sessionId, agentId], - ); - - return ( - - ); -} function formatChartTime(ts: number, rangeMs: number): string { const d = new Date(ts); diff --git a/packages/demo/CLAUDE.md b/packages/demo/CLAUDE.md index c2df33f3..0fce20ee 100644 --- a/packages/demo/CLAUDE.md +++ b/packages/demo/CLAUDE.md @@ -9,8 +9,8 @@ consumes it with the same hooks real users would use. ## Dual purpose 1. **Manual demo** — `bun run dev` to try the full stack in a browser. -2. **E2E test** (`tests/app-builder.e2e.test.ts`) — spins up the real server, - talks to it over REST+WS, snapshots LLM calls so CI can replay deterministically. +2. **E2E test** (`tests/app-builder.e2e.test.ts`) — always checks the REST + surface; opt-in record/replay modes exercise the LLM-backed build turn. ## Key files @@ -19,17 +19,21 @@ consumes it with the same hooks real users would use. - `server.ts` — thin launcher around `startStandaloneServer({ presets: [...] })`. - `spa/App.tsx` — landing form → `useChat()` workspace. Talks to standalone REST at `PLATFORM_URL` (derived from `window.location`, port 2486). -- `tests/app-builder.e2e.test.ts` — runs on port 0, uses - `createSnapshotLLMMiddleware` for deterministic LLM. Test skips the live-turn - assertion when neither `ANTHROPIC_API_KEY` nor snapshots are present. +- `tests/app-builder.e2e.test.ts` — runs on port 0 and uses + `createSnapshotLLMMiddleware`. The REST smoke test always runs. Recording the + build turn needs `ROJ_E2E_RECORD=1` plus a key; replay needs `LIVE_TESTS=1` + plus committed snapshots and never calls the provider. ## Snapshot workflow ```bash -# Record (local, once) -ANTHROPIC_API_KEY=sk-... bun test packages/demo/tests/ +# Record or refresh snapshots +ROJ_E2E_RECORD=1 ANTHROPIC_API_KEY=sk-... bun test packages/demo/tests/ -# Replay (CI, default) +# Replay the build turn without network access +LIVE_TESTS=1 bun test packages/demo/tests/ + +# Default CI: REST smoke only bun test packages/demo/tests/ ``` diff --git a/packages/demo/README.md b/packages/demo/README.md index 7dc3e471..ae570bcf 100644 --- a/packages/demo/README.md +++ b/packages/demo/README.md @@ -31,11 +31,9 @@ preview proxy. ## E2E test -`tests/app-builder.e2e.test.ts` starts the full standalone server on an -OS-assigned port, creates a session, sends a message, and asserts the -session reaches idle. LLM calls are snapshotted via -`createSnapshotLLMMiddleware` — the first run records, subsequent runs -replay from disk. +`tests/app-builder.e2e.test.ts` always starts the standalone server and checks +its REST surface. The LLM-backed build turn is opt-in so the default CI command +never calls a provider or trusts potentially stale snapshots. ```bash # Record snapshots (first time, or after preset changes) @@ -43,14 +41,17 @@ ROJ_E2E_RECORD=1 OPENROUTER_API_KEY=sk-or-... bun test packages/demo/tests/ # or with Anthropic directly ROJ_E2E_RECORD=1 ANTHROPIC_API_KEY=sk-ant-... bun test packages/demo/tests/ -# Replay (CI, default — no network, strict replay once snapshots exist) +# Replay the build turn without network access +LIVE_TESTS=1 bun test packages/demo/tests/ + +# Default CI: REST-surface smoke test only bun test packages/demo/tests/ ``` Commit `tests/__snapshots__/app-builder/` so CI can replay without an API key. When the preset prompt/tools/model change, re-record with `ROJ_E2E_RECORD=1` -(or just delete the affected snapshot files and let the next run -auto-record against the live API). +and a provider key. Replay is strict: a missing snapshot fails instead of +falling through to a live provider call. ### How deterministic is it? diff --git a/packages/demo/tests/app-builder.e2e.test.ts b/packages/demo/tests/app-builder.e2e.test.ts index 76d9bd28..af5408f6 100644 --- a/packages/demo/tests/app-builder.e2e.test.ts +++ b/packages/demo/tests/app-builder.e2e.test.ts @@ -2,15 +2,16 @@ * End-to-end test for the App Builder preset over the full @roj-ai/standalone-server * HTTP/WS surface. * - * LLM calls are snapshotted with createSnapshotLLMMiddleware — the first run - * with ANTHROPIC_API_KEY records; subsequent runs replay from disk without a - * network call. Commit the generated __snapshots__/ directory. + * LLM calls are snapshotted with createSnapshotLLMMiddleware. Recording and + * replay are explicit so stale snapshots never fall through to a live call. * * To (re-)record: - * ANTHROPIC_API_KEY=sk-... bun test packages/demo/tests/app-builder.e2e.test.ts + * ROJ_E2E_RECORD=1 ANTHROPIC_API_KEY=sk-... bun test packages/demo/tests/app-builder.e2e.test.ts * - * To replay (CI, default): - * bun test packages/demo/tests/app-builder.e2e.test.ts + * To replay the build turn without network access: + * LIVE_TESTS=1 bun test packages/demo/tests/app-builder.e2e.test.ts + * + * The default CI command runs only the REST-surface smoke test. */ import { afterAll, beforeAll, describe, expect, test } from 'bun:test' @@ -32,9 +33,13 @@ const WORKSPACE_DIR = '/tmp/roj-demo-e2e' const hasApiKey = !!process.env.ANTHROPIC_API_KEY || !!process.env.OPENROUTER_API_KEY const hasSnapshots = existsSync(SNAPSHOTS_DIR) && readdirSync(SNAPSHOTS_DIR).some((f) => f.endsWith('.json')) -const canRunLiveTurn = hasApiKey || hasSnapshots +const recordSnapshots = process.env.ROJ_E2E_RECORD === '1' +if (recordSnapshots && !hasApiKey) { + throw new Error('ROJ_E2E_RECORD=1 requires ANTHROPIC_API_KEY or OPENROUTER_API_KEY') +} +const canRunBuildTurn = recordSnapshots || (process.env.LIVE_TESTS === '1' && hasSnapshots) -describe.skip('App Builder e2e', () => { +describe('App Builder e2e', () => { let handle: StandaloneHandle let client: ReturnType @@ -67,9 +72,7 @@ describe.skip('App Builder e2e', () => { // Strip session UUIDs and randomly-assigned dev-service ports from // the request before hashing, so snapshots match across runs. normalize: normalizeStripRuntime, - // Explicit ROJ_E2E_RECORD=1 to (re-)record; default is strict replay - // in CI + auto in dev when snapshots are missing for a new branch. - mode: process.env.ROJ_E2E_RECORD === '1' ? 'record' : hasSnapshots ? 'replay' : 'auto', + mode: recordSnapshots ? 'record' : 'replay', }), ], }) @@ -91,7 +94,7 @@ describe.skip('App Builder e2e', () => { expect(listed.instances[0].instanceId).toBe(handle.instance.id) }) - test.skipIf(!canRunLiveTurn)('session completes a simple build turn', async () => { + test.skipIf(!canRunBuildTurn)('session completes a simple build turn', async () => { const session = await client.sessions.create({ instanceId: handle.instance.id, presetId: 'app-builder', diff --git a/packages/platform-cli/package.json b/packages/platform-cli/package.json index 8786e71d..b8dec521 100644 --- a/packages/platform-cli/package.json +++ b/packages/platform-cli/package.json @@ -22,6 +22,7 @@ "url": "https://github.com/contember/roj/issues" }, "dependencies": { + "@roj-ai/client": "workspace:*", "@roj-ai/sandbox-runtime": "workspace:*" }, "devDependencies": { diff --git a/packages/platform-cli/src/resource.ts b/packages/platform-cli/src/resource.ts index 7312b12b..b8966a18 100644 --- a/packages/platform-cli/src/resource.ts +++ b/packages/platform-cli/src/resource.ts @@ -1,4 +1,5 @@ import { execSync } from 'node:child_process' +import { postFile, sha256Hex } from '@roj-ai/client/platform' import { mkdtempSync, statSync } from 'node:fs' import { basename, join, resolve } from 'node:path' import { tmpdir } from 'node:os' @@ -51,7 +52,7 @@ export async function uploadResource(pathOrDir: string, options: { contentHash, filename, mimeType, - body: { buf, filename, mimeType }, + body: new Blob([buf], { type: mimeType }), }) } @@ -132,57 +133,9 @@ export async function uploadResource(pathOrDir: string, options: { } } -interface PostFileArgs { - url: string - apiKey: string - contentHash: string - filename: string - mimeType: string - body?: { buf: ArrayBuffer; filename: string; mimeType: string } -} - -interface PostFileResult { - status: number - body: { - ok?: boolean - error?: string - fileId?: string - filename?: string - mimeType?: string - size?: number - r2Key?: string - deduped?: boolean - } | null -} -async function postFile(args: PostFileArgs): Promise { - const formData = new FormData() - formData.append('contentHash', args.contentHash) - formData.append('filename', args.filename) - formData.append('mimeType', args.mimeType) - if (args.body) { - formData.append('file', new Blob([args.body.buf], { type: args.body.mimeType }), args.body.filename) - } - - const response = await fetch(`${args.url}/api/v1/files/upload`, { - method: 'POST', - headers: { Authorization: `Bearer ${args.apiKey}` }, - body: formData, - }) - const body = await response.json().catch(() => null) as PostFileResult['body'] - return { status: response.status, body } -} -async function sha256Hex(buf: ArrayBuffer): Promise { - const digest = await crypto.subtle.digest('SHA-256', buf) - const bytes = new Uint8Array(digest) - let hex = '' - for (let i = 0; i < bytes.length; i++) { - hex += bytes[i].toString(16).padStart(2, '0') - } - return hex -} function guessMimeType(filename: string): string { const ext = filename.split('.').pop()?.toLowerCase() diff --git a/packages/platform-cli/tsconfig.json b/packages/platform-cli/tsconfig.json index 6c679034..1561dd94 100644 --- a/packages/platform-cli/tsconfig.json +++ b/packages/platform-cli/tsconfig.json @@ -9,6 +9,7 @@ "include": ["src/**/*"], "exclude": ["node_modules", "dist"], "references": [ + { "path": "../client" }, { "path": "../sandbox-runtime" } ] } diff --git a/packages/sandbox-runtime/src/server.ts b/packages/sandbox-runtime/src/server.ts index 3011caa1..0b8d9897 100644 --- a/packages/sandbox-runtime/src/server.ts +++ b/packages/sandbox-runtime/src/server.ts @@ -11,7 +11,7 @@ import { type AppEnv, createApp } from '@roj-ai/sdk/transport/http/app' import { createAgentTransport, type IAgentTransport, ServerAdapter } from '@roj-ai/sdk/transport/adapter' import { bunWebSocketFactory, createBunWebSocketHandlers } from '@roj-ai/transport/bun' import type { Hono } from 'hono' -import { createBunPlatform } from '@roj-ai/sdk/bun-platform' +import { createBunPlatform, createServerLifecycle, runServerStartup, shutdownFromSignal } from '@roj-ai/sdk/bun-platform' import { SANDBOX_RUNTIME_NAME, SANDBOX_RUNTIME_VERSION } from './info.js' // ============================================================================ @@ -83,19 +83,37 @@ export async function startServer(options: StartServerOptions): Promise | undefined + let transportStopErrorReported = false + const stopTransportOnce = async () => { + transportStopPromise ??= Promise.resolve().then(() => transport.stop()) + try { + await transportStopPromise + } catch (error) { + if (transportStopErrorReported) return + transportStopErrorReported = true + throw error + } } - try { - await transport.start() - } catch (error) { - logger.error('Transport connection failed (will retry via reconnect)', error instanceof Error ? error : new Error(String(error))) - } + const lifecycle = createServerLifecycle({ + stopIngress: () => server.stop(true), + cleanupSteps: [ + ...options.onShutdown ? [options.onShutdown] : [], + () => sessionManager.shutdown(), + stopTransportOnce, + ], + runSignalShutdown: shutdown => { + void shutdownFromSignal(shutdown) + }, + }) + + // Load persisted sessions after HTTP server is up (health checks pass during loading). + await runServerStartup(lifecycle, [ + { run: () => sessionManager.loadAllSessions() }, + // A pending connection is canceled before the remaining ordered cleanup. + { run: () => transport.start(), cancel: stopTransportOnce }, + ]) logger.info('Agent server started', { host: config.host, @@ -104,24 +122,7 @@ export async function startServer(options: StartServerOptions): Promise { - logger.info('Shutting down...') - if (options.onShutdown) { - await options.onShutdown() - } - await sessionManager.shutdown() - await transport.stop() - server.stop() - } - - process.on('SIGINT', () => { - void shutdown().then(() => process.exit(0)) - }) - process.on('SIGTERM', () => { - void shutdown().then(() => process.exit(0)) - }) - - return { config, logger, shutdown } + return { config, logger, shutdown: lifecycle.shutdown } } // ============================================================================ diff --git a/packages/sandbox-runtime/tests/server.test.ts b/packages/sandbox-runtime/tests/server.test.ts new file mode 100644 index 00000000..38552379 --- /dev/null +++ b/packages/sandbox-runtime/tests/server.test.ts @@ -0,0 +1,326 @@ +import { describe, expect, it } from 'bun:test' +import { + createServerLifecycle, + runServerStartup, + ServerCleanupError, + shutdownFromSignal, + StartupInterruptedError, + type ServerSignal, + type SignalListenerRegistry, +} from '@roj-ai/sdk/bun-platform' + +class FakeSignalListeners implements SignalListenerRegistry { + private readonly listeners = new Map void>>() + + add(signal: ServerSignal, listener: () => void): void { + const listeners = this.listeners.get(signal) ?? new Set() + listeners.add(listener) + this.listeners.set(signal, listeners) + } + + remove(signal: ServerSignal, listener: () => void): void { + this.listeners.get(signal)?.delete(listener) + } + + emit(signal: ServerSignal): void { + for (const listener of this.listeners.get(signal) ?? []) listener() + } + + count(signal: ServerSignal): number { + return this.listeners.get(signal)?.size ?? 0 + } +} + +describe('server lifecycle', () => { + it('coalesces shutdown, stops ingress synchronously, and attempts every cleanup step once', async () => { + const firstFailure = new Error('stop failed') + const secondFailure = new Error('session cleanup failed') + let rejectIngress = (error: unknown) => {} + const ingressPending = new Promise((_resolve, reject) => { + rejectIngress = reject + }) + const calls: string[] = [] + const signals = new FakeSignalListeners() + const lifecycle = createServerLifecycle({ + stopIngress: () => { + calls.push('ingress') + return ingressPending + }, + cleanupSteps: [ + () => calls.push('callback'), + () => { + calls.push('sessions') + throw secondFailure + }, + () => calls.push('transport'), + ], + runSignalShutdown: shutdown => { + void shutdown() + }, + }, signals) + + const firstShutdown = lifecycle.shutdown() + const observedShutdown = firstShutdown.catch(error => error) + const secondShutdown = lifecycle.shutdown() + expect(secondShutdown).toBe(firstShutdown) + expect(calls).toEqual(['ingress']) + expect(signals.count('SIGINT')).toBe(1) + expect(signals.count('SIGTERM')).toBe(1) + + rejectIngress(firstFailure) + const caught: unknown = await observedShutdown + + expect(caught).toBeInstanceOf(ServerCleanupError) + if (!(caught instanceof ServerCleanupError)) throw new Error('Expected ServerCleanupError') + expect(caught.orderedErrors).toEqual([firstFailure, secondFailure]) + expect(calls).toEqual(['ingress', 'callback', 'sessions', 'transport']) + expect(signals.count('SIGINT')).toBe(0) + expect(signals.count('SIGTERM')).toBe(0) + expect(lifecycle.shutdown()).toBe(firstShutdown) + }) + + it('installs named signal listeners and handles SIGINT plus SIGTERM only once', async () => { + const calls: string[] = [] + const signals = new FakeSignalListeners() + let signalRuns = 0 + let signalShutdown: Promise | undefined + const lifecycle = createServerLifecycle({ + stopIngress: () => calls.push('ingress'), + cleanupSteps: [], + runSignalShutdown: shutdown => { + signalRuns++ + signalShutdown = shutdown() + }, + }, signals) + + expect(signals.count('SIGINT')).toBe(1) + expect(signals.count('SIGTERM')).toBe(1) + signals.emit('SIGINT') + signals.emit('SIGTERM') + await signalShutdown + + expect(signalRuns).toBe(1) + expect(calls).toEqual(['ingress']) + expect(lifecycle.isShuttingDown()).toBe(true) + }) + + it('treats startup rejection as fatal and preserves cleanup errors after it', async () => { + const startupFailure = new Error('load failed') + const ingressFailure = new Error('stop failed') + const transportFailure = new Error('transport cleanup failed') + const calls: string[] = [] + const lifecycle = createServerLifecycle({ + stopIngress: () => { + calls.push('ingress') + throw ingressFailure + }, + cleanupSteps: [ + () => calls.push('sessions'), + () => { + calls.push('transport') + throw transportFailure + }, + ], + runSignalShutdown: shutdown => { + void shutdown() + }, + }, new FakeSignalListeners()) + + let caught: unknown + try { + await runServerStartup(lifecycle, [{ run: async () => { + throw startupFailure + } }]) + } catch (error) { + caught = error + } + + expect(caught).toBeInstanceOf(AggregateError) + if (!(caught instanceof AggregateError)) throw new Error('Expected AggregateError') + expect(caught.errors).toEqual([startupFailure, ingressFailure, transportFailure]) + expect(caught.cause).toBe(startupFailure) + expect(calls).toEqual(['ingress', 'sessions', 'transport']) + }) + + it.each(['load', 'start'])('defers cleanup until a pending %s stage settles', async (stageName) => { + let releaseStartup = () => {} + const pendingStartup = new Promise(resolve => { + releaseStartup = resolve + }) + let markStageEntered = () => {} + const stageEntered = new Promise(resolve => { + markStageEntered = resolve + }) + const signals = new FakeSignalListeners() + let signalTask: Promise | undefined + const exitCodes: number[] = [] + let signalRuns = 0 + let ingressRuns = 0 + let nextStepRuns = 0 + const cleanupCalls: string[] = [] + const lifecycle = createServerLifecycle({ + stopIngress: () => { + ingressRuns++ + }, + cleanupSteps: [ + () => cleanupCalls.push('sessions'), + () => cleanupCalls.push('transport'), + ], + runSignalShutdown: shutdown => { + signalRuns++ + signalTask = shutdownFromSignal(shutdown, undefined, code => exitCodes.push(code)) + }, + }, signals) + + const deferredStage = () => { + markStageEntered() + return pendingStartup + } + const followingStage = async () => { + nextStepRuns++ + } + const startup = runServerStartup(lifecycle, stageName === 'load' + ? [{ run: deferredStage }, { run: followingStage }] + : [{ run: async () => {} }, { run: deferredStage }]) + await stageEntered + signals.emit('SIGTERM') + signals.emit('SIGINT') + + expect(ingressRuns).toBe(1) + expect(cleanupCalls).toEqual([]) + expect(exitCodes).toEqual([]) + expect(signalRuns).toBe(1) + expect(signals.count('SIGINT')).toBe(1) + expect(signals.count('SIGTERM')).toBe(1) + + releaseStartup() + + await expect(startup).rejects.toBeInstanceOf(StartupInterruptedError) + await signalTask + expect(nextStepRuns).toBe(0) + expect(cleanupCalls).toEqual(['sessions', 'transport']) + expect(exitCodes).toEqual([0]) + expect(signals.count('SIGINT')).toBe(0) + expect(signals.count('SIGTERM')).toBe(0) + }) + + it('joins shutdown when a pending startup rejection races a signal', async () => { + const startupFailure = new Error('load failed') + let rejectStartup = (_error: unknown) => {} + const pendingStartup = new Promise((_resolve, reject) => { + rejectStartup = reject + }) + const signals = new FakeSignalListeners() + let signalTask: Promise | undefined + let cleanupRuns = 0 + const lifecycle = createServerLifecycle({ + stopIngress: () => {}, + cleanupSteps: [() => { + cleanupRuns++ + }], + runSignalShutdown: shutdown => { + signalTask = shutdownFromSignal(shutdown, undefined, () => {}) + }, + }, signals) + + const startup = runServerStartup(lifecycle, [{ run: () => pendingStartup }]) + signals.emit('SIGINT') + expect(cleanupRuns).toBe(0) + rejectStartup(startupFailure) + + await expect(startup).rejects.toBeInstanceOf(StartupInterruptedError) + await signalTask + expect(cleanupRuns).toBe(1) + }) + + it('cancels a blocking transport start and stops transport only once', async () => { + let settleStart = () => {} + const calls: string[] = [] + const transport = { + start() { + calls.push('transport.start') + return new Promise(resolve => { + settleStart = resolve + }) + }, + async stop() { + calls.push('transport.stop') + settleStart() + }, + } + let stopPromise: Promise | undefined + const stopTransportOnce = () => { + stopPromise ??= transport.stop() + return stopPromise + } + const signals = new FakeSignalListeners() + const exitCodes: number[] = [] + let signalTask: Promise | undefined + let markStartEntered = () => {} + const startEntered = new Promise(resolve => { + markStartEntered = resolve + }) + const lifecycle = createServerLifecycle({ + stopIngress: () => calls.push('ingress'), + cleanupSteps: [ + () => calls.push('callback'), + () => calls.push('sessions'), + stopTransportOnce, + ], + runSignalShutdown: shutdown => { + signalTask = shutdownFromSignal(shutdown, undefined, code => exitCodes.push(code)) + }, + }, signals) + const startup = runServerStartup(lifecycle, [ + { run: async () => {} }, + { + run: () => { + markStartEntered() + return transport.start() + }, + cancel: stopTransportOnce, + }, + ]) + + await startEntered + signals.emit('SIGTERM') + await expect(startup).rejects.toBeInstanceOf(StartupInterruptedError) + await signalTask + + expect(calls).toEqual([ + 'transport.start', + 'ingress', + 'transport.stop', + 'callback', + 'sessions', + ]) + expect(exitCodes).toEqual([0]) + expect(signals.count('SIGINT')).toBe(0) + expect(signals.count('SIGTERM')).toBe(0) + }) +}) + +describe('shutdownFromSignal', () => { + it('reports a rejected shutdown and exits with failure', async () => { + const failure = new Error('shutdown failed') + const reported: Array<{ message: string; error: unknown }> = [] + const exitCodes: number[] = [] + + await shutdownFromSignal( + async () => { + throw failure + }, + (message, error) => reported.push({ message, error }), + (code) => exitCodes.push(code), + ) + + expect(reported).toEqual([{ message: 'Shutdown failed', error: failure }]) + expect(exitCodes).toEqual([1]) + }) + + it('exits successfully after clean shutdown', async () => { + const exitCodes: number[] = [] + await shutdownFromSignal(async () => {}, undefined, code => exitCodes.push(code)) + expect(exitCodes).toEqual([0]) + }) +}) diff --git a/packages/sdk/CLAUDE.md b/packages/sdk/CLAUDE.md index a08060b1..22bbcf72 100644 --- a/packages/sdk/CLAUDE.md +++ b/packages/sdk/CLAUDE.md @@ -2,16 +2,14 @@ Bun-based agent server: LLM sessions, plugin architecture, event sourcing. -**Not linted by Biome** (excluded in root biome.json). Uses its own conventions. - ## Commands +Run from the repo root — this package declares only `type-check`. + ```bash -bun run dev # Watch mode with example config -bun run start # Production start -bun run build # Bundle to dist/ (bun build, single file) -bun run type-check # tsc --noEmit -bun test # Bun native test runner +bun run ts:build # Build all packages (tsc --build + tsc-alias) +bun run lint # Biome (this package IS linted — 262 files) +bun test packages/sdk/src ``` ## Architecture @@ -24,9 +22,9 @@ bun test # Bun native test runner ``` src/ - main.ts # CLI entry (bun src/main.ts ) - server.ts # startServer() high-level API + index.ts # Public API surface bootstrap.ts # Composition root — wires all services + builtin-events.ts # Event definitions shared across core config.ts # Config interface, loadConfig (env vars) user-config.ts # defineConfig for roj.config.ts files core/ @@ -37,12 +35,17 @@ src/ tools/ # Tool definitions, executor preset/ # defineAgent, createPreset, createOrchestrator events/ # EventStore (file/memory), types - plugins/ # 15+ built-in plugins (mailbox, filesystem, shell, etc.) + file-store/ # SessionFileStore — agent-visible path resolution + image/ # Image processing and resizing + plugins/ # 22 built-in plugins (mailbox, filesystem, shell, services, etc.) transport/ http/ # Hono routes: /rpc, /health, uploads, files adapter/ # ServerAdapter (standalone) / ClientAdapter (worker mode) rpc/ # RPC protocol types - testing/ # TestHarness, NotificationCollector + testing/ # TestHarness, NotificationCollector (published as ./testing) + lib/ # Logger, Result, small utilities + platform/ # Platform interfaces (fs, process) + bun-platform/ # Bun implementations (published as ./bun-platform) ``` ## Plugin System diff --git a/packages/sdk/package.json b/packages/sdk/package.json index d57450ef..9e1b8c83 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -113,11 +113,14 @@ "types": "./dist/transport/rpc/index.d.ts", "import": "./dist/transport/rpc/index.js", "default": "./dist/transport/rpc/index.js" - } + }, + "./package.json": "./package.json" }, "files": [ "src", - "dist" + "dist", + "!src/**/*.test.ts", + "!src/__tests__" ], "publishConfig": { "access": "public" @@ -136,17 +139,14 @@ }, "dependencies": { "@roj-ai/transport": "workspace:*", - "@hono/zod-validator": "0.7.6", "hono": "4.13.1", "ignore": "7.0.5", "tokenx": "^1.3.0", "uuidv7": "^1.1.0", - "ws": "^8.18.0", "zod": "4.3.6" }, "devDependencies": { "@types/bun": "latest", - "@types/ws": "^8.5.10", "typescript": "^5.7.2" }, "types": "./dist/index.d.ts" diff --git a/packages/sdk/src/bootstrap.ts b/packages/sdk/src/bootstrap.ts index c78fa919..7f6a2758 100644 --- a/packages/sdk/src/bootstrap.ts +++ b/packages/sdk/src/bootstrap.ts @@ -257,6 +257,8 @@ export function createSystemFromServices( dataFileStore: services.dataFileStore, onUserOutput: options?.onUserOutput, preprocessorRegistry: services.preprocessorRegistry, + uploadArchiveLimits: services.config.uploadArchiveLimits, + resourceArchiveLimits: services.config.resourceArchiveLimits, llmLogger: services.llmLogger, portPool: services.portPool, pidRegistry: services.pidRegistry, diff --git a/packages/sdk/src/bun-platform/index.ts b/packages/sdk/src/bun-platform/index.ts index 47700773..2d5f762e 100644 --- a/packages/sdk/src/bun-platform/index.ts +++ b/packages/sdk/src/bun-platform/index.ts @@ -19,3 +19,15 @@ export function createBunPlatform(): Platform { } export { createBunFileSystem, createBunProcessRunner } +export { + createServerLifecycle, + runServerStartup, + ServerCleanupError, + shutdownFromSignal, + StartupInterruptedError, + type ServerLifecycle, + type ServerLifecycleOptions, + type ServerSignal, + type SignalListenerRegistry, + type StartupStage, +} from './server-lifecycle.js' diff --git a/packages/sdk/src/bun-platform/server-lifecycle.ts b/packages/sdk/src/bun-platform/server-lifecycle.ts new file mode 100644 index 00000000..79e95973 --- /dev/null +++ b/packages/sdk/src/bun-platform/server-lifecycle.ts @@ -0,0 +1,218 @@ +export type ServerSignal = 'SIGINT' | 'SIGTERM' + +export interface SignalListenerRegistry { + add(signal: ServerSignal, listener: () => void): void + remove(signal: ServerSignal, listener: () => void): void +} + +export interface ServerLifecycle { + shutdown(): Promise + isShuttingDown(): boolean + runStartupStage(stage: StartupStage): Promise +} + +export interface StartupStage { + run: () => Promise + cancel?: () => Promise | void +} + +export interface ServerLifecycleOptions { + stopIngress: () => Promise | void + cleanupSteps: ReadonlyArray<() => Promise | void> + runSignalShutdown: (shutdown: () => Promise) => void +} + +export class ServerCleanupError extends AggregateError { + readonly orderedErrors: readonly unknown[] + + constructor(errors: readonly unknown[]) { + super(errors, 'Server shutdown failed') + this.name = 'ServerCleanupError' + this.orderedErrors = errors + } +} + +export class StartupInterruptedError extends Error { + constructor() { + super('Server startup was interrupted by shutdown') + this.name = 'StartupInterruptedError' + } +} + +const processSignalListeners: SignalListenerRegistry = { + add(signal, listener) { + process.on(signal, listener) + }, + remove(signal, listener) { + process.off(signal, listener) + }, +} + +export function createServerLifecycle( + options: ServerLifecycleOptions, + signalListeners: SignalListenerRegistry = processSignalListeners, +): ServerLifecycle { + let shutdownPromise: Promise | undefined + let shuttingDown = false + let activeStartupStage: { promise: Promise; cancel?: () => Promise | void } | undefined + let listenersInstalled = true + let signalHandled = false + + const removeSignalListeners = () => { + if (!listenersInstalled) return + listenersInstalled = false + signalListeners.remove('SIGINT', handleSigint) + signalListeners.remove('SIGTERM', handleSigterm) + } + + const finishCleanup = async ( + ingressResult: Promise | void, + startupStage: { promise: Promise; cancel?: () => Promise | void } | undefined, + ) => { + const errors: unknown[] = [] + let cancellationError: unknown + let cancellationFailed = false + let cancellationSettled: Promise | undefined + if (startupStage?.cancel) { + try { + cancellationSettled = Promise.resolve(startupStage.cancel()).catch((error) => { + cancellationFailed = true + cancellationError = error + }) + } catch (error) { + cancellationFailed = true + cancellationError = error + } + } + + try { + await ingressResult + } catch (error) { + errors.push(error) + } + + await cancellationSettled + if (cancellationFailed) errors.push(cancellationError) + + if (startupStage) { + try { + await startupStage.promise + } catch { + // Startup reports its own failure after joining shutdown. + } + } + + for (const step of options.cleanupSteps) { + try { + await step() + } catch (error) { + errors.push(error) + } + } + + if (errors.length === 1) throw errors[0] + if (errors.length > 1) throw new ServerCleanupError(errors) + } + + const shutdown = (): Promise => { + if (shutdownPromise) return shutdownPromise + + shuttingDown = true + const startupStage = activeStartupStage + let resolveShutdown = () => {} + let rejectShutdown = (_error: unknown) => {} + shutdownPromise = new Promise((resolve, reject) => { + resolveShutdown = resolve + rejectShutdown = reject + }) + + let ingressResult: Promise | void + try { + ingressResult = options.stopIngress() + } catch (error) { + ingressResult = Promise.reject(error) + } + + void (async () => { + try { + await finishCleanup(ingressResult, startupStage) + removeSignalListeners() + resolveShutdown() + } catch (error) { + removeSignalListeners() + rejectShutdown(error) + } + })() + return shutdownPromise + } + + const beginSignalShutdown = () => { + if (signalHandled) return + signalHandled = true + options.runSignalShutdown(shutdown) + } + + function handleSigint() { + beginSignalShutdown() + } + + function handleSigterm() { + beginSignalShutdown() + } + + signalListeners.add('SIGINT', handleSigint) + signalListeners.add('SIGTERM', handleSigterm) + + return { + shutdown, + isShuttingDown: () => shuttingDown, + async runStartupStage(stage) { + if (shuttingDown) throw new StartupInterruptedError() + const runningStage = Promise.resolve().then(stage.run) + const activeStage = { promise: runningStage, cancel: stage.cancel } + activeStartupStage = activeStage + try { + await runningStage + } finally { + if (activeStartupStage === activeStage) activeStartupStage = undefined + } + }, + } +} + +export async function runServerStartup(lifecycle: ServerLifecycle, steps: ReadonlyArray): Promise { + try { + for (const step of steps) { + await lifecycle.runStartupStage(step) + if (lifecycle.isShuttingDown()) throw new StartupInterruptedError() + } + } catch (startupError) { + const reportedStartupError = + lifecycle.isShuttingDown() && !(startupError instanceof StartupInterruptedError) ? new StartupInterruptedError() : startupError + try { + await lifecycle.shutdown() + } catch (cleanupError) { + const cleanupErrors = cleanupError instanceof ServerCleanupError ? cleanupError.orderedErrors : [cleanupError] + throw new AggregateError([reportedStartupError, ...cleanupErrors], 'Server startup failed and cleanup failed', { + cause: reportedStartupError, + }) + } + throw reportedStartupError + } +} + +export async function shutdownFromSignal( + shutdown: () => Promise, + reportError: (message: string, error: unknown) => void = (message, error) => console.error(message, error), + exit: (code: number) => void = (code) => process.exit(code), +): Promise { + let exitCode = 0 + try { + await shutdown() + } catch (error) { + exitCode = 1 + reportError('Shutdown failed', error) + } finally { + exit(exitCode) + } +} diff --git a/packages/sdk/src/config.test.ts b/packages/sdk/src/config.test.ts index b992f2a5..941e74a1 100644 --- a/packages/sdk/src/config.test.ts +++ b/packages/sdk/src/config.test.ts @@ -23,6 +23,10 @@ describe('config', () => { delete process.env.LOG_FORMAT delete process.env.WORKER_URL delete process.env.AGENT_TOKEN + delete process.env.UPLOAD_ARCHIVE_MAX_ENTRIES + delete process.env.UPLOAD_ARCHIVE_MAX_UNCOMPRESSED_BYTES + delete process.env.RESOURCE_ARCHIVE_MAX_ENTRIES + delete process.env.RESOURCE_ARCHIVE_MAX_UNCOMPRESSED_BYTES const config = loadConfig() @@ -36,6 +40,8 @@ describe('config', () => { expect(config.logFormat).toBe('console') expect(config.workerUrl).toBeUndefined() expect(config.agentToken).toBeUndefined() + expect(config.uploadArchiveLimits).toBeUndefined() + expect(config.resourceArchiveLimits).toBeUndefined() }) test('loads values from environment', () => { @@ -49,6 +55,10 @@ describe('config', () => { process.env.LOG_FORMAT = 'json' process.env.WORKER_URL = 'https://worker.example.com' process.env.AGENT_TOKEN = 'secret-token' + process.env.UPLOAD_ARCHIVE_MAX_ENTRIES = '750' + process.env.UPLOAD_ARCHIVE_MAX_UNCOMPRESSED_BYTES = '209715200' + process.env.RESOURCE_ARCHIVE_MAX_ENTRIES = '2500' + process.env.RESOURCE_ARCHIVE_MAX_UNCOMPRESSED_BYTES = '1073741824' const config = loadConfig() @@ -62,6 +72,14 @@ describe('config', () => { expect(config.logFormat).toBe('json') expect(config.workerUrl).toBe('https://worker.example.com') expect(config.agentToken).toBe('secret-token') + expect(config.uploadArchiveLimits).toEqual({ + maxEntries: 750, + maxTotalUncompressedSize: 209715200, + }) + expect(config.resourceArchiveLimits).toEqual({ + maxEntries: 2500, + maxTotalUncompressedSize: 1073741824, + }) }) }) @@ -170,5 +188,18 @@ describe('config', () => { const errors = validateConfig(config) expect(errors.length).toBeGreaterThanOrEqual(3) }) + + test('rejects invalid archive limits', () => { + const config: Config = { + ...validConfig, + uploadArchiveLimits: { maxEntries: -1 }, + resourceArchiveLimits: { maxTotalUncompressedSize: Number.NaN }, + } + + expect(validateConfig(config)).toEqual([ + 'Invalid uploadArchiveLimits.maxEntries: -1', + 'Invalid resourceArchiveLimits.maxTotalUncompressedSize: NaN', + ]) + }) }) }) diff --git a/packages/sdk/src/config.ts b/packages/sdk/src/config.ts index d6bc3c97..da936dad 100644 --- a/packages/sdk/src/config.ts +++ b/packages/sdk/src/config.ts @@ -4,6 +4,7 @@ import { resolve } from 'node:path' import type { MockInferenceHandler } from './core/llm/mock.js' +import type { ArchiveLimitOverrides } from './lib/archive/index.js' import type { LogLevel } from './lib/logger/logger.js' /** @@ -33,6 +34,11 @@ export interface Config { /** Max concurrent vision LLM calls when classifying uploaded images. Default 10. */ imageClassifierConcurrency?: number + /** Aggregate limits across every ZIP nested in one attachment upload. */ + uploadArchiveLimits?: ArchiveLimitOverrides + /** Per-archive limits for resource ZIP injection. */ + resourceArchiveLimits?: ArchiveLimitOverrides + /** * Identity of the application embedding this SDK. Reported via `/status` * so platform health-checks can surface "what's actually running" in debug @@ -78,6 +84,8 @@ export const loadConfig = (): Config => { imageClassifierConcurrency: process.env.IMAGE_CLASSIFIER_CONCURRENCY ? parseInt(process.env.IMAGE_CLASSIFIER_CONCURRENCY, 10) : undefined, + uploadArchiveLimits: archiveLimitsFromEnv('UPLOAD_ARCHIVE'), + resourceArchiveLimits: archiveLimitsFromEnv('RESOURCE_ARCHIVE'), logLevel: (process.env.LOG_LEVEL ?? 'info') as LogLevel, logFormat: (process.env.LOG_FORMAT ?? 'console') as 'console' | 'json', workerUrl: process.env.WORKER_URL, @@ -118,5 +126,33 @@ export const validateConfig = (config: Config): string[] => { errors.push(`Invalid persistence type: ${config.persistence}`) } + validateArchiveLimitOverrides('uploadArchiveLimits', config.uploadArchiveLimits, errors) + validateArchiveLimitOverrides('resourceArchiveLimits', config.resourceArchiveLimits, errors) + return errors } + +function archiveLimitsFromEnv(prefix: 'UPLOAD_ARCHIVE' | 'RESOURCE_ARCHIVE'): ArchiveLimitOverrides | undefined { + const maxEntries = process.env[`${prefix}_MAX_ENTRIES`] + const maxTotalUncompressedSize = process.env[`${prefix}_MAX_UNCOMPRESSED_BYTES`] + if (maxEntries === undefined && maxTotalUncompressedSize === undefined) return undefined + return { + maxEntries: maxEntries === undefined ? undefined : parseInt(maxEntries, 10), + maxTotalUncompressedSize: maxTotalUncompressedSize === undefined + ? undefined + : parseInt(maxTotalUncompressedSize, 10), + } +} + +function validateArchiveLimitOverrides( + name: 'uploadArchiveLimits' | 'resourceArchiveLimits', + limits: ArchiveLimitOverrides | undefined, + errors: string[], +): void { + if (!limits) return + for (const [field, value] of Object.entries(limits)) { + if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) { + errors.push(`Invalid ${name}.${field}: ${value}`) + } + } +} diff --git a/packages/sdk/src/core/agents/agent-shutdown.test.ts b/packages/sdk/src/core/agents/agent-shutdown.test.ts index 7e567fc2..326991c2 100644 --- a/packages/sdk/src/core/agents/agent-shutdown.test.ts +++ b/packages/sdk/src/core/agents/agent-shutdown.test.ts @@ -95,6 +95,9 @@ async function createTestAgent( emitEvent: async (event) => { await store.emit(withSessionId(TEST_SESSION_ID, event)) }, + emitEvents: async (events) => { + await store.emitBatch(events.map((event) => withSessionId(TEST_SESSION_ID, event))) + }, notify: () => {}, } diff --git a/packages/sdk/src/core/agents/agent.ts b/packages/sdk/src/core/agents/agent.ts index 7d50f373..a2d310e4 100644 --- a/packages/sdk/src/core/agents/agent.ts +++ b/packages/sdk/src/core/agents/agent.ts @@ -1081,6 +1081,7 @@ export class Agent { platform: this.sessionContext.platform, logger: this.logger, emitEvent: this.sessionContext.emitEvent, + emitEvents: this.sessionContext.emitEvents, notify: this.sessionContext.notify, // AgentContext fields agentId: this.id, diff --git a/packages/sdk/src/core/agents/communicator.ts b/packages/sdk/src/core/agents/communicator.ts deleted file mode 100644 index b04d59e2..00000000 --- a/packages/sdk/src/core/agents/communicator.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { AgentDefinition, CommunicatorConfig } from './config.js' - -export function createCommunicatorDefinition( - config: CommunicatorConfig, -): AgentDefinition { - return { - name: '__communicator__', - system: config.system, - agents: config.agents ?? [], - tools: config.tools ?? [], - debounceMs: config.debounceMs ?? 100, // Fast response for UX - debounceCallback: config.debounceCallback, - checkIntervalMs: config.checkIntervalMs, - model: config.model, - } -} diff --git a/packages/sdk/src/core/agents/retry.test.ts b/packages/sdk/src/core/agents/retry.test.ts new file mode 100644 index 00000000..4b871a4f --- /dev/null +++ b/packages/sdk/src/core/agents/retry.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, jest, spyOn, test } from 'bun:test' +import type { LLMError } from '~/core/llm/provider.js' +import type { Logger } from '~/lib/logger/logger.js' +import { Err, Ok } from '~/lib/utils/result.js' +import { withLLMRetry, withRetry } from './retry.js' + +const timeout: LLMError = { type: 'timeout', message: 'Request timed out' } + +describe('withRetry abort handling', () => { + test('a cancel after a retryable failure reports the cancel, not the failure', async () => { + const controller = new AbortController() + const listenerSpy = spyOn(controller.signal, 'addEventListener') + let attempts = 0 + + const resultPromise = withLLMRetry( + async () => { + attempts++ + return Err(timeout) + }, + { baseDelayMs: 60_000, maxDelayMs: 60_000, signal: controller.signal }, + ) + await Promise.resolve() + expect(listenerSpy).toHaveBeenCalledTimes(1) + + controller.abort() + const result = await resultPromise + + expect(attempts).toBe(1) + expect(result.ok).toBe(false) + if (!result.ok) expect(result.error.type).toBe('aborted') + }) + + test('a cancel before the first attempt reports the cancel', async () => { + const controller = new AbortController() + controller.abort() + let attempts = 0 + + const result = await withLLMRetry( + async () => { + attempts++ + return Err(timeout) + }, + { signal: controller.signal }, + ) + + expect(attempts).toBe(0) + expect(result.ok).toBe(false) + if (!result.ok) expect(result.error.type).toBe('aborted') + }) + + test('without a cancel the last error still surfaces after maxAttempts', async () => { + let attempts = 0 + + const result = await withLLMRetry( + async () => { + attempts++ + return Err(timeout) + }, + { maxAttempts: 3, baseDelayMs: 1, maxDelayMs: 1 }, + ) + + expect(attempts).toBe(3) + expect(result.ok).toBe(false) + if (!result.ok) expect(result.error.type).toBe('timeout') + }) + + test('the final failed attempt neither calculates nor logs another retry', async () => { + jest.useFakeTimers() + let retryDelayReads = 0 + let retryWarnings = 0 + const logger: Logger = { + debug: () => {}, + info: () => {}, + warn: () => { + retryWarnings++ + }, + error: () => {}, + child: () => logger, + level: 'debug', + } + + try { + const resultPromise = withRetry(async () => Err('exhausted'), { + isRetryable: () => true, + getRetryDelay: () => { + retryDelayReads++ + return 60_000 + }, + maxAttempts: 1, + logger, + }) + await Promise.resolve() + + expect(retryDelayReads).toBe(0) + expect(retryWarnings).toBe(0) + expect(jest.getTimerCount()).toBe(0) + expect(await resultPromise).toEqual(Err('exhausted')) + } finally { + jest.useRealTimers() + } + }) + + test('a cancel coincident with the final failure returns the abort error', async () => { + const controller = new AbortController() + + const result = await withRetry( + async () => { + controller.abort() + return Err('transient') + }, + { + isRetryable: () => true, + maxAttempts: 1, + signal: controller.signal, + abortError: 'cancelled', + }, + ) + + expect(result).toEqual(Err('cancelled')) + }) + + test('a cancel with no abortError configured still falls back to the last error', async () => { + const controller = new AbortController() + + const result = await withRetry( + async () => { + controller.abort() + return Err('transient') + }, + { isRetryable: () => true, baseDelayMs: 1, maxDelayMs: 1, signal: controller.signal }, + ) + + expect(result).toEqual(Err('transient')) + }) + + test('success short-circuits the retry loop', async () => { + let attempts = 0 + + const result = await withLLMRetry( + async () => { + attempts++ + return attempts === 1 ? Err(timeout) : Ok('done') + }, + { baseDelayMs: 1, maxDelayMs: 1 }, + ) + + expect(attempts).toBe(2) + expect(result).toEqual(Ok('done')) + }) +}) diff --git a/packages/sdk/src/core/agents/retry.ts b/packages/sdk/src/core/agents/retry.ts index a115d595..0f1a9004 100644 --- a/packages/sdk/src/core/agents/retry.ts +++ b/packages/sdk/src/core/agents/retry.ts @@ -2,6 +2,7 @@ import type { LLMError } from '~/core/llm/provider.js' import type { Result } from '~/lib/utils/result.js' import { Err } from '~/lib/utils/result.js' import type { Logger } from '../../lib/logger/logger.js' +import { sleep } from '~/lib/utils/sleep.js' // ============================================================================ // Retry Options @@ -26,7 +27,7 @@ export const DEFAULT_RETRY_OPTIONS: Required = { export interface WithRetryOptions extends RetryOptions { isRetryable: (error: E) => boolean getRetryDelay?: (error: E) => number | undefined - /** Error to return when aborted before first attempt */ + /** Error to return when the caller's signal aborts, whichever attempt that happens on. */ abortError?: E logger?: Logger context?: string @@ -51,8 +52,13 @@ export async function withRetry( while (attempt < opts.maxAttempts) { if (options.signal?.aborted) { - const error = lastError ?? options.abortError - if (error !== undefined) { + // abortError wins over lastError: the caller cancelled, and that is what + // the cancellation means regardless of how the previous attempt failed. + // Returning the previous failure instead makes a cancel indistinguishable + // from a genuine one — Agent.runInference then emits inference_failed and + // notifies the parent for what was really a shutdown. + const error = options.abortError ?? lastError + if (error !== null && error !== undefined) { return Err(error) } break @@ -71,12 +77,16 @@ export async function withRetry( } lastError = result.error + attempt++ + + if (options.signal?.aborted) { + return Err(options.abortError ?? lastError) + } - if (!options.isRetryable(lastError)) { + if (attempt >= opts.maxAttempts || !options.isRetryable(lastError)) { return result } - attempt++ const delay = calculateDelay(attempt, lastError, opts, options.getRetryDelay) if (options.logger) { @@ -114,17 +124,6 @@ function calculateDelay( return Math.min(exponentialDelay + jitter, opts.maxDelayMs) } -function sleep(ms: number, signal?: AbortSignal): Promise { - if (signal?.aborted) return Promise.resolve() - return new Promise((resolve) => { - const timer = setTimeout(resolve, ms) - signal?.addEventListener('abort', () => { - clearTimeout(timer) - resolve() - }, { once: true }) - }) -} - // ============================================================================ // LLM-specific helpers // ============================================================================ @@ -158,7 +157,7 @@ export async function withLLMRetry( ...options, isRetryable: isRetryableLLMError, getRetryDelay: getLLMRetryDelay, - abortError: { type: 'aborted', message: 'Aborted before first attempt' }, + abortError: { type: 'aborted', message: 'Request was aborted' }, context: 'LLM inference', }) } diff --git a/packages/sdk/src/core/image/types.ts b/packages/sdk/src/core/image/types.ts index 192c2f43..fc9ee0e1 100644 --- a/packages/sdk/src/core/image/types.ts +++ b/packages/sdk/src/core/image/types.ts @@ -5,6 +5,8 @@ export interface ImageResizeOptions { maxFileSizeBytes?: number /** Override the resizer's default max dimension (long side, px). */ maxDimension?: number + /** Cancels external image-processing commands. */ + signal?: AbortSignal } export interface ImageResizer { diff --git a/packages/sdk/src/core/image/vips-resizer.test.ts b/packages/sdk/src/core/image/vips-resizer.test.ts index 3ffdbc5d..7be10b07 100644 --- a/packages/sdk/src/core/image/vips-resizer.test.ts +++ b/packages/sdk/src/core/image/vips-resizer.test.ts @@ -1,8 +1,9 @@ import { afterEach, describe, expect, it, spyOn } from 'bun:test' import { tmpdir } from 'node:os' -import { VipsImageResizer } from './vips-resizer.js' -import { createNodeFileSystem } from '~/testing/node-platform.js' +import type { FileSystem } from '~/platform/fs.js' import type { ExecFileResult, ProcessRunner } from '~/platform/process.js' +import { createNodeFileSystem } from '~/testing/node-platform.js' +import { VipsImageResizer } from './vips-resizer.js' type ExecFileCallback = (error: Error | null, stdout: string, stderr: string) => void let execFileImpl: (cmd: string, args: string[], opts: unknown, cb: ExecFileCallback) => void = () => {} @@ -26,9 +27,9 @@ function createFakeProcessRunner(): ProcessRunner { } } -function createResizer(maxDimension?: number): InstanceType { +function createResizer(maxDimension?: number, fs: FileSystem = createNodeFileSystem()): InstanceType { return new VipsImageResizer({ - fs: createNodeFileSystem(), + fs, process: createFakeProcessRunner(), tmpDir: tmpdir(), maxDimension, @@ -67,6 +68,21 @@ describe('VipsImageResizer', () => { expect(result).toEqual({ path: testJpegPath, mimeType: 'image/jpeg' }) }) + it('rejects when vipsheader resolves after aborting a within-limit request', async () => { + const controller = new AbortController() + const reason = new Error('cancel dimensions') + execFileImpl = (cmd, _args, _opts, cb) => { + if (cmd === 'vipsheader') { + controller.abort(reason) + cb(null, '4000\n3000\n', '') + } + } + + await expect( + createResizer().resize(testJpegPath, 'image/jpeg', { signal: controller.signal }), + ).rejects.toBe(reason) + }) + it('converts png to jpeg even when within dimension limits', async () => { execFileImpl = (cmd, _args, _opts, cb) => { if (cmd === 'vipsheader') cb(null, '4000\n3000\n', '') @@ -182,6 +198,69 @@ describe('VipsImageResizer', () => { expect(warnSpy).toHaveBeenCalled() }) + it('removes a partial resize output when vipsthumbnail fails', async () => { + let partialPath = '' + execFileImpl = (cmd, args, _opts, cb) => { + if (cmd === 'vipsheader') { + cb(null, '10000\n10000\n', '') + } else if (cmd === 'vipsthumbnail') { + partialPath = args[args.indexOf('-o') + 1] + Bun.write(partialPath, Buffer.from('partial')).then(() => cb(new Error('resize failed'), '', '')) + } + } + + const result = await createResizer().resize(testJpegPath, 'image/png') + + expect(result).toEqual({ path: testJpegPath, mimeType: 'image/png' }) + expect(await Bun.file(partialPath).exists()).toBe(false) + }) + + it('removes a partial resize output and rethrows an abort', async () => { + const controller = new AbortController() + const reason = new Error('cancel resize') + let partialPath = '' + execFileImpl = (cmd, args, _opts, cb) => { + if (cmd === 'vipsheader') { + cb(null, '10000\n10000\n', '') + } else if (cmd === 'vipsthumbnail') { + partialPath = args[args.indexOf('-o') + 1] + Bun.write(partialPath, Buffer.from('partial')).then(() => { + controller.abort(reason) + cb(reason, '', '') + }) + } + } + + await expect( + createResizer().resize(testJpegPath, 'image/png', { + signal: controller.signal, + }), + ).rejects.toBe(reason) + expect(await Bun.file(partialPath).exists()).toBe(false) + }) + + it('removes a generated resize when vipsthumbnail resolves after aborting', async () => { + const controller = new AbortController() + const reason = new Error('cancel successful resize') + let outputPath = '' + execFileImpl = (cmd, args, _opts, cb) => { + if (cmd === 'vipsheader') { + cb(null, '10000\n10000\n', '') + } else if (cmd === 'vipsthumbnail') { + outputPath = args[args.indexOf('-o') + 1] + Bun.write(outputPath, Buffer.from('complete')).then(() => { + controller.abort(reason) + cb(null, '', '') + }) + } + } + + await expect( + createResizer().resize(testJpegPath, 'image/png', { signal: controller.signal }), + ).rejects.toBe(reason) + expect(await Bun.file(outputPath).exists()).toBe(false) + }) + it('returns original path when vipsheader returns unparseable output for jpeg', async () => { execFileImpl = (cmd, _args, _opts, cb) => { if (cmd === 'vipsheader') cb(null, 'not-a-number\n', '') @@ -211,6 +290,71 @@ describe('VipsImageResizer', () => { }) describe('compression (maxFileSizeBytes)', () => { + it('rejects when the size stat resolves after aborting', async () => { + const testPath = '/tmp/test-abort-stat.jpg' + await Bun.write(testPath, Buffer.alloc(100)) + const controller = new AbortController() + const reason = new Error('cancel size check') + const baseFs = createNodeFileSystem() + const fs: FileSystem = { + ...baseFs, + async stat(path) { + const result = await baseFs.stat(path) + controller.abort(reason) + return result + }, + } + execFileImpl = (cmd, _args, _opts, cb) => { + if (cmd === 'vipsheader') cb(null, '4000\n3000\n', '') + } + + await expect( + createResizer(undefined, fs).resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 1000, + signal: controller.signal, + }), + ).rejects.toBe(reason) + + await import('node:fs/promises').then(fs => fs.unlink(testPath).catch(() => {})) + }) + + it('removes compression output when its stat resolves after aborting', async () => { + const testPath = '/tmp/test-abort-compression-stat.jpg' + await Bun.write(testPath, Buffer.alloc(2000)) + const controller = new AbortController() + const reason = new Error('cancel compression stat') + const baseFs = createNodeFileSystem() + let statCount = 0 + let outputPath = '' + const fs: FileSystem = { + ...baseFs, + async stat(path) { + const result = await baseFs.stat(path) + statCount++ + if (statCount === 2) controller.abort(reason) + return result + }, + } + execFileImpl = (cmd, args, _opts, cb) => { + if (cmd === 'vipsheader') { + cb(null, '4000\n3000\n', '') + } else if (cmd === 'vipsthumbnail') { + outputPath = args[args.indexOf('-o') + 1].replace(/\[.*\]$/, '') + Bun.write(outputPath, Buffer.alloc(50)).then(() => cb(null, '', '')) + } + } + + await expect( + createResizer(undefined, fs).resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 100, + signal: controller.signal, + }), + ).rejects.toBe(reason) + expect(await Bun.file(outputPath).exists()).toBe(false) + + await import('node:fs/promises').then(fs => fs.unlink(testPath).catch(() => {})) + }) + it('skips compression when file fits within limit', async () => { const testPath = '/tmp/test-small.jpg' // Write a small file for size check @@ -221,7 +365,9 @@ describe('VipsImageResizer', () => { } const resizer = createResizer() - const result = await resizer.resize(testPath, 'image/jpeg', { maxFileSizeBytes: 1000 }) + const result = await resizer.resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 1000, + }) expect(result).toEqual({ path: testPath, mimeType: 'image/jpeg' }) @@ -247,7 +393,9 @@ describe('VipsImageResizer', () => { } const resizer = createResizer() - const result = await resizer.resize(testPath, 'image/jpeg', { maxFileSizeBytes: 100 }) + const result = await resizer.resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 100, + }) expect(result.mimeType).toBe('image/jpeg') expect(result.tempFile).toBeDefined() @@ -282,7 +430,9 @@ describe('VipsImageResizer', () => { } const resizer = createResizer() - const result = await resizer.resize(testPath, 'image/jpeg', { maxFileSizeBytes: 100 }) + const result = await resizer.resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 100, + }) expect(compressionAttempts).toBe(3) // Q=85, Q=70 failed, Q=50 succeeded expect(result.mimeType).toBe('image/jpeg') @@ -315,7 +465,9 @@ describe('VipsImageResizer', () => { } const resizer = createResizer() - const result = await resizer.resize(testPath, 'image/jpeg', { maxFileSizeBytes: 100 }) + const result = await resizer.resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 100, + }) // 4 full-dim attempts + 3 half-dim attempts expect(compressionAttempts).toBe(7) @@ -346,7 +498,9 @@ describe('VipsImageResizer', () => { } const resizer = createResizer() - const result = await resizer.resize(testPath, 'image/jpeg', { maxFileSizeBytes: 100 }) + const result = await resizer.resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 100, + }) // Returns the last attempt even though it doesn't fit expect(result.mimeType).toBe('image/jpeg') @@ -358,6 +512,41 @@ describe('VipsImageResizer', () => { await import('node:fs/promises').then(fs => fs.unlink(testPath).catch(() => {})) }) + it('removes the current and superseded compression outputs when stat fails', async () => { + const testPath = '/tmp/test-stat-failure.jpg' + await Bun.write(testPath, Buffer.alloc(2000)) + const outputPaths: string[] = [] + + execFileImpl = (cmd, args, _opts, cb) => { + if (cmd === 'vipsheader') { + cb(null, '4000\n3000\n', '') + } else if (cmd === 'vipsthumbnail') { + const outputArg = args[args.indexOf('-o') + 1] + const outputPath = outputArg.replace(/\[.*\]$/, '') + outputPaths.push(outputPath) + if (outputPaths.length === 1) { + Bun.write(outputPath, Buffer.alloc(2000)).then(() => cb(null, '', '')) + } else { + Bun.write(outputPath, Buffer.alloc(2000)).then(async () => { + await import('node:fs/promises').then(fs => fs.unlink(outputPath)) + cb(null, '', '') + }) + } + } + } + + const result = await createResizer().resize(testPath, 'image/jpeg', { + maxFileSizeBytes: 100, + }) + + expect(result).toEqual({ path: testPath, mimeType: 'image/jpeg' }) + expect(outputPaths).toHaveLength(2) + for (const outputPath of outputPaths) { + expect(await Bun.file(outputPath).exists()).toBe(false) + } + await import('node:fs/promises').then(fs => fs.unlink(testPath).catch(() => {})) + }) + it('always outputs jpeg when compressing', async () => { const testPath = '/tmp/test-png-compress.png' await Bun.write(testPath, Buffer.alloc(2000)) @@ -373,7 +562,9 @@ describe('VipsImageResizer', () => { } const resizer = createResizer() - const result = await resizer.resize(testPath, 'image/png', { maxFileSizeBytes: 100 }) + const result = await resizer.resize(testPath, 'image/png', { + maxFileSizeBytes: 100, + }) // Even though input was PNG, compression outputs JPEG expect(result.mimeType).toBe('image/jpeg') diff --git a/packages/sdk/src/core/image/vips-resizer.ts b/packages/sdk/src/core/image/vips-resizer.ts index aeda0e5c..f7d48bba 100644 --- a/packages/sdk/src/core/image/vips-resizer.ts +++ b/packages/sdk/src/core/image/vips-resizer.ts @@ -25,31 +25,59 @@ export class VipsImageResizer implements ImageResizer { async resize(filePath: string, mimeType: string, options?: ImageResizeOptions): Promise { const effectiveMaxDimension = options?.maxDimension ?? this.maxDimension + const signal = options?.signal + let ownedTempFile: string | undefined try { + signal?.throwIfAborted() // Step 1: Dimension resize if needed - const result = await this.dimensionResize(filePath, mimeType, effectiveMaxDimension) + const result = await this.dimensionResize(filePath, mimeType, effectiveMaxDimension, signal) + ownedTempFile = result.tempFile + signal?.throwIfAborted() // Step 2: If no size constraint, done - if (!options?.maxFileSizeBytes) return result + if (!options?.maxFileSizeBytes) { + signal?.throwIfAborted() + return result + } // Step 3: Check if result fits const fileSize = (await this.fs.stat(result.path)).size - if (fileSize <= options.maxFileSizeBytes) return result + signal?.throwIfAborted() + if (fileSize <= options.maxFileSizeBytes) { + signal?.throwIfAborted() + return result + } // Step 4: Compress to fit — clean up dimension resize temp first if (result.tempFile) { await this.fs.unlink(result.tempFile).catch(() => {}) + ownedTempFile = undefined } - return await this.compressToFit(filePath, options.maxFileSizeBytes, effectiveMaxDimension) + signal?.throwIfAborted() + const compressed = await this.compressToFit(filePath, options.maxFileSizeBytes, effectiveMaxDimension, signal) + ownedTempFile = compressed.tempFile + signal?.throwIfAborted() + return compressed } catch (e) { + if (ownedTempFile) { + await this.fs.unlink(ownedTempFile).catch(() => {}) + } + if (signal?.aborted) signal.throwIfAborted() console.warn('[image-resize] failed, using original image:', e instanceof Error ? e.message : e) return { path: filePath, mimeType } } } - private async getImageDimensions(filePath: string): Promise<{ width: number; height: number } | null> { - const { stdout } = await this.process.execFile('vipsheader', ['-f', 'width', '-f', 'height', filePath], { timeout: 30_000 }) + private async getImageDimensions( + filePath: string, + signal?: AbortSignal, + ): Promise<{ width: number; height: number } | null> { + const { stdout } = await this.process.execFile('vipsheader', ['-f', 'width', '-f', 'height', filePath], { + timeout: 30_000, + signal, + }) + signal?.throwIfAborted() const lines = stdout.trim().split('\n') if (lines.length < 2) return null const width = parseInt(lines[0], 10) @@ -58,12 +86,19 @@ export class VipsImageResizer implements ImageResizer { return { width, height } } - private async dimensionResize(filePath: string, mimeType: string, maxDimension: number): Promise { - const dims = await this.getImageDimensions(filePath) + private async dimensionResize( + filePath: string, + mimeType: string, + maxDimension: number, + signal?: AbortSignal, + ): Promise { + const dims = await this.getImageDimensions(filePath, signal) + signal?.throwIfAborted() const needsResize = dims !== null && (dims.width > maxDimension || dims.height > maxDimension) // JPEGs within dimension limits pass through unchanged if (mimeType === 'image/jpeg' && !needsResize) { + signal?.throwIfAborted() return { path: filePath, mimeType } } @@ -71,17 +106,27 @@ export class VipsImageResizer implements ImageResizer { const id = `${Date.now()}-${Math.random().toString(36).slice(2)}` const outputPath = join(this.tmpDir, `roj-resize-${id}.jpg`) - await this.process.execFile('vipsthumbnail', [ - filePath, - '--size', - `${maxDimension}x${maxDimension}`, - '-o', - outputPath, - ], { timeout: 30_000 }) + try { + await this.process.execFile( + 'vipsthumbnail', + [filePath, '--size', `${maxDimension}x${maxDimension}`, '-o', outputPath], + { timeout: 30_000, signal }, + ) + signal?.throwIfAborted() + } catch (error) { + await this.fs.unlink(outputPath).catch(() => {}) + throw error + } + signal?.throwIfAborted() return { path: outputPath, mimeType: 'image/jpeg', tempFile: outputPath } } - private async compressToFit(filePath: string, maxFileSizeBytes: number, maxDimension: number): Promise { + private async compressToFit( + filePath: string, + maxFileSizeBytes: number, + maxDimension: number, + signal?: AbortSignal, + ): Promise { const halfDim = Math.floor(maxDimension / 2) const attempts = [ { dimension: maxDimension, quality: 85 }, @@ -95,31 +140,51 @@ export class VipsImageResizer implements ImageResizer { let lastResult: ImageResizeResult | undefined - for (const { dimension, quality } of attempts) { - const id = `${Date.now()}-${Math.random().toString(36).slice(2)}` - const outputPath = join(this.tmpDir, `roj-compress-${id}.jpg`) - - await this.process.execFile('vipsthumbnail', [ - filePath, - '--size', - `${dimension}x${dimension}`, - '-o', - `${outputPath}[Q=${quality}]`, - ], { timeout: 30_000 }) + try { + for (const { dimension, quality } of attempts) { + signal?.throwIfAborted() + const id = `${Date.now()}-${Math.random().toString(36).slice(2)}` + const outputPath = join(this.tmpDir, `roj-compress-${id}.jpg`) + + try { + await this.process.execFile( + 'vipsthumbnail', + [filePath, '--size', `${dimension}x${dimension}`, '-o', `${outputPath}[Q=${quality}]`], + { timeout: 30_000, signal }, + ) + signal?.throwIfAborted() + + const outputSize = (await this.fs.stat(outputPath)).size + signal?.throwIfAborted() + if (lastResult?.tempFile) { + await this.fs.unlink(lastResult.tempFile).catch(() => {}) + } + signal?.throwIfAborted() + + lastResult = { + path: outputPath, + mimeType: 'image/jpeg', + tempFile: outputPath, + } + if (outputSize <= maxFileSizeBytes) { + signal?.throwIfAborted() + return lastResult + } + } catch (error) { + await this.fs.unlink(outputPath).catch(() => {}) + throw error + } + } - // Clean up previous attempt + // Return best effort (most compressed) — caller decides what to do + if (!lastResult) throw new Error('Image compression produced no result') + signal?.throwIfAborted() + return lastResult + } catch (error) { if (lastResult?.tempFile) { await this.fs.unlink(lastResult.tempFile).catch(() => {}) } - - lastResult = { path: outputPath, mimeType: 'image/jpeg', tempFile: outputPath } - - if ((await this.fs.stat(outputPath)).size <= maxFileSizeBytes) { - return lastResult - } + throw error } - - // Return best effort (most compressed) — caller decides what to do - return lastResult! } } diff --git a/packages/sdk/src/core/llm/anthropic.test.ts b/packages/sdk/src/core/llm/anthropic.test.ts index 3dc5f35a..3940c439 100644 --- a/packages/sdk/src/core/llm/anthropic.test.ts +++ b/packages/sdk/src/core/llm/anthropic.test.ts @@ -1,10 +1,14 @@ import { describe, expect, test } from 'bun:test' import type { LLMMessage } from '~/core/agents/state.js' +import { isRetryableLLMError } from '~/core/agents/retry.js' import { ModelId } from '~/core/llm/schema.js' import { ToolCallId } from '~/core/tools/schema.js' -import { AnthropicProvider } from './anthropic.js' +import { type AnthropicConfig, AnthropicProvider } from './anthropic.js' import { applyCacheBreakpoint } from './cache-breakpoints.js' -import type { RawInferenceRequest } from './provider.js' +import { SessionFileStore } from '~/core/file-store/file-store.js' +import { createNodeFileSystem } from '~/testing/node-platform.js' +import type { InferenceContext, RawInferenceRequest } from './provider.js' +import { LLMMessageFactory, mapProviderError } from './provider.js' // ============================================================================ // Helpers — access private methods via prototype for testing @@ -25,7 +29,6 @@ const mergeConsecutiveMessages = (provider as any).mergeConsecutiveMessages.bind msgs: { role: string; content: unknown }[], ) => { role: string; content: unknown }[] const mapStopReason = (provider as any).mapStopReason.bind(provider) as (reason: string | null) => string -const mapError = (provider as any).mapError.bind(provider) as (err: unknown) => unknown // ============================================================================ // Message Mapping @@ -486,27 +489,198 @@ describe('AnthropicProvider stop reason mapping', () => { // Error Mapping // ============================================================================ -describe('AnthropicProvider error mapping', () => { - test('maps AbortError', () => { +describe('provider error mapping', () => { + const abortError = () => { const err = new Error('aborted') err.name = 'AbortError' - const result = mapError(err) as any - expect(result.type).toBe('aborted') + return err + } + + test('maps a caller-initiated AbortError to aborted', () => { + expect(mapProviderError(abortError()).type).toBe('aborted') + }) + + test('maps the same AbortError to timeout when our timeout fired', () => { + const result = mapProviderError(abortError(), { timedOut: true }) + expect(result.type).toBe('timeout') + expect(isRetryableLLMError(result)).toBe(true) + }) + + test('an aborted request stays non-retryable', () => { + expect(isRetryableLLMError(mapProviderError(abortError()))).toBe(false) }) test('maps unknown error to network_error', () => { - const result = mapError(new Error('something')) as any + const result = mapProviderError(new Error('something')) expect(result.type).toBe('network_error') expect(result.message).toBe('something') }) test('maps string error', () => { - const result = mapError('boom') as any + const result = mapProviderError('boom') expect(result.type).toBe('network_error') expect(result.message).toBe('boom') }) }) +describe('AnthropicProvider request timeout', () => { + type ProviderFetch = NonNullable + + const abortError = () => { + const error = new Error('aborted') + error.name = 'AbortError' + return error + } + + /** Provider whose fetch never settles on its own — only the abort signal ends it. */ + const stallingProvider = (timeout: number, deferAbortRejection = false) => { + let onFetchCalled: () => void + const fetchCalled = new Promise((resolve) => { + onFetchCalled = resolve + }) + let releaseRejection = () => {} + const fetchFn: ProviderFetch = (_input, init) => { + onFetchCalled() + return new Promise((_resolve, reject) => { + const rejectAbort = () => reject(abortError()) + if (init?.signal?.aborted) { + rejectAbort() + return + } + init?.signal?.addEventListener('abort', () => { + if (deferAbortRejection) { + releaseRejection = rejectAbort + } else { + rejectAbort() + } + }, { once: true }) + }) + } + const provider = new AnthropicProvider({ + apiKey: 'test-key', + timeout, + imageProcessor: { resolveContent: async (content) => content }, + fetch: fetchFn, + }) + return { provider, fetchCalled, releaseRejection: () => releaseRejection() } + } + + const bodyStallingProvider = (timeout: number, status: number) => { + let markBodyReadStarted = () => {} + const bodyReadStarted = new Promise((resolve) => { + markBodyReadStarted = resolve + }) + let releaseBodyFailure = () => {} + const fetchFn: ProviderFetch = (_input, init) => { + const bodyResult = new Promise((_resolve, reject) => { + const rejectAbort = () => reject(abortError()) + releaseBodyFailure = () => reject(new Error('body failed')) + if (init?.signal?.aborted) { + rejectAbort() + } else { + init?.signal?.addEventListener('abort', rejectAbort, { once: true }) + } + }) + class StallingResponse extends Response { + override readonly text = (): Promise => { + markBodyReadStarted() + return bodyResult + } + + override readonly json = (): Promise => { + markBodyReadStarted() + return bodyResult + } + } + return Promise.resolve(new StallingResponse(null, { status })) + } + const provider = new AnthropicProvider({ + apiKey: 'test-key', + timeout, + imageProcessor: { resolveContent: async (content) => content }, + fetch: fetchFn, + }) + return { provider, bodyReadStarted, releaseBodyFailure: () => releaseBodyFailure() } + } + + const request = { messages: [LLMMessageFactory.user('hi')], model: ModelId('claude-opus-4-6'), systemPrompt: '' } + // Never touched — a plain-text message resolves no file:// URLs — but the type requires it. + const fileStore = new SessionFileStore('/tmp/roj-anthropic-test', undefined, false, createNodeFileSystem(), 'session') + const contextWith = (signal: AbortSignal): InferenceContext => ({ + sessionId: 'session-1', + agentId: 'agent-1', + signal, + fileStore, + }) + + test('reports a stalled provider as timeout, not aborted', async () => { + const result = await stallingProvider(10).provider.inference(request) + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('timeout') + expect(isRetryableLLMError(result.error)).toBe(true) + }) + + test('reports a caller cancel as aborted', async () => { + const { provider, fetchCalled } = stallingProvider(60_000) + const controller = new AbortController() + const promise = provider.inference(request, contextWith(controller.signal)) + await fetchCalled + controller.abort() + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) + + test('reports an already-aborted caller as aborted', async () => { + const { provider } = stallingProvider(60_000) + const controller = new AbortController() + controller.abort() + const result = await provider.inference(request, contextWith(controller.signal)) + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) + + test('times out while consuming a success response body', async () => { + const { provider, bodyReadStarted } = bodyStallingProvider(10, 200) + const promise = provider.inference(request) + await bodyReadStarted + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('timeout') + }) + + test('caller can cancel while consuming an error response body', async () => { + const { provider, bodyReadStarted, releaseBodyFailure } = bodyStallingProvider(60_000, 429) + const controller = new AbortController() + const promise = provider.inference(request, contextWith(controller.signal)) + await bodyReadStarted + controller.abort() + releaseBodyFailure() + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) + + test('caller cancellation remains the cause when timeout fires before fetch rejects', async () => { + const { provider, fetchCalled, releaseRejection } = stallingProvider(40, true) + const controller = new AbortController() + const promise = provider.inference(request, contextWith(controller.signal)) + await fetchCalled + controller.abort() + await new Promise((resolve) => setTimeout(resolve, 80)) + releaseRejection() + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) +}) + // ============================================================================ // Model Routing (canHandle / normalizeModel) // ============================================================================ diff --git a/packages/sdk/src/core/llm/anthropic.ts b/packages/sdk/src/core/llm/anthropic.ts index ea75ede9..31b18776 100644 --- a/packages/sdk/src/core/llm/anthropic.ts +++ b/packages/sdk/src/core/llm/anthropic.ts @@ -17,7 +17,9 @@ import type { RawInferenceRequest, RawToolSpec, } from './provider.js' -import { ProviderMessageValidationError, sanitizeProviderMessages } from './message-sanitization.js' +import { mapProviderError } from './provider.js' +import { ProviderRequestAbortError, runProviderRequest } from './provider-request.js' +import { sanitizeProviderMessages } from './message-sanitization.js' import type { RoutableLLMProvider } from './routing-provider.js' // ============================================================================ @@ -290,18 +292,8 @@ export class AnthropicProvider implements RoutableLLMProvider { } const httpRequest = await this.buildHttpRequest(rawRequest, context) - - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), this.timeout) - - // Combine with external signal if provided - if (context?.signal) { - context.signal.addEventListener('abort', () => controller.abort(), { once: true }) - } - - let response: Response - try { - response = await this.fetchFn(httpRequest.url, { + return await runProviderRequest({ callerSignal: context?.signal, timeoutMs: this.timeout }, async (signal) => { + const response = await this.fetchFn(httpRequest.url, { method: httpRequest.method, headers: { ...httpRequest.headers, @@ -309,66 +301,70 @@ export class AnthropicProvider implements RoutableLLMProvider { 'anthropic-dangerous-direct-browser-access': 'true', }, body: JSON.stringify(httpRequest.body), - signal: controller.signal, + signal, }) - } finally { - clearTimeout(timeoutId) - } - if (!response.ok) { - const body = await response.text() - return Err(this.mapHttpError(response.status, body)) - } + if (!response.ok) { + const body = await response.text() + return Err(this.mapHttpError(response.status, body)) + } - const data = await response.json() as AnthropicMessageResponse - const latencyMs = Date.now() - startTime - - const textContent = data.content - .filter((block): block is AnthropicTextBlock => block.type === 'text') - .map((block) => block.text) - .join('') - - // Kept whole, in order, signatures included: the API verifies them on replay. - // `redacted_thinking` has no readable text but still has to travel. - const thinkingBlocks = data.content.filter(isThinkingBlock) - - const reasoning = thinkingBlocks - .filter((block): block is AnthropicThinkingBlock => block.type === 'thinking') - .map((block) => block.thinking) - .join('') - - const toolCalls: ToolCall[] = data.content - .filter((block): block is AnthropicToolUseBlock => block.type === 'tool_use') - .map((block) => ({ - id: ToolCallId(block.id), - name: block.name, - input: block.input, - })) - - const metrics: LLMMetrics = { - promptTokens: data.usage.input_tokens, - completionTokens: data.usage.output_tokens, - totalTokens: data.usage.input_tokens + data.usage.output_tokens, - latencyMs, - model: data.model, - provider: this.name, - cost: this.calculateCost(data.model, data.usage), - cachedTokens: data.usage.cache_read_input_tokens || undefined, - cacheWriteTokens: data.usage.cache_creation_input_tokens || undefined, - } + const data = await response.json() as AnthropicMessageResponse + const latencyMs = Date.now() - startTime + + const textContent = data.content + .filter((block): block is AnthropicTextBlock => block.type === 'text') + .map((block) => block.text) + .join('') + + // Kept whole, in order, signatures included: the API verifies them on replay. + // `redacted_thinking` has no readable text but still has to travel. + const thinkingBlocks = data.content.filter(isThinkingBlock) + + const reasoning = thinkingBlocks + .filter((block): block is AnthropicThinkingBlock => block.type === 'thinking') + .map((block) => block.thinking) + .join('') + + const toolCalls: ToolCall[] = data.content + .filter((block): block is AnthropicToolUseBlock => block.type === 'tool_use') + .map((block) => ({ + id: ToolCallId(block.id), + name: block.name, + input: block.input, + })) + + const metrics: LLMMetrics = { + promptTokens: data.usage.input_tokens, + completionTokens: data.usage.output_tokens, + totalTokens: data.usage.input_tokens + data.usage.output_tokens, + latencyMs, + model: data.model, + provider: this.name, + cost: this.calculateCost(data.model, data.usage), + cachedTokens: data.usage.cache_read_input_tokens || undefined, + cacheWriteTokens: data.usage.cache_creation_input_tokens || undefined, + } - return Ok({ - content: textContent || null, - toolCalls, - finishReason: this.mapStopReason(data.stop_reason), - metrics, - providerRequestId: data.id, - reasoning: reasoning || undefined, - // Normalized to undefined so an empty array never reaches the wire on the way back. - thinkingBlocks: thinkingBlocks.length > 0 ? thinkingBlocks : undefined, + return Ok({ + content: textContent || null, + toolCalls, + finishReason: this.mapStopReason(data.stop_reason), + metrics, + providerRequestId: data.id, + reasoning: reasoning || undefined, + // Normalized to undefined so an empty array never reaches the wire on the way back. + thinkingBlocks: thinkingBlocks.length > 0 ? thinkingBlocks : undefined, + }) }) } catch (error) { - return Err(this.mapError(error)) + if (error instanceof ProviderRequestAbortError) { + return Err({ + type: error.abortCause === 'caller' ? 'aborted' : 'timeout', + message: error.message, + }) + } + return Err(mapProviderError(error)) } } @@ -642,16 +638,4 @@ export class AnthropicProvider implements RoutableLLMProvider { return { type: 'server_error', message, statusCode: status, responseBody: body } } - private mapError(err: unknown): LLMError { - if (err instanceof ProviderMessageValidationError) { - return { type: 'invalid_request', message: err.message } - } - if (err instanceof Error && err.name === 'AbortError') { - return { type: 'aborted', message: 'Request was aborted' } - } - if (err instanceof TypeError && (err.message.includes('fetch') || err.message.includes('network'))) { - return { type: 'network_error', message: err.message, cause: err } - } - return { type: 'network_error', message: err instanceof Error ? err.message : String(err), cause: err } - } } diff --git a/packages/sdk/src/core/llm/openrouter.test.ts b/packages/sdk/src/core/llm/openrouter.test.ts index 9190f989..2f3a58ec 100644 --- a/packages/sdk/src/core/llm/openrouter.test.ts +++ b/packages/sdk/src/core/llm/openrouter.test.ts @@ -1,7 +1,11 @@ import { beforeEach, describe, expect, mock, test } from 'bun:test' +import { SessionFileStore } from '~/core/file-store/file-store.js' import type { Logger } from '~/lib/logger/logger.js' import { Err, isErr, isOk, Ok } from '~/lib/utils/result.js' -import type { InferenceRequest, InferenceResponse, LLMError } from './provider.js' +import { createNodeFileSystem } from '~/testing/node-platform.js' +import { type OpenRouterConfig, OpenRouterProvider } from './openrouter.js' +import type { InferenceContext, InferenceRequest, InferenceResponse, LLMError } from './provider.js' +import { LLMMessageFactory } from './provider.js' import { ModelId } from './schema.js' // ============================================================================ @@ -497,3 +501,162 @@ describe('Retry Logic', () => { }) }) }) + +describe('OpenRouterProvider request timeout', () => { + type ProviderFetch = NonNullable + + const abortError = () => { + const error = new Error('aborted') + error.name = 'AbortError' + return error + } + + const stallingProvider = (timeout: number, deferAbortRejection = false) => { + let markFetchCalled = () => {} + const fetchCalled = new Promise((resolve) => { + markFetchCalled = resolve + }) + let releaseRejection = () => {} + const fetchFn: ProviderFetch = (_input, init) => { + markFetchCalled() + return new Promise((_resolve, reject) => { + const rejectAbort = () => reject(abortError()) + if (init?.signal?.aborted) { + rejectAbort() + return + } + init?.signal?.addEventListener('abort', () => { + if (deferAbortRejection) { + releaseRejection = rejectAbort + } else { + rejectAbort() + } + }, { once: true }) + }) + } + const provider = new OpenRouterProvider({ + apiKey: 'test-key', + timeout, + imageProcessor: { resolveContent: async (content) => content }, + fetch: fetchFn, + }) + return { provider, fetchCalled, releaseRejection: () => releaseRejection() } + } + + const bodyStallingProvider = (timeout: number, status: number) => { + let markBodyReadStarted = () => {} + const bodyReadStarted = new Promise((resolve) => { + markBodyReadStarted = resolve + }) + let releaseBodyFailure = () => {} + const fetchFn: ProviderFetch = (_input, init) => { + const bodyResult = new Promise((_resolve, reject) => { + const rejectAbort = () => reject(abortError()) + releaseBodyFailure = () => reject(new Error('body failed')) + if (init?.signal?.aborted) { + rejectAbort() + } else { + init?.signal?.addEventListener('abort', rejectAbort, { once: true }) + } + }) + class StallingResponse extends Response { + override readonly text = (): Promise => { + markBodyReadStarted() + return bodyResult + } + + override readonly json = (): Promise => { + markBodyReadStarted() + return bodyResult + } + } + return Promise.resolve(new StallingResponse(null, { status })) + } + const provider = new OpenRouterProvider({ + apiKey: 'test-key', + timeout, + imageProcessor: { resolveContent: async (content) => content }, + fetch: fetchFn, + }) + return { provider, bodyReadStarted, releaseBodyFailure: () => releaseBodyFailure() } + } + + const request = { + messages: [LLMMessageFactory.user('hi')], + model: ModelId('anthropic/claude-opus-4.6'), + systemPrompt: '', + } + const fileStore = new SessionFileStore('/tmp/roj-openrouter-test', undefined, false, createNodeFileSystem(), 'session') + const contextWith = (signal: AbortSignal): InferenceContext => ({ + sessionId: 'session-1', + agentId: 'agent-1', + signal, + fileStore, + }) + + test('reports a stalled provider as timeout', async () => { + const result = await stallingProvider(10).provider.inference(request) + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('timeout') + }) + + test('reports a caller cancel as aborted', async () => { + const { provider, fetchCalled } = stallingProvider(60_000) + const controller = new AbortController() + const promise = provider.inference(request, contextWith(controller.signal)) + await fetchCalled + controller.abort() + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) + + test('reports an already-aborted caller as aborted', async () => { + const { provider } = stallingProvider(60_000) + const controller = new AbortController() + controller.abort() + const result = await provider.inference(request, contextWith(controller.signal)) + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) + + test('times out while consuming a success response body', async () => { + const { provider, bodyReadStarted } = bodyStallingProvider(10, 200) + const promise = provider.inference(request) + await bodyReadStarted + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('timeout') + }) + + test('caller can cancel while consuming an error response body', async () => { + const { provider, bodyReadStarted, releaseBodyFailure } = bodyStallingProvider(60_000, 429) + const controller = new AbortController() + const promise = provider.inference(request, contextWith(controller.signal)) + await bodyReadStarted + controller.abort() + releaseBodyFailure() + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) + + test('caller cancellation remains the cause when timeout fires before fetch rejects', async () => { + const { provider, fetchCalled, releaseRejection } = stallingProvider(40, true) + const controller = new AbortController() + const promise = provider.inference(request, contextWith(controller.signal)) + await fetchCalled + controller.abort() + await new Promise((resolve) => setTimeout(resolve, 80)) + releaseRejection() + const result = await promise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.type).toBe('aborted') + }) +}) diff --git a/packages/sdk/src/core/llm/openrouter.ts b/packages/sdk/src/core/llm/openrouter.ts index 5816d681..62ddf0d4 100644 --- a/packages/sdk/src/core/llm/openrouter.ts +++ b/packages/sdk/src/core/llm/openrouter.ts @@ -18,7 +18,9 @@ import type { ProviderHttpRequest, RawInferenceRequest, } from './provider.js' -import { ProviderMessageValidationError, sanitizeProviderMessages } from './message-sanitization.js' +import { mapProviderError } from './provider.js' +import { ProviderRequestAbortError, runProviderRequest } from './provider-request.js' +import { sanitizeProviderMessages } from './message-sanitization.js' // ============================================================================ // Configuration @@ -213,79 +215,74 @@ export class OpenRouterProvider implements LLMProvider { } const httpRequest = await this.buildHttpRequest(rawRequest, context) - - const controller = new AbortController() - const timeoutId = setTimeout(() => controller.abort(), this.timeout) - - if (context?.signal) { - context.signal.addEventListener('abort', () => controller.abort(), { once: true }) - } - - let response: Response - try { - response = await this.fetchFn(httpRequest.url, { + return await runProviderRequest({ callerSignal: context?.signal, timeoutMs: this.timeout }, async (signal) => { + const response = await this.fetchFn(httpRequest.url, { method: httpRequest.method, headers: { ...httpRequest.headers, 'Authorization': `Bearer ${this.apiKey}`, }, body: JSON.stringify(httpRequest.body), - signal: controller.signal, + signal, }) - } finally { - clearTimeout(timeoutId) - } - if (!response.ok) { - const body = await response.text() - return Err(this.mapHttpError(response.status, body)) - } + if (!response.ok) { + const body = await response.text() + return Err(this.mapHttpError(response.status, body)) + } - const data = await response.json() as OpenRouterResponse - const latencyMs = Date.now() - startTime + const data = await response.json() as OpenRouterResponse + const latencyMs = Date.now() - startTime - const choice = data.choices[0] - if (!choice) { - return Err({ type: 'server_error', message: 'No choices returned from LLM' }) - } + const choice = data.choices[0] + if (!choice) { + return Err({ type: 'server_error', message: 'No choices returned from LLM' }) + } - const toolCalls: ToolCall[] = (choice.message.tool_calls ?? []).map( - (tc) => ({ - id: ToolCallId(tc.id), - name: tc.function.name, - input: this.parseToolArguments(tc.function.arguments), - }), - ) - - const promptTokens = data.usage?.prompt_tokens ?? 0 - const completionTokens = data.usage?.completion_tokens ?? 0 - const cost = data.usage?.cost - - const metrics: LLMMetrics = { - promptTokens, - completionTokens, - totalTokens: data.usage?.total_tokens ?? 0, - latencyMs, - model: data.model, - provider: this.name, - cost, - cachedTokens: data.usage?.prompt_tokens_details?.cached_tokens || undefined, - cacheWriteTokens: data.usage?.prompt_tokens_details?.cache_write_tokens || undefined, - reasoningTokens: data.usage?.completion_tokens_details?.reasoning_tokens || undefined, - } + const toolCalls: ToolCall[] = (choice.message.tool_calls ?? []).map( + (tc) => ({ + id: ToolCallId(tc.id), + name: tc.function.name, + input: this.parseToolArguments(tc.function.arguments), + }), + ) + + const promptTokens = data.usage?.prompt_tokens ?? 0 + const completionTokens = data.usage?.completion_tokens ?? 0 + const cost = data.usage?.cost + + const metrics: LLMMetrics = { + promptTokens, + completionTokens, + totalTokens: data.usage?.total_tokens ?? 0, + latencyMs, + model: data.model, + provider: this.name, + cost, + cachedTokens: data.usage?.prompt_tokens_details?.cached_tokens || undefined, + cacheWriteTokens: data.usage?.prompt_tokens_details?.cache_write_tokens || undefined, + reasoningTokens: data.usage?.completion_tokens_details?.reasoning_tokens || undefined, + } - return Ok({ - content: this.extractContent(choice.message.content), - toolCalls, - finishReason: this.mapFinishReason(choice.finish_reason), - metrics, - providerRequestId: data.id, - reasoning: choice.message.reasoning || undefined, - // Normalized to undefined here so an empty array never reaches the wire on the way back. - reasoningDetails: choice.message.reasoning_details?.length ? choice.message.reasoning_details : undefined, + return Ok({ + content: this.extractContent(choice.message.content), + toolCalls, + finishReason: this.mapFinishReason(choice.finish_reason), + metrics, + providerRequestId: data.id, + reasoning: choice.message.reasoning || undefined, + // Normalized to undefined here so an empty array never reaches the wire on the way back. + reasoningDetails: choice.message.reasoning_details?.length ? choice.message.reasoning_details : undefined, + }) }) } catch (error) { - return Err(this.mapError(error)) + if (error instanceof ProviderRequestAbortError) { + return Err({ + type: error.abortCause === 'caller' ? 'aborted' : 'timeout', + message: error.message, + }) + } + return Err(mapProviderError(error)) } } @@ -484,16 +481,4 @@ export class OpenRouterProvider implements LLMProvider { return { type: 'server_error', message, statusCode: status, responseBody: body } } - private mapError(err: unknown): LLMError { - if (err instanceof ProviderMessageValidationError) { - return { type: 'invalid_request', message: err.message } - } - if (err instanceof Error && err.name === 'AbortError') { - return { type: 'aborted', message: 'Request was aborted' } - } - if (err instanceof TypeError && (err.message.includes('fetch') || err.message.includes('network'))) { - return { type: 'network_error', message: err.message, cause: err } - } - return { type: 'network_error', message: err instanceof Error ? err.message : String(err), cause: err } - } } diff --git a/packages/sdk/src/core/llm/provider-request.ts b/packages/sdk/src/core/llm/provider-request.ts new file mode 100644 index 00000000..8ed9df34 --- /dev/null +++ b/packages/sdk/src/core/llm/provider-request.ts @@ -0,0 +1,40 @@ +export type ProviderRequestAbortCause = 'caller' | 'timeout' + +export class ProviderRequestAbortError extends Error { + constructor(readonly abortCause: ProviderRequestAbortCause) { + super(abortCause === 'caller' ? 'Request was aborted' : 'Request timed out') + this.name = 'ProviderRequestAbortError' + } +} + +export async function runProviderRequest( + options: { callerSignal?: AbortSignal; timeoutMs: number }, + run: (signal: AbortSignal) => Promise, +): Promise { + if (options.callerSignal?.aborted) throw new ProviderRequestAbortError('caller') + + const controller = new AbortController() + let abortCause: ProviderRequestAbortCause | undefined + const abort = (cause: ProviderRequestAbortCause) => { + if (abortCause) return + abortCause = cause + controller.abort() + } + const abortFromCaller = () => abort('caller') + const timeoutId = setTimeout(() => abort('timeout'), options.timeoutMs) + options.callerSignal?.addEventListener('abort', abortFromCaller, { + once: true, + }) + + try { + const response = await run(controller.signal) + if (abortCause) throw new ProviderRequestAbortError(abortCause) + return response + } catch (error) { + if (abortCause) throw new ProviderRequestAbortError(abortCause) + throw error + } finally { + clearTimeout(timeoutId) + options.callerSignal?.removeEventListener('abort', abortFromCaller) + } +} diff --git a/packages/sdk/src/core/llm/provider.ts b/packages/sdk/src/core/llm/provider.ts index 0be4840c..cb0dc7cf 100644 --- a/packages/sdk/src/core/llm/provider.ts +++ b/packages/sdk/src/core/llm/provider.ts @@ -13,6 +13,7 @@ import type { ToolResultContent } from '~/core/llm/llm-log-types.js' import type { ToolDefinition } from '~/core/tools/definition.js' import type { ToolCallId } from '~/core/tools/schema.js' import type { Result } from '~/lib/utils/result.js' +import { ProviderMessageValidationError } from './message-sanitization.js' import { ModelId } from './schema.js' // Re-export LLMMessage types from agents/state for backwards compatibility @@ -281,3 +282,30 @@ export const LLMMessageFactory = { system: (content: string): SystemLLMMessage => ({ role: 'system', content }), } + +// ============================================================================ +// Error mapping +// ============================================================================ + +/** + * Maps a thrown provider error onto an LLMError. + * + * `timedOut` distinguishes our own request timeout from a caller-initiated + * cancel: both abort the same controller and surface as an indistinguishable + * AbortError, but only the timeout is retryable. Mapping a timeout to 'aborted' + * makes Agent.runInference bail silently and leave the agent stuck 'inferring'. + */ +export function mapProviderError(err: unknown, opts?: { timedOut?: boolean }): LLMError { + if (err instanceof ProviderMessageValidationError) { + return { type: 'invalid_request', message: err.message } + } + if (err instanceof Error && err.name === 'AbortError') { + return opts?.timedOut + ? { type: 'timeout', message: 'Request timed out', cause: err } + : { type: 'aborted', message: 'Request was aborted' } + } + if (err instanceof TypeError && (err.message.includes('fetch') || err.message.includes('network'))) { + return { type: 'network_error', message: err.message, cause: err } + } + return { type: 'network_error', message: err instanceof Error ? err.message : String(err), cause: err } +} diff --git a/packages/sdk/src/core/sessions/context.ts b/packages/sdk/src/core/sessions/context.ts index 7c44346d..a3608342 100644 --- a/packages/sdk/src/core/sessions/context.ts +++ b/packages/sdk/src/core/sessions/context.ts @@ -31,6 +31,8 @@ export type SessionContext = { readonly logger: Logger emitEvent: (event: Omit, 'sessionId'>) => Promise + /** Atomically persist and apply related domain events. */ + emitEvents: (events: Array, 'sessionId'>>) => Promise /** Send a notification to connected clients via transport (ephemeral, not persisted) */ notify: (type: string, payload: unknown) => void } diff --git a/packages/sdk/src/core/sessions/session-disposal.integration.test.ts b/packages/sdk/src/core/sessions/session-disposal.integration.test.ts new file mode 100644 index 00000000..77586ebb --- /dev/null +++ b/packages/sdk/src/core/sessions/session-disposal.integration.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it, spyOn } from 'bun:test' +import { Agent } from '~/core/agents/agent.js' +import { MockLLMProvider } from '~/core/llm/mock.js' +import { definePlugin } from '~/core/plugins/plugin-builder.js' +import { createTestPreset, TestHarness } from '~/testing/index.js' + +const createDeferred = () => { + let resolveDeferred: (() => void) | undefined + const promise = new Promise((resolve) => { + resolveDeferred = resolve + }) + if (!resolveDeferred) { + throw new Error('Deferred resolver was not initialized') + } + return { promise, resolve: resolveDeferred } +} + +const waitForMacrotaskTurn = () => new Promise((resolve) => setImmediate(resolve)) + +const createHarness = ( + systemPlugins: ConstructorParameters[0]['systemPlugins'], + eventStore?: ConstructorParameters[0]['eventStore'], +) => new TestHarness({ + presets: [createTestPreset()], + llmProvider: MockLLMProvider.withFixedResponse({ content: 'Ok', toolCalls: [] }), + systemPlugins, + eventStore, +}) + +describe('session disposal', () => { + it('waits for deferred close hooks during manager shutdown', async () => { + const hookStarted = createDeferred() + const releaseHook = createDeferred() + const plugin = definePlugin('deferred-close') + .sessionHook('onSessionClose', async () => { + hookStarted.resolve() + await releaseHook.promise + }) + .build() + const harness = createHarness([plugin]) + await harness.createSession('test') + + let shutdownFinished = false + const shutdown = harness.sessionManager.shutdown().then(() => { + shutdownFinished = true + }) + + await hookStarted.promise + await Promise.resolve() + expect(shutdownFinished).toBe(false) + + releaseHook.resolve() + await shutdown + expect(shutdownFinished).toBe(true) + + await harness.shutdown() + }) + + it('keeps manager shutdown pending during a concurrent domain close', async () => { + const hookStarted = createDeferred() + const releaseHook = createDeferred() + const plugin = definePlugin('blocked-domain-close') + .sessionHook('onSessionClose', async () => { + hookStarted.resolve() + await releaseHook.promise + }) + .build() + const harness = createHarness([plugin]) + const session = await harness.createSession('test') + + const close = session.close() + await hookStarted.promise + + let shutdownFinished = false + const shutdown = harness.sessionManager.shutdown().then(() => { + shutdownFinished = true + }) + await waitForMacrotaskTurn() + + try { + expect(shutdownFinished).toBe(false) + } finally { + releaseHook.resolve() + await Promise.all([close, shutdown]) + await harness.shutdown() + } + + expect(shutdownFinished).toBe(true) + }) + + it('shares concurrent disposal and cleans hooks and agents once', async () => { + let hookCalls = 0 + const plugin = definePlugin('counted-close') + .sessionHook('onSessionClose', async () => { + hookCalls++ + }) + .build() + const harness = createHarness([plugin]) + const testSession = await harness.createSession('test') + const sessionResult = await harness.sessionManager.getSession(testSession.sessionId) + if (!sessionResult.ok) { + throw new Error(`Failed to get session: ${sessionResult.error.message}`) + } + + const shutdownSpy = spyOn(Agent.prototype, 'shutdown') + shutdownSpy.mockClear() + try { + await Promise.all([ + sessionResult.value.dispose(), + sessionResult.value.dispose(), + ]) + + expect(hookCalls).toBe(1) + expect(shutdownSpy).toHaveBeenCalledTimes(1) + expect(sessionResult.value.getEntryAgent()).toBeNull() + } finally { + shutdownSpy.mockRestore() + await harness.shutdown() + } + }) + + it('keeps a gracefully disposed session persisted and reloadable', async () => { + const firstHarness = createHarness([]) + const testSession = await firstHarness.createSession('test') + const sessionId = testSession.sessionId + const eventStore = firstHarness.eventStore + + await firstHarness.shutdown() + + const events = await eventStore.load(sessionId) + expect(events.some((event) => event.type === 'session_closed')).toBe(false) + + const restartedHarness = createHarness([], eventStore) + try { + const reopened = await restartedHarness.openSession(sessionId) + expect(reopened.state.status).toBe('active') + } finally { + await restartedHarness.shutdown() + } + }) + + it('continues cleanup after a hook failure and does not repeat domain-close cleanup', async () => { + const calls: string[] = [] + const laterPlugin = definePlugin('later-close') + .sessionHook('onSessionClose', async () => { + calls.push('later') + }) + .build() + const failingPlugin = definePlugin('failing-close') + .sessionHook('onSessionClose', async () => { + calls.push('failing') + throw new Error('close failed') + }) + .build() + const harness = createHarness([laterPlugin, failingPlugin]) + const testSession = await harness.createSession('test') + const sessionResult = await harness.sessionManager.getSession(testSession.sessionId) + if (!sessionResult.ok) { + throw new Error(`Failed to get session: ${sessionResult.error.message}`) + } + + await testSession.close() + + expect(calls).toEqual(['failing', 'later']) + expect(sessionResult.value.getEntryAgent()).toBeNull() + + await harness.shutdown() + expect(calls).toEqual(['failing', 'later']) + }) +}) diff --git a/packages/sdk/src/core/sessions/session-manager.ts b/packages/sdk/src/core/sessions/session-manager.ts index ce798690..d61f2c76 100644 --- a/packages/sdk/src/core/sessions/session-manager.ts +++ b/packages/sdk/src/core/sessions/session-manager.ts @@ -28,6 +28,7 @@ import type { SessionCreatedEvent, SessionOverridesPatch } from '~/core/sessions import { checkRecoveryNeeded, isSessionCreatedEvent, reconstructSessionState, sessionEvents } from '~/core/sessions/state.js' import type { ToolExecutor } from '~/core/tools' import { FileLogger } from '~/lib/logger/file.js' +import type { ArchiveLimitOverrides } from '~/lib/archive/index.js' import type { Logger } from '~/lib/logger/logger.js' import { TeeLogger } from '~/lib/logger/tee.js' import type { Platform } from '~/platform/index.js' @@ -77,6 +78,8 @@ export interface SessionManagerOptions { dataFileStore: FileStore onUserOutput?: UserOutputCallback preprocessorRegistry?: PreprocessorRegistry + uploadArchiveLimits?: ArchiveLimitOverrides + resourceArchiveLimits?: ArchiveLimitOverrides llmLogger?: LLMLogger portPool?: PortPool pidRegistry?: ServicePidRegistry @@ -109,6 +112,8 @@ export class SessionManager { private readonly dataFileStore: FileStore private readonly onUserOutput?: UserOutputCallback private readonly preprocessorRegistry?: PreprocessorRegistry + private readonly uploadArchiveLimits?: ArchiveLimitOverrides + private readonly resourceArchiveLimits?: ArchiveLimitOverrides private readonly llmLogger?: LLMLogger private readonly portPool?: PortPool private readonly pidRegistry?: ServicePidRegistry @@ -126,6 +131,8 @@ export class SessionManager { this.dataFileStore = options.dataFileStore this.onUserOutput = options.onUserOutput this.preprocessorRegistry = options.preprocessorRegistry + this.uploadArchiveLimits = options.uploadArchiveLimits + this.resourceArchiveLimits = options.resourceArchiveLimits this.llmLogger = options.llmLogger this.portPool = options.portPool this.pidRegistry = options.pidRegistry @@ -661,11 +668,18 @@ export class SessionManager { Array.from(this.sessions.values()).map((p) => p.catch(() => null)), ) - for (const session of sessions) { - if (session) { - session.shutdown() + await Promise.all(sessions.map(async (session) => { + if (!session) return + try { + await session.dispose() + } catch (error) { + this.logger.error( + 'Failed to dispose session during shutdown', + error instanceof Error ? error : new Error(String(error)), + { sessionId: session.id }, + ) } - } + })) this.sessions.clear() for (const cleanup of this.sessionListenerCleanup.values()) { cleanup() @@ -786,8 +800,13 @@ export class SessionManager { configs.set('uploads', { dataFileStore: this.dataFileStore, preprocessorRegistry: this.preprocessorRegistry, + archiveLimits: this.uploadArchiveLimits, }) + if (this.resourceArchiveLimits) { + configs.set('resources', { archiveLimits: this.resourceArchiveLimits }) + } + // services plugin — collect services from all agent definitions const allAgentConfigs = [preset.orchestrator, ...(preset.communicator ? [preset.communicator] : []), ...preset.agents] const servicesByType = new Map() @@ -818,11 +837,6 @@ export class SessionManager { plugins: ConfiguredPlugin[], opts: { skipReadyHooks?: boolean } = {}, ): Promise { - // Only register cache eviction listener for active sessions (not closed) - if (store.getState().status !== 'closed') { - this.registerSessionEventListener(store.sessionId, store) - } - const sessionDir = this.getSessionDir(store.sessionId) const sessionLogger = new TeeLogger([ this.logger.child({ sessionId: store.sessionId }), @@ -845,6 +859,11 @@ export class SessionManager { platform: this.platform, }) + // Only register cache eviction listener for active sessions (not closed) + if (store.getState().status !== 'closed') { + this.registerSessionEventListener(store.sessionId, store, session) + } + // Ensure session and workspace directories exist before plugins run await this.platform.fs.mkdir(sessionDir, { recursive: true }) const workspaceDir = store.getState().workspaceDir @@ -896,14 +915,29 @@ export class SessionManager { * Cleans up any previous listener for this sessionId (from a prior load) * to prevent duplicate listeners firing on old stores. */ - private registerSessionEventListener(sessionId: SessionId, store: SessionStore): void { + private registerSessionEventListener(sessionId: SessionId, store: SessionStore, session: Session): void { const prevCleanup = this.sessionListenerCleanup.get(sessionId) if (prevCleanup) prevCleanup() + const evict = () => { + if (this.sessionListenerCleanup.get(sessionId) !== unsubscribe) return + this.sessions.delete(sessionId) + unsubscribe() + this.sessionListenerCleanup.delete(sessionId) + } + const unsubscribe = store.onEvent((event) => { - if (event.type === 'session_closed' || event.type === 'session_reopened') { - this.sessions.delete(sessionId) - this.sessionListenerCleanup.delete(sessionId) + if (event.type === 'session_closed') { + session.dispose().then(evict, (error) => { + this.logger.error( + 'Failed to dispose closed session', + error instanceof Error ? error : new Error(String(error)), + { sessionId }, + ) + evict() + }) + } else if (event.type === 'session_reopened') { + evict() } }) this.sessionListenerCleanup.set(sessionId, unsubscribe) diff --git a/packages/sdk/src/core/sessions/session.ts b/packages/sdk/src/core/sessions/session.ts index f4ab3708..5e6bac78 100644 --- a/packages/sdk/src/core/sessions/session.ts +++ b/packages/sdk/src/core/sessions/session.ts @@ -100,6 +100,7 @@ export class Session { private readonly agents = new Map() /** Cached plugin contexts created by plugin.createContext() */ private readonly pluginContexts = new Map() + private disposalPromise?: Promise constructor(deps: SessionDependencies) { this.id = deps.store.sessionId @@ -189,7 +190,7 @@ export class Session { /** * Close the session. - * Emits session_closed event — hooks and agent shutdown are handled reactively by handleSessionClosed(). + * Emits session_closed, then awaits the shared runtime disposal path. */ async close(): Promise> { if (this.store.isClosed()) { @@ -197,6 +198,7 @@ export class Session { } await this.store.emit(withSessionId(this.id, sessionEvents.create('session_closed', {}))) + await this.dispose() return Ok(undefined) } @@ -425,16 +427,11 @@ export class Session { } /** - * Shutdown the session - stop all agent processing. + * Dispose runtime resources without changing persisted session state. */ - shutdown(): void { - for (const agent of this.agents.values()) { - try { - agent.shutdown() - } catch { - // Suppress errors during shutdown - } - } + dispose(): Promise { + this.disposalPromise ??= Promise.resolve().then(() => this.performDisposal()) + return this.disposalPromise } /** @@ -601,6 +598,9 @@ export class Session { emitEvent: async (event) => { await this.store.emit(withSessionId(this.id, event)) }, + emitEvents: async (events) => { + await this.store.emitBatch(events.map((event) => withSessionId(this.id, event))) + }, notify: (type, payload) => { this.onUserOutput?.({ pluginName: '_agent', type, payload }) }, @@ -634,8 +634,8 @@ export class Session { break } case 'session_closed': { - this.handleSessionClosed().catch((err) => { - this.logger.error('Unhandled error in handleSessionClosed()', err instanceof Error ? err : undefined, { sessionId: this.id }) + this.dispose().catch((err) => { + this.logger.error('Unhandled error in session disposal', err instanceof Error ? err : undefined, { sessionId: this.id }) }) break } @@ -643,13 +643,13 @@ export class Session { } /** - * Handle session_closed event — call close hooks and shutdown agents. + * Call close hooks and release all in-memory runtime state. * * Unlike onSessionReady (which re-throws), onSessionClose intentionally * swallows per-plugin errors so that all plugins get a chance to clean up * and agents are always shut down, even if one plugin's close hook fails. */ - private async handleSessionClosed(): Promise { + private async performDisposal(): Promise { // Call onSessionClose for all plugins in REVERSE order (per-plugin isolation) const reversedPlugins = [...this.plugins].reverse() for (const plugin of reversedPlugins) { @@ -680,7 +680,7 @@ export class Session { this.pluginContexts.clear() this.store.clearListeners() - this.logger.info('Session closed', { sessionId: this.id }) + this.logger.info('Session runtime disposed', { sessionId: this.id }) } /** @@ -753,6 +753,9 @@ export class Session { emitEvent: async (event) => { await this.store.emit(withSessionId(this.id, event)) }, + emitEvents: async (events) => { + await this.store.emitBatch(events.map((event) => withSessionId(this.id, event))) + }, notify: (type, payload) => { this.onUserOutput?.({ pluginName: '_session', type, payload }) }, diff --git a/packages/sdk/src/core/system.ts b/packages/sdk/src/core/system.ts index 12fca50f..7edca13e 100644 --- a/packages/sdk/src/core/system.ts +++ b/packages/sdk/src/core/system.ts @@ -9,6 +9,7 @@ import type z4 from 'zod/v4' import type { Logger } from '~/lib/logger/logger.js' +import type { ArchiveLimitOverrides } from '~/lib/archive/index.js' import type { Platform } from '~/platform/index.js' import type { ServicePidRegistry } from '~/plugins/services/pid-registry.js' import type { PortPool } from '~/plugins/services/port-pool.js' @@ -119,6 +120,8 @@ export interface CreateSystemOptions { platform: createNodePlatform(), logger: silentLogger, emitEvent: async () => {}, + emitEvents: async () => {}, notify: () => {}, agentId, agentState, diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 45630ffb..5f2f2797 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -67,6 +67,10 @@ export { applyEvent } from '~/core/sessions/apply-event.js' export { selectPluginState } from '~/core/sessions/reducer.js' export { SessionId } from '~/core/sessions/schema.js' export type { SessionMetadata } from '~/core/sessions/schema.js' +// Exported so a consumer can validate a callManagerMethod('sessions.list') +// result — that call is typed Result, so the plugin's output schema +// does not reach the caller. +export { sessionMetadataSchema } from '~/core/sessions/schema.js' export type { AgentOverrides, SessionOverrides, SessionOverridesPatch } from '~/core/sessions/state.js' export { agentOverridesSchema, @@ -129,6 +133,7 @@ export type { export { RESOURCE_MANIFEST_FILENAME, ResourceManifestSchema } from '~/plugins/resources/manifest.js' export type { ResourceManifest } from '~/plugins/resources/manifest.js' export { uploadsPlugin } from '~/plugins/uploads/plugin.js' +export type { ArchiveLimitOverrides, ArchiveLimits } from '~/lib/archive/index.js' export { userChatPlugin } from '~/plugins/user-chat/plugin.js' export type { UserChatAgentConfig, UserChatPresetConfig, UserCommunicationMode } from '~/plugins/user-chat/plugin.js' diff --git a/packages/sdk/src/lib/archive/archive-inspection.fixtures.ts b/packages/sdk/src/lib/archive/archive-inspection.fixtures.ts new file mode 100644 index 00000000..82a60e98 --- /dev/null +++ b/packages/sdk/src/lib/archive/archive-inspection.fixtures.ts @@ -0,0 +1,138 @@ +// Captured from Info-ZIP 6.00; unrelated verbose fields were omitted. +export const SAFE_INFO_ZIP_6_FIXTURE = `Archive: fixture.zip +There is no zipfile comment. + +End-of-central-directory record: +------------------------------- + + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains 3 entries. + +Central directory entry #1: +--------------------------- + + regular.txt + + file system or operating system of origin: Unix + uncompressed size: 1 bytes + length of filename: 11 characters + Unix file attributes (100664 octal): -rw-rw-r-- + MS-DOS file attributes (00 hex): none + +Central directory entry #2: +--------------------------- + + space ž.txt + + file system or operating system of origin: Unix + uncompressed size: 7 bytes + length of filename: 12 characters + Unix file attributes (100664 octal): -rw-rw-r-- + MS-DOS file attributes (00 hex): none + +Central directory entry #3: +--------------------------- + + dir/ + + file system or operating system of origin: Unix + uncompressed size: 0 bytes + length of filename: 4 characters + Unix file attributes (040775 octal): drwxrwxr-x + MS-DOS file attributes (10 hex): dir +` + +export const BACKSLASH_INFO_ZIP_6_FIXTURE = `Archive: backslash.zip +There is no zipfile comment. + + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains 1 entry. + +Central directory entry #1: +--------------------------- + + trailing\\ + + file system or operating system of origin: Unix + uncompressed size: 5 bytes + length of filename: 9 characters + Unix file attributes (100664 octal): -rw-rw-r-- + MS-DOS file attributes (00 hex): none +` + +export const SYMLINK_INFO_ZIP_6_FIXTURE = `Archive: symlink.zip +There is no zipfile comment. + + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains 1 entry. + +Central directory entry #1: +--------------------------- + + dir/link + + file system or operating system of origin: Unix + uncompressed size: 14 bytes + length of filename: 8 characters + Unix file attributes (120777 octal): lrwxrwxrwx + MS-DOS file attributes (00 hex): none +` + +export const DOS_INFO_ZIP_6_FIXTURE = `Archive: windows.zip +There is no zipfile comment. + + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains 2 entries. + +Central directory entry #1: +--------------------------- + + win-file.txt + + file system or operating system of origin: MS-DOS, OS/2 or NT FAT + uncompressed size: 1 bytes + length of filename: 12 characters + non-MSDOS external file attributes: 000000 hex + MS-DOS file attributes (20 hex): arc + +Central directory entry #2: +--------------------------- + + win-dir/ + + file system or operating system of origin: MS-DOS, OS/2 or NT FAT + uncompressed size: 0 bytes + length of filename: 8 characters + non-MSDOS external file attributes: 000000 hex + MS-DOS file attributes (10 hex): dir +` + +export const EMPTY_INFO_ZIP_6_FIXTURE = `Archive: empty.zip +There is no zipfile comment. + +End-of-central-directory record: +------------------------------- + + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains 0 entries. + The central directory is 0 (0000000000000000h) bytes long, + and its (expected) offset in bytes from the beginning of the zipfile + is 0 (0000000000000000h). + + Empty zipfile. +` + +export const TRUNCATED_INFO_ZIP_6_FIXTURE = `Archive: truncated.zip +There is no zipfile comment. + + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains 1 entry. + +Central directory entry #1: +--------------------------- + + truncated.txt + + file system or operating system of origin: Unix + uncompressed size: 1 bytes +` diff --git a/packages/sdk/src/lib/archive/archive-inspection.test.ts b/packages/sdk/src/lib/archive/archive-inspection.test.ts new file mode 100644 index 00000000..d2667921 --- /dev/null +++ b/packages/sdk/src/lib/archive/archive-inspection.test.ts @@ -0,0 +1,281 @@ +import { describe, expect, test } from 'bun:test' +import type { ProcessRunner } from '~/platform/process.js' +import { + ArchiveBudget, + DEFAULT_ARCHIVE_LIMITS, + inspectZipArchive, + parseZipInfoVerbose, + validateArchiveEntries, + type ArchiveEntry, +} from './archive-inspection.js' +import { + BACKSLASH_INFO_ZIP_6_FIXTURE, + DOS_INFO_ZIP_6_FIXTURE, + EMPTY_INFO_ZIP_6_FIXTURE, + SAFE_INFO_ZIP_6_FIXTURE, + SYMLINK_INFO_ZIP_6_FIXTURE, + TRUNCATED_INFO_ZIP_6_FIXTURE, +} from './archive-inspection.fixtures.js' + +const MEBIBYTE = 1024 * 1024 + +function file(name: string, uncompressedSize = 0): ArchiveEntry { + return { name, uncompressedSize, type: 'file' } +} + +function directory(name: string): ArchiveEntry { + return { name, uncompressedSize: 0, type: 'directory' } +} + +describe('validateArchiveEntries', () => { + test('accepts the exact entry count and total size boundaries', () => { + const entries = Array.from({ length: 500 }, (_, index) => file(`nested/file-${index}.txt`, index === 0 ? 100 * MEBIBYTE : 0)) + + const result = validateArchiveEntries(entries) + + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value.entries).toHaveLength(DEFAULT_ARCHIVE_LIMITS.maxEntries) + expect(result.value.totalUncompressedSize).toBe(DEFAULT_ARCHIVE_LIMITS.maxTotalUncompressedSize) + }) + + test('rejects 501 entries even when they are directories', () => { + const result = validateArchiveEntries(Array.from({ length: 501 }, (_, index) => directory(`dir-${index}/`))) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('too_many_entries') + }) + + test('rejects one byte over the total size limit', () => { + const result = validateArchiveEntries([file('large.bin', 100 * MEBIBYTE + 1)]) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('too_large') + }) + + test('accepts nested files and counts directories separately', () => { + const result = validateArchiveEntries([ + directory('nested/'), + directory('nested/deeper/'), + file('nested/deeper/file.txt', 12), + ]) + + expect(result).toEqual({ + ok: true, + value: { + entries: [directory('nested/'), directory('nested/deeper/'), file('nested/deeper/file.txt', 12)], + fileCount: 1, + directoryCount: 2, + totalUncompressedSize: 12, + }, + }) + }) + + test('rejects an empty archive', () => { + const result = validateArchiveEntries([]) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('empty_archive') + }) + + test.each([ + ['directory without slash', { name: 'dir', uncompressedSize: 0, type: 'directory' }], + ['non-empty directory', { name: 'dir/', uncompressedSize: 1, type: 'directory' }], + ['regular file with slash', { name: 'file/', uncompressedSize: 0, type: 'file' }], + ] satisfies ReadonlyArray)('rejects inconsistent %s', (_label, entry) => { + const result = validateArchiveEntries([entry]) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('unsupported_entry_type') + }) + + test.each([ + ['absolute POSIX path', '/etc/passwd'], + ['absolute platform path', '\\server\\share'], + ['drive path', 'C:\\temp\\file'], + ['parent traversal', 'safe/../outside'], + ['platform-separator traversal', 'safe\\..\\outside'], + ['NUL byte', 'safe\0outside'], + ])('rejects %s', (_label, name) => { + const result = validateArchiveEntries([file(name)]) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('unsafe_path') + expect(result.error.entryName).toBe(name) + }) +}) + +describe('ArchiveBudget', () => { + test('enforces aggregate entry and expanded-size limits across inspections', () => { + const budget = new ArchiveBudget({ maxEntries: 3, maxTotalUncompressedSize: 10 }) + const first = validateArchiveEntries([file('nested.zip', 4)], budget.limits) + const second = validateArchiveEntries([file('one.txt', 3), file('two.txt', 4)], budget.limits) + if (!first.ok || !second.ok) throw new Error('Expected fixtures to fit their per-archive limits') + + expect(budget.consume(first.value).ok).toBe(true) + const result = budget.consume(second.value) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('too_large') + expect(result.error.message).toContain('aggregate') + }) + + test('merges partial overrides with the default limits', () => { + const budget = new ArchiveBudget({ maxEntries: 7 }) + + expect(budget.limits).toEqual({ + maxEntries: 7, + maxTotalUncompressedSize: DEFAULT_ARCHIVE_LIMITS.maxTotalUncompressedSize, + }) + }) +}) + +describe('parseZipInfoVerbose', () => { + test('parses captured regular, Unicode/space, and directory entries', () => { + expect(parseZipInfoVerbose(SAFE_INFO_ZIP_6_FIXTURE)).toEqual({ + ok: true, + value: [file('regular.txt', 1), file('space ž.txt', 7), directory('dir/')], + }) + }) + + test('parses captured MS-DOS regular and directory attributes', () => { + expect(parseZipInfoVerbose(DOS_INFO_ZIP_6_FIXTURE)).toEqual({ + ok: true, + value: [file('win-file.txt', 1), directory('win-dir/')], + }) + }) + + test('reports captured truncated CLI output explicitly', () => { + const result = parseZipInfoVerbose(TRUNCATED_INFO_ZIP_6_FIXTURE) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('invalid_listing') + }) + + test('rejects a captured trailing-backslash regular file during validation', () => { + const parsed = parseZipInfoVerbose(BACKSLASH_INFO_ZIP_6_FIXTURE) + expect(parsed.ok).toBe(true) + if (!parsed.ok) return + + const result = validateArchiveEntries(parsed.value) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('unsafe_path') + }) + + test('rejects a captured symlink entry', () => { + const result = parseZipInfoVerbose(SYMLINK_INFO_ZIP_6_FIXTURE) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('unsupported_entry_type') + }) + + test.each([ + ['FIFO', '010644'], + ['character device', '020666'], + ['block device', '060660'], + ['socket', '140777'], + ])('rejects a Unix %s entry', (_label, mode) => { + const result = parseZipInfoVerbose(SYMLINK_INFO_ZIP_6_FIXTURE.replace('120777', mode)) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('unsupported_entry_type') + }) + + test('rejects a captured empty ZIP', () => { + const result = parseZipInfoVerbose(EMPTY_INFO_ZIP_6_FIXTURE) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('empty_archive') + }) +}) + +describe('inspectZipArchive', () => { + test('uses unzip listing mode and preserves timeout and cancellation', async () => { + const controller = new AbortController() + let receivedSignal: AbortSignal | undefined + const process: ProcessRunner = { + async execFile(command, args, options) { + expect(command).toBe('unzip') + expect(args).toEqual(['-Z', '-v', '/archive.zip']) + expect(options?.timeout).toBe(1234) + receivedSignal = options?.signal + return { stdout: SAFE_INFO_ZIP_6_FIXTURE, stderr: '' } + }, + spawn() { + throw new Error('not used') + }, + } + + const result = await inspectZipArchive(process, '/archive.zip', { signal: controller.signal, timeoutMs: 1234 }) + + expect(result.ok).toBe(true) + expect(receivedSignal).toBe(controller.signal) + }) + + test('applies partial caller limits before extraction', async () => { + const process: ProcessRunner = { + async execFile() { + return { stdout: SAFE_INFO_ZIP_6_FIXTURE, stderr: '' } + }, + spawn() { + throw new Error('not used') + }, + } + + const result = await inspectZipArchive(process, '/archive.zip', { limits: { maxEntries: 2 } }) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('too_many_entries') + }) + + test('fails closed when Info-ZIP reports a non-zero warning exit', async () => { + const process: ProcessRunner = { + async execFile() { + throw new Error('unzip exited with code 1 after warnings') + }, + spawn() { + throw new Error('not used') + }, + } + + const result = await inspectZipArchive(process, '/archive.zip') + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('command_failed') + }) + + test('propagates abort through ProcessRunner and reports it distinctly', async () => { + const controller = new AbortController() + const process: ProcessRunner = { + async execFile(_command, _args, options) { + expect(options?.signal).toBe(controller.signal) + controller.abort(new Error('cancelled')) + throw new Error('process aborted') + }, + spawn() { + throw new Error('not used') + }, + } + + const result = await inspectZipArchive(process, '/archive.zip', { signal: controller.signal }) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.code).toBe('aborted') + expect(result.error.cause).toBe(controller.signal.reason) + }) +}) diff --git a/packages/sdk/src/lib/archive/archive-inspection.ts b/packages/sdk/src/lib/archive/archive-inspection.ts new file mode 100644 index 00000000..3d34d3a6 --- /dev/null +++ b/packages/sdk/src/lib/archive/archive-inspection.ts @@ -0,0 +1,359 @@ +import type { Result } from '~/lib/utils/result.js' +import { Err, Ok } from '~/lib/utils/result.js' +import type { ProcessRunner } from '~/platform/process.js' + +const MEBIBYTE = 1024 * 1024 +const ZIPINFO_TIMEOUT_MS = 60_000 +const ZIPINFO_MAX_BUFFER = 10 * MEBIBYTE + +export interface ArchiveLimits { + maxEntries: number + maxTotalUncompressedSize: number +} + +export type ArchiveLimitOverrides = Partial + +export const DEFAULT_ARCHIVE_LIMITS: Readonly = { + maxEntries: 500, + maxTotalUncompressedSize: 100 * MEBIBYTE, +} + +export interface ArchiveEntry { + name: string + uncompressedSize: number + type: 'file' | 'directory' +} + +export interface ArchiveInspection { + entries: readonly ArchiveEntry[] + fileCount: number + directoryCount: number + totalUncompressedSize: number +} + +export type ArchiveInspectionErrorCode = + | 'aborted' + | 'command_failed' + | 'empty_archive' + | 'invalid_listing' + | 'unsafe_path' + | 'too_many_entries' + | 'too_large' + | 'unsupported_entry_type' + +export class ArchiveInspectionError extends Error { + readonly code: ArchiveInspectionErrorCode + readonly entryName?: string + + constructor(code: ArchiveInspectionErrorCode, message: string, options?: { cause?: unknown; entryName?: string }) { + super(message, options?.cause === undefined ? undefined : { cause: options.cause }) + this.name = 'ArchiveInspectionError' + this.code = code + this.entryName = options?.entryName + } +} + +export interface InspectZipArchiveOptions { + signal?: AbortSignal + timeoutMs?: number + limits?: ArchiveLimitOverrides +} + +export function resolveArchiveLimits(overrides: ArchiveLimitOverrides = {}): Readonly { + return { ...DEFAULT_ARCHIVE_LIMITS, ...overrides } +} + +/** Tracks one aggregate extraction budget across a top-level archive and every nested archive. */ +export class ArchiveBudget { + readonly limits: Readonly + private consumedEntries = 0 + private consumedUncompressedSize = 0 + + constructor(limits: ArchiveLimitOverrides = {}) { + this.limits = resolveArchiveLimits(limits) + } + + consume(inspection: ArchiveInspection): Result { + if (!isValidLimit(this.limits.maxEntries) || !isValidLimit(this.limits.maxTotalUncompressedSize)) { + return invalidListing('Archive limits must be non-negative safe integers') + } + + const nextEntries = this.consumedEntries + inspection.entries.length + if (!Number.isSafeInteger(nextEntries) || nextEntries > this.limits.maxEntries) { + return Err(new ArchiveInspectionError( + 'too_many_entries', + `Nested ZIP archives exceed the aggregate ${this.limits.maxEntries} entry limit`, + )) + } + + const nextSize = this.consumedUncompressedSize + inspection.totalUncompressedSize + if (!Number.isSafeInteger(nextSize) || nextSize > this.limits.maxTotalUncompressedSize) { + return Err(new ArchiveInspectionError( + 'too_large', + `Nested ZIP archives exceed the aggregate ${this.limits.maxTotalUncompressedSize} byte uncompressed size limit`, + )) + } + + this.consumedEntries = nextEntries + this.consumedUncompressedSize = nextSize + return Ok(undefined) + } +} + +/** Inspect the central directory before a caller performs extraction. */ +export async function inspectZipArchive( + process: ProcessRunner, + archivePath: string, + options: InspectZipArchiveOptions = {}, +): Promise> { + if (options.signal?.aborted) { + return Err(abortedError(options.signal)) + } + + let stdout: string + try { + const result = await process.execFile('unzip', ['-Z', '-v', archivePath], { + timeout: options.timeoutMs ?? ZIPINFO_TIMEOUT_MS, + maxBuffer: ZIPINFO_MAX_BUFFER, + signal: options.signal, + }) + stdout = result.stdout + } catch (cause) { + if (options.signal?.aborted) { + return Err(abortedError(options.signal)) + } + // Info-ZIP uses non-zero exits for warnings; inspection fails closed on all of them. + return Err(new ArchiveInspectionError('command_failed', 'Failed to inspect ZIP archive', { cause })) + } + + const parsed = parseZipInfoVerbose(stdout) + if (!parsed.ok) return parsed + return validateArchiveEntries(parsed.value, resolveArchiveLimits(options.limits)) +} + +/** Parse the stable fields emitted by Info-ZIP's verbose central-directory listing. */ +export function parseZipInfoVerbose(output: string): Result { + const normalized = output.replaceAll('\r\n', '\n') + const countMatches = [...normalized.matchAll(/central directory contains (\d+) entr(?:y|ies)\./g)] + if (countMatches.length !== 1) { + return invalidListing('ZIP listing does not contain one central-directory entry count') + } + + const expectedCountText = countMatches[0]?.[1] + const expectedCount = expectedCountText === undefined ? Number.NaN : Number(expectedCountText) + if (!Number.isSafeInteger(expectedCount) || expectedCount < 0) { + return invalidListing('ZIP listing contains an invalid central-directory entry count') + } + if (expectedCount === 0) { + return Err(new ArchiveInspectionError('empty_archive', 'Empty ZIP archives are not accepted')) + } + + const markers = [...normalized.matchAll(/^Central directory entry #(\d+):\n-+\n/gm)] + if (markers.length !== expectedCount) { + return invalidListing(`ZIP listing declared ${expectedCount} entries but described ${markers.length}`) + } + + const entries: ArchiveEntry[] = [] + for (let index = 0; index < markers.length; index++) { + const marker = markers[index] + const nextMarker = markers[index + 1] + if (marker === undefined || marker.index === undefined) { + return invalidListing('ZIP listing contains an entry without a location') + } + + const declaredIndex = marker[1] === undefined ? Number.NaN : Number(marker[1]) + if (declaredIndex !== index + 1) { + return invalidListing('ZIP listing entry numbers are not sequential') + } + + const bodyStart = marker.index + marker[0].length + const bodyEnd = nextMarker?.index ?? normalized.length + const body = normalized.slice(bodyStart, bodyEnd) + const nameMatch = /^\n {2}([^\n]*)\n\n/.exec(body) + if (nameMatch?.[1] === undefined || nameMatch[1].length === 0) { + return invalidListing(`ZIP listing entry #${index + 1} has no unambiguous filename`) + } + + const name = nameMatch[1] + const size = parseSingleIntegerField(body, 'uncompressed size') + const filenameLength = parseSingleIntegerField(body, 'length of filename') + if (!size.ok) return size + if (!filenameLength.ok) return filenameLength + if (new TextEncoder().encode(name).byteLength !== filenameLength.value) { + return invalidListing(`ZIP listing entry #${index + 1} has an ambiguous filename`) + } + + const type = parseEntryType(body, name) + if (!type.ok) return type + + entries.push({ + name, + uncompressedSize: size.value, + type: type.value, + }) + } + + return Ok(entries) +} + +export function validateArchiveEntries( + entries: readonly ArchiveEntry[], + limits: Readonly = DEFAULT_ARCHIVE_LIMITS, +): Result { + if (!isValidLimit(limits.maxEntries) || !isValidLimit(limits.maxTotalUncompressedSize)) { + return invalidListing('Archive limits must be non-negative safe integers') + } + if (entries.length === 0) { + return Err(new ArchiveInspectionError('empty_archive', 'Empty ZIP archives are not accepted')) + } + if (entries.length > limits.maxEntries) { + return Err(new ArchiveInspectionError( + 'too_many_entries', + `ZIP archive exceeds the ${limits.maxEntries} entry limit`, + )) + } + + let fileCount = 0 + let directoryCount = 0 + let totalUncompressedSize = 0 + + for (const entry of entries) { + const pathError = validateArchivePath(entry.name) + if (pathError !== null) { + return Err(new ArchiveInspectionError('unsafe_path', pathError, { entryName: entry.name })) + } + if (!isValidLimit(entry.uncompressedSize)) { + return invalidListing(`ZIP entry has an invalid uncompressed size: ${entry.name}`) + } + + if (entry.type === 'directory') { + if (!entry.name.endsWith('/') || entry.uncompressedSize !== 0) { + return Err(new ArchiveInspectionError( + 'unsupported_entry_type', + 'ZIP directory entry has an inconsistent name or size', + { entryName: entry.name }, + )) + } + directoryCount++ + continue + } + + if (entry.name.endsWith('/')) { + return Err(new ArchiveInspectionError( + 'unsupported_entry_type', + 'ZIP regular file entry has a directory name', + { entryName: entry.name }, + )) + } + fileCount++ + totalUncompressedSize += entry.uncompressedSize + if (!Number.isSafeInteger(totalUncompressedSize) || totalUncompressedSize > limits.maxTotalUncompressedSize) { + return Err(new ArchiveInspectionError( + 'too_large', + `ZIP archive exceeds the ${limits.maxTotalUncompressedSize} byte uncompressed size limit`, + )) + } + } + + return Ok({ entries, fileCount, directoryCount, totalUncompressedSize }) +} + +function parseEntryType( + body: string, + name: string, +): Result { + const origin = parseSingleTextField(body, 'file system or operating system of origin') + if (!origin.ok) return origin + + let type: ArchiveEntry['type'] + if (origin.value === 'Unix') { + const modeMatches = [...body.matchAll(/^ {2}Unix file attributes \(([0-7]{6}) octal\):\s+.*$/gm)] + const modeText = modeMatches[0]?.[1] + if (modeMatches.length !== 1 || modeText === undefined) { + return invalidListing('ZIP listing entry has invalid Unix file attributes') + } + const kind = Number.parseInt(modeText, 8) & 0o170000 + if (kind === 0o100000) type = 'file' + else if (kind === 0o040000) type = 'directory' + else return unsupportedEntryType(name) + } else if (origin.value === 'MS-DOS, OS/2 or NT FAT') { + const attributeMatches = [...body.matchAll(/^ {2}MS-DOS file attributes \(([0-9A-Fa-f]{2}) hex\):\s+.*$/gm)] + const attributesText = attributeMatches[0]?.[1] + if (attributeMatches.length !== 1 || attributesText === undefined) { + return invalidListing('ZIP listing entry has invalid MS-DOS file attributes') + } + const attributes = Number.parseInt(attributesText, 16) + if ((attributes & 0x08) !== 0) return unsupportedEntryType(name) + type = (attributes & 0x10) !== 0 ? 'directory' : 'file' + } else { + return unsupportedEntryType(name) + } + + if ((type === 'directory') !== name.endsWith('/')) { + return Err(new ArchiveInspectionError( + 'unsupported_entry_type', + 'ZIP entry name and file type are inconsistent', + { entryName: name }, + )) + } + return Ok(type) +} + +function parseSingleIntegerField( + body: string, + field: 'uncompressed size' | 'length of filename', +): Result { + const expression = new RegExp(`^ ${field}:\\s+(\\d+)(?: bytes| characters)$`, 'gm') + const matches = [...body.matchAll(expression)] + const valueText = matches[0]?.[1] + const value = valueText === undefined ? Number.NaN : Number(valueText) + if (matches.length !== 1 || !Number.isSafeInteger(value) || value < 0) { + return invalidListing(`ZIP listing entry has an invalid ${field} field`) + } + return Ok(value) +} + +function parseSingleTextField( + body: string, + field: 'file system or operating system of origin', +): Result { + const expression = new RegExp(`^ ${field}:\\s+(.+)$`, 'gm') + const matches = [...body.matchAll(expression)] + const value = matches[0]?.[1] + if (matches.length !== 1 || value === undefined || value.length === 0) { + return invalidListing(`ZIP listing entry has an invalid ${field} field`) + } + return Ok(value) +} + +function validateArchivePath(name: string): string | null { + if (name.includes('\0')) return 'ZIP entry name contains a NUL byte' + if (name.includes('\\')) return 'ZIP entry name contains a backslash' + if (name.startsWith('/')) return 'ZIP entry path is absolute' + if (/^[A-Za-z]:/.test(name)) return 'ZIP entry path contains a drive prefix' + + if (name.split('/').some(segment => segment === '..')) { + return 'ZIP entry path escapes the extraction root' + } + return null +} + +function unsupportedEntryType(name: string): Result { + return Err(new ArchiveInspectionError( + 'unsupported_entry_type', + 'ZIP archive contains a symlink or unsupported special entry', + { entryName: name }, + )) +} + +function invalidListing(message: string): Result { + return Err(new ArchiveInspectionError('invalid_listing', message)) +} + +function isValidLimit(value: number): boolean { + return Number.isSafeInteger(value) && value >= 0 +} + +function abortedError(signal: AbortSignal): ArchiveInspectionError { + return new ArchiveInspectionError('aborted', 'ZIP archive inspection was aborted', { cause: signal.reason }) +} diff --git a/packages/sdk/src/lib/archive/index.ts b/packages/sdk/src/lib/archive/index.ts new file mode 100644 index 00000000..b46821af --- /dev/null +++ b/packages/sdk/src/lib/archive/index.ts @@ -0,0 +1,17 @@ +export { + ArchiveBudget, + ArchiveInspectionError, + DEFAULT_ARCHIVE_LIMITS, + inspectZipArchive, + parseZipInfoVerbose, + resolveArchiveLimits, + validateArchiveEntries, +} from './archive-inspection.js' +export type { + ArchiveEntry, + ArchiveInspection, + ArchiveInspectionErrorCode, + ArchiveLimitOverrides, + ArchiveLimits, + InspectZipArchiveOptions, +} from './archive-inspection.js' diff --git a/packages/sdk/src/lib/logger/console.ts b/packages/sdk/src/lib/logger/console.ts index 13ed744c..e1981453 100644 --- a/packages/sdk/src/lib/logger/console.ts +++ b/packages/sdk/src/lib/logger/console.ts @@ -1,5 +1,5 @@ import type { LogContext, Logger, LogLevel } from './logger.js' -import { shouldLog } from './logger.js' +import { BaseLogger, shouldLog } from './logger.js' export interface ConsoleLoggerConfig { level: LogLevel @@ -15,45 +15,20 @@ const COLORS = { error: '\x1b[31m', // red } -export class ConsoleLogger implements Logger { +export class ConsoleLogger extends BaseLogger { readonly level: LogLevel private useColors: boolean private showTimestamps: boolean private baseContext: LogContext constructor(config: ConsoleLoggerConfig, baseContext: LogContext = {}) { + super() this.level = config.level this.useColors = config.colors ?? true this.showTimestamps = config.timestamps ?? true this.baseContext = baseContext } - debug(message: string, context?: LogContext): void { - this.log('debug', message, context) - } - - info(message: string, context?: LogContext): void { - this.log('info', message, context) - } - - warn(message: string, context?: LogContext): void { - this.log('warn', message, context) - } - - error(message: string, error?: Error, context?: LogContext): void { - const errorContext = error - ? { - ...context, - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, - } - : context - - this.log('error', message, errorContext) - } child(context: LogContext): Logger { return new ConsoleLogger( @@ -66,7 +41,7 @@ export class ConsoleLogger implements Logger { ) } - private log(level: LogLevel, message: string, context?: LogContext): void { + protected log(level: LogLevel, message: string, context?: LogContext): void { if (!shouldLog(level, this.level)) return const fullContext = { ...this.baseContext, ...context } @@ -118,47 +93,22 @@ export class ConsoleLogger implements Logger { /** * JSON Logger for production (structured output) */ -export class JsonLogger implements Logger { +export class JsonLogger extends BaseLogger { readonly level: LogLevel private baseContext: LogContext constructor(level: LogLevel, baseContext: LogContext = {}) { + super() this.level = level this.baseContext = baseContext } - debug(message: string, context?: LogContext): void { - this.log('debug', message, context) - } - - info(message: string, context?: LogContext): void { - this.log('info', message, context) - } - - warn(message: string, context?: LogContext): void { - this.log('warn', message, context) - } - - error(message: string, error?: Error, context?: LogContext): void { - const errorContext = error - ? { - ...context, - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, - } - : context - - this.log('error', message, errorContext) - } child(context: LogContext): Logger { return new JsonLogger(this.level, { ...this.baseContext, ...context }) } - private log(level: LogLevel, message: string, context?: LogContext): void { + protected log(level: LogLevel, message: string, context?: LogContext): void { if (!shouldLog(level, this.level)) return const entry = { diff --git a/packages/sdk/src/lib/logger/file.ts b/packages/sdk/src/lib/logger/file.ts index 358cc9e9..b4e5dfbb 100644 --- a/packages/sdk/src/lib/logger/file.ts +++ b/packages/sdk/src/lib/logger/file.ts @@ -1,54 +1,30 @@ import type { FileSystem } from '~/platform/fs.js' import type { LogContext, Logger, LogLevel } from './logger.js' +import { BaseLogger } from './logger.js' /** * FileLogger - writes JSONL to a file, always at debug level. * Each line is a JSON object with timestamp, level, message, and context. */ -export class FileLogger implements Logger { +export class FileLogger extends BaseLogger { readonly level: LogLevel = 'debug' private filePath: string private baseContext: LogContext private fs: FileSystem constructor(filePath: string, fs: FileSystem, baseContext: LogContext = {}) { + super() this.filePath = filePath this.fs = fs this.baseContext = baseContext } - debug(message: string, context?: LogContext): void { - this.log('debug', message, context) - } - - info(message: string, context?: LogContext): void { - this.log('info', message, context) - } - - warn(message: string, context?: LogContext): void { - this.log('warn', message, context) - } - - error(message: string, error?: Error, context?: LogContext): void { - const errorContext = error - ? { - ...context, - error: { - name: error.name, - message: error.message, - stack: error.stack, - }, - } - : context - - this.log('error', message, errorContext) - } child(context: LogContext): Logger { return new FileLogger(this.filePath, this.fs, { ...this.baseContext, ...context }) } - private log(level: LogLevel, message: string, context?: LogContext): void { + protected log(level: LogLevel, message: string, context?: LogContext): void { const entry = { timestamp: new Date().toISOString(), level, diff --git a/packages/sdk/src/lib/logger/logger.ts b/packages/sdk/src/lib/logger/logger.ts index a41a3022..8908f788 100644 --- a/packages/sdk/src/lib/logger/logger.ts +++ b/packages/sdk/src/lib/logger/logger.ts @@ -112,3 +112,51 @@ export interface ToolLogContext extends LogContext { toolCallId: string durationMs?: number } + +/** + * Shared base for sink-backed loggers. + * + * The four level methods and the Error->context flattening in error() were + * copied verbatim into ConsoleLogger, JsonLogger and FileLogger. The flattening + * is the part worth having in one place — it is the only path by which a stack + * trace reaches a structured log, so a sink that misses it drops the field + * silently. + * + * Subclasses supply only log() and child(); child() stays per-class because each + * returns its own type with its own config. TeeLogger does NOT extend this — it + * fans out to other loggers rather than writing to a sink. + */ +export abstract class BaseLogger implements Logger { + abstract readonly level: LogLevel + + protected abstract log(level: LogLevel, message: string, context?: LogContext): void + + abstract child(context: LogContext): Logger + + debug(message: string, context?: LogContext): void { + this.log('debug', message, context) + } + + info(message: string, context?: LogContext): void { + this.log('info', message, context) + } + + warn(message: string, context?: LogContext): void { + this.log('warn', message, context) + } + + error(message: string, error?: Error, context?: LogContext): void { + const errorContext = error + ? { + ...context, + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + } + : context + + this.log('error', message, errorContext) + } +} diff --git a/packages/sdk/src/lib/never.ts b/packages/sdk/src/lib/never.ts deleted file mode 100644 index 1c29261a..00000000 --- a/packages/sdk/src/lib/never.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const assertNever = (value: never): never => { - throw new Error(`Unexpected value: ${JSON.stringify(value)}`) -} diff --git a/packages/sdk/src/lib/utils/concurrency.test.ts b/packages/sdk/src/lib/utils/concurrency.test.ts index 8e5c017c..cb68b2e3 100644 --- a/packages/sdk/src/lib/utils/concurrency.test.ts +++ b/packages/sdk/src/lib/utils/concurrency.test.ts @@ -1,7 +1,11 @@ -import { describe, expect, it } from 'bun:test' +import { describe, expect, it, spyOn } from 'bun:test' import { mapWithConcurrency, Semaphore } from './concurrency.js' -function defer(): { promise: Promise; resolve: (v: T) => void; reject: (e: unknown) => void } { +function defer(): { + promise: Promise + resolve: (v: T) => void + reject: (e: unknown) => void +} { let resolve!: (v: T) => void let reject!: (e: unknown) => void const promise = new Promise((res, rej) => { @@ -64,7 +68,7 @@ describe('mapWithConcurrency', () => { it('propagates errors thrown by the worker fn', async () => { await expect( - mapWithConcurrency([1, 2, 3], 2, async (n) => { + mapWithConcurrency([1, 2, 3], 2, async n => { if (n === 2) throw new Error('boom') return n }), @@ -142,15 +146,106 @@ describe('Semaphore', () => { it('releases the slot when the body throws', async () => { const gate = new Semaphore(1) - await expect(gate.run(async () => { - throw new Error('boom') - })).rejects.toThrow('boom') + await expect( + gate.run(async () => { + throw new Error('boom') + }), + ).rejects.toThrow('boom') // Slot must be free again — this would deadlock otherwise. const result = await gate.run(async () => 42) expect(result).toBe(42) }) + it('rejects a pre-aborted run without invoking its body', async () => { + const gate = new Semaphore(1) + const controller = new AbortController() + const reason = new Error('cancelled before acquire') + let invoked = false + controller.abort(reason) + + await expect( + gate.run(async () => { + invoked = true + }, controller.signal), + ).rejects.toBe(reason) + expect(invoked).toBe(false) + + await expect(gate.run(async () => 'next')).resolves.toBe('next') + }) + + it('removes an aborted queued waiter and preserves FIFO capacity', async () => { + const gate = new Semaphore(1) + const holderRelease = defer() + const firstRelease = defer() + const events: string[] = [] + const controller = new AbortController() + const reason = new Error('cancel queued waiter') + + const holder = gate.run(async () => { + events.push('holder') + await holderRelease.promise + }) + const first = gate.run(async () => { + events.push('first') + await firstRelease.promise + }) + let cancelledInvoked = false + const cancelled = gate.run(async () => { + cancelledInvoked = true + }, controller.signal) + const third = gate.run(async () => { + events.push('third') + }) + + controller.abort(reason) + await expect(cancelled).rejects.toBe(reason) + expect(cancelledInvoked).toBe(false) + + holderRelease.resolve() + await new Promise(resolve => setTimeout(resolve, 0)) + expect(events).toEqual(['holder', 'first']) + + firstRelease.resolve() + await Promise.all([holder, first, third]) + expect(events).toEqual(['holder', 'first', 'third']) + + await expect(gate.run(async () => 'capacity remains usable')).resolves.toBe('capacity remains usable') + }) + + it('releases a transferred permit when abort wins before the waiter continuation', async () => { + const gate = new Semaphore(1) + const holderStarted = defer() + const holderRelease = defer() + const controller = new AbortController() + const reason = new Error('cancel during permit handoff') + const removeListenerSpy = spyOn(controller.signal, 'removeEventListener') + let cancelledInvoked = false + let trailingInvoked = false + + const holder = gate.run(async () => { + holderStarted.resolve() + await holderRelease.promise + }) + await holderStarted.promise + const cancelled = gate.run(async () => { + cancelledInvoked = true + }, controller.signal) + const trailing = gate.run(async () => { + trailingInvoked = true + }) + + holderRelease.resolve() + queueMicrotask(() => queueMicrotask(() => controller.abort(reason))) + + await expect(cancelled).rejects.toBe(reason) + await Promise.all([holder, trailing]) + expect(cancelledInvoked).toBe(false) + expect(trailingInvoked).toBe(true) + expect(removeListenerSpy).toHaveBeenCalled() + await expect(gate.run(async () => 'still usable')).resolves.toBe('still usable') + }) + it('serializes work under limit=1', async () => { const gate = new Semaphore(1) const events: string[] = [] diff --git a/packages/sdk/src/lib/utils/concurrency.ts b/packages/sdk/src/lib/utils/concurrency.ts index e793ef08..9f4a72ad 100644 --- a/packages/sdk/src/lib/utils/concurrency.ts +++ b/packages/sdk/src/lib/utils/concurrency.ts @@ -33,7 +33,11 @@ export async function mapWithConcurrency( */ export class Semaphore { private active = 0 - private readonly waiters: Array<() => void> = [] + private readonly waiters: Array<{ + resolve: () => void + signal?: AbortSignal + onAbort?: () => void + }> = [] constructor(private readonly limit: number) { if (!Number.isInteger(limit) || limit < 1) { @@ -41,22 +45,44 @@ export class Semaphore { } } - async run(fn: () => Promise): Promise { - await this.acquire() + async run(fn: () => Promise, signal?: AbortSignal): Promise { + let acquired = false try { + await this.acquire(signal) + acquired = true + signal?.throwIfAborted() return await fn() } finally { - this.release() + if (acquired) this.release() } } - private acquire(): Promise { + private acquire(signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(signal.reason) + } if (this.active < this.limit) { this.active++ return Promise.resolve() } - return new Promise(resolve => { - this.waiters.push(resolve) + return new Promise((resolve, reject) => { + const waiter: { + resolve: () => void + signal?: AbortSignal + onAbort?: () => void + } = { resolve, signal } + if (signal) { + const onAbort = () => { + const index = this.waiters.indexOf(waiter) + if (index === -1) return + this.waiters.splice(index, 1) + signal.removeEventListener('abort', onAbort) + reject(signal.reason) + } + waiter.onAbort = onAbort + signal.addEventListener('abort', onAbort, { once: true }) + } + this.waiters.push(waiter) }) } @@ -64,7 +90,10 @@ export class Semaphore { const next = this.waiters.shift() if (next) { // Slot transfers directly to the next waiter; active stays unchanged. - next() + if (next.signal && next.onAbort) { + next.signal.removeEventListener('abort', next.onAbort) + } + next.resolve() } else { this.active-- } diff --git a/packages/sdk/src/lib/utils/result.ts b/packages/sdk/src/lib/utils/result.ts index eb771555..72bf1b4f 100644 --- a/packages/sdk/src/lib/utils/result.ts +++ b/packages/sdk/src/lib/utils/result.ts @@ -1,35 +1,11 @@ /** * Result type for explicit error handling without exceptions. - * Inspired by Rust/Go approach. + * + * Re-exported from @roj-ai/transport, which owns the definition — sdk already + * depends on it, and three byte-identical copies (here, transport, shared) meant + * any addition to the vocabulary had to be written three times or silently + * diverge. The ~58 in-package importers keep using `~/lib/utils/result.js`. */ -export type Result = - | { ok: true; value: T } - | { ok: false; error: E } - -// Constructors -export const Ok = (value: T): Result => ({ ok: true, value }) -export const Err = (error: E): Result => ({ ok: false, error }) - -// Type guards -export const isOk = (result: Result): result is { ok: true; value: T } => result.ok -export const isErr = (result: Result): result is { ok: false; error: E } => !result.ok - -// Helper functions -export const mapResult = ( - result: Result, - fn: (value: T) => U, -): Result => result.ok ? Ok(fn(result.value)) : result - -export const flatMapResult = ( - result: Result, - fn: (value: T) => Result, -): Result => result.ok ? fn(result.value) : result - -// Unwrap functions -export const unwrapOr = (result: Result, defaultValue: T): T => result.ok ? result.value : defaultValue - -export const unwrapOrThrow = (result: Result): T => { - if (result.ok) return result.value - throw result.error -} +export type { Result } from '@roj-ai/transport' +export { Err, flatMapResult, isErr, isOk, mapResult, Ok, unwrapOr, unwrapOrThrow } from '@roj-ai/transport' diff --git a/packages/sdk/src/lib/utils/sleep.test.ts b/packages/sdk/src/lib/utils/sleep.test.ts new file mode 100644 index 00000000..002ffbbe --- /dev/null +++ b/packages/sdk/src/lib/utils/sleep.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, spyOn, test } from 'bun:test' +import { sleep } from './sleep.js' + +describe('sleep', () => { + test('does not retain abort listeners across completed waits', async () => { + const controller = new AbortController() + const addListenerSpy = spyOn(controller.signal, 'addEventListener') + const removeListenerSpy = spyOn(controller.signal, 'removeEventListener') + + for (let wait = 0; wait < 3; wait++) { + await sleep(0, controller.signal) + } + + expect(addListenerSpy).toHaveBeenCalledTimes(3) + expect(removeListenerSpy).toHaveBeenCalledTimes(3) + for (let wait = 0; wait < 3; wait++) { + expect(removeListenerSpy.mock.calls[wait]?.[1]).toBe( + addListenerSpy.mock.calls[wait]?.[1], + ) + } + }) + + test('removes its abort listener when aborted', async () => { + const controller = new AbortController() + const addListenerSpy = spyOn(controller.signal, 'addEventListener') + const removeListenerSpy = spyOn(controller.signal, 'removeEventListener') + const sleepPromise = sleep(60_000, controller.signal) + + controller.abort() + await sleepPromise + + expect(addListenerSpy).toHaveBeenCalledTimes(1) + expect(removeListenerSpy).toHaveBeenCalledTimes(1) + expect(removeListenerSpy.mock.calls[0]?.[1]).toBe(addListenerSpy.mock.calls[0]?.[1]) + }) + + test('an already-aborted signal resolves without adding a listener', async () => { + const controller = new AbortController() + controller.abort() + const addListenerSpy = spyOn(controller.signal, 'addEventListener') + + await sleep(60_000, controller.signal) + + expect(addListenerSpy).not.toHaveBeenCalled() + }) +}) diff --git a/packages/sdk/src/lib/utils/sleep.ts b/packages/sdk/src/lib/utils/sleep.ts new file mode 100644 index 00000000..1ae7e8c5 --- /dev/null +++ b/packages/sdk/src/lib/utils/sleep.ts @@ -0,0 +1,23 @@ +/** + * Cancellable sleep. + * + * Resolves early (rather than rejecting) when `signal` aborts, so a retry or + * poll loop can check its own abort condition and exit on the next iteration. + * Prefer this over a bare `setTimeout` promise anywhere inside such a loop — + * a non-cancellable wait is what makes shutdown take as long as the backoff. + */ +export function sleep(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.resolve() + return new Promise((resolve) => { + let settled = false + const finish = () => { + if (settled) return + settled = true + clearTimeout(timer) + signal?.removeEventListener('abort', finish) + resolve() + } + const timer = setTimeout(finish, ms) + signal?.addEventListener('abort', finish, { once: true }) + }) +} diff --git a/packages/sdk/src/plugins/filesystem/schema.ts b/packages/sdk/src/plugins/filesystem/schema.ts deleted file mode 100644 index c60c6560..00000000 --- a/packages/sdk/src/plugins/filesystem/schema.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface DirectoryEntry { - name: string - path: string - type: 'file' | 'directory' | 'symlink' | 'other' - size?: number -} diff --git a/packages/sdk/src/plugins/resources/filename.ts b/packages/sdk/src/plugins/resources/filename.ts new file mode 100644 index 00000000..b30308d3 --- /dev/null +++ b/packages/sdk/src/plugins/resources/filename.ts @@ -0,0 +1,19 @@ +import { posix, win32 } from 'node:path' +import z from 'zod/v4' + +export function isResourceBasename(filename: string): boolean { + return filename.length > 0 + && !filename.includes('\0') + && !filename.includes('/') + && !filename.includes('\\') + && filename !== '.' + && filename !== '..' + && !posix.isAbsolute(filename) + && !win32.isAbsolute(filename) + && posix.basename(filename) === filename + && win32.basename(filename) === filename +} + +export const ResourceBasenameSchema = z.string().refine(isResourceBasename, { + message: 'Resource filename must be a basename', +}) diff --git a/packages/sdk/src/plugins/resources/plugin.ts b/packages/sdk/src/plugins/resources/plugin.ts index de6a6893..a65b8378 100644 --- a/packages/sdk/src/plugins/resources/plugin.ts +++ b/packages/sdk/src/plugins/resources/plugin.ts @@ -1,9 +1,11 @@ -import { join, relative, resolve } from 'node:path' +import { join, posix, resolve } from 'node:path' import z from 'zod/v4' import { definePlugin } from '~/core/plugins/plugin-builder.js' +import { type ArchiveLimitOverrides, inspectZipArchive } from '~/lib/archive/index.js' import { Ok } from '~/lib/utils/result.js' import type { FileSystem } from '~/platform/fs.js' import type { ProcessRunner } from '~/platform/process.js' +import { ResourceBasenameSchema } from './filename.js' import { RESOURCE_MANIFEST_FILENAME, type ResourceManifest, ResourceManifestSchema } from './manifest.js' import { type PostInjectContext, @@ -11,9 +13,10 @@ import { type PostInjectHook, postInjectRules, } from './post-inject.js' -import { type InjectedResource, resourceEvents, type ResourcesState } from './state.js' +import { type InjectedResource, type ResourcesState, resourceEvents } from './state.js' -const MAX_LISTED_PATHS = 100 +const ARCHIVE_TIMEOUT_MS = 120_000 +const ARCHIVE_MAX_BUFFER = 50 * 1024 * 1024 export interface ResourcesTargetDirArgs { sessionId: string @@ -25,6 +28,8 @@ export type ResourcesTargetDir = string | ((args: ResourcesTargetDirArgs) => str export interface ResourcesPluginConfig { targetDir?: ResourcesTargetDir + /** Entry and expanded-size limits for each injected ZIP resource. */ + archiveLimits?: ArchiveLimitOverrides /** * Called after a resource is written/extracted into `targetDir`, before the * `resource_injected` event is emitted. Use `postInjectRules` for a declarative @@ -40,33 +45,52 @@ function makeExec(processRunner: ProcessRunner) { options?: PostInjectExecOptions, ): Promise<{ stdout: string; stderr: string }> { return processRunner.execFile(cmd, args, { - timeout: options?.timeout ?? 120_000, - maxBuffer: 50 * 1024 * 1024, + timeout: options?.timeout ?? ARCHIVE_TIMEOUT_MS, + maxBuffer: ARCHIVE_MAX_BUFFER, cwd: options?.cwd, env: options?.env ? { ...process.env, ...options.env } : undefined, }) } } -async function listFiles(fs: FileSystem, dir: string, maxEntries: number): Promise { - const results: string[] = [] - - async function walk(current: string): Promise { - if (results.length >= maxEntries) return - const entries = await fs.readdir(current, { withFileTypes: true }) - for (const entry of entries) { - if (results.length >= maxEntries) break - const fullPath = join(current, entry.name) - if (entry.isDirectory()) { - await walk(fullPath) - } else { - results.push(relative(dir, fullPath)) - } - } +function normalizeArchiveEntryPath(path: string): string { + return posix.normalize(path) +} + +function isExcludedGitPath(path: string): boolean { + const normalized = normalizeArchiveEntryPath(path) + return normalized === '.git' || normalized.startsWith('.git/') +} + +function getErrorCode(error: unknown): string | undefined { + return error instanceof Error && 'code' in error && typeof error.code === 'string' + ? error.code + : undefined +} + +async function unlinkIfPresent(fs: FileSystem, path: string): Promise { + try { + await fs.unlink(path) + } catch (error) { + if (getErrorCode(error) !== 'ENOENT') throw error } +} - await walk(dir) - return results +async function verifiedExtractedPaths( + fs: FileSystem, + stagingDir: string, + entryPaths: readonly string[], +): Promise { + const paths = [...new Set(entryPaths + .map(normalizeArchiveEntryPath) + .filter(path => !isExcludedGitPath(path)))] + for (const path of paths) { + const stats = await fs.stat(join(stagingDir, path)) + if (!stats.isFile()) { + throw new Error(`ZIP extraction did not produce a regular file: ${path}`) + } + } + return paths } async function resolveTargetDir(targetDir: ResourcesTargetDir | undefined, args: ResourcesTargetDirArgs): Promise { @@ -113,6 +137,14 @@ export const resourcesPlugin = definePlugin('resources') slug: z.string().optional(), name: z.string().optional(), }).optional(), + }).superRefine((input, refinement) => { + if (input.mimeType !== 'application/zip' && !ResourceBasenameSchema.safeParse(input.filename).success) { + refinement.addIssue({ + code: 'custom', + path: ['filename'], + message: 'Resource filename must be a basename', + }) + } }), output: z.object({ resourceId: z.string(), @@ -126,59 +158,73 @@ export const resourcesPlugin = definePlugin('resources') sessionDir: ctx.environment.sessionDir, workspaceDir: ctx.environment.workspaceDir, }) - await fs.mkdir(targetDir, { recursive: true }) const resourceId = crypto.randomUUID() let paths: string[] + let manifest: ResourceManifest | null = null if (input.mimeType === 'application/zip') { - // Write to temp file, extract, clean up - const tempPath = join(ctx.environment.sessionDir, `_tmp_resource_${resourceId}.zip`) - await fs.writeFile(tempPath, input.fileBuffer) + const tempRoot = join(ctx.environment.sessionDir, `_tmp_resource_${resourceId}`) + const tempPath = join(tempRoot, 'resource.zip') + const stagingDir = join(tempRoot, 'staging') try { + await fs.mkdir(stagingDir, { recursive: true }) + await fs.writeFile(tempPath, input.fileBuffer) + + const inspection = await inspectZipArchive(ctx.platform.process, tempPath, { + timeoutMs: ARCHIVE_TIMEOUT_MS, + limits: ctx.pluginConfig?.archiveLimits, + }) + if (!inspection.ok) { + throw new Error(`ZIP inspection failed: ${inspection.error.message}`, { cause: inspection.error }) + } + // `-x .git .git/*` so a stray .git entry in the ZIP can't overwrite the // worktree's gitdir pointer (which silently breaks every subsequent git // command in the workspace). - await exec('unzip', ['-o', '-q', tempPath, '-d', targetDir, '-x', '.git', '.git/*']) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - // unzip returns exit code 1 for warnings — still usable - if (!message.includes('exit code 1')) { - await fs.unlink(tempPath).catch(() => {}) - throw new Error(`unzip failed: ${message}`) - } - } + await exec('unzip', ['-q', tempPath, '-d', stagingDir, '-x', '.git', '.git/*']) - await fs.unlink(tempPath).catch(() => {}) - paths = await listFiles(fs, targetDir, MAX_LISTED_PATHS) - } else { - // Copy file directly to target dir - const filePath = join(targetDir, input.filename) - await fs.writeFile(filePath, input.fileBuffer) - paths = [input.filename] - } + paths = await verifiedExtractedPaths( + fs, + stagingDir, + inspection.value.entries + .filter(entry => entry.type === 'file') + .map(entry => entry.name), + ) - let manifest: ResourceManifest | null = null - if (input.mimeType === 'application/zip') { - const manifestPath = join(targetDir, RESOURCE_MANIFEST_FILENAME) - try { - const raw = await fs.readFile(manifestPath, 'utf-8') - manifest = ResourceManifestSchema.parse(JSON.parse(raw)) - await fs.unlink(manifestPath).catch(() => {}) - paths = paths.filter((p) => p !== RESOURCE_MANIFEST_FILENAME) - ctx.logger.info('resources.inject: loaded resource manifest', { - filename: RESOURCE_MANIFEST_FILENAME, - postInjectRules: manifest.postInject?.length ?? 0, - }) - } catch (err) { - const code = (err as NodeJS.ErrnoException)?.code - if (code !== 'ENOENT') { - ctx.logger.warn('resources.inject: invalid resource manifest, skipping', { + const manifestPath = join(stagingDir, RESOURCE_MANIFEST_FILENAME) + try { + const raw = await fs.readFile(manifestPath, 'utf-8') + manifest = ResourceManifestSchema.parse(JSON.parse(raw)) + ctx.logger.info('resources.inject: loaded resource manifest', { filename: RESOURCE_MANIFEST_FILENAME, - error: err instanceof Error ? err.message : String(err), + postInjectRules: manifest.postInject?.length ?? 0, }) + } catch (error) { + if (getErrorCode(error) !== 'ENOENT') { + ctx.logger.warn('resources.inject: invalid resource manifest, skipping', { + filename: RESOURCE_MANIFEST_FILENAME, + error: error instanceof Error ? error.message : String(error), + }) + } + } finally { + await unlinkIfPresent(fs, manifestPath) } + + paths = paths.filter(path => normalizeArchiveEntryPath(path) !== RESOURCE_MANIFEST_FILENAME) + // Exclude every root .git spelling before promoting the complete staging tree. + await fs.rm(join(stagingDir, '.git'), { recursive: true, force: true }) + await fs.cp(stagingDir, targetDir, { recursive: true, force: true }) + } finally { + await fs.rm(tempRoot, { recursive: true, force: true }) } + } else { + // Keep this check adjacent to the filesystem write as defense in depth. + ResourceBasenameSchema.parse(input.filename) + await fs.mkdir(targetDir, { recursive: true }) + const filePath = join(targetDir, input.filename) + await fs.writeFile(filePath, input.fileBuffer) + paths = [input.filename] } const postInjectCtx: PostInjectContext = { diff --git a/packages/sdk/src/plugins/resources/resources.integration.test.ts b/packages/sdk/src/plugins/resources/resources.integration.test.ts index b655c635..bf849dcd 100644 --- a/packages/sdk/src/plugins/resources/resources.integration.test.ts +++ b/packages/sdk/src/plugins/resources/resources.integration.test.ts @@ -1,9 +1,14 @@ import { afterEach, describe, expect, it } from 'bun:test' -import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { dirname, join } from 'node:path' +import { Hono } from 'hono' +import { bootstrap, createSessionManager } from '~/bootstrap.js' import { MockLLMProvider } from '~/core/llm/mock.js' -import { createTestPreset, TestHarness } from '~/testing/index.js' +import { SAFE_INFO_ZIP_6_FIXTURE } from '~/lib/archive/archive-inspection.fixtures.js' +import { createNodePlatform, createTestPreset, TestHarness } from '~/testing/index.js' +import type { AppEnv, AppServices } from '~/transport/http/context.js' +import { createResourceRoutes } from '~/transport/http/routes/resources.js' import { resourcesPlugin } from './plugin.js' import { resourceEvents, type ResourcesState } from './state.js' @@ -16,6 +21,64 @@ afterEach(async () => { } }) +interface ZipInfoEntry { + name: string + size: number + type: 'file' | 'directory' +} + +function zipInfoFixture(entries: readonly ZipInfoEntry[]): string { + const entryWord = entries.length === 1 ? 'entry' : 'entries' + const details = entries.map((entry, index) => { + const mode = entry.type === 'directory' ? '040755' : '100644' + return `Central directory entry #${index + 1}: +--------------------------- + + ${entry.name} + + file system or operating system of origin: Unix + uncompressed size: ${entry.size} bytes + length of filename: ${Buffer.byteLength(entry.name)} characters + Unix file attributes (${mode} octal): attributes + MS-DOS file attributes (00 hex): none +` + }).join('\n') + + return `Archive: fixture.zip + This zipfile constitutes the sole disk of a single-part archive; its + central directory contains ${entries.length} ${entryWord}. + +${details}` +} + +function resourceInput(overrides: Partial<{ + filename: string + mimeType: string + fileBuffer: Buffer +}> = {}) { + const fileBuffer = overrides.fileBuffer ?? Buffer.from('content') + return { + sessionId: 'test-session', + filename: overrides.filename ?? 'resource.txt', + mimeType: overrides.mimeType ?? 'text/plain', + size: fileBuffer.length, + fileBuffer, + } +} + +async function createResourceHarness( + workspaceDir: string, + config: Parameters[0] = {}, +) { + const harness = new TestHarness({ + presets: [createTestPreset({ workspaceDir, plugins: [resourcesPlugin.configure(config)] })], + llmProvider: MockLLMProvider.withFixedResponse({ content: 'Ok', toolCalls: [] }), + }) + currentHarness = harness + const session = await harness.createSession('test') + return { harness, session } +} + describe('resources plugin', () => { it('resolves targetDir callback relative to workspace and records it in state', async () => { const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-')) @@ -65,4 +128,345 @@ describe('resources plugin', () => { await rm(workspaceDir, { recursive: true, force: true }) } }) + + const invalidFilenames = [ + '../outside.txt', + 'dir/file.txt', + 'dir\\file.txt', + '/absolute.txt', + 'C:\\absolute.txt', + 'C:relative.txt', + 'bad\0name.txt', + '.', + '..', + ] + + it.each(invalidFilenames)('rejects a non-basename direct resource filename: %s', async (filename) => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-name-')) + + try { + const { session } = await createResourceHarness(workspaceDir) + const result = await session.callPluginMethod('resources.inject', resourceInput({ filename })) + + expect(result.ok).toBe(false) + expect(await readdir(workspaceDir)).toEqual([]) + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } + }) + + const rejectedListings: ReadonlyArray = [ + ['too many entries', zipInfoFixture(Array.from({ length: 501 }, (_, index): ZipInfoEntry => ({ + name: `dir-${index}/`, + size: 0, + type: 'directory', + })))], + ['too many bytes', zipInfoFixture([{ name: 'large.bin', size: 100 * 1024 * 1024 + 1, type: 'file' }])], + ['unsafe path', zipInfoFixture([{ name: '../outside.txt', size: 1, type: 'file' }])], + ] + + it.each(rejectedListings)('rejects %s before extraction and leaves the target unchanged', async (_case, listing) => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-inspect-')) + await writeFile(join(workspaceDir, 'existing.txt'), 'unchanged') + + try { + const { harness, session } = await createResourceHarness(workspaceDir) + let extractionCalled = false + let tempRoot: string | undefined + const process = harness.sessionManager.getPlatform().process + const originalExec = process.execFile.bind(process) + process.execFile = async (file, args, options) => { + if (file === 'unzip' && args[0] === '-Z') { + const archivePath = args[2] + if (archivePath !== undefined) tempRoot = dirname(archivePath) + return { stdout: listing, stderr: '' } + } + if (file === 'unzip' && args[0] === '-q') { + extractionCalled = true + throw new Error('extraction must not run') + } + return originalExec(file, args, options) + } + + await expect(session.callPluginMethod('resources.inject', resourceInput({ + filename: 'resource.zip', + mimeType: 'application/zip', + }))).rejects.toThrow('ZIP inspection failed') + + expect(extractionCalled).toBe(false) + expect(await readdir(workspaceDir)).toEqual(['existing.txt']) + expect(await readFile(join(workspaceDir, 'existing.txt'), 'utf-8')).toBe('unchanged') + expect(tempRoot).toBeDefined() + if (tempRoot !== undefined) { + expect(await harness.sessionManager.getPlatform().fs.exists(tempRoot)).toBe(false) + } + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } + }) + + it('applies resource-specific archive limits before extraction', async () => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-custom-limit-')) + const listing = zipInfoFixture([ + { name: 'one.txt', size: 1, type: 'file' }, + { name: 'two.txt', size: 1, type: 'file' }, + ]) + + try { + const { harness, session } = await createResourceHarness(workspaceDir, { + archiveLimits: { maxEntries: 1 }, + }) + let extractionCalled = false + const process = harness.sessionManager.getPlatform().process + const originalExec = process.execFile.bind(process) + process.execFile = async (file, args, options) => { + if (file === 'unzip' && args[0] === '-Z') return { stdout: listing, stderr: '' } + if (file === 'unzip' && args[0] === '-q') extractionCalled = true + return originalExec(file, args, options) + } + + await expect(session.callPluginMethod('resources.inject', resourceInput({ + filename: 'resource.zip', + mimeType: 'application/zip', + }))).rejects.toThrow('ZIP inspection failed') + + expect(extractionCalled).toBe(false) + expect(await readdir(workspaceDir)).toEqual([]) + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } + }) + + it('applies server-configured resource archive limits', async () => { + const baseDir = await mkdtemp(join(tmpdir(), 'roj-resource-server-limit-')) + const workspaceDir = join(baseDir, 'workspace') + await mkdir(workspaceDir) + const platform = createNodePlatform() + let extractionCalled = false + const listing = zipInfoFixture([ + { name: 'one.txt', size: 1, type: 'file' }, + { name: 'two.txt', size: 1, type: 'file' }, + ]) + const originalExec = platform.process.execFile.bind(platform.process) + platform.process.execFile = async (file, args, options) => { + if (file === 'unzip' && args[0] === '-Z') return { stdout: listing, stderr: '' } + if (file === 'unzip' && args[0] === '-q') extractionCalled = true + return originalExec(file, args, options) + } + const services = bootstrap({ + port: 0, + host: 'localhost', + dataPath: baseDir, + persistence: 'memory', + logLevel: 'error', + logFormat: 'console', + resourceArchiveLimits: { maxEntries: 1 }, + llmMock: () => ({ + content: 'Mock response', + toolCalls: [], + finishReason: 'stop', + metrics: { promptTokens: 0, completionTokens: 0, totalTokens: 0, latencyMs: 0, model: 'mock' }, + }), + }, { presets: [createTestPreset({ workspaceDir })] }, platform) + const sessionManager = createSessionManager(services) + + try { + const sessionResult = await sessionManager.createSession('test', { workspaceDir }) + if (!sessionResult.ok) throw new Error(sessionResult.error.message) + + await expect(sessionResult.value.callPluginMethod('resources.inject', resourceInput({ + filename: 'resource.zip', + mimeType: 'application/zip', + }))).rejects.toThrow('ZIP inspection failed') + + expect(extractionCalled).toBe(false) + expect(await readdir(workspaceDir)).toEqual([]) + } finally { + await sessionManager.shutdown() + await rm(baseDir, { recursive: true, force: true }) + } + }) + + it('cleans staging and leaves the target unchanged when extraction fails', async () => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-extract-')) + await writeFile(join(workspaceDir, 'existing.txt'), 'unchanged') + + try { + const { harness, session } = await createResourceHarness(workspaceDir) + const calls: string[][] = [] + let tempRoot: string | undefined + const process = harness.sessionManager.getPlatform().process + const originalExec = process.execFile.bind(process) + process.execFile = async (file, args, options) => { + if (file !== 'unzip') return originalExec(file, args, options) + calls.push(args) + if (args[0] === '-Z') { + const archivePath = args[2] + if (archivePath !== undefined) tempRoot = dirname(archivePath) + return { stdout: SAFE_INFO_ZIP_6_FIXTURE, stderr: '' } + } + throw new Error('unzip exited with code 2') + } + + await expect(session.callPluginMethod('resources.inject', resourceInput({ + filename: 'resource.zip', + mimeType: 'application/zip', + }))).rejects.toThrow('unzip exited with code 2') + + expect(calls.map(args => args[0])).toEqual(['-Z', '-q']) + expect(await readdir(workspaceDir)).toEqual(['existing.txt']) + expect(await readFile(join(workspaceDir, 'existing.txt'), 'utf-8')).toBe('unchanged') + if (tempRoot !== undefined) { + expect(await harness.sessionManager.getPlatform().fs.exists(tempRoot)).toBe(false) + } + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } + }) + + it('promotes a validated overlay, excludes unrelated files, and runs manifest hooks afterward', async () => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-overlay-')) + const sourceDir = await mkdtemp(join(tmpdir(), 'roj-resources-source-')) + const archivePath = join(sourceDir, 'resource.zip') + await writeFile(join(workspaceDir, 'existing.txt'), 'keep') + await writeFile(join(sourceDir, 'new.txt'), 'new') + await writeFile(join(sourceDir, 'roj.resource.json'), JSON.stringify({ + postInject: [{ run: ['sh', '-c', 'printf manifest > manifest-ran.txt'] }], + })) + await mkdir(join(sourceDir, '.git')) + await writeFile(join(sourceDir, '.git', 'HEAD'), 'do not inject') + + try { + let configuredHookRan = false + const { harness, session } = await createResourceHarness(workspaceDir, { + postInject: async (ctx) => { + configuredHookRan = true + expect(ctx.paths).toEqual(['new.txt']) + expect(await ctx.fs.readFile(join(ctx.targetDir, 'new.txt'), 'utf-8')).toBe('new') + expect(await ctx.fs.exists(join(ctx.targetDir, 'roj.resource.json'))).toBe(false) + }, + }) + const process = harness.sessionManager.getPlatform().process + await process.execFile('zip', ['-q', '-r', archivePath, 'new.txt', 'roj.resource.json', '.git'], { cwd: sourceDir }) + const archive = await readFile(archivePath) + + const originalExec = process.execFile.bind(process) + const calls: string[] = [] + let tempRoot: string | undefined + process.execFile = async (file, args, options) => { + if (file === 'unzip') { + calls.push(args[0] ?? '') + if (args[0] === '-Z' && args[2] !== undefined) tempRoot = dirname(args[2]) + } + return originalExec(file, args, options) + } + + const result = await session.callPluginMethod('resources.inject', resourceInput({ + filename: 'resource.zip', + mimeType: 'application/zip', + fileBuffer: archive, + })) + + expect(result).toMatchObject({ ok: true, value: { paths: ['new.txt'] } }) + expect(calls.slice(0, 2)).toEqual(['-Z', '-q']) + expect(configuredHookRan).toBe(true) + expect(await readFile(join(workspaceDir, 'existing.txt'), 'utf-8')).toBe('keep') + expect(await readFile(join(workspaceDir, 'new.txt'), 'utf-8')).toBe('new') + expect(await readFile(join(workspaceDir, 'manifest-ran.txt'), 'utf-8')).toBe('manifest') + expect(await harness.sessionManager.getPlatform().fs.exists(join(workspaceDir, '.git'))).toBe(false) + expect(await harness.sessionManager.getPlatform().fs.exists(join(workspaceDir, 'roj.resource.json'))).toBe(false) + if (tempRoot !== undefined) { + expect(await harness.sessionManager.getPlatform().fs.exists(tempRoot)).toBe(false) + } + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + await rm(sourceDir, { recursive: true, force: true }) + } + }) + + it('normalizes archive aliases before excluding .git and the manifest', async () => { + const workspaceDir = await mkdtemp(join(tmpdir(), 'roj-resources-aliases-')) + const listing = zipInfoFixture([ + { name: './.git/HEAD', size: 3, type: 'file' }, + { name: 'nested//file.txt', size: 4, type: 'file' }, + { name: './roj.resource.json', size: 2, type: 'file' }, + ]) + + try { + const { harness, session } = await createResourceHarness(workspaceDir) + const process = harness.sessionManager.getPlatform().process + const originalExec = process.execFile.bind(process) + process.execFile = async (file, args, options) => { + if (file !== 'unzip') return originalExec(file, args, options) + if (args[0] === '-Z') return { stdout: listing, stderr: '' } + + const stagingDir = args[3] + if (stagingDir === undefined) throw new Error('missing staging directory') + await mkdir(join(stagingDir, '.git'), { recursive: true }) + await writeFile(join(stagingDir, '.git', 'HEAD'), 'git') + await mkdir(join(stagingDir, 'nested'), { recursive: true }) + await writeFile(join(stagingDir, 'nested', 'file.txt'), 'file') + await writeFile(join(stagingDir, 'roj.resource.json'), '{}') + return { stdout: '', stderr: '' } + } + + const result = await session.callPluginMethod('resources.inject', resourceInput({ + filename: 'resource.zip', + mimeType: 'application/zip', + })) + + expect(result).toMatchObject({ ok: true, value: { paths: ['nested/file.txt'] } }) + expect(await readFile(join(workspaceDir, 'nested', 'file.txt'), 'utf-8')).toBe('file') + expect(await harness.sessionManager.getPlatform().fs.exists(join(workspaceDir, '.git'))).toBe(false) + expect(await harness.sessionManager.getPlatform().fs.exists(join(workspaceDir, 'roj.resource.json'))).toBe(false) + } finally { + await rm(workspaceDir, { recursive: true, force: true }) + } + }) + + it('returns a controlled validation response for non-string HTTP fields', async () => { + const baseDir = await mkdtemp(join(tmpdir(), 'roj-resource-route-')) + const workspaceDir = join(baseDir, 'workspace') + await mkdir(workspaceDir) + const platform = createNodePlatform() + const services = bootstrap({ + port: 0, + host: 'localhost', + dataPath: baseDir, + persistence: 'memory', + logLevel: 'error', + logFormat: 'console', + llmMock: () => ({ + content: 'Mock response', + toolCalls: [], + finishReason: 'stop', + metrics: { promptTokens: 0, completionTokens: 0, totalTokens: 0, latencyMs: 0, model: 'mock' }, + }), + }, { presets: [createTestPreset()] }, platform) + const sessionRuntime = createSessionManager(services) + + try { + const sessionResult = await sessionRuntime.createSession('test', { workspaceDir }) + if (!sessionResult.ok) throw new Error(sessionResult.error.message) + const appServices: AppServices = { ...services, sessionRuntime } + const app = new Hono() + app.use('*', async (context, next) => { + context.set('services', appServices) + await next() + }) + app.route('/sessions', createResourceRoutes()) + + const response = await app.request(`/sessions/${sessionResult.value.id}/inject-resource`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: 'https://example.test/resource', filename: 42, mimeType: 'text/plain' }), + }) + + expect(response.status).toBe(400) + expect(await response.text()).toContain('validation_error') + } finally { + await sessionRuntime.shutdown() + await rm(baseDir, { recursive: true, force: true }) + } + }) }) diff --git a/packages/sdk/src/plugins/services/plugin.ts b/packages/sdk/src/plugins/services/plugin.ts index 5493dddf..516637f4 100644 --- a/packages/sdk/src/plugins/services/plugin.ts +++ b/packages/sdk/src/plugins/services/plugin.ts @@ -414,7 +414,7 @@ export const servicePlugin = definePlugin('services') // A queued revival outlives the session otherwise: its timer holds the // executor alive and spawns a process into a session that is already gone. const revivalQueued = ctx.pluginContext.executor.hasScheduledRestart(svcConfig.type) - if (status === 'ready' || status === 'starting' || revivalQueued) { + if (status === 'ready' || status === 'starting' || status === 'paused' || revivalQueued) { await ctx.pluginContext.executor.stop(svcConfig.type, ctx.sessionId) } } diff --git a/packages/sdk/src/plugins/services/service.ts b/packages/sdk/src/plugins/services/service.ts index f36d9129..8e04167b 100644 --- a/packages/sdk/src/plugins/services/service.ts +++ b/packages/sdk/src/plugins/services/service.ts @@ -61,6 +61,7 @@ interface RunningService { cwd?: string command: string logs: RingBuffer + forgetPid: () => Promise } export interface ServiceStatusChangeDetails { @@ -108,7 +109,22 @@ export interface ServiceExecutorDeps { fs: FileSystem process: ProcessRunner /** Durable pid record, swept at agent boot. Optional so embedders without a data dir still work. */ - pidRegistry?: ServicePidRegistry + pidRegistry?: Pick + /** Test seam for process-group failure handling. */ + kill?: typeof process.kill +} + +type ProcessGroupProbe = { state: 'alive' | 'gone' } | { state: 'error'; error: Error } + +let serviceExecutorObserverForTesting: ((executor: ServiceExecutor) => void) | undefined + +/** Observe executor creation in integration tests without exposing lifecycle methods as plugin API. */ +export function setServiceExecutorObserverForTesting(observer: (executor: ServiceExecutor) => void): () => void { + const previous = serviceExecutorObserverForTesting + serviceExecutorObserverForTesting = observer + return () => { + if (serviceExecutorObserverForTesting === observer) serviceExecutorObserverForTesting = previous + } } export class ServiceExecutor { @@ -117,6 +133,8 @@ export class ServiceExecutor { private readonly waiters = new Map) => void; timer: ReturnType }>>() /** Per-type lock collapsing concurrent start() calls onto a single in-flight start. */ private readonly startInFlight = new Map>>() + /** Registry deletions that must settle before a replacement can record its PID. */ + private readonly pidForgets = new Map>() /** Per-type counter bounding automatic EADDRINUSE port re-allocation retries. */ private readonly portConflictRetries = new Map() /** Per-type counter bounding `restartPolicy` revivals after an unexpected exit. */ @@ -127,7 +145,8 @@ export class ServiceExecutor { private readonly portPool: PortPool private readonly fs: FileSystem private readonly processRunner: ProcessRunner - private readonly pidRegistry?: ServicePidRegistry + private readonly pidRegistry?: Pick + private readonly killProcess: typeof process.kill /** Optional callback invoked on every service status change */ onStatusChanged?: ( @@ -143,6 +162,8 @@ export class ServiceExecutor { this.fs = deps.fs this.processRunner = deps.process this.pidRegistry = deps.pidRegistry + this.killProcess = deps.kill ?? process.kill + serviceExecutorObserverForTesting?.(this) } private notifyStatusChanged( @@ -279,9 +300,14 @@ export class ServiceExecutor { this.cancelPendingRestart(config.type) const existing = this.services.get(config.type) - if (existing && (existing.status === 'starting' || existing.status === 'ready')) { + if (existing && (existing.status === 'starting' || existing.status === 'ready' || existing.status === 'paused')) { return Ok(undefined) } + if (existing?.status === 'stopping') { + return Err({ message: `Service '${config.type}' is stopping, cannot start`, recoverable: true }) + } + if (existing) await existing.forgetPid() + await this.pidForgets.get(config.type) const availability = await this.isAvailable(config, sessionId, workspaceDir) if (!availability.ok) return availability @@ -313,10 +339,23 @@ export class ServiceExecutor { } const cwd = cwdResult.value - const logBufferSize = config.logBufferSize ?? 200 + const logBufferSize = Math.max(1, config.logBufferSize ?? 200) const startupTimeoutMs = config.startupTimeoutMs ?? 30_000 const readyRegex = config.readyPattern ? new RegExp(config.readyPattern) : undefined + const logs = new RingBuffer(logBufferSize) + const maxDiagnosticLength = 16_384 + const startTime = Date.now() + let portConflictDetected = false + let setupComplete = false + let retryPortConflictDuringSetup = false + let readyDetectedDuringSetup = false + let readyLineDuringSetup: string | undefined + let stdoutPartial = '' + let stderrPartial = '' + let outputSequence = 0 + let stdoutPartialSequence = 0 + let stderrPartialSequence = 0 const startArgs = { port, @@ -379,6 +418,82 @@ export class ServiceExecutor { return Err({ message: 'Failed to spawn service process', recoverable: true }) } + const matches = (regex: RegExp, content: string): boolean => { + regex.lastIndex = 0 + const matched = regex.test(content) + regex.lastIndex = 0 + return matched + } + + const recordDiagnostic = (line: string, prefix: string) => { + const maxContentLength = Math.max(0, maxDiagnosticLength - prefix.length) + const bounded = line.length > maxContentLength ? line.slice(-maxContentLength) : line + logs.push(`${prefix}${bounded}`) + } + + // Detection sees the full incoming content before diagnostics are truncated. + const appendOutput = (partial: string, data: Buffer, prefix: string): string => { + const combined = partial + data.toString() + if (matches(PORT_CONFLICT_PATTERN, combined)) portConflictDetected = true + const lines = combined.split('\n') + const nextPartial = lines.pop() ?? '' + const readyMatched = readyRegex ? matches(readyRegex, combined) || lines.some((line) => matches(readyRegex, line)) : false + if (readyMatched && !setupComplete) { + readyDetectedDuringSetup = true + readyLineDuringSetup = combined.slice(-maxDiagnosticLength) + } + + for (const line of lines) recordDiagnostic(line, prefix) + + if (setupComplete) { + const current = this.services.get(config.type) + if (current?.process === child && current.status === 'starting') { + this.logger.debug('Service output', { serviceType: config.type }) + if (readyMatched) markReady(combined.slice(-maxDiagnosticLength)) + } + void checkReadyWhen() + } + + // Retain enough suffix for markers split across chunks, never an entire line. + return nextPartial.length > maxDiagnosticLength ? nextPartial.slice(-maxDiagnosticLength) : nextPartial + } + + const onStdout = (data: Buffer) => { + stdoutPartial = appendOutput(stdoutPartial, data, '') + stdoutPartialSequence = stdoutPartial ? ++outputSequence : 0 + } + const onStderr = (data: Buffer) => { + stderrPartial = appendOutput(stderrPartial, data, '[stderr] ') + stderrPartialSequence = stderrPartial ? ++outputSequence : 0 + } + + let exitDuringSetup: { code: number | null } | undefined + let forgetPromise: Promise | undefined + const forgetPid = (): Promise => { + if (forgetPromise) return forgetPromise + const previous = this.pidForgets.get(config.type) + const runForget = () => this.pidRegistry?.forget(String(sessionId), config.type) ?? Promise.resolve() + const operation = previous ? previous.then(runForget, runForget) : runForget() + forgetPromise = operation + this.pidForgets.set(config.type, operation) + void operation.then( + () => { + if (this.pidForgets.get(config.type) === operation) this.pidForgets.delete(config.type) + }, + () => { + if (this.pidForgets.get(config.type) === operation) this.pidForgets.delete(config.type) + if (forgetPromise === operation) forgetPromise = undefined + }, + ) + return operation + } + const bufferClose = (code: number | null) => { + exitDuringSetup = { code } + } + child.stdout?.on('data', onStdout) + child.stderr?.on('data', onStderr) + child.on('close', bufferClose) + // Capture start time immediately so a later PID-reuse check can distinguish // "our process" from "an unrelated process that grabbed this PID after ours died" const pidStartTime = await getProcessStartTime(this.fs, child.pid) @@ -391,12 +506,6 @@ export class ServiceExecutor { // Emit starting event with PID, port, resolved cwd/command, and start time. this.notifyStatusChanged(sessionId, config.type, 'starting', { port, pid: child.pid, pidStartTime, cwd, command }) - const logs = new RingBuffer(logBufferSize) - const startTime = Date.now() - // Set when the child reports a port bind-conflict; read by the close handler - // to decide between a fresh-port retry and a terminal failure. - let portConflictDetected = false - const entry: RunningService = { config, process: child, @@ -406,6 +515,7 @@ export class ServiceExecutor { cwd, command, logs, + forgetPid, } this.services.set(config.type, entry) @@ -467,27 +577,6 @@ export class ServiceExecutor { } } - const processLine = (line: string) => { - logs.push(line) - if (PORT_CONFLICT_PATTERN.test(line)) { - portConflictDetected = true - } - const current = this.services.get(config.type) - if (!current || current.process !== child) return - - if (current.status === 'starting') { - this.logger.debug('Service output', { serviceType: config.type, line }) - } - - if (readyRegex && current.status === 'starting') { - if (readyRegex.test(line)) { - markReady(line) - } - } - - void checkReadyWhen() - } - // Startup timeout — mark as failed if not ready in time if (readyRegex || config.readyWhen) { startupTimer = setTimeout(() => { @@ -519,44 +608,26 @@ export class ServiceExecutor { readyCheckTimer = setInterval(() => { void checkReadyWhen() }, intervalMs) - void checkReadyWhen() } - // Pipe stdout/stderr line by line - let stdoutPartial = '' - child.stdout?.on('data', (data: Buffer) => { - stdoutPartial += data.toString() - const lines = stdoutPartial.split('\n') - stdoutPartial = lines.pop()! - for (const line of lines) { - processLine(line) - } - }) - - let stderrPartial = '' - child.stderr?.on('data', (data: Buffer) => { - stderrPartial += data.toString() - const lines = stderrPartial.split('\n') - stderrPartial = lines.pop()! - for (const line of lines) { - processLine(`[stderr] ${line}`) - } - }) - - // Handle unexpected exit - child.on('close', (code) => { + // Handle unexpected exit. Named and guarded because it also has to be + // replayable — see the exit-during-setup check after markReady() below. + let closeHandled = false + const handleClose = (code: number | null) => { + if (closeHandled) return + closeHandled = true clearReadinessTimers() - // The process is gone, so its durable record has nothing left to reap. - void this.pidRegistry?.forget(String(sessionId), config.type) - // Flush remaining partial lines - if (stdoutPartial) { - processLine(stdoutPartial) - stdoutPartial = '' - } - if (stderrPartial) { - processLine(`[stderr] ${stderrPartial}`) - stderrPartial = '' + // A replacement must not be recorded until this deletion finishes. + void forgetPid() + const partials = [ + { content: stdoutPartial, prefix: '', sequence: stdoutPartialSequence }, + { content: stderrPartial, prefix: '[stderr] ', sequence: stderrPartialSequence }, + ] + for (const partial of partials.filter(({ content }) => content).sort((left, right) => left.sequence - right.sequence)) { + recordDiagnostic(partial.content, partial.prefix) } + stdoutPartial = '' + stderrPartial = '' const current = this.services.get(config.type) if (!current || current.process !== child) return @@ -565,9 +636,9 @@ export class ServiceExecutor { // Expected stop current.status = 'stopped' this.notifyStatusChanged(sessionId, config.type, 'stopped') - } else if (current.status === 'starting' || current.status === 'ready') { + } else if (current.status === 'starting' || current.status === 'ready' || current.status === 'paused') { const retries = this.portConflictRetries.get(config.type) ?? 0 - if (portConflictDetected && retries < MAX_PORT_CONFLICT_RETRIES) { + if (current.status === 'starting' && portConflictDetected && retries < MAX_PORT_CONFLICT_RETRIES) { // The chosen port is held by a foreign process (e.g. a service // leaked from another session, or a survivor of a previous boot). // Re-allocate a fresh port and retry so the service recovers on a @@ -584,7 +655,11 @@ export class ServiceExecutor { }) this.allocatedPorts.delete(config.type) this.services.delete(config.type) - void this.start(config, sessionId, workspaceDir) + if (!setupComplete) { + retryPortConflictDuringSetup = true + } else { + setTimeout(() => void this.start(config, sessionId, workspaceDir), 0) + } return } @@ -607,7 +682,9 @@ export class ServiceExecutor { code, }) } - }) + } + child.on('close', handleClose) + child.off('close', bufferClose) this.logger.info('Service starting', { serviceType: config.type, @@ -619,9 +696,33 @@ export class ServiceExecutor { startupTimeoutMs, }) - // If no ready pattern, immediately mark as ready - if (!readyRegex && !config.readyWhen) { + // A service that died inside the bookkeeping above closed while only the + // setup close collector was listening, so replay that close now. + // `alreadyReaped` covers the in-between state: the exit is recorded but + // 'close' has not fired yet because it waits for the stdio EOF. The listener + // above is guaranteed to receive it, so calling handleClose here would only + // drop the tail of the log the failure has to explain — while marking such a + // child ready would be a lie. + const alreadyReaped = child.exitCode != null || child.signalCode != null + if (exitDuringSetup) { + handleClose(exitDuringSetup.code) + } + + setupComplete = true + if (retryPortConflictDuringSetup) { + await forgetPid() + return this.startInternal(config, sessionId, workspaceDir) + } + if (exitDuringSetup || alreadyReaped) { + return Ok(undefined) + } + if (readyDetectedDuringSetup) { + markReady(readyLineDuringSetup) + } else if (!readyRegex && !config.readyWhen) { + // If no ready condition is configured, a live child is ready immediately. markReady() + } else if (config.readyWhen) { + void checkReadyWhen() } return Ok(undefined) @@ -698,6 +799,111 @@ export class ServiceExecutor { return this.restartTimers.has(serviceType) } + private errorWithContext(error: unknown, context: string): Error { + const cause = error instanceof Error ? error.message : String(error) + return new Error(`${context}: ${cause}`) + } + + private hasErrorCode(error: unknown, code: string): boolean { + return error instanceof Error && 'code' in error && error.code === code + } + + private async hasNonZombieProcessGroupMember(pid: number): Promise { + if (process.platform !== 'linux') return true + + let entries: string[] + try { + entries = await this.fs.readdir('/proc') + } catch { + return true + } + + let foundGroupMember = false + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue + try { + const stat = await this.fs.readFile(`/proc/${entry}/stat`, 'utf-8') + const rparen = stat.lastIndexOf(')') + if (rparen === -1) continue + const fields = stat.slice(rparen + 2).split(' ') + if (Number(fields[2]) !== pid) continue + foundGroupMember = true + if (fields[0] !== 'Z') return true + } catch { + // Processes can disappear while /proc is scanned. + } + } + // A successful group probe with no readable member is still evidence of life. + return !foundGroupMember + } + + private async probeProcessGroup(pid: number): Promise { + try { + this.killProcess(-pid, 0) + } catch (error) { + if (this.hasErrorCode(error, 'ESRCH')) return { state: 'gone' } + if (this.hasErrorCode(error, 'EPERM')) return { state: 'alive' } + return { state: 'error', error: this.errorWithContext(error, `Failed to probe process group ${pid}`) } + } + + return await this.hasNonZombieProcessGroupMember(pid) ? { state: 'alive' } : { state: 'gone' } + } + + private async signalProcessGroup(pid: number, signal: NodeJS.Signals): Promise { + try { + this.killProcess(-pid, signal) + return { state: 'alive' } + } catch (error) { + if (this.hasErrorCode(error, 'ESRCH')) return { state: 'gone' } + return { state: 'error', error: this.errorWithContext(error, `Failed to send ${signal} to process group ${pid}`) } + } + } + + private async waitForProcessGroupExit(pid: number, timeoutMs: number): Promise { + const initial = await this.probeProcessGroup(pid) + if (initial.state !== 'alive') return initial + + return new Promise((resolve) => { + let settled = false + const finish = (result: ProcessGroupProbe) => { + if (settled) return + settled = true + clearInterval(checkInterval) + clearTimeout(timeout) + resolve(result) + } + const checkInterval = setInterval(() => { + void this.probeProcessGroup(pid).then((result) => { + if (result.state !== 'alive') finish(result) + }) + }, 50) + const timeout = setTimeout(() => { + void this.probeProcessGroup(pid).then(finish) + }, timeoutMs) + }) + } + + private async terminateProcessGroup(pid: number, gracefulStopMs: number): Promise> { + const initial = await this.probeProcessGroup(pid) + if (initial.state === 'gone') return Ok(undefined) + if (initial.state === 'error') return Err(initial.error) + + const term = await this.signalProcessGroup(pid, 'SIGTERM') + if (term.state === 'gone') return Ok(undefined) + if (term.state === 'error') return Err(term.error) + const graceful = await this.waitForProcessGroupExit(pid, gracefulStopMs) + if (graceful.state === 'gone') return Ok(undefined) + if (graceful.state === 'error') return Err(graceful.error) + + const kill = await this.signalProcessGroup(pid, 'SIGKILL') + if (kill.state === 'gone') return Ok(undefined) + if (kill.state === 'error') return Err(kill.error) + const forced = await this.waitForProcessGroupExit(pid, 5000) + if (forced.state === 'gone') return Ok(undefined) + if (forced.state === 'error') return Err(forced.error) + return Err(new Error(`Process group ${pid} survived SIGKILL`)) + } + /** * Stop a running service gracefully. * Port is NOT released — kept for session-level stability across restarts. @@ -713,6 +919,11 @@ export class ServiceExecutor { // A failed service with a revival queued is really "about to restart", // and calling that off is a legitimate stop rather than an error. if (hadPendingRestart) { + try { + await entry.forgetPid() + } catch (error) { + return Err({ message: this.errorWithContext(error, `Failed to forget PID for service '${serviceType}'`).message, recoverable: true }) + } entry.status = 'stopped' this.notifyStatusChanged(sessionId, serviceType, 'stopped') return Ok(undefined) @@ -720,46 +931,35 @@ export class ServiceExecutor { return Err({ message: `Service '${serviceType}' is ${entry.status}, cannot stop`, recoverable: false }) } + const previousStatus = entry.status entry.status = 'stopping' this.notifyStatusChanged(sessionId, serviceType, 'stopping') const gracefulStopMs = entry.config.gracefulStopMs ?? 5000 - // Send SIGTERM + const stopped = await this.terminateProcessGroup(entry.pid, gracefulStopMs) + if (!stopped.ok) { + if (entry.status === 'stopping') { + entry.status = previousStatus + this.notifyStatusChanged(sessionId, serviceType, previousStatus, { + port: entry.port, + cwd: entry.cwd, + command: entry.command, + error: stopped.error.message, + }) + } + return Err({ message: stopped.error.message, recoverable: true }) + } try { - process.kill(-entry.pid, 'SIGTERM') - } catch { - // Process already gone + await entry.forgetPid() + } catch (error) { + return Err({ message: this.errorWithContext(error, `Failed to forget PID for service '${serviceType}'`).message, recoverable: true }) + } + if (entry.status === 'stopping') { entry.status = 'stopped' this.notifyStatusChanged(sessionId, serviceType, 'stopped') - return Ok(undefined) } - // Wait for graceful shutdown, then SIGKILL - await new Promise((resolve) => { - const checkInterval = setInterval(() => { - try { - // Check if process is still alive (signal 0 doesn't kill, just checks) - process.kill(entry.pid, 0) - } catch { - // Process gone - clearInterval(checkInterval) - clearTimeout(killTimeout) - resolve() - } - }, 200) - - const killTimeout = setTimeout(() => { - clearInterval(checkInterval) - try { - process.kill(-entry.pid, 'SIGKILL') - } catch { - // Already gone - } - resolve() - }, gracefulStopMs) - }) - this.logger.info('Service stopped', { serviceType }) return Ok(undefined) } @@ -863,7 +1063,7 @@ export class ServiceExecutor { */ isRunning(serviceType: string): boolean { const status = this.getStatus(serviceType) - return status === 'starting' || status === 'ready' + return status === 'starting' || status === 'ready' || status === 'paused' } /** @@ -878,35 +1078,12 @@ export class ServiceExecutor { const promises: Promise[] = [] for (const [serviceType, entry] of this.services) { - if (entry.status === 'starting' || entry.status === 'ready' || entry.status === 'paused') { + if (entry.status === 'starting' || entry.status === 'ready' || entry.status === 'paused' || entry.status === 'stopping') { const gracefulStopMs = entry.config.gracefulStopMs ?? 5000 - const killPromise = new Promise((resolve) => { - try { - process.kill(-entry.pid, 'SIGTERM') - } catch { - resolve() - return - } - - const killTimeout = setTimeout(() => { - try { - process.kill(-entry.pid, 'SIGKILL') - } catch { - // Already gone - } - resolve() - }, gracefulStopMs) - - const checkInterval = setInterval(() => { - try { - process.kill(entry.pid, 0) - } catch { - clearInterval(checkInterval) - clearTimeout(killTimeout) - resolve() - } - }, 200) + const killPromise = this.terminateProcessGroup(entry.pid, gracefulStopMs).then(async (stopped) => { + if (!stopped.ok) throw stopped.error + await entry.forgetPid() }) promises.push(killPromise) @@ -916,6 +1093,7 @@ export class ServiceExecutor { } await Promise.all(promises) + await Promise.all(this.pidForgets.values()) this.services.clear() // Drain all waiters with error diff --git a/packages/sdk/src/plugins/services/services.integration.test.ts b/packages/sdk/src/plugins/services/services.integration.test.ts index 7dead3bc..e4eec30a 100644 --- a/packages/sdk/src/plugins/services/services.integration.test.ts +++ b/packages/sdk/src/plugins/services/services.integration.test.ts @@ -1,4 +1,6 @@ import { afterEach, describe, expect, it } from 'bun:test' +import { ChildProcess } from 'node:child_process' +import { EventEmitter } from 'node:events' import { mkdir, mkdtemp, readFile, rm } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -8,6 +10,7 @@ import { selectPluginState } from '~/core/sessions/reducer.js' import { SessionId } from '~/core/sessions/schema.js' import { ToolCallId } from '~/core/tools/schema.js' import { silentLogger } from '~/lib/logger/logger.js' +import type { ExecFileResult, ProcessRunner } from '~/platform/process.js' import { createNodePlatform } from '~/testing/node-platform.js' import { createTestPreset, TestHarness } from '~/testing/index.js' import { serviceEvents, servicePlugin } from './plugin.js' @@ -16,7 +19,7 @@ import { PortPool } from './port-pool.js' import { buildServiceStatusMessage } from './prompt.js' import type { ServiceCommandArgs, ServiceConfig, ServiceCwdArgs, ServiceEntry, ServiceStatus } from './schema.js' import type { ServiceStatusChangeDetails } from './service.js' -import { ServiceExecutor } from './service.js' +import { ServiceExecutor, setServiceExecutorObserverForTesting } from './service.js' // ============================================================================ // Test Service Configs @@ -108,6 +111,12 @@ async function waitForServiceStatus( } await new Promise((r) => setTimeout(r, 50)) } + const seen = (await session.getEventsByType(serviceEvents, 'service_status_changed')) + .filter((e) => e.serviceType === serviceType) + .map((e) => e.toStatus) + throw new Error( + `Timed out after ${timeoutMs}ms waiting for '${serviceType}' to reach '${targetStatus}'; saw [${seen.join(', ')}]`, + ) } /** Wait for the services plugin state slice to reflect a given status for a serviceType */ @@ -123,6 +132,33 @@ async function waitForServiceStateStatus( if (entry?.status === targetStatus) return await new Promise((r) => setTimeout(r, 20)) } + const actual = selectPluginState>(session.state, 'services')?.get(serviceType)?.status + throw new Error( + `Timed out after ${timeoutMs}ms waiting for '${serviceType}' state to be '${targetStatus}'; it is '${actual ?? 'absent'}'`, + ) +} + +/** Wait for a condition observed directly off a ServiceExecutor, not a session. */ +async function waitFor( + condition: () => boolean, + describeState: () => string, + timeoutMs = 5000, +): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (condition()) return + await new Promise((r) => setTimeout(r, 20)) + } + throw new Error(`Timed out after ${timeoutMs}ms; saw ${describeState()}`) +} + +async function waitForAsync(condition: () => Promise, describeState: () => string, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs + while (Date.now() < deadline) { + if (await condition()) return + await new Promise((resolve) => setTimeout(resolve, 20)) + } + throw new Error(`Timed out after ${timeoutMs}ms; saw ${describeState()}`) } // ============================================================================ @@ -388,7 +424,7 @@ describe('services plugin', () => { const events = await session.getEventsByType(serviceEvents, 'service_status_changed') const quickEvents = events.filter((e) => e.serviceType === 'quick') - expect(quickEvents.length).toBeGreaterThanOrEqual(1) + expect(quickEvents.map((e) => e.toStatus)).toContain('ready') }) it('agent calls service_status → returns status info', async () => { @@ -457,7 +493,7 @@ describe('services plugin', () => { const events = await session.getEventsByType(serviceEvents, 'service_status_changed') const autoEvents = events.filter((e) => e.serviceType === 'auto-start') - expect(autoEvents.length).toBeGreaterThanOrEqual(1) + expect(autoEvents.map((e) => e.toStatus)).toContain('ready') }) it('availableWhen false skips auto-start and explicit start returns an error', async () => { @@ -519,6 +555,295 @@ describe('services plugin', () => { expect(failEvent).toBeDefined() expect(failEvent!.error).toBeDefined() }) + + it('a service that closes during spawn setup skips ready and schedules restart', async () => { + const child = new ChildProcess() + const stdout = new EventEmitter() + Object.defineProperties(child, { + pid: { value: 424_242 }, + stdin: { value: null }, + stdout: { value: stdout }, + stderr: { value: null }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + // The death lands in the window between spawn and the handlers — + // the case a runtime never replays. exitCode is deliberately left + // unset so only the event can reveal it. + queueMicrotask(() => { + stdout.emit('data', Buffer.from('READY\n')) + child.emit('close', 1) + }) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + }) + const observed: Array<{ status: ServiceStatus; details: ServiceStatusChangeDetails }> = [] + executor.onStatusChanged = (_sessionId, _serviceType, status, details) => { + observed.push({ status, details }) + } + + try { + const result = await executor.start({ + type: 'missed-close', + description: 'Process is already gone when spawn returns', + command: 'unused', + readyPattern: 'READY', + restartPolicy: { maxRetries: 1, initialDelayMs: 60_000 }, + }, SessionId('s-missed-close')) + + expect(result.ok).toBe(true) + expect(observed.map(({ status }) => status)).toEqual(['starting', 'failed']) + expect(observed.filter(({ status }) => status === 'failed')).toHaveLength(1) + const failure = observed.find(({ status }) => status === 'failed') + expect(failure?.details.restartAttempt).toBe(1) + expect(failure?.details.restartMaxRetries).toBe(1) + expect(failure?.details.restartAt).toBeGreaterThan(Date.now()) + } finally { + await executor.shutdown() + } + }) + + it('bounds output produced during spawn setup while preserving its diagnostic tail', async () => { + const child = new ChildProcess() + // Bare emitters, not streams: a real child's pipe drops what it emitted + // before anything listened, and a buffering stream would hide exactly the + // loss this test is about. + const stderr = new EventEmitter() + Object.defineProperties(child, { + pid: { value: 424_243 }, + stdin: { value: null }, + stdout: { value: new EventEmitter() }, + stderr: { value: stderr }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + queueMicrotask(() => { + stderr.emit('data', Buffer.from('line-0\nline-1\nline-2\nline-3\nline-4\n')) + stderr.emit('data', Buffer.concat([Buffer.alloc(100_000, 'x'), Buffer.from('\n')])) + child.emit('close', 7) + }) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + }) + const observed: ServiceStatus[] = [] + executor.onStatusChanged = (_sessionId, _serviceType, status) => { + observed.push(status) + } + + try { + await executor.start({ + type: 'loud-crash', + description: 'Explains itself on stderr, then exits', + command: 'unused', + logBufferSize: 3, + }, SessionId('s-loud-crash')) + + await waitFor(() => observed.includes('failed'), () => `[${observed.join(', ')}]`) + + const logs = executor.getLogs('loud-crash') + expect(logs.ok).toBe(true) + if (logs.ok) { + expect(logs.value).toHaveLength(3) + expect(logs.value[0]).toBe('[stderr] line-3') + expect(logs.value[1]).toBe('[stderr] line-4') + expect(logs.value[2]?.startsWith('[stderr] xxx')).toBe(true) + expect(logs.value[2]?.length).toBe(16_384) + } + } finally { + await executor.shutdown() + } + }) + + it('does not mark an already-reaped child ready before close drains its output', async () => { + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_244 }, + exitCode: { value: 9 }, + stdin: { value: null }, + stdout: { value: new EventEmitter() }, + stderr: { value: new EventEmitter() }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + }) + const observed: ServiceStatus[] = [] + executor.onStatusChanged = (_sessionId, _serviceType, status) => observed.push(status) + + try { + const result = await executor.start({ + type: 'already-reaped', + description: 'Exit is recorded before stdio closes', + command: 'unused', + }, SessionId('s-already-reaped')) + + expect(result.ok).toBe(true) + expect(observed).toEqual(['starting']) + child.emit('close', 9) + await waitFor(() => observed.includes('failed'), () => `[${observed.join(', ')}]`) + expect(observed).toEqual(['starting', 'failed']) + } finally { + await executor.shutdown() + } + }) + + it('detects a setup ready marker before truncating an oversized partial line', async () => { + const child = new ChildProcess() + const stdout = new EventEmitter() + Object.defineProperties(child, { + pid: { value: 424_245 }, + stdin: { value: null }, + stdout: { value: stdout }, + stderr: { value: new EventEmitter() }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const recordStarted = Promise.withResolvers() + const releaseRecord = Promise.withResolvers() + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + pidRegistry: { + record: async () => { + recordStarted.resolve() + await releaseRecord.promise + }, + forget: async () => {}, + }, + }) + + try { + const started = executor.start({ + type: 'ready-before-truncation', + description: 'Ready marker precedes an oversized diagnostic tail', + command: 'unused', + readyPattern: 'READY', + logBufferSize: 1, + }, SessionId('s-ready-before-truncation')) + await recordStarted.promise + stdout.emit('data', Buffer.concat([Buffer.from('READY'), Buffer.alloc(100_000, 'x')])) + releaseRecord.resolve() + expect((await started).ok).toBe(true) + expect(executor.getStatus('ready-before-truncation')).toBe('ready') + + child.emit('close', 1) + await waitFor(() => executor.getStatus('ready-before-truncation') === 'failed', () => String(executor.getStatus('ready-before-truncation'))) + } finally { + releaseRecord.resolve() + await executor.shutdown() + } + }) + + it('keeps completed stdout and stderr diagnostics in emission order', async () => { + const child = new ChildProcess() + const stdout = new EventEmitter() + const stderr = new EventEmitter() + Object.defineProperties(child, { + pid: { value: 424_246 }, + stdin: { value: null }, + stdout: { value: stdout }, + stderr: { value: stderr }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + queueMicrotask(() => { + stdout.emit('data', Buffer.from('out-')) + stderr.emit('data', Buffer.from('err\n')) + stdout.emit('data', Buffer.from('done\n')) + child.emit('close', 1) + }) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: processRunner }) + + try { + await executor.start({ + type: 'interleaved-output', + description: 'Interleaved stream output', + command: 'unused', + }, SessionId('s-interleaved-output')) + await waitFor(() => executor.getStatus('interleaved-output') === 'failed', () => String(executor.getStatus('interleaved-output'))) + const logs = executor.getLogs('interleaved-output') + expect(logs.ok).toBe(true) + if (logs.ok) expect(logs.value).toEqual(['[stderr] err', 'out-done']) + } finally { + await executor.shutdown() + } + }) + + it('flushes unterminated stdout and stderr diagnostics in callback order', async () => { + const child = new ChildProcess() + const stdout = new EventEmitter() + const stderr = new EventEmitter() + Object.defineProperties(child, { + pid: { value: 424_247 }, + stdin: { value: null }, + stdout: { value: stdout }, + stderr: { value: stderr }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + queueMicrotask(() => { + stderr.emit('data', Buffer.from('stderr-partial')) + stdout.emit('data', Buffer.from('stdout-partial')) + child.emit('close', 1) + }) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: processRunner }) + + try { + await executor.start({ + type: 'partial-output-order', + description: 'Unterminated interleaved stream output', + command: 'unused', + }, SessionId('s-partial-output-order')) + await waitFor(() => executor.getStatus('partial-output-order') === 'failed', () => String(executor.getStatus('partial-output-order'))) + const logs = executor.getLogs('partial-output-order') + expect(logs.ok).toBe(true) + if (logs.ok) expect(logs.value).toEqual(['[stderr] stderr-partial', 'stdout-partial']) + } finally { + await executor.shutdown() + } + }) }) // ========================================================================= @@ -623,9 +948,10 @@ describe('services plugin', () => { it('kills orphaned process group from previous server instance', async () => { const eventStore = new MemoryEventStore() + const platform = createNodePlatform() - // Harness 1: start service, capture pid + port, then "crash" (shutdown - // without running onSessionClose — matches session.shutdown() behavior). + // Create the durable session first, then inject the process record that a + // crashed runtime would leave behind without calling lifecycle hooks. const harness1 = new TestHarness({ presets: [createServicesPreset([quickService], ['quick'], new PortPool())], llmProvider: MockLLMProvider.withFixedResponse({ content: 'Ok', toolCalls: [] }), @@ -634,33 +960,27 @@ describe('services plugin', () => { }) const session1 = await harness1.createSession('test') - await session1.sendAndWaitForIdle('Hi') + const sessionId = session1.sessionId + await harness1.shutdown() - const entryAgentId = session1.getEntryAgentId()! - await session1.callPluginMethod('services.start', { - sessionId: String(session1.sessionId), - agentId: String(entryAgentId), - serviceType: 'quick', + const orphan = platform.process.spawn('/bin/sh', ['-c', 'sleep 60'], { + detached: true, + stdio: 'ignore', }) - await waitForServiceStateStatus(session1, 'quick', 'ready') - - const stateBefore = selectPluginState>(session1.state, 'services')?.get('quick') - const orphanPid = stateBefore?.pid - const orphanPort = stateBefore?.port + const orphanPid = orphan.pid + const orphanPort = 41_234 expect(orphanPid).toBeDefined() - expect(orphanPort).toBeDefined() - - // Process should be alive before restart - expect(() => process.kill(orphanPid!, 0)).not.toThrow() - - const sessionId = session1.sessionId - - // Simulate server crash: sessionManager.shutdown() clears in-memory state - // but does NOT run onSessionClose, so the detached service process survives. - await harness1.sessionManager.shutdown() - - // Orphan must still be alive after "crash" - expect(() => process.kill(orphanPid!, 0)).not.toThrow() + if (orphanPid === undefined) throw new Error('Detached orphan did not receive a pid') + await eventStore.append(sessionId, { + ...serviceEvents.create('service_status_changed', { + serviceType: 'quick', + toStatus: 'starting', + port: orphanPort, + pid: orphanPid, + }), + sessionId, + }) + expect(() => process.kill(orphanPid, 0)).not.toThrow() // Harness 2: fresh SessionManager over the same event store const harness2 = new TestHarness({ @@ -679,7 +999,7 @@ describe('services plugin', () => { // Give the OS a beat to reap the killed process for (let i = 0; i < 20; i++) { try { - process.kill(orphanPid!, 0) + process.kill(orphanPid, 0) } catch { break } @@ -688,14 +1008,14 @@ describe('services plugin', () => { let isAlive = true try { - process.kill(orphanPid!, 0) + process.kill(orphanPid, 0) } catch { isAlive = false } // Safety net in case reconcile didn't kill it — don't leave zombies behind if (isAlive) { try { - process.kill(-orphanPid!, 'SIGKILL') + process.kill(-orphanPid, 'SIGKILL') } catch { // already gone } @@ -705,7 +1025,7 @@ describe('services plugin', () => { // Port preserved in state — next start() would receive it via preferredPort const stateAfter = selectPluginState>(session2.state, 'services')?.get('quick') expect(stateAfter?.status).toBe('stopped') - expect(stateAfter?.port).toBe(orphanPort!) + expect(stateAfter?.port).toBe(orphanPort) expect(stateAfter?.pid).toBeUndefined() }) }) @@ -739,6 +1059,104 @@ describe('services plugin', () => { const stoppedEvent = events.find((e) => e.serviceType === 'quick' && e.toStatus === 'stopped') expect(stoppedEvent).toBeDefined() }) + + it('closing a session stops a paused service through its lifecycle hook', async () => { + const config: ServiceConfig = { + type: 'paused-close', + description: 'Paused process closed with its session', + command: 'sleep 60', + gracefulStopMs: 50, + } + const executorCreated = Promise.withResolvers() + const stopObserving = setServiceExecutorObserverForTesting((executor) => executorCreated.resolve(executor)) + try { + const harness = createServicesHarness({ + presets: [createServicesPreset([config], ['paused-close'], new PortPool())], + llmProvider: MockLLMProvider.withFixedResponse({ content: 'Ok', toolCalls: [] }), + }) + const session = await harness.createSession('test') + const executor = await executorCreated.promise + + await session.callPluginMethod('services.start', { serviceType: 'paused-close' }) + await waitForServiceStateStatus(session, 'paused-close', 'ready') + const paused = await executor.pause('paused-close', session.sessionId) + expect(paused.ok).toBe(true) + await waitForServiceStateStatus(session, 'paused-close', 'paused') + const pid = selectPluginState>(session.state, 'services')?.get('paused-close')?.pid + if (pid === undefined) throw new Error('Service did not report its pid') + + await session.close() + await waitForServiceStateStatus(session, 'paused-close', 'stopped') + expect(() => process.kill(pid, 0)).toThrow() + } finally { + stopObserving() + } + }) + + it('restarting a paused service replaces its process', async () => { + const config: ServiceConfig = { + type: 'paused-restart', + description: 'Paused process restarted through the plugin method', + command: 'sleep 60', + gracefulStopMs: 50, + } + const executorCreated = Promise.withResolvers() + const stopObserving = setServiceExecutorObserverForTesting((executor) => executorCreated.resolve(executor)) + try { + const harness = createServicesHarness({ + presets: [createServicesPreset([config], ['paused-restart'], new PortPool())], + llmProvider: MockLLMProvider.withFixedResponse({ content: 'Ok', toolCalls: [] }), + }) + const session = await harness.createSession('test') + const executor = await executorCreated.promise + + await session.callPluginMethod('services.start', { serviceType: 'paused-restart' }) + await waitForServiceStateStatus(session, 'paused-restart', 'ready') + const originalPid = selectPluginState>(session.state, 'services')?.get('paused-restart')?.pid + if (originalPid === undefined) throw new Error('Service did not report its pid') + const paused = await executor.pause('paused-restart', session.sessionId) + expect(paused.ok).toBe(true) + await waitForServiceStateStatus(session, 'paused-restart', 'paused') + + const restarted = await session.callPluginMethod('services.restart', { serviceType: 'paused-restart' }) + expect(restarted.ok).toBe(true) + await waitForServiceStateStatus(session, 'paused-restart', 'ready') + const replacementPid = selectPluginState>(session.state, 'services')?.get('paused-restart')?.pid + expect(replacementPid).toBeDefined() + expect(replacementPid).not.toBe(originalPid) + expect(() => process.kill(originalPid, 0)).toThrow() + } finally { + stopObserving() + } + }) + + it('an externally killed paused service transitions to failed', async () => { + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process }) + const config: ServiceConfig = { + type: 'paused-crash', + description: 'Paused process killed externally', + command: 'sleep 60', + } + let pid: number | undefined + executor.onStatusChanged = (_sessionId, _serviceType, status, details) => { + if (status === 'starting') pid = details.pid + } + + try { + await executor.start(config, SessionId('s-paused-crash')) + const paused = await executor.pause('paused-crash', SessionId('s-paused-crash')) + expect(paused.ok).toBe(true) + if (pid === undefined) throw new Error('Service did not report its pid') + process.kill(-pid, 'SIGKILL') + await waitFor( + () => executor.getStatus('paused-crash') === 'failed', + () => String(executor.getStatus('paused-crash')), + ) + } finally { + await executor.shutdown() + } + }) }) // ========================================================================= @@ -804,8 +1222,12 @@ describe('services plugin', () => { } try { - const result = await executor.start(config, SessionId('s-flaky')) - expect(result.ok).toBe(true) + const [firstResult, concurrentResult] = await Promise.all([ + executor.start(config, SessionId('s-flaky')), + executor.start(config, SessionId('s-flaky')), + ]) + expect(firstResult.ok).toBe(true) + expect(concurrentResult.ok).toBe(true) await waitUntil(() => executor.getStatus('flaky-port') === 'ready') expect(executor.getStatus('flaky-port')).toBe('ready') @@ -822,6 +1244,412 @@ describe('services plugin', () => { await rm(marker, { force: true }) } }) + + it('retries a setup-window conflict inside the shared start and waits for registry deletion', async () => { + const children = [new ChildProcess(), new ChildProcess()] + const streams = children.map(() => ({ stdout: new EventEmitter(), stderr: new EventEmitter() })) + for (const [index, child] of children.entries()) { + Object.defineProperties(child, { + pid: { value: 425_000 + index }, + stdin: { value: null }, + stdout: { value: streams[index]?.stdout }, + stderr: { value: streams[index]?.stderr }, + }) + } + let spawnCount = 0 + const processRunner: ProcessRunner = { + spawn: () => { + const child = children[spawnCount] + if (!child) throw new Error(`Unexpected spawn ${spawnCount + 1}`) + spawnCount += 1 + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const firstRecordStarted = Promise.withResolvers() + const releaseFirstRecord = Promise.withResolvers() + const forgetStarted = Promise.withResolvers() + const releaseForget = Promise.withResolvers() + const operations: string[] = [] + let currentRecordedPid: number | undefined + let recordCount = 0 + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + pidRegistry: { + record: async (record) => { + recordCount += 1 + operations.push(`record:${record.pid}`) + currentRecordedPid = record.pid + if (recordCount === 1) { + firstRecordStarted.resolve() + await releaseFirstRecord.promise + } + }, + forget: async () => { + operations.push('forget:start') + forgetStarted.resolve() + await releaseForget.promise + currentRecordedPid = undefined + operations.push('forget:end') + }, + }, + }) + const observed: ServiceStatus[] = [] + executor.onStatusChanged = (_sessionId, _serviceType, status) => observed.push(status) + const config: ServiceConfig = { + type: 'controlled-conflict', + description: 'Conflict during blocked setup', + command: 'unused', + readyPattern: 'READY', + } + + try { + const firstStart = executor.start(config, SessionId('s-controlled-conflict')) + const concurrentStart = executor.start(config, SessionId('s-controlled-conflict')) + await firstRecordStarted.promise + streams[0]?.stderr.emit('data', Buffer.from('listen EADDRINUSE\n')) + children[0]?.emit('close', 1) + releaseFirstRecord.resolve() + await forgetStarted.promise + expect(spawnCount).toBe(1) + expect(recordCount).toBe(1) + + releaseForget.resolve() + await waitFor(() => spawnCount === 2, () => `spawnCount=${spawnCount}`) + streams[1]?.stdout.emit('data', Buffer.from('READY\n')) + const [firstResult, concurrentResult] = await Promise.all([firstStart, concurrentStart]) + expect(firstResult).toBe(concurrentResult) + expect(firstResult.ok).toBe(true) + expect(spawnCount).toBe(2) + expect(recordCount).toBe(2) + expect(currentRecordedPid).toBe(425_001) + expect(operations).toEqual(['record:425000', 'forget:start', 'forget:end', 'record:425001']) + expect(observed).toEqual(['starting', 'starting', 'ready']) + expect(executor.getStatus('controlled-conflict')).toBe('ready') + + children[1]?.emit('close', 1) + await waitFor(() => executor.getStatus('controlled-conflict') === 'failed', () => String(executor.getStatus('controlled-conflict'))) + } finally { + releaseFirstRecord.resolve() + releaseForget.resolve() + await executor.shutdown() + } + }) + + const exercisePidForgetBarrier = async (mode: 'explicit' | 'automatic'): Promise => { + const children = [new ChildProcess(), new ChildProcess()] + for (const [index, child] of children.entries()) { + Object.defineProperties(child, { + pid: { value: 425_100 + index }, + stdin: { value: null }, + stdout: { value: new EventEmitter() }, + stderr: { value: new EventEmitter() }, + }) + } + let spawnCount = 0 + const processRunner: ProcessRunner = { + spawn: () => { + const child = children[spawnCount] + if (!child) throw new Error(`Unexpected spawn ${spawnCount + 1}`) + spawnCount += 1 + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const forgetStarted = Promise.withResolvers() + const releaseForget = Promise.withResolvers() + const operations: string[] = [] + let currentRecordedPid: number | undefined + const terminated = new Set() + const processGone = (): Error => { + const error = new Error('ESRCH') + Object.defineProperty(error, 'code', { value: 'ESRCH' }) + return error + } + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + pidRegistry: { + record: async (record) => { + operations.push(`record:${record.pid}`) + currentRecordedPid = record.pid + }, + forget: async () => { + operations.push('forget:start') + forgetStarted.resolve() + await releaseForget.promise + currentRecordedPid = undefined + operations.push('forget:end') + }, + }, + kill: (pid, signal) => { + const processId = Math.abs(pid) + if (signal === 0) { + if (terminated.has(processId)) throw processGone() + return true + } + terminated.add(processId) + children[processId - 425_100]?.emit('close', 0) + return true + }, + }) + const config: ServiceConfig = { + type: `pid-forget-${mode}`, + description: 'Replacement waits for durable PID deletion', + command: 'unused', + restartPolicy: mode === 'automatic' ? { maxRetries: 1, initialDelayMs: 0 } : undefined, + } + + try { + await executor.start(config, SessionId(`s-pid-forget-${mode}`)) + let explicitRestart: ReturnType | undefined + if (mode === 'explicit') { + explicitRestart = executor.restart(config, SessionId('s-pid-forget-explicit')) + } else { + children[0]?.emit('close', 1) + } + + await forgetStarted.promise + await new Promise((resolve) => setTimeout(resolve, 25)) + expect(spawnCount).toBe(1) + expect(currentRecordedPid).toBe(425_100) + + releaseForget.resolve() + if (explicitRestart) expect((await explicitRestart).ok).toBe(true) + await waitFor(() => spawnCount === 2 && executor.getStatus(config.type) === 'ready', () => `spawnCount=${spawnCount}, status=${executor.getStatus(config.type)}`) + expect(currentRecordedPid).toBe(425_101) + expect(operations.slice(0, 4)).toEqual(['record:425100', 'forget:start', 'forget:end', 'record:425101']) + } finally { + releaseForget.resolve() + await executor.shutdown() + } + } + + it('blocks explicit replacement until the old PID record is deleted', async () => { + await exercisePidForgetBarrier('explicit') + }) + + it('blocks zero-delay automatic replacement until the old PID record is deleted', async () => { + await exercisePidForgetBarrier('automatic') + }) + }) + + describe('process group termination', () => { + const systemError = (code: string): Error => { + const error = new Error(code) + Object.defineProperty(error, 'code', { value: code }) + return error + } + + const isPidTerminated = async (pid: number): Promise => { + try { + const stat = await readFile(`/proc/${pid}/stat`, 'utf-8') + const rparen = stat.lastIndexOf(')') + return rparen !== -1 && stat.slice(rparen + 2).startsWith('Z ') + } catch { + return true + } + } + + it('keeps stop retryable across permission and probe errors, then accepts ESRCH', async () => { + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 426_000 }, + stdin: { value: null }, + stdout: { value: new EventEmitter() }, + stderr: { value: new EventEmitter() }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + let mode: 'permission' | 'probe-error' | 'signal-error' | 'gone' = 'permission' + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + kill: (_pid, signal) => { + if (mode === 'permission') throw systemError('EPERM') + if (mode === 'probe-error') throw systemError('EIO') + if (mode === 'signal-error') throw systemError(signal === 0 ? 'EPERM' : 'EIO') + if (signal === 0) throw systemError('ESRCH') + return true + }, + }) + await executor.start({ + type: 'retryable-stop', + description: 'Controlled process-group errors', + command: 'unused', + }, SessionId('s-retryable-stop')) + + const permission = await executor.stop('retryable-stop', SessionId('s-retryable-stop')) + expect(permission.ok).toBe(false) + expect(executor.getStatus('retryable-stop')).toBe('ready') + mode = 'probe-error' + const unexpected = await executor.stop('retryable-stop', SessionId('s-retryable-stop')) + expect(unexpected.ok).toBe(false) + expect(executor.getStatus('retryable-stop')).toBe('ready') + mode = 'signal-error' + const signalFailure = await executor.stop('retryable-stop', SessionId('s-retryable-stop')) + expect(signalFailure.ok).toBe(false) + expect(executor.getStatus('retryable-stop')).toBe('ready') + mode = 'gone' + const gone = await executor.stop('retryable-stop', SessionId('s-retryable-stop')) + expect(gone.ok).toBe(true) + expect(executor.getStatus('retryable-stop')).toBe('stopped') + await executor.shutdown() + }) + + it('does not restore a live status when close wins a later probe error', async () => { + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 426_002 }, + stdin: { value: null }, + stdout: { value: new EventEmitter() }, + stderr: { value: new EventEmitter() }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + let probeCount = 0 + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + kill: (_pid, signal) => { + if (signal !== 0) return true + probeCount += 1 + if (probeCount === 3) { + child.emit('close', 0) + throw systemError('EIO') + } + return true + }, + }) + const observed: ServiceStatus[] = [] + executor.onStatusChanged = (_sessionId, _serviceType, status) => observed.push(status) + await executor.start({ + type: 'close-before-probe-error', + description: 'Close races with a process-group probe error', + command: 'unused', + }, SessionId('s-close-before-probe-error')) + + const stopped = await executor.stop('close-before-probe-error', SessionId('s-close-before-probe-error')) + expect(stopped.ok).toBe(false) + expect(executor.getStatus('close-before-probe-error')).toBe('stopped') + expect(observed).toEqual(['starting', 'ready', 'stopping', 'stopped']) + await executor.shutdown() + }) + + it('a failed shutdown retries entries already marked stopping', async () => { + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 426_001 }, + stdin: { value: null }, + stdout: { value: new EventEmitter() }, + stderr: { value: new EventEmitter() }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + let permissionDenied = true + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { + fs: platform.fs, + process: processRunner, + kill: () => { + throw systemError(permissionDenied ? 'EPERM' : 'ESRCH') + }, + }) + await executor.start({ + type: 'retryable-shutdown', + description: 'Shutdown retries stopping entries', + command: 'unused', + }, SessionId('s-retryable-shutdown')) + + await expect(executor.shutdown()).rejects.toThrow('SIGTERM') + expect(executor.getStatus('retryable-shutdown')).toBe('stopping') + permissionDenied = false + await executor.shutdown() + expect(executor.getStatus('retryable-shutdown')).toBeNull() + }) + + const exerciseTermination = async (mode: 'stop' | 'shutdown'): Promise => { + const platform = createNodePlatform() + const executor = new ServiceExecutor(silentLogger, new PortPool(), { fs: platform.fs, process: platform.process }) + const fixtureDir = await mkdtemp(join(tmpdir(), `roj-svc-group-${mode}-`)) + const childFile = join(fixtureDir, 'child.pid') + const grandchildFile = join(fixtureDir, 'grandchild.pid') + const serviceType = `group-${mode}` + let leaderPid: number | undefined + executor.onStatusChanged = (_sessionId, _serviceType, status, details) => { + if (status === 'starting') leaderPid = details.pid + } + const config: ServiceConfig = { + type: serviceType, + description: 'Leader exits on TERM while descendants ignore it', + command: + `sh -c 'trap "" TERM; sleep 60 & echo $! > "${grandchildFile}"; wait' & echo $! > "${childFile}"; trap 'exit 0' TERM; echo READY; wait`, + readyPattern: 'READY', + gracefulStopMs: 50, + } + + try { + await executor.start(config, SessionId(`s-${serviceType}`)) + await waitFor(() => executor.getStatus(serviceType) === 'ready', () => String(executor.getStatus(serviceType))) + await waitForAsync( + async () => platform.fs.exists(childFile).then(async (childExists) => childExists && await platform.fs.exists(grandchildFile)), + () => 'waiting for child and grandchild pid files', + ) + const childPid = Number.parseInt(await readFile(childFile, 'utf-8'), 10) + const grandchildPid = Number.parseInt(await readFile(grandchildFile, 'utf-8'), 10) + expect(await isPidTerminated(childPid)).toBe(false) + expect(await isPidTerminated(grandchildPid)).toBe(false) + + if (mode === 'stop') { + const result = await executor.stop(serviceType, SessionId(`s-${serviceType}`)) + expect(result.ok).toBe(true) + } else { + await executor.shutdown() + } + + expect(await isPidTerminated(childPid)).toBe(true) + expect(await isPidTerminated(grandchildPid)).toBe(true) + } finally { + await executor.shutdown() + if (leaderPid !== undefined) { + try { + process.kill(-leaderPid, 'SIGKILL') + } catch { + // Already gone. + } + } + await rm(fixtureDir, { recursive: true, force: true }) + } + } + + it('stop waits for child and grandchild processes after the leader exits', async () => { + await exerciseTermination('stop') + }) + + it('shutdown waits for child and grandchild processes after the leader exits', async () => { + await exerciseTermination('shutdown') + }) }) // ========================================================================= diff --git a/packages/sdk/src/plugins/session-lifecycle/plugin.ts b/packages/sdk/src/plugins/session-lifecycle/plugin.ts index 3543f22e..7f481107 100644 --- a/packages/sdk/src/plugins/session-lifecycle/plugin.ts +++ b/packages/sdk/src/plugins/session-lifecycle/plugin.ts @@ -12,7 +12,7 @@ import { COMMUNICATOR_ROLE, ORCHESTRATOR_ROLE } from '~/core/agents/agent-roles. import { type DomainError, PresetErrors, SessionErrors, ValidationErrors } from '~/core/errors.js' import { definePlugin } from '~/core/plugins/index.js' import { knownDefinitionNames, unknownOverrideTargets } from '~/core/preset/overrides.js' -import { SessionId, sessionIdSchema } from '~/core/sessions/schema.js' +import { SessionId, sessionIdSchema, sessionMetadataSchema } from '~/core/sessions/schema.js' import { agentOverridesSchema, getEntryAgentId, sessionEvents, sessionOverridesPatchSchema } from '~/core/sessions/state.js' import { Err, Ok } from '~/lib/utils/result.js' @@ -158,7 +158,10 @@ export const sessionLifecyclePlugin = definePlugin('sessions') order: z4.enum(['asc', 'desc']).optional(), }), output: z4.object({ - sessions: z4.array(z4.unknown()), + // The handler returns SessionMetadata[]; z.unknown() used to hide that + // from every caller, which is how standalone-server's sessions.list + // drifted from the platform contract unnoticed. + sessions: z4.array(sessionMetadataSchema), total: z4.number(), }), handler: async (ctx, input) => { diff --git a/packages/sdk/src/plugins/shell/executor.ts b/packages/sdk/src/plugins/shell/executor.ts index e5f6c162..4ce286e1 100644 --- a/packages/sdk/src/plugins/shell/executor.ts +++ b/packages/sdk/src/plugins/shell/executor.ts @@ -337,6 +337,13 @@ export class ShellExecutor { let stdout = '' let stderr = '' let timedOut = false + let processClosed = false + let processError: Error | undefined + let exitCode: number | null = null + let exitSignal: NodeJS.Signals | null = null + let stdinSettled = input.stdin === undefined + let stdinError: Error | undefined + let settled = false let child: ChildProcess if (sandboxEnabled) { @@ -367,14 +374,6 @@ export class ShellExecutor { }) } - // Handle stdin - if (input.stdin) { - child.stdin?.write(input.stdin) - child.stdin?.end() - } else { - child.stdin?.end() - } - // Collect stdout with size cap let stdoutBytes = 0 let stdoutTruncated = false @@ -411,6 +410,55 @@ export class ShellExecutor { // Timeout handler — SIGTERM first, then SIGKILL after grace period let killTimeoutId: ReturnType | undefined + const clearTimers = () => { + clearTimeout(timeoutId) + if (killTimeoutId) clearTimeout(killTimeoutId) + } + const finishExecution = () => { + if (settled) return + if (!processClosed) return + + const durationMs = Date.now() - startTime + if (processError) { + settled = true + clearTimers() + resolve(Err({ + message: `Failed to execute command: ${processError.message}`, + recoverable: false, + details: { durationMs }, + })) + return + } + + if (!stdinSettled) return + + settled = true + clearTimers() + if (stdinError) { + resolve(Err({ + message: `Failed to deliver command stdin: ${stdinError.message}`, + recoverable: false, + details: { + stdout: stdout.trim(), + stderr: stderr.trim(), + durationMs, + exitCode: exitCode ?? -1, + signal: exitSignal ?? undefined, + timedOut, + }, + })) + return + } + + resolve(Ok({ + stdout: stdout.trim(), + stderr: stderr.trim(), + exitCode: exitCode ?? -1, + signal: exitSignal ?? undefined, + timedOut, + durationMs, + })) + } const timeoutId = setTimeout(() => { timedOut = true try { @@ -431,32 +479,57 @@ export class ShellExecutor { // Process exit child.on('close', (code, signal) => { - clearTimeout(timeoutId) - if (killTimeoutId) clearTimeout(killTimeoutId) - const durationMs = Date.now() - startTime - - resolve(Ok({ - stdout: stdout.trim(), - stderr: stderr.trim(), - exitCode: code ?? -1, - signal: signal ?? undefined, - timedOut, - durationMs, - })) + processClosed = true + exitCode = code + exitSignal = signal + if (input.stdin !== undefined && !stdinSettled) { + if (child.stdin?.writableFinished) { + stdinSettled = true + } else { + stdinError = new Error('child process closed before accepting all stdin input') + stdinSettled = true + } + } + finishExecution() }) // Process error child.on('error', (error) => { - clearTimeout(timeoutId) - if (killTimeoutId) clearTimeout(killTimeoutId) - const durationMs = Date.now() - startTime - - resolve(Err({ - message: `Failed to execute command: ${error.message}`, - recoverable: false, - details: { durationMs }, - })) + processError = error + finishExecution() }) + + // A failed stdin write must not be hidden by a successful process exit. + if (input.stdin !== undefined) { + if (!child.stdin) { + stdinError = new Error('child process stdin is unavailable') + stdinSettled = true + } else { + child.stdin.once('finish', () => { + if (stdinSettled) return + stdinSettled = true + finishExecution() + }) + child.stdin.on('error', (error) => { + if (stdinSettled) return + stdinError = error + stdinSettled = true + finishExecution() + }) + child.stdin.once('close', () => { + if (stdinSettled) return + stdinError = new Error('child process stdin closed before accepting all input') + stdinSettled = true + finishExecution() + }) + child.stdin.end(input.stdin) + } + } else { + // Keep late pipe errors contained when no input delivery was requested. + child.stdin?.on('error', () => {}) + child.stdin?.end() + } + finishExecution() }) } } diff --git a/packages/sdk/src/plugins/shell/shell.test.ts b/packages/sdk/src/plugins/shell/shell.test.ts index a41a7e49..194529e5 100644 --- a/packages/sdk/src/plugins/shell/shell.test.ts +++ b/packages/sdk/src/plugins/shell/shell.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'bun:test' +import { ChildProcess } from 'node:child_process' +import { PassThrough, Writable } from 'node:stream' import type { SessionEnvironment } from '~/core/sessions/session-environment.js' +import type { ExecFileResult, ProcessRunner } from '~/platform/process.js' import { createNodePlatform } from '~/testing/node-platform.js' import { buildBwrapArgs, type ShellConfig, ShellExecutor } from './executor.js' @@ -218,6 +221,248 @@ describe('ShellExecutor', () => { expect(result.value.exitCode).toBe(0) }) + it('returns an error when stdin delivery fails before a zero exit', async () => { + const stdin = new Writable({ + write(_chunk, _encoding, callback) { + const error = new Error('broken pipe') + Object.defineProperty(error, 'code', { value: 'EPIPE' }) + callback(error) + }, + }) + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_243 }, + stdin: { value: stdin }, + stdout: { value: null }, + stderr: { value: null }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + setTimeout(() => child.emit('close', 0, null), 0) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const executor = new ShellExecutor(defaultConfig, { + fs: testPlatform.fs, + process: processRunner, + }) + + const result = await executor.execute( + { command: 'ignores-stdin', stdin: 'data after exit' }, + createTestEnvironment(), + ) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toContain('Failed to deliver command stdin') + expect(result.error.message).toContain('broken pipe') + expect(result.error.details).toEqual(expect.objectContaining({ exitCode: 0 })) + }) + + it('returns success when stdin delivery finishes before a zero exit', async () => { + let written = '' + const stdin = new Writable({ + write(chunk, _encoding, callback) { + written += chunk.toString() + callback() + }, + }) + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_244 }, + stdin: { value: stdin }, + stdout: { value: null }, + stderr: { value: null }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + setTimeout(() => child.emit('close', 0, null), 0) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const executor = new ShellExecutor(defaultConfig, { + fs: testPlatform.fs, + process: processRunner, + }) + + const result = await executor.execute( + { command: 'reads-stdin', stdin: 'delivered input' }, + createTestEnvironment(), + ) + + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value.exitCode).toBe(0) + expect(written).toBe('delivered input') + }) + + it('gives stdin delivery failure precedence over a nonzero exit', async () => { + const stdout = new PassThrough() + const stderr = new PassThrough() + const stdin = new Writable({ + write(_chunk, _encoding, callback) { + callback(new Error('stdin rejected')) + }, + }) + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_245 }, + stdin: { value: stdin }, + stdout: { value: stdout }, + stderr: { value: stderr }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + stdout.end('partial output\n') + stderr.end('command diagnostics\n') + setTimeout(() => child.emit('close', 17, 'SIGTERM'), 0) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const executor = new ShellExecutor(defaultConfig, { + fs: testPlatform.fs, + process: processRunner, + }) + + const result = await executor.execute( + { command: 'rejects-stdin', stdin: 'undelivered input' }, + createTestEnvironment(), + ) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toContain('stdin rejected') + expect(result.error.details).toEqual(expect.objectContaining({ + stdout: 'partial output', + stderr: 'command diagnostics', + exitCode: 17, + signal: 'SIGTERM', + timedOut: false, + durationMs: expect.any(Number), + })) + }) + + it('waits for close after a process error and keeps timeout termination active', async () => { + const killSignals: NodeJS.Signals[] = [] + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_246 }, + stdin: { value: null }, + stdout: { value: null }, + stderr: { value: null }, + kill: { + value: (signal: NodeJS.Signals) => { + killSignals.push(signal) + return true + }, + }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const executor = new ShellExecutor({ ...defaultConfig, timeout: 10 }, { + fs: testPlatform.fs, + process: processRunner, + }) + let completed = false + const resultPromise = executor.execute({ command: 'runtime-error' }, createTestEnvironment()) + resultPromise.then(() => { + completed = true + }) + + child.emit('error', new Error('runtime process error')) + await Bun.sleep(30) + expect(completed).toBe(false) + expect(killSignals).toEqual(['SIGTERM']) + + child.emit('close', null, 'SIGTERM') + const result = await resultPromise + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toContain('runtime process error') + }) + + it('fails when a child closes before stdin settles', async () => { + let finishWrite: (() => void) | undefined + const stdin = new Writable({ + write(_chunk, _encoding, callback) { + finishWrite = callback + }, + }) + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_247 }, + stdin: { value: stdin }, + stdout: { value: null }, + stderr: { value: null }, + }) + const processRunner: ProcessRunner = { + spawn: () => child, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const executor = new ShellExecutor(defaultConfig, { + fs: testPlatform.fs, + process: processRunner, + }) + const resultPromise = executor.execute( + { command: 'closes-early', stdin: 'pending input' }, + createTestEnvironment(), + ) + + child.emit('close', 0, null) + const result = await resultPromise + finishWrite?.() + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toContain('closed before accepting all stdin input') + }) + + it('fails when requested stdin is unavailable', async () => { + const child = new ChildProcess() + Object.defineProperties(child, { + pid: { value: 424_248 }, + stdin: { value: null }, + stdout: { value: null }, + stderr: { value: null }, + }) + const processRunner: ProcessRunner = { + spawn: () => { + setTimeout(() => child.emit('close', 0, null), 0) + return child + }, + execFile: async (): Promise => { + throw new Error('Unexpected execFile call') + }, + } + const executor = new ShellExecutor(defaultConfig, { + fs: testPlatform.fs, + process: processRunner, + }) + + const result = await executor.execute( + { command: 'missing-stdin', stdin: 'input' }, + createTestEnvironment(), + ) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toContain('stdin is unavailable') + }) + it( 'times out long-running commands', async () => { diff --git a/packages/sdk/src/plugins/uploads/plugin.ts b/packages/sdk/src/plugins/uploads/plugin.ts index 070c4c59..90416884 100644 --- a/packages/sdk/src/plugins/uploads/plugin.ts +++ b/packages/sdk/src/plugins/uploads/plugin.ts @@ -1,11 +1,13 @@ import z from 'zod/v4' import { ValidationErrors } from '~/core/errors.js' import type { FileStore } from '~/core/file-store/types.js' +import type { InferenceContext } from '~/core/llm/provider.js' import { definePlugin } from '~/core/plugins/plugin-builder.js' import { SessionId } from '~/core/sessions/schema.js' import { getEntryAgentId } from '~/core/sessions/state.js' -import { Err, Ok } from '~/lib/utils/result.js' -import type { PreprocessorRegistry } from './preprocessor.js' +import { ArchiveBudget, type ArchiveLimitOverrides } from '~/lib/archive/index.js' +import { Err, Ok, type Result } from '~/lib/utils/result.js' +import type { PreprocessorRegistry, PreprocessorResult } from './preprocessor.js' import { generateUploadId, type MessageAttachment, UploadId, type UploadMetadata } from './schema.js' import { type PendingUpload, uploadEvents, type UploadsState } from './state.js' @@ -45,6 +47,7 @@ const ALLOWED_MIME_TYPES = [ ] const PROCESSING_TIMEOUT_MS = 120_000 // 120 seconds +const PROCESSING_ABORT_GRACE_MS = 1_000 // ============================================================================ // Config @@ -53,6 +56,51 @@ const PROCESSING_TIMEOUT_MS = 120_000 // 120 seconds export interface UploadsPluginConfig { dataFileStore: FileStore preprocessorRegistry?: PreprocessorRegistry + processingTimeoutMs?: number + processingAbortGraceMs?: number + archiveLimits?: ArchiveLimitOverrides +} + +interface UploadInput { + sessionId: string + filename: string + mimeType: string + size: number + fileBuffer: Buffer +} + +interface UploadResult { + status: 'ready' | 'failed' + extractedContent?: string + derivedPaths?: string[] + error?: string +} + +interface ActiveUpload { + controller: AbortController + completion: Promise +} + +interface UploadsPluginContext { + activeUploads: Map + deletedUploads: Set + operationQueues: Map> + closing: boolean +} + +interface PreparedUpload { + uploadId: ReturnType + uploadIdStr: string + uploadStore: FileStore + filePath: string + createdAt: number + lifecycle: UploadLifecycle +} + +interface UploadLifecycle { + controller: AbortController + start(run: (controller: AbortController) => Promise): Promise + abandon(): void } // ============================================================================ @@ -60,15 +108,7 @@ export interface UploadsPluginConfig { // ============================================================================ function isAllowedMimeType(mimeType: string): boolean { - return ALLOWED_MIME_TYPES.some((allowed) => - allowed.endsWith('/') - ? mimeType.startsWith(allowed) - : mimeType === allowed - ) -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) + return ALLOWED_MIME_TYPES.some((allowed) => (allowed.endsWith('/') ? mimeType.startsWith(allowed) : mimeType === allowed)) } function formatUploadsForLLM(uploads: PendingUpload[], sessionRoot: string): string { @@ -80,67 +120,262 @@ function formatUploadsForLLM(uploads: PendingUpload[], sessionRoot: string): str return blocks.join('\n') } -/** - * Run preprocessor (with timeout) and persist final upload metadata to disk. - * Returns the resolved status + extracted/derived data for the caller to emit. - */ -async function runPreprocessAndPersist(args: { - uploadId: string - sessionId: SessionId +function validateUploadInput(input: Pick): string | undefined { + if (input.size > MAX_FILE_SIZE) return `File too large: max ${MAX_FILE_SIZE / (1024 * 1024)}MB` + if (!isAllowedMimeType(input.mimeType)) return `Unsupported file type: ${input.mimeType}` + return undefined +} + +function reserveUploadLifecycle(pluginContext: UploadsPluginContext, uploadId: string): UploadLifecycle { + const controller = new AbortController() + if (pluginContext.closing) controller.abort(new Error('Session is closing')) + let resolveCompletion = () => {} + const completion = new Promise((resolve) => { + resolveCompletion = resolve + }) + const operation: ActiveUpload = { controller, completion } + pluginContext.activeUploads.set(uploadId, operation) + let settled = false + const settle = () => { + if (settled) return + settled = true + resolveCompletion() + if (pluginContext.activeUploads.get(uploadId) === operation) pluginContext.activeUploads.delete(uploadId) + } + return { + controller, + start(run: (signalController: AbortController) => Promise): Promise { + const result = run(controller) + result.then(settle, settle) + return result + }, + abandon: settle, + } +} + +async function prepareUpload(args: { pluginContext: UploadsPluginContext; dataFileStore: FileStore; input: UploadInput }): Promise> { + const validationError = validateUploadInput(args.input) + if (validationError) return Err(validationError) + + const uploadId = generateUploadId() + const uploadIdStr = String(uploadId) + const lifecycle = reserveUploadLifecycle(args.pluginContext, uploadIdStr) + if (lifecycle.controller.signal.aborted) { + lifecycle.abandon() + return Err('Session is closing') + } + const uploadStore = args.dataFileStore.scoped(`sessions/${args.input.sessionId}/uploads/${uploadId}`) + const writeResult = await uploadStore.write(args.input.filename, args.input.fileBuffer) + if (!writeResult.ok) { + lifecycle.abandon() + return Err('Failed to write file') + } + if (args.pluginContext.closing) { + await uploadStore.remove(args.input.filename) + lifecycle.abandon() + return Err('Session is closing') + } + return Ok({ + uploadId, + uploadIdStr, + uploadStore, + filePath: writeResult.value.path, + createdAt: Date.now(), + lifecycle, + }) +} + +type PreprocessorOutcome = { kind: 'completed'; result: Result } | { kind: 'threw'; error: unknown } + +async function runPreprocessor(args: { uploadStore: FileStore filePath: string - filename: string mimeType: string - size: number - createdAt: number preprocessorRegistry?: PreprocessorRegistry -}): Promise<{ - status: 'ready' | 'failed' - extractedContent?: string - derivedPaths?: string[] - error?: string -}> { + processingTimeoutMs?: number + processingAbortGraceMs?: number + archiveLimits?: ArchiveLimitOverrides + controller: AbortController + inferenceContext?: Omit +}): Promise { const preprocessor = args.preprocessorRegistry?.getForMimeType(args.mimeType) + if (!preprocessor) return { status: 'ready' } - let status: 'ready' | 'failed' = 'ready' - let extractedContent: string | undefined - let derivedPaths: string[] | undefined - let errorMessage: string | undefined - - if (preprocessor) { - const processPromise = preprocessor.process(args.filePath, args.mimeType, { - files: args.uploadStore, + let timedOut = false + const processPromise: Promise = (async () => { + try { + return { + kind: 'completed', + result: await preprocessor.process(args.filePath, args.mimeType, { + files: args.uploadStore, + signal: args.controller.signal, + inferenceContext: args.inferenceContext, + archiveBudget: new ArchiveBudget(args.archiveLimits), + }), + } + } catch (error) { + return { kind: 'threw', error } + } + })() + let resolveAbort: (() => void) | undefined + const aborted = new Promise<{ kind: 'aborted' }>((resolve) => { + if (args.controller.signal.aborted) { + resolve({ kind: 'aborted' }) + return + } + resolveAbort = () => resolve({ kind: 'aborted' }) + args.controller.signal.addEventListener('abort', resolveAbort, { + once: true, }) - const timeoutPromise = sleep(PROCESSING_TIMEOUT_MS).then(() => ({ - ok: false as const, - error: new Error('Processing timeout'), - })) - const result = await Promise.race([processPromise, timeoutPromise]) - if (result.ok) { - extractedContent = result.value.extractedContent - derivedPaths = result.value.derivedPaths + }) + const timeoutId = setTimeout(() => { + timedOut = true + args.controller.abort(new Error('Processing timeout')) + }, args.processingTimeoutMs ?? PROCESSING_TIMEOUT_MS) + + let outcome: PreprocessorOutcome | undefined + let graceTimeoutId: ReturnType | undefined + try { + const first = await Promise.race([processPromise, aborted]) + if (first.kind === 'aborted') { + const grace = new Promise<{ kind: 'grace-expired' }>((resolve) => { + graceTimeoutId = setTimeout(() => resolve({ kind: 'grace-expired' }), args.processingAbortGraceMs ?? PROCESSING_ABORT_GRACE_MS) + }) + const afterAbort = await Promise.race([processPromise, grace]) + if (afterAbort.kind !== 'grace-expired') outcome = afterAbort } else { - status = 'failed' - errorMessage = result.error.message + outcome = first } + } finally { + clearTimeout(timeoutId) + if (graceTimeoutId) clearTimeout(graceTimeoutId) + if (resolveAbort) args.controller.signal.removeEventListener('abort', resolveAbort) } - const metadata: UploadMetadata = { - uploadId: UploadId(args.uploadId), + if (args.controller.signal.aborted) { + return { + status: 'failed', + error: timedOut ? 'Processing timeout' : 'Processing cancelled', + } + } + if (outcome?.kind === 'threw') { + return { + status: 'failed', + error: outcome.error instanceof Error ? outcome.error.message : String(outcome.error), + } + } + if (!outcome) return { status: 'failed', error: 'Processing failed' } + if (!outcome.result.ok) return { status: 'failed', error: outcome.result.error.message } + return { + status: 'ready', + extractedContent: outcome.result.value.extractedContent, + derivedPaths: outcome.result.value.derivedPaths, + } +} + +async function writeMetadata(uploadStore: FileStore, metadata: UploadMetadata): Promise { + const result = await uploadStore.write('meta.json', JSON.stringify(metadata, null, 2)) + if (!result.ok) throw new Error(result.error) +} + +function createFinalMetadata(args: { prepared: PreparedUpload; sessionId: SessionId; input: UploadInput; result: UploadResult }): UploadMetadata { + return { + uploadId: args.prepared.uploadId, sessionId: args.sessionId, - filename: args.filename, - mimeType: args.mimeType, - size: args.size, - path: args.filePath, - status, - extractedContent, - derivedPaths, - createdAt: args.createdAt, + filename: args.input.filename, + mimeType: args.input.mimeType, + size: args.input.size, + path: args.prepared.filePath, + status: args.result.status, + extractedContent: args.result.extractedContent, + derivedPaths: args.result.derivedPaths, + error: args.result.error, + createdAt: args.prepared.createdAt, completedAt: Date.now(), } - await args.uploadStore.write('meta.json', JSON.stringify(metadata, null, 2)) +} + +function createUploadResultEvent(input: UploadInput, prepared: PreparedUpload, result: UploadResult) { + return uploadEvents.create('attachment_uploaded', { + uploadId: prepared.uploadId, + filename: input.filename, + mimeType: input.mimeType, + size: input.size, + status: result.status, + extractedContent: result.extractedContent, + derivedPaths: result.derivedPaths, + error: result.error, + }) +} + +async function withUploadLock(pluginContext: UploadsPluginContext, uploadId: string, run: () => Promise): Promise { + const previous = pluginContext.operationQueues.get(uploadId) ?? Promise.resolve() + let release = () => {} + const current = new Promise((resolve) => { + release = resolve + }) + pluginContext.operationQueues.set(uploadId, current) + await previous + try { + return await run() + } finally { + release() + if (pluginContext.operationQueues.get(uploadId) === current) pluginContext.operationQueues.delete(uploadId) + } +} + +async function processUpload(args: { + pluginContext: UploadsPluginContext + prepared: PreparedUpload + sessionId: SessionId + input: UploadInput + config: UploadsPluginConfig + inferenceContext?: Omit + onFinal: (result: UploadResult) => Promise +}): Promise { + const result = await runPreprocessor({ + uploadStore: args.prepared.uploadStore, + filePath: args.prepared.filePath, + mimeType: args.input.mimeType, + preprocessorRegistry: args.config.preprocessorRegistry, + processingTimeoutMs: args.config.processingTimeoutMs, + processingAbortGraceMs: args.config.processingAbortGraceMs, + archiveLimits: args.config.archiveLimits, + controller: args.prepared.lifecycle.controller, + inferenceContext: args.inferenceContext, + }) + await withUploadLock(args.pluginContext, args.prepared.uploadIdStr, async () => { + if (args.pluginContext.deletedUploads.has(args.prepared.uploadIdStr)) return + await writeMetadata( + args.prepared.uploadStore, + createFinalMetadata({ + prepared: args.prepared, + sessionId: args.sessionId, + input: args.input, + result, + }), + ) + if (!args.pluginContext.closing) await args.onFinal(result) + }) + return result +} - return { status, extractedContent, derivedPaths, error: errorMessage } +async function removeUploadFiles(uploadStore: FileStore, path = ''): Promise> { + const listResult = await uploadStore.list(path, { maxDepth: 1 }) + if (!listResult.ok) return listResult + for (const entry of listResult.value) { + const entryPath = path ? `${path}/${entry.name}` : entry.name + if (entry.type === 'directory') { + const nested = await removeUploadFiles(uploadStore, entryPath) + if (!nested.ok) return nested + continue + } + if (entryPath === 'meta.json') continue + if (entry.type !== 'file' && entry.type !== 'symlink') return Err(`Unsupported upload entry: ${entryPath}`) + const removed = await uploadStore.remove(entryPath) + if (!removed.ok) return removed + } + return Ok(undefined) } // ============================================================================ @@ -157,7 +392,8 @@ export const uploadsPlugin = definePlugin('uploads') reduce: (state, event) => { switch (event.type) { case 'attachment_uploaded': { - if (event.status !== 'ready') return state + const pending = state.pending.filter((upload) => String(upload.uploadId) !== String(event.uploadId)) + if (event.status !== 'ready') return { ...state, pending } const upload: PendingUpload = { uploadId: event.uploadId, filename: event.filename, @@ -167,7 +403,7 @@ export const uploadsPlugin = definePlugin('uploads') extractedContent: event.extractedContent, derivedPaths: event.derivedPaths, } - return { ...state, pending: [...state.pending, upload] } + return { ...state, pending: [...pending, upload] } } case 'attachments_consumed': { const consumedIds = new Set(event.uploadIds.map(String)) @@ -176,11 +412,25 @@ export const uploadsPlugin = definePlugin('uploads') pending: state.pending.filter((u) => !consumedIds.has(String(u.uploadId))), } } + case 'attachment_deletion_completed': { + return { + ...state, + pending: state.pending.filter((upload) => String(upload.uploadId) !== String(event.uploadId)), + } + } default: return state } }, }) + .context( + async (): Promise => ({ + activeUploads: new Map(), + deletedUploads: new Set(), + operationQueues: new Map(), + closing: false, + }), + ) .dequeue({ hasPendingMessages: (ctx) => { const uploads = ctx.pluginState @@ -191,29 +441,35 @@ export const uploadsPlugin = definePlugin('uploads') if (uploads.pending.length === 0) return null const sessionRoot = ctx.files.getRoots().session return { - messages: [{ - role: 'user', - content: formatUploadsForLLM(uploads.pending, sessionRoot), - }], + messages: [ + { + role: 'user', + content: formatUploadsForLLM(uploads.pending, sessionRoot), + }, + ], token: uploads.pending.map((u) => u.uploadId), } }, markConsumed: async (ctx, token) => { - await ctx.emitEvent(uploadEvents.create('attachments_consumed', { - agentId: ctx.agentId, - uploadIds: token.map(UploadId), - })) + await ctx.emitEvent( + uploadEvents.create('attachments_consumed', { + agentId: ctx.agentId, + uploadIds: token.map(UploadId), + }), + ) // Also mark as used on disk const { dataFileStore } = ctx.pluginConfig for (const uploadIdStr of token) { - const uploadStore = dataFileStore.scoped(`sessions/${ctx.sessionId}/uploads/${uploadIdStr}`) - const metaResult = await uploadStore.read('meta.json') - if (metaResult.ok) { + await withUploadLock(ctx.pluginContext, uploadIdStr, async () => { + const uploadStore = dataFileStore.scoped(`sessions/${ctx.sessionId}/uploads/${uploadIdStr}`) + const metaResult = await uploadStore.read('meta.json') + if (!metaResult.ok) return const meta: UploadMetadata = JSON.parse(metaResult.value) + if (meta.status === 'deleted') return meta.usedInMessageId = 'auto-dequeued' - await uploadStore.write('meta.json', JSON.stringify(meta, null, 2)) - } + await writeMetadata(uploadStore, meta) + }) } }, }) @@ -222,14 +478,16 @@ export const uploadsPlugin = definePlugin('uploads') sessionId: z.string(), }), output: z.object({ - uploads: z.array(z.object({ - uploadId: z.string(), - filename: z.string(), - mimeType: z.string(), - size: z.number(), - status: z.enum(['processing', 'ready', 'failed']), - createdAt: z.number(), - })), + uploads: z.array( + z.object({ + uploadId: z.string(), + filename: z.string(), + mimeType: z.string(), + size: z.number(), + status: z.enum(['processing', 'ready', 'failed']), + createdAt: z.number(), + }), + ), }), handler: async (ctx, input) => { const { dataFileStore } = ctx.pluginConfig @@ -288,27 +546,39 @@ export const uploadsPlugin = definePlugin('uploads') handler: async (ctx, input) => { const { dataFileStore } = ctx.pluginConfig const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${input.uploadId}`) - const metaResult = await uploadStore.read('meta.json') - - if (!metaResult.ok) { - return Err(ValidationErrors.invalid(`Upload not found: ${input.uploadId}`)) - } - - const meta: UploadMetadata = JSON.parse(metaResult.value) - - if (meta.sessionId !== input.sessionId) { - return Err(ValidationErrors.invalid('Upload does not belong to this session')) - } - - if (meta.usedInMessageId) { - return Err(ValidationErrors.invalid('Cannot delete an upload that has been sent in a message')) - } - - // Mark as deleted - meta.status = 'deleted' - await uploadStore.write('meta.json', JSON.stringify(meta, null, 2)) + const reservation = await withUploadLock(ctx.pluginContext, input.uploadId, async () => { + const metaResult = await uploadStore.read('meta.json') + if (!metaResult.ok) return Err(ValidationErrors.invalid(`Upload not found: ${input.uploadId}`)) + const meta: UploadMetadata = JSON.parse(metaResult.value) + if (meta.sessionId !== input.sessionId) { + return Err(ValidationErrors.invalid('Upload does not belong to this session')) + } + if (meta.usedInMessageId) { + return Err(ValidationErrors.invalid('Cannot delete an upload that has been sent in a message')) + } + ctx.pluginContext.deletedUploads.add(input.uploadId) + const active = ctx.pluginContext.activeUploads.get(input.uploadId) + active?.controller.abort(new Error('Upload deleted')) + return Ok(active?.completion) + }) + if (!reservation.ok) return reservation + await reservation.value - return Ok({}) + return withUploadLock(ctx.pluginContext, input.uploadId, async () => { + const metaResult = await uploadStore.read('meta.json') + if (!metaResult.ok) return Err(ValidationErrors.invalid(`Upload not found: ${input.uploadId}`)) + const meta: UploadMetadata = JSON.parse(metaResult.value) + meta.status = 'deleted' + await writeMetadata(uploadStore, meta) + const cleanup = await removeUploadFiles(uploadStore) + if (!cleanup.ok) return Err(ValidationErrors.invalid(`Could not remove upload: ${input.uploadId}`)) + await ctx.emitEvent( + uploadEvents.create('attachment_deletion_completed', { + uploadId: UploadId(input.uploadId), + }), + ) + return Ok({}) + }) }, }) .method('loadAttachments', { @@ -365,13 +635,19 @@ export const uploadsPlugin = definePlugin('uploads') const { dataFileStore } = ctx.pluginConfig for (const uploadIdStr of input.uploadIds) { - const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${uploadIdStr}`) - const metaResult = await uploadStore.read('meta.json') - if (metaResult.ok) { + const result = await withUploadLock(ctx.pluginContext, uploadIdStr, async () => { + const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${uploadIdStr}`) + const metaResult = await uploadStore.read('meta.json') + if (!metaResult.ok || ctx.pluginContext.deletedUploads.has(uploadIdStr)) { + return Err(ValidationErrors.invalid(`Upload not found: ${uploadIdStr}`)) + } const meta: UploadMetadata = JSON.parse(metaResult.value) + if (meta.status === 'deleted') return Err(ValidationErrors.invalid(`Upload not found: ${uploadIdStr}`)) meta.usedInMessageId = input.messageId - await uploadStore.write('meta.json', JSON.stringify(meta, null, 2)) - } + await writeMetadata(uploadStore, meta) + return Ok({}) + }) + if (!result.ok) return result } return Ok({}) @@ -391,56 +667,43 @@ export const uploadsPlugin = definePlugin('uploads') extractedContent: z.string().optional(), }), handler: async (ctx, input) => { - const { dataFileStore, preprocessorRegistry } = ctx.pluginConfig - - if (input.size > MAX_FILE_SIZE) { - return Err(ValidationErrors.invalid(`File too large: max ${MAX_FILE_SIZE / (1024 * 1024)}MB`)) - } - if (!isAllowedMimeType(input.mimeType)) { - return Err(ValidationErrors.invalid(`Unsupported file type: ${input.mimeType}`)) - } - - const uploadId = generateUploadId() - const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${uploadId}`) - - const writeResult = await uploadStore.write(input.filename, input.fileBuffer) - if (!writeResult.ok) { - return Err(ValidationErrors.invalid('Failed to write file')) - } - - const result = await runPreprocessAndPersist({ - uploadId: String(uploadId), - sessionId: ctx.sessionId, - uploadStore, - filePath: writeResult.value.path, - filename: input.filename, - mimeType: input.mimeType, - size: input.size, - createdAt: Date.now(), - preprocessorRegistry, + const preparedResult = await prepareUpload({ + pluginContext: ctx.pluginContext, + dataFileStore: ctx.pluginConfig.dataFileStore, + input, }) - - await ctx.emitEvent(uploadEvents.create('attachment_uploaded', { - uploadId, - filename: input.filename, - mimeType: input.mimeType, - size: input.size, - status: result.status, - extractedContent: result.extractedContent, - derivedPaths: result.derivedPaths, - error: result.error, - })) - + if (!preparedResult.ok) return Err(ValidationErrors.invalid(preparedResult.error)) + const prepared = preparedResult.value const entryAgentId = getEntryAgentId(ctx.sessionState) - if (result.status === 'ready' && entryAgentId) { - ctx.scheduleAgent(entryAgentId) + try { + const result = await prepared.lifecycle.start(() => + processUpload({ + pluginContext: ctx.pluginContext, + prepared, + sessionId: ctx.sessionId, + input, + config: ctx.pluginConfig, + inferenceContext: entryAgentId + ? { + sessionId: String(ctx.sessionId), + agentId: String(entryAgentId), + fileStore: ctx.files, + } + : undefined, + onFinal: async (finalResult) => { + await ctx.emitEvent(createUploadResultEvent(input, prepared, finalResult)) + if (finalResult.status === 'ready' && entryAgentId) ctx.scheduleAgent(entryAgentId) + }, + }), + ) + return Ok({ + uploadId: prepared.uploadIdStr, + status: result.status, + extractedContent: result.extractedContent, + }) + } catch (error) { + return Err(ValidationErrors.invalid(error instanceof Error ? error.message : 'Upload processing failed')) } - - return Ok({ - uploadId: String(uploadId), - status: result.status, - extractedContent: result.extractedContent, - }) }, }) .method('uploadAsync', { @@ -456,126 +719,146 @@ export const uploadsPlugin = definePlugin('uploads') status: z.enum(['processing']), }), handler: async (ctx, input) => { - const { dataFileStore, preprocessorRegistry } = ctx.pluginConfig - - if (input.size > MAX_FILE_SIZE) { - return Err(ValidationErrors.invalid(`File too large: max ${MAX_FILE_SIZE / (1024 * 1024)}MB`)) - } - if (!isAllowedMimeType(input.mimeType)) { - return Err(ValidationErrors.invalid(`Unsupported file type: ${input.mimeType}`)) - } - - const uploadId = generateUploadId() - const uploadIdStr = String(uploadId) - const uploadStore = dataFileStore.scoped(`sessions/${input.sessionId}/uploads/${uploadId}`) - - const writeResult = await uploadStore.write(input.filename, input.fileBuffer) - if (!writeResult.ok) { - return Err(ValidationErrors.invalid('Failed to write file')) - } - - const filePath = writeResult.value.path - const createdAt = Date.now() - - // Persist initial 'processing' metadata so listPending sees it before preprocessor finishes. + const preparedResult = await prepareUpload({ + pluginContext: ctx.pluginContext, + dataFileStore: ctx.pluginConfig.dataFileStore, + input, + }) + if (!preparedResult.ok) return Err(ValidationErrors.invalid(preparedResult.error)) + const prepared = preparedResult.value const processingMeta: UploadMetadata = { - uploadId, + uploadId: prepared.uploadId, sessionId: ctx.sessionId, filename: input.filename, mimeType: input.mimeType, size: input.size, - path: filePath, + path: prepared.filePath, status: 'processing', - createdAt, - completedAt: createdAt, + createdAt: prepared.createdAt, + } + try { + await writeMetadata(prepared.uploadStore, processingMeta) + await ctx.emitEvent( + uploadEvents.create('attachment_uploaded', { + uploadId: prepared.uploadId, + filename: input.filename, + mimeType: input.mimeType, + size: input.size, + status: 'processing', + }), + ) + } catch (error) { + prepared.lifecycle.abandon() + return Err(ValidationErrors.invalid(error instanceof Error ? error.message : 'Could not start upload')) } - await uploadStore.write('meta.json', JSON.stringify(processingMeta, null, 2)) - - await ctx.emitEvent(uploadEvents.create('attachment_uploaded', { - uploadId, - filename: input.filename, - mimeType: input.mimeType, - size: input.size, - status: 'processing', - })) ctx.notify('uploadStatusChanged', { sessionId: input.sessionId, - uploadId: uploadIdStr, + uploadId: prepared.uploadIdStr, status: 'processing', }) - // Capture refs from ctx before the handler returns — `notify`/`emitEvent` - // closures stay valid for the lifetime of the session, which in roj - // outlives any single handler call. const { emitEvent, notify, logger, scheduleAgent } = ctx const sessionId = ctx.sessionId const entryAgentId = getEntryAgentId(ctx.sessionState) - - void (async () => { - try { - const result = await runPreprocessAndPersist({ - uploadId: uploadIdStr, + void prepared.lifecycle + .start(() => + processUpload({ + pluginContext: ctx.pluginContext, + prepared, sessionId, - uploadStore, - filePath, - filename: input.filename, - mimeType: input.mimeType, - size: input.size, - createdAt, - preprocessorRegistry, - }) - - await emitEvent(uploadEvents.create('attachment_uploaded', { - uploadId, - filename: input.filename, - mimeType: input.mimeType, - size: input.size, - status: result.status, - extractedContent: result.extractedContent, - derivedPaths: result.derivedPaths, - error: result.error, - })) - if (result.status === 'ready' && entryAgentId) { - scheduleAgent(entryAgentId) - } - notify('uploadStatusChanged', { - sessionId: input.sessionId, - uploadId: uploadIdStr, - status: result.status, - extractedContent: result.extractedContent, - error: result.error, - }) - } catch (err) { - const message = err instanceof Error ? err.message : String(err) - logger.error('Async upload processing crashed', err instanceof Error ? err : undefined, { - uploadId: uploadIdStr, + input, + config: ctx.pluginConfig, + inferenceContext: entryAgentId + ? { + sessionId: String(sessionId), + agentId: String(entryAgentId), + fileStore: ctx.files, + } + : undefined, + onFinal: async (result) => { + try { + await emitEvent(createUploadResultEvent(input, prepared, result)) + } catch (error) { + logger.error('Could not persist upload result event', error instanceof Error ? error : undefined, { + uploadId: prepared.uploadIdStr, + }) + } + if (result.status === 'ready' && entryAgentId) scheduleAgent(entryAgentId) + notify('uploadStatusChanged', { + sessionId: input.sessionId, + uploadId: prepared.uploadIdStr, + status: result.status, + extractedContent: result.extractedContent, + error: result.error, + }) + }, + }), + ) + .catch((error) => { + logger.error('Async upload processing crashed', error instanceof Error ? error : undefined, { + uploadId: prepared.uploadIdStr, filename: input.filename, }) - try { - await emitEvent(uploadEvents.create('attachment_uploaded', { - uploadId, - filename: input.filename, - mimeType: input.mimeType, - size: input.size, - status: 'failed', - error: message, - })) - } catch { - // Even event emission failed — best-effort; nothing useful left to do. - } - notify('uploadStatusChanged', { - sessionId: input.sessionId, - uploadId: uploadIdStr, - status: 'failed', - error: message, - }) - } - })() + }) return Ok({ - uploadId: uploadIdStr, - status: 'processing' as const, + uploadId: prepared.uploadIdStr, + status: 'processing', }) }, }) + .sessionHook('onSessionReady', async (ctx) => { + const uploadsPath = `sessions/${ctx.sessionId}/uploads` + const listResult = await ctx.pluginConfig.dataFileStore.list(uploadsPath) + if (!listResult.ok) return + + for (const entry of listResult.value) { + if (entry.type !== 'directory') continue + const uploadStore = ctx.pluginConfig.dataFileStore.scoped(`${uploadsPath}/${entry.name}`) + try { + const metaResult = await uploadStore.read('meta.json') + if (!metaResult.ok) continue + const metadata: UploadMetadata = JSON.parse(metaResult.value) + if (String(metadata.sessionId) !== String(ctx.sessionId)) continue + if (metadata.status === 'deleted') { + ctx.pluginContext.deletedUploads.add(String(metadata.uploadId)) + continue + } + if (metadata.status !== 'processing') continue + + metadata.status = 'failed' + metadata.error = 'Processing interrupted by restart' + metadata.completedAt = Date.now() + await writeMetadata(uploadStore, metadata) + await ctx.emitEvent( + uploadEvents.create('attachment_uploaded', { + uploadId: metadata.uploadId, + filename: metadata.filename, + mimeType: metadata.mimeType, + size: metadata.size, + status: 'failed', + error: metadata.error, + }), + ) + ctx.notify('uploadStatusChanged', { + sessionId: String(ctx.sessionId), + uploadId: String(metadata.uploadId), + status: 'failed', + error: metadata.error, + }) + } catch (error) { + ctx.logger.warn('Could not recover interrupted upload', { + uploadId: entry.name, + error: error instanceof Error ? error.message : String(error), + }) + } + } + }) + .sessionHook('onSessionClose', async (ctx) => { + ctx.pluginContext.closing = true + const active = [...ctx.pluginContext.activeUploads.values()] + for (const upload of active) upload.controller.abort(new Error('Session is closing')) + await Promise.all(active.map((upload) => upload.completion)) + ctx.pluginContext.activeUploads.clear() + }) .build() diff --git a/packages/sdk/src/plugins/uploads/preprocessor.ts b/packages/sdk/src/plugins/uploads/preprocessor.ts index b08df5a0..d4d4a747 100644 --- a/packages/sdk/src/plugins/uploads/preprocessor.ts +++ b/packages/sdk/src/plugins/uploads/preprocessor.ts @@ -9,6 +9,8 @@ */ import type { FileStore } from '~/core/file-store/types.js' +import type { InferenceContext } from '~/core/llm/provider.js' +import type { ArchiveBudget } from '~/lib/archive/index.js' import type { Result } from '~/lib/utils/result.js' // ============================================================================ @@ -21,6 +23,29 @@ import type { Result } from '~/lib/utils/result.js' export interface PreprocessorContext { /** FileStore scoped to upload directory for writing derived files */ files: FileStore + /** + * Cancels subprocesses, inference calls, and derived-file generation. + * Implementations should settle promptly; the host stops awaiting after a bounded grace period. + */ + signal?: AbortSignal + /** Optional metadata required to pass cancellation through LLM providers. */ + inferenceContext?: Omit + /** Shared extraction budget for all ZIPs nested within one upload. */ + archiveBudget?: ArchiveBudget +} + +const NEVER_ABORTED_SIGNAL = new AbortController().signal + +export function getPreprocessingSignal(ctx: PreprocessorContext): AbortSignal { + return ctx.signal ?? NEVER_ABORTED_SIGNAL +} + +export function preprocessingAbortError(signal: AbortSignal): Error { + return signal.reason instanceof Error ? signal.reason : new Error('Processing cancelled') +} + +export function throwIfPreprocessingAborted(signal: AbortSignal): void { + if (signal.aborted) throw preprocessingAbortError(signal) } // ============================================================================ diff --git a/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.test.ts b/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.test.ts index 67d87fd8..7e5ce00a 100644 --- a/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.test.ts +++ b/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.test.ts @@ -3,15 +3,18 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { SessionFileStore } from '~/core/file-store/file-store.js' +import type { ImageResizer } from '~/core/image/types.js' import { MockLLMProvider } from '~/core/llm/mock.js' +import type { InferenceContext, LLMProvider } from '~/core/llm/provider.js' import { Semaphore } from '~/lib/utils/concurrency.js' +import { Ok } from '~/lib/utils/result.js' import { silentLogger } from '../../../lib/logger/logger.js' import { createNodePlatform } from '../../../testing/node-platform.js' import { ImageClassifierPreprocessor } from './image-classifier.js' function defer(): { promise: Promise; resolve: (v: T) => void } { let resolve!: (v: T) => void - const promise = new Promise((res) => { + const promise = new Promise(res => { resolve = res }) return { promise, resolve } @@ -71,9 +74,13 @@ describe('ImageClassifierPreprocessor.gate', () => { ) const fileStore = new SessionFileStore(workDir, undefined, false, platform.fs, 'session') + const signal = new AbortController().signal const tasks = imagePaths.map((p, i) => - classifier.process(p, 'image/png', { files: fileStore.scoped(`img-${i}-meta`) }), + classifier.process(p, 'image/png', { + files: fileStore.scoped(`img-${i}-meta`), + signal, + }), ) // Let the workers queue up; the first LIMIT should be in-flight. @@ -129,7 +136,9 @@ describe('ImageClassifierPreprocessor.gate', () => { const fileStore = new SessionFileStore(workDir, undefined, false, platform.fs, 'session') const tasks = imagePaths.map((p, i) => - classifier.process(p, 'image/png', { files: fileStore.scoped(`nogate-${i}-meta`) }), + classifier.process(p, 'image/png', { + files: fileStore.scoped(`nogate-${i}-meta`), + }), ) await new Promise(r => setTimeout(r, 20)) @@ -139,4 +148,138 @@ describe('ImageClassifierPreprocessor.gate', () => { await Promise.all(tasks) expect(peak).toBe(N) }) + + it('aborts while queued without invoking the provider', async () => { + const gate = new Semaphore(1) + const holderRelease = defer() + const holder = gate.run(async () => holderRelease.promise) + const llmProvider = MockLLMProvider.withFixedResponse({ content: 'desc' }) + const classifier = new ImageClassifierPreprocessor({ + llmProvider, + logger: silentLogger, + fs: platform.fs, + gate, + }) + const imagePath = join(workDir, 'queued-abort.png') + await writeFile(imagePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])) + const fileStore = new SessionFileStore(workDir, undefined, false, platform.fs, 'session') + const controller = new AbortController() + const reason = new Error('cancel queued classifier') + + const processing = classifier.process(imagePath, 'image/png', { + files: fileStore.scoped('queued-abort-meta'), + signal: controller.signal, + }) + await new Promise(resolve => setTimeout(resolve, 0)) + controller.abort(reason) + const result = await processing + + expect(result).toEqual({ ok: false, error: reason }) + expect(llmProvider.getCallCount()).toBe(0) + holderRelease.resolve() + await holder + }) + + it('deletes a resized temp when cancellation wins after resize', async () => { + const controller = new AbortController() + const reason = new Error('cancel after resize') + const tempPath = join(workDir, 'cancelled-resize.jpg') + const imagePath = join(workDir, 'cancel-after-resize.png') + await writeFile(imagePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])) + const imageResizer: ImageResizer = { + async resize() { + await writeFile(tempPath, Buffer.from('resized')) + controller.abort(reason) + return { path: tempPath, mimeType: 'image/jpeg', tempFile: tempPath } + }, + } + const classifier = new ImageClassifierPreprocessor({ + llmProvider: MockLLMProvider.withFixedResponse({ content: 'unused' }), + logger: silentLogger, + fs: platform.fs, + imageResizer, + }) + const fileStore = new SessionFileStore(workDir, undefined, false, platform.fs, 'session') + + const result = await classifier.process(imagePath, 'image/png', { + files: fileStore.scoped('cancel-after-resize-meta'), + signal: controller.signal, + }) + + expect(result).toEqual({ ok: false, error: reason }) + expect(await Bun.file(tempPath).exists()).toBe(false) + }) + + it('deletes a resized temp when reading it fails before ownership is returned', async () => { + const tempPath = join(workDir, 'unreadable-resize.jpg') + const missingPath = join(workDir, 'missing-resize.jpg') + const imagePath = join(workDir, 'read-failure.png') + await writeFile(imagePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])) + const imageResizer: ImageResizer = { + async resize() { + await writeFile(tempPath, Buffer.from('resized')) + return { + path: missingPath, + mimeType: 'image/jpeg', + tempFile: tempPath, + } + }, + } + const classifier = new ImageClassifierPreprocessor({ + llmProvider: MockLLMProvider.withFixedResponse({ + content: 'fallback description', + }), + logger: silentLogger, + fs: platform.fs, + imageResizer, + }) + const fileStore = new SessionFileStore(workDir, undefined, false, platform.fs, 'session') + + const result = await classifier.process(imagePath, 'image/png', { + files: fileStore.scoped('read-failure-meta'), + }) + + expect(result.ok).toBe(true) + expect(await Bun.file(tempPath).exists()).toBe(false) + }) + + it('passes the preprocessing signal to provider inference context', async () => { + const contexts: Array = [] + const llmProvider: LLMProvider = { + name: 'context-capture', + async inference(_request, context) { + contexts.push(context) + return Ok({ + content: 'desc', + toolCalls: [], + finishReason: 'stop', + metrics: MockLLMProvider.defaultMetrics(), + }) + }, + } + const classifier = new ImageClassifierPreprocessor({ + llmProvider, + logger: silentLogger, + fs: platform.fs, + }) + const imagePath = join(workDir, 'provider-signal.png') + await writeFile(imagePath, Buffer.from([0x89, 0x50, 0x4e, 0x47])) + const fileStore = new SessionFileStore(workDir, undefined, false, platform.fs, 'session') + const controller = new AbortController() + const inferenceContext = { + sessionId: 'session', + agentId: 'agent', + fileStore, + } + + const result = await classifier.process(imagePath, 'image/png', { + files: fileStore.scoped('provider-signal-meta'), + signal: controller.signal, + inferenceContext, + }) + + expect(result.ok).toBe(true) + expect(contexts).toHaveLength(1) + expect(contexts[0]?.signal).toBe(controller.signal) + }) }) diff --git a/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.ts b/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.ts index b93770db..8bdae671 100644 --- a/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.ts +++ b/packages/sdk/src/plugins/uploads/preprocessors/image-classifier.ts @@ -13,7 +13,14 @@ import type { Result } from '~/lib/utils/result.js' import { Err, Ok } from '~/lib/utils/result.js' import type { FileSystem } from '~/platform/fs.js' import type { Logger } from '../../../lib/logger/logger.js' -import type { Preprocessor, PreprocessorContext, PreprocessorResult } from '../preprocessor.js' +import { + getPreprocessingSignal, + preprocessingAbortError, + throwIfPreprocessingAborted, + type Preprocessor, + type PreprocessorContext, + type PreprocessorResult, +} from '../preprocessor.js' /** * Anthropic vision API internally downsamples images to ~1568px long side. @@ -92,7 +99,9 @@ export class ImageClassifierPreprocessor implements Preprocessor { ctx: PreprocessorContext, ): Promise> { const totalStart = Date.now() + const signal = getPreprocessingSignal(ctx) try { + throwIfPreprocessingAborted(signal) // Check + stat image file if (!(await this.fs.exists(filePath))) { return Err(new Error(`Image file not found: ${filePath}`)) @@ -110,11 +119,12 @@ export class ImageClassifierPreprocessor implements Preprocessor { // Try vision inference const inferenceStart = Date.now() - const description = await this.describeImage(filePath, mimeType) + const description = await this.describeImage(filePath, mimeType, ctx, signal) const inferenceDurationMs = Date.now() - inferenceStart if (description) { // Save description to file + throwIfPreprocessingAborted(signal) const writeResult = await ctx.files.write('description.txt', description) this.logger.info('Image described successfully', { @@ -143,6 +153,7 @@ export class ImageClassifierPreprocessor implements Preprocessor { extractedContent: `[Image: ${filename}, ${this.formatSize(size)}, ${mimeType}]`, }) } catch (error) { + if (signal.aborted) return Err(preprocessingAbortError(signal)) this.logger.error( 'Image classification failed', error instanceof Error ? error : undefined, @@ -164,34 +175,42 @@ export class ImageClassifierPreprocessor implements Preprocessor { private async describeImage( filePath: string, mimeType: string, + ctx: PreprocessorContext, + signal: AbortSignal, ): Promise { try { - const { url: imageUrl, cleanup } = await this.prepareImageUrl(filePath, mimeType) + const { url: imageUrl, cleanup } = await this.prepareImageUrl(filePath, mimeType, signal) try { - const inferenceCall = () => this.llmProvider.inference({ - model: this.visionModel, - systemPrompt: 'You are an image description assistant. Describe images concisely in 1-2 sentences.', - messages: [ - { - role: 'user', - content: [ - { - type: 'text', - text: 'Please describe this image concisely in 1-2 sentences. Focus on the main subject and any text visible.', - }, - { - type: 'image_url', - imageUrl: { url: imageUrl }, - }, - ], - }, - ], - maxTokens: 200, - temperature: 0.3, - }) + const inferenceCall = () => { + throwIfPreprocessingAborted(signal) + return this.llmProvider.inference({ + model: this.visionModel, + systemPrompt: 'You are an image description assistant. Describe images concisely in 1-2 sentences.', + messages: [ + { + role: 'user', + content: [ + { + type: 'text', + text: 'Please describe this image concisely in 1-2 sentences. Focus on the main subject and any text visible.', + }, + { + type: 'image_url', + imageUrl: { url: imageUrl }, + }, + ], + }, + ], + maxTokens: 200, + temperature: 0.3, + }, ctx.inferenceContext ? { + ...ctx.inferenceContext, + signal, + } : undefined) + } - const result = await (this.gate ? this.gate.run(inferenceCall) : inferenceCall()) + const result = await (this.gate ? this.gate.run(inferenceCall, signal) : inferenceCall()) if (result.ok && result.value.content) { return result.value.content.trim() @@ -202,6 +221,7 @@ export class ImageClassifierPreprocessor implements Preprocessor { await cleanup() } } catch (error) { + if (signal.aborted) throw preprocessingAbortError(signal) this.logger.warn('Vision inference failed', { error: error instanceof Error ? error.message : String(error), }) @@ -221,17 +241,24 @@ export class ImageClassifierPreprocessor implements Preprocessor { private async prepareImageUrl( filePath: string, mimeType: string, + signal: AbortSignal, ): Promise<{ url: string; cleanup: () => Promise }> { + throwIfPreprocessingAborted(signal) if (!this.imageResizer) { return { url: `file://${filePath}`, cleanup: async () => {} } } + let resizedTempFile: string | undefined try { const resized = await this.imageResizer.resize(filePath, mimeType, { maxDimension: CLASSIFY_MAX_DIMENSION, maxFileSizeBytes: CLASSIFY_MAX_FILE_SIZE_BYTES, + signal, }) + resizedTempFile = resized.tempFile + throwIfPreprocessingAborted(signal) const buffer = await this.fs.readFile(resized.path) + throwIfPreprocessingAborted(signal) const base64 = buffer.toString('base64') return { url: `data:${resized.mimeType};base64,${base64}`, @@ -242,6 +269,10 @@ export class ImageClassifierPreprocessor implements Preprocessor { }, } } catch (error) { + if (resizedTempFile) { + await this.fs.unlink(resizedTempFile).catch(() => {}) + } + if (signal.aborted) throw preprocessingAbortError(signal) this.logger.warn('Pre-resize for classification failed, falling back to file:// URL', { filePath, error: error instanceof Error ? error.message : String(error), diff --git a/packages/sdk/src/plugins/uploads/preprocessors/markitdown-preprocessor.ts b/packages/sdk/src/plugins/uploads/preprocessors/markitdown-preprocessor.ts index 9472d57b..71af17e9 100644 --- a/packages/sdk/src/plugins/uploads/preprocessors/markitdown-preprocessor.ts +++ b/packages/sdk/src/plugins/uploads/preprocessors/markitdown-preprocessor.ts @@ -23,7 +23,15 @@ import type { FileSystem } from '~/platform/fs.js' import type { ProcessRunner } from '~/platform/process.js' import type { FileStore } from '../../../core/file-store/types.js' import type { Logger } from '../../../lib/logger/logger.js' -import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, PreprocessorResult } from '../preprocessor.js' +import { + getPreprocessingSignal, + preprocessingAbortError, + throwIfPreprocessingAborted, + type Preprocessor, + type PreprocessorContext, + type PreprocessorRegistry, + type PreprocessorResult, +} from '../preprocessor.js' const MAX_IMAGES = 20 const IMAGE_CLASSIFY_CONCURRENCY = 10 @@ -49,8 +57,8 @@ const MARKITDOWN_TIMEOUT_MS = 60_000 const IMAGE_EXTRACT_TIMEOUT_MS = 5 * 60_000 function makeExec(processRunner: ProcessRunner) { - return (cmd: string, args: string[], timeoutMs: number = MARKITDOWN_TIMEOUT_MS) => - processRunner.execFile(cmd, args, { timeout: timeoutMs, maxBuffer: 50 * 1024 * 1024 }) + return (cmd: string, args: string[], timeoutMs: number = MARKITDOWN_TIMEOUT_MS, signal?: AbortSignal) => + processRunner.execFile(cmd, args, { timeout: timeoutMs, maxBuffer: 50 * 1024 * 1024, signal }) } /** MIME types where markitdown converts to markdown (non-ZIP, non-image, non-PDF) */ @@ -97,7 +105,7 @@ export class MarkitdownPreprocessor implements Preprocessor { private readonly logger: Logger private readonly fs: FileSystem private readonly processRunner: ProcessRunner - private readonly exec: (cmd: string, args: string[], timeoutMs?: number) => Promise<{ stdout: string; stderr: string }> + private readonly exec: (cmd: string, args: string[], timeoutMs?: number, signal?: AbortSignal) => Promise<{ stdout: string; stderr: string }> constructor(config: MarkitdownPreprocessorConfig) { this.registry = config.registry @@ -113,6 +121,8 @@ export class MarkitdownPreprocessor implements Preprocessor { ctx: PreprocessorContext, ): Promise> { const totalStart = Date.now() + const signal = getPreprocessingSignal(ctx) + throwIfPreprocessingAborted(signal) this.logger.info('Markitdown processing started', { filePath, mimeType }) @@ -126,12 +136,13 @@ export class MarkitdownPreprocessor implements Preprocessor { // independent, so there's no reason to serialize them. For documents // where pandoc extraction isn't applicable, the image task resolves // immediately. - const markdownTask = this.runMarkitdown(filePath, mimeType, contentPathResult.value) + const markdownTask = this.runMarkitdown(filePath, mimeType, contentPathResult.value, signal) const imageTask = PANDOC_EXTRACT_MIMES.has(mimeType) - ? this.extractImagesWithPandoc(filePath, mimeType, ctx) + ? this.extractImagesWithPandoc(filePath, mimeType, ctx, signal) : Promise.resolve>([]) const [markdownResult, images] = await Promise.all([markdownTask, imageTask]) + if (signal.aborted) return Err(preprocessingAbortError(signal)) if (!markdownResult.ok) return markdownResult @@ -166,11 +177,13 @@ export class MarkitdownPreprocessor implements Preprocessor { filePath: string, mimeType: string, outputPath: string, + signal: AbortSignal, ): Promise> { const markitdownStart = Date.now() try { - await this.exec('markitdown', [filePath, '-o', outputPath]) + await this.exec('markitdown', [filePath, '-o', outputPath], MARKITDOWN_TIMEOUT_MS, signal) } catch (error) { + if (signal.aborted) return Err(preprocessingAbortError(signal)) const message = error instanceof Error ? error.message : String(error) this.logger.error( 'markitdown CLI failed', @@ -182,6 +195,7 @@ export class MarkitdownPreprocessor implements Preprocessor { } return Err(new Error(`markitdown failed: ${message}`)) } + if (signal.aborted) return Err(preprocessingAbortError(signal)) let markdown = '' try { @@ -204,7 +218,9 @@ export class MarkitdownPreprocessor implements Preprocessor { filePath: string, mimeType: string, ctx: PreprocessorContext, + signal: AbortSignal, ): Promise> { + if (signal.aborted) return [] const mediaStore = ctx.files.scoped('media') const mediaDirResult = mediaStore.realPath('') if (!mediaDirResult.ok) return [] @@ -219,8 +235,10 @@ export class MarkitdownPreprocessor implements Preprocessor { 'pandoc', ['-f', format, '-t', 'gfm', filePath, '-o', '/dev/null', `--extract-media=${mediaDirResult.value}`], IMAGE_EXTRACT_TIMEOUT_MS, + signal, ) } catch (error) { + if (signal.aborted) return [] extractSucceeded = false this.logger.warn('pandoc --extract-media failed (will classify any partial output)', { filePath, @@ -228,6 +246,7 @@ export class MarkitdownPreprocessor implements Preprocessor { error: error instanceof Error ? error.message : String(error), }) } + if (signal.aborted) return [] if (extractSucceeded) { this.logger.info('pandoc --extract-media complete', { filePath, @@ -302,12 +321,13 @@ export function shouldClassifyImage(meta: { width: number; height: number; sizeB export async function getImageDimensions( filePath: string, processRunner: ProcessRunner, + signal?: AbortSignal, ): Promise<{ width: number; height: number } | null> { try { const { stdout } = await processRunner.execFile( 'vipsheader', ['-f', 'width', '-f', 'height', filePath], - { timeout: 10_000 }, + { timeout: 10_000, signal }, ) const lines = stdout.trim().split('\n') if (lines.length < 2) return null @@ -315,7 +335,8 @@ export async function getImageDimensions( const height = parseInt(lines[1], 10) if (!Number.isFinite(width) || !Number.isFinite(height)) return null return { width, height } - } catch { + } catch (error) { + if (signal?.aborted) throw error return null } } @@ -329,6 +350,8 @@ export async function classifyExtractedImages( fs: FileSystem, processRunner: ProcessRunner, ): Promise> { + const signal = getPreprocessingSignal(ctx) + if (signal.aborted) return [] const listResult = await imageStore.list('', { maxDepth: 3 }) if (!listResult.ok) return [] @@ -336,6 +359,7 @@ export async function classifyExtractedImages( // Stat + density filter, then keep the top MAX_IMAGES by file size. const inspected = await mapWithConcurrency(candidates, 8, async (entry) => { + if (signal.aborted) return null const pathResult = imageStore.realPath(entry.name) if (!pathResult.ok) return null @@ -346,7 +370,7 @@ export async function classifyExtractedImages( return null } - const dims = await getImageDimensions(pathResult.value, processRunner) + const dims = await getImageDimensions(pathResult.value, processRunner, signal) if (!dims) { // Unknown dims — include but warn; better to classify than silently drop. return { name: entry.name, sizeBytes, width: 0, height: 0, kept: true } @@ -357,7 +381,7 @@ export async function classifyExtractedImages( }) const filtered = inspected - .filter((r): r is NonNullable => r !== null && r.kept) + .filter((r): r is NonNullable => r?.kept === true) .sort((a, b) => b.sizeBytes - a.sizeBytes) .slice(0, MAX_IMAGES) @@ -373,6 +397,7 @@ export async function classifyExtractedImages( } const settled = await mapWithConcurrency(filtered, IMAGE_CLASSIFY_CONCURRENCY, async (imgFile) => { + if (signal.aborted) return null const imgPathResult = imageStore.realPath(imgFile.name) if (!imgPathResult.ok) return null @@ -382,6 +407,7 @@ export async function classifyExtractedImages( const classifier = registry.getForMimeType(imgMime) if (classifier) { const classifyResult = await classifier.process(imgPathResult.value, imgMime, { + ...ctx, files: ctx.files.scoped(`${relativePrefix}/${imgFile.name}-meta`), }) if (classifyResult.ok && classifyResult.value.extractedContent) { diff --git a/packages/sdk/src/plugins/uploads/preprocessors/pdf-preprocessor.ts b/packages/sdk/src/plugins/uploads/preprocessors/pdf-preprocessor.ts index ab35c0f6..4d550e93 100644 --- a/packages/sdk/src/plugins/uploads/preprocessors/pdf-preprocessor.ts +++ b/packages/sdk/src/plugins/uploads/preprocessors/pdf-preprocessor.ts @@ -20,13 +20,22 @@ * layers before the vision call. */ +import { sleep } from '../../../lib/utils/sleep.js' import { dirname } from 'node:path' import type { Result } from '~/lib/utils/result.js' import { Err, Ok } from '~/lib/utils/result.js' import type { FileSystem } from '~/platform/fs.js' import type { ProcessRunner } from '~/platform/process.js' import type { Logger } from '../../../lib/logger/logger.js' -import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, PreprocessorResult } from '../preprocessor.js' +import { + getPreprocessingSignal, + preprocessingAbortError, + throwIfPreprocessingAborted, + type Preprocessor, + type PreprocessorContext, + type PreprocessorRegistry, + type PreprocessorResult, +} from '../preprocessor.js' import { getImageDimensions, guessImageMime, @@ -74,6 +83,8 @@ export class PdfPreprocessor implements Preprocessor { ctx: PreprocessorContext, ): Promise> { const totalStart = Date.now() + const signal = getPreprocessingSignal(ctx) + throwIfPreprocessingAborted(signal) this.logger.info('PDF processing started', { filePath }) const contentPathResult = ctx.files.realPath('content.md') @@ -88,9 +99,10 @@ export class PdfPreprocessor implements Preprocessor { // Run text extraction and image extraction (with streaming classification) // in parallel. They share no state and don't block each other. const [textResult, images] = await Promise.all([ - this.extractText(filePath, contentPathResult.value), + this.extractText(filePath, contentPathResult.value, signal), this.extractAndClassifyImages(filePath, imagesDirResult.value, ctx), ]) + if (signal.aborted) return Err(preprocessingAbortError(signal)) const markdown = textResult.ok ? textResult.value : '' @@ -127,19 +139,21 @@ export class PdfPreprocessor implements Preprocessor { * `-layout` preserves the original visual layout (columns, tables), * which is what users typically expect when looking at PDFs. */ - private async extractText(filePath: string, outputPath: string): Promise> { + private async extractText(filePath: string, outputPath: string, signal: AbortSignal): Promise> { const start = Date.now() try { await this.processRunner.execFile( 'pdftotext', ['-layout', filePath, outputPath], - { timeout: PDFTOTEXT_TIMEOUT_MS, maxBuffer: 50 * 1024 * 1024 }, + { timeout: PDFTOTEXT_TIMEOUT_MS, maxBuffer: 50 * 1024 * 1024, signal }, ) } catch (error) { + if (signal.aborted) return Err(preprocessingAbortError(signal)) const message = error instanceof Error ? error.message : String(error) this.logger.warn('pdftotext failed', { filePath, durationMs: Date.now() - start, error: message }) return Err(new Error(`pdftotext failed: ${message}`)) } + if (signal.aborted) return Err(preprocessingAbortError(signal)) let text = '' try { @@ -178,6 +192,7 @@ export class PdfPreprocessor implements Preprocessor { imagesDir: string, ctx: PreprocessorContext, ): Promise> { + const signal = getPreprocessingSignal(ctx) const extractStart = Date.now() const seen = new Set() const acceptedQueue: Array<{ name: string; sizeBytes: number; width: number; height: number }> = [] @@ -202,6 +217,7 @@ export class PdfPreprocessor implements Preprocessor { const classifyOne = async (name: string): Promise<{ relativePath: string; description: string } | null> => { await acquire() try { + if (signal.aborted) return null const mime = guessImageMime(name) const fullPath = `${imagesDir}/${name}` const imageStore = ctx.files.scoped('images') @@ -210,6 +226,7 @@ export class PdfPreprocessor implements Preprocessor { const classifier = this.registry.getForMimeType(mime) if (classifier) { const result = await classifier.process(fullPath, mime, { + ...ctx, files: ctx.files.scoped(`images/${name}-meta`), }) if (result.ok && result.value.extractedContent) { @@ -224,7 +241,7 @@ export class PdfPreprocessor implements Preprocessor { } const inspectAndMaybeClassify = async (name: string) => { - if (seen.has(name) || stopAccepting) return + if (signal.aborted || seen.has(name) || stopAccepting) return seen.add(name) if (!IMAGE_EXT_RE.test(name)) { @@ -240,7 +257,8 @@ export class PdfPreprocessor implements Preprocessor { return } - const dims = await getImageDimensions(fullPath, this.processRunner) + const dims = await getImageDimensions(fullPath, this.processRunner, signal) + if (signal.aborted) return const hasDims = dims !== null const passesFilter = hasDims ? shouldClassifyImage({ width: dims.width, height: dims.height, sizeBytes }) @@ -272,7 +290,7 @@ export class PdfPreprocessor implements Preprocessor { const pdfimagesPromise = this.processRunner.execFile( 'pdfimages', ['-all', filePath, `${imagesDir}/img`], - { timeout: PDFIMAGES_TIMEOUT_MS, maxBuffer: 1024 * 1024 }, + { timeout: PDFIMAGES_TIMEOUT_MS, maxBuffer: 1024 * 1024, signal }, ).then(() => true).catch((error) => { this.logger.warn('pdfimages failed (will classify any partial output)', { filePath, @@ -284,13 +302,15 @@ export class PdfPreprocessor implements Preprocessor { let extractionDone = false const poll = async () => { - while (!extractionDone) { + while (!extractionDone && !signal.aborted) { await this.scanAndDispatch(imagesDir, inspectAndMaybeClassify) - await sleep(STREAM_POLL_INTERVAL_MS) + await sleep(STREAM_POLL_INTERVAL_MS, signal) } // Final sweep — pick up anything that landed between the last poll // and pdfimages exiting. - await this.scanAndDispatch(imagesDir, inspectAndMaybeClassify) + if (!signal.aborted) { + await this.scanAndDispatch(imagesDir, inspectAndMaybeClassify) + } } const pollPromise = poll() @@ -336,7 +356,3 @@ export class PdfPreprocessor implements Preprocessor { await Promise.all(entries.map(handle)) } } - -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)) -} diff --git a/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.test.ts b/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.test.ts new file mode 100644 index 00000000..2520ec18 --- /dev/null +++ b/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.test.ts @@ -0,0 +1,307 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { SessionFileStore } from '~/core/file-store/file-store.js' +import { ArchiveBudget } from '~/lib/archive/index.js' +import { SYMLINK_INFO_ZIP_6_FIXTURE } from '~/lib/archive/archive-inspection.fixtures.js' +import type { ProcessRunner } from '~/platform/process.js' +import { silentLogger } from '../../../lib/logger/logger.js' +import { createNodePlatform } from '../../../testing/node-platform.js' +import { PreprocessorRegistry } from '../preprocessor.js' +import { ZipPreprocessor } from './zip-preprocessor.js' + +interface ListingEntry { + name: string + size: number + type?: 'file' | 'directory' +} + +function zipListing(entries: readonly ListingEntry[]): string { + const noun = entries.length === 1 ? 'entry' : 'entries' + const bodies = entries.map((entry, index) => { + const type = entry.type ?? 'file' + const mode = type === 'directory' ? '040775' : '100664' + return `Central directory entry #${index + 1}: +--------------------------- + + ${entry.name} + + file system or operating system of origin: Unix + uncompressed size: ${entry.size} bytes + length of filename: ${new TextEncoder().encode(entry.name).byteLength} characters + Unix file attributes (${mode} octal): attributes + MS-DOS file attributes (00 hex): none +` + }) + return `Archive: fixture.zip + central directory contains ${entries.length} ${noun}. + +${bodies.join('\n')}` +} + +describe('ZipPreprocessor archive inspection', () => { + const platform = createNodePlatform() + let workDir: string + + beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), 'roj-zip-preprocessor-')) + }) + + afterEach(async () => { + await rm(workDir, { recursive: true, force: true }) + }) + + function createContext() { + return { + files: new SessionFileStore(workDir, undefined, false, platform.fs, 'session'), + } + } + + it.each([ + ['parent traversal', zipListing([{ name: '../outside.txt', size: 1 }])], + ['backslash path', zipListing([{ name: 'dir\\outside.txt', size: 1 }])], + ['symlink', SYMLINK_INFO_ZIP_6_FIXTURE], + [ + 'entry limit', + zipListing( + Array.from({ length: 501 }, (_, index) => ({ + name: `file-${index}.txt`, + size: 0, + })), + ), + ], + ['size limit', zipListing([{ name: 'large.bin', size: 100 * 1024 * 1024 + 1 }])], + ])('rejects %s before extraction or derived writes', async (_label, listing) => { + const calls: string[][] = [] + const process: ProcessRunner = { + async execFile(command, args) { + calls.push([command, ...args]) + return { stdout: listing, stderr: '' } + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/unsafe.zip', 'application/zip', createContext()) + + expect(result.ok).toBe(false) + expect(calls).toEqual([['unzip', '-Z', '-v', '/unsafe.zip']]) + expect(await platform.fs.exists(join(workDir, 'extracted'))).toBe(false) + expect(await platform.fs.exists(join(workDir, 'content.txt'))).toBe(false) + }) + + it('fails a warning from archive inspection without extracting', async () => { + let callCount = 0 + const process: ProcessRunner = { + async execFile() { + callCount++ + throw new Error('unzip exited with code 1 after warnings') + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/warning.zip', 'application/zip', createContext()) + + expect(result.ok).toBe(false) + expect(callCount).toBe(1) + expect(await platform.fs.exists(join(workDir, 'extracted'))).toBe(false) + }) + + it('fails an extraction warning without listing or writing derived files', async () => { + const calls: string[][] = [] + const process: ProcessRunner = { + async execFile(command, args) { + calls.push([command, ...args]) + if (args[0] === '-Z') { + return { stdout: zipListing([{ name: 'safe.txt', size: 4 }]), stderr: '' } + } + throw new Error('unzip exited with code 1 after warnings') + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/warning.zip', 'application/zip', createContext()) + + expect(result.ok).toBe(false) + expect(calls).toHaveLength(2) + expect(calls[0]).toEqual(['unzip', '-Z', '-v', '/warning.zip']) + expect(calls[1]?.slice(0, 4)).toEqual(['unzip', '-o', '-q', '/warning.zip']) + expect(await platform.fs.exists(join(workDir, 'content.txt'))).toBe(false) + }) + + it('propagates an inspection abort without extracting', async () => { + const controller = new AbortController() + const reason = new Error('cancel inspection') + let callCount = 0 + const process: ProcessRunner = { + async execFile(_command, _args, options) { + callCount++ + expect(options?.signal).toBe(controller.signal) + controller.abort(reason) + throw reason + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/aborted.zip', 'application/zip', { + ...createContext(), + signal: controller.signal, + }) + + expect(result).toEqual({ ok: false, error: reason }) + expect(callCount).toBe(1) + expect(await platform.fs.exists(join(workDir, 'extracted'))).toBe(false) + }) + + it('does not extract when inspection resolves after aborting', async () => { + const controller = new AbortController() + const reason = new Error('cancel after listing') + const calls: string[][] = [] + const process: ProcessRunner = { + async execFile(command, args) { + calls.push([command, ...args]) + controller.abort(reason) + return { stdout: zipListing([{ name: 'safe.txt', size: 4 }]), stderr: '' } + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/abort-after-listing.zip', 'application/zip', { + ...createContext(), + signal: controller.signal, + }) + + expect(result).toEqual({ ok: false, error: reason }) + expect(calls).toEqual([['unzip', '-Z', '-v', '/abort-after-listing.zip']]) + expect(await platform.fs.exists(join(workDir, 'extracted'))).toBe(false) + }) + + it('inspects each nested archive before extracting that depth', async () => { + const calls: string[] = [] + const process: ProcessRunner = { + async execFile(command, args) { + if (args[0] === '-Z') { + const archivePath = args[args.length - 1] + calls.push(`inspect:${archivePath}`) + return archivePath.endsWith('nested.zip') + ? { + stdout: zipListing([{ name: 'leaf.txt', size: 4 }]), + stderr: '', + } + : { + stdout: zipListing([{ name: 'nested.zip', size: 4 }]), + stderr: '', + } + } + + const archivePath = args[2] + const destination = args[args.indexOf('-d') + 1] + calls.push(`extract:${archivePath}`) + await mkdir(destination, { recursive: true }) + if (archivePath.endsWith('nested.zip')) { + await writeFile(join(destination, 'leaf.txt'), 'leaf') + } else { + await writeFile(join(destination, 'nested.zip'), 'nested') + } + return { stdout: '', stderr: '' } + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/outer.zip', 'application/zip', createContext()) + + expect(result.ok).toBe(true) + expect(calls).toEqual([ + 'inspect:/outer.zip', + 'extract:/outer.zip', + `inspect:${join(workDir, 'extracted/nested.zip')}`, + `extract:${join(workDir, 'extracted/nested.zip')}`, + ]) + }) + + it('shares one configured budget across nested archives', async () => { + const calls: string[] = [] + const process: ProcessRunner = { + async execFile(_command, args) { + if (args[0] === '-Z') { + const archivePath = args[args.length - 1] + calls.push(`inspect:${archivePath}`) + return archivePath.endsWith('nested.zip') + ? { stdout: zipListing([{ name: 'one.txt', size: 3 }, { name: 'two.txt', size: 3 }]), stderr: '' } + : { stdout: zipListing([{ name: 'nested.zip', size: 4 }]), stderr: '' } + } + + const archivePath = args[2] + const destination = args[args.indexOf('-d') + 1] + calls.push(`extract:${archivePath}`) + await mkdir(destination, { recursive: true }) + await writeFile(join(destination, 'nested.zip'), 'nested') + return { stdout: '', stderr: '' } + }, + spawn() { + throw new Error('not used') + }, + } + const preprocessor = new ZipPreprocessor({ + registry: new PreprocessorRegistry(), + logger: silentLogger, + process, + }) + + const result = await preprocessor.process('/outer.zip', 'application/zip', { + ...createContext(), + archiveBudget: new ArchiveBudget({ maxEntries: 2, maxTotalUncompressedSize: 100 }), + }) + + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toContain('aggregate 2 entry limit') + expect(calls).toEqual([ + 'inspect:/outer.zip', + 'extract:/outer.zip', + `inspect:${join(workDir, 'extracted/nested.zip')}`, + ]) + }) +}) diff --git a/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.ts b/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.ts index 4bd3643c..aa67a2a1 100644 --- a/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.ts +++ b/packages/sdk/src/plugins/uploads/preprocessors/zip-preprocessor.ts @@ -8,18 +8,30 @@ */ import { extname } from 'node:path' +import { ArchiveBudget, inspectZipArchive } from '~/lib/archive/index.js' import { mapWithConcurrency } from '~/lib/utils/concurrency.js' import type { Result } from '~/lib/utils/result.js' import { Err, Ok } from '~/lib/utils/result.js' import type { ProcessRunner } from '~/platform/process.js' import type { Logger } from '../../../lib/logger/logger.js' -import type { Preprocessor, PreprocessorContext, PreprocessorRegistry, PreprocessorResult } from '../preprocessor.js' +import { + getPreprocessingSignal, + preprocessingAbortError, + type Preprocessor, + type PreprocessorContext, + type PreprocessorRegistry, + type PreprocessorResult, +} from '../preprocessor.js' const MAX_DEPTH = 3 -const MAX_FILES = 500 -const MAX_TOTAL_SIZE = 100 * 1024 * 1024 // 100MB const ZIP_FILE_CONCURRENCY = 10 +interface ProcessedArchiveEntry { + manifestEntry: string + derivedPaths: string[] + nestedArchiveError?: Error +} + const MIME_MAP: Record = { '.pdf': 'application/pdf', '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', @@ -49,7 +61,7 @@ function getMimeType(filename: string): string | null { } function makeExec(processRunner: ProcessRunner) { - return (cmd: string, args: string[]) => processRunner.execFile(cmd, args, { timeout: 60_000, maxBuffer: 50 * 1024 * 1024 }) + return (cmd: string, args: string[], signal?: AbortSignal) => processRunner.execFile(cmd, args, { timeout: 60_000, maxBuffer: 50 * 1024 * 1024, signal }) } function formatSize(bytes: number): string { @@ -72,7 +84,7 @@ export class ZipPreprocessor implements Preprocessor { private readonly registry: PreprocessorRegistry private readonly logger: Logger private readonly processRunner: ProcessRunner - private readonly exec: (cmd: string, args: string[]) => Promise<{ stdout: string; stderr: string }> + private readonly exec: (cmd: string, args: string[], signal?: AbortSignal) => Promise<{ stdout: string; stderr: string }> private readonly depth: number constructor(config: ZipPreprocessorConfig) { @@ -88,9 +100,27 @@ export class ZipPreprocessor implements Preprocessor { _mimeType: string, ctx: PreprocessorContext, ): Promise> { + const signal = getPreprocessingSignal(ctx) + if (signal.aborted) return Err(preprocessingAbortError(signal)) if (this.depth >= MAX_DEPTH) { return Err(new Error(`ZIP nesting depth limit reached (max ${MAX_DEPTH})`)) } + const archiveBudget = ctx.archiveBudget ?? new ArchiveBudget() + const inspection = await inspectZipArchive(this.processRunner, filePath, { + signal, + limits: archiveBudget.limits, + }) + if (!inspection.ok) { + if (signal.aborted) return Err(preprocessingAbortError(signal)) + return Err( + new Error(`ZIP inspection failed: ${inspection.error.message}`, { + cause: inspection.error, + }), + ) + } + if (signal.aborted) return Err(preprocessingAbortError(signal)) + const budgetResult = archiveBudget.consume(inspection.value) + if (!budgetResult.ok) return Err(budgetResult.error) // Extract to disk via unzip const extractStore = ctx.files.scoped('extracted') @@ -98,54 +128,39 @@ export class ZipPreprocessor implements Preprocessor { if (!extractDirResult.ok) { return Err(new Error('Failed to resolve extraction directory')) } + if (signal.aborted) return Err(preprocessingAbortError(signal)) try { - await this.exec('unzip', ['-o', '-q', filePath, '-d', extractDirResult.value]) + await this.exec('unzip', ['-o', '-q', filePath, '-d', extractDirResult.value], signal) } catch (error) { + if (signal.aborted) return Err(preprocessingAbortError(signal)) const message = error instanceof Error ? error.message : String(error) if (message.includes('ENOENT')) { return Err(new Error('unzip not found')) } - // unzip returns exit code 1 for warnings (e.g. skipped dirs) — still usable - if (!message.includes('exit code 1')) { - return Err(new Error(`unzip failed: ${message}`)) - } + return Err(new Error(`unzip failed: ${message}`)) } + if (signal.aborted) return Err(preprocessingAbortError(signal)) // List extracted files const listResult = await extractStore.list('', { maxDepth: 10 }) if (!listResult.ok) { return Err(new Error('Failed to list extracted files')) } + if (signal.aborted) return Err(preprocessingAbortError(signal)) const files = listResult.value .filter(e => e.type === 'file') .sort((a, b) => a.name.localeCompare(b.name)) - // Pick eligible files first (limits depend on cumulative iteration order, so this stays sequential) - const eligible: typeof files = [] - let totalSize = 0 - let truncationNotice: string | null = null - - for (const file of files) { - if (eligible.length >= MAX_FILES) { - truncationNotice = `... (truncated, ${files.length - eligible.length} more files)` - break - } - const fileSize = file.size ?? 0 - if (totalSize + fileSize > MAX_TOTAL_SIZE) { - truncationNotice = '... (total size limit reached)' - break - } - totalSize += fileSize - eligible.push(file) - } - - const fileCount = eligible.length + const fileCount = files.length - // Process eligible files in parallel with bounded concurrency - const processed = await mapWithConcurrency(eligible, ZIP_FILE_CONCURRENCY, async (file) => { + // Process files in parallel with bounded concurrency + const processed = await mapWithConcurrency(files, ZIP_FILE_CONCURRENCY, async (file): Promise => { const collectedPaths: string[] = [] + if (signal.aborted) { + return { manifestEntry: '', derivedPaths: collectedPaths } + } const fileRealPath = extractStore.realPath(file.name) if (!fileRealPath.ok) { @@ -175,7 +190,9 @@ export class ZipPreprocessor implements Preprocessor { if (preprocessor) { const subResult = await preprocessor.process(fileRealPath.value, mime, { + ...ctx, files: ctx.files.scoped(`extracted/${file.name}-content`), + archiveBudget, }) if (subResult.ok) { if (subResult.value.derivedPaths) { @@ -191,6 +208,13 @@ export class ZipPreprocessor implements Preprocessor { } } else { this.logger.warn('Sub-preprocessor failed', { file: file.name, error: subResult.error.message }) + if (mime === 'application/zip') { + return { + manifestEntry: `- ${file.name} (nested ZIP rejected)`, + derivedPaths: collectedPaths, + nestedArchiveError: subResult.error, + } + } } } } @@ -200,6 +224,9 @@ export class ZipPreprocessor implements Preprocessor { derivedPaths: collectedPaths, } }) + if (signal.aborted) return Err(preprocessingAbortError(signal)) + const nestedArchiveError = processed.find(item => item.nestedArchiveError)?.nestedArchiveError + if (nestedArchiveError) return Err(nestedArchiveError) const derivedPaths: string[] = [] const manifest: string[] = [] @@ -207,18 +234,17 @@ export class ZipPreprocessor implements Preprocessor { derivedPaths.push(...item.derivedPaths) manifest.push(item.manifestEntry) } - if (truncationNotice) manifest.push(truncationNotice) - const fullManifest = `## ZIP Contents (${fileCount} files)\n\n${manifest.join('\n')}` // Write full manifest to disk + if (signal.aborted) return Err(preprocessingAbortError(signal)) await ctx.files.write('content.txt', fullManifest) derivedPaths.push('content.txt') this.logger.debug('ZIP processed', { filePath, filesExtracted: fileCount, - totalSize, + totalSize: inspection.value.totalUncompressedSize, depth: this.depth, }) diff --git a/packages/sdk/src/plugins/uploads/state.ts b/packages/sdk/src/plugins/uploads/state.ts index ca18780b..8018eeaf 100644 --- a/packages/sdk/src/plugins/uploads/state.ts +++ b/packages/sdk/src/plugins/uploads/state.ts @@ -18,11 +18,15 @@ export const uploadEvents = createEventsFactory({ agentId: z4.string(), uploadIds: z4.array(uploadIdSchema), }), + attachment_deletion_completed: z4.object({ + uploadId: uploadIdSchema, + }), }, }) export type AttachmentUploadedEvent = (typeof uploadEvents)['Events']['attachment_uploaded'] export type AttachmentsConsumedEvent = (typeof uploadEvents)['Events']['attachments_consumed'] +export type AttachmentDeletionCompletedEvent = (typeof uploadEvents)['Events']['attachment_deletion_completed'] /** A pending upload tracked in session state */ export interface PendingUpload { diff --git a/packages/sdk/src/plugins/workers/context.ts b/packages/sdk/src/plugins/workers/context.ts index 159beb0f..6df7f438 100644 --- a/packages/sdk/src/plugins/workers/context.ts +++ b/packages/sdk/src/plugins/workers/context.ts @@ -17,8 +17,8 @@ import { generateMessageId } from '~/plugins/mailbox/schema.js' import { mailboxEvents } from '~/plugins/mailbox/state.js' import type { Logger } from '../../lib/logger/logger.js' import type { WorkerSubEvent } from './definition.js' -import type { EmitEvent } from './plugin.js' -import { workerEvents } from './plugin.js' +import type { EmitEvent } from './state.js' +import { workerEvents } from './state.js' import { WorkerId } from './worker.js' // ============================================================================ diff --git a/packages/sdk/src/plugins/workers/index.ts b/packages/sdk/src/plugins/workers/index.ts index 4431f8ae..f3dae964 100644 --- a/packages/sdk/src/plugins/workers/index.ts +++ b/packages/sdk/src/plugins/workers/index.ts @@ -3,8 +3,8 @@ export { workerPlugin } from './plugin.js' export type { WorkerAgentConfig, WorkerPresetConfig } from './plugin.js' // Events (now in plugin.ts) -export { workerEvents } from './plugin.js' -export type { WorkerCompletedEvent, WorkerFailedEvent, WorkerStartedEvent, WorkerStatusChangedEvent, WorkerSubEventEmittedEvent } from './plugin.js' +export { workerEvents } from './state.js' +export type { WorkerCompletedEvent, WorkerFailedEvent, WorkerStartedEvent, WorkerStatusChangedEvent, WorkerSubEventEmittedEvent } from './state.js' // Definition export type { WorkerDefinition, WorkerSubEvent } from './definition.js' @@ -18,4 +18,4 @@ export { WorkerContextImpl } from './context.js' export type { WorkerError, WorkerResult } from './worker.js' // Types -export type { EmitEvent } from './plugin.js' +export type { EmitEvent } from './state.js' diff --git a/packages/sdk/src/plugins/workers/plugin.ts b/packages/sdk/src/plugins/workers/plugin.ts index 38838cea..81a74cda 100644 --- a/packages/sdk/src/plugins/workers/plugin.ts +++ b/packages/sdk/src/plugins/workers/plugin.ts @@ -1,59 +1,17 @@ import z from 'zod/v4' import { agentIdSchema } from '~/core/agents/schema.js' import { ValidationErrors } from '~/core/errors.js' -import { createEventsFactory } from '~/core/events/types.js' import type { BaseEvent } from '~/core/events/types.js' import { definePlugin } from '~/core/plugins/plugin-builder.js' import { createTool, type ToolDefinition } from '~/core/tools/definition.js' import { Err, Ok } from '~/lib/utils/result.js' import type { Logger } from '../../lib/logger/logger.js' import { WorkerContextImpl } from './context.js' +import { workerEvents } from './state.js' +import type { EmitEvent } from './state.js' import type { WorkerCommandDefinition, WorkerDefinition, WorkerSubEvent } from './definition.js' import { generateWorkerId, type WorkerEntry, WorkerId, type WorkerId as WorkerIdType, workerIdSchema } from './worker.js' -export const workerEvents = createEventsFactory({ - events: { - worker_started: z.object({ - workerId: workerIdSchema, - agentId: agentIdSchema, - workerType: z.string(), - config: z.unknown(), - }), - worker_sub_event: z.object({ - workerId: workerIdSchema, - workerType: z.string(), - subEvent: z.record(z.string(), z.unknown()).and(z.object({ - type: z.string(), - })), - }), - worker_status_changed: z.object({ - workerId: workerIdSchema, - fromStatus: z.enum(['running', 'paused', 'completed', 'failed', 'cancelled']), - toStatus: z.enum(['running', 'paused', 'completed', 'failed', 'cancelled']), - reason: z.string().optional(), - }), - worker_completed: z.object({ - workerId: workerIdSchema, - result: z.object({ - status: z.string(), - resultsPath: z.string().optional(), - summary: z.string(), - data: z.unknown().optional(), - }), - }), - worker_failed: z.object({ - workerId: workerIdSchema, - error: z.string(), - resumable: z.boolean(), - }), - }, -}) - -export type WorkerStartedEvent = (typeof workerEvents)['Events']['worker_started'] -export type WorkerSubEventEmittedEvent = (typeof workerEvents)['Events']['worker_sub_event'] -export type WorkerStatusChangedEvent = (typeof workerEvents)['Events']['worker_status_changed'] -export type WorkerCompletedEvent = (typeof workerEvents)['Events']['worker_completed'] -export type WorkerFailedEvent = (typeof workerEvents)['Events']['worker_failed'] /** * Session-wide worker configuration. @@ -75,10 +33,6 @@ export interface WorkerAgentConfig { // Types moved from executor.ts // ============================================================================ -/** - * Event emitter callback - emits events without sessionId (added automatically). - */ -export type EmitEvent = (event: Omit, 'sessionId'>) => Promise /** * Represents a running worker instance. diff --git a/packages/sdk/src/plugins/workers/state.ts b/packages/sdk/src/plugins/workers/state.ts new file mode 100644 index 00000000..53f963db --- /dev/null +++ b/packages/sdk/src/plugins/workers/state.ts @@ -0,0 +1,64 @@ +/** + * Worker domain events and the emitter shape. + * + * Lives here rather than in plugin.ts so that context.ts can import + * `workerEvents` (a runtime value, produced by a top-level + * createEventsFactory call) without depending on the plugin that also imports + * WorkerContextImpl back from it. That was the repo's only two-way value cycle, + * and a module-initialisation-order one at that. Matches mailbox, resources and + * uploads, which all keep their events in state.ts. + */ +import z from 'zod/v4' +import { agentIdSchema } from '~/core/agents/schema.js' +import { createEventsFactory } from '~/core/events/types.js' +import type { BaseEvent } from '~/core/events/types.js' +import { workerIdSchema } from './worker.js' + +export const workerEvents = createEventsFactory({ + events: { + worker_started: z.object({ + workerId: workerIdSchema, + agentId: agentIdSchema, + workerType: z.string(), + config: z.unknown(), + }), + worker_sub_event: z.object({ + workerId: workerIdSchema, + workerType: z.string(), + subEvent: z.record(z.string(), z.unknown()).and(z.object({ + type: z.string(), + })), + }), + worker_status_changed: z.object({ + workerId: workerIdSchema, + fromStatus: z.enum(['running', 'paused', 'completed', 'failed', 'cancelled']), + toStatus: z.enum(['running', 'paused', 'completed', 'failed', 'cancelled']), + reason: z.string().optional(), + }), + worker_completed: z.object({ + workerId: workerIdSchema, + result: z.object({ + status: z.string(), + resultsPath: z.string().optional(), + summary: z.string(), + data: z.unknown().optional(), + }), + }), + worker_failed: z.object({ + workerId: workerIdSchema, + error: z.string(), + resumable: z.boolean(), + }), + }, +}) + +export type WorkerStartedEvent = (typeof workerEvents)['Events']['worker_started'] +export type WorkerSubEventEmittedEvent = (typeof workerEvents)['Events']['worker_sub_event'] +export type WorkerStatusChangedEvent = (typeof workerEvents)['Events']['worker_status_changed'] +export type WorkerCompletedEvent = (typeof workerEvents)['Events']['worker_completed'] +export type WorkerFailedEvent = (typeof workerEvents)['Events']['worker_failed'] + +/** + * Event emitter callback - emits events without sessionId (added automatically). + */ +export type EmitEvent = (event: Omit, 'sessionId'>) => Promise diff --git a/packages/sdk/src/testing/test-harness.ts b/packages/sdk/src/testing/test-harness.ts index 7429a5e1..41a08d61 100644 --- a/packages/sdk/src/testing/test-harness.ts +++ b/packages/sdk/src/testing/test-harness.ts @@ -1,3 +1,4 @@ +import { rm } from 'node:fs/promises' import type { AgentId } from '~/core/agents/schema.js' import type { DomainError } from '~/core/errors.js' import { MemoryEventStore } from '~/core/events/memory.js' @@ -76,6 +77,8 @@ export class TestHarness { readonly llmProvider: MockLLMProvider readonly notifications: NotificationCollector readonly sessionManager: SessionManager + /** Per-instance scratch dir under /tmp, removed by shutdown(). */ + private readonly basePath: string constructor(options: { presets: Preset[] @@ -118,6 +121,7 @@ export class TestHarness { } const basePath = `/tmp/roj-test-${Math.random().toString(36).slice(2)}` + this.basePath = basePath const toolExecutor = new ToolExecutor(silentLogger) const platform = createNodePlatform() const dataFileStore = new SessionFileStore(basePath, undefined, false, platform.fs, 'session') @@ -177,10 +181,15 @@ export class TestHarness { } /** - * Shutdown all sessions. + * Shutdown all sessions and remove the harness's own scratch directory. + * + * The directory is exclusively ours (a per-instance random path under /tmp), + * so nothing else can be relying on it. Idempotent — safe from an afterEach + * that also runs after a failed test. */ async shutdown(): Promise { await this.sessionManager.shutdown() + await rm(this.basePath, { recursive: true, force: true }) } } diff --git a/packages/sdk/src/testing/wait-helpers.ts b/packages/sdk/src/testing/wait-helpers.ts index c44159a2..3eb32277 100644 --- a/packages/sdk/src/testing/wait-helpers.ts +++ b/packages/sdk/src/testing/wait-helpers.ts @@ -1,3 +1,4 @@ +import { sleep } from '~/lib/utils/sleep.js' import type { AgentId } from '~/core/agents/schema.js' import type { Session } from '~/core/sessions/session.js' @@ -78,7 +79,3 @@ function areAllAgentsIdle(session: Session): boolean { } return true } - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} diff --git a/packages/sdk/src/transport/http/app.ts b/packages/sdk/src/transport/http/app.ts index fdd69dfe..ce68f4ce 100644 --- a/packages/sdk/src/transport/http/app.ts +++ b/packages/sdk/src/transport/http/app.ts @@ -7,9 +7,8 @@ import { Hono } from 'hono' import { cors } from 'hono/cors' import { SDK_VERSION } from '~/info.js' -import type { PreprocessorRegistry } from '~/plugins/uploads/preprocessor.js' -import type { Services } from '../../bootstrap.js' -import type { SessionManager } from '../../core/sessions/session-manager.js' +import type { AppEnv, AppServices } from './context.js' +import { getServices } from './context.js' import { createBearerAuth } from './middleware/bearer-auth.js' import { errorHandler } from './middleware/error-handler.js' import { createFileRoutes } from './routes/files.js' @@ -17,38 +16,10 @@ import { createResourceRoutes } from './routes/resources.js' import { createRpcRoutes } from './routes/rpc.js' import { createUploadRoutes } from './routes/upload.js' -/** - * Extended services with SessionManager for HTTP routes. - */ -export type AppServices = Services & { - sessionRuntime: SessionManager - /** Bearer token for authenticating HTTP requests. Optional - only used in worker mode. */ - agentToken?: string - /** File preprocessor registry for upload routes. Optional - only available when uploads plugin is configured. */ - preprocessorRegistry?: PreprocessorRegistry -} - -/** - * Environment type for Hono app with injected services. - */ -export type AppEnv = { - Variables: { - services: AppServices - } -} - -/** - * Hono context type for routes. - */ -export type AppContext = import('hono').Context - -/** - * Type-safe accessor for services from Hono context. - * Guarantees services are present (set by middleware). - */ -export function getServices(c: AppContext): AppServices { - return c.get('services') -} +// Re-exported so `from './app.js'` keeps working for consumers; the +// declarations live in context.js, which routes import directly. +export type { AppContext, AppEnv, AppServices } from './context.js' +export { getServices } from './context.js' /** * Creates the Hono application with all middleware and routes. diff --git a/packages/sdk/src/transport/http/context.ts b/packages/sdk/src/transport/http/context.ts new file mode 100644 index 00000000..0ea3f2da --- /dev/null +++ b/packages/sdk/src/transport/http/context.ts @@ -0,0 +1,46 @@ +/** + * Hono context types and the services accessor. + * + * Kept out of app.ts on purpose. The route factories need `getServices` — a + * runtime value, not just a type — while app.ts imports every route factory to + * mount them. With both in app.ts the ESM module graph really does contain a + * cycle (app -> routes/* -> app), not merely a type-shape artifact. This module + * imports nothing from app.ts, so the graph stays acyclic. + */ + +import type { PreprocessorRegistry } from '~/plugins/uploads/preprocessor.js' +import type { Services } from '../../bootstrap.js' +import type { SessionManager } from '../../core/sessions/session-manager.js' + +/** + * Extended services with SessionManager for HTTP routes. + */ +export type AppServices = Services & { + sessionRuntime: SessionManager + /** Bearer token for authenticating HTTP requests. Optional - only used in worker mode. */ + agentToken?: string + /** File preprocessor registry for upload routes. Optional - only available when uploads plugin is configured. */ + preprocessorRegistry?: PreprocessorRegistry +} + +/** + * Environment type for Hono app with injected services. + */ +export type AppEnv = { + Variables: { + services: AppServices + } +} + +/** + * Hono context type for routes. + */ +export type AppContext = import('hono').Context + +/** + * Type-safe accessor for services from Hono context. + * Guarantees services are present (set by middleware). + */ +export function getServices(c: AppContext): AppServices { + return c.get('services') +} diff --git a/packages/sdk/src/transport/http/index.ts b/packages/sdk/src/transport/http/index.ts index e6b498e8..4e87810f 100644 --- a/packages/sdk/src/transport/http/index.ts +++ b/packages/sdk/src/transport/http/index.ts @@ -3,5 +3,5 @@ */ export { createApp } from './app.js' -export type { AppEnv } from './app.js' +export type { AppEnv } from './context.js' export { errorHandler } from './middleware/error-handler.js' diff --git a/packages/sdk/src/transport/http/middleware/bearer-auth.ts b/packages/sdk/src/transport/http/middleware/bearer-auth.ts index 3ea748a8..72420e13 100644 --- a/packages/sdk/src/transport/http/middleware/bearer-auth.ts +++ b/packages/sdk/src/transport/http/middleware/bearer-auth.ts @@ -6,7 +6,7 @@ */ import type { MiddlewareHandler } from 'hono' -import type { AppEnv } from '../app.js' +import type { AppEnv } from '../context.js' /** * Creates bearer auth middleware. diff --git a/packages/sdk/src/transport/http/middleware/error-handler.ts b/packages/sdk/src/transport/http/middleware/error-handler.ts index 2ae93d1c..cb798dc9 100644 --- a/packages/sdk/src/transport/http/middleware/error-handler.ts +++ b/packages/sdk/src/transport/http/middleware/error-handler.ts @@ -12,7 +12,7 @@ import type { Context } from 'hono' import { HTTPException } from 'hono/http-exception' import type { ContentfulStatusCode } from 'hono/utils/http-status' import { isDomainError } from '~/core/errors.js' -import type { AppEnv } from '../app.js' +import type { AppEnv } from '../context.js' /** * Global error handler for Hono app. diff --git a/packages/sdk/src/transport/http/responses.ts b/packages/sdk/src/transport/http/responses.ts new file mode 100644 index 00000000..110d6141 --- /dev/null +++ b/packages/sdk/src/transport/http/responses.ts @@ -0,0 +1,15 @@ +/** + * Error responses shared by the HTTP routes. + * + * Every route used to hand-write `{ error: { type, message } }` inline — four + * copies of the session 404 and four of the parse 400 — so nothing enforced the + * envelope and a client parsing errors had no single contract to code against. + * The status codes and the `type` strings are the wire contract; keep them here. + */ +import type { AppContext } from './context.js' + +export const sessionNotFound = (c: AppContext, sessionId: string) => + c.json({ error: { type: 'session_not_found', message: `Session not found: ${sessionId}` } }, 404) + +export const parseError = (c: AppContext, message: string) => + c.json({ error: { type: 'parse_error', message } }, 400) diff --git a/packages/sdk/src/transport/http/routes/files.test.ts b/packages/sdk/src/transport/http/routes/files.test.ts new file mode 100644 index 00000000..f0905816 --- /dev/null +++ b/packages/sdk/src/transport/http/routes/files.test.ts @@ -0,0 +1,141 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' +import { mkdir, mkdtemp, rm, symlink, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { Hono } from 'hono' +import { bootstrap, createSessionManager } from '../../../bootstrap.js' +import { createTestPreset } from '../../../testing/preset-helpers.js' +import { createNodePlatform } from '../../../testing/node-platform.js' +import type { AppEnv, AppServices } from '../context.js' +import { createFileRoutes } from './files.js' + +interface FileRouteFixture { + app: Hono + baseDir: string + sessionDir: string + sessionId: string + workspaceDir: string + services: AppServices +} + +async function createFixture(): Promise { + const baseDir = await mkdtemp(join(tmpdir(), 'roj-file-routes-')) + const workspaceDir = join(baseDir, 'workspace') + await mkdir(workspaceDir) + + const services = bootstrap({ + port: 0, + host: 'localhost', + dataPath: baseDir, + persistence: 'memory', + logLevel: 'error', + logFormat: 'console', + llmMock: () => ({ + content: 'Mock response', + toolCalls: [], + finishReason: 'stop', + metrics: { promptTokens: 0, completionTokens: 0, totalTokens: 0, latencyMs: 0, model: 'mock' }, + }), + }, { presets: [createTestPreset()] }, createNodePlatform()) + const sessionRuntime = createSessionManager(services) + const sessionResult = await sessionRuntime.createSession('test', { workspaceDir }) + if (!sessionResult.ok) { + await rm(baseDir, { recursive: true, force: true }) + throw new Error(`Failed to create test session: ${sessionResult.error.message}`) + } + + const appServices: AppServices = { ...services, sessionRuntime } + const app = new Hono() + app.use('*', async (c, next) => { + c.set('services', appServices) + await next() + }) + app.route('/sessions', createFileRoutes()) + + const sessionId = String(sessionResult.value.id) + const sessionDir = join(baseDir, 'sessions', sessionId) + await mkdir(sessionDir, { recursive: true }) + + return { app, baseDir, sessionDir, sessionId, workspaceDir, services: appServices } +} + +describe('file routes', () => { + let fixture: FileRouteFixture | undefined + + beforeEach(async () => { + fixture = await createFixture() + }) + + afterEach(async () => { + if (!fixture) return + await fixture.services.sessionRuntime.shutdown() + await rm(fixture.baseDir, { recursive: true, force: true }) + fixture = undefined + }) + + function currentFixture(): FileRouteFixture { + if (!fixture) throw new Error('Test fixture is not initialized') + return fixture + } + + it('serves a normal session file', async () => { + const { app, sessionDir, sessionId } = currentFixture() + await writeFile(join(sessionDir, 'hello.txt'), 'hello') + + const response = await app.request(`/sessions/${sessionId}/files/hello.txt`) + + expect(response.status).toBe(200) + expect(response.headers.get('Content-Type')).toBe('text/plain') + expect(await response.text()).toBe('hello') + }) + + it('serves an internal session symlink', async () => { + const { app, sessionDir, sessionId } = currentFixture() + await writeFile(join(sessionDir, 'target.txt'), 'internal') + await symlink(join(sessionDir, 'target.txt'), join(sessionDir, 'link.txt')) + + const response = await app.request(`/sessions/${sessionId}/files/link.txt`) + + expect(response.status).toBe(200) + expect(await response.text()).toBe('internal') + }) + + it('rejects a session symlink that escapes the session root', async () => { + const { app, baseDir, sessionDir, sessionId } = currentFixture() + const outsidePath = join(baseDir, 'outside-session.txt') + await writeFile(outsidePath, 'secret') + await symlink(outsidePath, join(sessionDir, 'escape.txt')) + + const response = await app.request(`/sessions/${sessionId}/files/escape.txt`) + + expect(response.status).toBe(403) + }) + + it('rejects a workspace symlink that escapes the workspace root', async () => { + const { app, baseDir, sessionId, workspaceDir } = currentFixture() + const outsidePath = join(baseDir, 'outside-workspace.txt') + await writeFile(outsidePath, 'secret') + await symlink(outsidePath, join(workspaceDir, 'escape.txt')) + + const response = await app.request(`/sessions/${sessionId}/workspace/escape.txt`) + + expect(response.status).toBe(403) + }) + + it('rejects lexical parent traversal', async () => { + const { app, sessionId } = currentFixture() + const traversal = encodeURIComponent('../../outside.txt') + + const response = await app.request(`/sessions/${sessionId}/files/${traversal}`) + + expect(response.status).toBe(403) + }) + + it('keeps missing files as not found', async () => { + const { app, sessionId } = currentFixture() + + const response = await app.request(`/sessions/${sessionId}/files/missing.txt`) + + expect(response.status).toBe(404) + }) +}) diff --git a/packages/sdk/src/transport/http/routes/files.ts b/packages/sdk/src/transport/http/routes/files.ts index a1de17d4..084d5800 100644 --- a/packages/sdk/src/transport/http/routes/files.ts +++ b/packages/sdk/src/transport/http/routes/files.ts @@ -6,150 +6,15 @@ */ import { Hono } from 'hono' -import { extname, resolve } from 'node:path' +import { isAbsolute, relative, resolve, sep } from 'node:path' +import { getMimeType, preventTraversal } from '~/plugins/filesystem/listing.js' import { SessionId } from '~/core/sessions/schema.js' -import { type AppContext, type AppEnv, getServices } from '../app.js' - -// ============================================================================ -// Constants -// ============================================================================ - -/** Known MIME types for specific extensions. */ -const MIME_TYPES: Record = { - // Images - '.jpg': 'image/jpeg', - '.jpeg': 'image/jpeg', - '.png': 'image/png', - '.gif': 'image/gif', - '.webp': 'image/webp', - '.svg': 'image/svg+xml', - '.ico': 'image/x-icon', - '.bmp': 'image/bmp', - '.avif': 'image/avif', - // Video - '.mp4': 'video/mp4', - '.webm': 'video/webm', - '.mov': 'video/quicktime', - // Audio - '.mp3': 'audio/mpeg', - '.wav': 'audio/wav', - '.ogg': 'audio/ogg', - // Documents - '.pdf': 'application/pdf', - // Data/markup - '.json': 'application/json', - '.html': 'text/html', - '.css': 'text/css', - '.js': 'application/javascript', - '.mjs': 'application/javascript', - '.xml': 'text/xml', - '.yaml': 'text/yaml', - '.yml': 'text/yaml', - '.md': 'text/markdown', - // Archives - '.zip': 'application/zip', - '.tar': 'application/x-tar', - '.gz': 'application/gzip', - // Fonts - '.woff': 'font/woff', - '.woff2': 'font/woff2', - '.ttf': 'font/ttf', - '.otf': 'font/otf', - // Binary - '.wasm': 'application/wasm', -} - -/** Extensions known to be binary — files that cannot be displayed as text. */ -const BINARY_EXTENSIONS = new Set([ - // Images - '.jpg', - '.jpeg', - '.png', - '.gif', - '.webp', - '.bmp', - '.ico', - '.tiff', - '.tif', - '.avif', - // Video - '.mp4', - '.webm', - '.avi', - '.mov', - '.mkv', - '.flv', - '.wmv', - // Audio - '.mp3', - '.wav', - '.ogg', - '.flac', - '.aac', - '.wma', - '.m4a', - // Archives - '.zip', - '.tar', - '.gz', - '.bz2', - '.xz', - '.7z', - '.rar', - '.zst', - // Documents (binary) - '.pdf', - '.doc', - '.docx', - '.xls', - '.xlsx', - '.ppt', - '.pptx', - '.odt', - // Fonts - '.woff', - '.woff2', - '.ttf', - '.otf', - '.eot', - // Compiled/binary - '.wasm', - '.exe', - '.dll', - '.so', - '.dylib', - '.o', - '.a', - '.class', - '.pyc', - '.pyo', - // Database - '.sqlite', - '.db', - '.sqlite3', - // Other - '.bin', - '.dat', -]) +import { type AppContext, type AppEnv, getServices } from '../context.js' // ============================================================================ // Helpers // ============================================================================ -/** - * Determine MIME type for a file. - * Known extensions get their specific MIME type, known binary extensions - * get `application/octet-stream`, everything else defaults to `text/plain` - * so that code/config files (.astro, .vue, .svelte, .go, .rs, etc.) - * are previewable as text. - */ -function getMimeType(filePath: string): string { - const ext = extname(filePath).toLowerCase() - if (MIME_TYPES[ext]) return MIME_TYPES[ext] - if (BINARY_EXTENSIONS.has(ext)) return 'application/octet-stream' - return 'text/plain' -} - /** * Extract the wildcard path suffix from a request. * @@ -159,18 +24,42 @@ function getMimeType(filePath: string): string { function extractWildcardPath(c: AppContext, marker: string): string { const idx = c.req.path.indexOf(`/${marker}/`) if (idx === -1) return '' - return c.req.path.slice(idx + marker.length + 2) + try { + return decodeURIComponent(c.req.path.slice(idx + marker.length + 2)) + } catch { + return '' + } } -function preventTraversal(baseDir: string, requestedPath: string): string | null { - const resolved = resolve(baseDir, requestedPath) - if (!resolved.startsWith(baseDir + '/') && resolved !== baseDir) { - return null +type CanonicalPathResult = + | { status: 'ok'; path: string } + | { status: 'not_found' } + | { status: 'forbidden' } + +async function resolveCanonicalPath(c: AppContext, rootPath: string, targetPath: string): Promise { + const { platform } = getServices(c) + let canonicalRoot: string + let canonicalTarget: string + try { + canonicalRoot = await platform.fs.realpath(rootPath) + canonicalTarget = await platform.fs.realpath(targetPath) + } catch { + return { status: 'not_found' } } - return resolved + + const relativeTarget = relative(canonicalRoot, canonicalTarget) + const isContained = relativeTarget === '' || ( + !isAbsolute(relativeTarget) + && relativeTarget !== '..' + && !relativeTarget.startsWith(`..${sep}`) + ) + + return isContained + ? { status: 'ok', path: canonicalTarget } + : { status: 'forbidden' } } -async function serveFile(c: AppContext, filePath: string): Promise { +async function serveFile(c: AppContext, filePath: string, mimePath: string): Promise { const { platform } = getServices(c) let data: Buffer try { @@ -182,7 +71,7 @@ async function serveFile(c: AppContext, filePath: string): Promise { ) } - const contentType = getMimeType(filePath) + const contentType = getMimeType(mimePath) return new Response(data, { headers: { @@ -236,7 +125,21 @@ export function createFileRoutes(): Hono { ) } - return serveFile(c, resolvedPath) + const canonicalPath = await resolveCanonicalPath(c, sessionDir, resolvedPath) + if (canonicalPath.status === 'forbidden') { + return c.json( + { error: { type: 'forbidden', message: 'Symlink traversal not allowed' } }, + 403, + ) + } + if (canonicalPath.status === 'not_found') { + return c.json( + { error: { type: 'not_found', message: 'File not found' } }, + 404, + ) + } + + return serveFile(c, canonicalPath.path, resolvedPath) }) // --- Serve workspace file --- @@ -267,7 +170,21 @@ export function createFileRoutes(): Hono { ) } - return serveFile(c, resolvedPath) + const canonicalPath = await resolveCanonicalPath(c, workspaceDir, resolvedPath) + if (canonicalPath.status === 'forbidden') { + return c.json( + { error: { type: 'forbidden', message: 'Symlink traversal not allowed' } }, + 403, + ) + } + if (canonicalPath.status === 'not_found') { + return c.json( + { error: { type: 'not_found', message: 'File not found' } }, + 404, + ) + } + + return serveFile(c, canonicalPath.path, resolvedPath) }) return app diff --git a/packages/sdk/src/transport/http/routes/resources.ts b/packages/sdk/src/transport/http/routes/resources.ts index c074ca36..dc495049 100644 --- a/packages/sdk/src/transport/http/routes/resources.ts +++ b/packages/sdk/src/transport/http/routes/resources.ts @@ -8,9 +8,35 @@ */ import { Hono } from 'hono' -import type { AppContext, AppEnv } from '../app.js' -import { getServices } from '../app.js' +import z from 'zod/v4' import { SessionId } from '~/core/sessions/schema.js' +import { ResourceBasenameSchema } from '~/plugins/resources/filename.js' +import type { AppContext, AppEnv } from '../context.js' +import { getServices } from '../context.js' +import { parseError, sessionNotFound } from '../responses.js' + +const ResourceRequestSchema = z.object({ + url: z.string().min(1), + filename: z.string().min(1), + mimeType: z.string().min(1), + metadata: z.object({ + slug: z.string().optional(), + name: z.string().optional(), + }).optional(), +}).superRefine((body, refinement) => { + if (body.mimeType !== 'application/zip' && !ResourceBasenameSchema.safeParse(body.filename).success) { + refinement.addIssue({ + code: 'custom', + path: ['filename'], + message: 'Resource filename must be a basename', + }) + } +}) + +const ResourceInjectResultSchema = z.object({ + resourceId: z.string(), + paths: z.array(z.string()), +}) export function createResourceRoutes(): Hono { const app = new Hono() @@ -22,29 +48,25 @@ export function createResourceRoutes(): Hono { // 1. Verify session exists const sessionResult = await sessionRuntime.getSession(sessionId) if (!sessionResult.ok) { - return c.json( - { error: { type: 'session_not_found', message: `Session not found: ${sessionId}` } }, - 404, - ) + return sessionNotFound(c, sessionId) } // 2. Parse JSON body - let body: { url: string; filename: string; mimeType: string; metadata?: { slug?: string; name?: string } } + let rawBody: unknown try { - body = await c.req.json() + rawBody = await c.req.json() } catch { - return c.json( - { error: { type: 'parse_error', message: 'Failed to parse JSON body' } }, - 400, - ) + return parseError(c, 'Failed to parse JSON body') } - if (!body.url || !body.filename || !body.mimeType) { + const parsedBody = ResourceRequestSchema.safeParse(rawBody) + if (!parsedBody.success) { return c.json( - { error: { type: 'validation_error', message: 'Missing required fields: url, filename, mimeType' } }, + { error: { type: 'validation_error', message: parsedBody.error.message } }, 400, ) } + const body = parsedBody.data // 3. Fetch URL const maxSize = 50 * 1024 * 1024 // 50MB @@ -102,12 +124,19 @@ export function createResourceRoutes(): Hono { ) } - const injectResult = result.value as { resourceId: string; paths: string[] } + const injectResult = ResourceInjectResultSchema.safeParse(result.value) + if (!injectResult.success) { + logger.error('Resource injection returned an invalid result', undefined, { sessionId: String(sessionId) }) + return c.json( + { error: { type: 'internal_error', message: 'Resource injection returned an invalid result' } }, + 500, + ) + } return c.json({ ok: true, - resourceId: injectResult.resourceId, - paths: injectResult.paths, + resourceId: injectResult.data.resourceId, + paths: injectResult.data.paths, }, 201) }) diff --git a/packages/sdk/src/transport/http/routes/rpc.integration.test.ts b/packages/sdk/src/transport/http/routes/rpc.integration.test.ts index 57d20872..79aa483f 100644 --- a/packages/sdk/src/transport/http/routes/rpc.integration.test.ts +++ b/packages/sdk/src/transport/http/routes/rpc.integration.test.ts @@ -4,12 +4,12 @@ * Tests for the RPC dispatch system using TestHarness + Hono. * Covers batch calls, method dispatch, error handling. */ -import { beforeEach, describe, expect, it } from 'bun:test' +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' import { Hono } from 'hono' import { MockLLMProvider } from '~/core/llm/mock.js' import { createTestPreset, TestHarness } from '~/testing/index.js' import { bootstrapForTesting } from '../../../testing/bootstrap-for-testing.js' -import type { AppEnv } from '../app.js' +import type { AppEnv } from '../context.js' import { createRpcRoutes } from './rpc.js' interface RpcResponse { @@ -42,6 +42,10 @@ describe('RPC integration', () => { let app: Hono let harness: TestHarness + afterEach(async () => { + await harness.shutdown() + }) + beforeEach(() => { harness = new TestHarness({ presets: [createTestPreset()], diff --git a/packages/sdk/src/transport/http/routes/rpc.test.ts b/packages/sdk/src/transport/http/routes/rpc.test.ts index 30afd124..0259fb02 100644 --- a/packages/sdk/src/transport/http/routes/rpc.test.ts +++ b/packages/sdk/src/transport/http/routes/rpc.test.ts @@ -7,7 +7,7 @@ import { ModelId } from '~/core/llm/schema.js' import type { Preset } from '~/core/preset/index.js' import { createSessionManager } from '../../../bootstrap.js' import { bootstrapForTesting } from '../../../testing/bootstrap-for-testing.js' -import type { AppEnv, AppServices } from '../app.js' +import type { AppEnv, AppServices } from '../context.js' import { createRpcRoutes } from './rpc.js' /** Minimal preset for testing */ diff --git a/packages/sdk/src/transport/http/routes/rpc.ts b/packages/sdk/src/transport/http/routes/rpc.ts index 607031dc..fab227c8 100644 --- a/packages/sdk/src/transport/http/routes/rpc.ts +++ b/packages/sdk/src/transport/http/routes/rpc.ts @@ -21,8 +21,8 @@ import type { DomainError } from '~/core/errors.js' import { type CallerContext, DEFAULT_CALLER } from '~/core/plugins/plugin-builder.js' import { SessionId } from '~/core/sessions/schema.js' import type { SessionManager } from '~/core/sessions/session-manager.js' -import { getServices } from '../app.js' -import type { AppEnv } from '../app.js' +import { getServices } from '../context.js' +import type { AppEnv } from '../context.js' type MethodResult = | { ok: true; value: unknown } diff --git a/packages/sdk/src/transport/http/routes/upload.ts b/packages/sdk/src/transport/http/routes/upload.ts index 786df353..7c52debe 100644 --- a/packages/sdk/src/transport/http/routes/upload.ts +++ b/packages/sdk/src/transport/http/routes/upload.ts @@ -6,9 +6,10 @@ * Business logic delegated to uploads plugin. */ +import { parseError, sessionNotFound } from '../responses.js' import { Hono } from 'hono' import { SessionId } from '~/core/sessions/schema.js' -import { type AppContext, type AppEnv, getServices } from '../app.js' +import { type AppContext, type AppEnv, getServices } from '../context.js' // ============================================================================ // Routes @@ -42,10 +43,7 @@ export function createUploadRoutes(): Hono { // 1. Verify session exists const sessionResult = await sessionRuntime.getSession(sessionId) if (!sessionResult.ok) { - return c.json( - { error: { type: 'session_not_found', message: `Session not found: ${sessionId}` } }, - 404, - ) + return sessionNotFound(c, sessionId) } // 2. Parse multipart form data (transport concern — stays in HTTP layer) @@ -53,10 +51,7 @@ export function createUploadRoutes(): Hono { try { body = await c.req.parseBody() } catch { - return c.json( - { error: { type: 'parse_error', message: 'Failed to parse multipart form data' } }, - 400, - ) + return parseError(c, 'Failed to parse multipart form data') } const file = body.file @@ -134,20 +129,14 @@ export function createUploadRoutes(): Hono { const sessionResult = await sessionRuntime.getSession(sessionId) if (!sessionResult.ok) { - return c.json( - { error: { type: 'session_not_found', message: `Session not found: ${sessionId}` } }, - 404, - ) + return sessionNotFound(c, sessionId) } let body: Record try { body = await c.req.parseBody() } catch { - return c.json( - { error: { type: 'parse_error', message: 'Failed to parse multipart form data' } }, - 400, - ) + return parseError(c, 'Failed to parse multipart form data') } const file = body.file @@ -218,10 +207,7 @@ export function createUploadRoutes(): Hono { // 1. Verify session exists const sessionResult = await sessionRuntime.getSession(sessionId) if (!sessionResult.ok) { - return c.json( - { error: { type: 'session_not_found', message: `Session not found: ${sessionId}` } }, - 404, - ) + return sessionNotFound(c, sessionId) } // 2. Parse JSON body @@ -229,10 +215,7 @@ export function createUploadRoutes(): Hono { try { body = await c.req.json() } catch { - return c.json( - { error: { type: 'parse_error', message: 'Failed to parse JSON body' } }, - 400, - ) + return parseError(c, 'Failed to parse JSON body') } if (!body.url || !body.filename || !body.mimeType) { diff --git a/packages/sdk/tsconfig.json b/packages/sdk/tsconfig.json index 8393f983..41068f7d 100644 --- a/packages/sdk/tsconfig.json +++ b/packages/sdk/tsconfig.json @@ -11,7 +11,7 @@ } }, "include": ["src/**/*"], - "exclude": ["node_modules", "dist", ".state"], + "exclude": ["node_modules", "dist", ".state", "src/**/*.test.ts", "src/__tests__"], "references": [ { "path": "../transport" } ] diff --git a/packages/sdk/tsconfig.test.json b/packages/sdk/tsconfig.test.json new file mode 100644 index 00000000..5a8fd6db --- /dev/null +++ b/packages/sdk/tsconfig.test.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "composite": false, + "noEmit": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", ".state"], + "references": [ + { "path": "../transport" } + ] +} diff --git a/packages/shared/package.json b/packages/shared/package.json index 95be476c..fb256181 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -35,10 +35,10 @@ "type-check": "tsc --noEmit" }, "dependencies": { + "@roj-ai/sdk": "workspace:*", "zod": "catalog:libs" }, "devDependencies": { - "@roj-ai/sdk": "workspace:*", "typescript": "^5.7.2" }, "types": "./dist/index.d.ts" diff --git a/packages/shared/src/lib/ids.ts b/packages/shared/src/lib/ids.ts index f45b39e2..74b63463 100644 --- a/packages/shared/src/lib/ids.ts +++ b/packages/shared/src/lib/ids.ts @@ -2,7 +2,11 @@ * Branded ID types and constructors. * * Uses Zod brands for structural compatibility with agent-server types. - * These are the canonical definitions — other packages import from here. + * + * NOT canonical: @roj-ai/sdk declares its own SessionId/AgentId/ChatMessageId + * (core/sessions/schema.ts, core/agents/schema.ts) and is the side the domain + * vocabulary is migrating to — see sdk/src/index.ts. The brands are structural, + * so the two sets stay assignable, but nothing asserts that they agree. */ import z from 'zod/v4' diff --git a/packages/shared/src/lib/result.ts b/packages/shared/src/lib/result.ts index eb771555..e9db63ec 100644 --- a/packages/shared/src/lib/result.ts +++ b/packages/shared/src/lib/result.ts @@ -1,6 +1,12 @@ /** * Result type for explicit error handling without exceptions. - * Inspired by Rust/Go approach. + * + * @roj-ai/transport owns the canonical definition and @roj-ai/sdk re-exports it + * from there. This copy stays standalone on purpose: @roj-ai/shared declares no + * runtime dependency beyond zod, and the client tier depends on shared. The + * type is structural, so the two are mutually assignable — but keep them in + * sync by hand, or move shared onto transport if that dependency ever becomes + * acceptable. */ export type Result = diff --git a/packages/standalone-server/CLAUDE.md b/packages/standalone-server/CLAUDE.md index 6fbd596e..13af9e6f 100644 --- a/packages/standalone-server/CLAUDE.md +++ b/packages/standalone-server/CLAUDE.md @@ -14,7 +14,7 @@ What this does: What this does NOT do: - Multi-tenancy (one `instanceId` per process, generated on startup) - Sandbox isolation (agent runs directly on the host) -- Authentication (no tokens, no cookies — bind to localhost) +- Authentication (no tokens, no cookies — binds to `127.0.0.1` by default) - Bundle management (`bundles.*` RPC returns `method_not_found`) - Publishing (`sessions.publish` returns `method_not_found`) @@ -62,8 +62,8 @@ Implemented: process-singleton) - `sessions.create/list` — delegates to SDK `callManagerMethod`. `initialPrompt` is delivered as a `user-chat.sendMessage` after creation, mirroring - roj-platform `activatePendingSession`. `resourceIds` are matched against the - local registry (see below). + roj-platform `activatePendingSession`. This method does not accept resource or + file IDs; those belong only to `instances.create.autoCreateSession`. - `tokens.create` — returns `{ token: '' }` - `sessionFiles.createDownloadUrl` — HMAC-signed URL pointing at `GET /api/v1/instances/:id/sessions/:sid/files/{workspace|session}/{path}?token=...`, @@ -113,14 +113,19 @@ left alone (so previously uploaded revisions survive restarts). At session start, resources to inject are resolved in this order (mirroring roj-platform's project-init): -1. `input.resourceIds` (from `instances.create.autoCreateSession` or - `sessions.create`) — each value is matched against the registry first - by `id`, then by `slug`. Unmatched values are warned-and-skipped. -2. If nothing matched, fall back to `preset.defaultResourceSlugs` - (looked up by slug). +1. `instances.create.autoCreateSession.resourceIds` — each value is matched + against the registry first by `id`, then by `slug`. +2. `instances.create.autoCreateSession.fileIds` — each value is matched directly + against a registry file. +3. If neither explicit list contains an ID, fall back to + `preset.defaultResourceSlugs` (looked up by slug). -For each resolved resource, the server reads the latest revision's file -bytes from the registry and calls `resources.inject` directly on the +Unmatched IDs are warned-and-skipped. Files are injected in the listed order, +with resources before direct files. A registry file selected more than once is +injected once. + +For each selected file, the server reads its bytes from the registry and calls +`resources.inject` directly on the session — same plugin method the SDK's `POST /sessions/:sid/inject-resource` HTTP route uses, just bypassing the URL fetch. @@ -155,10 +160,15 @@ Lifecycle: if `repo.git` already exists). The bare gets an empty `Initial commit` on `main` via plumbing (`mktree`, `commit-tree`, `update-ref`) so subsequent `worktree add -b session/{sid}` calls have something to branch from. -- `sessions.create` mints the session id locally, `git worktree add`s its +- Session creation mints the session id locally, `git worktree add`s its worktree, and passes the worktree path to the SDK as `workspaceDir`. If `sessionManager.callManagerMethod('sessions.create', ...)` fails, the worktree is rolled back so retries aren't blocked by an orphaned dir. +- If resource injection fails after SDK creation, standalone closes the domain + session (which awaits runtime disposal) and removes the worktree before + returning the original injection error. The closed session events remain in + the event store and can appear in history; standalone does not delete domain + history as part of rollback. - Worktrees persist across `roj-standalone` restarts. There is currently no automatic cleanup — to nuke instance state, stop the server and `rm -rf {dataPath}/instances/`. @@ -166,3 +176,10 @@ Lifecycle: No auto-commit. Resources extracted via `resources.inject` land in the worktree as untracked files; whether to commit them is the agent's call, matching platform behavior. + +## Network binding + +Without an explicit host, standalone binds to `127.0.0.1`. `config.host` takes +precedence over the `HOST` environment variable. Explicit non-loopback hosts +are allowed for intentional network access, but startup logs a warning because +standalone does not authenticate requests. Protect such a listener externally. diff --git a/packages/standalone-server/package.json b/packages/standalone-server/package.json index 84fea8f9..c331b6d2 100644 --- a/packages/standalone-server/package.json +++ b/packages/standalone-server/package.json @@ -35,9 +35,9 @@ "dependencies": { "@roj-ai/client": "workspace:*", "@roj-ai/sdk": "workspace:*", - "@roj-ai/shared": "workspace:*", "@roj-ai/transport": "workspace:*", - "hono": "catalog:libs" + "hono": "catalog:libs", + "zod": "catalog:libs" }, "devDependencies": { "@types/bun": "catalog:", diff --git a/packages/standalone-server/src/platform-api.ts b/packages/standalone-server/src/platform-api.ts index 0792d74c..b443a3b2 100644 --- a/packages/standalone-server/src/platform-api.ts +++ b/packages/standalone-server/src/platform-api.ts @@ -14,42 +14,244 @@ * - instances.archive — no-op; shutdown the server instead */ -import type { Logger, Preset, SessionManager } from '@roj-ai/sdk' -import { SessionId } from '@roj-ai/sdk' +import type { MethodInput, MethodOutput, PlatformMethodName, PlatformMethods } from '@roj-ai/client/platform' +import type { Logger, Preset, Result, Session } from '@roj-ai/sdk' +import { SessionId, sessionMetadataSchema } from '@roj-ai/sdk' +import z from 'zod/v4' import { randomUUID } from 'node:crypto' import { Hono } from 'hono' -import type { GitInstanceFs } from './git-instance-fs.js' import type { InstanceState } from './instance.js' import type { LocalRegistry } from './local-registry.js' import { signFileToken } from './signed-token.js' +interface RpcError { + type: string + message: string + httpStatus: number +} + +interface PlatformSessionManager { + callManagerMethod(method: string, input: unknown): Promise> + callPluginMethod(sessionId: SessionId, method: string, input: unknown): Promise> + getSession(sessionId: SessionId): Promise, RpcError>> + getStats(): Promise<{ + sessions: Array<{ id: SessionId; presetId: string; status: string }> + }> +} + +interface SessionGitFs { + addSessionWorktree(instanceId: string, sessionId: string): Promise + removeSessionWorktree(instanceId: string, sessionId: string): Promise +} + interface Deps { instance: InstanceState - sessionManager: SessionManager + sessionManager: PlatformSessionManager logger: Logger presets: Preset[] registry: LocalRegistry /** Per-instance bare repo + per-session worktree manager. */ - gitFs: GitInstanceFs + gitFs: SessionGitFs /** HMAC secret for signing download tokens (`sessionFiles.createDownloadUrl`). */ tokenSecret: string /** Externally-reachable base URL (e.g. `http://localhost:8765`) used when minting download URLs. */ - publicBaseUrl: string + getPublicBaseUrl(): string } -interface RpcEnvelope { - method?: string - input?: unknown - batch?: Array<{ method: string; input?: unknown }> -} +const RpcCallSchema = z.strictObject({ + method: z.string(), + input: z.unknown().optional(), +}) + +const RpcEnvelopeSchema = z.union([ + RpcCallSchema, + z.strictObject({ batch: z.array(RpcCallSchema) }), +]) + +const AutoCreateSessionSchema = z.strictObject({ + presetId: z.string().min(1), + blocking: z.boolean().optional(), + initialPrompt: z.string().optional(), + resourceIds: z.array(z.string()).optional(), + fileIds: z.array(z.string()).optional(), +}) + +const inputSchemas = { + 'instances.create': z.strictObject({ + templateSlug: z.string(), + bundleSlug: z.string().optional(), + bundleRevisionId: z.string().optional(), + name: z.string(), + vcsType: z.enum(['github', 'gitLocal', 'none']).optional(), + metadata: z.record(z.string(), z.unknown()).optional(), + autoCreateSession: AutoCreateSessionSchema.optional(), + }), + 'instances.get': z.strictObject({ instanceId: z.string() }), + 'instances.list': z.strictObject({ + limit: z.number().optional(), + offset: z.number().optional(), + }), + 'instances.status': z.strictObject({ instanceId: z.string() }), + 'instances.archive': z.strictObject({ instanceId: z.string() }), + 'sessions.create': z.strictObject({ + instanceId: z.string(), + presetId: z.string().min(1), + blocking: z.boolean().optional(), + origin: z.string().optional(), + expiresIn: z.number().optional(), + initialPrompt: z.string().optional(), + }), + 'sessions.list': z.strictObject({ instanceId: z.string() }), + 'tokens.create': z.strictObject({ + instanceId: z.string(), + origin: z.string().optional(), + expiresIn: z.number().optional(), + meta: z.record(z.string(), z.unknown()).optional(), + }), + 'resources.create': z.strictObject({ + slug: z.string(), + name: z.string().optional(), + description: z.string().optional(), + fileId: z.string(), + label: z.string().optional(), + }), + 'resources.addRevision': z.strictObject({ + resourceId: z.string().optional(), + resourceSlug: z.string().optional(), + fileId: z.string(), + label: z.string().optional(), + }), + 'resources.get': z.strictObject({ + resourceId: z.string().optional(), + resourceSlug: z.string().optional(), + }), + 'resources.list': z.strictObject({ + limit: z.number().optional(), + offset: z.number().optional(), + }), + 'resources.delete': z.strictObject({ resourceId: z.string() }), + 'sessionFiles.createDownloadUrl': z.strictObject({ + instanceId: z.string(), + sessionId: z.string(), + scope: z.enum(['workspace', 'session']), + path: z.string(), + ttlSeconds: z.number().optional(), + }), +} satisfies Partial> + +type ImplementedMethod = keyof typeof inputSchemas + +const InstanceSummaryOutputSchema = z.strictObject({ + instanceId: z.string(), + name: z.string(), + status: z.string(), + templateSlug: z.string(), + bundleSlug: z.string(), + bundleRevisionId: z.string(), + vcsType: z.string(), + metadata: z.record(z.string(), z.unknown()).nullable(), + createdAt: z.string(), +}) + +const SessionSummaryOutputSchema = z.strictObject({ + id: z.string(), + presetId: z.string().nullable(), + status: z.string(), + createdAt: z.string(), +}) + +const ResourceOutputSchema = z.strictObject({ + id: z.string(), + slug: z.string(), + name: z.string().nullable(), + description: z.string().nullable(), + latestRevision: z.strictObject({ + id: z.string(), + label: z.string().nullable(), + file: z.strictObject({ + id: z.string(), + filename: z.string(), + mimeType: z.string(), + size: z.number(), + }), + createdAt: z.string(), + }).nullable(), + createdAt: z.string(), +}) + +const outputSchemas = { + 'instances.create': z.strictObject({ + instanceId: z.string(), + status: z.enum(['created', 'initializing', 'ready']), + sessionId: z.string().optional(), + wsToken: z.string().optional(), + }), + 'instances.get': InstanceSummaryOutputSchema, + 'instances.list': z.strictObject({ + instances: z.array(InstanceSummaryOutputSchema), + total: z.number(), + }), + 'instances.status': z.strictObject({ + instanceId: z.string(), + status: z.string(), + sandbox: z.strictObject({ + state: z.enum(['stopped', 'starting', 'running', 'pausing', 'paused', 'failed']), + e2bId: z.string().optional(), + lastActivityAt: z.string().optional(), + versions: z.strictObject({ + sdk: z.string(), + runtime: z.strictObject({ name: z.string(), version: z.string() }).nullable(), + }).optional(), + }).nullable(), + sessions: z.array(SessionSummaryOutputSchema), + lifecycleEvents: z.array(z.strictObject({ + event: z.string(), + detail: z.string().optional(), + createdAt: z.string(), + })), + serviceUrls: z.array(z.strictObject({ + code: z.string(), + sessionId: z.string().nullable(), + serviceType: z.string().nullable(), + port: z.number(), + })), + }), + 'instances.archive': z.strictObject({ ok: z.boolean() }), + 'sessions.create': z.strictObject({ + sessionId: z.string(), + status: z.enum(['creating', 'active']), + wsToken: z.string().optional(), + }), + 'sessions.list': z.strictObject({ sessions: z.array(SessionSummaryOutputSchema) }), + 'tokens.create': z.strictObject({ token: z.string(), expiresAt: z.string() }), + 'resources.create': z.strictObject({ resourceId: z.string(), revisionId: z.string() }), + 'resources.addRevision': z.strictObject({ + revisionId: z.string(), + noop: z.boolean().optional(), + }), + 'resources.get': ResourceOutputSchema, + 'resources.list': z.strictObject({ resources: z.array(ResourceOutputSchema) }), + 'resources.delete': z.strictObject({ ok: z.boolean() }), + 'sessionFiles.createDownloadUrl': z.strictObject({ url: z.string(), expiresAt: z.string() }), +} satisfies { [M in ImplementedMethod]: z.ZodType> } export function createPlatformApi(deps: Deps): Hono { const app = new Hono() app.post('/rpc', async (c) => { - const body = await c.req.json().catch(() => ({} as RpcEnvelope)) + let rawBody: unknown + try { + rawBody = await c.req.json() + } catch { + return c.json({ ok: false, error: { type: 'invalid_request', message: 'Invalid JSON body' } }, 400) + } + const parsedBody = RpcEnvelopeSchema.safeParse(rawBody) + if (!parsedBody.success) { + return c.json({ ok: false, error: { type: 'invalid_request', message: parsedBody.error.message } }, 400) + } + const body = parsedBody.data - if (Array.isArray(body.batch)) { + if ('batch' in body) { const results = [] for (const call of body.batch) { results.push(await dispatch(deps, call.method, call.input)) @@ -57,10 +259,6 @@ export function createPlatformApi(deps: Deps): Hono { return c.json({ results }) } - if (typeof body.method !== 'string') { - return c.json({ ok: false, error: { type: 'invalid_request', message: 'Missing method' } }, 400) - } - const result = await dispatch(deps, body.method, body.input) return c.json(result) }) @@ -73,33 +271,107 @@ async function dispatch( method: string, input: unknown, ): Promise<{ ok: true; value: unknown } | { ok: false; error: { type: string; message: string } }> { - const handler = handlers[method] - if (!handler) { - return { ok: false, error: { type: 'method_not_found', message: `Method not supported in standalone: ${method}` } } - } - try { - const value = await handler(deps, input ?? {}) + const value = await callHandler(deps, method, input ?? {}) return { ok: true, value } } catch (err) { + if (err instanceof UnsupportedMethodError) { + return { ok: false, error: { type: 'method_not_found', message: err.message } } + } const message = err instanceof Error ? err.message : String(err) deps.logger.error(`Platform RPC handler failed: ${method}`, err instanceof Error ? err : new Error(message)) return { ok: false, error: { type: 'handler_error', message } } } } -type Handler = (deps: Deps, input: any) => Promise +async function callHandler(deps: Deps, method: string, input: unknown): Promise { + switch (method) { + case 'instances.create': + return outputSchemas['instances.create'].parse( + await handlers['instances.create'](deps, inputSchemas['instances.create'].parse(input)), + ) + case 'instances.get': + return outputSchemas['instances.get'].parse( + await handlers['instances.get'](deps, inputSchemas['instances.get'].parse(input)), + ) + case 'instances.list': + return outputSchemas['instances.list'].parse( + await handlers['instances.list'](deps, inputSchemas['instances.list'].parse(input)), + ) + case 'instances.status': + return outputSchemas['instances.status'].parse( + await handlers['instances.status'](deps, inputSchemas['instances.status'].parse(input)), + ) + case 'instances.archive': + return outputSchemas['instances.archive'].parse( + await handlers['instances.archive'](deps, inputSchemas['instances.archive'].parse(input)), + ) + case 'sessions.create': + return outputSchemas['sessions.create'].parse( + await handlers['sessions.create'](deps, inputSchemas['sessions.create'].parse(input)), + ) + case 'sessions.list': + return outputSchemas['sessions.list'].parse( + await handlers['sessions.list'](deps, inputSchemas['sessions.list'].parse(input)), + ) + case 'tokens.create': + return outputSchemas['tokens.create'].parse( + await handlers['tokens.create'](deps, inputSchemas['tokens.create'].parse(input)), + ) + case 'resources.create': + return outputSchemas['resources.create'].parse( + await handlers['resources.create'](deps, inputSchemas['resources.create'].parse(input)), + ) + case 'resources.addRevision': + return outputSchemas['resources.addRevision'].parse( + await handlers['resources.addRevision'](deps, inputSchemas['resources.addRevision'].parse(input)), + ) + case 'resources.get': + return outputSchemas['resources.get'].parse( + await handlers['resources.get'](deps, inputSchemas['resources.get'].parse(input)), + ) + case 'resources.list': + return outputSchemas['resources.list'].parse( + await handlers['resources.list'](deps, inputSchemas['resources.list'].parse(input)), + ) + case 'resources.delete': + return outputSchemas['resources.delete'].parse( + await handlers['resources.delete'](deps, inputSchemas['resources.delete'].parse(input)), + ) + case 'sessionFiles.createDownloadUrl': + return outputSchemas['sessionFiles.createDownloadUrl'].parse( + await handlers['sessionFiles.createDownloadUrl']( + deps, + inputSchemas['sessionFiles.createDownloadUrl'].parse(input), + ), + ) + default: + throw new UnsupportedMethodError(method) + } +} -interface AutoCreateSessionInput { - presetId: string - initialPrompt?: string - resourceIds?: string[] - fileIds?: string[] - blocking?: boolean +class UnsupportedMethodError extends Error { + constructor(readonly method: string) { + super(`Method not supported in standalone: ${method}`) + } } -// Shared by `instances.create.autoCreateSession` and `sessions.create`. Creates -// a session via the SDK session manager, injects any matching local resources, +/** + * A handler for one platform method, typed against the shared contract. + * + * `MethodInput`/`MethodOutput` come from @roj-ai/client/platform, so renaming a + * method or reshaping its payload there is a compile error here instead of a + * runtime `method_not_found` or a wrong-shaped JSON body. Two divergences had + * already shipped before this was wired up. + */ +type Handler = ( + deps: Deps, + input: MethodInput, +) => Promise> + +type PlatformHandlers = { [M in ImplementedMethod]: Handler } + +// Creates a session via the SDK session manager, injects selected local files, // then (if set) pushes `initialPrompt` as a user-chat message — order matters: // resources must land in the workspace before the agent's first inference so // the agent sees a non-empty workspace. Mirrors roj-platform's project-init @@ -110,8 +382,8 @@ interface AutoCreateSessionInput { // — the worktree path is then passed through as `workspaceDir`. async function startSession( deps: Deps, - input: { presetId: string; initialPrompt?: string; resourceIds?: string[] }, -): Promise<{ sessionId: string }> { + input: { presetId: string; initialPrompt?: string; resourceIds?: string[]; fileIds?: string[] }, +): Promise<{ sessionId: string; status: 'active' }> { const sessionId = randomUUID() const workspaceDir = await deps.gitFs.addSessionWorktree(deps.instance.id, sessionId) @@ -121,15 +393,18 @@ async function startSession( workspaceDir, }) if (!created.ok) { - // Roll back the worktree we just created so a retry isn't blocked by an - // orphaned dir. Best-effort — failure here is logged inside removeSessionWorktree. - await deps.gitFs.removeSessionWorktree(deps.instance.id, sessionId) + await removeWorktreeAfterFailure(deps, sessionId) throw new Error(created.error.message) } - const resources = resolveSessionResources(deps, input.presetId, input.resourceIds) - for (const resource of resources) { - await injectRegistryResource(deps, sessionId, resource) + try { + const selections = resolveSessionFiles(deps, input.presetId, input.resourceIds, input.fileIds) + for (const selection of selections) { + await injectRegistryFile(deps, sessionId, selection) + } + } catch (error) { + await rollbackCreatedSession(deps, sessionId) + throw error } if (input.initialPrompt) { @@ -144,36 +419,44 @@ async function startSession( } } - return { sessionId } + // Creation is synchronous here — there is no provisioning step to wait on, + // so the session is active by the time this returns. The contract's other + // value, 'creating', belongs to the CF platform. + return { sessionId, status: 'active' } } -interface ResolvedResource { - resourceId: string - slug: string - name: string | null +interface ResolvedFile { fileId: string - filename: string - mimeType: string + metadata?: { slug?: string; name?: string } } // Resolution mirrors roj-platform's project-init.ts:206: -// 1. explicit input.resourceIds — matched against the local registry by ID -// first, then by slug (since local-dev callers often pass slugs as ids). -// 2. fallback to preset.defaultResourceSlugs (looked up by slug). -function resolveSessionResources( +// 1. explicit resourceIds, followed by explicit fileIds. +// 2. preset.defaultResourceSlugs only when no explicit IDs were supplied. +// Repeated selections of the same registry file are injected once. +function resolveSessionFiles( deps: Deps, presetId: string, inputResourceIds: string[] | undefined, -): ResolvedResource[] { - if (inputResourceIds && inputResourceIds.length > 0) { - const matched: ResolvedResource[] = [] + inputFileIds: string[] | undefined, +): ResolvedFile[] { + const hasExplicitFiles = (inputResourceIds?.length ?? 0) > 0 || (inputFileIds?.length ?? 0) > 0 + const resolved: ResolvedFile[] = [] + const seenFileIds = new Set() + + if (hasExplicitFiles) { const unmatched: string[] = [] - for (const id of inputResourceIds) { + for (const id of inputResourceIds ?? []) { const resource = deps.registry.getResource({ resourceId: id }) ?? deps.registry.getResource({ resourceSlug: id }) - const resolved = toResolvedResource(resource) - if (resolved) matched.push(resolved) - else unmatched.push(id) + const selection = toResolvedFile(resource) + if (!selection) { + unmatched.push(id) + continue + } + if (seenFileIds.has(selection.fileId)) continue + seenFileIds.add(selection.fileId) + resolved.push(selection) } if (unmatched.length > 0) { deps.logger.warn('Some input resourceIds did not match the local registry; ignoring', { @@ -181,17 +464,26 @@ function resolveSessionResources( availableSlugs: deps.registry.listResources().map(r => r.slug), }) } - if (matched.length > 0) return matched + for (const fileId of inputFileIds ?? []) { + if (seenFileIds.has(fileId)) continue + seenFileIds.add(fileId) + resolved.push({ fileId }) + } + return resolved } const preset = deps.presets.find(p => p.id === presetId) const slugs = preset?.defaultResourceSlugs ?? [] - const resolved: ResolvedResource[] = [] const missing: string[] = [] for (const slug of slugs) { - const r = toResolvedResource(deps.registry.getResource({ resourceSlug: slug })) - if (r) resolved.push(r) - else missing.push(slug) + const selection = toResolvedFile(deps.registry.getResource({ resourceSlug: slug })) + if (!selection) { + missing.push(slug) + continue + } + if (seenFileIds.has(selection.fileId)) continue + seenFileIds.add(selection.fileId) + resolved.push(selection) } if (missing.length > 0) { deps.logger.warn( @@ -202,48 +494,96 @@ function resolveSessionResources( return resolved } -function toResolvedResource(resource: ReturnType): ResolvedResource | null { +function toResolvedFile(resource: ReturnType): ResolvedFile | null { if (!resource || !resource.latestRevision) return null const file = resource.latestRevision.file return { - resourceId: resource.id, - slug: resource.slug, - name: resource.name, fileId: file.id, - filename: file.filename, - mimeType: file.mimeType, + metadata: { slug: resource.slug, name: resource.name ?? resource.slug }, } } -async function injectRegistryResource(deps: Deps, sessionId: string, resource: ResolvedResource): Promise { - const file = await deps.registry.readFileById(resource.fileId) +async function injectRegistryFile(deps: Deps, sessionId: string, selection: ResolvedFile): Promise { + const file = await deps.registry.readFileById(selection.fileId) if (!file) { - throw new Error(`Registry file not found for resource ${resource.slug} (fileId=${resource.fileId})`) + if (selection.metadata?.slug) { + throw new Error( + `Registry file not found for resource ${selection.metadata.slug} (fileId=${selection.fileId})`, + ) + } + deps.logger.warn('Selected fileId did not match the local registry; ignoring', { + fileId: selection.fileId, + }) + return } const result = await deps.sessionManager.callPluginMethod(SessionId(sessionId), 'resources.inject', { sessionId, - filename: resource.filename, - mimeType: resource.mimeType, + filename: file.meta.filename, + mimeType: file.meta.mimeType, size: file.buffer.length, fileBuffer: file.buffer, - metadata: { slug: resource.slug, name: resource.name ?? resource.slug }, + metadata: selection.metadata, }) if (!result.ok) { deps.logger.error('Registry resource injection failed', undefined, { sessionId, - slug: resource.slug, + fileId: selection.fileId, error: result.error, }) - throw new Error(`Failed to inject resource '${resource.slug}': ${result.error.message ?? result.error.type}`) + throw new Error(`Failed to inject registry file '${selection.fileId}': ${result.error.message}`) + } +} + +async function rollbackCreatedSession(deps: Deps, sessionId: string): Promise { + let sessionResult: Awaited> | null = null + try { + sessionResult = await deps.sessionManager.getSession(SessionId(sessionId)) + } catch (error) { + deps.logger.error( + 'Failed to load session for rollback', + error instanceof Error ? error : new Error(String(error)), + { sessionId }, + ) + } + + if (sessionResult && !sessionResult.ok) { + deps.logger.error('Failed to load session for rollback', undefined, { + sessionId, + error: sessionResult.error, + }) + } else if (sessionResult) { + try { + const closed = await sessionResult.value.close() + if (!closed.ok) { + deps.logger.error('Failed to close session during rollback', undefined, { sessionId, error: closed.error }) + } + } catch (error) { + deps.logger.error( + 'Failed to close session during rollback', + error instanceof Error ? error : new Error(String(error)), + { sessionId }, + ) + } + } + + await removeWorktreeAfterFailure(deps, sessionId) +} + +async function removeWorktreeAfterFailure(deps: Deps, sessionId: string): Promise { + try { + await deps.gitFs.removeSessionWorktree(deps.instance.id, sessionId) + } catch (error) { + deps.logger.error( + 'Failed to remove session worktree during rollback', + error instanceof Error ? error : new Error(String(error)), + { sessionId }, + ) } } -const handlers: Record = { - 'instances.create': async ( - deps, - input: { metadata?: Record; autoCreateSession?: AutoCreateSessionInput }, - ) => { +const handlers: PlatformHandlers = { + 'instances.create': async (deps, input) => { if (input.metadata !== undefined) { deps.instance.metadata = input.metadata } @@ -254,6 +594,7 @@ const handlers: Record = { presetId: input.autoCreateSession.presetId, initialPrompt: input.autoCreateSession.initialPrompt, resourceIds: input.autoCreateSession.resourceIds, + fileIds: input.autoCreateSession.fileIds, }) sessionId = result.sessionId } @@ -291,51 +632,46 @@ const handlers: Record = { 'instances.archive': async () => ({ ok: true }), - 'sessions.create': async ( - deps, - input: { presetId: string; initialPrompt?: string; resourceIds?: string[] }, - ) => startSession(deps, input), + 'sessions.create': async (deps, input) => startSession(deps, input), 'sessions.list': async ({ sessionManager }) => { const result = await sessionManager.callManagerMethod('sessions.list', {}) if (!result.ok) throw new Error(result.error.message) - return result.value as { sessions: unknown[]; total: number } + // callManagerMethod is typed Result, so the plugin's output schema + // does not reach us — validate here rather than assert. + const listed = z.object({ sessions: z.array(sessionMetadataSchema) }).parse(result.value) + return { + sessions: listed.sessions.map(s => ({ + id: String(s.sessionId), + presetId: s.presetId, + status: s.status, + createdAt: new Date(s.createdAt).toISOString(), + })), + } }, - 'tokens.create': async () => ({ token: '' }), + // Standalone has no auth: the token is empty and never checked. expiresAt is + // still part of the contract, so hand back a real timestamp rather than + // omitting the field and hoping no caller reads it. + 'tokens.create': async () => ({ token: '', expiresAt: new Date(Date.now() + 3600_000).toISOString() }), - 'resources.create': async ( - deps, - input: { slug: string; name?: string; description?: string; fileId: string; label?: string }, - ) => deps.registry.createResource(input), + 'resources.create': async (deps, input) => deps.registry.createResource(input), - 'resources.addRevision': async ( - deps, - input: { resourceId?: string; resourceSlug?: string; fileId: string; label?: string }, - ) => deps.registry.addRevision(input), + 'resources.addRevision': async (deps, input) => deps.registry.addRevision(input), - 'resources.get': async (deps, input: { resourceId?: string; resourceSlug?: string }) => { + 'resources.get': async (deps, input) => { const resource = deps.registry.getResource(input) if (!resource) throw new Error('Resource not found') return resource }, - 'resources.list': async (deps, _input: { limit?: number; offset?: number }) => ({ + 'resources.list': async (deps, _input) => ({ resources: deps.registry.listResources(), }), - 'resources.delete': async (deps, input: { resourceId: string }) => deps.registry.deleteResource(input.resourceId), - - 'sessionFiles.createDownloadUrl': async ( - deps, - input: { - instanceId: string - sessionId: string - scope: 'workspace' | 'session' - path: string - ttlSeconds?: number - }, - ) => { + 'resources.delete': async (deps, input) => deps.registry.deleteResource(input.resourceId), + + 'sessionFiles.createDownloadUrl': async (deps, input) => { if (input.scope !== 'workspace' && input.scope !== 'session') { throw new Error(`Invalid scope: ${input.scope}`) } @@ -358,7 +694,7 @@ const handlers: Record = { .split('/') .map(seg => encodeURIComponent(seg)) .join('/') - const url = `${deps.publicBaseUrl}/api/v1/instances/${input.instanceId}/sessions/${input.sessionId}/files/${input.scope}/${encodedPath}?token=${encodeURIComponent(token)}` + const url = `${deps.getPublicBaseUrl()}/api/v1/instances/${input.instanceId}/sessions/${input.sessionId}/files/${input.scope}/${encodedPath}?token=${encodeURIComponent(token)}` return { url, expiresAt: new Date(expiresAt).toISOString() } }, diff --git a/packages/standalone-server/src/server.ts b/packages/standalone-server/src/server.ts index 1cedb7a6..ab672d9e 100644 --- a/packages/standalone-server/src/server.ts +++ b/packages/standalone-server/src/server.ts @@ -16,7 +16,7 @@ import type { Config, LLMMiddleware, LocalResource, Logger, Preset, SessionId, S import { bootstrap, createSystemFromServices, loadConfig, validateConfig } from '@roj-ai/sdk' import { createApp } from '@roj-ai/sdk/transport/http/app' import { createAgentTransport, ServerAdapter } from '@roj-ai/sdk/transport/adapter' -import { createBunPlatform } from '@roj-ai/sdk/bun-platform' +import { createBunPlatform, createServerLifecycle, runServerStartup, shutdownFromSignal } from '@roj-ai/sdk/bun-platform' import { createBunWebSocketHandlers } from '@roj-ai/transport/bun' import { Hono } from 'hono' import { cors } from 'hono/cors' @@ -58,22 +58,28 @@ export interface StandaloneHandle { export async function startStandaloneServer(options: StartStandaloneOptions): Promise { const envConfig = loadConfig() - const config: Config = options.config ? { ...envConfig, ...options.config } : envConfig + const config: Config = { + ...envConfig, + ...options.config, + host: resolveStandaloneHost(options.config?.host, process.env.HOST), + } const errors = validateConfig(config) if (errors.length > 0) { throw new Error(`Configuration errors:\n${errors.map(e => ` - ${e}`).join('\n')}`) } - const presets = options.llmMiddleware?.length + const llmMiddleware = options.llmMiddleware + const presets = llmMiddleware?.length ? options.presets.map(p => ({ ...p, - llmMiddleware: [...options.llmMiddleware!, ...(p.llmMiddleware ?? [])], + llmMiddleware: [...llmMiddleware, ...(p.llmMiddleware ?? [])], })) : options.presets const services = bootstrap(config, { presets }, createBunPlatform()) const { logger } = services + warnIfStandaloneExposed(config.host, logger) // Reap service processes left behind by a previous agent, before any session can load // and start its own. At boot this agent owns nothing, so every survivor is an orphan. @@ -103,8 +109,8 @@ export async function startStandaloneServer(options: StartStandaloneOptions): Pr }) const tokenSecret = generateTokenSecret() - const publicHost = config.host === '0.0.0.0' ? 'localhost' : config.host - const publicBaseUrl = `http://${publicHost}:${config.port}` + let publicPort = config.port + const getPublicBaseUrl = () => `http://${formatPublicHost(config.host)}:${publicPort}` const registry = new LocalRegistry(config.dataPath, logger) await registry.init() @@ -123,7 +129,7 @@ export async function startStandaloneServer(options: StartStandaloneOptions): Pr registry, gitFs, tokenSecret, - publicBaseUrl, + getPublicBaseUrl, }) const outerApp = new Hono() @@ -168,44 +174,74 @@ export async function startStandaloneServer(options: StartStandaloneOptions): Pr }) const server = startBunServer(config, outerApp, serverAdapter) - - try { - await sessionManager.loadAllSessions() - } catch (err) { - logger.error('Failed to load persisted sessions', err instanceof Error ? err : new Error(String(err))) + publicPort = server.port ?? config.port + let transportStopPromise: Promise | undefined + let transportStopErrorReported = false + const stopTransportOnce = async () => { + transportStopPromise ??= Promise.resolve().then(() => transport.stop()) + try { + await transportStopPromise + } catch (error) { + if (transportStopErrorReported) return + transportStopErrorReported = true + throw error + } } - try { - await transport.start() - } catch (err) { - logger.error('Transport start failed', err instanceof Error ? err : new Error(String(err))) - } + const lifecycle = createServerLifecycle({ + stopIngress: () => server.stop(true), + cleanupSteps: [ + ...options.onShutdown ? [options.onShutdown] : [], + () => sessionManager.shutdown(), + stopTransportOnce, + ], + runSignalShutdown: shutdown => { + void shutdownFromSignal(shutdown) + }, + }) + + await runServerStartup(lifecycle, [ + { run: () => sessionManager.loadAllSessions() }, + // A pending connection is canceled before the remaining ordered cleanup. + { run: () => transport.start(), cancel: stopTransportOnce }, + ]) logger.info('Standalone server started', { host: config.host, - port: config.port, + port: publicPort, instanceId: instance.id, - url: `http://${config.host === '0.0.0.0' ? 'localhost' : config.host}:${config.port}`, + url: getPublicBaseUrl(), }) - const shutdown = async () => { - logger.info('Shutting down standalone server...') - if (options.onShutdown) { - await options.onShutdown() - } - await sessionManager.shutdown() - await transport.stop() - server.stop() - } + return { config, logger, instance, port: publicPort, sessionManager, shutdown: lifecycle.shutdown } +} - process.on('SIGINT', () => { - void shutdown().then(() => process.exit(0)) - }) - process.on('SIGTERM', () => { - void shutdown().then(() => process.exit(0)) +export function resolveStandaloneHost(configHost: string | undefined, envHost: string | undefined): string { + return configHost ?? envHost ?? '127.0.0.1' +} + +export function isLoopbackHost(host: string): boolean { + const normalized = host.toLowerCase().replace(/^\[|\]$/g, '') + if (normalized === 'localhost' || normalized === '::1' || normalized === '0:0:0:0:0:0:0:1') return true + if (normalized.startsWith('::ffff:')) return isLoopbackHost(normalized.slice('::ffff:'.length)) + const ipv4 = normalized.split('.') + return ipv4.length === 4 + && ipv4.every(part => /^\d{1,3}$/.test(part) && Number(part) <= 255) + && ipv4[0] === '127' +} + +export function warnIfStandaloneExposed(host: string, logger: Pick): void { + if (isLoopbackHost(host)) return + logger.warn('Standalone server is listening on a non-loopback host without authentication', { + host, + action: 'Bind to 127.0.0.1 unless network access is intentional and protected externally', }) +} - return { config, logger, instance, port: server.port ?? config.port, sessionManager, shutdown } +function formatPublicHost(host: string): string { + if (host === '0.0.0.0' || host === '::' || host === '[::]') return 'localhost' + const unwrapped = host.replace(/^\[|\]$/g, '') + return unwrapped.includes(':') ? `[${unwrapped}]` : unwrapped } interface WSData { @@ -251,4 +287,3 @@ function startBunServer( websocket: wsHandlers ?? { open() {}, close() {}, message() {} }, }) } - diff --git a/packages/standalone-server/tests/platform-api.test.ts b/packages/standalone-server/tests/platform-api.test.ts new file mode 100644 index 00000000..21d07dd1 --- /dev/null +++ b/packages/standalone-server/tests/platform-api.test.ts @@ -0,0 +1,413 @@ +import { afterEach, describe, expect, it } from 'bun:test' +import type { Logger } from '@roj-ai/sdk' +import { ModelId, SessionId } from '@roj-ai/sdk' +import type { Preset } from '@roj-ai/sdk' +import { rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { z } from 'zod/v4' +import { createInstance } from '../src/instance.js' +import { LocalRegistry } from '../src/local-registry.js' +import { createPlatformApi } from '../src/platform-api.js' + +const RpcResultSchema = z.union([ + z.object({ ok: z.literal(true), value: z.unknown() }), + z.object({ + ok: z.literal(false), + error: z.object({ type: z.string(), message: z.string() }), + }), +]) + +const CreateInstanceValueSchema = z.object({ + instanceId: z.string(), + status: z.enum(['created', 'initializing', 'ready']), + sessionId: z.string().optional(), +}) + +const InjectionInputSchema = z.object({ + filename: z.string(), + mimeType: z.string(), + fileBuffer: z.custom(), +}) + +const SessionCreateInputSchema = z.object({ sessionId: z.string() }) +const PromptInputSchema = z.object({ content: z.string() }) + +const cleanupPaths: string[] = [] +const cleanupFailureCases: Array<{ + stage: 'getSession' | 'close' | 'worktree' + expectedLog: string +}> = [ + { stage: 'getSession', expectedLog: 'Failed to load session for rollback' }, + { stage: 'close', expectedLog: 'Failed to close session during rollback' }, + { stage: 'worktree', expectedLog: 'Failed to remove session worktree during rollback' }, +] + +afterEach(async () => { + await Promise.all(cleanupPaths.splice(0).map(path => rm(path, { recursive: true, force: true }))) +}) + +async function createFixture(options: { presets?: Preset[] } = {}) { + const path = join(tmpdir(), `roj-standalone-platform-${crypto.randomUUID()}`) + cleanupPaths.push(path) + const calls: string[] = [] + const warnings: Array<{ message: string; context?: Record }> = [] + const errors: string[] = [] + const activeSessions = new Set() + const worktrees = new Set() + let failNextInjection = false + let malformedSessionList = false + let cleanupFailure: 'getSession' | 'close' | 'worktree' | undefined + + const logger: Logger = { + debug() {}, + info() {}, + warn(message, context) { + warnings.push({ message, context }) + }, + error(message) { + errors.push(message) + }, + child() { + return logger + }, + level: 'debug', + } + + const registry = new LocalRegistry(path, logger) + await registry.init() + + const sessionManager = { + async callManagerMethod(method: string, input: unknown) { + if (method === 'sessions.create') { + const { sessionId } = SessionCreateInputSchema.parse(input) + calls.push(`session:create:${sessionId}`) + activeSessions.add(sessionId) + return { ok: true, value: { sessionId } } + } + if (method === 'sessions.list') { + return malformedSessionList + ? { ok: true, value: { sessions: [{ malformed: true }], total: 1 } } + : { ok: true, value: { sessions: [], total: 0 } } + } + return { + ok: false, + error: { type: 'validation_error', message: `Unknown method: ${method}`, httpStatus: 400 }, + } + }, + async callPluginMethod(_sessionId: SessionId, method: string, input: unknown) { + if (method === 'resources.inject') { + const parsed = InjectionInputSchema.parse(input) + calls.push(`inject:${parsed.filename}`) + if (failNextInjection) { + failNextInjection = false + return { + ok: false, + error: { type: 'injection_failed', message: 'deterministic injection failure', httpStatus: 400 }, + } + } + return { ok: true, value: { resourceId: 'injected', paths: [parsed.filename] } } + } + if (method === 'user-chat.sendMessage') { + const parsed = PromptInputSchema.parse(input) + calls.push(`prompt:${parsed.content}`) + return { ok: true, value: {} } + } + return { + ok: false, + error: { type: 'validation_error', message: `Unknown method: ${method}`, httpStatus: 400 }, + } + }, + async getSession(sessionId: SessionId) { + const id = String(sessionId) + if (cleanupFailure === 'getSession') throw new Error('getSession cleanup failed') + if (!activeSessions.has(id)) { + return { + ok: false, + error: { type: 'session_not_found', message: `Session not found: ${id}`, httpStatus: 404 }, + } + } + return { + ok: true, + value: { + async close() { + if (cleanupFailure === 'close') throw new Error('close cleanup failed') + calls.push(`session:close:${id}`) + activeSessions.delete(id) + return { ok: true, value: undefined } + }, + }, + } + }, + async getStats() { + return { + sessions: [...activeSessions].map(id => ({ id: SessionId(id), presetId: 'edit', status: 'active' })), + } + }, + } satisfies Parameters[0]['sessionManager'] + + const gitFs = { + async addSessionWorktree(_instanceId: string, sessionId: string) { + calls.push(`worktree:add:${sessionId}`) + worktrees.add(sessionId) + return join(path, 'sessions', sessionId) + }, + async removeSessionWorktree(_instanceId: string, sessionId: string) { + if (cleanupFailure === 'worktree') throw new Error('worktree cleanup failed') + calls.push(`worktree:remove:${sessionId}`) + worktrees.delete(sessionId) + }, + } + + const app = createPlatformApi({ + instance: createInstance({ id: 'instance-1', presetIds: ['edit'] }), + sessionManager, + logger, + presets: options.presets ?? [], + registry, + gitFs, + tokenSecret: 'test-secret', + getPublicBaseUrl: () => 'http://127.0.0.1:2486', + }) + + return { + app, + registry, + calls, + warnings, + errors, + activeSessions, + worktrees, + failNextInjection() { + failNextInjection = true + }, + returnMalformedSessionList() { + malformedSessionList = true + }, + failCleanupAt(stage: 'getSession' | 'close' | 'worktree') { + cleanupFailure = stage + }, + } +} + +function presetWithDefaults(...defaultResourceSlugs: string[]): Preset { + return { + id: 'edit', + name: 'Edit', + orchestrator: { + system: 'Test orchestrator', + model: ModelId('mock'), + tools: [], + agents: [], + }, + agents: [], + defaultResourceSlugs, + } +} + +async function rpc(app: ReturnType, method: string, input: unknown) { + const response = await app.request('/rpc', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ method, input }), + }) + return RpcResultSchema.parse(await response.json()) +} + +function createInstanceInput(autoCreateSession?: Record) { + return { + templateSlug: 'standalone', + name: 'Standalone', + ...(autoCreateSession ? { autoCreateSession } : {}), + } +} + +describe('standalone platform RPC', () => { + it.each(['resourceIds', 'fileIds'])('rejects %s on public sessions.create', async (field) => { + const fixture = await createFixture() + const result = await rpc(fixture.app, 'sessions.create', { + instanceId: 'instance-1', + presetId: 'edit', + [field]: ['forbidden'], + }) + + expect(result.ok).toBe(false) + if (result.ok) throw new Error('Expected sessions.create validation failure') + expect(result.error.type).toBe('handler_error') + expect(result.error.message).toContain('Unrecognized key') + expect(fixture.calls).toEqual([]) + }) + + it('injects resourceIds and fileIds once before initialPrompt', async () => { + const fixture = await createFixture() + const resourceFile = await fixture.registry.uploadFile({ + buffer: Buffer.from('resource'), + filename: 'resource.txt', + mimeType: 'text/plain', + }) + if ('error' in resourceFile) throw new Error('Expected resource registry file') + const directFile = await fixture.registry.uploadFile({ + buffer: Buffer.from('direct'), + filename: 'direct.txt', + mimeType: 'text/plain', + }) + if ('error' in directFile) throw new Error('Expected direct registry file') + const resource = await fixture.registry.createResource({ + slug: 'context', + name: 'Context', + fileId: resourceFile.fileId, + }) + + const result = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + resourceIds: [resource.resourceId, 'context'], + fileIds: [directFile.fileId, directFile.fileId, resourceFile.fileId], + initialPrompt: 'Start now', + })) + + expect(result.ok).toBe(true) + if (!result.ok) throw new Error(result.error.message) + const created = CreateInstanceValueSchema.parse(result.value) + expect(created.sessionId).toBeDefined() + const sessionId = created.sessionId + if (!sessionId) throw new Error('Expected auto-created session') + expect(fixture.calls).toEqual([ + `worktree:add:${sessionId}`, + `session:create:${sessionId}`, + 'inject:resource.txt', + 'inject:direct.txt', + 'prompt:Start now', + ]) + }) + + it('warns and skips missing resource and file IDs', async () => { + const fixture = await createFixture() + const result = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + resourceIds: ['missing-resource'], + fileIds: ['missing-file'], + initialPrompt: 'Continue', + })) + + expect(result.ok).toBe(true) + expect(fixture.calls.filter(call => call.startsWith('inject:'))).toEqual([]) + expect(fixture.calls.at(-1)).toBe('prompt:Continue') + expect(fixture.warnings.map(warning => warning.message)).toEqual([ + 'Some input resourceIds did not match the local registry; ignoring', + 'Selected fileId did not match the local registry; ignoring', + ]) + }) + + it('uses preset defaults when both explicit ID arrays are empty', async () => { + const fixture = await createFixture({ presets: [presetWithDefaults('default-context')] }) + const uploaded = await fixture.registry.uploadFile({ + buffer: Buffer.from('default'), + filename: 'default.txt', + mimeType: 'text/plain', + }) + if ('error' in uploaded) throw new Error('Expected default registry file') + await fixture.registry.createResource({ slug: 'default-context', fileId: uploaded.fileId }) + + const result = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + resourceIds: [], + fileIds: [], + })) + + expect(result.ok).toBe(true) + expect(fixture.calls.filter(call => call.startsWith('inject:'))).toEqual(['inject:default.txt']) + }) + + it('does not fall back to preset defaults when an explicit ID is missing', async () => { + const fixture = await createFixture({ presets: [presetWithDefaults('default-context')] }) + const uploaded = await fixture.registry.uploadFile({ + buffer: Buffer.from('default'), + filename: 'default.txt', + mimeType: 'text/plain', + }) + if ('error' in uploaded) throw new Error('Expected default registry file') + await fixture.registry.createResource({ slug: 'default-context', fileId: uploaded.fileId }) + + const result = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + resourceIds: ['missing-resource'], + fileIds: [], + })) + + expect(result.ok).toBe(true) + expect(fixture.calls.filter(call => call.startsWith('inject:'))).toEqual([]) + expect(fixture.warnings.map(warning => warning.message)).toContain( + 'Some input resourceIds did not match the local registry; ignoring', + ) + }) + + it('returns a controlled RPC error for malformed session-manager output', async () => { + const fixture = await createFixture() + fixture.returnMalformedSessionList() + + const result = await rpc(fixture.app, 'sessions.list', { instanceId: 'instance-1' }) + + expect(result.ok).toBe(false) + if (result.ok) throw new Error('Expected malformed collaborator output to fail') + expect(result.error.type).toBe('handler_error') + expect(result.error.message).toContain('sessions') + }) + + it('closes the SDK session and removes its worktree after injection failure', async () => { + const fixture = await createFixture() + const uploaded = await fixture.registry.uploadFile({ + buffer: Buffer.from('broken'), + filename: 'broken.txt', + mimeType: 'text/plain', + }) + if ('error' in uploaded) throw new Error('Expected uploaded registry file') + fixture.failNextInjection() + + const first = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + fileIds: [uploaded.fileId], + })) + expect(first.ok).toBe(false) + if (first.ok) throw new Error('Expected injection failure') + expect(first.error.message).toContain('deterministic injection failure') + expect(fixture.activeSessions.size).toBe(0) + expect(fixture.worktrees.size).toBe(0) + const closeIndex = fixture.calls.findIndex(call => call.startsWith('session:close:')) + const removeIndex = fixture.calls.findIndex(call => call.startsWith('worktree:remove:')) + expect(closeIndex).toBeGreaterThan(-1) + expect(removeIndex).toBeGreaterThan(closeIndex) + + const retry = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + fileIds: [uploaded.fileId], + })) + expect(retry.ok).toBe(true) + expect(fixture.activeSessions.size).toBe(1) + expect(fixture.worktrees.size).toBe(1) + }) + + it.each(cleanupFailureCases)('preserves the injection error when $stage cleanup fails', async ({ + stage, + expectedLog, + }) => { + const fixture = await createFixture() + const uploaded = await fixture.registry.uploadFile({ + buffer: Buffer.from('broken'), + filename: 'broken.txt', + mimeType: 'text/plain', + }) + if ('error' in uploaded) throw new Error('Expected uploaded registry file') + fixture.failNextInjection() + fixture.failCleanupAt(stage) + + const result = await rpc(fixture.app, 'instances.create', createInstanceInput({ + presetId: 'edit', + fileIds: [uploaded.fileId], + })) + + expect(result.ok).toBe(false) + if (result.ok) throw new Error('Expected injection failure') + expect(result.error.message).toContain('deterministic injection failure') + expect(fixture.errors).toContain(expectedLog) + }) +}) diff --git a/packages/standalone-server/tests/server.test.ts b/packages/standalone-server/tests/server.test.ts new file mode 100644 index 00000000..44f1370c --- /dev/null +++ b/packages/standalone-server/tests/server.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it } from 'bun:test' +import { ModelId } from '@roj-ai/sdk' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { z } from 'zod/v4' +import { + isLoopbackHost, + resolveStandaloneHost, + startStandaloneServer, + warnIfStandaloneExposed, +} from '../src/server.js' + +const DownloadUrlRpcSchema = z.object({ + ok: z.literal(true), + value: z.object({ url: z.string(), expiresAt: z.string() }), +}) + +describe('standalone network boundary', () => { + it('defaults to loopback and gives explicit config precedence over HOST', () => { + expect(resolveStandaloneHost(undefined, undefined)).toBe('127.0.0.1') + expect(resolveStandaloneHost(undefined, '192.0.2.10')).toBe('192.0.2.10') + expect(resolveStandaloneHost('localhost', '192.0.2.10')).toBe('localhost') + }) + + it('recognizes IPv4, IPv6, and mapped loopback hosts', () => { + expect(isLoopbackHost('localhost')).toBe(true) + expect(isLoopbackHost('127.3.2.1')).toBe(true) + expect(isLoopbackHost('[::1]')).toBe(true) + expect(isLoopbackHost('::ffff:127.0.0.1')).toBe(true) + expect(isLoopbackHost('0.0.0.0')).toBe(false) + expect(isLoopbackHost('192.0.2.10')).toBe(false) + }) + + it('warns clearly when an explicit host exposes the unauthenticated server', () => { + const warnings: Array<{ message: string; context?: Record }> = [] + const logger = { + warn(message: string, context?: Record) { + warnings.push({ message, context }) + }, + } + + warnIfStandaloneExposed('127.0.0.1', logger) + warnIfStandaloneExposed('0.0.0.0', logger) + + expect(warnings).toEqual([{ + message: 'Standalone server is listening on a non-loopback host without authentication', + context: { + host: '0.0.0.0', + action: 'Bind to 127.0.0.1 unless network access is intentional and protected externally', + }, + }]) + }) + + it('uses the OS-assigned port in generated public URLs', async () => { + const dataPath = await mkdtemp(join(tmpdir(), 'roj-standalone-port-')) + let handle: Awaited> | undefined + try { + handle = await startStandaloneServer({ + presets: [{ + id: 'edit', + name: 'Edit', + orchestrator: { + system: 'Test orchestrator', + model: ModelId('mock'), + tools: [], + agents: [], + }, + agents: [], + }], + config: { + port: 0, + host: '127.0.0.1', + dataPath, + persistence: 'memory', + logLevel: 'error', + logFormat: 'console', + llmMock: () => ({ + content: 'unused', + toolCalls: [], + finishReason: 'stop', + metrics: { + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + latencyMs: 0, + model: 'mock', + }, + }), + }, + }) + + const response = await fetch(`http://127.0.0.1:${handle.port}/api/v1/rpc`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + method: 'sessionFiles.createDownloadUrl', + input: { + instanceId: handle.instance.id, + sessionId: 'session-1', + scope: 'workspace', + path: 'artifact.zip', + }, + }), + }) + const result = DownloadUrlRpcSchema.parse(await response.json()) + const downloadUrl = new URL(result.value.url) + + expect(handle.port).toBeGreaterThan(0) + expect(downloadUrl.hostname).toBe('127.0.0.1') + expect(downloadUrl.port).toBe(String(handle.port)) + } finally { + await handle?.shutdown() + await rm(dataPath, { recursive: true, force: true }) + } + }, 15_000) +}) diff --git a/packages/transport/package.json b/packages/transport/package.json index e9ee6b71..8ecef4d0 100644 --- a/packages/transport/package.json +++ b/packages/transport/package.json @@ -32,7 +32,9 @@ }, "files": [ "src", - "dist" + "dist", + "!src/**/*.test.ts", + "!src/__tests__" ], "publishConfig": { "access": "public" diff --git a/packages/transport/src/core/connection.test.ts b/packages/transport/src/core/connection.test.ts new file mode 100644 index 00000000..a3d5d760 --- /dev/null +++ b/packages/transport/src/core/connection.test.ts @@ -0,0 +1,85 @@ +import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from 'bun:test' +import { Connection } from './connection.js' +import type { ProtocolDef } from './protocol.js' +import type { ICloseEvent, IWebSocket } from '../platform/types.js' +import { WebSocketReadyState } from '../platform/types.js' + +class RecordingWebSocket implements IWebSocket { + readyState: WebSocketReadyState = WebSocketReadyState.OPEN + onopen: (() => void) | null = null + onclose: ((event: ICloseEvent) => void) | null = null + onerror: ((error: Error) => void) | null = null + onmessage: ((data: string) => void) | null = null + readonly sent: string[] = [] + + send(data: string): void { + this.sent.push(data) + } + + close(): void { + this.readyState = WebSocketReadyState.CLOSED + } +} + +class TestConnection extends Connection { + async connect(): Promise {} + async disconnect(): Promise {} + + attach(ws: IWebSocket): void { + this.setupWebSocket(ws) + this.handleOpen() + this.flushSendBuffer() + } +} + +let warnSpy: Mock + +beforeEach(() => { + warnSpy = spyOn(console, 'warn').mockImplementation(() => {}) +}) + +afterEach(() => { + warnSpy.mockRestore() +}) + +describe('Connection send buffer', () => { + it('keeps the newest messages and reports one overflow episode', () => { + const connection = new TestConnection({}) + + for (let index = 0; index < 503; index++) { + connection.send(`message-${index}`) + } + + expect(connection.bufferedMessageCount).toBe(500) + expect(warnSpy).toHaveBeenCalledTimes(1) + expect(warnSpy).toHaveBeenCalledWith( + '[transport] send buffer full (500), dropping oldest notifications', + ) + + const ws = new RecordingWebSocket() + connection.attach(ws) + + expect(ws.sent).toHaveLength(500) + expect(ws.sent[0]).toBe('message-3') + expect(ws.sent.at(-1)).toBe('message-502') + expect(connection.bufferedMessageCount).toBe(0) + expect(warnSpy).toHaveBeenCalledTimes(2) + expect(warnSpy).toHaveBeenLastCalledWith( + '[transport] send buffer drained; 3 notification(s) were dropped', + ) + }) + + it('starts a new warning episode after the previous buffer drains', () => { + const connection = new TestConnection({}) + + for (let index = 0; index < 501; index++) connection.send(`first-${index}`) + connection.attach(new RecordingWebSocket()) + + const disconnected = new TestConnection({}) + for (let index = 0; index < 501; index++) disconnected.send(`second-${index}`) + + expect( + warnSpy.mock.calls.filter(([message]) => String(message).includes('send buffer full')), + ).toHaveLength(2) + }) +}) diff --git a/packages/transport/src/core/connection.ts b/packages/transport/src/core/connection.ts index 99e7ec65..959eccc2 100644 --- a/packages/transport/src/core/connection.ts +++ b/packages/transport/src/core/connection.ts @@ -92,12 +92,31 @@ export abstract class Connection= this.maxBufferSize) { + this.sendBuffer.shift() + if (this.droppedSinceLastFlush === 0) { + console.warn(`[transport] send buffer full (${this.maxBufferSize}), dropping oldest notifications`) + } + this.droppedSinceLastFlush++ + } + this.sendBuffer.push(data) + } send(data: string): boolean { if (!this.ws || this.ws.readyState !== WebSocketReadyState.OPEN) { - if (this.sendBuffer.length < this.maxBufferSize) { - this.sendBuffer.push(data) - } + this.buffer(data) return false } this.flushSendBuffer() @@ -105,9 +124,7 @@ export abstract class Connection 0) { + console.warn(`[transport] send buffer drained; ${this.droppedSinceLastFlush} notification(s) were dropped`) + this.droppedSinceLastFlush = 0 + } } clearSendBuffer(): void { diff --git a/scripts/npm-publish/artifact-validation.mjs b/scripts/npm-publish/artifact-validation.mjs new file mode 100644 index 00000000..a228446a --- /dev/null +++ b/scripts/npm-publish/artifact-validation.mjs @@ -0,0 +1,252 @@ +import { createHash } from 'node:crypto' +import { spawnSync } from 'node:child_process' +import { readFile, readdir, stat } from 'node:fs/promises' +import path from 'node:path' +import { compareSemver, parseSemver } from './release-policy.mjs' +import { dependencyFields, publishedDependencyFields } from './workspace-plan.mjs' + +export const fileHashes = async (filePath) => { + const contents = await readFile(filePath) + return { + sha256: createHash('sha256').update(contents).digest('hex'), + integrity: `sha512-${createHash('sha512').update(contents).digest('base64')}`, + } +} + +export const assertPreparedDependencies = (workspace) => { + for (const field of dependencyFields) { + for (const [name, value] of Object.entries(workspace.pkg[field] ?? {})) { + if (typeof value === 'string' && (value.startsWith('workspace:') || value.startsWith('catalog:'))) { + throw new Error(`${workspace.pkg.name} ${field}.${name} was not prepared for publishing: ${value}`) + } + } + } +} + +export const validatePackedManifest = (entry, manifest) => { + if (!manifest || typeof manifest !== 'object' || Array.isArray(manifest)) { + throw new Error(`${entry.name} packed package.json is not an object`) + } + if (manifest.name !== entry.name || manifest.version !== entry.version) { + throw new Error(`Packed manifest does not match ${entry.name}@${entry.version}`) + } + for (const field of publishedDependencyFields) { + const dependencies = manifest[field] + if (dependencies === undefined) continue + if (!dependencies || typeof dependencies !== 'object' || Array.isArray(dependencies)) { + throw new Error(`${entry.name} packed ${field} is not an object`) + } + for (const [name, value] of Object.entries(dependencies)) { + if (!name || typeof value !== 'string' || !value) { + throw new Error(`${entry.name} packed ${field} has an invalid dependency entry`) + } + } + } + return manifest +} + +export const readPackedManifest = (entry, runner = (args) => spawnSync('tar', args, { encoding: 'utf8' })) => { + const result = runner(['-xOf', entry.tarball, 'package/package.json']) + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error(`Could not read package.json from ${entry.tarball}: ${result.stderr.trim() || `tar exited ${result.status}`}`) + } + let manifest + try { + manifest = JSON.parse(result.stdout) + } catch { + throw new Error(`${entry.name} packed package.json is invalid JSON`) + } + return validatePackedManifest(entry, manifest) +} + +const satisfiesComparator = (candidate, operator, boundary) => { + const comparison = compareSemver(candidate, boundary) + switch (operator) { + case '': + case '=': + return comparison === 0 + case '>': + return comparison > 0 + case '>=': + return comparison >= 0 + case '<': + return comparison < 0 + case '<=': + return comparison <= 0 + default: + return false + } +} + +const parseSimpleRange = (range) => { + if (/^(?:\*|[xX])$/.test(range)) return (candidate) => !parseSemver(candidate).prerelease + const wildcard = range.match(/^(\d+)(?:\.(\d+|[xX*]))?(?:\.(\d+|[xX*]))?$/) + if (wildcard && (wildcard[2] === undefined || /[xX*]/.test(wildcard[2]) || wildcard[3] === undefined || /[xX*]/.test(wildcard[3]))) { + const major = wildcard[1] + if (wildcard[2] === undefined || /[xX*]/.test(wildcard[2])) { + return (candidate) => { + const parsed = parseSemver(candidate) + return parsed.major === major && !parsed.prerelease + } + } + const minor = wildcard[2] + return (candidate) => { + const parsed = parseSemver(candidate) + return parsed.major === major && parsed.minor === minor && !parsed.prerelease + } + } + + const prefixed = range.match(/^([~^])\s*(\d+\.\d+\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)$/) + if (prefixed) { + const lower = prefixed[2] + const parsedLower = parseSemver(lower) + const upper = prefixed[1] === '~' + ? `${parsedLower.major}.${BigInt(parsedLower.minor) + 1n}.0` + : parsedLower.major !== '0' + ? `${BigInt(parsedLower.major) + 1n}.0.0` + : parsedLower.minor !== '0' + ? `0.${BigInt(parsedLower.minor) + 1n}.0` + : `0.0.${BigInt(parsedLower.patch) + 1n}` + return (candidate) => compareSemver(candidate, lower) >= 0 + && compareSemver(candidate, upper) < 0 + && (!parseSemver(candidate).prerelease || compareSemver(candidate, lower) === 0) + } + + const exact = range.replace(/^=/, '') + if (parseSemver(exact)) return (candidate) => compareSemver(candidate, exact) === 0 + + const comparatorTokens = range.split(/\s+/) + if (comparatorTokens.length > 0) { + const comparators = comparatorTokens.map((token) => { + const match = token.match(/^(<=|>=|<|>)(\d+\.\d+\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)$/) + if (!match) return undefined + return { operator: match[1], boundary: match[2] } + }) + if (comparators.every(Boolean)) { + return (candidate) => comparators.every(({ operator, boundary }) => satisfiesComparator(candidate, operator, boundary)) + && (!parseSemver(candidate).prerelease || comparators.some(({ boundary }) => { + const parsedCandidate = parseSemver(candidate) + const parsedBoundary = parseSemver(boundary) + return parsedBoundary.prerelease && parsedCandidate.major === parsedBoundary.major + && parsedCandidate.minor === parsedBoundary.minor && parsedCandidate.patch === parsedBoundary.patch + })) + } + } + return undefined +} + +export const validateInternalDependencySpec = (source, field, name, spec, dependencyVersion) => { + if (/^(?:workspace|catalog|file|link|npm|https?|git(?:\+[^:]*)?):/i.test(spec) || /^(?:git@|github:|gitlab:|bitbucket:)/i.test(spec)) { + throw new Error(`${source} packed ${field}.${name} uses unsupported internal dependency spec ${spec}`) + } + const candidate = parseSemver(dependencyVersion) + if (!candidate) throw new Error(`${name} has invalid packed version ${dependencyVersion}`) + const alternatives = spec.split('||').map((part) => part.trim()) + if (alternatives.some((part) => !part)) { + throw new Error(`${source} packed ${field}.${name} uses unsupported internal dependency spec ${spec}`) + } + const predicates = alternatives.map(parseSimpleRange) + if (predicates.some((predicate) => predicate === undefined)) { + throw new Error(`${source} packed ${field}.${name} uses unsupported internal dependency spec ${spec}`) + } + if (!predicates.some((predicate) => predicate(dependencyVersion))) { + throw new Error(`${source} packed ${field}.${name} requires ${name}@${spec}, but the packed version is ${dependencyVersion}`) + } +} + +export const buildPackedDependencyGraph = (packed, manifestsByName) => { + const packedNames = new Set(packed.map(({ name }) => name)) + const graph = new Map() + for (const entry of packed) { + const manifest = validatePackedManifest(entry, manifestsByName.get(entry.name)) + const dependencies = [] + for (const field of publishedDependencyFields) { + for (const [name, spec] of Object.entries(manifest[field] ?? {})) { + if (packedNames.has(name)) dependencies.push({ field, name, spec }) + } + } + graph.set(entry.name, dependencies.sort((left, right) => left.name.localeCompare(right.name) + || left.field.localeCompare(right.field) || left.spec.localeCompare(right.spec))) + } + const visiting = new Set() + const visited = new Set() + const visit = (name, chain = []) => { + if (visiting.has(name)) throw new Error(`Packed dependency cycle: ${[...chain, name].join(' -> ')}`) + if (visited.has(name)) return + visiting.add(name) + for (const dependency of graph.get(name) ?? []) visit(dependency.name, [...chain, name]) + visiting.delete(name) + visited.add(name) + } + for (const name of [...graph.keys()].sort()) visit(name) + return graph +} + +export const collectInternalDependencyClosure = (targetName, graph) => { + const result = [] + const visited = new Set([targetName]) + const visit = (name) => { + if (visited.has(name)) return + for (const dependency of graph.get(name) ?? []) visit(dependency.name) + visited.add(name) + result.push(name) + } + for (const dependency of graph.get(targetName) ?? []) visit(dependency.name) + return result +} + +export const createIsolatedInstallPlan = (entry, packedByName, graph) => { + const overrides = {} + const closure = collectInternalDependencyClosure(entry.name, graph) + for (const source of [entry.name, ...closure]) { + for (const { field, name, spec } of graph.get(source) ?? []) { + const dependency = packedByName.get(name) + if (!dependency) throw new Error(`Missing validated tarball for ${name}`) + validateInternalDependencySpec(source, field, name, spec, dependency.version) + } + } + for (const name of closure) { + const dependency = packedByName.get(name) + if (!dependency) throw new Error(`Missing validated tarball for ${name}`) + overrides[name] = `file:${dependency.tarball}` + } + return { + dependencies: { [entry.name]: `file:${entry.tarball}` }, + overrides, + } +} + +export const pathExists = async (target) => { + try { + await stat(target) + return true + } catch (error) { + if (error?.code === 'ENOENT') return false + throw error + } +} + +export const collectTargets = (value, result = []) => { + if (typeof value === 'string') result.push(value) + else if (Array.isArray(value)) value.forEach((entry) => collectTargets(entry, result)) + else if (value && typeof value === 'object') Object.values(value).forEach((entry) => collectTargets(entry, result)) + return result +} + +/** Test sources can ship outside dist through source maps and declaration maps. */ +export const findTestArtifacts = async (dir, relative = '') => { + if (!(await pathExists(dir))) return [] + const found = [] + for (const entry of await readdir(dir, { withFileTypes: true })) { + const entryRelative = path.join(relative, entry.name) + if (entry.isDirectory()) { + if (entry.name === 'node_modules') continue + if (entry.name === '__tests__') found.push(entryRelative) + else found.push(...await findTestArtifacts(path.join(dir, entry.name), entryRelative)) + } else if (/\.(?:test|spec)\.(?:[cm]?[jt]sx?|d\.ts)(?:\.map)?$/.test(entry.name)) { + found.push(entryRelative) + } + } + return found +} diff --git a/scripts/npm-publish/artifact-validation.test.mjs b/scripts/npm-publish/artifact-validation.test.mjs new file mode 100644 index 00000000..128b7152 --- /dev/null +++ b/scripts/npm-publish/artifact-validation.test.mjs @@ -0,0 +1,281 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { spawnSync } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { + assertPreparedDependencies, + buildPackedDependencyGraph, + collectTargets, + createIsolatedInstallPlan, + fileHashes, + findTestArtifacts, + readPackedManifest, + validateInternalDependencySpec, + validatePackedManifest, +} from './artifact-validation.mjs' + +const tempDirs = [] + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +const createTempDir = async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'roj-artifact-validation-test-')) + tempDirs.push(dir) + return dir +} + +const run = (command, args, cwd) => { + const result = spawnSync(command, args, { cwd, encoding: 'utf8' }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`${command} failed: ${result.stderr}`) + return result.stdout.trim() +} + +const writeFixturePackage = async (root, { name, dependencies = {}, source }) => { + const dir = path.join(root, name.split('/').at(-1)) + await mkdir(dir, { recursive: true }) + await writeFile(path.join(dir, 'package.json'), `${JSON.stringify({ + name, + version: '1.0.0', + type: 'module', + main: './index.js', + dependencies, + }, null, '\t')}\n`) + await writeFile(path.join(dir, 'index.js'), source) + return dir +} + +describe('artifact validation', () => { + test('records both tamper and npm-compatible integrity hashes', async () => { + const dir = await createTempDir() + const file = path.join(dir, 'package.tgz') + await writeFile(file, 'artifact') + + expect(await fileHashes(file)).toEqual({ + sha256: 'c7c5c1d70c5dec4416ab6158afd0b223ef40c29b1dc1f97ed9428b94d4cadb1c', + integrity: 'sha512-FGl0QHAcOIX3yNX6pZ8za0ccqGMyA07/DT/dwC3JsYuDVuhA21SCPI/S8svQkGlpzxMs+Lucc9x2m0/9gXvSPQ==', + }) + }) + + test('derives a deterministic transitive install plan from packed manifests', () => { + const packed = [ + { name: '@roj-ai/client', version: '1.0.0', tarball: '/packs/client.tgz' }, + { name: '@roj-ai/shared', version: '1.0.0', tarball: '/packs/shared.tgz' }, + { name: '@roj-ai/sdk', version: '1.0.0', tarball: '/packs/sdk.tgz' }, + { name: '@roj-ai/transport', version: '1.0.0', tarball: '/packs/transport.tgz' }, + { name: '@roj-ai/undeclared', version: '1.0.0', tarball: '/packs/undeclared.tgz' }, + ] + const manifests = new Map([ + ['@roj-ai/client', { name: '@roj-ai/client', version: '1.0.0', dependencies: { '@roj-ai/shared': '^1.0.0' } }], + ['@roj-ai/shared', { name: '@roj-ai/shared', version: '1.0.0', dependencies: { '@roj-ai/sdk': '^1.0.0' } }], + ['@roj-ai/sdk', { name: '@roj-ai/sdk', version: '1.0.0', dependencies: { '@roj-ai/transport': '^1.0.0' } }], + ['@roj-ai/transport', { name: '@roj-ai/transport', version: '1.0.0' }], + ['@roj-ai/undeclared', { name: '@roj-ai/undeclared', version: '1.0.0' }], + ]) + const graph = buildPackedDependencyGraph(packed, manifests) + expect(graph.get('@roj-ai/client')).toEqual([ + { field: 'dependencies', name: '@roj-ai/shared', spec: '^1.0.0' }, + ]) + + expect(createIsolatedInstallPlan(packed[0], new Map(packed.map((entry) => [entry.name, entry])), graph)).toEqual({ + dependencies: { '@roj-ai/client': 'file:/packs/client.tgz' }, + overrides: { + '@roj-ai/transport': 'file:/packs/transport.tgz', + '@roj-ai/sdk': 'file:/packs/sdk.tgz', + '@roj-ai/shared': 'file:/packs/shared.tgz', + }, + }) + }) + + test('installs a deep packed graph without exposing an undeclared transitive dependency', async () => { + const root = await createTempDir() + const packageRoot = path.join(root, 'packages') + const packRoot = path.join(root, 'packs') + const consumer = path.join(root, 'consumer') + await Promise.all([mkdir(packageRoot), mkdir(packRoot), mkdir(consumer)]) + const fixturePackages = [ + { + name: '@fixture/transport', + source: `export const value = 'transport'\n`, + }, + { + name: '@fixture/sdk', + dependencies: { '@fixture/transport': '^1.0.0' }, + source: `import { value } from '@fixture/transport'; export const sdk = value + ':sdk'\n`, + }, + { + name: '@fixture/shared', + dependencies: { '@fixture/sdk': '^1.0.0' }, + source: `import { sdk } from '@fixture/sdk'; export const shared = sdk + ':shared'\n`, + }, + { + name: '@fixture/client', + dependencies: { '@fixture/shared': '^1.0.0' }, + source: `import { shared } from '@fixture/shared'; export const value = shared + ':client'; export const loadTransitive = () => import('@fixture/sdk')\n`, + }, + { + name: '@fixture/undeclared', + source: `export const leaked = true\n`, + }, + ] + const sourceDirs = new Map() + for (const fixturePackage of fixturePackages) { + sourceDirs.set(fixturePackage.name, await writeFixturePackage(packageRoot, fixturePackage)) + } + const packed = [] + for (const fixturePackage of fixturePackages) { + const output = JSON.parse(run('npm', [ + 'pack', + sourceDirs.get(fixturePackage.name), + '--json', + '--ignore-scripts', + '--pack-destination', + packRoot, + ], root)) + packed.push({ + name: fixturePackage.name, + version: '1.0.0', + tarball: path.join(packRoot, output[0].filename), + }) + } + + // Source drift after packing must not change the artifact dependency graph. + const clientSourceManifestPath = path.join(sourceDirs.get('@fixture/client'), 'package.json') + const clientSourceManifest = JSON.parse(await readFile(clientSourceManifestPath, 'utf8')) + clientSourceManifest.dependencies = { '@fixture/undeclared': '^1.0.0' } + await writeFile(clientSourceManifestPath, `${JSON.stringify(clientSourceManifest, null, '\t')}\n`) + + const manifests = new Map(packed.map((entry) => [entry.name, readPackedManifest(entry)])) + const graph = buildPackedDependencyGraph(packed, manifests) + const clientEntry = packed.find(({ name }) => name === '@fixture/client') + const plan = createIsolatedInstallPlan(clientEntry, new Map(packed.map((entry) => [entry.name, entry])), graph) + expect(plan.overrides).not.toHaveProperty('@fixture/undeclared') + await writeFile(path.join(consumer, 'package.json'), `${JSON.stringify({ + name: 'fixture-consumer', + private: true, + type: 'module', + ...plan, + }, null, '\t')}\n`) + run('npm', [ + 'install', + '--offline', + '--install-strategy=nested', + '--ignore-scripts', + '--no-audit', + '--no-fund', + '--package-lock=false', + ], consumer) + await writeFile(path.join(consumer, 'verify.mjs'), ` +import assert from 'node:assert/strict' +import { loadTransitive, value } from '@fixture/client' +assert.equal(value, 'transport:sdk:shared:client') +await assert.rejects(loadTransitive(), { code: 'ERR_MODULE_NOT_FOUND' }) +`) + run('node', ['verify.mjs'], consumer) + }, { timeout: 20_000 }) + + test('rejects packed internal dependency version mismatches', () => { + const packed = [ + { name: '@roj-ai/client', version: '1.0.0', tarball: '/packs/client.tgz' }, + { name: '@roj-ai/shared', version: '1.0.0', tarball: '/packs/shared.tgz' }, + ] + const graph = buildPackedDependencyGraph(packed, new Map([ + ['@roj-ai/client', { name: '@roj-ai/client', version: '1.0.0', dependencies: { '@roj-ai/shared': '^2.0.0' } }], + ['@roj-ai/shared', { name: '@roj-ai/shared', version: '1.0.0' }], + ])) + + expect(() => createIsolatedInstallPlan( + packed[0], + new Map(packed.map((entry) => [entry.name, entry])), + graph, + )).toThrow('requires @roj-ai/shared@^2.0.0, but the packed version is 1.0.0') + }) + + test('rejects unsupported packed internal dependency protocols', () => { + for (const spec of ['workspace:*', 'catalog:', 'file:../shared', 'link:../shared', 'npm:other@^1.0.0']) { + expect(() => validateInternalDependencySpec( + '@roj-ai/client', + 'dependencies', + '@roj-ai/shared', + spec, + '1.0.0', + )).toThrow('uses unsupported internal dependency spec') + } + }) + + test('accepts supported semver forms only when the packed version satisfies them', () => { + for (const spec of ['1.2.3', '^1.2.0', '~1.2.0', '>=1.0.0 <2.0.0', '1.2.x', '^1.2.3 || ^2.0.0']) { + expect(() => validateInternalDependencySpec( + '@roj-ai/client', + 'dependencies', + '@roj-ai/shared', + spec, + '1.2.3', + )).not.toThrow() + } + expect(() => validateInternalDependencySpec( + '@roj-ai/client', + 'dependencies', + '@roj-ai/shared', + 'latest', + '1.2.3', + )).toThrow('uses unsupported internal dependency spec') + }) + + test('validates identity and dependency fields from packed manifests', () => { + const entry = { name: '@roj-ai/client', version: '1.0.0' } + expect(() => validatePackedManifest(entry, { + name: '@roj-ai/other', + version: '1.0.0', + })).toThrow('does not match') + expect(() => validatePackedManifest(entry, { + name: '@roj-ai/client', + version: '1.0.0', + dependencies: [], + })).toThrow('dependencies is not an object') + }) + + test('rejects dependency cycles found only in packed manifests', () => { + const packed = [ + { name: '@roj-ai/a', version: '1.0.0' }, + { name: '@roj-ai/b', version: '1.0.0' }, + ] + expect(() => buildPackedDependencyGraph(packed, new Map([ + ['@roj-ai/a', { name: '@roj-ai/a', version: '1.0.0', dependencies: { '@roj-ai/b': '^1.0.0' } }], + ['@roj-ai/b', { name: '@roj-ai/b', version: '1.0.0', dependencies: { '@roj-ai/a': '^1.0.0' } }], + ]))).toThrow('Packed dependency cycle') + }) + + test('rejects unprepared workspace and catalog references', () => { + expect(() => assertPreparedDependencies({ + pkg: { name: '@roj-ai/a', dependencies: { '@roj-ai/b': 'workspace:*' } }, + })).toThrow('was not prepared for publishing') + expect(() => assertPreparedDependencies({ + pkg: { name: '@roj-ai/a', devDependencies: { typescript: 'catalog:build' } }, + })).toThrow('was not prepared for publishing') + }) + + test('finds test artifacts anywhere in a packed tree', async () => { + const dir = await createTempDir() + await mkdir(path.join(dir, 'src', '__tests__'), { recursive: true }) + await mkdir(path.join(dir, 'dist'), { recursive: true }) + await writeFile(path.join(dir, 'src', 'worker.test.ts'), '') + await writeFile(path.join(dir, 'dist', 'worker.spec.js.map'), '') + + expect((await findTestArtifacts(dir)).sort()).toEqual([ + path.join('dist', 'worker.spec.js.map'), + path.join('src', '__tests__'), + path.join('src', 'worker.test.ts'), + ].sort()) + }) + + test('collects nested package targets', () => { + expect(collectTargets({ '.': { import: './dist/index.js', types: './dist/index.d.ts' } })).toEqual([ + './dist/index.js', + './dist/index.d.ts', + ]) + }) +}) diff --git a/scripts/npm-publish/check-release-ancestry.mjs b/scripts/npm-publish/check-release-ancestry.mjs new file mode 100644 index 00000000..8d199044 --- /dev/null +++ b/scripts/npm-publish/check-release-ancestry.mjs @@ -0,0 +1,21 @@ +import { spawnSync } from 'node:child_process' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +export const assertReleaseAncestry = ({ commit, releaseRef = 'origin/main', cwd = process.cwd(), runner } = {}) => { + if (!commit) throw new Error('Release commit is required') + const run = runner ?? ((args) => spawnSync('git', args, { cwd, encoding: 'utf8' })) + const result = run(['merge-base', '--is-ancestor', commit, releaseRef]) + if (result.error) throw result.error + if (result.status === 0) return + if (result.status === 1) throw new Error(`Release commit ${commit} is not reachable from ${releaseRef}`) + throw new Error(`Could not verify release ancestry against ${releaseRef}: ${result.stderr.trim() || `git exited ${result.status}`}`) +} + +const isMain = process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) +if (isMain) { + const commit = process.argv[2] + const releaseRef = process.argv[3] ?? process.env.ROJ_RELEASE_REF ?? 'origin/main' + assertReleaseAncestry({ commit, releaseRef }) + console.log(`Verified release commit ${commit} is reachable from ${releaseRef}`) +} diff --git a/scripts/npm-publish/check-release-ancestry.test.mjs b/scripts/npm-publish/check-release-ancestry.test.mjs new file mode 100644 index 00000000..05f0af05 --- /dev/null +++ b/scripts/npm-publish/check-release-ancestry.test.mjs @@ -0,0 +1,63 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { execFileSync } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { assertReleaseAncestry } from './check-release-ancestry.mjs' + +const tempDirs = [] + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +const git = (cwd, ...args) => execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() + +const createRepository = async () => { + const cwd = await mkdtemp(path.join(tmpdir(), 'roj-release-ancestry-test-')) + tempDirs.push(cwd) + git(cwd, 'init', '--initial-branch=main') + git(cwd, 'config', 'user.name', 'Test User') + git(cwd, 'config', 'user.email', 'test@example.com') + await writeFile(path.join(cwd, 'release.txt'), 'main\n') + git(cwd, 'add', 'release.txt') + git(cwd, 'commit', '-m', 'main release') + const mainCommit = git(cwd, 'rev-parse', 'HEAD') + git(cwd, 'update-ref', 'refs/remotes/origin/main', mainCommit) + return { cwd, mainCommit } +} + +describe('release ancestry', () => { + test('accepts a commit reachable from origin/main', async () => { + const repository = await createRepository() + expect(() => assertReleaseAncestry({ + commit: repository.mainCommit, + releaseRef: 'origin/main', + cwd: repository.cwd, + })).not.toThrow() + }) + + test('rejects a feature-only commit', async () => { + const repository = await createRepository() + git(repository.cwd, 'switch', '-c', 'feature') + await writeFile(path.join(repository.cwd, 'feature.txt'), 'feature\n') + git(repository.cwd, 'add', 'feature.txt') + git(repository.cwd, 'commit', '-m', 'feature release') + const featureCommit = git(repository.cwd, 'rev-parse', 'HEAD') + + expect(() => assertReleaseAncestry({ + commit: featureCommit, + releaseRef: 'origin/main', + cwd: repository.cwd, + })).toThrow('is not reachable') + }) + + test('reports a missing release ref as a verification error', async () => { + const repository = await createRepository() + expect(() => assertReleaseAncestry({ + commit: repository.mainCommit, + releaseRef: 'origin/missing', + cwd: repository.cwd, + })).toThrow('Could not verify release ancestry') + }) +}) diff --git a/scripts/npm-publish/init.sh b/scripts/npm-publish/init.sh index d121f602..37184bdd 100755 --- a/scripts/npm-publish/init.sh +++ b/scripts/npm-publish/init.sh @@ -26,6 +26,10 @@ fi REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$REPO_ROOT" +RELEASE_COMMIT="${ROJ_RELEASE_COMMIT:-HEAD}" +RELEASE_REF="${ROJ_RELEASE_REF:-origin/main}" +node ./scripts/npm-publish/check-release-ancestry.mjs "$RELEASE_COMMIT" "$RELEASE_REF" + if ! command -v npm > /dev/null; then echo "npm not found in PATH" >&2 exit 1 diff --git a/scripts/npm-publish/pack-and-validate.mjs b/scripts/npm-publish/pack-and-validate.mjs new file mode 100644 index 00000000..1288e417 --- /dev/null +++ b/scripts/npm-publish/pack-and-validate.mjs @@ -0,0 +1,194 @@ +import { spawnSync } from 'node:child_process' +import { appendFile, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { + discoverWorkspacePackages, + publishedDependencyFields, + topologicallySortWorkspacePackages, +} from './workspace-plan.mjs' +import { + assertPreparedDependencies, + buildPackedDependencyGraph, + collectTargets, + createIsolatedInstallPlan, + fileHashes, + findTestArtifacts, + pathExists, + readPackedManifest, +} from './artifact-validation.mjs' + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)) +const repoRoot = path.resolve(scriptDir, '../..') +const workspaces = await discoverWorkspacePackages(repoRoot) +const publishOrder = topologicallySortWorkspacePackages(workspaces, { + fields: publishedDependencyFields, + publicOnly: true, +}) + +if (process.argv.includes('--check-order')) { + console.log(publishOrder.map(({ pkg }) => pkg.name).join('\n')) + process.exit(0) +} + +const packRoot = process.env.ROJ_PACK_DIR + ? path.resolve(process.env.ROJ_PACK_DIR) + : await mkdtemp(path.join(tmpdir(), 'roj-npm-pack-')) +const tarballDir = path.join(packRoot, 'tarballs') +await mkdir(tarballDir, { recursive: true }) + +const run = (command, args, options = {}) => { + const result = spawnSync(command, args, { stdio: 'inherit', ...options }) + if (result.error) throw result.error + if (result.status !== 0) { + throw new Error(`${command} ${args.join(' ')} failed with exit code ${result.status ?? 'unknown'}`) + } +} + +const packed = [] +for (const workspace of publishOrder) { + const version = workspace.pkg.version + if (!version || version === '0.0.0') throw new Error(`${workspace.pkg.name} has invalid publish version ${version ?? '(missing)'}`) + assertPreparedDependencies(workspace) + + const filename = `${workspace.dir}-${version}.tgz` + const tarball = path.join(tarballDir, filename) + run('bun', ['pm', 'pack', '--filename', tarball, '--quiet'], { cwd: workspace.absDir }) + await stat(tarball) + const hashes = await fileHashes(tarball) + packed.push({ + name: workspace.pkg.name, + dir: workspace.dir, + version, + tarball, + ...hashes, + }) +} + +const rootPackage = JSON.parse(await readFile(path.join(repoRoot, 'package.json'), 'utf8')) +const clientReact = workspaces.find(({ pkg }) => pkg.name === '@roj-ai/client-react')?.pkg +const packedByName = new Map(packed.map((entry) => [entry.name, entry])) +const packedManifests = new Map(packed.map((entry) => [entry.name, readPackedManifest(entry)])) +const packedDependencyGraph = buildPackedDependencyGraph(packed, packedManifests) +for (const entry of packed) { + const consumerDir = await mkdtemp(path.join(tmpdir(), `roj-npm-consumer-${entry.dir}-`)) + const installPlan = createIsolatedInstallPlan(entry, packedByName, packedDependencyGraph) + const consumerPackage = { + name: `roj-published-artifact-smoke-${entry.dir}`, + private: true, + type: 'module', + ...installPlan, + devDependencies: { + typescript: rootPackage.devDependencies.typescript, + '@types/bun': rootPackage.workspaces.catalog['@types/bun'], + '@types/react': clientReact?.devDependencies?.['@types/react'], + '@types/react-dom': clientReact?.devDependencies?.['@types/react-dom'], + }, + } + await writeFile(path.join(consumerDir, 'package.json'), `${JSON.stringify(consumerPackage, null, '\t')}\n`) + // Nested layout preserves package-local visibility for the declared graph. + run('npm', [ + 'install', + '--install-strategy=nested', + '--ignore-scripts', + '--no-audit', + '--no-fund', + '--package-lock=false', + ], { cwd: consumerDir }) + + const installedDir = path.join(consumerDir, 'node_modules', ...entry.name.split('/')) + const manifest = JSON.parse(await readFile(path.join(installedDir, 'package.json'), 'utf8')) + if (manifest.name !== entry.name || manifest.version !== entry.version) { + throw new Error(`Installed ${entry.name} does not match packed version ${entry.version}`) + } + + const targets = new Set([ + ...collectTargets(manifest.exports), + ...collectTargets(manifest.bin), + ...collectTargets(manifest.main), + ...collectTargets(manifest.types), + ]) + for (const target of targets) { + if (!target.startsWith('./')) throw new Error(`${entry.name} has non-relative package target: ${target}`) + if (target.includes('*')) throw new Error(`${entry.name} has an unvalidated wildcard package target: ${target}`) + if (!(await pathExists(path.resolve(installedDir, target)))) { + throw new Error(`${entry.name} package target does not exist: ${target}`) + } + } + + const testArtifacts = await findTestArtifacts(installedDir) + if (testArtifacts.length > 0) { + throw new Error(`${entry.name} ships tests:\n${testArtifacts.map((file) => ` ${file}`).join('\n')}`) + } + + for (const [binName, binTarget] of Object.entries(manifest.bin ?? {})) { + const binPath = path.resolve(installedDir, binTarget) + const firstLine = (await readFile(binPath, 'utf8')).split(/\r?\n/, 1)[0] + if (!firstLine.startsWith('#!')) throw new Error(`${entry.name} bin ${binName} has no shebang`) + } + + const importSpecifiers = [] + for (const [subpath, definition] of Object.entries(manifest.exports ?? {})) { + const importTarget = typeof definition === 'string' + ? definition + : definition?.import ?? definition?.default + if (typeof importTarget !== 'string' || !/\.[cm]?js$/.test(importTarget)) continue + importSpecifiers.push(subpath === '.' ? entry.name : `${entry.name}${subpath.slice(1)}`) + } + + if (entry.name === '@roj-ai/cli') { + const cliTarget = await readFile(path.join(installedDir, 'dist', 'main.js'), 'utf8') + if (!cliTarget.startsWith('#!/usr/bin/env bun\n')) throw new Error('@roj-ai/cli does not ship the Bun shebang') + run(path.join(consumerDir, 'node_modules', '.bin', 'roj-cli'), ['--help'], { cwd: consumerDir }) + } + if (entry.name === '@roj-ai/platform-cli') { + run(path.join(consumerDir, 'node_modules', '.bin', 'roj'), ['--help'], { cwd: consumerDir }) + } + + const uniqueImportSpecifiers = [...new Set(importSpecifiers)] + const esmSmokePath = path.join(consumerDir, 'esm-smoke.mjs') + await writeFile(esmSmokePath, ` +const specifiers = ${JSON.stringify(uniqueImportSpecifiers)} +for (const specifier of specifiers) await import(specifier) +`) + run('node', [esmSmokePath], { cwd: consumerDir }) + + const typeSmokePath = path.join(consumerDir, 'smoke.ts') + await writeFile(typeSmokePath, [ + ...uniqueImportSpecifiers.map((specifier) => `import '${specifier}'`), + ...(entry.name === '@roj-ai/sdk' + ? [`import packageManifest from '@roj-ai/sdk/package.json' with { type: 'json' }`, 'void packageManifest'] + : []), + '', + ].join('\n')) + await writeFile(path.join(consumerDir, 'tsconfig.json'), `${JSON.stringify({ + compilerOptions: { + allowSyntheticDefaultImports: true, + lib: ['ES2022', 'DOM'], + module: 'NodeNext', + moduleResolution: 'NodeNext', + noEmit: true, + resolveJsonModule: true, + skipLibCheck: false, + strict: true, + target: 'ES2022', + types: ['bun', 'react', 'react-dom'], + }, + include: ['smoke.ts'], + }, null, '\t')}\n`) + run(path.join(consumerDir, 'node_modules', '.bin', 'tsc'), ['--project', 'tsconfig.json'], { cwd: consumerDir }) + await rm(consumerDir, { recursive: true, force: true }) +} + +const manifestPath = path.join(packRoot, 'manifest.json') +await writeFile(manifestPath, `${JSON.stringify({ + schemaVersion: 2, + validatedAt: new Date().toISOString(), + packages: packed, +}, null, '\t')}\n`) + +if (process.env.GITHUB_ENV) { + await appendFile(process.env.GITHUB_ENV, `PACKAGE_TARBALL_MANIFEST=${manifestPath}\n`) +} +console.log(`Validated ${packed.length} package tarballs: ${manifestPath}`) diff --git a/scripts/npm-publish/prepare-packages.mjs b/scripts/npm-publish/prepare-packages.mjs index 59970236..54a330e7 100644 --- a/scripts/npm-publish/prepare-packages.mjs +++ b/scripts/npm-publish/prepare-packages.mjs @@ -1,6 +1,7 @@ -import { readdir, readFile, writeFile } from 'node:fs/promises' +import { readFile, writeFile } from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' +import { dependencyFields, discoverWorkspacePackages } from './workspace-plan.mjs' const versionArg = process.argv[2] @@ -16,8 +17,6 @@ if (!match) { const releaseVersion = match[1] const __dirname = path.dirname(fileURLToPath(import.meta.url)) const repoRoot = path.resolve(__dirname, '../..') -const packagesDir = path.resolve(repoRoot, 'packages') -const dependencyFields = ['dependencies', 'devDependencies', 'peerDependencies', 'optionalDependencies'] const rootPkg = JSON.parse(await readFile(path.resolve(repoRoot, 'package.json'), 'utf8')) const defaultCatalog = rootPkg.workspaces?.catalog ?? {} @@ -36,20 +35,7 @@ const resolveCatalogRef = (depName, value) => { return resolved } -const packageDirs = await readdir(packagesDir, { withFileTypes: true }) -const packages = [] - -for (const entry of packageDirs) { - if (!entry.isDirectory()) continue - const packageJsonPath = path.join(packagesDir, entry.name, 'package.json') - try { - const source = await readFile(packageJsonPath, 'utf8') - const pkg = JSON.parse(source) - packages.push({ dir: entry.name, path: packageJsonPath, pkg }) - } catch { - // Skip directories without package.json. - } -} +const packages = await discoverWorkspacePackages(repoRoot) const publicPackageNames = new Set( packages.filter(({ pkg }) => pkg.private !== true).map(({ pkg }) => pkg.name), diff --git a/scripts/npm-publish/publish-packed.mjs b/scripts/npm-publish/publish-packed.mjs new file mode 100644 index 00000000..a32e53b2 --- /dev/null +++ b/scripts/npm-publish/publish-packed.mjs @@ -0,0 +1,70 @@ +import { readFile, stat } from 'node:fs/promises' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { + discoverWorkspacePackages, + publishedDependencyFields, + topologicallySortWorkspacePackages, +} from './workspace-plan.mjs' +import { fileHashes } from './artifact-validation.mjs' +import { assertReleaseAncestry } from './check-release-ancestry.mjs' +import { decidePublish, runNpm } from './release-policy.mjs' + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)) +const repoRoot = path.resolve(scriptDir, '../..') +const manifestInput = process.argv.slice(2).find((arg) => !arg.startsWith('--')) ?? process.env.PACKAGE_TARBALL_MANIFEST +if (!manifestInput) throw new Error('PACKAGE_TARBALL_MANIFEST is required') +const manifestPath = path.resolve(manifestInput) + +const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) +if (manifest.schemaVersion !== 2 || !manifest.validatedAt || !Array.isArray(manifest.packages)) { + throw new Error(`Invalid validated tarball manifest: ${manifestPath}`) +} + +const workspaces = await discoverWorkspacePackages(repoRoot) +const publishOrder = topologicallySortWorkspacePackages(workspaces, { + fields: publishedDependencyFields, + publicOnly: true, +}) +const expectedNames = publishOrder.map(({ pkg }) => pkg.name) +const actualNames = manifest.packages.map(({ name }) => name) +if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { + throw new Error(`Tarball manifest package order mismatch\nExpected: ${expectedNames.join(', ')}\nActual: ${actualNames.join(', ')}`) +} +if (new Set(actualNames).size !== actualNames.length) throw new Error('Tarball manifest contains duplicate packages') + +for (let index = 0; index < publishOrder.length; index++) { + const workspace = publishOrder[index] + const packed = manifest.packages[index] + if (packed.dir !== workspace.dir || packed.version !== workspace.pkg.version) { + throw new Error(`Tarball manifest entry does not match ${workspace.pkg.name}`) + } + await stat(packed.tarball) + const actualHashes = await fileHashes(packed.tarball) + if (actualHashes.sha256 !== packed.sha256 || actualHashes.integrity !== packed.integrity) { + throw new Error(`Validated tarball changed after validation: ${packed.tarball}`) + } +} + +console.log(`Verified ${manifest.packages.length} validated tarballs before publishing`) +if (process.argv.includes('--check')) process.exit(0) + +assertReleaseAncestry({ + commit: process.env.ROJ_RELEASE_COMMIT ?? process.env.GITHUB_SHA ?? 'HEAD', + releaseRef: process.env.ROJ_RELEASE_REF ?? 'origin/main', +}) + +const npmTag = process.env.NPM_TAG ?? 'latest' +for (const entry of manifest.packages) { + const decision = decidePublish(entry, npmTag) + if (decision.action === 'skip') { + console.log(`\n→ Skipping ${entry.name}@${entry.version}: ${decision.reason}`) + continue + } + console.log(`\n→ Publishing ${entry.name} (tag: ${npmTag})`) + const result = runNpm(['publish', entry.tarball, '--tag', npmTag, '--access', 'public', '--provenance'], { + stdio: 'inherit', + }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`npm publish failed for ${entry.name}`) +} diff --git a/scripts/npm-publish/release-entrypoints.test.mjs b/scripts/npm-publish/release-entrypoints.test.mjs new file mode 100644 index 00000000..889ed431 --- /dev/null +++ b/scripts/npm-publish/release-entrypoints.test.mjs @@ -0,0 +1,60 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { spawnSync } from 'node:child_process' +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..') +const tempDirs = [] + +afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))) +}) + +const createTempDir = async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'roj-release-entrypoint-test-')) + tempDirs.push(dir) + return dir +} + +describe('release entrypoints', () => { + test('run.sh verifies ancestry before delegating to the publisher', async () => { + const tempDir = await createTempDir() + const binDir = path.join(tempDir, 'bin') + const logPath = path.join(tempDir, 'node-calls.log') + await mkdir(binDir) + const fakeNode = path.join(binDir, 'node') + await writeFile(fakeNode, '#!/bin/sh\nprintf \'%s\\n\' "$*" >> "$ROJ_NODE_CALL_LOG"\n') + await chmod(fakeNode, 0o755) + + const result = spawnSync('bash', ['./scripts/npm-publish/run.sh'], { + cwd: repoRoot, + encoding: 'utf8', + env: { + ...process.env, + GITHUB_SHA: 'release-commit', + PACKAGE_TARBALL_MANIFEST: '/validated/manifest.json', + PATH: `${binDir}:${process.env.PATH}`, + ROJ_NODE_CALL_LOG: logPath, + ROJ_RELEASE_REF: 'origin/release-main', + }, + }) + expect(result.status).toBe(0) + expect((await readFile(logPath, 'utf8')).trim().split('\n')).toEqual([ + './scripts/npm-publish/check-release-ancestry.mjs release-commit origin/release-main', + './scripts/npm-publish/publish-packed.mjs /validated/manifest.json', + ]) + }) + + test('direct publishing and bootstrap cannot bypass ancestry verification', async () => { + const [publisher, bootstrap] = await Promise.all([ + readFile(path.join(repoRoot, 'scripts/npm-publish/publish-packed.mjs'), 'utf8'), + readFile(path.join(repoRoot, 'scripts/npm-publish/init.sh'), 'utf8'), + ]) + expect(publisher.indexOf('assertReleaseAncestry({')).toBeGreaterThan(-1) + expect(publisher.indexOf('assertReleaseAncestry({')).toBeLessThan(publisher.indexOf('decidePublish(entry, npmTag)')) + expect(bootstrap.indexOf('check-release-ancestry.mjs')).toBeGreaterThan(-1) + expect(bootstrap.indexOf('check-release-ancestry.mjs')).toBeLessThan(bootstrap.indexOf('npm whoami')) + }) +}) diff --git a/scripts/npm-publish/release-policy.mjs b/scripts/npm-publish/release-policy.mjs new file mode 100644 index 00000000..bf0f5e2b --- /dev/null +++ b/scripts/npm-publish/release-policy.mjs @@ -0,0 +1,123 @@ +import { spawnSync } from 'node:child_process' + +const parseJsonOutput = (stdout, description) => { + const value = stdout.trim() + if (!value) return undefined + try { + return JSON.parse(value) + } catch { + throw new Error(`npm returned invalid JSON for ${description}`) + } +} + +const isNotFound = (result) => /(?:\bE404\b|404 Not Found)/i.test(`${result.stderr}\n${result.stdout}`) + +export const runNpm = (args, options = {}) => spawnSync(process.env.ROJ_NPM_COMMAND ?? 'npm', args, { + encoding: 'utf8', + ...options, +}) + +const queryNpm = (args, description, runner) => { + const result = runner(args) + if (result.error) throw result.error + if (result.status !== 0) { + if (isNotFound(result)) return { found: false } + throw new Error(`npm query failed for ${description}: ${result.stderr.trim() || `exit ${result.status}`}`) + } + return { found: true, value: parseJsonOutput(result.stdout, description) } +} + +export const getPublishedIntegrity = (name, version, runner = runNpm) => { + const result = queryNpm(['view', `${name}@${version}`, 'dist.integrity', '--json'], `${name}@${version}`, runner) + if (!result.found) return undefined + if (typeof result.value !== 'string' || !result.value.startsWith('sha512-')) { + throw new Error(`${name}@${version} exists without a usable sha512 integrity`) + } + return result.value +} + +export const getDistTagVersion = (name, tag, runner = runNpm) => { + const result = queryNpm(['view', name, 'dist-tags', '--json'], `${name} dist-tags`, runner) + if (!result.found || result.value === undefined || result.value === null) return undefined + if (typeof result.value !== 'object' || Array.isArray(result.value)) { + throw new Error(`${name} dist-tags metadata is not an object`) + } + const version = result.value[tag] + if (version === undefined) return undefined + if (typeof version !== 'string') throw new Error(`${name} dist-tag ${tag} is not a version string`) + return version +} + +export const parseSemver = (value) => { + const match = value.match(/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*)(?:\.(?:0|[1-9]\d*|\d*[A-Za-z-][0-9A-Za-z-]*))*))?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?$/) + if (!match) return undefined + return { + major: match[1], + minor: match[2], + patch: match[3], + prerelease: match[4]?.split('.'), + } +} + +const compareNumericIdentifier = (left, right) => { + if (left.length !== right.length) return left.length < right.length ? -1 : 1 + if (left === right) return 0 + return left < right ? -1 : 1 +} + +const compareIdentifier = (left, right) => { + const leftNumeric = /^\d+$/.test(left) + const rightNumeric = /^\d+$/.test(right) + if (leftNumeric && rightNumeric) return compareNumericIdentifier(left, right) + if (leftNumeric) return -1 + if (rightNumeric) return 1 + if (left === right) return 0 + return left < right ? -1 : 1 +} + +export const compareSemver = (leftValue, rightValue) => { + const left = parseSemver(leftValue) + const right = parseSemver(rightValue) + if (!left || !right) throw new Error(`Cannot compare invalid semver: ${leftValue}, ${rightValue}`) + for (const key of ['major', 'minor', 'patch']) { + const comparison = compareNumericIdentifier(left[key], right[key]) + if (comparison !== 0) return comparison + } + if (!left.prerelease && !right.prerelease) return 0 + if (!left.prerelease) return 1 + if (!right.prerelease) return -1 + for (let index = 0; index < Math.max(left.prerelease.length, right.prerelease.length); index++) { + if (left.prerelease[index] === undefined) return -1 + if (right.prerelease[index] === undefined) return 1 + const comparison = compareIdentifier(left.prerelease[index], right.prerelease[index]) + if (comparison !== 0) return comparison + } + return 0 +} + +export const assertDistTagCanAdvance = ({ candidate, current, tag }) => { + const candidateVersion = parseSemver(candidate) + if (!candidateVersion) throw new Error(`Invalid candidate version: ${candidate}`) + // latest is stable-only; other tags accept semver but still cannot regress. + if (tag === 'latest' && candidateVersion.prerelease) { + throw new Error(`Refusing to assign stable dist-tag latest to prerelease ${candidate}`) + } + if (current === undefined) return + if (!parseSemver(current)) throw new Error(`Current ${tag} dist-tag is not valid semver: ${current}`) + if (compareSemver(candidate, current) < 0) { + throw new Error(`Refusing to move ${tag} dist-tag backward from ${current} to ${candidate}`) + } +} + +export const decidePublish = (entry, tag, runner = runNpm) => { + const publishedIntegrity = getPublishedIntegrity(entry.name, entry.version, runner) + if (publishedIntegrity !== undefined) { + if (publishedIntegrity !== entry.integrity) { + throw new Error(`${entry.name}@${entry.version} already exists with different artifact integrity`) + } + return { action: 'skip', reason: 'identical artifact already published' } + } + const current = getDistTagVersion(entry.name, tag, runner) + assertDistTagCanAdvance({ candidate: entry.version, current, tag }) + return { action: 'publish', current } +} diff --git a/scripts/npm-publish/release-policy.test.mjs b/scripts/npm-publish/release-policy.test.mjs new file mode 100644 index 00000000..cf1d05ad --- /dev/null +++ b/scripts/npm-publish/release-policy.test.mjs @@ -0,0 +1,118 @@ +import { describe, expect, test } from 'bun:test' +import { + assertDistTagCanAdvance, + compareSemver, + decidePublish, + getPublishedIntegrity, +} from './release-policy.mjs' + +const response = (status, value, stderr = '') => ({ + status, + stdout: value === undefined ? '' : `${JSON.stringify(value)}\n`, + stderr, +}) + +const registry = (responses) => { + const calls = [] + return { + calls, + run: (args) => { + calls.push(args) + const next = responses.shift() + if (!next) throw new Error(`Unexpected npm call: ${args.join(' ')}`) + return next + }, + } +} + +describe('release policy', () => { + test('resumes by skipping an identical published artifact', () => { + const mock = registry([response(0, 'sha512-identical')]) + expect(decidePublish({ + name: '@roj-ai/sdk', + version: '1.2.3', + integrity: 'sha512-identical', + }, 'latest', mock.run)).toEqual({ + action: 'skip', + reason: 'identical artifact already published', + }) + expect(mock.calls).toHaveLength(1) + }) + + test('rejects resume when registry integrity differs', () => { + const mock = registry([response(0, 'sha512-other')]) + expect(() => decidePublish({ + name: '@roj-ai/sdk', + version: '1.2.3', + integrity: 'sha512-local', + }, 'latest', mock.run)).toThrow('different artifact integrity') + }) + + test('publishes an absent version without moving a tag backward', () => { + const mock = registry([ + response(1, undefined, 'npm error code E404'), + response(0, { latest: '1.2.2' }), + ]) + expect(decidePublish({ + name: '@roj-ai/sdk', + version: '1.2.3', + integrity: 'sha512-local', + }, 'latest', mock.run)).toEqual({ action: 'publish', current: '1.2.2' }) + }) + + test('allows an absent dist-tag', () => { + const mock = registry([ + response(1, undefined, 'npm error code E404'), + response(0, { latest: '1.2.2' }), + ]) + expect(decidePublish({ + name: '@roj-ai/sdk', + version: '1.2.3', + integrity: 'sha512-local', + }, 'next', mock.run)).toEqual({ action: 'publish', current: undefined }) + }) + + test('refuses stable and prerelease tag regressions', () => { + expect(() => assertDistTagCanAdvance({ candidate: '1.2.2', current: '1.2.3', tag: 'latest' })) + .toThrow('backward') + expect(() => assertDistTagCanAdvance({ candidate: '2.0.0-next.1', current: undefined, tag: 'latest' })) + .toThrow('stable dist-tag latest') + expect(() => assertDistTagCanAdvance({ candidate: '2.0.0-next.1', current: '2.0.0-next.2', tag: 'next' })) + .toThrow('backward') + }) + + test('orders semver prereleases before stable releases', () => { + expect(compareSemver('2.0.0-next.2', '2.0.0-next.10')).toBeLessThan(0) + expect(compareSemver('2.0.0-next.10', '2.0.0')).toBeLessThan(0) + }) + + test('implements SemVer precedence without locale or number precision errors', () => { + const precedence = [ + '1.0.0-alpha', + '1.0.0-alpha.1', + '1.0.0-alpha.beta', + '1.0.0-beta', + '1.0.0-beta.2', + '1.0.0-beta.11', + '1.0.0-rc.1', + '1.0.0', + ] + for (let index = 1; index < precedence.length; index++) { + expect(compareSemver(precedence[index - 1], precedence[index])).toBeLessThan(0) + } + expect(compareSemver('1.0.0-A', '1.0.0-a')).toBeLessThan(0) + expect(compareSemver('1.0.0-9007199254740992', '1.0.0-9007199254740993')).toBeLessThan(0) + expect(compareSemver('9007199254740992.0.0', '9007199254740993.0.0')).toBeLessThan(0) + }) + + test('rejects leading zeroes and malformed build metadata', () => { + expect(() => compareSemver('01.0.0', '1.0.0')).toThrow('invalid semver') + expect(() => compareSemver('1.0.0-alpha.01', '1.0.0-alpha.1')).toThrow('invalid semver') + expect(() => compareSemver('1.0.0+build..1', '1.0.0')).toThrow('invalid semver') + }) + + test('does not treat registry failures as an absent version', () => { + const mock = registry([response(1, undefined, 'npm error code E500')]) + expect(() => getPublishedIntegrity('@roj-ai/sdk', '1.2.3', mock.run)).toThrow('npm query failed') + }) +}) diff --git a/scripts/npm-publish/run.sh b/scripts/npm-publish/run.sh index 8b48468f..c035eb90 100755 --- a/scripts/npm-publish/run.sh +++ b/scripts/npm-publish/run.sh @@ -1,28 +1,15 @@ #!/bin/bash # -# Publish all public @roj-ai/* packages to npm. Run from CI after -# prepare-packages.mjs has rewritten versions and workspace:* refs. +# Publish the exact public @roj-ai/* tarballs previously packed and validated by +# pack-and-validate.mjs. Run from CI after prepare-packages.mjs. # # Requires NPM_TAG env (defaults to "latest"). Uses --provenance, so the # workflow must have `id-token: write` and a configured npm trusted publisher. # set -euo pipefail -NPM_TAG="${NPM_TAG:-latest}" - -for dir in packages/*; do - [ -f "$dir/package.json" ] || continue - if grep -q '"private": true' "$dir/package.json"; then - continue - fi - pkg_name="$(node -p "require('./$dir/package.json').name")" - echo "" - echo "→ Publishing $pkg_name (tag: $NPM_TAG)" - tarball="$(cd "$dir" && bun pm pack 2>&1 | grep -Eo '[^[:space:]]+\.tgz' | tail -n1)" - if [ -z "$tarball" ]; then - echo "Failed to pack $pkg_name" >&2 - exit 1 - fi - (cd "$dir" && npm publish "$tarball" --tag "$NPM_TAG" --access public --provenance) - rm -f "$dir/$tarball" -done +: "${PACKAGE_TARBALL_MANIFEST:?pack-and-validate.mjs did not provide PACKAGE_TARBALL_MANIFEST}" +RELEASE_COMMIT="${ROJ_RELEASE_COMMIT:-${GITHUB_SHA:-HEAD}}" +RELEASE_REF="${ROJ_RELEASE_REF:-origin/main}" +node ./scripts/npm-publish/check-release-ancestry.mjs "$RELEASE_COMMIT" "$RELEASE_REF" +exec node ./scripts/npm-publish/publish-packed.mjs "$PACKAGE_TARBALL_MANIFEST" diff --git a/scripts/npm-publish/workspace-plan.mjs b/scripts/npm-publish/workspace-plan.mjs new file mode 100644 index 00000000..796ed807 --- /dev/null +++ b/scripts/npm-publish/workspace-plan.mjs @@ -0,0 +1,97 @@ +import { readdir, readFile } from 'node:fs/promises' +import path from 'node:path' + +export const dependencyFields = [ + 'dependencies', + 'devDependencies', + 'peerDependencies', + 'optionalDependencies', +] + +export const publishedDependencyFields = [ + 'dependencies', + 'peerDependencies', + 'optionalDependencies', +] + +export async function discoverWorkspacePackages(repoRoot) { + const packagesDir = path.resolve(repoRoot, 'packages') + const entries = await readdir(packagesDir, { withFileTypes: true }) + const packages = [] + + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (!entry.isDirectory()) continue + const packageJsonPath = path.join(packagesDir, entry.name, 'package.json') + let source + try { + source = await readFile(packageJsonPath, 'utf8') + } catch (error) { + if (error?.code === 'ENOENT') continue + throw error + } + const pkg = JSON.parse(source) + if (!pkg.name) throw new Error(`${packageJsonPath} has no package name`) + packages.push({ + dir: entry.name, + absDir: path.join(packagesDir, entry.name), + path: packageJsonPath, + pkg, + }) + } + + const names = new Set() + for (const entry of packages) { + if (names.has(entry.pkg.name)) throw new Error(`Duplicate workspace package name: ${entry.pkg.name}`) + names.add(entry.pkg.name) + } + + return packages +} + +export function topologicallySortWorkspacePackages(packages, options = {}) { + const fields = options.fields ?? dependencyFields + const selected = options.publicOnly ? packages.filter(({ pkg }) => pkg.private !== true) : [...packages] + const selectedByName = new Map(selected.map((entry) => [entry.pkg.name, entry])) + const allByName = new Map(packages.map((entry) => [entry.pkg.name, entry])) + const dependencies = new Map() + + for (const entry of selected) { + const internal = new Set() + for (const field of fields) { + for (const dependencyName of Object.keys(entry.pkg[field] ?? {})) { + if (!allByName.has(dependencyName)) continue + if (!selectedByName.has(dependencyName)) { + throw new Error(`${entry.pkg.name} ${field} references excluded workspace package ${dependencyName}`) + } + internal.add(dependencyName) + } + } + dependencies.set(entry.pkg.name, [...internal].sort()) + } + + const result = [] + const visiting = new Set() + const visited = new Set() + + const visit = (name, chain = []) => { + if (visited.has(name)) return + if (visiting.has(name)) throw new Error(`Workspace dependency cycle: ${[...chain, name].join(' -> ')}`) + visiting.add(name) + for (const dependencyName of dependencies.get(name) ?? []) { + visit(dependencyName, [...chain, name]) + } + visiting.delete(name) + visited.add(name) + result.push(selectedByName.get(name)) + } + + for (const entry of selected.sort((a, b) => a.dir.localeCompare(b.dir))) { + visit(entry.pkg.name) + } + + if (result.length !== selected.length || new Set(result.map(({ pkg }) => pkg.name)).size !== selected.length) { + throw new Error('Workspace ordering did not include every selected package exactly once') + } + + return result +} diff --git a/scripts/ts-build.mjs b/scripts/ts-build.mjs index 590f68c8..25081e99 100644 --- a/scripts/ts-build.mjs +++ b/scripts/ts-build.mjs @@ -1,33 +1,22 @@ #!/usr/bin/env bun import { spawnSync } from 'node:child_process' import { existsSync } from 'node:fs' -import { join } from 'node:path' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { discoverWorkspacePackages, topologicallySortWorkspacePackages } from './npm-publish/workspace-plan.mjs' -// Topological build order. Each package is built (tsc) and its dist -// has alias paths resolved (tsc-alias) before downstream packages compile, -// so consumers see fully-resolved relative paths in upstream .d.ts files. -const ORDER = [ - 'transport', - 'sdk', - 'shared', - 'client', - 'sandbox-runtime', - 'platform-cli', - 'cli', - 'debug', - 'client-react', - 'standalone-server', - 'demo', -] +const repoRoot = dirname(dirname(fileURLToPath(import.meta.url))) +const workspaces = await discoverWorkspacePackages(repoRoot) +const buildOrder = topologicallySortWorkspacePackages(workspaces) const run = (cmd, args) => { const r = spawnSync(cmd, args, { stdio: 'inherit' }) if (r.status !== 0) process.exit(r.status ?? 1) } -for (const pkg of ORDER) { - const tsconfig = join('packages', pkg, 'tsconfig.json') +for (const workspace of buildOrder) { + const tsconfig = join('packages', workspace.dir, 'tsconfig.json') if (!existsSync(tsconfig)) continue - run('bunx', ['tsc', '--build', `packages/${pkg}`]) - run('bunx', ['tsc-alias', '-p', tsconfig]) + run('bunx', ['tsc', '--build', `packages/${workspace.dir}`]) + run('bunx', ['tsc-alias', '-p', tsconfig, '--resolve-full-paths', '--resolve-full-extension', '.js']) }