diff --git a/cli/__tests__/adapters.dsh.confined.test.mjs b/cli/__tests__/adapters.dsh.confined.test.mjs new file mode 100644 index 0000000000..38beb4c36d --- /dev/null +++ b/cli/__tests__/adapters.dsh.confined.test.mjs @@ -0,0 +1,280 @@ +/** + * The confined dsh path, end to end with real processes and no third-party + * code: a stub dsh (a node script laid out as an npm package) runs a turn + * against the fake provider through the pinned proxy, reaches the instance + * through the relay, and is refused by the egress proxy. Seatbelt itself is + * macOS-only, so the wrapper is injected here and its options are checked by + * building the real profile from them; the host probe covers enforcement. + */ +import { jest } from '@jest/globals'; +import { spawn as spawnReal } from 'child_process'; +import { createServer } from 'http'; +import { existsSync, readFileSync, realpathSync } from 'fs'; +import { chmod, mkdir, mkdtemp, rm, symlink, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import dsh, { isConfinedDsh, TESTED_DSH_VERSIONS } from '../src/lib/adapters/dsh.js'; +import { buildSeatbeltProfile } from '../src/lib/sandbox/seatbelt.js'; +import { OWN_PROCESS_GROUP } from '@commonlyai/seats/process-group'; +import { superviseRun } from '@commonlyai/seats/supervisor'; +import { createTraceStore } from '@commonlyai/seats/trace/store'; +import { startFakeProvider } from '../../packages/seats/test-support/fake-provider.js'; +import { isRunning, readPid, reap, waitFor } from '../../packages/seats/test-support/processes.js'; + +const PIN = 'deepseek-flash-1017'; +const CONFINED = { mode: 'workspace', network: { policy: 'restricted' } }; + +// Runs in the workspace (dsh's child env is closed, so it reports there). +// Reads its patch, makes one completion through the proxy it was given, then +// probes the relay and the egress proxy, and reports what it saw. +const STUB = String.raw`#!/usr/bin/env node +const fs = require('fs'); +const http = require('http'); +const args = process.argv.slice(2); +if (args[0] === '--version') { process.stdout.write('dsh ${TESTED_DSH_VERSIONS[0]}\n'); process.exit(0); } +const emit = (event) => process.stdout.write(JSON.stringify(event) + '\n'); +const rows = JSON.parse(fs.readFileSync(args[args.indexOf('--patch') + 1], 'utf8')); +const prompt = args[args.length - 1]; +if (prompt.includes('[[hang]]')) { + const { spawn } = require('child_process'); + const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' }); + fs.writeFileSync('.stub-grandchild.pid', String(child.pid)); + setInterval(() => {}, 1000); + return; +} +const provider = rows.find((row) => row.id === 'llm-pi-ai').config.providers.commonly; +const pinned = rows.find((row) => row.id === 'agent-default-model').config.model; +const mcp = rows.at(-1).insert[0].config; +const asked = /\[\[model:([^\]]+)\]\]/.exec(prompt)?.[1] || pinned; +const connect = (target) => new Promise((resolve) => { + const proxy = new URL(process.env.HTTPS_PROXY); + const request = http.request({ host: proxy.hostname, port: proxy.port, method: 'CONNECT', path: target }); + request.on('connect', (response, socket) => { socket.destroy(); resolve(response.statusCode); }); + request.on('error', (error) => resolve(error.message)); + request.end(); +}); +(async () => { + const response = await fetch(provider.baseURL.replace(/\/$/, '') + '/chat/completions', { + method: 'POST', + headers: { authorization: 'Bearer ' + process.env[provider.apiKeyEnv], 'content-type': 'application/json' }, + body: JSON.stringify({ model: asked, messages: [{ role: 'user', content: prompt }] }), + }); + const body = await response.text(); + const relay = await fetch(mcp.env.COMMONLY_API_URL + '/api/health').then((r) => r.text(), (e) => e.message); + const egress = process.env.HTTPS_PROXY ? await connect('exfil.example:443') : null; + fs.writeFileSync('.stub-report.json', JSON.stringify({ + status: response.status, relay, egress, mcp, + sandboxPolicy: rows.find((row) => row.id === 'sandbox-policy').config, + env: { HTTPS_PROXY: process.env.HTTPS_PROXY, NO_PROXY: process.env.NO_PROXY, TMPDIR: process.env.TMPDIR }, + })); + emit({ type: 'session', sessionId: 'session-11111111-1111-1111-1111-111111111111' }); + if (!response.ok) { emit({ type: 'error', message: 'provider said ' + response.status }); process.exit(1); } + emit({ type: 'final', text: JSON.parse(body).choices[0].message.content }); +})(); +`; + +const withSeat = async (fn, { provider: providerOptions = {} } = {}) => { + const root = await mkdtemp(join(tmpdir(), 'dsh-confined-')); + const cwd = join(root, 'workspace'); + const home = join(root, 'dsh-home'); + const pkg = join(root, 'prefix', 'node_modules', '@deepseek-ai', 'dsh'); + const mcpDir = join(root, 'mcp-home', '0.3.20'); + const mcpPkg = join(mcpDir, 'node_modules', '@commonlyai', 'mcp'); + await Promise.all([cwd, join(pkg, 'lib'), join(root, 'prefix', 'bin'), join(mcpPkg, 'src')] + .map((dir) => mkdir(dir, { recursive: true }))); + await writeFile(join(pkg, 'package.json'), '{"name":"@deepseek-ai/dsh","version":"0.2.1-rc.2"}'); + await writeFile(join(pkg, 'lib', 'bin.js'), STUB); + await chmod(join(pkg, 'lib', 'bin.js'), 0o755); + await symlink(join(pkg, 'lib', 'bin.js'), join(root, 'prefix', 'bin', 'dsh')); + await writeFile(join(mcpPkg, 'package.json'), '{"name":"@commonlyai/mcp","version":"0.3.20"}'); + await writeFile(join(mcpPkg, 'src', 'index.js'), ''); + const keyFile = join(root, 'provider-key'); + await writeFile(keyFile, 'upstream-secret-123', { mode: 0o600 }); + + const instanceSeen = []; + const instance = createServer((request, response) => { instanceSeen.push(request.url); response.end('instance-ok'); }); + await new Promise((resolve) => instance.listen(0, '127.0.0.1', resolve)); + const provider = await startFakeProvider({ model: PIN, ...providerOptions }); + const wraps = []; + const report = join(cwd, '.stub-report.json'); + const ctx = { + cwd, _dshHome: home, _dshBinary: join(root, 'prefix', 'bin', 'dsh'), agentName: 'dsh-confined', + runtimeToken: 'cm_agent_secret-123', instanceUrl: `http://127.0.0.1:${instance.address().port}`, + env: { PATH: process.env.PATH }, + environment: { + model: PIN, provider: { id: 'deepseek', baseURL: provider.baseURL, keyFile }, sandbox: CONFINED, + }, + _platform: 'darwin', + _detectSeatbelt: () => ({ available: true, path: '/usr/bin/sandbox-exec' }), + // The stub runs unwrapped: the options are what is under test here. + _wrapArgvWithSeatbelt: (inner, options) => { wraps.push({ inner, options }); return inner; }, + _prepareMcpSpawn: (command, options) => { + wraps.push({ prepared: command, prepareOptions: options }); + return ['node', join(mcpPkg, 'src', 'index.js')]; + }, + _selectedMcpVersionDir: () => mcpDir, + }; + try { + await fn({ + ctx, root, wraps, provider, instanceSeen, pkg, mcpDir, home, + report: () => JSON.parse(readFileSync(report, 'utf8')), + }); + } finally { + await provider.close(); + instance.closeAllConnections(); + await new Promise((resolve) => instance.close(resolve)); + await rm(root, { recursive: true, force: true }); + } +}; + +let warn; +beforeEach(() => { warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); }); +afterEach(() => { warn.mockRestore(); }); + +test('a confined turn runs through the pinned proxy, the relay and the egress fence', async () => { + await withSeat(async ({ ctx, wraps, provider, instanceSeen, pkg, mcpDir, home, report }) => { + const result = await dsh.spawn('[[reply:confined-ok]]', ctx); + expect(result).toEqual({ text: 'confined-ok', newSessionId: 'session-11111111-1111-1111-1111-111111111111' }); + + const seen = report(); + expect(seen.status).toBe(200); + expect(provider.requests.map((request) => request.model)).toEqual([PIN]); + // The relay reached the instance; the MCP was handed the relay, not the instance. + expect(seen.relay).toBe('instance-ok'); + expect(instanceSeen).toEqual(['/api/health']); + expect(seen.mcp.env.COMMONLY_API_URL).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(seen.mcp.env.COMMONLY_API_URL).not.toBe(ctx.instanceUrl); + expect(seen.mcp.command).toBe(process.execPath); + // The egress proxy is live and refuses what is not on its list. + expect(seen.egress).toBe(403); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('exfil.example:443 (not allowlisted)')); + expect(seen.env.HTTPS_PROXY).toMatch(/^http:\/\/127\.0\.0\.1:\d+$/); + expect(seen.env.NO_PROXY).toBe('127.0.0.1,localhost'); + expect(seen.env.TMPDIR).toBe(join(home, 'tmp')); + expect(seen.sandboxPolicy).toEqual({ mode: 'danger-full-access', workspaceRoot: ctx.cwd }); + + const prepared = wraps.find((entry) => entry.prepared); + expect(prepared.prepareOptions).toEqual({ apiUrl: ctx.instanceUrl, publicSeat: true }); + const { inner, options } = wraps.find((entry) => entry.inner); + expect(inner.slice(0, 2)).toEqual([process.execPath, realpathSync(join(pkg, 'lib', 'bin.js'))]); + expect(options).toMatchObject({ + workspacePath: ctx.cwd, workspaceAccess: 'write', executablePath: process.execPath, + statePath: home, network: 'loopback', + readOnlyDirectories: [realpathSync(join(pkg, '..', '..')), mcpDir], + }); + const ports = options.loopbackNetworkPorts; + expect(ports).toHaveLength(3); + expect(new Set(ports).size).toBe(3); + expect(ports).toContain(Number(new URL(seen.env.HTTPS_PROXY).port)); + expect(ports).toContain(Number(new URL(seen.mcp.env.COMMONLY_API_URL).port)); + + // The profile those options build: loopback only, those three ports. + const profile = buildSeatbeltProfile(options); + expect(profile).not.toMatch(/^\(allow network-outbound\)$/m); + expect(profile).not.toContain('com.apple.SecurityServer'); + expect(profile.split('\n').filter((line) => line.includes('network-outbound')).sort()) + .toEqual(ports.map((port) => `(allow network-outbound (remote tcp "localhost:${port}"))`).sort()); + }); +}); + +test('a response naming another model fails the turn as a broken pin', async () => { + await withSeat(async ({ ctx, report }) => { + await expect(dsh.spawn('hello', ctx)).rejects.toThrow(/served model check failed: served model "deepseek-pro-other" is not the pinned "deepseek-flash-1017"/); + expect(report().status).toBe(502); + }, { provider: { servedModel: 'deepseek-pro-other' } }); +}); + +test('a request for another model is refused before the provider and fails the turn', async () => { + await withSeat(async ({ ctx, provider, report }) => { + await expect(dsh.spawn('[[model:deepseek-pro-other]] hi', ctx)) + .rejects.toThrow(/1 request\(s\) for a model other than deepseek-flash-1017 refused/); + expect(report().status).toBe(403); + expect(provider.requests).toEqual([]); + }); +}); + +test('confinement fails closed before any child starts', async () => { + await withSeat(async ({ ctx, root }) => { + const spawned = jest.fn(); + const cases = [ + [{ _platform: 'linux' }, /macOS only/], + [{ _detectSeatbelt: () => ({ available: false, error: 'no sandbox-exec' }) }, /needs Seatbelt: no sandbox-exec/], + [{ _prepareMcpSpawn: () => null }, /warmed Commonly MCP build/], + [{ _selectedMcpVersionDir: () => null }, /warmed Commonly MCP build/], + ]; + for (const [override, message] of cases) { + await expect(dsh.spawn('hi', { ...ctx, ...override, _spawnImpl: spawned })).rejects.toThrow(message); + } + expect(spawned).not.toHaveBeenCalled(); + expect(existsSync(join(ctx.cwd, '.stub-report.json'))).toBe(false); + }); +}); + +test('only the one confined shape is accepted', () => { + expect(isConfinedDsh({})).toBe(false); + expect(isConfinedDsh({ sandbox: CONFINED })).toBe(true); + for (const sandbox of [ + { mode: 'workspace' }, + { mode: 'workspace', network: { policy: 'unrestricted' } }, + { mode: 'workspace', network: { policy: 'restricted', 'allow-hosts': ['evil.example'] } }, + { ...CONFINED, filesystem: { 'write-outside': ['/'] } }, + { mode: 'read-only', network: { policy: 'restricted' } }, + { mode: 'bwrap', network: { policy: 'restricted' } }, + ]) { + expect(() => isConfinedDsh({ sandbox })).toThrow(/one confined shape/); + } + expect(() => isConfinedDsh({ sandbox: { ...CONFINED, trust: 'public' } })).toThrow(/public-trust/); +}); + +test('an unconfined seat keeps its old spawn: no wrap, no fence, no pin', async () => { + await withSeat(async ({ ctx, wraps, report }) => { + const { sandbox, ...environment } = ctx.environment; + const result = await dsh.spawn('[[model:any-model]] hi', { ...ctx, environment }); + expect(result.text).toBe('PONG'); + expect(wraps).toEqual([]); + const seen = report(); + expect(seen.env.HTTPS_PROXY).toBeUndefined(); + expect(seen.egress).toBeNull(); + expect(seen.mcp.env.COMMONLY_API_URL).toBe(ctx.instanceUrl); + expect(seen.sandboxPolicy).toEqual({ mode: 'workspace-write', workspaceRoot: ctx.cwd }); + }); +}); + +(process.platform === 'win32' ? test.skip : test)('a supervised confined turn that stalls is killed with what it started', async () => { + await withSeat(async ({ ctx, root }) => { + const store = createTraceStore({ root: join(root, 'trace') }); + let grandchild; + try { + const pending = superviseRun({ + adapter: dsh, prompt: '[[hang]]', ctx, store, + policy: { stallMs: 400, cancelGraceMs: 300, timeoutMs: 20000 }, + }); + grandchild = await readPid(join(ctx.cwd, '.stub-grandchild.pid')); + expect(isRunning(grandchild)).toBe(true); + const out = await pending; + expect(out.outcome).toBe('stuck'); + expect(await waitFor(() => !isRunning(grandchild))).toBe(true); + } finally { + if (grandchild) reap(grandchild); + } + }); +}, 20000); + +test('the supervisor flag is what puts a dsh child in its own group', async () => { + await withSeat(async ({ ctx }) => { + const calls = []; + const record = (flag) => ({ + ...ctx, + ...(flag ? { [OWN_PROCESS_GROUP]: true } : {}), + _spawnImpl: (command, args, options) => { + calls.push(options); + return spawnReal(command, args, options); + }, + }); + await dsh.spawn('hi', record(false)); + await dsh.spawn('hi', record(true)); + expect(Object.hasOwn(calls[0], 'detached')).toBe(false); + expect(calls[1].detached).toBe(process.platform !== 'win32'); + }); +}); diff --git a/cli/__tests__/egress-proxy.test.mjs b/cli/__tests__/egress-proxy.test.mjs new file mode 100644 index 0000000000..635e5da4fc --- /dev/null +++ b/cli/__tests__/egress-proxy.test.mjs @@ -0,0 +1,124 @@ +import { once } from 'events'; +import { connect, createServer } from 'net'; + +import { GITHUB_CONTENT_HOSTS, isNonPublicAddress, startEgressProxy } from '../src/lib/egress-proxy.js'; + +// A raw CONNECT through the proxy; resolves with the status line and the socket. +const tunnel = async (port, target) => { + const socket = connect(port, '127.0.0.1'); + await once(socket, 'connect'); + socket.write(`CONNECT ${target} HTTP/1.1\r\nHost: ${target}\r\n\r\n`); + let buffered = ''; + while (!buffered.includes('\r\n\r\n')) { + const [chunk] = await once(socket, 'data'); + buffered += String(chunk); + } + return { status: buffered.split('\r\n')[0], rest: buffered.slice(buffered.indexOf('\r\n\r\n') + 4), socket }; +}; + +describe('egress proxy', () => { + let upstream; + let upstreamPort; + let proxy; + + beforeEach(async () => { + // An echo server standing in for an allowlisted host. + upstream = createServer((socket) => socket.pipe(socket)); + await new Promise((resolve) => upstream.listen(0, '127.0.0.1', resolve)); + upstreamPort = upstream.address().port; + }); + + afterEach(async () => { + await proxy?.close(); + proxy = null; + await new Promise((resolve) => upstream.close(resolve)); + }); + + const start = (options = {}) => startEgressProxy({ + allowHosts: ['allowed.test'], + allowPorts: [upstreamPort], + _resolve: async () => [{ address: '127.0.0.1' }], + _allowPrivate: true, + ...options, + }); + + test('opens a tunnel to an allowlisted host and port', async () => { + proxy = await start(); + const { status, socket } = await tunnel(proxy.port, `allowed.test:${upstreamPort}`); + expect(status).toBe('HTTP/1.1 200 Connection Established'); + socket.write('ping'); + const [echo] = await once(socket, 'data'); + expect(String(echo)).toBe('ping'); + socket.destroy(); + expect(proxy.stats()).toEqual({ opened: 1, refusals: [] }); + }); + + test.each([ + ['a host not on the list', () => 'other.test:PORT', 'not allowlisted'], + ['an allowlisted host on another port', () => 'allowed.test:22', 'not allowlisted'], + ['an address literal', () => '127.0.0.1:PORT', 'address literal'], + ['a malformed target', () => 'allowed.test', 'malformed target'], + ])('refuses %s', async (_label, target, reason) => { + proxy = await start(); + const asked = target().replace('PORT', String(upstreamPort)); + const { status, socket } = await tunnel(proxy.port, asked); + expect(status).toMatch(/^HTTP\/1\.1 (403|400) /); + socket.destroy(); + expect(proxy.stats()).toEqual({ opened: 0, refusals: [{ target: asked, reason }] }); + }); + + test('refuses a name that resolves to a non-public address, even partly', async () => { + proxy = await start({ + _allowPrivate: false, + _resolve: async () => [{ address: '140.82.112.3' }, { address: '10.0.0.5' }], + }); + const { status, socket } = await tunnel(proxy.port, `allowed.test:${upstreamPort}`); + expect(status).toBe('HTTP/1.1 403 Forbidden'); + socket.destroy(); + expect(proxy.stats().refusals).toEqual([ + { target: `allowed.test:${upstreamPort}`, reason: 'resolves to a non-public address' }, + ]); + }); + + test('refuses plain HTTP requests: it carries tunnels only', async () => { + proxy = await start(); + const response = await fetch(`${proxy.url}/`, { headers: { host: 'allowed.test' } }); + expect(response.status).toBe(405); + expect(proxy.stats().opened).toBe(0); + }); + + test('reports an unreachable upstream without opening a tunnel', async () => { + proxy = await start(); + const closedPort = upstreamPort; + await new Promise((resolve) => upstream.close(resolve)); + upstream = createServer(); + await new Promise((resolve) => upstream.listen(0, '127.0.0.1', resolve)); + const { status, socket } = await tunnel(proxy.port, `allowed.test:${closedPort}`); + expect(status).toBe('HTTP/1.1 502 Bad Gateway'); + socket.destroy(); + expect(proxy.stats().opened).toBe(0); + }); + + test('defaults to GitHub\'s download hosts on 443, and refuses the hosts that take writes', async () => { + expect(GITHUB_CONTENT_HOSTS).toEqual([ + 'codeload.github.com', 'raw.githubusercontent.com', 'objects.githubusercontent.com', + ]); + proxy = await startEgressProxy({ _resolve: async () => [{ address: '140.82.112.3' }] }); + for (const target of ['api.deepseek.com:443', 'github.com:443', 'api.github.com:443', 'gist.github.com:443', 'uploads.github.com:443']) { + // eslint-disable-next-line no-await-in-loop + const { status, socket } = await tunnel(proxy.port, target); + expect([target, status]).toEqual([target, 'HTTP/1.1 403 Forbidden']); + socket.destroy(); + } + expect(proxy.stats().refusals.map((r) => r.reason)).toEqual(Array(5).fill('not allowlisted')); + }); + + test.each([ + ['127.0.0.1', true], ['10.1.2.3', true], ['172.20.0.1', true], ['192.168.1.1', true], + ['169.254.169.254', true], ['100.64.0.1', true], ['0.0.0.0', true], ['::1', true], + ['fd00::1', true], ['fe80::1', true], ['::ffff:127.0.0.1', true], ['not-an-ip', true], + ['140.82.112.3', false], ['185.199.108.133', false], ['2606:50c0:8000::153', false], + ])('isNonPublicAddress(%s) is %s', (address, expected) => { + expect(isNonPublicAddress(address)).toBe(expected); + }); +}); diff --git a/cli/__tests__/instance-proxy.test.mjs b/cli/__tests__/instance-proxy.test.mjs index 9d9867f404..a0e6e6ff37 100644 --- a/cli/__tests__/instance-proxy.test.mjs +++ b/cli/__tests__/instance-proxy.test.mjs @@ -82,3 +82,35 @@ test('closes the downstream client when an upstream response ends incomplete', a await rm(root, { recursive: true, force: true }); } }); + +test('without a socket path the relay listens on a loopback port to the same origin', async () => { + let seen; + const target = createServer((request, response) => { + seen = { path: request.url, host: request.headers.host, proxyAuth: request.headers['proxy-authorization'] }; + response.end('ok'); + }); + await new Promise((resolve) => target.listen(0, '127.0.0.1', resolve)); + const origin = `127.0.0.1:${target.address().port}`; + let proxy; + try { + proxy = await startInstanceProxy({ instanceUrl: `http://${origin}` }); + expect(proxy.socketPath).toBeNull(); + expect(proxy.url).toBe(`http://127.0.0.1:${proxy.port}`); + const response = await fetch(`${proxy.url}/api/agents/runtime/pods/p/context`, { + headers: { 'proxy-authorization': 'Basic x' }, + }); + expect(await response.text()).toBe('ok'); + expect(seen).toEqual({ path: '/api/agents/runtime/pods/p/context', host: origin, proxyAuth: undefined }); + // An absolute-form target is refused, not followed to another origin. + const client = connect(proxy.port, '127.0.0.1'); + await once(client, 'connect'); + client.write('GET //elsewhere.example/ HTTP/1.1\r\nHost: x\r\n\r\n'); + const [chunk] = await once(client, 'data'); + expect(String(chunk)).toMatch(/^HTTP\/1\.1 400/); + client.destroy(); + } finally { + await proxy?.close(); + target.closeAllConnections(); + await new Promise((resolve) => target.close(resolve)); + } +}); diff --git a/cli/__tests__/seatbelt.loopback.smoke.test.mjs b/cli/__tests__/seatbelt.loopback.smoke.test.mjs new file mode 100644 index 0000000000..c854ca4df7 --- /dev/null +++ b/cli/__tests__/seatbelt.loopback.smoke.test.mjs @@ -0,0 +1,19 @@ +/** + * The loopback Seatbelt profile a confined dsh seat runs under, enforced by + * the kernel: writes outside the workspace, the seat's secrets on disk, other + * network routes and the keychain's Security services are all denied, while + * the workspace and the one admitted loopback port work. macOS only. + */ +import { probeSeatbeltConfinement } from '../scripts/probe-seatbelt-confinement.mjs'; + +const macTest = process.platform === 'darwin' ? test : test.skip; + +macTest('a loopback profile denies what a confined seat must not reach', async () => { + const report = await probeSeatbeltConfinement(); + const failed = report.results.filter((row) => !row.pass); + expect(failed).toEqual([]); + expect(report.results.map((row) => row.probe)).toEqual(expect.arrayContaining([ + 'write outside the workspace', 'open ~/.commonly', 'open ~/.config/gh/hosts.yml', + 'open the login keychain file', 'connect to 1.1.1.1:443', 'connect to another loopback port', + ])); +}, 60000); diff --git a/cli/__tests__/seatbelt.test.mjs b/cli/__tests__/seatbelt.test.mjs index 36f583872c..61688ad6e3 100644 --- a/cli/__tests__/seatbelt.test.mjs +++ b/cli/__tests__/seatbelt.test.mjs @@ -217,6 +217,36 @@ describe('macOS Seatbelt profile', () => { })).toThrow(/valid TCP ports/); }); + test('a loopback profile drops open network and the Security services, and nothing else', () => { + const port = 54321; + const base = { + workspacePath: workspace, + executablePath: '/usr/bin/true', + statePath: state, + mcpConfigDir: mcp, + loopbackNetworkPorts: [port], + }; + const open = buildSeatbeltProfile(base); + expect(buildSeatbeltProfile({ ...base, network: 'open' })).toBe(open); + expect(open).toMatch(/^\(allow network-outbound\)$/m); + expect(open).toContain('(global-name "com.apple.SecurityServer")'); + + const loopback = buildSeatbeltProfile({ ...base, network: 'loopback' }); + expect(loopback).not.toMatch(/^\(allow network-outbound\)$/m); + expect(loopback).not.toContain('AF_SYSTEM'); + for (const service of ['SecurityServer', 'securityd', 'ocspd', 'networkd', 'SystemConfiguration']) { + expect(loopback).not.toContain(`com.apple.${service}`); + } + expect(loopback).toContain(`(allow network-outbound (remote tcp "localhost:${port}"))`); + // Every line a loopback profile carries, the open one carries too. + const openLines = new Set(open.split('\n')); + expect(loopback.split('\n').filter((line) => !openLines.has(line))).toEqual([]); + const outbound = loopback.split('\n').filter((line) => line.includes('network-outbound')); + expect(outbound).toEqual([`(allow network-outbound (remote tcp "localhost:${port}"))`]); + + expect(() => buildSeatbeltProfile({ ...base, network: 'none' })).toThrow(/open or loopback/); + }); + test('Claude gets the login-keychain file on request, and never the shared Claude temp root', () => { const keychains = join(homedir(), 'Library', 'Keychains'); const claudeTmp = `/private/tmp/claude-${typeof process.getuid === 'function' ? process.getuid() : '0'}`; diff --git a/cli/scripts/probe-seatbelt-confinement.mjs b/cli/scripts/probe-seatbelt-confinement.mjs new file mode 100644 index 0000000000..d01cf4c070 --- /dev/null +++ b/cli/scripts/probe-seatbelt-confinement.mjs @@ -0,0 +1,172 @@ +#!/usr/bin/env node +/** + * Probe what a `network: 'loopback'` Seatbelt profile actually denies, on this + * Mac, with nothing but node and (when the toolchain is present) a ten-line C + * program of our own. It is the enforcement half of the confined dsh seat: + * the adapter tests prove the options, this proves the kernel holds them. + * + * node cli/scripts/probe-seatbelt-confinement.mjs [--json] + * + * Every probe only opens, connects or looks up; none reads a file's content, + * and nothing here calls /usr/bin/security. Exit 0 when every probe matched. + */ +import { spawnSync } from 'child_process'; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from 'fs'; +import { createServer } from 'net'; +import { homedir } from 'os'; +import { join } from 'path'; +import { pathToFileURL } from 'url'; + +import { buildSeatbeltProfile } from '../src/lib/sandbox/seatbelt.js'; + +// Runs inside the profile. Each probe prints one JSON line: { name, ok, code }. +const CHILD = String.raw` +const fs = require('fs'); +const net = require('net'); +const [workspace, outside, allowedPort, deniedPort, paths] = JSON.parse(process.argv[1]); +const out = (name, ok, code) => process.stdout.write(JSON.stringify({ name, ok, code }) + '\n'); +const tryOpen = (name, path) => { + try { fs.closeSync(fs.openSync(path, 'r')); out(name, true, null); } catch (e) { out(name, false, e.code); } +}; +const tryWrite = (name, path) => { + try { fs.writeFileSync(path, 'probe'); out(name, true, null); } catch (e) { out(name, false, e.code); } +}; +const tryConnect = (name, host, port) => new Promise((resolve) => { + const socket = net.connect({ host, port, timeout: 3000 }); + socket.on('connect', () => { socket.destroy(); out(name, true, null); resolve(); }); + socket.on('timeout', () => { socket.destroy(); out(name, false, 'TIMEOUT'); resolve(); }); + socket.on('error', (e) => { out(name, false, e.code); resolve(); }); +}); +(async () => { + tryWrite('write inside the workspace', workspace + '/probe.txt'); + tryWrite('write outside the workspace', outside + '/probe.txt'); + for (const [name, path] of paths) tryOpen(name, path); + await tryConnect('connect to the allowed loopback port', '127.0.0.1', Number(allowedPort)); + await tryConnect('connect to another loopback port', '127.0.0.1', Number(deniedPort)); + await tryConnect('connect to 1.1.1.1:443', '1.1.1.1', 443); +})(); +`; + +// bootstrap_look_up only: it asks launchd for a port and never talks to it. +const MACH_LOOKUP_C = `#include +#include +int main(int argc, char **argv) { + mach_port_t port; + kern_return_t kr = bootstrap_look_up(bootstrap_port, argv[1], &port); + printf("%d\\n", kr); + return kr == KERN_SUCCESS ? 0 : 1; +} +`; + +const listen = () => new Promise((resolve) => { + const server = createServer((socket) => socket.destroy()); + server.listen(0, '127.0.0.1', () => resolve(server)); +}); + +const compileMachLookup = (dir) => { + // xcode-select fails without a toolchain; /usr/bin/clang would then offer an + // install dialog, so it is not run at all. + if (spawnSync('/usr/bin/xcode-select', ['-p'], { encoding: 'utf8' }).status !== 0) return null; + const source = join(dir, 'mach-lookup.c'); + const binary = join(dir, 'mach-lookup'); + writeFileSync(source, MACH_LOOKUP_C); + const built = spawnSync('/usr/bin/clang', ['-o', binary, source], { encoding: 'utf8' }); + return built.status === 0 ? binary : null; +}; + +export const probeSeatbeltConfinement = async () => { + if (process.platform !== 'darwin') throw new Error('the Seatbelt probe runs on macOS only'); + const root = mkdtempSync(join(homedir(), '.commonly-seatbelt-probe-')); + const workspace = join(root, 'workspace'); + const state = join(root, 'state'); + const outside = join(root, 'outside'); + const tools = join(root, 'tools'); + for (const dir of [workspace, state, outside, tools]) spawnSync('/bin/mkdir', ['-p', dir]); + const allowed = await listen(); + const denied = await listen(); + const results = []; + try { + const machLookup = compileMachLookup(tools); + const profile = buildSeatbeltProfile({ + workspacePath: workspace, + workspaceAccess: 'write', + executablePath: process.execPath, + statePath: state, + network: 'loopback', + loopbackNetworkPorts: [allowed.address().port], + executablePaths: machLookup ? [machLookup] : [], + }); + const secretPaths = [ + ['open ~/.commonly', join(homedir(), '.commonly')], + ['open ~/.config/gh/hosts.yml', join(homedir(), '.config', 'gh', 'hosts.yml')], + ['open the login keychain file', join(homedir(), 'Library', 'Keychains', 'login.keychain-db')], + ]; + const child = spawnSync('/usr/bin/sandbox-exec', ['-p', profile, process.execPath, '-e', CHILD, JSON.stringify([ + workspace, outside, allowed.address().port, denied.address().port, secretPaths, + ])], { encoding: 'utf8', timeout: 30_000 }); + const seen = Object.fromEntries(String(child.stdout || '').split('\n').filter(Boolean) + .map((line) => JSON.parse(line)).map((entry) => [entry.name, entry])); + const expect = (name, wantOk, detail = '') => { + const got = seen[name]; + results.push({ + probe: name, expected: wantOk ? 'allowed' : 'denied', + got: got ? (got.ok ? 'allowed' : `denied (${got.code})`) : 'no result', + pass: !!got && got.ok === wantOk && (wantOk || got.code === 'EPERM' || got.code === 'EACCES'), + detail, + }); + }; + expect('write inside the workspace', true); + expect('write outside the workspace', false); + for (const [name, path] of secretPaths) { + // A denial of a path that is not there proves less, so say which it was. + expect(name, false, existsSync(path) ? 'exists on this host' : 'absent on this host'); + } + expect('connect to the allowed loopback port', true); + expect('connect to another loopback port', false); + expect('connect to 1.1.1.1:443', false); + + if (machLookup) { + const lookup = (service, sandboxed) => { + const argv = sandboxed ? ['/usr/bin/sandbox-exec', '-p', profile, machLookup, service] : [machLookup, service]; + const run = spawnSync(argv[0], argv.slice(1), { encoding: 'utf8', timeout: 10_000 }); + return Number(String(run.stdout).trim()); + }; + const rows = [ + // Positive controls: the program works, unconfined and for a service the profile admits. + ['look up com.apple.SecurityServer, unconfined', 'com.apple.SecurityServer', false, true], + ['look up com.apple.trustd (admitted)', 'com.apple.trustd', true, true], + ['look up com.apple.SecurityServer', 'com.apple.SecurityServer', true, false], + ['look up com.apple.securityd', 'com.apple.securityd', true, false], + ]; + for (const [probe, service, sandboxed, wantOk] of rows) { + const kr = lookup(service, sandboxed); + results.push({ + probe, expected: wantOk ? 'allowed' : 'denied', + got: kr === 0 ? 'allowed' : `denied (kern_return ${kr})`, + pass: wantOk ? kr === 0 : kr !== 0 && Number.isFinite(kr), + detail: '', + }); + } + } else { + results.push({ probe: 'mach-lookup probes', expected: 'run', got: 'skipped: no command-line toolchain', pass: true, detail: 'skipped' }); + } + return { ok: results.every((row) => row.pass), results, stderr: String(child.stderr || '').trim().slice(0, 500) }; + } finally { + allowed.close(); + denied.close(); + rmSync(root, { recursive: true, force: true }); + } +}; + +if (import.meta.url === pathToFileURL(process.argv[1]).href) { + const report = await probeSeatbeltConfinement(); + if (process.argv.includes('--json')) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + } else { + for (const row of report.results) { + process.stdout.write(`${row.pass ? 'ok ' : 'FAIL'} ${row.probe}: expected ${row.expected}, got ${row.got}${row.detail ? ` (${row.detail})` : ''}\n`); + } + if (report.stderr) process.stdout.write(`child stderr: ${report.stderr}\n`); + } + process.exitCode = report.ok ? 0 : 1; +} diff --git a/cli/src/lib/adapters/dsh.js b/cli/src/lib/adapters/dsh.js index 9bdaa78b80..4121143914 100644 --- a/cli/src/lib/adapters/dsh.js +++ b/cli/src/lib/adapters/dsh.js @@ -1,12 +1,28 @@ import { observeSpawn } from '../runtime-observation.js'; -/** DeepSeek Harness 0.2.1-rc.2, private seats only. Configuration is per spawn. */ +/** + * DeepSeek Harness 0.2.1-rc.2, private seats only. Configuration is per spawn. + * + * A seat runs plain, or confined: `sandbox: { mode: 'workspace', network: + * { policy: 'restricted' } }` runs dsh inside one Seatbelt profile that writes + * only the workspace and the seat's home, reaches only three loopback ports + * (the provider proxy, the Commonly instance relay and the GitHub egress + * proxy), and cannot reach the Security services the keychain sits behind. + * The provider proxy is pinned to the seat's model and checks the model each + * response names. dsh's own sandbox is turned off inside it, because a second + * Seatbelt cannot be applied under the first. + */ +import { spawnInOwnGroup, wantsOwnProcessGroup } from '@commonlyai/seats/process-group'; import { requireSeatModel } from './seat-model.js'; import { spawn as childSpawn, spawnSync } from 'child_process'; import { createHash } from 'crypto'; -import { existsSync } from 'fs'; +import { existsSync, realpathSync } from 'fs'; import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'fs/promises'; import { homedir, tmpdir } from 'os'; -import { basename, join } from 'path'; +import { basename, delimiter, isAbsolute, join, sep } from 'path'; +import { startEgressProxy } from '../egress-proxy.js'; +import { startInstanceProxy } from '../instance-proxy.js'; +import { prepareMcpSpawn, selectedMcpVersionDir } from '../mcp-home.js'; +import { detectSeatbelt, wrapArgvWithSeatbelt } from '../sandbox/seatbelt.js'; import { buildMemoryPreamble } from '../memory-bridge.js'; import { commonlyMcpServer } from '../default-environment.js'; import { effectiveSandboxTrust, refuseWorkspaceTrust, validateEnvironmentSpec } from '../environment.js'; @@ -21,15 +37,56 @@ const PROXY_KEY_ENV = 'COMMONLY_DSH_PROXY_KEY'; const MAX_TIMEOUT_MS = 15 * 60 * 1000; const SAFE_ENV = /^(?:PATH|LANG|LC_[A-Z_]+|TERM|USER|LOGNAME|NO_COLOR|CI|SSL_CERT_FILE|SSL_CERT_DIR)$/; -export const assertPrivateDsh = (environment = {}) => { +/** True for the one confined shape this adapter enforces; throws on any other sandbox. */ +export const isConfinedDsh = (environment = {}) => { const sandbox = environment?.sandbox || {}; refuseWorkspaceTrust(sandbox, 'dsh'); if (effectiveSandboxTrust(sandbox.trust) === 'public') { throw new Error('dsh adapter: public-trust seats are not supported; nested Seatbelt fails closed'); } - if (sandbox.mode !== undefined) { - throw new Error(`dsh adapter: sandbox.mode=${sandbox.mode} cannot be enforced by this adapter`); + if (sandbox.mode === undefined) return false; + const { mode, network, ...rest } = sandbox; + const networkKeys = Object.keys(network || {}); + if (mode === 'workspace' && network?.policy === 'restricted' + && networkKeys.length === 1 && Object.keys(rest).length === 0) return true; + throw new Error(`dsh adapter: sandbox.mode=${mode} cannot be enforced by this adapter; ` + + "the one confined shape is { mode: 'workspace', network: { policy: 'restricted' } }"); +}; + +export const assertPrivateDsh = (environment = {}) => { isConfinedDsh(environment); }; + +const onPath = (name, env) => { + if (isAbsolute(name)) return existsSync(name) ? name : null; + for (const dir of String(env?.PATH || '').split(delimiter)) { + if (dir && existsSync(join(dir, name))) return join(dir, name); } + return null; +}; + +/** + * The script a dsh executable runs, and the outermost node_modules holding it: + * a confined run starts that script with this node, and reads that tree only. + */ +export const resolveDshEntry = (binary, env) => { + const found = onPath(binary, env); + if (!found) throw new Error(`dsh adapter: a confined seat could not find ${binary} on PATH`); + const entry = realpathSync(found); + const parts = entry.split(sep); + const root = parts.indexOf('node_modules'); + if (root < 1 || !/\.(?:c|m)?js$/.test(entry)) { + throw new Error(`dsh adapter: a confined seat runs dsh from its node package; ${found} resolves to ${entry}`); + } + return { entry, packages: parts.slice(0, root + 1).join(sep) }; +}; + +/** Fail a confined turn whose provider traffic broke the model pin. */ +export const assertServedModel = (check, credentials) => { + if (!check.refused && !check.failures.length) return; + const parts = [ + ...(check.refused ? [`${check.refused} request(s) for a model other than ${check.pinned} refused`] : []), + ...check.failures, + ]; + throw adapterFailure('dsh', `served model check failed: ${parts.join('; ')}`, { credentials, limit: 500 }); }; /** A stable home per agent and Commonly instance, never the operator's ~/.dsh. */ @@ -71,9 +128,17 @@ export const resolveCommonlyMcp = (declared, ctx) => { if (servers.length !== 1 || servers[0]?.name !== 'commonly') { throw new Error('dsh adapter: PR 1 supports only the Commonly MCP server'); } - const server = servers[0]; - if ((server.transport || 'stdio') !== 'stdio' - || !isCommonlyCommand(server.command) || !describeMcpCommand(server.command)?.isCommonly) { + const isCommonly = (entry) => (entry.transport || 'stdio') === 'stdio' + && isCommonlyCommand(entry.command) && !!describeMcpCommand(entry.command)?.isCommonly; + if (!isCommonly(servers[0])) { + throw new Error('dsh adapter: Commonly MCP must be a recognized stdio @commonlyai/mcp command'); + } + // A confined seat runs the warmed build it can read, never a package runner, + // and that build is held to the same check as the declaration. + const server = ctx.prepareCommand + ? { ...servers[0], command: ctx.prepareCommand(servers[0].command) } + : servers[0]; + if (!isCommonly(server)) { throw new Error('dsh adapter: Commonly MCP must be a recognized stdio @commonlyai/mcp command'); } const delivery = deliverSeatCredential(server, { @@ -101,7 +166,9 @@ export const resolveCommonlyMcp = (declared, ctx) => { }; /** JSON is valid YAML; a generated patch has no expression or quoting ambiguity. */ -export const buildPatch = ({ proxyBaseURL, model, cwd, mcp, effort, deepseekOfficial = false }) => ([ +export const buildPatch = ({ + proxyBaseURL, model, cwd, mcp, effort, deepseekOfficial = false, confined = false, +}) => ([ { id: 'llm-pi-ai', config: { providers: { commonly: { apiKeyEnv: PROXY_KEY_ENV, api: 'openai-completions', baseURL: proxyBaseURL, @@ -124,7 +191,9 @@ export const buildPatch = ({ proxyBaseURL, model, cwd, mcp, effort, deepseekOffi { id: 'llm-deepseek', disabled: true }, { id: 'llm-deepseek-account', disabled: true }, { id: 'session-log-deepseek', config: { enabled: false } }, - { id: 'sandbox-policy', config: { mode: 'workspace-write', workspaceRoot: cwd } }, + // Confined, the outer Seatbelt is the sandbox: dsh's own would nest a second + // one, which macOS refuses, so it is off. + { id: 'sandbox-policy', config: { mode: confined ? 'danger-full-access' : 'workspace-write', workspaceRoot: cwd } }, { insert: [mcp] }, ]); @@ -169,8 +238,10 @@ const parseEvent = (line, state) => { } }; -export const runDsh = ({ binary, args, cwd, env, timeoutMs, credentials, spawnImpl = childSpawn, onSpawn, controls = {} }) => new Promise((resolve, reject) => { - const proc = spawnImpl(binary, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }); +export const runDsh = ({ + binary, args, cwd, env, timeoutMs, credentials, spawnImpl = childSpawn, onSpawn, controls = {}, ownGroup = false, +}) => new Promise((resolve, reject) => { + const proc = spawnInOwnGroup(spawnImpl, binary, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'] }, { ownGroup }); observeSpawn(proc, onSpawn, controls); const state = { sessionId: null, final: null, error: null, buffer: '' }; let stderr = ''; @@ -252,7 +323,14 @@ export default { async spawn(prompt, ctx = {}) { const seatModel = requireSeatModel(ctx.environment, 'dsh', true); - assertPrivateDsh(ctx.environment); + const confined = isConfinedDsh(ctx.environment); + if (confined) { + if ((ctx._platform || process.platform) !== 'darwin') { + throw new Error('dsh adapter: a confined seat needs Seatbelt, which is macOS only'); + } + const seatbelt = (ctx._detectSeatbelt || detectSeatbelt)(); + if (!seatbelt.available) throw new Error(`dsh adapter: a confined seat needs Seatbelt: ${seatbelt.error}`); + } if (!ctx.environment?.provider || !ctx.environment?.model) { throw new Error('dsh requires environment.provider and environment.model'); } @@ -270,19 +348,39 @@ export default { const tempDir = await mkdtemp(join(tmpdir(), 'commonly-dsh-')); await chmod(tempDir, 0o700); let proxy; + let relay; + let egress; try { const credential = writeCredentialFile(ctx.runtimeToken, { agentName: 'commonly', root: tempDir }); proxy = await startProviderProxy({ provider: ctx.environment.provider, keyFile, tokenFile: join(tempDir, 'provider-proxy-token'), adapterLabel: 'dsh', + ...(confined ? { pinnedModel: seatModel } : {}), }); const bearer = (await readFile(proxy.tokenFile, 'utf8')).trim(); + const mcpDirectories = []; + if (confined) { + relay = await startInstanceProxy({ instanceUrl: ctx.instanceUrl }); + egress = await (ctx._startEgressProxy || startEgressProxy)(); + } const mcp = resolveCommonlyMcp(ctx.environment?.mcp, { - credentialFile: credential.path, instanceUrl: ctx.instanceUrl, + credentialFile: credential.path, + instanceUrl: confined ? relay.url : ctx.instanceUrl, + ...(confined ? { + prepareCommand: (command) => { + const warmed = (ctx._prepareMcpSpawn || prepareMcpSpawn)(command, { apiUrl: ctx.instanceUrl, publicSeat: true }); + const directory = warmed && (ctx._selectedMcpVersionDir || selectedMcpVersionDir)(warmed); + if (!directory) { + throw new Error('dsh adapter: a confined seat runs only a warmed Commonly MCP build, and none is ready on this host'); + } + mcpDirectories.push(directory); + return [process.execPath, warmed[1]]; + }, + } : {}), }); const patchData = buildPatch({ proxyBaseURL: proxy.baseURL, model: seatModel, - cwd: ctx.cwd, mcp, effort: ctx.environment.effort, + cwd: ctx.cwd, mcp, effort: ctx.environment.effort, confined, deepseekOfficial: new URL(ctx.environment.provider.baseURL).hostname === 'api.deepseek.com', }); const patch = JSON.stringify(patchData, null, 2); @@ -300,11 +398,51 @@ export default { const requested = Number(ctx.timeoutMs); const timeoutMs = Number.isFinite(requested) && requested > 0 ? Math.min(requested, MAX_TIMEOUT_MS) : MAX_TIMEOUT_MS; - return await runDsh({ - binary, args, cwd: ctx.cwd, env, timeoutMs, - credentials: [...spawnCredentials(ctx, []), bearer], spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, controls: { model: passedModel || null, effort: passedProvider.reasoning || null }, - }); + let command = binary; + let commandArgs = args; + if (confined) { + const { entry, packages } = resolveDshEntry(binary, sourceEnv); + const tempWork = join(home, 'tmp'); + await mkdir(tempWork, { recursive: true, mode: 0o700 }); + env.TMPDIR = tempWork; + for (const name of ['HTTPS_PROXY', 'https_proxy', 'HTTP_PROXY', 'http_proxy']) env[name] = egress.url; + env.NO_PROXY = '127.0.0.1,localhost'; + env.no_proxy = env.NO_PROXY; + [command, ...commandArgs] = (ctx._wrapArgvWithSeatbelt || wrapArgvWithSeatbelt)([process.execPath, entry, ...args], { + workspacePath: ctx.cwd, + workspaceAccess: 'write', + executablePath: process.execPath, + statePath: home, + mcpConfigDir: tempDir, + network: 'loopback', + loopbackNetworkPorts: [proxy.port, relay.port, egress.port], + readOnlyDirectories: [packages, ...mcpDirectories], + }); + } + const credentials = [...spawnCredentials(ctx, []), bearer]; + let result; + let failure; + try { + result = await runDsh({ + binary: command, args: commandArgs, cwd: ctx.cwd, env, timeoutMs, credentials, + spawnImpl: ctx._spawnImpl, onSpawn: ctx.onSpawn, ownGroup: wantsOwnProcessGroup(ctx), + controls: { model: passedModel || null, effort: passedProvider.reasoning || null }, + }); + } catch (error) { + failure = error; + } + // A broken pin is the reason a turn failed, whatever dsh said about it. + if (confined) { + const { refusals } = egress.stats(); + // eslint-disable-next-line no-console + if (refusals.length) console.warn(`[dsh] egress refused: ${refusals.map((r) => `${r.target} (${r.reason})`).join(', ')}`); + assertServedModel(proxy.modelCheck(), credentials); + } + if (failure) throw failure; + return result; } finally { + try { await egress?.close(); } catch { /* best effort */ } + try { await relay?.close(); } catch { /* best effort */ } try { await proxy?.close(); } catch { /* best effort */ } await rm(tempDir, { recursive: true, force: true }); } diff --git a/cli/src/lib/egress-proxy.js b/cli/src/lib/egress-proxy.js new file mode 100644 index 0000000000..62c227b764 --- /dev/null +++ b/cli/src/lib/egress-proxy.js @@ -0,0 +1,160 @@ +/** + * A per-spawn loopback CONNECT proxy that reaches only a fixed list of hosts. + * + * A confined run gets no outbound network of its own (Seatbelt `network: + * 'loopback'`); HTTPS_PROXY points it here. Only CONNECT to an allowlisted + * name on an allowlisted port is opened, and the name is resolved here and + * the connection made to the address resolved, so a name that resolves to + * loopback or a private network is refused rather than followed. Plain HTTP + * requests are refused: the proxy never sees the content of what it carries. + * + * There is no proxy credential. Everything the proxy reaches, any process on + * the host already reaches; it gives the confined run a route, not authority. + */ +import { createServer } from 'http'; +import { BlockList, connect, isIP } from 'net'; +import { lookup } from 'dns/promises'; + +// GitHub's download hosts only. github.com and api.github.com accept writes +// (git push, gists, issues) made with any token a run is handed, and a tunnel +// cannot tell a read from a write, so they would be an outbound route for +// anything the run can read. These hosts only serve content. +export const GITHUB_CONTENT_HOSTS = Object.freeze([ + 'codeload.github.com', + 'raw.githubusercontent.com', + 'objects.githubusercontent.com', +]); + +const CONNECT_TIMEOUT_MS = 10_000; +const MAX_REFUSALS_KEPT = 20; + +const nonPublic = new BlockList(); +for (const [address, prefix] of [ + ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], + ['169.254.0.0', 16], ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.168.0.0', 16], + ['198.18.0.0', 15], ['224.0.0.0', 4], ['240.0.0.0', 4], +]) nonPublic.addSubnet(address, prefix, 'ipv4'); +for (const [address, prefix] of [ + ['::', 128], ['::1', 128], ['fc00::', 7], ['fe80::', 10], ['ff00::', 8], ['64:ff9b::', 96], +]) nonPublic.addSubnet(address, prefix, 'ipv6'); + +/** True for an address no public host name should resolve to. */ +export const isNonPublicAddress = (address) => { + const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(address); + if (mapped) return nonPublic.check(mapped[1], 'ipv4'); + const family = isIP(address); + if (!family) return true; + return nonPublic.check(address, family === 6 ? 'ipv6' : 'ipv4'); +}; + +/** `host:port` from a CONNECT target, or null when it is not one. */ +const parseTarget = (target) => { + const match = /^([A-Za-z0-9.-]+):(\d{1,5})$/.exec(target || ''); + if (!match) return null; + const port = Number(match[2]); + if (port < 1 || port > 65535) return null; + return { host: match[1].toLowerCase().replace(/\.$/, ''), port }; +}; + +/** + * startEgressProxy({ allowHosts?, allowPorts? }) → { port, url, refusals, close } + * + * Listens on 127.0.0.1 at an ephemeral port. `_resolve` and `_allowPrivate` + * exist for tests only, so a test can reach a loopback upstream. + */ +export const startEgressProxy = async ({ + allowHosts = GITHUB_CONTENT_HOSTS, + allowPorts = [443], + _resolve = (host) => lookup(host, { all: true, verbatim: true }), + _allowPrivate = false, +} = {}) => { + const hosts = new Set(allowHosts.map((host) => String(host).toLowerCase())); + const ports = new Set(allowPorts); + if (!hosts.size || [...ports].some((port) => !Number.isInteger(port) || port < 1 || port > 65535)) { + throw new Error('Egress proxy needs at least one host and valid TCP ports'); + } + const refusals = []; + let opened = 0; + const refuse = (socket, status, target, reason) => { + refusals.push({ target, reason }); + if (refusals.length > MAX_REFUSALS_KEPT) refusals.shift(); + if (!socket.destroyed) socket.end(`HTTP/1.1 ${status}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`); + }; + const sockets = new Set(); + const track = (socket) => { + sockets.add(socket); + socket.on('close', () => sockets.delete(socket)); + }; + + const server = createServer((incoming, outgoing) => { + incoming.resume(); + refusals.push({ target: String(incoming.url || '').slice(0, 200), reason: 'not CONNECT' }); + if (refusals.length > MAX_REFUSALS_KEPT) refusals.shift(); + outgoing.writeHead(405, { Connection: 'close' }).end(); + }); + server.on('connection', track); + server.on('connect', async (request, client, head) => { + client.on('error', () => {}); + const target = parseTarget(request.url); + const label = String(request.url || '').slice(0, 200); + if (!target) { refuse(client, '400 Bad Request', label, 'malformed target'); return; } + if (isIP(target.host)) { refuse(client, '403 Forbidden', label, 'address literal'); return; } + if (!hosts.has(target.host) || !ports.has(target.port)) { + refuse(client, '403 Forbidden', label, 'not allowlisted'); + return; + } + let addresses; + try { + addresses = (await _resolve(target.host)).map((entry) => entry.address); + } catch { + refuse(client, '502 Bad Gateway', label, 'did not resolve'); + return; + } + // Every address must be public, not just the one used: a name that + // resolves to both is not one this proxy was asked to reach. + if (!addresses.length || (!_allowPrivate && addresses.some(isNonPublicAddress))) { + refuse(client, '403 Forbidden', label, 'resolves to a non-public address'); + return; + } + if (client.destroyed) return; + const upstream = connect({ host: addresses[0], port: target.port }); + track(upstream); + let established = false; + client.on('close', () => upstream.destroy()); + upstream.setTimeout(CONNECT_TIMEOUT_MS, () => upstream.destroy()); + upstream.on('error', () => {}); + upstream.on('close', () => { + if (established) client.destroy(); + else refuse(client, '502 Bad Gateway', label, 'upstream unreachable'); + }); + upstream.once('connect', () => { + upstream.setTimeout(0); + established = true; + opened += 1; + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + if (head?.length) upstream.write(head); + upstream.pipe(client); + client.pipe(upstream); + }); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.removeListener('error', reject); + resolve(); + }); + }); + const { port } = server.address(); + return { + port, + url: `http://127.0.0.1:${port}`, + /** { opened, refusals: [{ target, reason }] } — the last 20 refusals. */ + stats: () => ({ opened, refusals: refusals.map((entry) => ({ ...entry })) }), + close: async () => { + const closed = new Promise((resolve) => server.close(resolve)); + for (const socket of sockets) socket.destroy(); + await closed; + }, + }; +}; diff --git a/cli/src/lib/instance-proxy.js b/cli/src/lib/instance-proxy.js index e93362b4a9..5755e049f2 100644 --- a/cli/src/lib/instance-proxy.js +++ b/cli/src/lib/instance-proxy.js @@ -1,4 +1,8 @@ -/** A per-spawn Unix-socket relay to one declared Commonly instance origin. */ +/** + * A per-spawn relay to one declared Commonly instance origin, on a Unix + * socket, or on an ephemeral loopback TCP port when no socketPath is given + * (for a Seatbelt run whose only network is loopback). + */ import { createServer, request as httpRequest } from 'http'; import { request as httpsRequest } from 'https'; import { chmod, unlink } from 'fs/promises'; @@ -48,20 +52,25 @@ export const startInstanceProxy = async ({ instanceUrl, socketPath }) => { }); await new Promise((resolve, reject) => { server.once('error', reject); - server.listen(socketPath, () => { + const listening = () => { server.removeListener('error', reject); resolve(); - }); + }; + if (socketPath) server.listen(socketPath, listening); + else server.listen(0, '127.0.0.1', listening); }); - await chmod(socketPath, 0o600); + if (socketPath) await chmod(socketPath, 0o600); + const port = socketPath ? null : server.address().port; return { - socketPath, + socketPath: socketPath || null, + port, + url: port ? `http://127.0.0.1:${port}` : null, close: async () => { const closed = new Promise((resolve) => server.close(resolve)); for (const upstream of upstreams) upstream.destroy(); server.closeAllConnections(); await closed; - await unlink(socketPath).catch((err) => { if (err.code !== 'ENOENT') throw err; }); + if (socketPath) await unlink(socketPath).catch((err) => { if (err.code !== 'ENOENT') throw err; }); }, }; }; diff --git a/cli/src/lib/sandbox/seatbelt.js b/cli/src/lib/sandbox/seatbelt.js index bfd24e4b9e..e782cd9f50 100644 --- a/cli/src/lib/sandbox/seatbelt.js +++ b/cli/src/lib/sandbox/seatbelt.js @@ -274,7 +274,12 @@ const PLATFORM_BASELINE = String.raw` ; Apple's App Sandbox grants every app this same domain. (allow user-preference-read (preference-domain "kCFPreferencesAnyApplication")) +`; +// Open network, the tail of the default baseline. A `network: 'loopback'` +// profile leaves it out: no outbound except the loopback ports it lists, and +// none of the Security framework services the keychain is reached through. +const OPEN_NETWORK = String.raw` ; The selected provider and declared MCP transports need outbound network. The model's ; WebSearch/WebFetch/Bash tools are denied independently in the adapter argv. (allow network-outbound) @@ -367,6 +372,8 @@ export const buildSeatbeltProfile = ({ readOnlyDirectories = [], loopbackNetworkPorts = [], allowClaudeRuntimeAccess = false, + // 'open' (the default) or 'loopback': only the listed loopback ports. + network = 'open', // Names directly inside a writable workspace that stay read-only. A // workspace seat granted write keeps its project's .git read-only, so every // change it makes is one `git status` shows the person (TASK-294). @@ -387,6 +394,9 @@ export const buildSeatbeltProfile = ({ throw new Error('Seatbelt loopbackNetworkPorts entries must be valid TCP ports'); } } + if (!['open', 'loopback'].includes(network)) { + throw new Error('Seatbelt network must be open or loopback'); + } if (!['read', 'write'].includes(workspaceAccess)) { throw new Error('Seatbelt workspaceAccess must be read or write'); } @@ -456,7 +466,7 @@ export const buildSeatbeltProfile = ({ })() : []; return [ - PLATFORM_BASELINE.trim(), + (network === 'open' ? `${PLATFORM_BASELINE}${OPEN_NETWORK}` : PLATFORM_BASELINE).trim(), executableRules, executableSymlinkRules, `(allow file-read-metadata file-test-existence (path-ancestors "${escapeSbplString(resolvedWorkspace)}"))`, diff --git a/docs/cli/dsh.md b/docs/cli/dsh.md index 8fca46c907..42d391319c 100644 --- a/docs/cli/dsh.md +++ b/docs/cli/dsh.md @@ -53,7 +53,8 @@ bearer, never the provider key. Use a pod that only its members can read. A private dsh seat runs with dsh's `workspace-write` permission mode. It is not a public-seat sandbox: the -adapter refuses `sandbox.trust: public` and any declared `sandbox.mode`. +adapter refuses `sandbox.trust: public` and every declared `sandbox` shape +except the one described under [Confined mode](#confined-mode-macos). The pod id is the last part of its URL: `commonly.me/v2/pods/`. ```bash @@ -81,3 +82,63 @@ outside the workspace; a second turn resumes only an id found in that home. DeepSeek Harness 0.2.1-rc.2 was walked with a non-admin Commonly account on macOS using CLI 0.1.117, then re-walked on CLI 0.1.122 on 2026-10-23. + +## Confined mode (macOS) + +A seat that declares exactly this, and nothing else under `sandbox`, runs +confined: + +```json +"sandbox": { "mode": "workspace", "network": { "policy": "restricted" } } +``` + +Any other `sandbox` shape is refused at spawn. A confined spawn also refuses to +start on a host other than macOS, when `sandbox-exec` is missing, and when no +warmed Commonly MCP build is ready. It never falls back to an unconfined run. + +What changes: + +- **Seatbelt.** dsh and the MCP server run under a deny-default Seatbelt + profile. Writes are allowed only in the workspace and the seat's private + home. Reads are limited to system paths, the dsh package, the warmed MCP + build and those two folders. dsh's own sandbox is set to + `danger-full-access` because a nested `sandbox-exec` fails; the outer + profile is the boundary. +- **Network.** The profile admits no outbound network except three loopback + ports opened for this spawn: the provider proxy, a relay to the Commonly + instance, and an egress proxy. The egress proxy carries `CONNECT` only to + GitHub's download hosts, `codeload.github.com`, + `raw.githubusercontent.com` and `objects.githubusercontent.com`, on port + 443, and refuses a name that resolves to a loopback or private address. + `github.com` and `api.github.com` are refused: they accept writes made with + any token a run is handed, and a tunnel cannot tell a read from a write. + Refused targets are printed after the turn. +- **Model pin.** The provider proxy refuses a request for any model other + than the seat's `model` (403, never forwarded), and fails the turn when the + provider's response names a different model than the one requested. +- **Process group.** The turn runs in its own process group, so a stalled + turn's children are killed with it. + +Limits to know: + +- The run holds no GitHub credential: `gh`'s token file and the keychain are + outside the profile. It can download public archives and raw files, and + cannot clone over git, call the GitHub API, or push. +- The seat's Commonly credential file is readable inside the profile, because + the MCP server needs it. +- Three routes out remain by design. The provider receives what the run + sends it. The instance relay carries whatever the seat's own Commonly token + allows, such as a post to a pod the seat is in. Anything the run can read, + that credential included, can leave through either of those two. The + egress proxy is the third: it opens tunnels to the three GitHub download + hosts only, and refuses github.com, api.github.com and every other host. + A request to a download host can still carry data in its URL. +- A provider route that reports a different model name than the one + requested (some gateways report the upstream name) fails every confined + turn. Check one turn before relying on it. +- `node cli/scripts/probe-seatbelt-confinement.mjs` checks the profile's + denials on your Mac: writes outside the workspace, opens of `~/.commonly`, + `gh`'s `hosts.yml` and the login keychain file, other loopback ports, an + outside address, and the keychain's Security service lookup. It never + calls `/usr/bin/security`. + diff --git a/packages/seats/src/wire/provider-proxy.js b/packages/seats/src/wire/provider-proxy.js index e1ce311e73..0b4c09260d 100644 --- a/packages/seats/src/wire/provider-proxy.js +++ b/packages/seats/src/wire/provider-proxy.js @@ -1,4 +1,5 @@ import { Transform } from 'node:stream'; +import { StringDecoder } from 'node:string_decoder'; /** The per-spawn loopback proxy shared by OpenAI-compatible CLI adapters. */ import { randomBytes, timingSafeEqual } from 'crypto'; import { constants as fsConstants, realpathSync } from 'fs'; @@ -40,6 +41,71 @@ export const redactProviderMaterial = (material) => { }); }; +const MAX_PINNED_REQUEST_BYTES = 8 * 1024 * 1024; +const MAX_PINNED_RESPONSE_BYTES = 16 * 1024 * 1024; + +/** + * Checks that every completion an upstream serves names the pinned model. + * SSE is checked line by line, and nothing is released until a line has named + * the pinned model; after that only checked lines are released, so a chunk + * naming another model never reaches the CLI. Any other body is held whole and + * released only after its `model` is checked. A body that names no model is + * unverified, which counts as a failure: the pin is only as good as the check. + */ +const servedModelCheck = ({ pinned, stream, onServed }) => { + // A chunk boundary can fall inside a multi-byte character; the decoder + // carries the partial bytes into the next chunk instead of mangling both. + const decoder = new StringDecoder('utf8'); + let pending = ''; + let held = ''; + let named = false; + let size = 0; + const check = (value) => { + if (typeof value !== 'string') return; + named = true; + onServed(value); + if (value !== pinned) throw new Error(`served model ${JSON.stringify(value)} is not the pinned ${JSON.stringify(pinned)}`); + }; + const checkLine = (line) => { + const data = /^data:\s?(.*)$/.exec(line.replace(/\r$/, ''))?.[1]; + if (data === undefined || data.trim() === '[DONE]' || !data.trim()) return; + let parsed; + try { parsed = JSON.parse(data); } catch { return; } + check(parsed?.model); + }; + return new Transform({ + transform(chunk, _encoding, done) { + size += chunk.length; + if (size > MAX_PINNED_RESPONSE_BYTES) { done(new Error('provider response exceeded the pinned-model check limit')); return; } + pending += decoder.write(chunk); + if (!stream) { done(); return; } + const end = pending.lastIndexOf('\n'); + if (end < 0) { done(); return; } + const complete = pending.slice(0, end + 1); + pending = pending.slice(end + 1); + try { complete.split('\n').forEach(checkLine); } catch (error) { done(error); return; } + held += complete; + if (!named) { done(); return; } + const release = held; + held = ''; + done(null, release); + }, + flush(done) { + pending += decoder.end(); + try { + if (stream) checkLine(pending); + else { + let parsed; + try { parsed = JSON.parse(pending); } catch { parsed = null; } + check(parsed?.model); + } + if (!named) throw new Error(`provider response named no model; the pinned ${JSON.stringify(pinned)} cannot be verified`); + } catch (error) { done(error); return; } + done(null, held + pending); + }, + }); +}; + const isPathWithin = (parent, candidate) => { const fromParent = relative(parent, candidate); return fromParent === '' || (fromParent !== '..' @@ -103,7 +169,13 @@ const readProviderKey = async (keyFile, adapterLabel) => { export const startProviderProxy = async ({ provider, keyFile, keyMaterial, tokenFile, socketPath, isolatedPort, adapterLabel = 'OpenCode', + // When set, every completion must request this model and be served by it. + pinnedModel, }) => { + if (pinnedModel !== undefined && (typeof pinnedModel !== 'string' || !pinnedModel.trim())) { + throw new Error(`${adapterLabel} pinnedModel must be a non-empty string`); + } + const modelState = { requested: new Set(), served: new Set(), refused: 0, failures: [] }; const upstream = new URL(provider.baseURL); if (socketPath && (!isAbsolute(socketPath) || !Number.isInteger(isolatedPort) || isolatedPort < 1 || isolatedPort > 65535)) { @@ -212,42 +284,98 @@ export const startProviderProxy = async ({ headers.authorization = `Bearer ${upstreamKey}`; headers.host = target.host; headers['accept-encoding'] = 'identity'; - const request = (target.protocol === 'https:' ? httpsRequest : httpRequest)({ - protocol: target.protocol, - hostname: target.hostname, - port: target.port, - path: target.path, - method: incoming.method, - headers, - }, (response) => { - if (response.statusCode >= 300 && response.statusCode < 400) { - response.resume(); - outgoing.writeHead(502).end('Provider redirects are not supported'); - return; - } - if (response.headers['content-encoding'] && response.headers['content-encoding'] !== 'identity') { - response.resume(); outgoing.writeHead(502).end('Compressed provider responses are not supported'); return; + const pinned = pinnedModel !== undefined && target.path.endsWith('/chat/completions'); + const forward = (body) => { + let streamed = false; + if (body) { + headers['content-length'] = String(body.length); + try { streamed = JSON.parse(body.toString('utf8'))?.stream === true; } catch { /* checked before forward */ } } - const responseHeaders = Object.fromEntries(Object.entries(response.headers) - .filter(([name]) => ![ - 'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', - 'te', 'trailer', 'transfer-encoding', 'upgrade', - ].includes(name.toLowerCase()))); - delete responseHeaders['content-length']; - for (const [name, value] of Object.entries(responseHeaders)) { - if (typeof value === 'string') responseHeaders[name] = value.split(upstreamKey).join('[redacted]'); - else if (Array.isArray(value)) responseHeaders[name] = value.map((entry) => entry.split(upstreamKey).join('[redacted]')); - } - outgoing.writeHead(response.statusCode || 502, responseHeaders); - response.pipe(redactProviderMaterial(upstreamKey)).pipe(outgoing); + const request = (target.protocol === 'https:' ? httpsRequest : httpRequest)({ + protocol: target.protocol, + hostname: target.hostname, + port: target.port, + path: target.path, + method: incoming.method, + headers, + }, (response) => { + if (response.statusCode >= 300 && response.statusCode < 400) { + response.resume(); + outgoing.writeHead(502).end('Provider redirects are not supported'); + return; + } + if (response.headers['content-encoding'] && response.headers['content-encoding'] !== 'identity') { + response.resume(); outgoing.writeHead(502).end('Compressed provider responses are not supported'); return; + } + const responseHeaders = Object.fromEntries(Object.entries(response.headers) + .filter(([name]) => ![ + 'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', + 'te', 'trailer', 'transfer-encoding', 'upgrade', + ].includes(name.toLowerCase()))); + delete responseHeaders['content-length']; + for (const [name, value] of Object.entries(responseHeaders)) { + if (typeof value === 'string') responseHeaders[name] = value.split(upstreamKey).join('[redacted]'); + else if (Array.isArray(value)) responseHeaders[name] = value.map((entry) => entry.split(upstreamKey).join('[redacted]')); + } + const ok = response.statusCode >= 200 && response.statusCode < 300; + if (!pinned || !ok) { + outgoing.writeHead(response.statusCode || 502, responseHeaders); + response.pipe(redactProviderMaterial(upstreamKey)).pipe(outgoing); + return; + } + const sse = streamed || /text\/event-stream/i.test(String(response.headers['content-type'] || '')); + const check = servedModelCheck({ + pinned: pinnedModel, stream: sse, onServed: (served) => modelState.served.add(served), + }); + const fail = (error) => { + modelState.failures.push(error.message); + process.stderr.write(`[${adapterLabel.toLowerCase()}] provider proxy: ${error.message}\n`); + response.destroy(); + if (!outgoing.headersSent) outgoing.writeHead(502, { 'content-type': 'text/plain; charset=utf-8' }).end('Served model check failed'); + else outgoing.destroy(); + }; + check.on('error', fail); + // A stream's headers go out at once; a whole body waits for its check. + if (sse) outgoing.writeHead(response.statusCode, responseHeaders); + else check.once('data', () => { if (!outgoing.headersSent) outgoing.writeHead(response.statusCode, responseHeaders); }); + response.pipe(check).pipe(redactProviderMaterial(upstreamKey)).pipe(outgoing); + }); + request.on('error', () => { + if (!outgoing.headersSent) outgoing.writeHead(502); + outgoing.end('Provider request failed'); + }); + outgoing.on('close', () => request.destroy()); + incoming.on('aborted', () => request.destroy()); + if (body) request.end(body); + else incoming.pipe(request); + }; + if (!pinned) { + forward(null); + return; + } + // A pinned proxy reads the request first: a completion for any other model + // is refused here, before the provider bills it. + const chunks = []; + let size = 0; + incoming.on('data', (chunk) => { + size += chunk.length; + if (size <= MAX_PINNED_REQUEST_BYTES) chunks.push(chunk); }); - request.on('error', () => { - if (!outgoing.headersSent) outgoing.writeHead(502); - outgoing.end('Provider request failed'); + incoming.on('end', () => { + if (size > MAX_PINNED_REQUEST_BYTES) { outgoing.writeHead(413).end(); return; } + const body = Buffer.concat(chunks); + let requested; + try { requested = JSON.parse(body.toString('utf8'))?.model; } catch { requested = undefined; } + if (requested !== pinnedModel) { + modelState.refused += 1; + process.stderr.write(`[${adapterLabel.toLowerCase()}] provider proxy refused a completion for an unpinned model\n`); + outgoing.writeHead(403, { 'content-type': 'text/plain; charset=utf-8' }); + outgoing.end(`This seat is pinned to ${pinnedModel}`); + return; + } + modelState.requested.add(requested); + forward(body); }); - outgoing.on('close', () => request.destroy()); - incoming.on('aborted', () => request.destroy()); - incoming.pipe(request); }); await new Promise((resolve, reject) => { @@ -274,6 +402,14 @@ export const startProviderProxy = async ({ tokenFile, socketPath, port, + /** The pin's verdict so far: what was asked for, what was served, and what failed. */ + modelCheck: () => ({ + pinned: pinnedModel ?? null, + requested: [...modelState.requested], + served: [...modelState.served], + refused: modelState.refused, + failures: [...modelState.failures], + }), close: () => new Promise((resolve) => { server.close(async () => { if (socketPath) await unlink(socketPath).catch(() => {}); diff --git a/packages/seats/test-support/fake-provider.js b/packages/seats/test-support/fake-provider.js index 62bb98bf77..440b783cb1 100644 --- a/packages/seats/test-support/fake-provider.js +++ b/packages/seats/test-support/fake-provider.js @@ -9,6 +9,13 @@ * [[refuse]] answer HTTP 400 with a content-policy error body * [[fail:CODE]] answer HTTP CODE * Serves POST /v1/chat/completions (stream and non-stream) and GET /v1/models. + * + * `servedModel` makes completions name a model other than the one requested, + * as a router that substitutes one would; `null` names none at all. + * `echoAuthorization` replies with the request's Authorization header, as a + * provider that echoes request headers in an error body would. + * `splitMultibyte` sends a completion in two writes, the cut inside its first + * multi-byte character, as a TCP chunk boundary can. */ import { createServer } from 'node:http'; @@ -22,7 +29,11 @@ const directiveOf = (body) => { return match ? { kind: match[1], arg: match[2] ?? '' } : { kind: 'reply', arg: '' }; }; -export const startFakeProvider = async ({ model = 'fake-model', defaultReply = 'PONG', port = 0 } = {}) => { +export const startFakeProvider = async ({ + model = 'fake-model', defaultReply = 'PONG', port = 0, servedModel, echoAuthorization = false, splitMultibyte = false, +} = {}) => { + const served = (requested) => (servedModel === undefined ? { model: requested || model } + : servedModel === null ? {} : { model: servedModel }); const requests = []; const open = new Set(); const server = createServer((req, res) => { @@ -54,22 +65,28 @@ export const startFakeProvider = async ({ model = 'fake-model', defaultReply = ' res.end(JSON.stringify({ error: { message: 'fake provider failure', type: 'server_error' } })); return; } - const text = directive.arg || defaultReply; + const text = echoAuthorization ? String(req.headers.authorization || '') : (directive.arg || defaultReply); const id = `chatcmpl-fake-${requests.length}`; const created = Math.floor(Date.now() / 1000); const usage = { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }; + const send = (payload) => { + const bytes = Buffer.from(payload); + const cut = splitMultibyte ? bytes.findIndex((byte) => byte >= 0x80) + 1 : 0; + if (cut <= 0) { res.end(bytes); return; } + res.write(bytes.subarray(0, cut)); + setTimeout(() => res.end(bytes.subarray(cut)), 30); + }; if (body.stream) { res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' }); - const chunk = (delta, finish = null, extra = {}) => res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model: body.model || model, choices: [{ index: 0, delta, finish_reason: finish }], ...extra })}\n\n`); - chunk({ role: 'assistant', content: '' }); - chunk({ content: text }); - chunk({}, 'stop', body.stream_options?.include_usage ? {} : { usage }); - if (body.stream_options?.include_usage) res.write(`data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, model: body.model || model, choices: [], usage })}\n\n`); - res.end('data: [DONE]\n\n'); + const chunk = (delta, finish = null, extra = {}) => `data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, ...served(body.model), choices: [{ index: 0, delta, finish_reason: finish }], ...extra })}\n\n`; + let events = chunk({ role: 'assistant', content: '' }) + chunk({ content: text }) + + chunk({}, 'stop', body.stream_options?.include_usage ? {} : { usage }); + if (body.stream_options?.include_usage) events += `data: ${JSON.stringify({ id, object: 'chat.completion.chunk', created, ...served(body.model), choices: [], usage })}\n\n`; + send(`${events}data: [DONE]\n\n`); return; } res.writeHead(200, { 'content-type': 'application/json' }); - res.end(JSON.stringify({ id, object: 'chat.completion', created, model: body.model || model, choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }], usage })); + send(JSON.stringify({ id, object: 'chat.completion', created, ...served(body.model), choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }], usage })); }); }); await new Promise((resolve) => server.listen(port, '127.0.0.1', resolve)); diff --git a/packages/seats/test/provider-proxy-pin.test.js b/packages/seats/test/provider-proxy-pin.test.js new file mode 100644 index 0000000000..df23984f56 --- /dev/null +++ b/packages/seats/test/provider-proxy-pin.test.js @@ -0,0 +1,150 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { startProviderProxy } from '../src/wire/provider-proxy.js'; +import { startFakeProvider } from '../test-support/fake-provider.js'; + +const PIN = 'deepseek-flash-1017'; +const KEY = 'test-provider-key-aaaaaaaaaaaaaaaa'; + +// `pinnedModel: null` runs the proxy with no pin at all. +const withProxy = async ({ provider: providerOptions = {}, pinnedModel = PIN } = {}, fn) => { + const dir = mkdtempSync(join(tmpdir(), 'seats-pin-')); + const provider = await startFakeProvider({ model: PIN, ...providerOptions }); + const proxy = await startProviderProxy({ + provider: { baseURL: provider.baseURL }, keyMaterial: KEY, + tokenFile: join(dir, 'token'), adapterLabel: 'test', pinnedModel: pinnedModel ?? undefined, + }); + const { readFileSync } = await import('node:fs'); + const bearer = readFileSync(proxy.tokenFile, 'utf8').trim(); + const complete = (body) => fetch(`${proxy.baseURL.replace(/\/$/, '')}/chat/completions`, { + method: 'POST', + headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json' }, + body: JSON.stringify({ messages: [{ role: 'user', content: 'hi' }], ...body }), + }); + try { return await fn({ complete, proxy, provider }); } finally { + await proxy.close(); await provider.close(); rmSync(dir, { recursive: true, force: true }); + } +}; + +// Every byte that arrived before the stream ended or was cut; a cut that lands +// before the headers flush is a failed fetch, which delivered nothing. +const received = async (pending) => { + let res; + try { res = await pending; } catch { return { status: null, text: '' }; } + const chunks = []; + const reader = res.body.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(Buffer.from(value)); + } + } catch { /* cut mid-stream */ } + return { status: res.status, text: Buffer.concat(chunks).toString('utf8') }; +}; + +// Stderr carries the proxy's refusals; keep test output clean. +const quiet = async (fn) => { + const write = process.stderr.write; + process.stderr.write = () => true; + try { return await fn(); } finally { process.stderr.write = write; } +}; + +test('the pinned model is served, stream and not: the reply passes and the verdict is clean', async () => { + await withProxy({}, async ({ complete, proxy, provider }) => { + const plain = await complete({ model: PIN }); + assert.equal(plain.status, 200); + assert.equal((await plain.json()).choices[0].message.content, 'PONG'); + const streamed = await complete({ model: PIN, stream: true }); + assert.equal(streamed.status, 200); + assert.match(await streamed.text(), /"content":"PONG"[\s\S]*\[DONE\]/); + assert.deepEqual(proxy.modelCheck(), { pinned: PIN, requested: [PIN], served: [PIN], refused: 0, failures: [] }); + assert.deepEqual(provider.requests.map((r) => r.authorization), ['present', 'present']); + }); +}); + +test('a completion for another model is refused before it reaches the provider', async () => { + await quiet(() => withProxy({}, async ({ complete, proxy, provider }) => { + const res = await complete({ model: 'deepseek-pro' }); + assert.equal(res.status, 403); + assert.equal(provider.requests.length, 0); + const missing = await complete({}); + assert.equal(missing.status, 403); + assert.equal(proxy.modelCheck().refused, 2); + })); +}); + +test('a substituted model fails the check: a whole body is withheld, a stream is cut before the chunk', async () => { + await quiet(() => withProxy({ provider: { servedModel: 'deepseek-flash-0901' } }, async ({ complete, proxy }) => { + const plain = await received(complete({ model: PIN })); + assert.equal(plain.status, 502); + assert.doesNotMatch(plain.text, /PONG|0901/); + // The first chunk already names the substitute: not one byte of it is released. + const streamed = await received(complete({ model: PIN, stream: true })); + assert.doesNotMatch(streamed.text, /PONG|0901|chat\.completion/); + const check = proxy.modelCheck(); + assert.deepEqual(check.served, ['deepseek-flash-0901']); + assert.equal(check.failures.length, 2); + assert.match(check.failures[0], /served model "deepseek-flash-0901" is not the pinned "deepseek-flash-1017"/); + })); +}); + +test('a reply that names no model cannot be verified, so it fails closed', async () => { + await quiet(() => withProxy({ provider: { servedModel: null } }, async ({ complete, proxy }) => { + const plain = await complete({ model: PIN }); + assert.equal(plain.status, 502); + const streamed = await received(complete({ model: PIN, stream: true })); + assert.doesNotMatch(streamed.text, /\[DONE\]/); + const check = proxy.modelCheck(); + assert.deepEqual(check.served, []); + assert.equal(check.failures.length, 2); + assert.match(check.failures[1], /named no model/); + })); +}); + +test('an echoed provider key is still redacted on the pinned path', async () => { + await withProxy({ provider: { echoAuthorization: true } }, async ({ complete }) => { + const plain = await complete({ model: PIN }); + const text = (await plain.json()).choices[0].message.content; + assert.equal(text, 'Bearer [redacted]'); + const streamed = await (await complete({ model: PIN, stream: true })).text(); + assert.doesNotMatch(streamed, new RegExp(KEY)); + assert.match(streamed, /Bearer \[redacted\]/); + }); +}); + +test('without a pin the proxy is unchanged: any model passes and nothing is checked', async () => { + await withProxy({ pinnedModel: null, provider: { servedModel: 'something-else' } }, async ({ complete, proxy }) => { + const res = await complete({ model: 'any-model' }); + assert.equal(res.status, 200); + assert.equal((await res.json()).model, 'something-else'); + assert.deepEqual(proxy.modelCheck(), { pinned: null, requested: [], served: [], refused: 0, failures: [] }); + }); +}); + +test('a pin must be a non-empty string', async () => { + const dir = mkdtempSync(join(tmpdir(), 'seats-pin-')); + try { + writeFileSync(join(dir, 'unused'), ''); + await assert.rejects(startProviderProxy({ + provider: { baseURL: 'http://127.0.0.1:9/v1' }, keyMaterial: KEY, tokenFile: join(dir, 't'), pinnedModel: ' ', + }), /pinnedModel must be a non-empty string/); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test('a multi-byte character split across chunks reaches the CLI intact, stream and not', async () => { + await withProxy({ provider: { splitMultibyte: true } }, async ({ complete, proxy }) => { + const plain = await complete({ model: PIN, messages: [{ role: 'user', content: '[[reply:你好,世界]]' }] }); + assert.equal(plain.status, 200); + assert.equal((await plain.json()).choices[0].message.content, '你好,世界'); + const streamed = await received(complete({ model: PIN, stream: true, messages: [{ role: 'user', content: '[[reply:你好,世界]]' }] })); + assert.equal(streamed.status, 200); + assert.match(streamed.text, /你好,世界/); + assert.doesNotMatch(streamed.text, /\uFFFD/); + assert.equal(proxy.modelCheck().failures.length, 0); + }); +}); +