Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 182 additions & 3 deletions cli/__tests__/adapters.claude.environment.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ const fakeChild = ({ stdout = '', code = 0 } = {}) => {
return proc;
};

const makeSpawnImpl = ({ code = 0 } = {}) => {
const makeSpawnImpl = ({ code = 0, stdout = 'ok' } = {}) => {
const calls = [];
const impl = (cmd, args, opts) => {
const configIndex = args.indexOf('--mcp-config');
Expand All @@ -56,11 +56,17 @@ const makeSpawnImpl = ({ code = 0 } = {}) => {
configMode: configPath ? fs.statSync(configPath).mode & 0o777 : null,
configDirMode: configPath ? fs.statSync(path.dirname(configPath)).mode & 0o777 : null,
});
return fakeChild({ stdout: 'ok', code });
return fakeChild({ stdout, code });
};
return { impl, calls };
};

// A public seat's turn posts only on a stream whose init shows Commonly connected (TASK-256).
const COMMONLY_CONNECTED = [
{ type: 'system', subtype: 'init', mcp_servers: [{ name: 'commonly', status: 'connected' }] },
{ type: 'result', subtype: 'success', is_error: false, result: 'ok' },
].map((event) => JSON.stringify(event)).join('\n');

describe('claude adapter — ctx.environment', () => {
let cwd;
beforeEach(() => {
Expand Down Expand Up @@ -636,7 +642,7 @@ describe('claude adapter — ctx.environment', () => {
fs.writeFileSync(path.join(home, 'current'), '0.3.13\n');
fs.writeFileSync(path.join(home, '.warm.lock'), 'warming\n');
const prior = process.env.COMMONLY_MCP_HOME;
const { impl, calls } = makeSpawnImpl();
const { impl, calls } = makeSpawnImpl({ stdout: COMMONLY_CONNECTED });
try {
process.env.COMMONLY_MCP_HOME = home;
Object.defineProperty(process, 'platform', { value: 'darwin' });
Expand Down Expand Up @@ -1764,3 +1770,176 @@ describe('claude adapter — a workspace seat\'s git tool', () => {
expect(warn).toHaveBeenCalledWith(expect.stringContaining('git was not found'));
});
});

// TASK-256: a public seat posts only when its own stream shows the Commonly MCP
// working. The stream shapes are the ones Claude Code 2.1.314 wrote against a
// stub MCP server: `mcp_servers[].status` in init, and a failed call as a
// `tool_result` with `is_error: true` whose content is the server's text.
describe('claude adapter: the public seat Commonly MCP witness (TASK-256)', () => {
const COMMONLY_MCP = [{ name: 'commonly', transport: 'stdio', command: ['npx', '-y', '@commonlyai/mcp@latest'] }];
const stream = (...events) => events.map((event) => JSON.stringify(event)).join('\n');
const init = (status) => ({ type: 'system', subtype: 'init', mcp_servers: [{ name: 'commonly', status }] });
const call = (id, name) => ({ type: 'assistant', message: { content: [{ type: 'tool_use', id, name, input: {} }] } });
const failed = (id, content) => ({
type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, is_error: true, content }] },
});
const succeeded = (id, content = '[]') => ({
type: 'user', message: { content: [{ type: 'tool_result', tool_use_id: id, content }] },
});
const result = (text) => ({ type: 'result', subtype: 'success', is_error: false, result: text });
const API_503 = JSON.stringify({ status: 503, body: '<html>503 Service Unavailable</html>', message: 'HTTP 503' });
const API_404 = JSON.stringify({ status: 404, body: { error: 'Pod not found' }, message: 'Pod not found' });

let cwd;
let home;
let publicState;
let prior;
const originalPlatform = process.platform;
beforeEach(() => {
cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'cli-claude-witness-'));
publicState = fs.mkdtempSync(path.join(os.tmpdir(), 'cli-claude-public-state-'));
home = fs.mkdtempSync(path.join(os.tmpdir(), 'cli-claude-mcp-home-'));
const pkgDir = path.join(home, '0.3.13', 'node_modules', '@commonlyai', 'mcp');
fs.mkdirSync(path.join(pkgDir, 'src'), { recursive: true });
fs.writeFileSync(path.join(pkgDir, 'package.json'), JSON.stringify({
name: '@commonlyai/mcp', version: '0.3.13', bin: { 'commonly-mcp': 'src/index.js' },
}));
fs.writeFileSync(path.join(pkgDir, 'src', 'index.js'), '');
fs.writeFileSync(path.join(home, 'current'), '0.3.13\n');
// A fresh registry answer naming the installed build, so no turn here starts
// a background warm: a detached `npm install` outlives the suite and writes
// into TMPDIR after it ends (the CLI temp-cleanup guard caught it).
fs.writeFileSync(path.join(home, '.registry.json'), JSON.stringify({ version: '0.3.13', checkedAt: Date.now() }));
prior = process.env.COMMONLY_MCP_HOME;
process.env.COMMONLY_MCP_HOME = home;
Object.defineProperty(process, 'platform', { value: 'darwin' });
spawnSync.mockImplementation((cmd) => (cmd === 'which'
? { status: 0, stdout: '/usr/bin/true\n' } : { status: 0, stdout: '' }));
});
afterEach(() => {
if (prior === undefined) delete process.env.COMMONLY_MCP_HOME;
else process.env.COMMONLY_MCP_HOME = prior;
Object.defineProperty(process, 'platform', { value: originalPlatform });
spawnSync.mockReset();
for (const dir of [cwd, publicState, home]) fs.rmSync(dir, { recursive: true, force: true });
});

const turn = (stdout, { trust = 'public', mcp = COMMONLY_MCP, impl = makeSpawnImpl({ stdout }).impl, sessionId } = {}) => (
claude.spawn('hi', {
agentName: 'public-test', cwd, ...(sessionId ? { sessionId } : {}),
env: { PATH: process.env.PATH, USER: 'safe-user', LOGNAME: 'safe-user' },
environment: { model: 'test-model', ...(trust ? { sandbox: { mode: 'workspace', trust } } : {}), mcp },
_publicClaudeState: publicState,
_spawnImpl: impl,
})
);

test('posts when init shows Commonly connected and no Commonly call failed', async () => {
await expect(turn(stream(init('connected'), result('Here you go.')))).resolves.toMatchObject({ text: 'Here you go.' });
});

test.each([
['the server failed to start', stream(init('failed'), result('HELLO')), /status failed/],
['the stream carries no init event', stream(result('HELLO')), /no init event/],
['the output is plain text', 'HELLO', /no init event/],
])('withholds the turn when %s', async (_label, stdout, reason) => {
const outcome = turn(stdout);
await expect(outcome).rejects.toMatchObject({ code: 'public_mcp_not_ready' });
await expect(outcome).rejects.toThrow(reason);
});

test.each([
['an API 503', API_503],
['a transport error with no status', 'MCP error -32000: Connection closed'],
['an empty error', ''],
['an error as text parts', [{ type: 'text', text: API_503 }]],
])('withholds the turn when a Commonly call fails with %s', async (_label, content) => {
const stdout = stream(init('connected'), call('t1', 'mcp__commonly__list_pods'), failed('t1', content), result('Every call is failing.'));
await expect(turn(stdout)).rejects.toMatchObject({ code: 'public_tool_failed' });
});

// `pending` at init is not a failure: the server may finish starting during
// the turn. Only a Commonly call that returned shows it working, so that is
// the evidence a pending server needs.
test('posts when init shows Commonly pending and a Commonly call then returned', async () => {
const stdout = stream(init('pending'), call('t1', 'mcp__commonly__list_pods'), succeeded('t1'), result('Two pods.'));
await expect(turn(stdout)).resolves.toMatchObject({ text: 'Two pods.' });
});

test.each([
['no Commonly call was made', stream(init('pending'), result('HELLO'))],
['only another server\'s call returned', stream(init('pending'), call('t1', 'mcp__other__ping'), succeeded('t1'), result('HELLO'))],
])('withholds the turn when init shows Commonly pending and %s', async (_label, stdout) => {
const outcome = turn(stdout);
await expect(outcome).rejects.toMatchObject({ code: 'public_mcp_not_ready' });
await expect(outcome).rejects.toThrow(/status pending/);
});

test('withholds a pending server whose Commonly call failed, even beside one that returned', async () => {
const stdout = stream(
init('pending'),
call('t1', 'mcp__commonly__list_pods'), succeeded('t1'),
call('t2', 'mcp__commonly__read_context'), failed('t2', API_503),
result('Every call is failing.'),
);
await expect(turn(stdout)).rejects.toMatchObject({ code: 'public_tool_failed' });
});

test('posts an answer to a refusal the person can be told about (a 404)', async () => {
const stdout = stream(init('connected'), call('t1', 'mcp__commonly__read_context'), failed('t1', API_404), result('I cannot see that pod.'));
await expect(turn(stdout)).resolves.toMatchObject({ text: 'I cannot see that pod.' });
});

test('withholds a failed call from a server whose name Claude Code normalises in tool names', async () => {
const mcp = [{ ...COMMONLY_MCP[0], name: 'commonly.me' }];
const stdout = stream(
{ type: 'system', subtype: 'init', mcp_servers: [{ name: 'commonly.me', status: 'connected' }] },
call('t1', 'mcp__commonly_me__list_pods'), failed('t1', API_503), result('Every call is failing.'),
);
await expect(turn(stdout, { mcp })).rejects.toMatchObject({ code: 'public_tool_failed' });
});

test('the fresh-session retry is witnessed too', async () => {
// The resumed session is refused, the retry runs fresh, and its server failed.
let spawns = 0;
const impl = () => {
spawns += 1;
const proc = new EventEmitter();
proc.stdout = new EventEmitter();
proc.stderr = new EventEmitter();
proc.kill = jest.fn();
const first = spawns === 1;
setTimeout(() => {
if (first) proc.stderr.emit('data', Buffer.from('Error: Session ID abc-123 is already in use.'));
else proc.stdout.emit('data', Buffer.from(stream(init('failed'), result('HELLO'))));
proc.emit('close', first ? 1 : 0);
}, 0);
return proc;
};
await expect(turn('', { impl, sessionId: 'abc-123' })).rejects.toMatchObject({ code: 'public_mcp_not_ready' });
expect(spawns).toBe(2);
});

test("ignores another server's failed call", async () => {
const stdout = stream(init('connected'), call('t1', 'mcp__other__lookup'), failed('t1', API_503), result('Done.'));
await expect(turn(stdout)).resolves.toMatchObject({ text: 'Done.' });
});

// As in the OpenCode check: only the shipped server's error contract is known,
// so a server run some other way is not judged by it.
test('a seat that does not run the shipped Commonly MCP is not witnessed', async () => {
// A hand-run copy, as the staging seats run: not the published package.
const mcp = [{ name: 'commonly', transport: 'stdio', command: ['node', '/opt/commonly-mcp-staging/src/index.js'] }];
await expect(turn(stream(init('failed'), result('HELLO')), { mcp })).resolves.toMatchObject({ text: 'HELLO' });
});

// A record still carrying `internal` is read as public (environment.js), so it is witnessed.
test('a legacy internal-trust seat is witnessed as public', async () => {
await expect(turn(stream(init('failed'), result('HELLO')), { trust: 'internal' }))
.rejects.toMatchObject({ code: 'public_mcp_not_ready' });
});

test.each([['a workspace seat', 'workspace'], ['an unconfined seat', null]])('%s is not witnessed', async (_label, trust) => {
await expect(turn(stream(init('failed'), result('HELLO')), { trust })).resolves.toMatchObject({ text: 'HELLO' });
});
});
34 changes: 34 additions & 0 deletions cli/__tests__/spawn-retry.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -322,3 +322,37 @@ describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)',
});
});
});

// TASK-256: a public seat's withheld turn carries the tool's own error text in
// its message, so a vendor that says "content_policy_violation", "quota" or
// "403 forbidden" must not decide the class. The code does: the event stays
// unacked and is retried, never acked as refused.
describe('a public witness failure is a runtime failure, whatever its text says', () => {
test.each([
['public_tool_failed', '{"status":500,"message":"content_policy_violation"}'],
['public_tool_failed', 'this content was flagged'],
['public_tool_failed', 'insufficient_quota'],
['public_tool_failed', '403 forbidden'],
['public_mcp_not_ready', 'connection reset'],
])('%s with "%s" classes as runtime', (code, reason) => {
const error = new Error(`public seat withheld this turn's reply: Commonly tool x failed (${reason})`);
error.code = code;
expect(classifySpawnFailure(error)).toBe(SPAWN_FAILURE_CLASS.RUNTIME);
});

// A throttle is still a throttle: it keeps the event unacked like runtime, and
// the longer backoff is the point, since a quick retry meets the same 429.
test.each([
['public_tool_failed', '{"status":429,"message":"Too Many Requests"}'],
['public_mcp_not_ready', 'rate limit while connecting'],
])('%s with "%s" keeps the rate-limit backoff', (code, reason) => {
const error = new Error(`public seat withheld this turn's reply: Commonly tool x failed (${reason})`);
error.code = code;
expect(classifySpawnFailure(error)).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT);
});

test('positive control: the same text without the code still classes as a refusal', () => {
expect(classifySpawnFailure(new Error('{"status":500,"message":"content_policy_violation"}')))
.toBe(SPAWN_FAILURE_CLASS.CONTENT_REFUSED);
});
});
Loading
Loading