diff --git a/backend/__tests__/unit/services/nativeRuntimeService.hostedEngine.test.js b/backend/__tests__/unit/services/nativeRuntimeService.hostedEngine.test.js index 4a9370de48..58f466e10b 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.hostedEngine.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.hostedEngine.test.js @@ -56,6 +56,27 @@ jest.mock('../../../services/nativeOutsiderGrantPolicy', () => ({ podHasOutsiderWritableGrant: jest.fn().mockResolvedValue(false), })); +// Whether this instance holds a search key and a sandbox key. Off by default, +// so the arms below that do not name them see an instance with neither. +let mockSearchConfigured = false; +let mockSandboxConfigured = false; +const mockSearchWeb = jest.fn(); +jest.mock('../../../services/webSearchService', () => ({ + isSearchConfigured: () => mockSearchConfigured, + searchWeb: (...args) => mockSearchWeb(...args), +})); +// A turn offered search asks whether the pod is bridged to an external chat; +// these pods are not. +jest.mock('../../../services/outsiderGrantPodEligibility', () => ({ + podIsBridged: jest.fn().mockResolvedValue(false), +})); +const mockSandboxRun = jest.fn(); +const mockSandboxClose = jest.fn(); +jest.mock('../../../services/turnSandboxService', () => ({ + isSandboxConfigured: () => mockSandboxConfigured, + TurnSandbox: jest.fn().mockImplementation(() => ({ run: mockSandboxRun, close: mockSandboxClose })), +})); + jest.mock('../../../models/Message', () => { const chain = { sort: () => chain, limit: () => chain, select: () => chain, populate: () => chain, lean: () => Promise.resolve([]), @@ -82,6 +103,21 @@ const hostedInstall = (hosted = { engine: 'native', provisionedAt: new Date(), a const mention = { type: 'chat.mention', eventId: '507f1f77bcf86cd799439012', payload: { content: '@maya-agent what did we decide?' } }; +const BASE_TOOLS = [ + 'commonly_create_task', 'commonly_post_message', 'commonly_read_context', 'commonly_read_memory', 'commonly_write_memory', +]; +const call = (id, name, args) => ({ id, type: 'function', function: { name, arguments: JSON.stringify(args) } }); +const toolCalls = (...calls) => ({ + status: 200, + headers: {}, + data: { + choices: [{ message: { role: 'assistant', content: null, tool_calls: calls } }], + usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 }, + }, +}); +const offeredIn = (callIndex) => axios.post.mock.calls[callIndex][1].tools.map((tool) => tool.function.name).sort(); +const systemPromptIn = (callIndex) => axios.post.mock.calls[callIndex][1].messages[0].content; + describe('nativeRuntimeService — hosted installs on the native engine', () => { const previous = { baseUrl: process.env.LITELLM_BASE_URL, @@ -89,15 +125,22 @@ describe('nativeRuntimeService — hosted installs on the native engine', () => turns: process.env.HOSTED_TURNS_PER_DAY, }; + let savedRun; beforeEach(() => { jest.clearAllMocks(); process.env.LITELLM_BASE_URL = 'http://litellm.test'; process.env.LITELLM_MASTER_KEY = 'test-key'; delete process.env.HOSTED_TURNS_PER_DAY; + mockSearchConfigured = false; + mockSandboxConfigured = false; + mockSearchWeb.mockResolvedValue({ provider: 'exa', results: [{ title: 'Node 26', url: 'https://example.com/node-26', text: 'Released.' }] }); + mockSandboxRun.mockResolvedValue({ ok: true, stdout: '4\n', stderr: '' }); + mockSandboxClose.mockResolvedValue(0); AgentRun.countDocuments.mockResolvedValue(0); - AgentRun.create.mockImplementation(async (doc) => ({ - _id: 'run-1', ...doc, save: jest.fn().mockResolvedValue(undefined), - })); + AgentRun.create.mockImplementation(async (doc) => { + savedRun = { _id: 'run-1', ...doc, save: jest.fn().mockResolvedValue(undefined) }; + return savedRun; + }); axios.post.mockResolvedValue({ status: 200, data: { @@ -127,12 +170,88 @@ describe('nativeRuntimeService — hosted installs on the native engine', () => agentName: 'maya-agent', userId: OWNER, trigger: expect.any(String), })); const body = axios.post.mock.calls[0][1]; - expect(body.tools.map((tool) => tool.function.name).sort()).toEqual([...HOSTED_NATIVE_TOOLS].sort()); + expect(offeredIn(0)).toEqual(BASE_TOOLS); expect(body.messages[0].role).toBe('system'); expect(body.messages[0].content).toMatch(/You are Maya, a hosted agent on Commonly/); expect(body.messages[0].content).toMatch(/Keeps the research log\./); }); + // Search and code runs are on every plan (Cofounder, 2026-10-16), offered + // only where this instance holds the key, and bounded per turn. + describe('web search and code runs', () => { + test('are on the hosted list, and an instance with both keys offers both', async () => { + mockSearchConfigured = true; + mockSandboxConfigured = true; + expect(HOSTED_NATIVE_TOOLS).toEqual(expect.arrayContaining(['commonly_web_search', 'commonly_run_code'])); + + await runAgent(hostedInstall(), mention); + + expect(offeredIn(0)).toEqual([...BASE_TOOLS, 'commonly_run_code', 'commonly_web_search'].sort()); + const prompt = systemPromptIn(0); + expect(prompt).toMatch(/Use commonly_web_search for anything current/); + expect(prompt).toMatch(/Use commonly_run_code to calculate/); + expect(prompt).toMatch(/for example, opening a link someone shares\)/); + }); + + test('an instance with neither key offers neither, and the prompt does not claim them', async () => { + await runAgent(hostedInstall(), mention); + + expect(offeredIn(0)).toEqual(BASE_TOOLS); + const prompt = systemPromptIn(0); + expect(prompt).not.toMatch(/commonly_web_search|commonly_run_code/); + expect(prompt).toMatch(/for example, running code, searching the web or opening a link someone shares\)/); + }); + + test('with one key, only that tool is offered and named', async () => { + mockSearchConfigured = true; + + await runAgent(hostedInstall(), mention); + + expect(offeredIn(0)).toEqual([...BASE_TOOLS, 'commonly_web_search'].sort()); + const prompt = systemPromptIn(0); + expect(prompt).toMatch(/Use commonly_web_search/); + expect(prompt).not.toMatch(/commonly_run_code/); + expect(prompt).toMatch(/for example, running code or opening a link someone shares\)/); + }); + + test('a hosted turn stops at three searches and records them for the spend alarm', async () => { + mockSearchConfigured = true; + axios.post + .mockResolvedValueOnce(toolCalls( + call('c1', 'commonly_web_search', { query: 'a' }), + call('c2', 'commonly_web_search', { query: 'b' }), + call('c3', 'commonly_web_search', { query: 'c' }), + call('c4', 'commonly_web_search', { query: 'd' }), + )); + + await runAgent(hostedInstall(), mention); + + expect(mockSearchWeb.mock.calls.map(([query]) => query)).toEqual(['a', 'b', 'c']); + expect(savedRun.webSearches).toBe(3); + }); + + test('a hosted code run goes through the turn sandbox, whose seconds are recorded for the spend alarm', async () => { + mockSandboxConfigured = true; + mockSandboxClose.mockResolvedValue(7); + axios.post + .mockResolvedValueOnce(toolCalls(call('c1', 'commonly_run_code', { language: 'python', code: 'print(2 + 2)' }))); + + await runAgent(hostedInstall(), mention); + + expect(mockSandboxRun).toHaveBeenCalledTimes(1); + expect(mockSandboxClose).toHaveBeenCalledTimes(1); + expect(savedRun.sandboxSeconds).toBe(7); + }); + + test('the install cannot add them when the instance has no key', async () => { + const install = hostedInstall(undefined, { tools: ['commonly_web_search', 'commonly_run_code'] }); + + await runAgent(install, mention); + + expect(offeredIn(0)).toEqual(BASE_TOOLS); + }); + }); + test('in a pod holding an outsider-writable grant, a hosted turn is narrowed like any native one', async () => { const { podHasOutsiderWritableGrant } = require('../../../services/nativeOutsiderGrantPolicy'); podHasOutsiderWritableGrant.mockResolvedValue(true); diff --git a/backend/services/hostedEngine.ts b/backend/services/hostedEngine.ts index 0492fd1062..b0e98882aa 100644 --- a/backend/services/hostedEngine.ts +++ b/backend/services/hostedEngine.ts @@ -18,6 +18,8 @@ * Every site that has to tell the two engines apart asks this module. */ import { isHostedInstallation, readRuntimeConfig } from './hostedRuntimeService'; +import { isSandboxConfigured } from './turnSandboxService'; +import { isSearchConfigured } from './webSearchService'; export const HOSTED_ENGINE_NATIVE = 'native'; export const HOSTED_ENGINE_DURABLE_OBJECT = 'durable-object'; @@ -149,20 +151,59 @@ export const hostedNativeDecline = (eventType: string, turnsPerDay: number): str return null; }; -/** The Commonly tools a hosted agent gets. Broker grants are appended per run. */ +/** + * The Commonly tools a hosted agent gets. Broker grants are appended per run. + * + * Web search and code runs are on every plan, free included (Cofounder, + * 2026-10-16): the out-of-box test is a stranger watching the agent do the + * work in its first minute, and a tool held back fails it. The native runtime + * still offers each only where this instance holds its key, and bounds it per + * turn: 3 searches (nativeRuntimeService), and 5 code runs in one sandbox that + * lives at most 3 minutes (turnSandboxService). Both are recorded on the + * AgentRun and priced into the per-user spend alarm. + */ export const HOSTED_NATIVE_TOOLS: readonly string[] = [ 'commonly_read_context', 'commonly_read_memory', 'commonly_write_memory', 'commonly_post_message', 'commonly_create_task', + 'commonly_web_search', + 'commonly_run_code', ]; /** Recent messages a hosted turn sees, as Scout does. */ export const HOSTED_NATIVE_RECENT_MESSAGES = 8; -export const hostedNativeSystemPrompt = (displayName: string, about?: string): string => { +/** The opt-in tools this instance can actually offer a hosted turn. */ +export interface HostedNativeAbilities { + webSearch: boolean; + runCode: boolean; +} + +const orList = (items: string[]): string => ( + items.length < 2 ? items.join('') : `${items.slice(0, -1).join(', ')} or ${items[items.length - 1]}` +); + +// The prompt names a tool only when the turn can be offered it, so an +// instance without a search or sandbox key never tells its agent it can +// search or run code. Opening a link stays on the "say so" list until a fetch +// tool exists (TASK-267b). +export const hostedNativeSystemPrompt = ( + displayName: string, + about?: string, + abilities: HostedNativeAbilities = { webSearch: false, runCode: false }, +): string => { const persona = about && about.trim() ? ` ${about.trim()}` : ''; + const uses = [ + abilities.webSearch ? 'Use commonly_web_search for anything current or outside the pod.' : '', + abilities.runCode ? 'Use commonly_run_code to calculate or check something by running it instead of guessing.' : '', + ].filter(Boolean); + const missing = [ + abilities.runCode ? '' : 'running code', + abilities.webSearch ? '' : 'searching the web', + 'opening a link someone shares', + ].filter(Boolean); return ( `You are ${displayName}, a hosted agent on Commonly. Commonly runs you for the person who ` + `set you up, in a shared pod with humans and other agents.${persona}\n` @@ -170,11 +211,11 @@ export const hostedNativeSystemPrompt = (displayName: string, about?: string): s + 'Reply to the message you were mentioned in by calling commonly_post_message once: concise, ' + 'useful, and addressed to the pod. Use commonly_read_context when the mention alone is not ' + 'enough. Keep durable facts with commonly_write_memory, and track work with ' - + 'commonly_create_task. If no reply is genuinely needed, do not post.\n' + + `commonly_create_task.${uses.length ? ` ${uses.join(' ')}` : ''} If no reply is genuinely ` + + 'needed, do not post.\n' + '\n' + 'Only claim what you actually did. If a task needs a tool you do not have (for example, ' - + 'running code or browsing the web), say so plainly instead of describing a result you did ' - + 'not observe.' + + `${orList(missing)}), say so plainly instead of describing a result you did not observe.` ); }; @@ -194,7 +235,10 @@ export const hostedNativeRunConfig = (installation: any, turnsPerDay: number): R return { runtime: { runtimeType: 'hosted' }, hosted: { engine: HOSTED_ENGINE_NATIVE }, - systemPrompt: hostedNativeSystemPrompt(displayName, about), + systemPrompt: hostedNativeSystemPrompt(displayName, about, { + webSearch: isSearchConfigured(), + runCode: isSandboxConfigured(), + }), tools: [...HOSTED_NATIVE_TOOLS], recentMessages: HOSTED_NATIVE_RECENT_MESSAGES, // The owner's plan line (TASK-268: 30 free, 200 pro), enforced through the