Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions backend/__tests__/unit/config/nativeAgents.scoutKnowledge.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -77,4 +77,14 @@ describe('Scout onboarding knowledge', () => {
expect(prompt).toMatch(/at most one question per turn/);
expect(prompt).not.toMatch(/within the hour|within an hour|you will hear back|you'll hear/i);
});

// QA 2026-10-16: Scout said "200 turns a day" while the card and the meter
// said 30, because the prompt kept its own copy of the plan line. The limits
// now reach Scout only through the hosted offer's description.
test('quotes no hosted limit from memory, and hands off to Run it here with a button', () => {
expect(prompt).not.toMatch(/\d+\s+turns (a|per) day/i);
expect(prompt).toContain('commonly_offer_hosted_agent');
expect(scoutApp.tools).toContain('commonly_offer_hosted_agent');
expect(prompt).toMatch(/with no invitation code/);
});
});
2 changes: 2 additions & 0 deletions backend/__tests__/unit/routes/hosted.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ jest.mock('../../../services/hostedRuntimeService', () => mockHosted);
const mockNativeAvailable = jest.fn(() => false);
jest.mock('../../../services/nativeRuntimeService', () => ({
isNativeRuntimeAvailable: (...args) => mockNativeAvailable(...args),
// The real rule, over this file's two switches.
hostedEngineAvailable: (engine) => (engine === 'native' ? mockNativeAvailable() : mockHosted.isConfigured()),
}));
const mockMarkByoOpened = jest.fn();
jest.mock('../../../services/onboardingMarks', () => ({ markByoOpened: (...args) => mockMarkByoOpened(...args) }));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
/**
* Scout's hosted offer (QA 2026-10-16): asked "can Commonly host it?", Scout
* answered with text directions and no link, while the Bring-your-own path got
* a card. commonly_offer_hosted_agent posts the model's reply with one
* server-composed button to Run it here, aimed at this pod.
*/
jest.mock('axios', () => ({ __esModule: true, default: { post: jest.fn() } }));
jest.mock('../../../models/Pod', () => ({
findById: jest.fn(() => ({ lean: jest.fn().mockResolvedValue({ name: 'My Workspace' }) })),
}));
jest.mock('../../../models/AgentRun', () => ({ create: jest.fn(), countDocuments: jest.fn() }));
jest.mock('../../../models/User', () => ({
findOne: jest.fn(() => ({ select: () => ({ lean: () => Promise.resolve(null) }) })),
}));
jest.mock('../../../services/agentTypingService', () => ({ emitAgentTypingStart: jest.fn(), emitAgentTypingStop: jest.fn() }));
jest.mock('../../../services/agentMessageService', () => ({ postMessage: jest.fn().mockResolvedValue({ success: true }) }));
jest.mock('../../../services/messageClaimService', () => ({
claim: jest.fn().mockResolvedValue({ claimed: true, expiresAt: new Date() }),
release: jest.fn().mockResolvedValue({ released: true }),
}));
jest.mock('../../../services/nativeOutsiderGrantPolicy', () => ({
podHasOutsiderWritableGrant: jest.fn().mockResolvedValue(false),
}));
const mockListHosted = jest.fn();
jest.mock('../../../services/hostedRuntimeService', () => ({
...jest.requireActual('../../../services/hostedRuntimeService'),
listHostedInstallationsForUser: (...args) => mockListHosted(...args),
}));

const axios = require('axios').default;
const AgentRun = require('../../../models/AgentRun');
const AgentMessageService = require('../../../services/agentMessageService');
const { runAgent } = require('../../../services/nativeRuntimeService');

const POD = '507f1f77bcf86cd799439011';
const OWNER = '507f1f77bcf86cd799439099';
const scoutInstall = {
podId: POD,
agentName: 'scout',
instanceId: 'u1234567890',
displayName: 'Scout',
installedBy: OWNER,
config: { runtime: { runtimeType: 'native' }, tools: ['commonly_post_message', 'commonly_offer_hosted_agent'] },
};
const ask = {
type: 'chat.mention',
eventId: '507f1f77bcf86cd799439012',
payload: { content: 'I\'d rather not install anything. Can Commonly host it for me?' },
};
const REPLY = 'Yes. Commonly can run it for you, nothing to install.';

const callsTool = (name, args) => ({
status: 200,
headers: {},
data: {
choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name, arguments: JSON.stringify(args) } }] } }],
usage: { prompt_tokens: 10, completion_tokens: 8, total_tokens: 18 },
},
});
const says = (content) => ({
status: 200,
headers: {},
data: { choices: [{ message: { role: 'assistant', content } }], usage: { prompt_tokens: 12, completion_tokens: 2, total_tokens: 14 } },
});

describe('commonly_offer_hosted_agent', () => {
const savedEnv = { ...process.env };
beforeEach(() => {
jest.clearAllMocks();
process.env.LITELLM_BASE_URL = 'http://litellm.test';
process.env.LITELLM_MASTER_KEY = 'test-key';
process.env.HOSTED_ENGINE = 'native';
AgentRun.countDocuments.mockResolvedValue(0);
AgentRun.create.mockImplementation(async (doc) => ({ _id: 'run-1', ...doc, save: jest.fn().mockResolvedValue(undefined) }));
mockListHosted.mockResolvedValue([]);
});
afterAll(() => { process.env = savedEnv; });

test('posts the reply once, with one Run it here button aimed at this pod with hosting chosen', async () => {
axios.post
.mockResolvedValueOnce(callsTool('commonly_offer_hosted_agent', { content: REPLY }))
.mockResolvedValueOnce(says('Posted the button above.'));

const result = await runAgent(scoutInstall, ask);

expect(result.status).toBe('succeeded');
expect(mockListHosted).toHaveBeenCalledWith(OWNER);
// The narration after the tool is not posted beside it.
expect(AgentMessageService.postMessage).toHaveBeenCalledTimes(1);
const posted = AgentMessageService.postMessage.mock.calls[0][0];
expect(posted.content).toBe(REPLY);
expect(posted.podId).toBe(POD);
expect(posted.payload).toEqual({
kind: 'starts',
starts: [{ id: 'run-hosted', label: 'Run it here', href: `/v2/agents/byo?pod=${POD}&mode=hosted` }],
});
// The thread renders a start's href only as an in-app /v2/ path.
expect(posted.payload.starts[0].href).toMatch(/^\/v2\/[^/\\]/);
});

test('posts nothing when the person already holds the beta\'s hosted agent, and names it', async () => {
mockListHosted.mockResolvedValue([{ agentName: 'lee-agent', instanceId: 'default', podId: POD }]);
axios.post
.mockResolvedValueOnce(callsTool('commonly_offer_hosted_agent', { content: REPLY }))
.mockResolvedValueOnce(says('You already have lee-agent. Mention it here.'));

await runAgent(scoutInstall, ask);

const toolMessage = axios.post.mock.calls[1][1].messages.find((m) => m.role === 'tool');
expect(toolMessage.content).toMatch(/already have 1 hosted agent/);
expect(toolMessage.content).toContain('lee-agent');
// Only the model's own follow-up reached the pod, with no button.
expect(AgentMessageService.postMessage).toHaveBeenCalledTimes(1);
expect(AgentMessageService.postMessage.mock.calls[0][0].payload).toBeUndefined();
});

test('is not offered where the instance cannot host, so no dead button is posted', async () => {
delete process.env.HOSTED_ENGINE;
delete process.env.HOSTED_RUNTIME_URL;
delete process.env.HOSTED_RUNTIME_ADMIN_TOKEN;
axios.post
.mockResolvedValueOnce(callsTool('commonly_offer_hosted_agent', { content: REPLY }))
.mockResolvedValueOnce(says('Hosting is not available here.'));

await runAgent(scoutInstall, ask);

const offered = axios.post.mock.calls[0][1].tools.map((tool) => tool.function.name);
expect(offered).not.toContain('commonly_offer_hosted_agent');
const toolMessage = axios.post.mock.calls[1][1].messages.find((m) => m.role === 'tool');
expect(toolMessage.content).toMatch(/not available to you in this turn/);
expect(AgentMessageService.postMessage.mock.calls.every(([arg]) => arg.payload === undefined)).toBe(true);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,37 @@
* filters exactly; an undeclared (legacy) config keeps the pre-gate surface
* MINUS approval proposing, which is opt-in by declaration only.
*/
const { toolsForConfig } = require('../../../services/nativeRuntimeService');
const { toolsForConfig, TOOLS } = require('../../../services/nativeRuntimeService');
const { hostedCaps } = require('../../../services/hostedRuntimeService');

const names = (tools) => tools.map((t) => t.function.name);

// Runs fn with these env vars set (undefined deletes), then restores them.
const withEnv = (vars, fn) => {
const saved = Object.fromEntries(Object.keys(vars).map((k) => [k, process.env[k]]));
try {
Object.entries(vars).forEach(([k, v]) => {
if (v === undefined) delete process.env[k];
else process.env[k] = v;
});
return fn();
} finally {
Object.entries(saved).forEach(([k, v]) => {
if (v === undefined) delete process.env[k];
else process.env[k] = v;
});
}
};
const NATIVE_HOSTING = { HOSTED_ENGINE: 'native', LITELLM_BASE_URL: 'http://litellm.test', LITELLM_MASTER_KEY: 'k' };
const NO_HOSTING = {
HOSTED_ENGINE: undefined,
HOSTED_RUNTIME_URL: undefined,
HOSTED_RUNTIME_ADMIN_TOKEN: undefined,
LITELLM_BASE_URL: undefined,
LITELLM_MASTER_KEY: undefined,
LITELLM_API_KEY: undefined,
};

describe('toolsForConfig', () => {
test('a declared allowlist filters exactly', () => {
const tools = toolsForConfig({ tools: ['commonly_post_message', 'commonly_propose_action'] });
Expand All @@ -29,13 +56,43 @@ describe('toolsForConfig', () => {
test('the scout manifest grants propose_action', () => {
// eslint-disable-next-line global-require
const { scoutApp } = require('../../../config/native-agents/scout');
const tools = toolsForConfig({ tools: [...scoutApp.tools] });
// On an instance that can host, so the hosted offer is offered too.
const tools = withEnv(NATIVE_HOSTING, () => toolsForConfig({ tools: [...scoutApp.tools] }));
expect(names(tools)).toContain('commonly_propose_action');
// The manifest and the runtime's schema must agree — a declared tool with
// no schema would silently vanish here.
expect(tools.length).toBe(scoutApp.tools.length);
});

test('the hosted offer is opt-in, and offered only where the instance can host', () => {
const declared = { tools: ['commonly_offer_hosted_agent'] };
withEnv(NO_HOSTING, () => {
expect(names(toolsForConfig(declared))).not.toContain('commonly_offer_hosted_agent');
});
withEnv(NATIVE_HOSTING, () => {
expect(names(toolsForConfig(declared))).toContain('commonly_offer_hosted_agent');
// Never on the default surface, even where hosting works.
expect(names(toolsForConfig({}))).not.toContain('commonly_offer_hosted_agent');
expect(names(toolsForConfig({ tools: ['commonly_post_message'] }))).not.toContain('commonly_offer_hosted_agent');
});
// The Durable Object engine counts when its worker is configured.
withEnv({ ...NO_HOSTING, HOSTED_RUNTIME_URL: 'https://worker.test', HOSTED_RUNTIME_ADMIN_TOKEN: 't' }, () => {
expect(names(toolsForConfig(declared))).toContain('commonly_offer_hosted_agent');
});
});

// Scout told strangers "200 turns a day" after the free line became 30
// (#2237): the number lived in a copy. The offer's description is built from
// the cap source, so the model is told the line the meter enforces.
test('the hosted offer states the limits the meter enforces', () => {
const offer = TOOLS.find((t) => t.function.name === 'commonly_offer_hosted_agent');
const free = hostedCaps();
const pro = hostedCaps({ pro: true });
expect(offer.function.description).toContain(`${free.turnsPerDay} turns a day (${pro.turnsPerDay} on Pro)`);
expect(offer.function.description).toContain(`${free.agentsPerUser} hosted agent${free.agentsPerUser === 1 ? '' : 's'} per account`);
expect(offer.function.description).toMatch(/no invitation code/);
});

test('agent_status is declaration-gated like every other tool', () => {
// Scout declares it; a legacy config or a minimal allowlist does not
// receive it implicitly.
Expand Down
13 changes: 10 additions & 3 deletions backend/config/native-agents/scout.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,11 @@ export const scoutApp = {
+ 'your own" button (or "Set it up" on the "Your own agent" card) opens '
+ 'the "Bring your own agent" page (/v2/agents/byo). It offers:\n'
+ ' * "Run it here": Commonly hosts the agent. Open to every signed-in user '
+ 'during beta, one hosted agent per account and 200 turns a day. Nothing to '
+ 'install; it is live in about ten seconds.\n'
+ 'during beta, with no invitation code. Nothing to install; it is live a few '
+ 'seconds after the click. When someone wants it, answer with '
+ 'commonly_offer_hosted_agent: your reply arrives with a button that opens '
+ 'this choice already made. Its limits are in that tool\'s description; '
+ 'quote them from there, never from memory.\n'
+ ' * "Bring your own runtime": their own Claude Code or Codex, on their own '
+ 'machine. The page issues a token once and shows these commands, which '
+ 'are the whole setup:\n'
Expand All @@ -94,7 +97,7 @@ export const scoutApp = {
+ 'terminal.\n'
+ '- The commands above are verified on macOS. Windows is not verified yet '
+ '(the export lines are bash, and the issue is open): on Windows say so '
+ 'plainly and offer "Run it here" instead.\n'
+ 'plainly and offer "Run it here" instead, with commonly_offer_hosted_agent.\n'
+ '- The listener runs Claude Code or Codex. Do not claim it runs any other '
+ 'tool.\n'
+ '- Developers building their own agent: Commonly agents use a small HTTP '
Expand Down Expand Up @@ -154,6 +157,8 @@ export const scoutApp = {
+ 'once, without narrating it.\n'
+ '- Asked for a new pod → commonly_propose_action with actionType '
+ 'create_pod.\n'
+ '- They want Commonly to run an agent for them, or would rather not '
+ 'install anything → commonly_offer_hosted_agent.\n'
+ '- commonly_read_context before answering questions about what happened '
+ 'here.',
'Never claim the user\'s own agent is connected or working — the '
Expand All @@ -173,6 +178,8 @@ export const scoutApp = {
// via approval card only.
'commonly_propose_action',
'commonly_agent_status',
// Answers "can Commonly host it?" with a button to Run it here.
'commonly_offer_hosted_agent',
],
iconUrl: '',
categories: ['onboarding', 'utility'],
Expand Down
9 changes: 4 additions & 5 deletions backend/routes/hosted.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ const { AgentInstallation } = require('../models/AgentRegistry');
const User = require('../models/User');
const AgentIdentityService = require('../services/agentIdentityService');
const hostedRuntime = require('../services/hostedRuntimeService');
const { isNativeRuntimeAvailable } = require('../services/nativeRuntimeService');
const { hostedEngineAvailable, isNativeRuntimeAvailable } = require('../services/nativeRuntimeService');
const {
HOSTED_ENGINE_NATIVE,
engineForNewHostedInstall,
Expand Down Expand Up @@ -145,11 +145,10 @@ const resolveOwnedHostedInstallation = async (req: any, source: Record<string, a

/**
* Whether an engine can run turns on this instance: the native engine needs
* LiteLLM, the Durable Object needs the worker (services/hostedEngine.ts).
* LiteLLM, the Durable Object needs the worker. One rule, shared with Scout's
* hosted offer (nativeRuntimeService.hostedEngineAvailable).
*/
const engineAvailable = (engine: string): boolean => (
engine === HOSTED_ENGINE_NATIVE ? isNativeRuntimeAvailable() : hostedRuntime.isConfigured()
);
const engineAvailable = (engine: string): boolean => hostedEngineAvailable(engine);

/**
* Refuses before any database read when no engine can run on this instance.
Expand Down
3 changes: 2 additions & 1 deletion backend/services/hostedRuntimeService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
* makes an unmetered public runtime safe to expose:
*
* HOSTED_AGENTS_PER_USER active hosted installations per owner (default 1)
* HOSTED_TURNS_PER_DAY acked events per hosted agent per UTC day (200)
* HOSTED_TURNS_PER_DAY acked events per hosted agent per UTC day (30;
* HOSTED_TURNS_PER_DAY_PRO, 200, on Pro)
*
* The daily cap is a RATE LIMIT ON REQUESTS, not a spend meter: it counts
* acked AgentEvents by ack time (`deliveredAt`, which ack() stamps). One
Expand Down
Loading
Loading