Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/__tests__/unit/routes/hosted.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -543,6 +543,8 @@ describe('/api/hosted on the native engine', () => {
expect(res.status).toBe(200);
expect(res.body.runtime).toEqual({ engine: 'native', ready: true, provisionedAt: PROVISIONED.toISOString() });
expect(res.body.meter).toMatchObject({ used: 3, cap: 200 });
// A native turn is a run: an event acked without one (a stand-down) is not a turn.
expect(mockHosted.meterAllowsTurn).toHaveBeenCalledWith('scout', 'default', expect.any(Number), { countRuns: true });
expect(mockHosted.getAgentStatus).not.toHaveBeenCalled();
});

Expand Down
3 changes: 2 additions & 1 deletion backend/__tests__/unit/services/firstContactService.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ describe('firstContactService', () => {

const { content } = AgentEventService.enqueue.mock.calls[0][0].payload;
expect(content).toMatch(/exactly ONE specific, answerable question/);
expect(content).toMatch(/add one more line, exactly: "\(Mention me as @cto-walk so I see your answer\.\)"/);
expect(content).toMatch(/End that same message, after the question, with one more line, exactly: "\(Mention me as @cto-walk so I see your answer\.\)"/);
expect(content).toMatch(/post one message, not two/);
});

test.each([
Expand Down
19 changes: 19 additions & 0 deletions backend/__tests__/unit/services/hostedRuntimeService.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ jest.mock('../../../models/AgentRegistry', () => ({
find: (...args) => mockInstallationFind(...args),
},
}));
const mockRunCount = jest.fn();
jest.mock('../../../models/AgentRun', () => ({
countDocuments: (...args) => mockRunCount(...args),
}));
jest.mock('../../../models/AgentEvent', () => ({
countDocuments: (...args) => mockEventCount(...args),
}));
Expand All @@ -25,6 +29,7 @@ describe('hostedRuntimeService', () => {
mockInstallationCount.mockReset();
mockInstallationFind.mockReset();
mockEventCount.mockReset();
mockRunCount.mockReset();
global.fetch = jest.fn();
});
afterEach(() => {
Expand Down Expand Up @@ -96,6 +101,20 @@ describe('hostedRuntimeService', () => {
expect(select).toHaveBeenCalledWith('agentName instanceId podId');
});

// QA walk 4: an addressed turn that stood down to a peer's claim was acked
// with no run, and still showed as a used turn. On the native engine the
// meter counts runs.
it('on the native engine, meters runs since the UTC day start, leaving out the greeting', async () => {
process.env.HOSTED_TURNS_PER_DAY = '2';
mockRunCount.mockResolvedValue(1);
const meter = await hosted.meterAllowsTurn('Scout', 'default', 2, { countRuns: true });
expect(meter).toMatchObject({ allowed: true, used: 1, cap: 2 });
expect(mockEventCount).not.toHaveBeenCalled();
const filter = mockRunCount.mock.calls[0][0];
expect(filter).toMatchObject({ agentName: 'scout', instanceId: 'default', trigger: { $ne: 'first_contact' } });
expect(filter.startedAt.$gte.toISOString()).toMatch(/T00:00:00\.000Z$/);
});

it('meters acked events by ACK time (deliveredAt) since the UTC day start and reports the reset', async () => {
process.env.HOSTED_TURNS_PER_DAY = '2';
mockEventCount.mockResolvedValue(1);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@
*
* Contract under test:
* - message-shaped triggers claim payload.messageId before any run/LLM work
* - a lost CAS stands down: no AgentRun row, no LLM call, no typing, and
* the result reads as a successful (acked) no-op
* - a lost CAS on an AMBIENT trigger stands down: no AgentRun row, no LLM
* call, no typing, and the result reads as a successful (acked) no-op
* - a lost CAS while DIRECTLY ADDRESSED runs anyway (the wrapper's rule),
* peer-aware only when the holder was addressed too
* - a won claim runs, then releases in the same cleanup that stops typing
* - claim infrastructure failure fails OPEN (#887) — the run proceeds
* - triggers without a message (heartbeat) never touch the claim service
Expand Down Expand Up @@ -48,6 +50,27 @@ jest.mock('../../../services/messageClaimHandoffService', () => ({
release: jest.fn(),
}));

// No outsider-writable grant, no connector tools, and no User rows. The real
// modules read Mongo, which unit mode does not connect, so every turn here
// waited out mongoose's 10 s buffer per read (about 20 s an arm before).
jest.mock('../../../services/nativeOutsiderGrantPolicy', () => ({
podHasOutsiderWritableGrant: jest.fn().mockResolvedValue(false),
}));
jest.mock('../../../models/User', () => ({
findOne: jest.fn(() => ({ select: () => ({ lean: () => Promise.resolve(null) }) })),
findById: jest.fn(() => ({ select: () => ({ lean: () => Promise.resolve(null) }) })),
}));
jest.mock('../../../services/grantBrokerProjectionService', () => ({
hostedBrokerToolsForRun: jest.fn().mockResolvedValue({ tools: [], dispatch: new Map() }),
dispatchHostedBrokerTool: jest.fn().mockResolvedValue(null),
}));

// Whether the holder of a lost claim was itself addressed by the message.
const mockHolderEvent = jest.fn();
jest.mock('../../../models/AgentEvent', () => ({
exists: (...args) => mockHolderEvent(...args),
}));

const axios = require('axios').default;
const AgentRun = require('../../../models/AgentRun');
const MessageClaimService = require('../../../services/messageClaimService');
Expand Down Expand Up @@ -123,9 +146,15 @@ describe('nativeRuntimeService claim-before-act (ADR-018 D3)', () => {
expect(typing.emitAgentTypingStop).toHaveBeenCalled();
});

test('a lost CAS stands down: no run row, no LLM call, no typing, acked no-op', async () => {
const ambientTrigger = {
type: 'message.posted',
eventId: 'evt-2',
payload: { messageId: 'msg-77', content: 'thinking out loud' },
};

test('a lost CAS on an ambient wake stands down: no run row, no LLM call, no typing, acked no-op', async () => {
MessageClaimService.claim.mockResolvedValue({ claimed: false, claimedBy: 'task-clerk' });
const result = await runAgent(installation, mentionTrigger);
const result = await runAgent(installation, ambientTrigger);
expect(result).toEqual({
runId: '', status: 'succeeded', totalTurns: 0, totalTokens: 0,
});
Expand All @@ -135,6 +164,62 @@ describe('nativeRuntimeService claim-before-act (ADR-018 D3)', () => {
expect(MessageClaimService.release).not.toHaveBeenCalled();
});

// QA walk 4 (2026-10-16, message 78340): Scout's ambient wake claimed a
// stranger's first @-ask to their hosted agent, answered NO_REPLY, and the
// hosted agent, having stood down, never ran. Addressed outranks a CAS.
const userMessageSent = () => axios.post.mock.calls[0][1].messages.find((m) => m.role === 'user').content;

test.each([
['a mention', mentionTrigger],
['a thread mention', { ...mentionTrigger, type: 'thread.mention' }],
['a DM', { ...mentionTrigger, type: 'dm.message' }],
['a reply to its own message', { ...ambientTrigger, payload: { ...ambientTrigger.payload, repliesToYourMessage: true } }],
])('directly addressed by %s, a lost CAS still runs the turn, unclaimed', async (_label, trigger) => {
MessageClaimService.claim.mockResolvedValue({ claimed: false, claimedBy: 'scout', instanceId: 'scout-1' });
mockHolderEvent.mockResolvedValue(null);

const result = await runAgent(installation, trigger);

expect(result.status).toBe('succeeded');
expect(AgentRun.create).toHaveBeenCalledTimes(1);
expect(axios.post).toHaveBeenCalledTimes(1);
// Nothing was held, so nothing is released.
expect(MessageClaimService.release).not.toHaveBeenCalled();
expect(MessageClaimHandoffService.release).not.toHaveBeenCalled();
});

test('beaten by an ambient listener, the addressed agent just answers: no peer notice', async () => {
MessageClaimService.claim.mockResolvedValue({ claimed: false, claimedBy: 'scout', instanceId: 'scout-1' });
mockHolderEvent.mockResolvedValue(null);

await runAgent(installation, mentionTrigger);

expect(mockHolderEvent).toHaveBeenCalledWith(expect.objectContaining({
agentName: 'scout', instanceId: 'scout-1', 'payload.messageId': 'msg-77',
}));
expect(userMessageSent()).not.toMatch(/Claim notice/);
});

test('when the holder was addressed too, the turn runs peer-aware', async () => {
MessageClaimService.claim.mockResolvedValue({ claimed: false, claimedBy: 'task-clerk', instanceId: 'default' });
mockHolderEvent.mockResolvedValue({ _id: 'evt-holder' });

await runAgent(installation, mentionTrigger);

expect(userMessageSent()).toMatch(/^\[Claim notice: @task-clerk holds message msg-77 and was addressed too/);
expect(userMessageSent()).toMatch(/otherwise return NO_REPLY\.\]/);
});

test('if it cannot tell whether the holder was addressed, it answers rather than defers', async () => {
MessageClaimService.claim.mockResolvedValue({ claimed: false, claimedBy: 'task-clerk', instanceId: 'default' });
mockHolderEvent.mockRejectedValue(new Error('mongo down'));

await runAgent(installation, mentionTrigger);

expect(AgentRun.create).toHaveBeenCalledTimes(1);
expect(userMessageSent()).not.toMatch(/Claim notice/);
});

test('a human wake that ends NO_REPLY declines into the shared one-seat handoff', async () => {
AgentMessageService.postMessage.mockResolvedValueOnce({ success: true, skipped: true });
const result = await runAgent(installation, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,34 @@ describe('nativeRuntimeService — hosted installs on the native engine', () =>
});
});

// QA walk 4 (2026-10-16): given a second round, the greeting run posted its
// "(Mention me as @…)" line as a second message. A hosted greeting is one post.
test('a hosted first-contact run ends at its first post', async () => {
const AgentMessageService = require('../../../services/agentMessageService');
axios.post
.mockResolvedValueOnce(toolCalls(call('c1', 'commonly_post_message', { podId: POD, content: 'Hi! What should we start on?' })))
.mockResolvedValueOnce(toolCalls(call('c2', 'commonly_post_message', { podId: POD, content: '(Mention me as @maya so I see your answer.)' })));

const result = await runAgent(hostedInstall(), {
type: 'first_contact', eventId: '507f1f77bcf86cd799439013', payload: { content: 'Greet them.', userId: OWNER },
});

expect(result.status).toBe('succeeded');
expect(axios.post).toHaveBeenCalledTimes(1);
expect(AgentMessageService.postMessage).toHaveBeenCalledTimes(1);
expect(savedRun.status).toBe('succeeded');
});

test('a hosted mention run is not cut short by that rule', async () => {
axios.post
.mockResolvedValueOnce(toolCalls(call('c1', 'commonly_post_message', { podId: POD, content: 'Here is the answer.' })))
.mockResolvedValueOnce(toolCalls(call('c2', 'commonly_write_memory', { content: 'answered' })));

await runAgent(hostedInstall(), mention);

expect(axios.post.mock.calls.length).toBeGreaterThanOrEqual(2);
});

test('in a pod holding an outsider-writable grant, a hosted turn is narrowed like any native one', async () => {
const { podHasOutsiderWritableGrant } = require('../../../services/nativeOutsiderGrantPolicy');
podHasOutsiderWritableGrant.mockResolvedValue(true);
Expand Down
2 changes: 1 addition & 1 deletion backend/routes/hosted.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ router.get('/status', hostedRateLimit, auth, async (req: any, res: any) => {
// and the runtime it needs is available (checked just above).
const { provisionedAt } = readHostedConfig(installation);
const { turnsPerDay } = await hostedRuntime.hostedCapsForOwner(getUserId(req));
const meter = await hostedRuntime.meterAllowsTurn(agentName, instanceId, turnsPerDay);
const meter = await hostedRuntime.meterAllowsTurn(agentName, instanceId, turnsPerDay, { countRuns: true });
return res.json({
agentName,
instanceId,
Expand Down
5 changes: 3 additions & 2 deletions backend/services/firstContactService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ const FIRST_CONTACT_CUE =
// The greeting's question invites a reply, and an agent that sees only its
// mentions would never see a plain one. So that greeting says how to reach it.
const mentionLineCue = (handle: string): string => (
` After the question, add one more line, exactly: "(Mention me as @${handle} so I see your answer.)" `
+ 'You only see messages that mention you, so without that line their answer would never reach you.'
` End that same message, after the question, with one more line, exactly: "(Mention me as @${handle} `
+ 'so I see your answer.)" It is part of the greeting: post one message, not two. You only see '
+ 'messages that mention you, so without that line their answer would never reach you.'
);
const MENTION_HANDLE = /^[a-z0-9][a-z0-9-]{0,63}$/;

Expand Down
21 changes: 20 additions & 1 deletion backend/services/hostedRuntimeService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,22 @@ export const turnsToday = async (agentName: string, instanceId: string): Promise
})
);

// On the native engine a turn is a run: an AgentRun is created only when the
// model is actually called. The event count above also counts an event that
// was acked without a run, such as an addressed turn that stood down to a
// peer's claim (QA walk 4, 2026-10-16: the ask got no answer and still showed
// as a used turn). The greeting is left out here too.
export const nativeTurnsToday = async (agentName: string, instanceId: string): Promise<number> => {
// eslint-disable-next-line global-require, @typescript-eslint/no-require-imports
const AgentRun = require('../models/AgentRun');
return AgentRun.countDocuments({
agentName: String(agentName).toLowerCase(),
instanceId: instanceId || 'default',
startedAt: { $gte: utcDayStart() },
trigger: { $ne: 'first_contact' },
});
};

export interface MeterResult {
allowed: boolean;
used: number;
Expand All @@ -167,8 +183,10 @@ export const meterAllowsTurn = async (
agentName: string,
instanceId: string,
turnsPerDay: number = hostedCaps().turnsPerDay,
// True for a native-engine install, whose turns are runs, not acked events.
{ countRuns = false }: { countRuns?: boolean } = {},
): Promise<MeterResult> => {
const used = await turnsToday(agentName, instanceId);
const used = countRuns ? await nativeTurnsToday(agentName, instanceId) : await turnsToday(agentName, instanceId);
const tomorrow = new Date(utcDayStart().getTime() + 24 * 60 * 60 * 1000);
return {
allowed: used < turnsPerDay,
Expand Down Expand Up @@ -269,6 +287,7 @@ module.exports = {
countHostedAgentsForUser,
listHostedInstallationsForUser,
turnsToday,
nativeTurnsToday,
meterAllowsTurn,
HostedRuntimeError,
provisionAgent,
Expand Down
Loading
Loading