diff --git a/backend/__tests__/unit/scripts/sweep-stale-digest.test.js b/backend/__tests__/unit/scripts/sweep-stale-digest.test.js new file mode 100644 index 0000000000..ed8168c8fb --- /dev/null +++ b/backend/__tests__/unit/scripts/sweep-stale-digest.test.js @@ -0,0 +1,84 @@ +const mongoose = require('mongoose'); +const AgentEvent = require('../../../models/AgentEvent'); +const { sweepStaleDigest, parseArgs, SWEEP_REASON } = require('../../../scripts/sweep-stale-digest'); +const { setupMongoDb, closeMongoDb, clearMongoDb } = require('../../utils/testUtils'); + +const HOUR_MS = 60 * 60 * 1000; +const NOW = new Date('2026-10-26T12:00:00.000Z'); +const POD_ID = new mongoose.Types.ObjectId(); + +describe('sweep-stale-digest', () => { + beforeAll(async () => { await setupMongoDb(); }); + afterEach(async () => { await clearMongoDb(); }); + afterAll(async () => { await closeMongoDb(); }); + + // Raw insert, so each row keeps the createdAt it is given. + const row = (overrides) => ({ + agentName: 'rhea', + instanceId: 'default', + podId: POD_ID, + type: 'message.posted', + deliveryMode: 'digest', + status: 'pending', + attempts: 0, + payload: { content: 'ambient' }, + ...overrides, + }); + const seed = async () => { + const ago = (hours) => new Date(NOW.getTime() - hours * HOUR_MS); + const docs = [ + row({ createdAt: ago(30) }), + row({ createdAt: ago(25) }), + row({ createdAt: ago(48), status: 'delivered', deliveryNonce: 'n-1' }), + row({ createdAt: ago(26), agentName: 'quire' }), + row({ createdAt: ago(2) }), // inside the day: stays unread + row({ createdAt: ago(30), deliveryMode: 'wake', type: 'chat.mention' }), // a task, never swept + row({ createdAt: ago(30), status: 'acked' }), // already read + ]; + const { insertedIds } = await AgentEvent.collection.insertMany(docs); + return Object.values(insertedIds); + }; + const statuses = async (ids) => Promise.all(ids.map(async (id) => (await AgentEvent.findById(id).lean()).status)); + + test('a dry run reports per seat and changes nothing', async () => { + const ids = await seed(); + const result = await sweepStaleDigest({ now: NOW }); + expect(result).toEqual({ + apply: false, + cutoff: '2026-10-25T12:00:00.000Z', + matched: 4, + settled: 0, + seats: [ + { agentName: 'rhea', instanceId: 'default', count: 3 }, + { agentName: 'quire', instanceId: 'default', count: 1 }, + ], + }); + expect(await statuses(ids)).toEqual(['pending', 'pending', 'delivered', 'pending', 'pending', 'pending', 'acked']); + }); + + test('apply settles only digest rows older than a day, with the sweep as their reason', async () => { + const ids = await seed(); + const result = await sweepStaleDigest({ apply: true, now: NOW }); + expect(result.matched).toBe(4); + expect(result.settled).toBe(4); + expect(await statuses(ids)).toEqual(['acked', 'acked', 'acked', 'acked', 'pending', 'pending', 'acked']); + const swept = await AgentEvent.findById(ids[2]).lean(); + expect(swept.deliveryNonce).toBeNull(); + expect(swept.delivery).toEqual(expect.objectContaining({ outcome: 'no_action', reason: SWEEP_REASON })); + expect((await AgentEvent.findById(ids[6]).lean()).delivery).toBeUndefined(); + }); + + test('a second apply finds nothing left', async () => { + await seed(); + await sweepStaleDigest({ apply: true, now: NOW }); + expect(await sweepStaleDigest({ apply: true, now: NOW })) + .toEqual(expect.objectContaining({ matched: 0, settled: 0, seats: [] })); + }); + + test('--dry-run wins over --apply, and no flag is a dry run', () => { + expect(parseArgs([])).toEqual({ apply: false }); + expect(parseArgs(['--dry-run'])).toEqual({ apply: false }); + expect(parseArgs(['--apply'])).toEqual({ apply: true }); + expect(parseArgs(['--apply', '--dry-run'])).toEqual({ apply: false }); + }); +}); diff --git a/backend/__tests__/unit/services/agentEventService.digestBacklog.test.js b/backend/__tests__/unit/services/agentEventService.digestBacklog.test.js new file mode 100644 index 0000000000..d7067d4ad4 --- /dev/null +++ b/backend/__tests__/unit/services/agentEventService.digestBacklog.test.js @@ -0,0 +1,233 @@ +// A consumed wake reads its pod's whole unread digest, not only the page it +// quoted. Pinned against mongodb-memory-server. +// +// The loop this exists for (rhea, 2026-10-25): a wake claimed up to 9 digest +// companions, the seat answered NO_REPLY, the server judged the reply +// `silent_or_empty`, and both the CLI and the ack treated that as "not read". +// The requeue put the companions back, the unread-digest heartbeat fired an +// hour later, and it repeated. Even a consumed wake settled only the rows it +// had quoted, so a backlog drained 9 rows per wake, oldest first, while about +// 1,000 rows arrived per seat per day. 22,568 were pending fleet-wide. +const { MongoMemoryServer } = require('mongodb-memory-server'); +const mongoose = require('mongoose'); + +jest.mock('../../../services/agentMemoryService', () => ({ + buildMemoryDigestBundle: jest.fn(() => ({})), +})); +jest.mock('../../../services/agentProvisionerService', () => ({ + getAgentSessionSizes: jest.fn(), + clearAgentRuntimeSessions: jest.fn(), + restartAgentRuntime: jest.fn(), + resolveOpenClawAccountId: jest.fn(() => 'acct'), +})); +jest.mock('../../../services/agentIdentityService', () => ({ + getAgentTypeConfig: jest.fn(() => ({ runtime: 'moltbot' })), +})); +jest.mock('../../../services/nativeRuntimeService', () => ({ runAgent: jest.fn() })); + +let mongod; +let AgentEvent; +let AgentEventService; + +const AGENT = 'pixel'; +const INSTANCE = 'default'; +const POD_ID = new mongoose.Types.ObjectId(); +const BACKLOG = 30; + +beforeAll(async () => { + mongod = await MongoMemoryServer.create(); + await mongoose.connect(mongod.getUri()); + AgentEvent = require('../../../models/AgentEvent'); + AgentEventService = require('../../../services/agentEventService'); +}); + +afterAll(async () => { + await mongoose.disconnect(); + await mongod.stop(); +}); + +beforeEach(async () => { + await AgentEvent.deleteMany({}); +}); + +// Raw insert so each row keeps the createdAt it is given: oldest first, one +// minute apart, each in its own thread so the frame can name which it quoted. +const seedBacklog = async () => { + const now = Date.now(); + await AgentEvent.collection.insertMany(Array.from({ length: BACKLOG }, (_, index) => ({ + agentName: AGENT, + instanceId: INSTANCE, + podId: POD_ID, + type: 'message.posted', + deliveryMode: 'digest', + status: 'pending', + attempts: 0, + payload: { messageId: `m${index + 1}`, threadRootId: index + 1, content: 'ambient body' }, + createdAt: new Date(now - (BACKLOG - index) * 60000), + updatedAt: new Date(now - (BACKLOG - index) * 60000), + }))); +}; + +const seedWake = () => AgentEvent.create({ + agentName: AGENT, instanceId: INSTANCE, podId: POD_ID, + type: 'chat.mention', payload: { content: 'question' }, +}); + +const unread = () => AgentEvent.countDocuments({ + deliveryMode: 'digest', status: { $in: ['pending', 'delivered'] }, +}); + +// The wrapper CLI's path: one pod-scoped page, wake plus digest companions. +const claimPage = async () => { + const page = await AgentEventService.listInboxPage({ + agentName: AGENT, instanceId: INSTANCE, podIds: [POD_ID], limit: 10, + }); + const [wake] = page.events.filter((event) => event.deliveryMode !== 'digest'); + return { page, wake }; +}; + +describe('a consumed wake settles the digest backlog up to its snapshot', () => { + test.each([ + [{ outcome: 'posted' }], + [{ outcome: 'acknowledged' }], + [{ outcome: 'no_action' }], + [{ outcome: 'no_action', reason: 'silent_or_empty' }], + ])('a 30-row backlog drains in one consumed ack: %j', async (result) => { + await seedBacklog(); + await seedWake(); + const { page, wake } = await claimPage(); + expect(page.events.filter((event) => event.deliveryMode === 'digest')).toHaveLength(9); + expect((await AgentEvent.findById(wake._id).lean()).digestSnapshotAt).toEqual(expect.any(Date)); + + await AgentEventService.acknowledge(wake._id, AGENT, INSTANCE, result, wake.deliveryNonce); + expect(await unread()).toBe(0); + }); + + test.each([ + [{ outcome: 'no_action', reason: 'claim-held' }], + [{ outcome: 'no_action', reason: 'duplicate_recent' }], + // Empty output is not the NO_REPLY sentinel: a seat failing silently every + // turn must leave its backlog pending, not drain it (sable, #2753). + [{ outcome: 'no_action', reason: 'empty-output' }], + [{ outcome: 'error' }], + ])('a non-consuming outcome settles nothing: %j', async (result) => { + await seedBacklog(); + await seedWake(); + const { wake } = await claimPage(); + + await AgentEventService.acknowledge(wake._id, AGENT, INSTANCE, result, wake.deliveryNonce); + expect(await unread()).toBe(BACKLOG); + expect(await AgentEvent.countDocuments({ deliveryMode: 'digest', status: 'acked' })).toBe(0); + }); + + test('a row created after the snapshot stays pending', async () => { + await seedBacklog(); + await seedWake(); + const { wake } = await claimPage(); + await new Promise((resolve) => setTimeout(resolve, 5)); + const late = await AgentEvent.create({ + agentName: AGENT, instanceId: INSTANCE, podId: POD_ID, + type: 'message.posted', deliveryMode: 'digest', payload: { content: 'arrived mid-turn' }, + }); + + await AgentEventService.acknowledge(wake._id, AGENT, INSTANCE, + { outcome: 'posted' }, wake.deliveryNonce); + expect((await AgentEvent.findById(late._id).lean()).status).toBe('pending'); + expect(await unread()).toBe(1); + }); + + test('another pod\'s backlog is untouched', async () => { + await seedBacklog(); + const otherPod = new mongoose.Types.ObjectId(); + const other = await AgentEvent.create({ + agentName: AGENT, instanceId: INSTANCE, podId: otherPod, + type: 'message.posted', deliveryMode: 'digest', payload: { content: 'elsewhere' }, + }); + await AgentEvent.updateOne({ _id: other._id }, { $set: { createdAt: new Date(Date.now() - 3600000) } }); + await seedWake(); + const { wake } = await claimPage(); + + await AgentEventService.acknowledge(wake._id, AGENT, INSTANCE, + { outcome: 'posted' }, wake.deliveryNonce); + expect((await AgentEvent.findById(other._id).lean()).status).toBe('pending'); + }); + + test('a late companion ack after the settle is still a success', async () => { + await seedBacklog(); + await seedWake(); + const { page, wake } = await claimPage(); + const companion = page.events.find((event) => event.deliveryMode === 'digest'); + + await AgentEventService.acknowledge(wake._id, AGENT, INSTANCE, + { outcome: 'posted' }, wake.deliveryNonce); + expect(await AgentEventService.acknowledge(companion._id, AGENT, INSTANCE, + { outcome: 'no_action', reason: 'digest-read' }, companion.deliveryNonce)).toBeNull(); + expect(await AgentEventService.isSupersededDelivery( + companion._id, AGENT, INSTANCE, companion.deliveryNonce, + )).toBe(false); + }); +}); + +describe('the legacy frame shows the newest rows and the true total', () => { + test('quotes the newest 8 rows and counts every unread row', async () => { + await seedBacklog(); + await seedWake(); + const [delivered] = await AgentEventService.list({ agentName: AGENT, instanceId: INSTANCE }); + const summary = delivered.payload.digestSummary; + + expect(summary).toContain(`${BACKLOG} ambient updates`); + expect(summary).toContain(`older pod activity: ${BACKLOG - 8} unread`); + expect(summary).toContain('thread 23: 1 unread'); + expect(summary).not.toContain('thread 22:'); + expect(summary).not.toContain('thread 1:'); + expect(summary).not.toContain('ambient body'); + + await AgentEventService.acknowledge(delivered._id, AGENT, INSTANCE, + { outcome: 'no_action', reason: 'silent_or_empty' }, delivered.payload.deliveryId); + expect(await unread()).toBe(0); + }); +}); + +describe('a native turn settles the backlog it was shown', () => { + const { runAgent } = require('../../../services/nativeRuntimeService'); + const { AgentInstallation } = require('../../../models/AgentRegistry'); + const nativeInstall = { + _id: new mongoose.Types.ObjectId(), + agentName: AGENT, + instanceId: INSTANCE, + podId: POD_ID, + status: 'active', + config: { runtime: { runtimeType: 'native' } }, + }; + + afterEach(() => { jest.restoreAllMocks(); }); + + const runNative = async (runResult) => { + runAgent.mockReset(); + runAgent.mockResolvedValue(runResult); + jest.spyOn(AgentInstallation, 'findOne').mockReturnValue({ lean: () => Promise.resolve(nativeInstall) }); + const event = await AgentEventService.enqueue({ + agentName: AGENT, instanceId: INSTANCE, podId: POD_ID, type: 'chat.mention', payload: { text: 'hi' }, + }); + const deadline = Date.now() + 5000; + while (Date.now() < deadline + && !['acked', 'failed'].includes((await AgentEvent.findById(event._id).lean()).status)) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(runAgent).toHaveBeenCalledTimes(1); + return runAgent.mock.calls[0][1].payload.content; + }; + + test('a succeeded turn reads every row up to its snapshot', async () => { + await seedBacklog(); + const content = await runNative({ status: 'succeeded', runId: 'run-1', totalTurns: 1 }); + expect(content).toContain(`${BACKLOG} ambient updates`); + expect(await unread()).toBe(0); + }); + + test('a turn that never ran the model leaves the backlog unread', async () => { + await seedBacklog(); + await runNative({ status: 'succeeded', runId: null, totalTurns: 0 }); + expect(await unread()).toBe(BACKLOG); + }); +}); diff --git a/backend/__tests__/unit/services/agentEventService.lifecycle.test.js b/backend/__tests__/unit/services/agentEventService.lifecycle.test.js index 96699066e9..891a599722 100644 --- a/backend/__tests__/unit/services/agentEventService.lifecycle.test.js +++ b/backend/__tests__/unit/services/agentEventService.lifecycle.test.js @@ -480,6 +480,7 @@ describe('driver paths reach a terminal state', () => { }; const duplicate = { ...ambient, _id: 'ambient-2' }; AgentEvent.find.mockReturnValue({ sort: () => ({ limit: () => ({ lean: async () => [ambient, duplicate] }) }) }); + AgentEvent.countDocuments.mockResolvedValue(2); AgentEvent.updateOne.mockResolvedValue({ matchedCount: 1 }); AgentEvent.updateMany.mockResolvedValue({ modifiedCount: 1 }); AgentEvent.create.mockResolvedValue({ @@ -501,8 +502,11 @@ describe('driver paths reach a terminal state', () => { expect(body.payload.digestSummary).toContain('thread 40: 1 unread'); expect(body.payload.digestSummary).toContain('1 ambient update in 1 conversation'); expect(body.payload.digestSummary).not.toContain('ambient note'); + // Accepted delivery reads the pod's whole backlog up to the wake's snapshot. expect(AgentEvent.updateMany).toHaveBeenCalledWith( - expect.objectContaining({ _id: { $in: ['ambient-1', 'ambient-2'] }, deliveryMode: 'digest' }), + expect.objectContaining({ + agentName: 'hookbot', podId: POD_P1, deliveryMode: 'digest', createdAt: { $lte: expect.any(Date) }, + }), expect.objectContaining({ $set: expect.objectContaining({ status: 'acked' }) }), ); }); @@ -575,9 +579,10 @@ describe('driver paths reach a terminal state', () => { expect(runAgent).not.toHaveBeenCalled(); }); - test('a later native wake carries digest context and settles only included rows on success', async () => { + test('a later native wake carries digest context and settles its backlog only on success', async () => { const digest = { _id: 'evt-d', payload: { username: 'alice', messageId: '42', content: 'daily update' } }; AgentEvent.find.mockReturnValue({ sort: () => ({ limit: () => ({ lean: () => Promise.resolve([digest]) }) }) }); + AgentEvent.countDocuments.mockResolvedValue(1); AgentEvent.updateMany.mockResolvedValue({ modifiedCount: 1 }); runAgent.mockResolvedValue({ status: 'succeeded', runId: 'run-1', totalTurns: 1 }); @@ -586,8 +591,8 @@ describe('driver paths reach a terminal state', () => { expect(runAgent.mock.calls[0][1].payload.content).toContain('pod conversation: 1 unread'); expect(runAgent.mock.calls[0][1].payload.content).not.toContain('daily update'); expect(AgentEvent.updateMany).toHaveBeenCalledWith( - { _id: { $in: ['evt-d'] }, status: 'pending', deliveryMode: 'digest' }, - { $set: { status: 'acked' } }, + expect.objectContaining({ agentName: 'clerk', deliveryMode: 'digest', createdAt: { $lte: expect.any(Date) } }), + { $set: expect.objectContaining({ status: 'acked' }) }, ); jest.clearAllMocks(); diff --git a/backend/__tests__/unit/services/agentEventService.test.js b/backend/__tests__/unit/services/agentEventService.test.js index 412617ac6e..605455627a 100644 --- a/backend/__tests__/unit/services/agentEventService.test.js +++ b/backend/__tests__/unit/services/agentEventService.test.js @@ -423,10 +423,18 @@ describe('AgentEventService', () => { status: 'delivered', payload: { messageId: '42' }, }) }); readFoldedDigest.mockResolvedValueOnce({ unreadCount: 3, lastEventIds: ['older-1'], revision: 4 }); - AgentEvent.updateOne.mockResolvedValueOnce({ modifiedCount: 1 }); + AgentEvent.updateOne + .mockResolvedValueOnce({ modifiedCount: 1 }) // the wake's digest snapshot + .mockResolvedValueOnce({ modifiedCount: 1 }); // the folded rollup it carries const page = await AgentEventService.listInboxPage({ agentName: 'inbox', podId: 'pod-a' }); expect(page.events.map((event) => event._id)).toEqual(['wake', 'digest']); expect(page.inboxCount).toBe(2); + // A wake that carried digest records when it read it, so its ack settles + // the pod's backlog up to that moment. + expect(AgentEvent.updateOne).toHaveBeenNthCalledWith(1, + expect.objectContaining({ _id: 'wake', status: 'delivered' }), + { $set: { digestSnapshotAt: expect.any(Date) } }); + expect(page.events[0].digestSnapshotAt).toEqual(expect.any(Date)); expect(page.events[0].payload.digestRollup).toEqual({ unreadCount: 3, lastEventIds: ['older-1'] }); expect(page.events[1].payload.digestRollup).toBeUndefined(); expect(AgentEvent.find).toHaveBeenNthCalledWith(3, expect.objectContaining({ diff --git a/backend/models/AgentEvent.ts b/backend/models/AgentEvent.ts index 278e57e11f..4ceaaa730d 100644 --- a/backend/models/AgentEvent.ts +++ b/backend/models/AgentEvent.ts @@ -117,6 +117,9 @@ export interface IAgentEvent extends Document { // Raw ambient rows presented inside a legacy wake or webhook delivery. // They settle only when that wake succeeds. digestEventIds?: Types.ObjectId[]; + // When this wake read its pod's unread digest. A consumed wake settles every + // pending digest row created at or before it, not only the rows it quoted. + digestSnapshotAt?: Date | null; createdAt: Date; updatedAt: Date; } @@ -137,6 +140,7 @@ const AgentEventSchema = new Schema( deliveryNonce: { type: String, default: null }, digestRollupRevision: { type: Number, default: null }, digestEventIds: [{ type: Schema.Types.ObjectId, ref: 'AgentEvent' }], + digestSnapshotAt: { type: Date, default: null }, delivery: { outcome: { type: String, diff --git a/backend/package.json b/backend/package.json index 01787ff3c7..876e03e633 100644 --- a/backend/package.json +++ b/backend/package.json @@ -29,6 +29,7 @@ "sweep:resolved-mention-attention": "ts-node scripts/sweep-resolved-mention-attention.ts", "sweep:done-task-handoffs": "ts-node scripts/sweep-done-task-handoffs.ts", "sweep:leftover-runtime-tokens": "ts-node scripts/sweep-leftover-runtime-tokens.ts", + "sweep:stale-digest": "ts-node scripts/sweep-stale-digest.ts", "backfill:hosted-mcp-client-id": "ts-node scripts/backfill-hosted-mcp-client-id.ts", "revive:native-installations": "ts-node scripts/revive-native-installations.ts", "set:agent-descriptions": "ts-node scripts/set-agent-descriptions.ts", diff --git a/backend/scripts/sweep-stale-digest.ts b/backend/scripts/sweep-stale-digest.ts new file mode 100644 index 0000000000..61fb6492fd --- /dev/null +++ b/backend/scripts/sweep-stale-digest.ts @@ -0,0 +1,94 @@ +/** + * Mark ambient digest rows older than a day as read. + * + * Until the backlog settle landed, a consumed wake settled only the digest + * rows it quoted (at most 9, oldest first), and a NO_REPLY the server reduced + * to `silent_or_empty` settled none. About 1,000 rows arrive per seat per day, + * so the backlog only grew: 22,568 pending fleet-wide on 2026-10-25, the oldest + * 31 hours old, each one keeping an hourly unread-digest heartbeat alive. The + * fix settles a pod's whole backlog on each consumed wake; this clears what + * built up before it. + * + * A digest row is context, never a task, and nothing waits on one a day old. + * Wake rows are never touched. + * + * Dry run is the default (`--dry-run` says so explicitly and beats `--apply`): + * npm run sweep:stale-digest + * Apply: + * npm run sweep:stale-digest -- --apply + */ +/* eslint-disable no-console */ +import mongoose from 'mongoose'; +import AgentEvent from '../models/AgentEvent'; + +const DAY_MS = 24 * 60 * 60 * 1000; +export const SWEEP_REASON = 'stale-digest-sweep'; + +export type StaleDigestSeat = { agentName: string; instanceId: string; count: number }; +export type StaleDigestResult = { + apply: boolean; + cutoff: string; + matched: number; + settled: number; + seats: StaleDigestSeat[]; +}; + +export const parseArgs = (argv: string[]): { apply: boolean } => ({ + apply: argv.includes('--apply') && !argv.includes('--dry-run'), +}); + +export const sweepStaleDigest = async ({ + apply = false, + now = new Date(), + olderThanMs = DAY_MS, +}: { apply?: boolean; now?: Date; olderThanMs?: number } = {}): Promise => { + const cutoff = new Date(now.getTime() - olderThanMs); + // `delivered` too: a companion claimed beside a wake that never consumed + // waits there for the requeue, and is just as stale. + const filter = { + deliveryMode: 'digest', + status: { $in: ['pending', 'delivered'] }, + createdAt: { $lt: cutoff }, + }; + const grouped = await AgentEvent.aggregate<{ _id: { agentName: string; instanceId: string }; count: number }>([ + { $match: filter }, + { $group: { _id: { agentName: '$agentName', instanceId: '$instanceId' }, count: { $sum: 1 } } }, + { $sort: { count: -1 } }, + ]); + const seats = grouped.map((row) => ({ + agentName: row._id.agentName, instanceId: row._id.instanceId || 'default', count: row.count, + })); + const matched = seats.reduce((sum, seat) => sum + seat.count, 0); + let settled = 0; + if (apply && matched) { + const updated = await AgentEvent.updateMany(filter, { + $set: { + status: 'acked', + deliveredAt: now, + deliveryNonce: null, + delivery: { outcome: 'no_action', reason: SWEEP_REASON, reasonRecorded: true, updatedAt: now }, + }, + }); + settled = updated.modifiedCount; + } + return { apply, cutoff: cutoff.toISOString(), matched, settled, seats }; +}; + +export const main = async (): Promise => { + if (!process.env.MONGO_URI) throw new Error('MONGO_URI is required'); + const { apply } = parseArgs(process.argv.slice(2)); + await mongoose.connect(process.env.MONGO_URI); + try { + console.log(JSON.stringify(await sweepStaleDigest({ apply }))); + if (!apply) console.log('DRY RUN — no digest rows changed. Re-run with --apply after review.'); + } finally { + await mongoose.disconnect(); + } +}; + +if (require.main === module) { + main().catch((error) => { + console.error('stale digest sweep failed:', error); + process.exit(1); + }); +} diff --git a/backend/services/agentEventService.ts b/backend/services/agentEventService.ts index ef5fbaef44..43f4641e5a 100644 --- a/backend/services/agentEventService.ts +++ b/backend/services/agentEventService.ts @@ -90,6 +90,23 @@ const digestConversationFrame = ( + `${lines.join('\n')}\nPull relevant messages with commonly_get_messages. On resume, read oldest unread first; at most 8 conversations, 5 messages each, 20 messages total.`; }; +// How many unread ambient rows a wake quotes; the rest are counted, not shown. +const DIGEST_FRAME_ROWS = 8; + +// Whether a wake's outcome means its seat read the digest it carried: a post, +// an ack, a plain decline, or a reply the server reduced to silence. +// `silent_or_empty` is what agentMessageService returns for an intentional +// NO_REPLY (a suppressed runtime failure is `runtime_*_suppressed` instead), so +// it is the seat's decision, not a refusal. Treating it as unread left a +// silent seat's digest pending, and the hourly unread-digest wake re-woke it to +// be silent again (rhea, 2026-10-25). A turn that printed nothing acks as +// `empty-output`, which is not a read: that is what a silently failing seat +// sends every turn, and reading on it would drain a backlog nobody saw +// (sable, #2753). +const digestConsumedBy = (outcome?: string, reason?: string): boolean => outcome === 'posted' + || outcome === 'acknowledged' + || (outcome === 'no_action' && (!reason || reason === 'silent_or_empty')); + // TASK-264: how many stuck in-process events one sweep re-runs, and how long a // 'queued'/'running' AgentRun may sit before it counts as abandoned. const STUCK_REDISPATCH_BATCH = 50; @@ -168,8 +185,11 @@ interface EventDoc { deliveryNonce?: string | null; digestRollupRevision?: number | null; digestEventIds?: unknown[]; + digestSnapshotAt?: Date | null; } +type DigestScope = { agentName: string; instanceId: string; podId: unknown }; + interface InstallationDoc { _id?: unknown; agentName?: string; @@ -376,7 +396,7 @@ const deliverEventViaWebhook = async (installation: InstallationDoc, event: Even deliveredAt: new Date(), delivery: AgentEventService.normalizeDeliveryMeta({ outcome, reason }), }); - if (outcome === 'posted' || outcome === 'acknowledged' || (outcome === 'no_action' && !reason)) { + if (digestConsumedBy(outcome, reason)) { await AgentEventService.settleLegacyDigest(event).catch((err: Error) => { console.warn('[webhook] digest settle failed:', err.message); }); @@ -1510,6 +1530,39 @@ class AgentEventService { conversations: state.conversations }; } + // The newest page of a pod's unread ambient rows, and how many older ones sit + // behind it. Reading oldest-first fed a backlogged seat day-old context while + // every newer row waited (22,568 pending on 2026-10-25, the oldest 31h old). + static async readDigestPage(scope: DigestScope): Promise<{ + rows: EventDoc[]; olderCount: number; snapshotAt: Date; + }> { + const snapshotAt = new Date(); + const filter = { ...scope, status: 'pending', deliveryMode: 'digest', createdAt: { $lte: snapshotAt } }; + const [newest, total] = await Promise.all([ + AgentEvent.find(filter).sort({ createdAt: -1 }).limit(DIGEST_FRAME_ROWS).lean() as Promise, + AgentEvent.countDocuments(filter), + ]); + return { rows: newest.reverse(), olderCount: Math.max(0, total - newest.length), snapshotAt }; + } + + // A digest row is ambient context, not a task. Once a wake that showed the + // unread count is consumed, every row it covered is read, not only the page it + // quoted; a row created after the snapshot stays pending for the next wake. + // `delivered` covers the companions claimed beside the wake: a CLI older than + // the silent_or_empty rule never acks them, and the requeue would bring them + // back as unread. + static async settleDigestBacklog(scope: DigestScope, snapshotAt: Date): Promise { + await AgentEvent.updateMany( + { + ...scope, + status: { $in: ['pending', 'delivered'] }, + deliveryMode: 'digest', + createdAt: { $lte: snapshotAt }, + }, + { $set: { status: 'acked', deliveredAt: new Date(), deliveryNonce: null } }, + ); + } + /** Add one bounded ambient frame to a wake consumed by an older poller or webhook. */ static async attachLegacyDigest(event: EventDoc): Promise { if (!event._id || !event.podId || !event.agentName || event.deliveryMode === 'digest') return; @@ -1518,13 +1571,14 @@ class AgentEventService { instanceId: String(event.instanceId || 'default'), podId: event.podId, }; - const [raw, rollup] = await Promise.all([ - AgentEvent.find({ ...scope, status: 'pending', deliveryMode: 'digest' }) - .sort({ createdAt: 1 }).limit(8).lean() as Promise, + const [page, rollup] = await Promise.all([ + this.readDigestPage(scope), readFoldedDigest(scope), ]); + const { rows: raw, olderCount } = page; if (!raw.length && !rollup?.unreadCount) return; const ids = raw.map((row) => row._id); + const snapshotAt = raw.length ? page.snapshotAt : null; const saved = await AgentEvent.updateOne( { _id: event._id, @@ -1535,12 +1589,16 @@ class AgentEventService { { $set: { digestEventIds: ids, digestRollupRevision: rollup?.revision ?? null, + digestSnapshotAt: snapshotAt, } }, ); if (!saved.matchedCount) return; event.digestEventIds = ids; event.digestRollupRevision = rollup?.revision ?? null; - const summary = digestConversationFrame(raw, rollup?.unreadCount || 0, rollup?.conversations); + event.digestSnapshotAt = snapshotAt; + const summary = digestConversationFrame( + raw, (rollup?.unreadCount || 0) + olderCount, rollup?.conversations, + ); const baseContent = [event.payload?.content, event.payload?.prompt, event.payload?.text] .find((value): value is string => typeof value === 'string' && value.length > 0) || ''; event.payload = { @@ -1561,7 +1619,10 @@ class AgentEventService { instanceId: String(event.instanceId || 'default'), podId: event.podId, }; - if (event.digestEventIds?.length) { + if (event.digestSnapshotAt) { + await this.settleDigestBacklog(scope, new Date(event.digestSnapshotAt)); + } else if (event.digestEventIds?.length) { + // A wake framed before digestSnapshotAt existed settles what it quoted. await AgentEvent.updateMany( { ...scope, _id: { $in: event.digestEventIds }, status: 'pending', deliveryMode: 'digest' }, { $set: { status: 'acked', deliveredAt: new Date() } }, @@ -1619,18 +1680,22 @@ class AgentEventService { console.warn('[native-runtime] folded digest lookup failed:', digestErr.message); return null; }); + const digestScope = { agentName: agentName.toLowerCase(), instanceId, podId: event.podId }; let digest: EventDoc[] = []; + let olderDigest = 0; + let digestSnapshotAt: Date | null = null; try { - digest = await AgentEvent.find({ - agentName: agentName.toLowerCase(), instanceId, podId: event.podId, - status: 'pending', deliveryMode: 'digest', - }).sort({ createdAt: 1 }).limit(8).lean() as EventDoc[]; + const page = await this.readDigestPage(digestScope); + digest = page.rows; + olderDigest = page.olderCount; + digestSnapshotAt = page.snapshotAt; } catch (digestErr) { console.warn('[native-runtime] digest lookup failed:', (digestErr as Error).message); } const baseContent = String(event.payload?.content || event.payload?.prompt || event.payload?.text || ''); const summary = digest.length || folded?.unreadCount - ? digestConversationFrame(digest, folded?.unreadCount || 0, folded?.conversations) : null; + ? digestConversationFrame(digest, (folded?.unreadCount || 0) + olderDigest, folded?.conversations) + : null; const payload = summary ? { ...event.payload, content: `${baseContent}\n\n${summary}`, @@ -1643,15 +1708,12 @@ class AgentEventService { // The model is done even if a digest/ack write stalls. Leave that // delivery for succeeded-run recovery instead of replaying its output. turnFinished(); - // Leave digest unread when the native turn fails. On a successful turn, - // mark only the rows actually included; older backlog remains pending. + // Leave digest unread when the native turn fails. A successful turn was + // shown the unread count, so it reads everything up to its snapshot. const modelTurnRan = result?.status === 'succeeded' && Boolean(result.runId) && Number(result.totalTurns) > 0; - if (modelTurnRan && digest.length) { - await AgentEvent.updateMany( - { _id: { $in: digest.map((item) => item._id) }, status: 'pending', deliveryMode: 'digest' }, - { $set: { status: 'acked' } }, - ).catch((digestErr: Error) => { + if (modelTurnRan && digest.length && digestSnapshotAt) { + await this.settleDigestBacklog(digestScope, digestSnapshotAt).catch((digestErr: Error) => { console.warn('[native-runtime] digest settle failed:', digestErr.message); }); } @@ -1967,15 +2029,19 @@ class AgentEventService { // one exists; otherwise keep the entire wake page. // At limit=1 the digest is a companion to the single wake, not another // runnable item. Return the pair so even that page exposes unread context. + // Newest first, so a backlogged seat sees current context; the wake that + // carries them settles everything up to this snapshot when it is consumed. const digestSlots = limit - wakeCandidates.length || 1; + const digestSnapshotAt = new Date(); const digestCandidates = onePod && wakeCandidates.length > 0 && digestSlots > 0 - ? await AgentEvent.find({ ...batchQuery, deliveryMode: 'digest' }) - .sort({ createdAt: 1 }) + ? await AgentEvent.find({ ...batchQuery, deliveryMode: 'digest', createdAt: { $lte: digestSnapshotAt } }) + .sort({ createdAt: -1 }) .limit(digestSlots) .select({ _id: 1 }) .lean() as Array<{ _id: unknown }> : []; if (limit > 1 && wakeCandidates.length === limit && digestCandidates.length > 0) wakeCandidates.pop(); + digestCandidates.reverse(); // the newest page, served in the order it happened const candidates = [...wakeCandidates, ...digestCandidates]; // A sibling poller can claim every candidate after our pre-claim count. @@ -2044,6 +2110,16 @@ class AgentEventService { } const firstWake = claimed.find((event) => event.deliveryMode !== 'digest'); + if (firstWake && claimed.some((event) => event.deliveryMode === 'digest')) { + const marked = await AgentEvent.updateOne( + { _id: firstWake._id, status: 'delivered', deliveryNonce: firstWake.deliveryNonce ?? null }, + { $set: { digestSnapshotAt } }, + ).catch((err: Error) => { + console.warn('[agent-event] digest snapshot mark failed:', err.message); + return null; + }); + if (marked?.modifiedCount) firstWake.digestSnapshotAt = digestSnapshotAt; + } const foldedRollup = onePod && firstWake ? await this.attachFoldedDigest(firstWake).catch((err: Error) => { console.warn('[agent-event] folded digest attachment failed:', err.message); @@ -2167,11 +2243,9 @@ class AgentEventService { ) as EventDoc | null; const consumedDigest = !normalizedDelivery - || normalizedDelivery.outcome === 'posted' - || normalizedDelivery.outcome === 'acknowledged' - || (normalizedDelivery.outcome === 'no_action' && !normalizedDelivery.reason); + || digestConsumedBy(normalizedDelivery.outcome, normalizedDelivery.reason); if (result && consumedDigest) { - if (result.digestEventIds?.length) { + if (result.digestSnapshotAt || result.digestEventIds?.length) { await this.settleLegacyDigest(result).catch((err: Error) => { console.warn('[agent-event] legacy digest settle failed:', err.message); }); diff --git a/cli/__tests__/claim-outcome.test.mjs b/cli/__tests__/claim-outcome.test.mjs index 4516af4d52..c2e8bd152a 100644 --- a/cli/__tests__/claim-outcome.test.mjs +++ b/cli/__tests__/claim-outcome.test.mjs @@ -140,6 +140,15 @@ describe('claimReleaseFor', () => { expect(claimReleaseFor(humanBroadcast, { outcome: 'no_action' })).toEqual({ outcome: 'declined' }); }); + // Empty output carries its own reason so a digest is not read on it (#2753), + // but it must still hand a human's message to another listener. + test('empty output on a human broadcast declines, and completes elsewhere', () => { + expect(claimReleaseFor(humanBroadcast, { outcome: 'no_action', reason: 'empty-output' })) + .toEqual({ outcome: 'declined' }); + expect(claimReleaseFor(agentMention, { outcome: 'no_action', reason: 'empty-output' })) + .toEqual({ outcome: 'completed' }); + }); + test('a skip that is not a refusal completes — the paired control', () => { // `claim-held` and `duplicate-delivery` carry a reason but nothing was // refused. diff --git a/cli/__tests__/lib.test.mjs b/cli/__tests__/lib.test.mjs index 997016316d..1c5466f513 100644 --- a/cli/__tests__/lib.test.mjs +++ b/cli/__tests__/lib.test.mjs @@ -343,7 +343,7 @@ describe('session-store.js handled-event ring', () => { // ── webhook-server.js tests ─────────────────────────────────────────────────── -const { startWebhookServer, forwardToLocalWebhook } = await import('../src/lib/webhook-server.js'); +const { startWebhookServer, forwardToLocalWebhook, logOnlyHandler } = await import('../src/lib/webhook-server.js'); describe('webhook-server.js', () => { test('startWebhookServer starts and resolves with { server, url, close }', async () => { @@ -456,6 +456,56 @@ describe('webhook-server.js', () => { await new Promise((r) => server.close(r)); }); + + // A body without a stated outcome must not read the pod digest (#2753). + test.each([ + ['an empty body', 204, ''], + ['a non-JSON body', 200, 'ok'], + ['JSON null', 200, 'null'], + ['a JSON array', 200, '[]'], + ['an object without an outcome', 200, '{"content":"hi"}'], + ['a non-string outcome', 200, '{"outcome":5}'], + ])('forwardToLocalWebhook reads %s as empty output', async (_label, status, body) => { + const server = http.createServer((req, res) => { + req.resume(); + req.on('end', () => { res.writeHead(status); res.end(body); }); + }); + await new Promise((r) => server.listen(0, r)); + const url = `http://localhost:${server.address().port}/cap`; + try { + expect(await forwardToLocalWebhook({ _id: 'evt-1', type: 'chat.mention' }, url, null)) + .toEqual({ outcome: 'no_action', reason: 'empty-output' }); + } finally { + await new Promise((r) => server.close(r)); + } + }); + + test('startWebhookServer answers empty output for a handler that returns nothing', async () => { + const { server, url } = await startOnFreePort({ onEvent: async () => undefined }); + try { + expect(await forwardToLocalWebhook({ _id: 'evt-1', type: 'chat.mention' }, url, null)) + .toEqual({ outcome: 'no_action', reason: 'empty-output' }); + } finally { + await new Promise((r) => server.close(r)); + } + }); + + test.each([ + ['{"outcome":"acknowledged"}', { outcome: 'acknowledged' }], + ['{"outcome":"no_action"}', { outcome: 'no_action' }], + ])('forwardToLocalWebhook keeps a stated outcome: %s', async (body, expected) => { + const server = http.createServer((req, res) => { + req.resume(); + req.on('end', () => { res.writeHead(200, { 'Content-Type': 'application/json' }); res.end(body); }); + }); + await new Promise((r) => server.listen(0, r)); + const url = `http://localhost:${server.address().port}/cap`; + try { + expect(await forwardToLocalWebhook({ _id: 'evt-1', type: 'chat.mention' }, url, null)).toEqual(expected); + } finally { + await new Promise((r) => server.close(r)); + } + }); }); // ── poller.js tests ─────────────────────────────────────────────────────────── @@ -649,6 +699,57 @@ describe('poller.js', () => { ]); }); + test('a handler that returns nothing acks the wake as empty output and leaves the digest unread', async () => { + const events = [ + { ...makeEvent('wake-1'), deliveryMode: 'wake' }, + { ...makeEvent('digest-1'), deliveryMode: 'digest', payload: { content: 'ambient' } }, + ]; + const mockPost = jest.fn().mockResolvedValue({}); + const onEvent = jest.fn().mockResolvedValue(undefined); + createClient.mockReturnValue({ get: jest.fn().mockResolvedValue({ events }), post: mockPost }); + const { stop } = startPoller({ + instanceUrl: 'http://localhost:5000', token: 'cm_test', agentName: 'my-agent', + intervalMs: 100_000, onEvent, + }); + await new Promise((r) => setTimeout(r, 50)); + stop(); + + expect(mockPost.mock.calls).toEqual([ + ['/api/agents/runtime/events/wake-1/ack', { result: { outcome: 'no_action', reason: 'empty-output' } }], + ]); + }); + + // `agent connect` forwards to its own log-only server; nothing read the digest. + test('a wake forwarded through the agent connect server leaves the digest unread', async () => { + const port = await new Promise((resolve) => { + const probe = http.createServer(); + probe.listen(0, () => { const p = probe.address().port; probe.close(() => resolve(p)); }); + }); + const log = jest.spyOn(console, 'log').mockImplementation(() => {}); + const { server, url } = await startWebhookServer({ port, onEvent: logOnlyHandler }); + const events = [ + { ...makeEvent('wake-1'), deliveryMode: 'wake' }, + { ...makeEvent('digest-1'), deliveryMode: 'digest', payload: { content: 'ambient' } }, + ]; + const mockPost = jest.fn().mockResolvedValue({}); + createClient.mockReturnValue({ get: jest.fn().mockResolvedValue({ events }), post: mockPost }); + try { + const { stop } = startPoller({ + instanceUrl: 'http://localhost:5000', token: 'cm_test', agentName: 'my-agent', + intervalMs: 100_000, onEvent: (event) => forwardToLocalWebhook(event, url, null), + }); + await new Promise((r) => setTimeout(r, 150)); + stop(); + } finally { + log.mockRestore(); + await new Promise((r) => server.close(r)); + } + + expect(mockPost.mock.calls).toEqual([ + ['/api/agents/runtime/events/wake-1/ack', { result: { outcome: 'no_action', reason: 'empty-output' } }], + ]); + }); + test('does not forward or acknowledge another pod\'s digest with a wake', async () => { const events = [ { ...makeEvent('wake-1'), deliveryMode: 'wake', payload: { content: 'pod A' } }, diff --git a/cli/__tests__/run-loop.test.mjs b/cli/__tests__/run-loop.test.mjs index 45e6235d8e..7612e23ff0 100644 --- a/cli/__tests__/run-loop.test.mjs +++ b/cli/__tests__/run-loop.test.mjs @@ -282,7 +282,7 @@ describe('performRun', () => { expect(lines.some((line) => line.includes('upstream refused'))).toBe(false); expect(mockPost).toHaveBeenCalledWith( '/api/agents/runtime/events/evt-1/ack', - { result: { outcome: 'no_action' } }, + { result: { outcome: 'no_action', reason: 'empty-output' } }, ); }); @@ -475,6 +475,79 @@ describe('performRun', () => { expect(ackPosts(post).map(([route]) => route)).toEqual(['/api/agents/runtime/events/evt-refused/ack']); }); + // The server answers an intentional NO_REPLY with `silent_or_empty`. That + // seat read the batch and chose silence, so its digest is read; any other + // skip leaves the digest for the next turn (rhea, 2026-10-25). + test.each([ + ['silent_or_empty', true], + ['duplicate_recent', false], + ])('a server skip as %s marks the digest read: %s', async (reason, read) => { + const events = [ + makeEvent({ _id: 'evt-wake', payload: { content: 'question' } }), + makeEvent({ _id: 'evt-ambient', type: 'message.posted', deliveryMode: 'digest', + payload: { content: 'background' } }), + ]; + const get = jest.fn(async (route) => ( + route.endsWith('/messages') ? { messages: [] } : { events, inboxCount: 2 } + )); + const post = jest.fn(async (route) => ( + route === '/api/agents/runtime/pods/pod-abc/messages' + ? { success: true, skipped: true, reason } + : {} + )); + createClient.mockReturnValue({ get, post }); + const spawn = jest.fn(async () => ({ text: 'Nothing to add.' })); + const { stop } = performRun({ + instanceUrl: 'http://localhost:5000', token: 'cm_agent_test', + adapter: { name: 'stub', detect: stubAdapter.detect, spawn }, + agentName: 'my-stub', setTimeoutImpl: noopTimeout, + }); + await drainMicrotasks(); + stop(); + + expect(spawn).toHaveBeenCalledTimes(1); + const acked = ackPosts(post).map(([route]) => route); + expect(acked).toContain('/api/agents/runtime/events/evt-wake/ack'); + expect(acked.includes('/api/agents/runtime/events/evt-ambient/ack')).toBe(read); + if (read) { + expect(post).toHaveBeenCalledWith('/api/agents/runtime/events/evt-ambient/ack', { + result: { outcome: 'no_action', reason: 'digest-read' }, + }); + } + }); + + // Only the sentinel is a choice. A seat that prints nothing every turn (a + // crash, a timeout, a lost route that says nothing) must not drain the + // digest it never saw (sable, #2753). + test.each([ + ['NO_REPLY', { outcome: 'no_action' }, true], + ['', { outcome: 'no_action', reason: 'empty-output' }, false], + ])('a reply of %j acks the wake as %j and reads the digest: %s', async (text, wakeResult, read) => { + const events = [ + makeEvent({ _id: 'evt-wake', payload: { content: 'question' } }), + makeEvent({ _id: 'evt-ambient', type: 'message.posted', deliveryMode: 'digest', + payload: { content: 'background' } }), + ]; + const get = jest.fn(async (route) => ( + route.endsWith('/messages') ? { messages: [] } : { events, inboxCount: 2 } + )); + const post = jest.fn(async () => ({})); + createClient.mockReturnValue({ get, post }); + const spawn = jest.fn(async () => ({ text })); + const { stop } = performRun({ + instanceUrl: 'http://localhost:5000', token: 'cm_agent_test', + adapter: { name: 'stub', detect: stubAdapter.detect, spawn }, + agentName: 'my-stub', setTimeoutImpl: noopTimeout, + }); + await drainMicrotasks(); + stop(); + + expect(spawn).toHaveBeenCalledTimes(1); + expect(post).toHaveBeenCalledWith('/api/agents/runtime/events/evt-wake/ack', { result: wakeResult }); + const acked = ackPosts(post).map(([route]) => route); + expect(acked.includes('/api/agents/runtime/events/evt-ambient/ack')).toBe(read); + }); + test('a lost binding item stays read-only context while another batch item may run', async () => { const events = [ makeEvent({ diff --git a/cli/src/commands/agent.js b/cli/src/commands/agent.js index db97652ab0..6a365f6dae 100644 --- a/cli/src/commands/agent.js +++ b/cli/src/commands/agent.js @@ -32,7 +32,7 @@ import { import { getToken, resolveInstanceUrl } from '../lib/config.js'; import { startPoller, terminalDeliveryAckError } from '../lib/poller.js'; import { formatDigestFrame } from '../lib/digest-frame.js'; -import { startWebhookServer, forwardToLocalWebhook } from '../lib/webhook-server.js'; +import { startWebhookServer, forwardToLocalWebhook, logOnlyHandler } from '../lib/webhook-server.js'; import { getAdapter, listAdapterNames } from '../lib/adapters/index.js'; import { getSession, @@ -76,7 +76,7 @@ import { formatPodFocusFrame, readPodFocus, } from '../lib/pod-focus.js'; -import { claimReleaseFor, ackResultFor } from '../lib/claim-outcome.js'; +import { claimReleaseFor, ackResultFor, EMPTY_OUTPUT } from '../lib/claim-outcome.js'; import { prepareCommitAttribution } from '../lib/commit-attribution.js'; import { detectBwrap } from '../lib/sandbox/bwrap.js'; import { resolvePublicSandboxMode } from '../lib/sandbox/mode.js'; @@ -2214,6 +2214,13 @@ export const performRun = ({ }, }; } + // Empty output is not the sentinel. A seat that answered `NO_REPLY` read + // its batch and chose silence; one that printed nothing may have crashed, + // timed out or lost its route without saying so. The bare `no_action` + // stays the sentinel's alone, because a digest counts as read only on + // that (sable, #2753). Release, hand-off and spawn counting treat + // `empty-output` exactly as they treat a bare decline. + if (!delivered && !replyText) return { outcome: 'no_action', reason: EMPTY_OUTPUT }; return { outcome: delivered ? 'posted' : 'no_action' }; }; @@ -2681,9 +2688,15 @@ export const performRun = ({ for (const entry of actionEntries) entry.result = turnResult; // A digest was context in this successful waking turn, never an // independent action. If the waking work stood down or the provider - // refused the batch, leave it unread for the next real turn. + // refused the batch, leave it unread for the next real turn. A reply + // the server skipped as `silent_or_empty` is the seat choosing silence + // (the server's answer to an intentional NO_REPLY), so it read the + // digest. Empty output (`empty-output`) is not a choice anyone can see: + // a broken seat produces it every turn, and reading on it would drain + // the backlog that seat never saw. if (turnResult?.outcome === 'posted' - || (turnResult?.outcome === 'no_action' && !turnResult?.reason)) { + || (turnResult?.outcome === 'no_action' + && (!turnResult?.reason || turnResult.reason === 'silent_or_empty'))) { for (const entry of activeEntries) { if (entry.event.deliveryMode === 'digest') { entry.result = { outcome: 'no_action', reason: 'digest-read' }; @@ -2914,7 +2927,8 @@ export const performRun = ({ } const batchSpawned = batch.entries.some((entry) => ( entry.result?.outcome === 'posted' - || (entry.result?.outcome === 'no_action' && !entry.result?.reason) + || (entry.result?.outcome === 'no_action' + && (!entry.result?.reason || entry.result.reason === EMPTY_OUTPUT)) )); if (batchSpawned) { consecutiveSpawnFailures = 0; @@ -3462,11 +3476,7 @@ Docs: port, path: opts.path, secret: opts.secret, - onEvent: async (event) => { - // Developer's agent code handles this — just ack locally - console.log(`[local] ${event.type} from pod ${event.podId}`); - return { outcome: 'acknowledged' }; - }, + onEvent: logOnlyHandler, onReady: (url) => console.log(`Local server ready: ${url}`), }); diff --git a/cli/src/lib/claim-outcome.js b/cli/src/lib/claim-outcome.js index 37e4c482b9..e2fda35853 100644 --- a/cli/src/lib/claim-outcome.js +++ b/cli/src/lib/claim-outcome.js @@ -20,8 +20,9 @@ * finite. On any other wake it is terminal. The routing is * the kernel's: this module only names the outcome. * - `declined` — a silent, normally completed human broadcast. The agent - * chose not to answer, so the kernel hands it to exactly one - * remaining original listener. + * chose not to answer, or printed nothing (`empty-output`), + * so the kernel hands it to exactly one remaining original + * listener. * - `completed` — everything else that produced a result: a posted reply, or * a skip with a reason (`no-prompt`, `claim-held`, * `duplicate-delivery`) where nothing was refused. @@ -43,6 +44,13 @@ export const REFUSAL_REASONS = ['upstream-refused', 'cascade-cap', 'delivery-ref /** What an unclassifiable refusal is recorded as, rather than a fourth class. */ export const DEFAULT_REFUSAL_REASON = 'delivery-refused'; +/** + * The reason a turn that printed nothing carries. It releases like a bare + * decline, so a human's message still reaches another listener, but it is not + * the `NO_REPLY` sentinel and does not mark a digest read. + */ +export const EMPTY_OUTPUT = 'empty-output'; + /** * The range the kernel accepts for a refusal's `status` — its * `MIN_UPSTREAM_STATUS` / `MAX_UPSTREAM_STATUS` in `messageClaimService`. @@ -93,7 +101,7 @@ export const claimReleaseFor = (event, turnResult) => { if (event?.type === 'message.posted' && event.payload?.senderIsHuman === true && turnResult.outcome === 'no_action' - && !turnResult.reason) { + && (!turnResult.reason || turnResult.reason === EMPTY_OUTPUT)) { return { outcome: 'declined' }; } return { outcome: 'completed' }; diff --git a/cli/src/lib/poller.js b/cli/src/lib/poller.js index b7efe9a464..9c6e2fd982 100644 --- a/cli/src/lib/poller.js +++ b/cli/src/lib/poller.js @@ -8,6 +8,7 @@ import { createClient } from './api.js'; import { formatDigestFrame } from './digest-frame.js'; +import { EMPTY_OUTPUT } from './claim-outcome.js'; // Delivery failures that must stop a consumer rather than fall through to the // normal at-least-once retry path. A stale delivery belongs to a replacement @@ -80,7 +81,8 @@ export const startPoller = ({ } : event; let result = { outcome: 'no_action' }; try { - result = (await onEvent(forwarded)) || { outcome: 'no_action' }; + // A handler that returns nothing said nothing: not a read of the digest. + result = (await onEvent(forwarded)) || { outcome: 'no_action', reason: EMPTY_OUTPUT }; } catch (handlerErr) { result = { outcome: 'error', reason: handlerErr.message }; } diff --git a/cli/src/lib/webhook-server.js b/cli/src/lib/webhook-server.js index 84d7066549..0ac54d8a39 100644 --- a/cli/src/lib/webhook-server.js +++ b/cli/src/lib/webhook-server.js @@ -8,6 +8,7 @@ import { createServer } from 'http'; import { createHmac } from 'crypto'; +import { EMPTY_OUTPUT } from './claim-outcome.js'; export const startWebhookServer = ({ port = 3001, @@ -47,9 +48,10 @@ export const startWebhookServer = ({ return; } - let result = { outcome: 'acknowledged' }; + // A handler that returns nothing said nothing (#2753). + let result = { outcome: 'no_action', reason: EMPTY_OUTPUT }; try { - result = (await onEvent(event)) || { outcome: 'acknowledged' }; + result = (await onEvent(event)) || { outcome: 'no_action', reason: EMPTY_OUTPUT }; } catch (err) { result = { outcome: 'error', reason: err.message }; } @@ -67,6 +69,17 @@ export const startWebhookServer = ({ }); }); +/** + * The handler `agent connect` serves on its own port. It only logs what + * arrives, and no agent has read the event, so it states no outcome a read + * would count as: the wake still settles, and the pod digest stays unread + * (Sable, #2753). + */ +export const logOnlyHandler = async (event) => { + console.log(`[local] ${event.type} from pod ${event.podId}`); + return { outcome: 'no_action', reason: EMPTY_OUTPUT }; +}; + /** * Forward a polled event to a local webhook server. * Used by `agent connect` to bridge polling → local HTTP. @@ -84,5 +97,13 @@ export const forwardToLocalWebhook = async (event, webhookUrl, secret = null) => const res = await fetch(webhookUrl, { method: 'POST', headers, body: payload }); if (!res.ok) throw new Error(`Local webhook returned HTTP ${res.status}`); - try { return await res.json(); } catch { return { outcome: 'acknowledged' }; } + // Only a stated outcome counts. An empty or non-JSON body, or one without + // an outcome, is a handler that said nothing, the same as a CLI turn that + // printed nothing, so it must not read the pod digest (Rhea, #2753). + let body = null; + try { body = await res.json(); } catch { body = null; } + if (!body || typeof body !== 'object' || typeof body.outcome !== 'string') { + return { outcome: 'no_action', reason: EMPTY_OUTPUT }; + } + return body; };