diff --git a/packages/agent-runtime/src/__tests__/run-agent-step-unfinished-tool-calls.test.ts b/packages/agent-runtime/src/__tests__/run-agent-step-unfinished-tool-calls.test.ts index 90afa94d21..af636da89e 100644 --- a/packages/agent-runtime/src/__tests__/run-agent-step-unfinished-tool-calls.test.ts +++ b/packages/agent-runtime/src/__tests__/run-agent-step-unfinished-tool-calls.test.ts @@ -208,4 +208,45 @@ describe('runAgentStep - unfinished tool calls', () => { capturedMessages.filter((message) => message.role === 'tool'), ).toHaveLength(2) }) + + it('stubs stale bulky read results but keeps small and newest ones verbatim', async () => { + const readResult = (toolCallId: string, content: string): Message => ({ + role: 'tool', + toolCallId, + toolName: 'read_files', + content: [{ type: 'json', value: [{ path: 'src/app.ts', content }] }], + }) + const bigRead = (marker: string) => `${marker}\n`.repeat(1_000) + const agentState = await runWithHistory([ + userMessage('do the thing'), + // Five bulky reads (~17k each) so past-window savings clear the + // pass's 20k floor; the newest-3 window is r3, r4 and the small + // read, which rides along despite its size floor being separate. + ...['first', 'second', 'third', 'fourth', 'fifth'].flatMap((name, i) => [ + toolCallMessage(`r${i}`), + readResult(`r${i}`, bigRead(`const ${name} = ${i};`)), + ]), + toolCallMessage('small'), + readResult('small', 'export const tiny = 1'), + userMessage('are you there?'), + ]) + + const request = JSON.stringify(capturedMessages) + // The three oldest bulky reads (newest-3 window = r3, r4, small) were + // replaced by the omit stub already understood by getPreviouslyReadFiles + // and the fitToolResults re-read hints... + expect(request).not.toContain('const first = 0;') + expect(request).not.toContain('const second = 1;') + expect(request).not.toContain('const third = 2;') + expect(request).toContain('contentOmittedForLength') + // ...the two newest bulky reads stay verbatim... + expect(request.split('const fourth = 3;').length).toBeGreaterThan(10) + expect(request.split('const fifth = 4;').length).toBeGreaterThan(10) + // ...and the small read rides along inside the window. + expect(request).toContain('export const tiny = 1') + // The stored history is healed to the same state. + const stored = JSON.stringify(agentState.messageHistory) + expect(stored).not.toContain('const first = 0;') + expect(stored).toContain('export const tiny = 1') + }) }) diff --git a/packages/agent-runtime/src/run-agent-step.ts b/packages/agent-runtime/src/run-agent-step.ts index 5863bc99c0..0f10e8dcbe 100644 --- a/packages/agent-runtime/src/run-agent-step.ts +++ b/packages/agent-runtime/src/run-agent-step.ts @@ -145,6 +145,7 @@ import { assertUserContentSize, UserContentTooLargeError, } from './util/context-size-guard' +import { simplifyOldToolResults } from './util/simplify-tool-results' import { countTokens, countTokensJson, @@ -384,9 +385,21 @@ export const runAgentStep = async ( // and replayed, that one orphan would fail every later turn, so drop it here // — the single point every step's request is built — and assign the cleaned // history back so the checkpointed state is valid too. - const history = dropUnansweredToolCalls( - expireMessages(agentState.messageHistory, 'agentStep'), - ) + // In a long session, stub tool results older than the newest few with the + // omit shapes fitToolResults/getPreviouslyReadFiles already understand: + // read_files and terminal outputs up to 100k/50k chars otherwise stay + // verbatim in history and are re-sent on every request until compaction. + // The pass waits for real past-window volume (see simplifyOldToolResults), + // so a history that still fits the window comfortably keeps everything; + // the newest results and anything this step later appends are untouched, + // and unreplaced messages keep reference identity for the extension check + // below. + const history = simplifyOldToolResults({ + messages: dropUnansweredToolCalls( + expireMessages(agentState.messageHistory, 'agentStep'), + ), + logger, + }) const agentMessagesUntruncated = buildArray( ...history, @@ -437,8 +450,7 @@ export const runAgentStep = async ( const systemTokens = countTokens(system) let cacheDebugCorrelation: - | ReturnType - | undefined + ReturnType | undefined if (CACHE_DEBUG_FULL_LOGGING) { try { cacheDebugCorrelation = createCacheDebugSnapshot({ diff --git a/packages/agent-runtime/src/util/__tests__/simplify-old-tool-results.test.ts b/packages/agent-runtime/src/util/__tests__/simplify-old-tool-results.test.ts new file mode 100644 index 0000000000..5f5b9e7399 --- /dev/null +++ b/packages/agent-runtime/src/util/__tests__/simplify-old-tool-results.test.ts @@ -0,0 +1,329 @@ +import { describe, expect, it } from 'bun:test' + +import { simplifyOldToolResults } from '../simplify-tool-results' + +import type { + CodebuffToolMessage, + CodebuffToolOutput, +} from '@codebuff/common/tools/list' +import type { Logger } from '@codebuff/common/types/contracts/logger' +import type { + Message, + ToolMessage, +} from '@codebuff/common/types/messages/codebuff-message' + +const logger: Logger = { + debug: () => {}, + info: () => {}, + warn: () => {}, + error: () => {}, +} as unknown as Logger + +type ReadFilesValue = Extract< + CodebuffToolOutput<'read_files'>[number], + { type: 'json' } +>['value'] + +function readFilesResult(id: string, files: ReadFilesValue): Message { + return { + role: 'tool', + toolName: 'read_files', + toolCallId: id, + content: [{ type: 'json', value: files }], + } as ToolMessage +} + +// The pass holds until past-window savings clear 20k chars, so fixtures that +// expect stubbing put several 8k results beyond the window. +const BULK = 'x'.repeat(8_000) + +function fullRead(id: string, ...paths: string[]): Message { + return readFilesResult( + id, + paths.map((path) => ({ path, content: `contents of ${path} ${BULK}` })), + ) +} + +function stubbedRead(id: string, ...paths: string[]): Message { + return readFilesResult( + id, + paths.map((path) => ({ path, contentOmittedForLength: true as const })), + ) +} + +function terminalResult( + id: string, + command: string, + overrides?: Record, +): Message { + return { + role: 'tool', + toolName: 'run_terminal_command', + toolCallId: id, + content: [ + { + type: 'json', + value: { + command, + stdout: `output of ${command} ${BULK}`, + exitCode: 0, + ...overrides, + }, + }, + ], + } as ToolMessage +} + +function toolContent(message: Message) { + return (message as CodebuffToolMessage).content[0] as { + type: string + value: any + } +} + +describe('simplifyOldToolResults', () => { + it('stubs terminal results older than the newest keep window', () => { + const messages = [ + ...Array.from({ length: 9 }, (_, i) => + terminalResult(`t${i}`, `cmd${i}`), + ), + ] + + const result = simplifyOldToolResults({ messages, logger }) + + expect(result).not.toBe(messages) + for (let i = 0; i < 9; i++) { + const value = toolContent(result[i]).value + if (i < 4) { + // Oldest four fall outside the newest-5 window. + expect(value).toEqual({ + command: `cmd${i}`, + stdoutOmittedForLength: true, + exitCode: 0, + }) + } else { + expect((value as { stdout: string }).stdout).toContain( + `output of cmd${i}`, + ) + expect(result[i]).toBe(messages[i]) + } + } + }) + + it('stubs read_files results older than the newest keep window', () => { + const messages = [ + ...Array.from({ length: 7 }, (_, i) => fullRead(`r${i}`, `src/f${i}.ts`)), + ] + + const result = simplifyOldToolResults({ messages, logger }) + + expect(result).not.toBe(messages) + for (let i = 0; i < 7; i++) { + const value = toolContent(result[i]).value + if (i < 4) { + // Oldest four fall outside the newest-3 window. + expect(value).toEqual([ + { path: `src/f${i}.ts`, contentOmittedForLength: true }, + ]) + } else { + expect(value[0].content).toContain(`contents of src/f${i}.ts`) + expect(result[i]).toBe(messages[i]) + } + } + }) + + it('keeps each tool type independently of the other', () => { + const messages = [ + // 7 reads + 6 terminals interleaved; both windows are exceeded, but + // each tool's window never counts the other tool's results. + ...Array.from({ length: 6 }, (_, i) => [ + fullRead(`r${i}`, `src/f${i}.ts`), + terminalResult(`t${i}`, `cmd${i}`), + ]).flat(), + fullRead('r6', 'src/f6.ts'), + ] + + const result = simplifyOldToolResults({ messages, logger }) + + expect(result).not.toBe(messages) + for (let i = 0; i < 6; i++) { + const read = result[i * 2] + const terminal = result[i * 2 + 1] + if (i >= 4) { + expect(read).toBe(messages[i * 2]) + } else { + expect(toolContent(read).value).toEqual([ + { path: `src/f${i}.ts`, contentOmittedForLength: true }, + ]) + } + if (i >= 1) { + expect(terminal).toBe(messages[i * 2 + 1]) + } else { + expect(toolContent(terminal).value).toEqual({ + command: `cmd${i}`, + stdoutOmittedForLength: true, + exitCode: 0, + }) + } + } + // The trailing read is the newest of seven: verbatim. + expect(result[12]).toBe(messages[12]) + }) + + it('passes already-stubbed results through with reference identity and without consuming window slots', () => { + const messages = [ + stubbedRead('already-stubbed', 'src/stubbed.ts'), + fullRead('r1', 'src/f1.ts'), + fullRead('r2', 'src/f2.ts'), + fullRead('r3', 'src/f3.ts'), + ] + + const result = simplifyOldToolResults({ messages, logger }) + + // Nothing to do: same array reference back. + expect(result).toBe(messages) + expect(result[0]).toBe(messages[0]) + expect(result[0]).toEqual(stubbedRead('already-stubbed', 'src/stubbed.ts')) + }) + + it('leaves terminal background-process and error results verbatim', () => { + const messages = [ + terminalResult('bg', 'npm start', { + processId: 12345, + backgroundProcessStatus: 'running', + }), + terminalResult('err', 'invalid-command', { + errorMessage: 'Command not found', + }), + ...Array.from({ length: 8 }, (_, i) => + terminalResult(`t${i}`, `cmd${i}`), + ), + ] + + const result = simplifyOldToolResults({ messages, logger }) + + expect(result[0]).toBe(messages[0]) + expect(result[1]).toBe(messages[1]) + // The three oldest regular results exceed the newest-5 window; the + // background and error entries never consume window slots. + for (let i = 2; i < 5; i++) { + expect(toolContent(result[i]).value).toEqual({ + command: `cmd${i - 2}`, + stdoutOmittedForLength: true, + exitCode: 0, + }) + } + for (let i = 5; i < 10; i++) { + expect(result[i]).toBe(messages[i]) + } + }) + + it('preserves reference identity for every unreplaced message', () => { + const user = { + role: 'user', + content: [{ type: 'text', text: 'do the thing' }], + } as Message + const otherTool = { + role: 'tool', + toolName: 'code_search', + toolCallId: 'cs1', + content: [{ type: 'json', value: { results: ['huge'] } }], + } as Message + const messages = [ + otherTool, + user, + fullRead('old-read', 'src/old.ts'), + ...Array.from({ length: 3 }, (_, i) => fullRead(`r${i}`, `src/f${i}.ts`)), + ...Array.from({ length: 9 }, (_, i) => + terminalResult(`t${i}`, `cmd${i}`), + ), + ] + const snapshot = [...messages] + + const result = simplifyOldToolResults({ messages, logger }) + + expect(result.length).toBe(messages.length) + for (let i = 0; i < result.length; i++) { + if (i === 2) { + // 4th-newest read of 4: stubbed. + expect(result[i]).not.toBe(messages[i]) + expect(toolContent(result[i]).value).toEqual([ + { path: 'src/old.ts', contentOmittedForLength: true }, + ]) + } else if (i >= 6 && i <= 9) { + // 6th-through-9th-newest terminals: stubbed. + expect(result[i]).not.toBe(messages[i]) + expect(toolContent(result[i]).value).toEqual({ + command: `cmd${i - 6}`, + stdoutOmittedForLength: true, + exitCode: 0, + }) + } else { + // Everything else — user, other tools, kept window — same reference. + expect(result[i]).toBe(messages[i]) + } + } + // Input array and shared messages are untouched (history snapshots safe). + expect(messages).toEqual(snapshot) + expect(messages[2]).toBe(snapshot[2]) + expect(toolContent(messages[2]).value[0].content).toContain( + 'contents of src/old.ts', + ) + }) + + it('leaves the history alone until past-window savings clear the floor', () => { + // A handful of bulky results might clear the per-result keep window, but + // stubbing one modest result trades content the model may still read for + // negligible re-send savings — and forfeits fitToolResults' graded + // prefix-clip in a tight window. + const messages = [ + ...Array.from({ length: 7 }, (_, i) => + terminalResult(`t${i}`, `cmd${i}`), + ), + fullRead('r0', 'src/f0.ts'), + ...Array.from({ length: 3 }, (_, i) => + fullRead(`r${i + 1}`, `src/f${i + 1}.ts`), + ), + ] + // past-window: 2 terminals + 1 read ≈ 24k > 20k floor — clipped version + // would differ; below shows the floor by removing one bulky terminal. + const belowFloor = messages.slice(1) + + expect(simplifyOldToolResults({ messages, logger })).not.toBe(messages) + expect(simplifyOldToolResults({ messages: belowFloor, logger })).toBe( + belowFloor, + ) + }) + + it('honours explicit keep-window sizes', () => { + const messages = [ + ...Array.from({ length: 5 }, (_, i) => + terminalResult(`t${i}`, `cmd${i}`), + ), + ...Array.from({ length: 5 }, (_, i) => fullRead(`r${i}`, `src/f${i}.ts`)), + ] + + const result = simplifyOldToolResults({ + messages, + numTerminalCommands: 1, + numReadFileResults: 2, + logger, + }) + + for (let i = 0; i < 5; i++) { + const value = toolContent(result[i]).value + if (i < 4) expect(value.stdoutOmittedForLength).toBe(true) + else expect(value.stdout).toContain('output of cmd4') + } + for (let i = 5; i < 10; i++) { + const value = toolContent(result[i]).value + if (i < 8) { + expect(value).toEqual([ + { path: `src/f${i - 5}.ts`, contentOmittedForLength: true }, + ]) + } else { + expect(result[i]).toBe(messages[i]) + } + } + }) +}) diff --git a/packages/agent-runtime/src/util/simplify-tool-results.ts b/packages/agent-runtime/src/util/simplify-tool-results.ts index 8ecd1458d1..33709faa8f 100644 --- a/packages/agent-runtime/src/util/simplify-tool-results.ts +++ b/packages/agent-runtime/src/util/simplify-tool-results.ts @@ -1,8 +1,15 @@ import { getErrorObject } from '@codebuff/common/util/error' -import { cloneDeep } from 'lodash' +import { cloneDeep, isEqual } from 'lodash' -import type { CodebuffToolOutput } from '@codebuff/common/tools/list' +import type { + CodebuffToolMessage, + CodebuffToolOutput, +} from '@codebuff/common/tools/list' import type { Logger } from '@codebuff/common/types/contracts/logger' +import type { + Message, + ToolMessage, +} from '@codebuff/common/types/messages/codebuff-message' export function simplifyReadFileResults( messageContent: CodebuffToolOutput<'read_files'>, @@ -59,3 +66,152 @@ export function simplifyTerminalCommandResults(params: { ] } } + +/** + * simplifyReadFileResults assumes content[0] is the json file list. Real + * histories can also carry media parts (images read as files); never feed + * those to the simplifier. + */ +function readFilesContentSimplifiable( + content: CodebuffToolOutput<'read_files'>, +): boolean { + const first = content[0] + return ( + first?.type === 'json' && + Array.isArray((first as { value?: unknown }).value) + ) +} + +// Number of terminal command outputs to keep in full form before simplifying. +// Scanned from the newest message backwards; the per-stub baseline matches +// simplifyTerminalHelper in util/messages.ts (background-process and error +// results simplify to themselves and stay outside the keep budget). +const numTerminalCommandsToKeep = 5 + +// Number of read_files results to keep verbatim, newest first. Matches the +// terminal-command convention (newest K verbatim, rest stubbed), with a +// smaller window: the read-path stub retains every path, and an omitted read +// can be re-read exactly with offset/limit, whereas a stubbed command cannot +// be repeated safely. +const numReadFileResultsToKeep = 3 + +// Only apply the pass once the past-window results it would stub carry real +// volume. Stubbing trades content for tokens on every later step: worth it +// when a long session would otherwise re-send hundreds of kilobytes, not when +// the savings are one modest result and the model may still want what it read +// a moment ago. On overflow a verbatim result also keeps fitToolResults' +// graded prefix-clip available, while a stub has nothing left to clip. +const minSavingsCharsToSimplify = 20_000 + +/** Approximate serialized size of a result without copying it. */ +function contentChars(content: unknown[]): number { + return JSON.stringify(content).length +} + +/** + * Replaces read_files and run_terminal_command results older than the newest + * few with the omit stubs ({ contentOmittedForLength: true } / + * { stdoutOmittedForLength: true }) that getPreviouslyReadFiles and the + * re-read hints already understand, so a long session stops re-sending stale + * multi-kilobyte outputs on every step until compaction. + * + * Pure copy-on-write: messages left alone keep reference identity, and the + * input array itself (same reference) is returned when nothing needed + * stubbing, so callers that detect history extension by === or + * every((m, i) => history[i] === m) see exactly the edits made here. Nothing + * from a caller's in-flight step should be included in `messages`. + */ +export function simplifyOldToolResults(params: { + messages: Message[] + numTerminalCommands?: number + numReadFileResults?: number + logger: Logger +}): Message[] { + const { messages, logger } = params + const numTerminalCommands = + params.numTerminalCommands ?? numTerminalCommandsToKeep + const numReadFileResults = + params.numReadFileResults ?? numReadFileResultsToKeep + + const stub = ( + message: ToolMessage, + ): + | { kind: 'run_terminal_command'; stubbed: ToolMessage } + | { kind: 'read_files'; stubbed: ToolMessage } + | undefined => { + if (message.toolName === 'run_terminal_command') { + const content = (message as CodebuffToolMessage<'run_terminal_command'>) + .content + try { + const stubbed = simplifyTerminalCommandResults({ + messageContent: content, + logger, + }) + // Already stubbed, or a background/error result the simplifier keeps + // verbatim (matching simplifyTerminalHelper in util/messages.ts). + return isEqual(stubbed, content) + ? undefined + : { + kind: 'run_terminal_command', + stubbed: { ...message, content: stubbed }, + } + } catch (error) { + logger.error( + { error: getErrorObject(error), message }, + 'Error simplifying old tool result; keeping it verbatim', + ) + return undefined + } + } + if (message.toolName === 'read_files') { + const content = (message as CodebuffToolMessage<'read_files'>).content + if (!readFilesContentSimplifiable(content)) return undefined + try { + const stubbed = simplifyReadFileResults(content) + // Already stubbed (nothing carries it either way). + return isEqual(stubbed, content) + ? undefined + : { kind: 'read_files', stubbed: { ...message, content: stubbed } } + } catch (error) { + logger.error( + { error: getErrorObject(error), message }, + 'Error simplifying old tool result; keeping it verbatim', + ) + return undefined + } + } + return undefined + } + + // The newest few results of each kind stay verbatim; everything older is + // stubbed. First measure what the pass would save, and bail out whole when + // the past-window volume is below minSavingsCharsToSimplify (the quality + // floor above). + let terminalKept = 0 + let readKept = 0 + let savings = 0 + const candidates = new Map>>() + for (let i = messages.length - 1; i >= 0; i--) { + const message = messages[i] + if (message.role !== 'tool') continue + const candidate = stub(message) + if (!candidate) continue + if (candidate.kind === 'run_terminal_command') { + if (terminalKept < numTerminalCommands) { + terminalKept++ + continue + } + } else if (readKept < numReadFileResults) { + readKept++ + continue + } + candidates.set(message, candidate) + savings += + contentChars(message.content) - contentChars(candidate.stubbed.content) + } + if (candidates.size === 0 || savings < minSavingsCharsToSimplify) { + return messages + } + + return messages.map((message) => candidates.get(message)?.stubbed ?? message) +}