From 2e6cf12df5524b7e38d5b31739db0f8a01bb988d Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:45:26 -0700 Subject: [PATCH 1/2] fix(native-runtime): when the recovery round is stopped too, set aside the rest of the web results A guard stop after web reads sets aside the result the guard stops on its own and gives the model one round to answer. On 10-25 that round was stopped again in all 6 of one outside user's guardrail failures: every result it kept passed the check alone, so the guard stops on them together. The turn then failed with nothing posted. Now a stop in that round sets aside every web result left and allows one last round, offered only a post. A third stop ends the run as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4VWLC9MiWprFWXFNjJSnr --- .../nativeRuntimeService.guardStop.test.js | 41 +++++++++++++++++++ backend/services/nativeRuntimeService.ts | 16 ++++++++ 2 files changed, 57 insertions(+) diff --git a/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js b/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js index 4648c82fa..0e69db5f2 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js @@ -298,6 +298,47 @@ describe('a hosted turn the guard stops, or that fails', () => { expect(posted()).toEqual([HOSTED_RUN_FAILED_LINE]); }); + // 10-25: the recovery round was stopped though every kept result passed the + // check on its own. The rest are set aside for one last round. + test('a stop in the extra round sets aside the web results left and answers in one last round', async () => { + chatReplies = [ + toolCalls(fetchCall('c1', CLEAN_URL), fetchCall('c2', LATER_URL)), + toolCalls(fetchCall('c3', STOPPED_URL)), + guardStop, + guardStop, + reply('Here is what I can say without those pages.'), + ]; + + const result = await runAgent(hostedInstall(), mention); + + expect(result.status).toBe('succeeded'); + expect(chatBodies).toHaveLength(5); + const last = JSON.stringify(chatBodies[4].messages); + expect(last).not.toContain(STOPPED_TEXT); + expect(last).not.toContain(CLEAN_TEXT); + expect(last).not.toContain(LATER_TEXT); + expect(offeredIn(chatBodies[4])).toEqual(['commonly_post_message']); + expect(savedRun.webSetAside).toBe(3); + expect(posted()).toEqual(['Here is what I can say without those pages.']); + }); + + test('a third stop ends the run with the fixed line', async () => { + chatReplies = [ + toolCalls(fetchCall('c1', CLEAN_URL)), + toolCalls(fetchCall('c2', STOPPED_URL)), + guardStop, + guardStop, + guardStop, + reply('Never asked.'), + ]; + + const result = await runAgent(hostedInstall(), mention); + + expect(result).toEqual(expect.objectContaining({ status: 'failed', errorKind: 'guardrail_blocked' })); + expect(chatBodies).toHaveLength(5); + expect(posted()).toEqual([HOSTED_RUN_FAILED_LINE]); + }); + test('a stop with no web read checks nothing and ends with the fixed line', async () => { chatReplies = [guardStop]; diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index 88c9af6d0..d2c508651 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -2072,6 +2072,9 @@ export async function runAgent( const webResults: Array<{ round: number; index: number }> = []; const roundEnds: number[] = []; let webSetAside = false; + // Set when the recovery round was stopped too, and every web result left + // was set aside for one last round. + let webSetAsideAll = false; // Returns how many results were set aside, and how many web results the // turn lost in all (those plus the ones in the dropped rounds). const setAsideStoppedWebResult = async (): Promise<{ setAside: number; removed: number }> => { @@ -2187,6 +2190,19 @@ export async function runAgent( run.webSetAside = removed; continue; } + } else if (webSetAside && !webSetAsideAll && !postedViaTool && webResults.length > 0) { + // The recovery round was stopped too, though every result it + // kept passed the check on its own: the guard stops on them + // together. One outside user's research turns failed this way 6 + // times in a day (10-25). Set aside the rest of the web results + // and give the same single round once more. + webResults.forEach(({ index }) => { + messages[index] = { ...messages[index], content: JSON.stringify({ ok: false, error: WEB_SET_ASIDE_NOTICE }) }; + }); + run.webSetAside = Number(run.webSetAside || 0) + webResults.length; + webResults.splice(0, webResults.length); + webSetAsideAll = true; + continue; } run.status = 'failed'; run.errorKind = 'guardrail_blocked'; From 64f382452345b91f292b70c7262215abf3bd698a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Thu, 29 Oct 2026 21:05:24 -0700 Subject: [PATCH 2/2] fix(native-runtime): clear the model's own text after a web read when setting the rest aside Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N4VWLC9MiWprFWXFNjJSnr --- .../nativeRuntimeService.guardStop.test.js | 21 +++++++++++++++++++ backend/services/nativeRuntimeService.ts | 6 ++++++ 2 files changed, 27 insertions(+) diff --git a/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js b/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js index 0e69db5f2..4d5fa272f 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.guardStop.test.js @@ -322,6 +322,27 @@ describe('a hosted turn the guard stops, or that fails', () => { expect(posted()).toEqual(['Here is what I can say without those pages.']); }); + test('text the model wrote after a web read is cleared too, its tool calls kept', async () => { + const quoting = toolCalls(fetchCall('c3', STOPPED_URL)); + quoting.data.choices[0].message.content = `Noting: ${CLEAN_TEXT}`; + chatReplies = [ + toolCalls(fetchCall('c1', CLEAN_URL), fetchCall('c2', LATER_URL)), + quoting, + guardStop, + guardStop, + reply('Answered without the pages.'), + ]; + + const result = await runAgent(hostedInstall(), mention); + + expect(result.status).toBe('succeeded'); + const last = chatBodies[4].messages; + expect(JSON.stringify(last)).not.toContain(CLEAN_TEXT); + // Every tool result still follows the assistant call that asked for it. + const calls = last.filter((m) => m.role === 'assistant').flatMap((m) => (m.tool_calls || []).map((c) => c.id)); + last.filter((m) => m.role === 'tool').forEach((m) => expect(calls).toContain(m.tool_call_id)); + }); + test('a third stop ends the run with the fixed line', async () => { chatReplies = [ toolCalls(fetchCall('c1', CLEAN_URL)), diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index d2c508651..3a0fac186 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -2199,6 +2199,12 @@ export async function runAgent( webResults.forEach(({ index }) => { messages[index] = { ...messages[index], content: JSON.stringify({ ok: false, error: WEB_SET_ASIDE_NOTICE }) }; }); + // The model's own text after the first web result may quote it. + // Keep its tool calls, so every tool result still has its call. + const firstWeb = Math.min(...webResults.map(({ index }) => index)); + for (let i = firstWeb + 1; i < messages.length; i += 1) { + if (messages[i]?.role === 'assistant' && messages[i].content) messages[i] = { ...messages[i], content: null }; + } run.webSetAside = Number(run.webSetAside || 0) + webResults.length; webResults.splice(0, webResults.length); webSetAsideAll = true;