From af082f822ef5eaf2fe559e1432a23ae2025a9f6b Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:11:49 -0700 Subject: [PATCH 1/4] fix(sandbox): a refused E2B sandbox returns a fixed busy line (TASK-596) When E2B refused a sandbox (a concurrency cap, a rate limit, or a create that timed out), the raw SDK error threw out of the tool loop, so the turn failed and the model never got a usable result. Return a fixed "sandbox busy, try again in a minute" tool result instead, so the turn carries on and the model can still reply. Other create failures still throw. The create call also gets its own request timeout. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: dart Agent-Model: claude/claude-haiku-5-5/default --- .../unit/services/turnSandboxService.test.js | 32 ++++++++++++++++++- backend/services/turnSandboxService.ts | 24 +++++++++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/backend/__tests__/unit/services/turnSandboxService.test.js b/backend/__tests__/unit/services/turnSandboxService.test.js index 053367411e..d3dcbe4681 100644 --- a/backend/__tests__/unit/services/turnSandboxService.test.js +++ b/backend/__tests__/unit/services/turnSandboxService.test.js @@ -9,7 +9,9 @@ jest.mock('@e2b/code-interpreter', () => ({ Sandbox: { create: (...args) => mockCreate(...args) }, })); -const { TurnSandbox, MAX_RUNS_PER_TURN, isSandboxConfigured } = require('../../../services/turnSandboxService'); +const { + TurnSandbox, MAX_RUNS_PER_TURN, isSandboxConfigured, SANDBOX_BUSY_MESSAGE, +} = require('../../../services/turnSandboxService'); describe('TurnSandbox', () => { const previousKey = process.env.E2B_API_KEY; @@ -104,6 +106,34 @@ describe('TurnSandbox', () => { expect(mockRunCode).toHaveBeenCalledTimes(MAX_RUNS_PER_TURN); }); + test('a refused sandbox returns the fixed busy line, not the SDK message, and the turn carries on', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const rateLimit = Object.assign(new Error('Rate limit exceeded: concurrent sandbox cap reached (acct 42)'), { name: 'RateLimitError' }); + mockCreate.mockRejectedValueOnce(rateLimit); + + const sandbox = new TurnSandbox({}); + // Resolves, so the tool loop gets a result and the model can still reply. + const result = await sandbox.run('python', 'print(1)'); + expect(result).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); + expect(JSON.stringify(result)).not.toMatch(/acct 42|Rate limit/); + // No sandbox exists, so close has nothing to kill and reports nothing. + expect(await sandbox.close()).toBe(0); + expect(mockKill).not.toHaveBeenCalled(); + + // A later run in the same turn may try again. + expect((await sandbox.run('python', 'print(1)')).ok).toBe(true); + expect(mockCreate).toHaveBeenCalledTimes(2); + warn.mockRestore(); + }); + + test('a create that times out is refused the same way, and the create call is bounded', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + mockCreate.mockRejectedValueOnce(Object.assign(new Error('context deadline exceeded'), { name: 'TimeoutError' })); + expect(await new TurnSandbox({}).run('python', 'print(1)')).toMatchObject({ ok: false, error: SANDBOX_BUSY_MESSAGE }); + expect(mockCreate.mock.calls[0][1].requestTimeoutMs).toBeGreaterThan(0); + warn.mockRestore(); + }); + test('is configured only when the instance has an E2B key', () => { expect(isSandboxConfigured()).toBe(true); delete process.env.E2B_API_KEY; diff --git a/backend/services/turnSandboxService.ts b/backend/services/turnSandboxService.ts index be121fde2e..9a8970fdc3 100644 --- a/backend/services/turnSandboxService.ts +++ b/backend/services/turnSandboxService.ts @@ -23,6 +23,11 @@ export const SANDBOX_TEMPLATE = 'code-interpreter-v1'; const SANDBOX_LIFETIME_MS = 3 * 60 * 1000; const RUN_TIMEOUT_MS = 30_000; const REQUEST_TIMEOUT_MS = 45_000; +// Bounds the create call itself: a hung E2B control plane must not hold the turn. +const CREATE_TIMEOUT_MS = 30_000; +// What the model is told when E2B refuses a sandbox (concurrency cap, rate +// limit, or a create that timed out). The raw SDK message stays server-side. +export const SANDBOX_BUSY_MESSAGE = 'sandbox busy, try again in a minute'; const MAX_CODE_CHARS = 20_000; const STDOUT_LIMIT = 4000; const STDERR_LIMIT = 2000; @@ -51,6 +56,13 @@ export interface RunCodeResult { error?: string; } +// E2B's own refusal classes: RateLimitError (cap or 429), TimeoutError (create +// ran past its request timeout). Anything else is a fault, and still throws. +const isSandboxRefusal = (error: unknown): boolean => { + const { name, status } = (error || {}) as { name?: string; status?: number }; + return name === 'RateLimitError' || name === 'TimeoutError' || status === 429; +}; + const clip = (text: string, limit: number): string => ( text.length > limit ? `${text.slice(0, limit)}\n… [${text.length - limit} more characters]` : text ); @@ -76,6 +88,7 @@ export class TurnSandbox { this.creating = Sandbox.create(SANDBOX_TEMPLATE, { apiKey: process.env.E2B_API_KEY, timeoutMs: SANDBOX_LIFETIME_MS, + requestTimeoutMs: CREATE_TIMEOUT_MS, allowInternetAccess: false, // E2B serves every sandbox port on a public URL unless told not to; // with this off, a URL needs the traffic token only this process holds, @@ -97,7 +110,16 @@ export class TurnSandbox { if (this.runs >= MAX_RUNS_PER_TURN) return refuse(`this turn has used its ${MAX_RUNS_PER_TURN} code runs`); this.runs += 1; - const sandbox = await this.open(); + let sandbox: E2BSandbox; + try { + sandbox = await this.open(); + } catch (error) { + if (!isSandboxRefusal(error)) throw error; + // Forget the failed create, so close() has nothing to kill and a later run may try again. + this.creating = null; + console.warn('[sandbox] E2B refused a sandbox:', (error as Error).name); + return refuse(SANDBOX_BUSY_MESSAGE); + } const execution = await sandbox.runCode(code, { language, timeoutMs: RUN_TIMEOUT_MS, requestTimeoutMs: REQUEST_TIMEOUT_MS, envs: {}, }); From 3f96224d96e81e54dbf29d2cbe458a10b51017ec Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:33:33 -0700 Subject: [PATCH 2/4] fix(sandbox): E2B capacity refusals (ServiceBusyError, 503) get the busy line (TASK-596) Sable's review on #2942: E2B's capacity refusal is an HTTP 503 that the SDK throws as ServiceBusyError, and isSandboxRefusal only matched RateLimitError, TimeoutError and status 429. A 503 fell through as a raw error. The model got the SDK's text and a later run in the same turn never retried. Match ServiceBusyError by name and 429/503 by statusCode. Tests: a 503 gets the fixed busy line and a later run retries; the create timeout is pinned to 30_000 rather than "> 0". Co-Authored-By: Claude Haiku 5.5 Co-authored-by: dart Agent-Model: claude/claude-haiku-5-5/default --- .../unit/services/turnSandboxService.test.js | 12 +++++++++++- backend/services/turnSandboxService.ts | 10 ++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/__tests__/unit/services/turnSandboxService.test.js b/backend/__tests__/unit/services/turnSandboxService.test.js index d3dcbe4681..ffab5941a2 100644 --- a/backend/__tests__/unit/services/turnSandboxService.test.js +++ b/backend/__tests__/unit/services/turnSandboxService.test.js @@ -130,7 +130,17 @@ describe('TurnSandbox', () => { const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockCreate.mockRejectedValueOnce(Object.assign(new Error('context deadline exceeded'), { name: 'TimeoutError' })); expect(await new TurnSandbox({}).run('python', 'print(1)')).toMatchObject({ ok: false, error: SANDBOX_BUSY_MESSAGE }); - expect(mockCreate.mock.calls[0][1].requestTimeoutMs).toBeGreaterThan(0); + expect(mockCreate.mock.calls[0][1].requestTimeoutMs).toBe(30_000); + warn.mockRestore(); + }); + + test('E2B capacity refusals (ServiceBusyError, 503) get the same busy line, and a later run retries', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + mockCreate.mockRejectedValueOnce(Object.assign(new Error('Service temporarily unavailable'), { name: 'ServiceBusyError', statusCode: 503 })); + const sandbox = new TurnSandbox({}); + expect(await sandbox.run('python', 'print(1)')).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); + expect((await sandbox.run('python', 'print(1)')).ok).toBe(true); + expect(mockCreate).toHaveBeenCalledTimes(2); warn.mockRestore(); }); diff --git a/backend/services/turnSandboxService.ts b/backend/services/turnSandboxService.ts index 9a8970fdc3..cbaac4fe59 100644 --- a/backend/services/turnSandboxService.ts +++ b/backend/services/turnSandboxService.ts @@ -56,11 +56,13 @@ export interface RunCodeResult { error?: string; } -// E2B's own refusal classes: RateLimitError (cap or 429), TimeoutError (create -// ran past its request timeout). Anything else is a fault, and still throws. +// E2B's own refusal classes: RateLimitError (cap or 429), ServiceBusyError (503, +// capacity), TimeoutError (create ran past its request timeout). Anything else is +// a fault, and still throws. const isSandboxRefusal = (error: unknown): boolean => { - const { name, status } = (error || {}) as { name?: string; status?: number }; - return name === 'RateLimitError' || name === 'TimeoutError' || status === 429; + const { name, status, statusCode } = (error || {}) as { name?: string; status?: number; statusCode?: number }; + return name === 'RateLimitError' || name === 'ServiceBusyError' || name === 'TimeoutError' + || status === 429 || statusCode === 429 || statusCode === 503; }; const clip = (text: string, limit: number): string => ( From 57800631ec7bc4f385951b44eee189c2e71cfc0b Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Wed, 28 Oct 2026 03:43:28 -0700 Subject: [PATCH 3/4] fix(sandbox): bound E2B create on our own clock (TASK-596) Vera's HOLD on #2942: the pinned @e2b/code-interpreter does not honour requestTimeoutMs on Sandbox.create, so a hung control plane held the turn with no bound at all. Race the create against our own 30 s timer. On timeout the run returns the busy line and forgets the create, so a later run may retry. A create that answers after the bound is killed on arrival, so a late sandbox is not billed until its 3 min lifetime. Tests: a create that never answers gets the busy line at the bound, and a create that answers late is killed. Both hang on the pre-change code. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: forge Agent-Model: claude/claude-haiku-5-5/default --- .../unit/services/turnSandboxService.test.js | 39 +++++++++++++++++++ backend/services/turnSandboxService.ts | 28 ++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/backend/__tests__/unit/services/turnSandboxService.test.js b/backend/__tests__/unit/services/turnSandboxService.test.js index ffab5941a2..bfaa52b590 100644 --- a/backend/__tests__/unit/services/turnSandboxService.test.js +++ b/backend/__tests__/unit/services/turnSandboxService.test.js @@ -134,6 +134,45 @@ describe('TurnSandbox', () => { warn.mockRestore(); }); + test('a create that never answers gets the busy line by our own bound, and a later run retries', async () => { + jest.useFakeTimers(); + try { + // The pinned SDK ignores requestTimeoutMs here, so only our own timer can end this. + mockCreate.mockImplementationOnce(() => new Promise(() => {})); + const sandbox = new TurnSandbox({}); + const pending = sandbox.run('python', 'print(1)'); + await jest.advanceTimersByTimeAsync(30_000); + expect(await pending).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); + expect(await sandbox.close()).toBe(0); + + expect((await sandbox.run('python', 'print(1)')).ok).toBe(true); + expect(mockCreate).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test('a create that answers after our bound is killed on arrival, not left to bill', async () => { + jest.useFakeTimers(); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + try { + let resolveLate; + mockCreate.mockImplementationOnce(() => new Promise((resolve) => { resolveLate = resolve; })); + const sandbox = new TurnSandbox({}); + const pending = sandbox.run('python', 'print(1)'); + await jest.advanceTimersByTimeAsync(30_000); + expect((await pending).error).toBe(SANDBOX_BUSY_MESSAGE); + + resolveLate({ runCode: mockRunCode, kill: mockKill }); + await jest.advanceTimersByTimeAsync(0); + expect(mockKill).toHaveBeenCalledTimes(1); + expect(mockRunCode).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + warn.mockRestore(); + } + }); + test('E2B capacity refusals (ServiceBusyError, 503) get the same busy line, and a later run retries', async () => { const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockCreate.mockRejectedValueOnce(Object.assign(new Error('Service temporarily unavailable'), { name: 'ServiceBusyError', statusCode: 503 })); diff --git a/backend/services/turnSandboxService.ts b/backend/services/turnSandboxService.ts index cbaac4fe59..53224e258c 100644 --- a/backend/services/turnSandboxService.ts +++ b/backend/services/turnSandboxService.ts @@ -65,6 +65,31 @@ const isSandboxRefusal = (error: unknown): boolean => { || status === 429 || statusCode === 429 || statusCode === 503; }; +// The pinned SDK does not honour requestTimeoutMs on Sandbox.create, so the +// bound is ours: a hung control plane must not hold the turn. A create that +// answers after the bound is killed on arrival, so it is not billed until its +// lifetime runs out. +const boundCreate = (create: Promise, ms: number): Promise => new Promise((resolve, reject) => { + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + const error = new Error(`sandbox create did not answer within ${ms}ms`); + error.name = 'TimeoutError'; + reject(error); + }, ms); + create.then((sandbox) => { + clearTimeout(timer); + if (timedOut) { + sandbox.kill().catch(() => {}); + return; + } + resolve(sandbox); + }, (error) => { + clearTimeout(timer); + reject(error); + }); +}); + const clip = (text: string, limit: number): string => ( text.length > limit ? `${text.slice(0, limit)}\n… [${text.length - limit} more characters]` : text ); @@ -87,7 +112,7 @@ export class TurnSandbox { // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { Sandbox } = require('@e2b/code-interpreter'); this.startedAt = Date.now(); - this.creating = Sandbox.create(SANDBOX_TEMPLATE, { + const create = Sandbox.create(SANDBOX_TEMPLATE, { apiKey: process.env.E2B_API_KEY, timeoutMs: SANDBOX_LIFETIME_MS, requestTimeoutMs: CREATE_TIMEOUT_MS, @@ -100,6 +125,7 @@ export class TurnSandbox { envs: {}, metadata: this.metadata, }) as Promise; + this.creating = boundCreate(create, CREATE_TIMEOUT_MS); } return this.creating; } From 50bb1154e5ea6040b16b3a1902c8b61b9bab5169 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Wed, 28 Oct 2026 04:02:09 -0700 Subject: [PATCH 4/4] fix(sandbox): retry a failed late kill once; pin the create bound to 30 s (TASK-596) CodeRabbit on #2942, both review-body items: - A late sandbox whose first kill() rejects is now killed once more. Only a second failure is logged. A failed kill otherwise bills to the 3 min lifetime. - The never-answers test asserts the run is still unsettled at 29_999 ms and settles at 30_000 ms, so a timer that fires early fails it. Tests: the late-kill retry is asserted; the bound test checks the early edge. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: forge Agent-Model: claude/claude-haiku-5-5/default --- .../unit/services/turnSandboxService.test.js | 31 +++++++++++++++++-- backend/services/turnSandboxService.ts | 10 +++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/backend/__tests__/unit/services/turnSandboxService.test.js b/backend/__tests__/unit/services/turnSandboxService.test.js index bfaa52b590..b3f8ace847 100644 --- a/backend/__tests__/unit/services/turnSandboxService.test.js +++ b/backend/__tests__/unit/services/turnSandboxService.test.js @@ -140,8 +140,11 @@ describe('TurnSandbox', () => { // The pinned SDK ignores requestTimeoutMs here, so only our own timer can end this. mockCreate.mockImplementationOnce(() => new Promise(() => {})); const sandbox = new TurnSandbox({}); - const pending = sandbox.run('python', 'print(1)'); - await jest.advanceTimersByTimeAsync(30_000); + let settled = false; + const pending = sandbox.run('python', 'print(1)').then((result) => { settled = true; return result; }); + await jest.advanceTimersByTimeAsync(29_999); + expect(settled).toBe(false); + await jest.advanceTimersByTimeAsync(1); expect(await pending).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); expect(await sandbox.close()).toBe(0); @@ -173,6 +176,30 @@ describe('TurnSandbox', () => { } }); + test('a late sandbox whose first kill fails is killed again once, and no warning is logged when the retry works', async () => { + jest.useFakeTimers(); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + try { + let resolveLate; + mockCreate.mockImplementationOnce(() => new Promise((resolve) => { resolveLate = resolve; })); + const sandbox = new TurnSandbox({}); + const pending = sandbox.run('python', 'print(1)'); + await jest.advanceTimersByTimeAsync(30_000); + expect((await pending).error).toBe(SANDBOX_BUSY_MESSAGE); + expect(await sandbox.close()).toBe(0); + + warn.mockClear(); + mockKill.mockRejectedValueOnce(new Error('network')).mockResolvedValueOnce(true); + resolveLate({ runCode: mockRunCode, kill: mockKill }); + await jest.advanceTimersByTimeAsync(0); + expect(mockKill).toHaveBeenCalledTimes(2); + expect(warn).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + warn.mockRestore(); + } + }); + test('E2B capacity refusals (ServiceBusyError, 503) get the same busy line, and a later run retries', async () => { const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockCreate.mockRejectedValueOnce(Object.assign(new Error('Service temporarily unavailable'), { name: 'ServiceBusyError', statusCode: 503 })); diff --git a/backend/services/turnSandboxService.ts b/backend/services/turnSandboxService.ts index 53224e258c..0a1cf5e162 100644 --- a/backend/services/turnSandboxService.ts +++ b/backend/services/turnSandboxService.ts @@ -69,6 +69,14 @@ const isSandboxRefusal = (error: unknown): boolean => { // bound is ours: a hung control plane must not hold the turn. A create that // answers after the bound is killed on arrival, so it is not billed until its // lifetime runs out. +// A failed kill leaves the sandbox billing until its lifetime, so one retry +// first. Only a second failure is logged. +const killLate = (sandbox: E2BSandbox): void => { + sandbox.kill().catch(() => sandbox.kill()).catch((error) => { + console.warn('[sandbox] late sandbox kill failed; E2B stops it at its lifetime:', (error as Error).message); + }); +}; + const boundCreate = (create: Promise, ms: number): Promise => new Promise((resolve, reject) => { let timedOut = false; const timer = setTimeout(() => { @@ -80,7 +88,7 @@ const boundCreate = (create: Promise, ms: number): Promise { clearTimeout(timer); if (timedOut) { - sandbox.kill().catch(() => {}); + killLate(sandbox); return; } resolve(sandbox);