diff --git a/backend/__tests__/unit/services/turnSandboxService.test.js b/backend/__tests__/unit/services/turnSandboxService.test.js index 053367411e..b3f8ace847 100644 --- a/backend/__tests__/unit/services/turnSandboxService.test.js +++ b/backend/__tests__/unit/services/turnSandboxService.test.js @@ -9,7 +9,9 @@ jest.mock('@e2b/code-interpreter', () => ({ Sandbox: { create: (...args) => mockCreate(...args) }, })); -const { TurnSandbox, MAX_RUNS_PER_TURN, isSandboxConfigured } = require('../../../services/turnSandboxService'); +const { + TurnSandbox, MAX_RUNS_PER_TURN, isSandboxConfigured, SANDBOX_BUSY_MESSAGE, +} = require('../../../services/turnSandboxService'); describe('TurnSandbox', () => { const previousKey = process.env.E2B_API_KEY; @@ -104,6 +106,110 @@ describe('TurnSandbox', () => { expect(mockRunCode).toHaveBeenCalledTimes(MAX_RUNS_PER_TURN); }); + test('a refused sandbox returns the fixed busy line, not the SDK message, and the turn carries on', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + const rateLimit = Object.assign(new Error('Rate limit exceeded: concurrent sandbox cap reached (acct 42)'), { name: 'RateLimitError' }); + mockCreate.mockRejectedValueOnce(rateLimit); + + const sandbox = new TurnSandbox({}); + // Resolves, so the tool loop gets a result and the model can still reply. + const result = await sandbox.run('python', 'print(1)'); + expect(result).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); + expect(JSON.stringify(result)).not.toMatch(/acct 42|Rate limit/); + // No sandbox exists, so close has nothing to kill and reports nothing. + expect(await sandbox.close()).toBe(0); + expect(mockKill).not.toHaveBeenCalled(); + + // A later run in the same turn may try again. + expect((await sandbox.run('python', 'print(1)')).ok).toBe(true); + expect(mockCreate).toHaveBeenCalledTimes(2); + warn.mockRestore(); + }); + + test('a create that times out is refused the same way, and the create call is bounded', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + mockCreate.mockRejectedValueOnce(Object.assign(new Error('context deadline exceeded'), { name: 'TimeoutError' })); + expect(await new TurnSandbox({}).run('python', 'print(1)')).toMatchObject({ ok: false, error: SANDBOX_BUSY_MESSAGE }); + expect(mockCreate.mock.calls[0][1].requestTimeoutMs).toBe(30_000); + warn.mockRestore(); + }); + + test('a create that never answers gets the busy line by our own bound, and a later run retries', async () => { + jest.useFakeTimers(); + try { + // The pinned SDK ignores requestTimeoutMs here, so only our own timer can end this. + mockCreate.mockImplementationOnce(() => new Promise(() => {})); + const sandbox = new TurnSandbox({}); + let settled = false; + const pending = sandbox.run('python', 'print(1)').then((result) => { settled = true; return result; }); + await jest.advanceTimersByTimeAsync(29_999); + expect(settled).toBe(false); + await jest.advanceTimersByTimeAsync(1); + expect(await pending).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); + expect(await sandbox.close()).toBe(0); + + expect((await sandbox.run('python', 'print(1)')).ok).toBe(true); + expect(mockCreate).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } + }); + + test('a create that answers after our bound is killed on arrival, not left to bill', async () => { + jest.useFakeTimers(); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + try { + let resolveLate; + mockCreate.mockImplementationOnce(() => new Promise((resolve) => { resolveLate = resolve; })); + const sandbox = new TurnSandbox({}); + const pending = sandbox.run('python', 'print(1)'); + await jest.advanceTimersByTimeAsync(30_000); + expect((await pending).error).toBe(SANDBOX_BUSY_MESSAGE); + + resolveLate({ runCode: mockRunCode, kill: mockKill }); + await jest.advanceTimersByTimeAsync(0); + expect(mockKill).toHaveBeenCalledTimes(1); + expect(mockRunCode).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + warn.mockRestore(); + } + }); + + test('a late sandbox whose first kill fails is killed again once, and no warning is logged when the retry works', async () => { + jest.useFakeTimers(); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + try { + let resolveLate; + mockCreate.mockImplementationOnce(() => new Promise((resolve) => { resolveLate = resolve; })); + const sandbox = new TurnSandbox({}); + const pending = sandbox.run('python', 'print(1)'); + await jest.advanceTimersByTimeAsync(30_000); + expect((await pending).error).toBe(SANDBOX_BUSY_MESSAGE); + expect(await sandbox.close()).toBe(0); + + warn.mockClear(); + mockKill.mockRejectedValueOnce(new Error('network')).mockResolvedValueOnce(true); + resolveLate({ runCode: mockRunCode, kill: mockKill }); + await jest.advanceTimersByTimeAsync(0); + expect(mockKill).toHaveBeenCalledTimes(2); + expect(warn).not.toHaveBeenCalled(); + } finally { + jest.useRealTimers(); + warn.mockRestore(); + } + }); + + test('E2B capacity refusals (ServiceBusyError, 503) get the same busy line, and a later run retries', async () => { + const warn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + mockCreate.mockRejectedValueOnce(Object.assign(new Error('Service temporarily unavailable'), { name: 'ServiceBusyError', statusCode: 503 })); + const sandbox = new TurnSandbox({}); + expect(await sandbox.run('python', 'print(1)')).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE }); + expect((await sandbox.run('python', 'print(1)')).ok).toBe(true); + expect(mockCreate).toHaveBeenCalledTimes(2); + warn.mockRestore(); + }); + test('is configured only when the instance has an E2B key', () => { expect(isSandboxConfigured()).toBe(true); delete process.env.E2B_API_KEY; diff --git a/backend/services/turnSandboxService.ts b/backend/services/turnSandboxService.ts index be121fde2e..0a1cf5e162 100644 --- a/backend/services/turnSandboxService.ts +++ b/backend/services/turnSandboxService.ts @@ -23,6 +23,11 @@ export const SANDBOX_TEMPLATE = 'code-interpreter-v1'; const SANDBOX_LIFETIME_MS = 3 * 60 * 1000; const RUN_TIMEOUT_MS = 30_000; const REQUEST_TIMEOUT_MS = 45_000; +// Bounds the create call itself: a hung E2B control plane must not hold the turn. +const CREATE_TIMEOUT_MS = 30_000; +// What the model is told when E2B refuses a sandbox (concurrency cap, rate +// limit, or a create that timed out). The raw SDK message stays server-side. +export const SANDBOX_BUSY_MESSAGE = 'sandbox busy, try again in a minute'; const MAX_CODE_CHARS = 20_000; const STDOUT_LIMIT = 4000; const STDERR_LIMIT = 2000; @@ -51,6 +56,48 @@ export interface RunCodeResult { error?: string; } +// E2B's own refusal classes: RateLimitError (cap or 429), ServiceBusyError (503, +// capacity), TimeoutError (create ran past its request timeout). Anything else is +// a fault, and still throws. +const isSandboxRefusal = (error: unknown): boolean => { + const { name, status, statusCode } = (error || {}) as { name?: string; status?: number; statusCode?: number }; + return name === 'RateLimitError' || name === 'ServiceBusyError' || name === 'TimeoutError' + || status === 429 || statusCode === 429 || statusCode === 503; +}; + +// The pinned SDK does not honour requestTimeoutMs on Sandbox.create, so the +// bound is ours: a hung control plane must not hold the turn. A create that +// answers after the bound is killed on arrival, so it is not billed until its +// lifetime runs out. +// A failed kill leaves the sandbox billing until its lifetime, so one retry +// first. Only a second failure is logged. +const killLate = (sandbox: E2BSandbox): void => { + sandbox.kill().catch(() => sandbox.kill()).catch((error) => { + console.warn('[sandbox] late sandbox kill failed; E2B stops it at its lifetime:', (error as Error).message); + }); +}; + +const boundCreate = (create: Promise, ms: number): Promise => new Promise((resolve, reject) => { + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + const error = new Error(`sandbox create did not answer within ${ms}ms`); + error.name = 'TimeoutError'; + reject(error); + }, ms); + create.then((sandbox) => { + clearTimeout(timer); + if (timedOut) { + killLate(sandbox); + return; + } + resolve(sandbox); + }, (error) => { + clearTimeout(timer); + reject(error); + }); +}); + const clip = (text: string, limit: number): string => ( text.length > limit ? `${text.slice(0, limit)}\n… [${text.length - limit} more characters]` : text ); @@ -73,9 +120,10 @@ export class TurnSandbox { // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { Sandbox } = require('@e2b/code-interpreter'); this.startedAt = Date.now(); - this.creating = Sandbox.create(SANDBOX_TEMPLATE, { + const create = Sandbox.create(SANDBOX_TEMPLATE, { apiKey: process.env.E2B_API_KEY, timeoutMs: SANDBOX_LIFETIME_MS, + requestTimeoutMs: CREATE_TIMEOUT_MS, allowInternetAccess: false, // E2B serves every sandbox port on a public URL unless told not to; // with this off, a URL needs the traffic token only this process holds, @@ -85,6 +133,7 @@ export class TurnSandbox { envs: {}, metadata: this.metadata, }) as Promise; + this.creating = boundCreate(create, CREATE_TIMEOUT_MS); } return this.creating; } @@ -97,7 +146,16 @@ export class TurnSandbox { if (this.runs >= MAX_RUNS_PER_TURN) return refuse(`this turn has used its ${MAX_RUNS_PER_TURN} code runs`); this.runs += 1; - const sandbox = await this.open(); + let sandbox: E2BSandbox; + try { + sandbox = await this.open(); + } catch (error) { + if (!isSandboxRefusal(error)) throw error; + // Forget the failed create, so close() has nothing to kill and a later run may try again. + this.creating = null; + console.warn('[sandbox] E2B refused a sandbox:', (error as Error).name); + return refuse(SANDBOX_BUSY_MESSAGE); + } const execution = await sandbox.runCode(code, { language, timeoutMs: RUN_TIMEOUT_MS, requestTimeoutMs: REQUEST_TIMEOUT_MS, envs: {}, });