fix(sandbox): a refused E2B sandbox returns a fixed busy line (TASK-596)

backend/__tests__/unit/services/turnSandboxService.test.js

@@ -9,7 +9,9 @@ jest.mock('@e2b/code-interpreter', () => ({
   Sandbox: { create: (...args) => mockCreate(...args) },
 }));
 
-const { TurnSandbox, MAX_RUNS_PER_TURN, isSandboxConfigured } = require('../../../services/turnSandboxService');
+const {
+  TurnSandbox, MAX_RUNS_PER_TURN, isSandboxConfigured, SANDBOX_BUSY_MESSAGE,
+} = require('../../../services/turnSandboxService');
 
 describe('TurnSandbox', () => {
   const previousKey = process.env.E2B_API_KEY;
@@ -104,6 +106,110 @@ describe('TurnSandbox', () => {
     expect(mockRunCode).toHaveBeenCalledTimes(MAX_RUNS_PER_TURN);
   });
 
+  test('a refused sandbox returns the fixed busy line, not the SDK message, and the turn carries on', async () => {
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    const rateLimit = Object.assign(new Error('Rate limit exceeded: concurrent sandbox cap reached (acct 42)'), { name: 'RateLimitError' });
+    mockCreate.mockRejectedValueOnce(rateLimit);
+
+    const sandbox = new TurnSandbox({});
+    // Resolves, so the tool loop gets a result and the model can still reply.
+    const result = await sandbox.run('python', 'print(1)');
+    expect(result).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE });
+    expect(JSON.stringify(result)).not.toMatch(/acct 42|Rate limit/);
+    // No sandbox exists, so close has nothing to kill and reports nothing.
+    expect(await sandbox.close()).toBe(0);
+    expect(mockKill).not.toHaveBeenCalled();
+
+    // A later run in the same turn may try again.
+    expect((await sandbox.run('python', 'print(1)')).ok).toBe(true);
+    expect(mockCreate).toHaveBeenCalledTimes(2);
+    warn.mockRestore();
+  });
+
+  test('a create that times out is refused the same way, and the create call is bounded', async () => {
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    mockCreate.mockRejectedValueOnce(Object.assign(new Error('context deadline exceeded'), { name: 'TimeoutError' }));
+    expect(await new TurnSandbox({}).run('python', 'print(1)')).toMatchObject({ ok: false, error: SANDBOX_BUSY_MESSAGE });
+    expect(mockCreate.mock.calls[0][1].requestTimeoutMs).toBe(30_000);
+    warn.mockRestore();
+  });
+
+  test('a create that never answers gets the busy line by our own bound, and a later run retries', async () => {
+    jest.useFakeTimers();
+    try {
+      // The pinned SDK ignores requestTimeoutMs here, so only our own timer can end this.
+      mockCreate.mockImplementationOnce(() => new Promise(() => {}));
+      const sandbox = new TurnSandbox({});
+      let settled = false;
+      const pending = sandbox.run('python', 'print(1)').then((result) => { settled = true; return result; });
+      await jest.advanceTimersByTimeAsync(29_999);
+      expect(settled).toBe(false);
+      await jest.advanceTimersByTimeAsync(1);
+      expect(await pending).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE });
+      expect(await sandbox.close()).toBe(0);
+
+      expect((await sandbox.run('python', 'print(1)')).ok).toBe(true);
+      expect(mockCreate).toHaveBeenCalledTimes(2);
+    } finally {
+      jest.useRealTimers();
+    }
+  });
+
+  test('a create that answers after our bound is killed on arrival, not left to bill', async () => {
+    jest.useFakeTimers();
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    try {
+      let resolveLate;
+      mockCreate.mockImplementationOnce(() => new Promise((resolve) => { resolveLate = resolve; }));
+      const sandbox = new TurnSandbox({});
+      const pending = sandbox.run('python', 'print(1)');
+      await jest.advanceTimersByTimeAsync(30_000);
+      expect((await pending).error).toBe(SANDBOX_BUSY_MESSAGE);
+
+      resolveLate({ runCode: mockRunCode, kill: mockKill });
+      await jest.advanceTimersByTimeAsync(0);
+      expect(mockKill).toHaveBeenCalledTimes(1);
+      expect(mockRunCode).not.toHaveBeenCalled();
+    } finally {
+      jest.useRealTimers();
+      warn.mockRestore();
+    }
+  });
+
+  test('a late sandbox whose first kill fails is killed again once, and no warning is logged when the retry works', async () => {
+    jest.useFakeTimers();
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    try {
+      let resolveLate;
+      mockCreate.mockImplementationOnce(() => new Promise((resolve) => { resolveLate = resolve; }));
+      const sandbox = new TurnSandbox({});
+      const pending = sandbox.run('python', 'print(1)');
+      await jest.advanceTimersByTimeAsync(30_000);
+      expect((await pending).error).toBe(SANDBOX_BUSY_MESSAGE);
+      expect(await sandbox.close()).toBe(0);
+
+      warn.mockClear();
+      mockKill.mockRejectedValueOnce(new Error('network')).mockResolvedValueOnce(true);
+      resolveLate({ runCode: mockRunCode, kill: mockKill });
+      await jest.advanceTimersByTimeAsync(0);
+      expect(mockKill).toHaveBeenCalledTimes(2);
+      expect(warn).not.toHaveBeenCalled();
+    } finally {
+      jest.useRealTimers();
+      warn.mockRestore();
+    }
+  });
+
+  test('E2B capacity refusals (ServiceBusyError, 503) get the same busy line, and a later run retries', async () => {
+    const warn = jest.spyOn(console, 'warn').mockImplementation(() => {});
+    mockCreate.mockRejectedValueOnce(Object.assign(new Error('Service temporarily unavailable'), { name: 'ServiceBusyError', statusCode: 503 }));
+    const sandbox = new TurnSandbox({});
+    expect(await sandbox.run('python', 'print(1)')).toEqual({ ok: false, stdout: '', stderr: '', error: SANDBOX_BUSY_MESSAGE });
+    expect((await sandbox.run('python', 'print(1)')).ok).toBe(true);
+    expect(mockCreate).toHaveBeenCalledTimes(2);
+    warn.mockRestore();
+  });
+
   test('is configured only when the instance has an E2B key', () => {
     expect(isSandboxConfigured()).toBe(true);
     delete process.env.E2B_API_KEY;

backend/services/turnSandboxService.ts

@@ -23,6 +23,11 @@ export const SANDBOX_TEMPLATE = 'code-interpreter-v1';
 const SANDBOX_LIFETIME_MS = 3 * 60 * 1000;
 const RUN_TIMEOUT_MS = 30_000;
 const REQUEST_TIMEOUT_MS = 45_000;
+// Bounds the create call itself: a hung E2B control plane must not hold the turn.
+const CREATE_TIMEOUT_MS = 30_000;
+// What the model is told when E2B refuses a sandbox (concurrency cap, rate
+// limit, or a create that timed out). The raw SDK message stays server-side.
+export const SANDBOX_BUSY_MESSAGE = 'sandbox busy, try again in a minute';
 const MAX_CODE_CHARS = 20_000;
 const STDOUT_LIMIT = 4000;
 const STDERR_LIMIT = 2000;
@@ -51,6 +56,48 @@ export interface RunCodeResult {
   error?: string;
 }
 
+// E2B's own refusal classes: RateLimitError (cap or 429), ServiceBusyError (503,
+// capacity), TimeoutError (create ran past its request timeout). Anything else is
+// a fault, and still throws.
+const isSandboxRefusal = (error: unknown): boolean => {
+  const { name, status, statusCode } = (error || {}) as { name?: string; status?: number; statusCode?: number };
+  return name === 'RateLimitError' || name === 'ServiceBusyError' || name === 'TimeoutError'
+    || status === 429 || statusCode === 429 || statusCode === 503;
+};
+
+// The pinned SDK does not honour requestTimeoutMs on Sandbox.create, so the
+// bound is ours: a hung control plane must not hold the turn. A create that
+// answers after the bound is killed on arrival, so it is not billed until its
+// lifetime runs out.
+// A failed kill leaves the sandbox billing until its lifetime, so one retry
+// first. Only a second failure is logged.
+const killLate = (sandbox: E2BSandbox): void => {
+  sandbox.kill().catch(() => sandbox.kill()).catch((error) => {
+    console.warn('[sandbox] late sandbox kill failed; E2B stops it at its lifetime:', (error as Error).message);
+  });
+};
+
+const boundCreate = (create: Promise<E2BSandbox>, ms: number): Promise<E2BSandbox> => new Promise((resolve, reject) => {
+  let timedOut = false;
+  const timer = setTimeout(() => {
+    timedOut = true;
+    const error = new Error(`sandbox create did not answer within ${ms}ms`);
+    error.name = 'TimeoutError';
+    reject(error);
+  }, ms);
+  create.then((sandbox) => {
+    clearTimeout(timer);
+    if (timedOut) {
+      killLate(sandbox);
+      return;
+    }
+    resolve(sandbox);
+  }, (error) => {
+    clearTimeout(timer);
+    reject(error);
+  });
+});
+
 const clip = (text: string, limit: number): string => (
   text.length > limit ? `${text.slice(0, limit)}\n… [${text.length - limit} more characters]` : text
 );
@@ -73,9 +120,10 @@ export class TurnSandbox {
       // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports
       const { Sandbox } = require('@e2b/code-interpreter');
       this.startedAt = Date.now();
-      this.creating = Sandbox.create(SANDBOX_TEMPLATE, {
+      const create = Sandbox.create(SANDBOX_TEMPLATE, {
         apiKey: process.env.E2B_API_KEY,
         timeoutMs: SANDBOX_LIFETIME_MS,
+        requestTimeoutMs: CREATE_TIMEOUT_MS,
         allowInternetAccess: false,
         // E2B serves every sandbox port on a public URL unless told not to;
         // with this off, a URL needs the traffic token only this process holds,
@@ -85,6 +133,7 @@ export class TurnSandbox {
         envs: {},
         metadata: this.metadata,
       }) as Promise<E2BSandbox>;
+      this.creating = boundCreate(create, CREATE_TIMEOUT_MS);
     }
     return this.creating;
   }
@@ -97,7 +146,16 @@ export class TurnSandbox {
     if (this.runs >= MAX_RUNS_PER_TURN) return refuse(`this turn has used its ${MAX_RUNS_PER_TURN} code runs`);
     this.runs += 1;
 
-    const sandbox = await this.open();
+    let sandbox: E2BSandbox;
+    try {
+      sandbox = await this.open();
+    } catch (error) {
+      if (!isSandboxRefusal(error)) throw error;
+      // Forget the failed create, so close() has nothing to kill and a later run may try again.
+      this.creating = null;
+      console.warn('[sandbox] E2B refused a sandbox:', (error as Error).name);
+      return refuse(SANDBOX_BUSY_MESSAGE);
+    }
     const execution = await sandbox.runCode(code, {
       language, timeoutMs: RUN_TIMEOUT_MS, requestTimeoutMs: REQUEST_TIMEOUT_MS, envs: {},
     });