diff --git a/services/cloud-agent-next/docs/control-plane.md b/services/cloud-agent-next/docs/control-plane.md index 7d5296f52a8..16225b78efb 100644 --- a/services/cloud-agent-next/docs/control-plane.md +++ b/services/cloud-agent-next/docs/control-plane.md @@ -114,7 +114,7 @@ terminal. Nothing else. | Outcome (status, reason, `lastMessageId`) | Every `accepted` message up to and including `lastMessageId` takes the status. If `lastMessageId` is unknown, every `accepted` message does. | | Stop | All `queued` and `accepted` → `cancelled`; send `abort` (best effort). | | Cancel one queued message | That message → `cancelled`. | -| Backstop alarm | `queued` older than 16.5 minutes → `failed` (`preparation_timeout`). `accepted` older than 125 minutes → `failed` (`no_outcome`). | +| Backstop alarm | `queued` older than its queued backstop → `failed` (`preparation_timeout`). `accepted` older than its accepted backstop → `failed` (`no_outcome`). | Stop still forwards a best-effort abort to an existing ready Kilo route when no message is queued or accepted and the wrapper has no active turn. It does not @@ -123,7 +123,7 @@ rewrite terminal message outcomes or prepare/wake a stopped sandbox. Late outcomes and notifications for terminal messages are ignored. The backstop exists only for lost notifications; the Sandbox DO and the wrapper settle every normal case earlier. Both values are derived from the owners' timers so the backstop does not end work within its owner's bound: -queued = reconnect window (90 s) + one preparation attempt (12 min) + 3 min, which covers the +queued = reconnect window (90 s) + one preparation attempt (the sum of its step bounds, below) + 3 min, which covers the longest normal wait (a message sent while the socket is down, then a fresh attempt on a new sandbox); accepted = turn hard cap (120 min) + 5 min. The hard cap bounds one native execution, not an accepted message, so a batch whose follow-up waits behind a serialized command spans several @@ -309,10 +309,15 @@ so a later report changes nothing. ### Route state (per session) -`preparing` → `ready` → `failed`. Each preparation attempt has a 12-minute deadline. An attempt +`preparing` → `ready` → `failed`. Each preparation attempt has a deadline that outlasts its +own steps in sequence (sandbox create, wrapper first connect and hello, snapshot restore with +the clone fallback, the workspace save, Kilo runtime start, session import), including the one +retry each of the last two steps may take. An attempt starts when a route enters `preparing`: the first `prepare`, a `prepare` after `failed`, or a re-prepare after a wrapper restart. A repeated `prepare`, a reconnect or a new allocation inside the -attempt keeps its deadline. +attempt keeps its deadline. Setup commands keep their own per-command limits, and how many run is +configuration, so the attempt deadline is their aggregate stop: a preparation whose setup work +together outlasts the attempt fails as `preparation_timeout`. `prepare` returns the route view that the Sandbox DO would notify: `ready` only when the route is ready and the wrapper is connected, `reconnecting` when the route is ready but the socket is down, @@ -853,7 +858,7 @@ them through one development-only override. | Owner | Timer | Value | On expiry | |---|---|---|---| -| Session DO | Queued backstop | 16.5 min from send (reconnect + attempt + 3 min) | `failed` (`preparation_timeout`) | +| Session DO | Queued backstop | From send: reconnect window + one preparation attempt + 3 min | `failed` (`preparation_timeout`) | | Session DO | Accepted backstop | 125 min from accept | `failed` (`no_outcome`) | | Session DO | Sandbox transport pass / best-effort abort | 2 s including RPC retries, not scaled | Retain queued intent / return from abort | | Session DO | Queued transport recovery | 15 s after exhaustion; development-only scaling | Consume once; passive status then bounded prepare/deliver, no self-rearm | @@ -863,7 +868,7 @@ them through one development-only override. | Sandbox DO | Unbound wrapper hello | 30 s from socket admission, not scaled | Close candidate only; attachment deadline survives hibernation and participates in the existing alarm even while stopped | | Sandbox DO | Heartbeat | 45 s | Treat the socket as lost | | Sandbox DO | Reconnect | 90 s from the last frame received | Stop the sandbox | -| Sandbox DO | Route preparation | 12 min per preparation attempt | Route `failed` (`preparation_timeout`) | +| Sandbox DO | Route preparation | Sum of one preparation attempt's step bounds (see Preparation) | Route `failed` (`preparation_timeout`) | | Sandbox DO | Session notification | 2 s from enqueue including queue wait and RPC retries, not scaled | Drop expired queued work before RPC; bounded best-effort loss, existing Session backstop | | Sandbox DO | Idle | 10 min without activity | Stop the sandbox | | Sandbox DO | Provider lease | Existing lease length, renewed while active | Provider may stop an inactive sandbox | @@ -913,7 +918,7 @@ them through one development-only override. | Native execution over 120 min | Native session supervisor | Bounded tree abort | Accepted fail (`execution_limit`), or routed error without a current turn | | Idle 10 min, question pending | Sandbox DO | Stop the sandbox | Accepted fail (`sandbox_stopped`); a later answer is a new message | | Sandbox gone | Provider via Sandbox DO | New allocation on next `prepare` | Accepted fail (`sandbox_lost`); queued re-prepare | -| Notification lost | Session DO backstop | — | Fail at 16.5 or 125 min | +| Notification lost | Session DO backstop | — | Fail at the queued or accepted backstop | | User Stop | Session DO | Abort the Kilo session only | Queued and accepted `cancelled` | | Auto-commit or condense fails | Wrapper | None | Warning event; turn `completed` | | Provider stop not confirmed | Sandbox DO | Logged; next `prepare` may create again | None; worktree deletion reports incomplete | @@ -1360,7 +1365,7 @@ counts. overflow drops newest, loss diagnostics stay bounded/internal and expired queue heads make no RPC. Heartbeats, alarms, stop and reconnect continue while notifications are held. Release retires references; reconstruction replays no notification backlog. Public events, reports, callbacks - and the 16.5/125-minute lost-notification backstops are unchanged. + and the queued and accepted lost-notification backstops are unchanged. 22. **Permanent versus transient creation failure.** Balance lost after send preflight fails queued work promptly with `billing_blocked`, not preparation timeout. Proven local invalid configuration fails promptly with `invalid_configuration`. The safe stream reason, durable route/message, diff --git a/services/cloud-agent-next/src/control-plane/sandbox/routes.ts b/services/cloud-agent-next/src/control-plane/sandbox/routes.ts index b21988b7a04..14addebae02 100644 --- a/services/cloud-agent-next/src/control-plane/sandbox/routes.ts +++ b/services/cloud-agent-next/src/control-plane/sandbox/routes.ts @@ -500,7 +500,7 @@ export async function failReadyRoutesForScope( } } -/** Fail preparing routes whose 12-minute attempt deadline has passed. */ +/** Fail preparing routes whose preparation attempt deadline has passed. */ export async function failExpiredRoutes(ctx: RouteContext): Promise { const now = ctx.now(); for (const route of await listRoutes(ctx.db)) { diff --git a/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts b/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts index c7d005caccd..7857c747e12 100644 --- a/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts +++ b/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts @@ -5443,7 +5443,7 @@ export class SandboxControlV2 extends DurableObject { private async armAlarm(state: AllocationState): Promise { // The one alarm is the earliest of the allocation timers and the route // preparation deadlines, so a hung wrapper's route cannot outlive its - // 12-minute attempt. + // preparation attempt. const allocationAt = nextAllocationAlarmAt( state.kind === 'connected' && [...this.repositoryCaptures.values()].some( diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts index 1b895ce5dfe..f485e6e6149 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { DEADLINE_MS } from '../sandbox-control/deadlines.js'; import { CONTROL_PLANE_TIMERS, resolveControlPlaneTimers, @@ -35,6 +36,42 @@ describe('control-plane idle timers', () => { }); }); +describe('control-plane preparation attempt budget', () => { + it('outlasts every step of one preparation attempt in sequence', () => { + const { sandbox, wrapper } = CONTROL_PLANE_TIMERS; + // A repository-backed container allocation runs its create under the wider + // repository startup bound, so that is the create step this covers. + const createMs = Math.max(sandbox.providerCreateMs, DEADLINE_MS.repositoryStartup); + const steps = + // A transient create failure creates again inside the same attempt. + createMs + + sandbox.providerCreateRetryMs + + createMs + + sandbox.wrapperFirstConnectMs + + sandbox.wrapperHelloMs + + wrapper.restoreMs + + wrapper.cloneMs + + wrapper.captureMs + + // Kilo runtime start and the Kilo session each retry once inside the + // attempt, so each can consume two of its own bounds. + 2 * wrapper.kiloRuntimeStartMs + + 2 * wrapper.kiloSessionMs; + + // A shorter attempt deadline would fail a slow but valid preparation as + // `preparation_timeout` while each step stayed inside its own bound. + expect(sandbox.routePreparationMs).toBeGreaterThanOrEqual(steps); + }); + + it('keeps the attempt budget real when the development divisor shrinks step timers', () => { + const scaled = resolveControlPlaneTimers({ CONTROL_PLANE_TIMER_DIVISOR: '10' }); + + expect(scaled.sandbox.routePreparationMs).toBe(CONTROL_PLANE_TIMERS.sandbox.routePreparationMs); + expect(scaled.sandbox.routePreparationMs).toBeGreaterThanOrEqual( + scaled.wrapper.cloneMs + scaled.wrapper.restoreMs + ); + }); +}); + describe('sandbox idle policy for route origins', () => { it('keeps the interactive window for a sandbox without routes', () => { expect(sandboxIdlePolicyForOrigins([])).toBe('interactive'); diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index 44fe2d18d99..b8651492cc7 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -72,23 +72,80 @@ export type ControlPlaneTimers = { }; }; +// External-work bounds for one preparation attempt. They stay real (never +// scaled) because they bound provider and container work. +const PROVIDER_CREATE_MS = 2 * MINUTE_MS; +const PROVIDER_CREATE_RETRY_MS = 10_000; +// A repository-backed container allocation creates its sandbox under the wider +// repository startup bound (`DEADLINE_MS.repositoryStartup` in +// `sandbox-control/deadlines.ts`), which is the create path every preparation +// attempt with a repository takes. It is restated here because this module is +// bundled into the wrapper without the worker's other sources; +// `control-plane-timers.test.ts` asserts the attempt budget against that bound. +const PREPARATION_CREATE_MS = 5 * MINUTE_MS; +const WRAPPER_FIRST_CONNECT_MS = 5 * MINUTE_MS; +const WRAPPER_HELLO_MS = 30_000; +const RESTORE_MS = 2 * MINUTE_MS; +const CLONE_MS = 6 * MINUTE_MS; +// The attempt's own last workspace step: the Sandbox DO saves the prepared +// workspace for a repository-backed route, and preparation waits for it. +const CAPTURE_MS = 5 * MINUTE_MS + 10_000; +const KILO_RUNTIME_START_MS = 2 * MINUTE_MS; +const KILO_SESSION_MS = 2 * MINUTE_MS; + +/** + * One preparation attempt runs these steps in order: sandbox create, wrapper + * first connect and hello, snapshot restore (falling back to a clone), the + * workspace save, Kilo runtime start, then the session import. Every step keeps + * its own bound, so the attempt deadline is their sum. A shorter deadline fails + * a slow but valid preparation as `preparation_timeout` while each step stayed + * inside its own budget — the failure names a timeout the plan never allowed to + * finish. + * + * Two steps are retried once inside the same attempt, so each counts twice. + * A transient create failure pauses and creates again inside the same attempt + * (`control-plane/sandbox/allocation.ts`), so one retried create is counted + * too. Further retries stay bounded by this deadline, which is what stops a + * create that keeps failing. + * + * Setup commands have no term here. Each command carries its own inactivity and + * hard timeout (`wrapper/src/control-plane/prepare.ts`), while how many run and + * how long each takes are the route's configuration (bounded per command, not in + * total). This deadline is therefore their aggregate stop by design: a route + * whose setup work together outlasts the attempt fails as + * `preparation_timeout`, because a bound that grew with the configured command + * count would also stretch how long a stuck preparation can hold a session. + */ +const ROUTE_PREPARATION_MS = + PREPARATION_CREATE_MS + + PROVIDER_CREATE_RETRY_MS + + PREPARATION_CREATE_MS + + WRAPPER_FIRST_CONNECT_MS + + WRAPPER_HELLO_MS + + RESTORE_MS + + CLONE_MS + + CAPTURE_MS + + 2 * KILO_RUNTIME_START_MS + + 2 * KILO_SESSION_MS; + // The divisor shortens only internal deadlines whose partner also scales, so // their orderings hold (the 90-s reconnect window stays longer than the 30-s // max backoff). Timers that bound external work stay real: provider create and -// container boot, the 12-min prep attempt that must outlast clone+runtime+ -// session, the pause before a failed create is retried, clone/runtime/session -// bounds, SSE silence (Kilo heartbeats every 10-15 s), health request, the -// real-SSE reconnect window, credential grants and the provider stop ladder. +// container boot, the prep attempt that must outlast create+connect+restore+ +// clone+save+runtime+session, the pause before a failed create is retried, +// restore/clone/save/runtime/session bounds, SSE silence (Kilo heartbeats every +// 10-15 s), health request, the real-SSE reconnect window, credential grants +// and the provider stop ladder. function buildControlPlaneTimers(divisor: number): ControlPlaneTimers { const ms = (value: number): number => Math.max(1, Math.round(value / divisor)); const sandbox = { - providerCreateMs: 2 * MINUTE_MS, - providerCreateRetryMs: 10_000, - wrapperFirstConnectMs: 5 * MINUTE_MS, - wrapperHelloMs: 30_000, + providerCreateMs: PROVIDER_CREATE_MS, + providerCreateRetryMs: PROVIDER_CREATE_RETRY_MS, + wrapperFirstConnectMs: WRAPPER_FIRST_CONNECT_MS, + wrapperHelloMs: WRAPPER_HELLO_MS, heartbeatMs: ms(45_000), reconnectMs: ms(90_000), - routePreparationMs: 12 * MINUTE_MS, + routePreparationMs: ROUTE_PREPARATION_MS, idleMs: ms(10 * MINUTE_MS), unattendedIdleMs: ms(MINUTE_MS), // Bounds one Sandbox -> Session notification; real (not scaled) like the @@ -108,11 +165,11 @@ function buildControlPlaneTimers(divisor: number): ControlPlaneTimers { heartbeatNegotiationMs: ms(1_000), // Scales with the reconnect window it must stay inside. handshakeMs: ms(30_000), - cloneMs: 6 * MINUTE_MS, - restoreMs: 2 * MINUTE_MS, - captureMs: 5 * MINUTE_MS + 10_000, - kiloRuntimeStartMs: 2 * MINUTE_MS, - kiloSessionMs: 2 * MINUTE_MS, + cloneMs: CLONE_MS, + restoreMs: RESTORE_MS, + captureMs: CAPTURE_MS, + kiloRuntimeStartMs: KILO_RUNTIME_START_MS, + kiloSessionMs: KILO_SESSION_MS, sseSilenceMs: 30_000, healthRequestMs: 5_000, sseReconnectLimit: 6, diff --git a/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts b/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts index bb73f8f69aa..c32bc7cf643 100644 --- a/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts +++ b/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts @@ -125,7 +125,7 @@ describe('SandboxSessionV2 queued transport recovery', () => { const scaled = resolveControlPlaneTimers({ CONTROL_PLANE_TIMER_DIVISOR: '10' }); expect(scaled.session.sandboxRpcDeadlineMs).toBe(2_000); expect(scaled.session.transportRecoveryMs).toBe(1_500); - expect(scaled.sandbox.routePreparationMs).toBe(12 * 60_000); + expect(scaled.sandbox.routePreparationMs).toBe(CONTROL_PLANE_TIMERS.sandbox.routePreparationMs); expect(resolveControlPlaneTimers({ CONTROL_PLANE_TIMER_DIVISOR: 'invalid' })).toBe( CONTROL_PLANE_TIMERS );