From 68b9409793612e6b7fb68b4151267b9f6f426f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:14:26 +0000 Subject: [PATCH 1/7] fix(cloud-agent-next): budget a preparation attempt for its own steps One preparation attempt runs the sandbox create, the wrapper's first connect and hello, the snapshot restore with its clone fallback, the Kilo runtime start and the session import in sequence. Each step keeps its own bound, so the attempt deadline is derived from their sum instead of a fixed 12 minutes: a shorter deadline fails a slow but valid preparation as preparation_timeout while every step stayed inside its own budget. The step bounds, a permanent create failure and the reconnect window still fail a stuck preparation earlier. --- .../cloud-agent-next/docs/control-plane.md | 4 +- .../src/control-plane/sandbox/routes.ts | 2 +- .../src/control-plane/sandbox/sandbox-do.ts | 2 +- .../src/shared/control-plane-timers.test.ts | 27 +++++++++ .../src/shared/control-plane-timers.ts | 55 ++++++++++++++----- ...control-plane-session-v2-transport.test.ts | 2 +- 6 files changed, 75 insertions(+), 17 deletions(-) diff --git a/services/cloud-agent-next/docs/control-plane.md b/services/cloud-agent-next/docs/control-plane.md index 7d5296f52a8..6d946a51e87 100644 --- a/services/cloud-agent-next/docs/control-plane.md +++ b/services/cloud-agent-next/docs/control-plane.md @@ -309,7 +309,9 @@ so a later report changes nothing. ### Route state (per session) -`preparing` → `ready` → `failed`. Each preparation attempt has a 12-minute deadline. An attempt +`preparing` → `ready` → `failed`. Each preparation attempt has a deadline that outlasts its +own steps in sequence (sandbox create, wrapper first connect and hello, snapshot restore with +the clone fallback, Kilo runtime start, session import). An attempt starts when a route enters `preparing`: the first `prepare`, a `prepare` after `failed`, or a re-prepare after a wrapper restart. A repeated `prepare`, a reconnect or a new allocation inside the attempt keeps its deadline. diff --git a/services/cloud-agent-next/src/control-plane/sandbox/routes.ts b/services/cloud-agent-next/src/control-plane/sandbox/routes.ts index b21988b7a04..14addebae02 100644 --- a/services/cloud-agent-next/src/control-plane/sandbox/routes.ts +++ b/services/cloud-agent-next/src/control-plane/sandbox/routes.ts @@ -500,7 +500,7 @@ export async function failReadyRoutesForScope( } } -/** Fail preparing routes whose 12-minute attempt deadline has passed. */ +/** Fail preparing routes whose preparation attempt deadline has passed. */ export async function failExpiredRoutes(ctx: RouteContext): Promise { const now = ctx.now(); for (const route of await listRoutes(ctx.db)) { diff --git a/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts b/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts index c7d005caccd..7857c747e12 100644 --- a/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts +++ b/services/cloud-agent-next/src/control-plane/sandbox/sandbox-do.ts @@ -5443,7 +5443,7 @@ export class SandboxControlV2 extends DurableObject { private async armAlarm(state: AllocationState): Promise { // The one alarm is the earliest of the allocation timers and the route // preparation deadlines, so a hung wrapper's route cannot outlive its - // 12-minute attempt. + // preparation attempt. const allocationAt = nextAllocationAlarmAt( state.kind === 'connected' && [...this.repositoryCaptures.values()].some( diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts index 1b895ce5dfe..e4055d0a9ba 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts @@ -35,6 +35,33 @@ describe('control-plane idle timers', () => { }); }); +describe('control-plane preparation attempt budget', () => { + it('outlasts every step of one preparation attempt in sequence', () => { + const { sandbox, wrapper } = CONTROL_PLANE_TIMERS; + const steps = + sandbox.providerCreateMs + + sandbox.wrapperFirstConnectMs + + sandbox.wrapperHelloMs + + wrapper.restoreMs + + wrapper.cloneMs + + wrapper.kiloRuntimeStartMs + + wrapper.kiloSessionMs; + + // A shorter attempt deadline would fail a slow but valid preparation as + // `preparation_timeout` while each step stayed inside its own bound. + expect(sandbox.routePreparationMs).toBeGreaterThanOrEqual(steps); + }); + + it('keeps the attempt budget real when the development divisor shrinks step timers', () => { + const scaled = resolveControlPlaneTimers({ CONTROL_PLANE_TIMER_DIVISOR: '10' }); + + expect(scaled.sandbox.routePreparationMs).toBe(CONTROL_PLANE_TIMERS.sandbox.routePreparationMs); + expect(scaled.sandbox.routePreparationMs).toBeGreaterThanOrEqual( + scaled.wrapper.cloneMs + scaled.wrapper.restoreMs + ); + }); +}); + describe('sandbox idle policy for route origins', () => { it('keeps the interactive window for a sandbox without routes', () => { expect(sandboxIdlePolicyForOrigins([])).toBe('interactive'); diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index 44fe2d18d99..12e8b97f4a4 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -72,23 +72,52 @@ export type ControlPlaneTimers = { }; }; +// External-work bounds for one preparation attempt. They stay real (never +// scaled) because they bound provider and container work. +const PROVIDER_CREATE_MS = 2 * MINUTE_MS; +const PROVIDER_CREATE_RETRY_MS = 10_000; +const WRAPPER_FIRST_CONNECT_MS = 5 * MINUTE_MS; +const WRAPPER_HELLO_MS = 30_000; +const RESTORE_MS = 2 * MINUTE_MS; +const CLONE_MS = 6 * MINUTE_MS; +const KILO_RUNTIME_START_MS = 2 * MINUTE_MS; +const KILO_SESSION_MS = 2 * MINUTE_MS; + +/** + * One preparation attempt runs these steps in order: sandbox create, wrapper + * first connect and hello, snapshot restore (falling back to a clone), Kilo + * runtime start, then the session import. Every step keeps its own bound, so + * the attempt deadline is their sum. A shorter deadline fails a slow but valid + * preparation as `preparation_timeout` while each step stayed inside its own + * budget — the failure names a timeout the plan never allowed to finish. + */ +const ROUTE_PREPARATION_MS = + PROVIDER_CREATE_MS + + WRAPPER_FIRST_CONNECT_MS + + WRAPPER_HELLO_MS + + RESTORE_MS + + CLONE_MS + + KILO_RUNTIME_START_MS + + KILO_SESSION_MS; + // The divisor shortens only internal deadlines whose partner also scales, so // their orderings hold (the 90-s reconnect window stays longer than the 30-s // max backoff). Timers that bound external work stay real: provider create and -// container boot, the 12-min prep attempt that must outlast clone+runtime+ -// session, the pause before a failed create is retried, clone/runtime/session -// bounds, SSE silence (Kilo heartbeats every 10-15 s), health request, the -// real-SSE reconnect window, credential grants and the provider stop ladder. +// container boot, the prep attempt that must outlast create+connect+clone+ +// runtime+session, the pause before a failed create is retried, clone/runtime/ +// session bounds, SSE silence (Kilo heartbeats every 10-15 s), health request, +// the real-SSE reconnect window, credential grants and the provider stop +// ladder. function buildControlPlaneTimers(divisor: number): ControlPlaneTimers { const ms = (value: number): number => Math.max(1, Math.round(value / divisor)); const sandbox = { - providerCreateMs: 2 * MINUTE_MS, - providerCreateRetryMs: 10_000, - wrapperFirstConnectMs: 5 * MINUTE_MS, - wrapperHelloMs: 30_000, + providerCreateMs: PROVIDER_CREATE_MS, + providerCreateRetryMs: PROVIDER_CREATE_RETRY_MS, + wrapperFirstConnectMs: WRAPPER_FIRST_CONNECT_MS, + wrapperHelloMs: WRAPPER_HELLO_MS, heartbeatMs: ms(45_000), reconnectMs: ms(90_000), - routePreparationMs: 12 * MINUTE_MS, + routePreparationMs: ROUTE_PREPARATION_MS, idleMs: ms(10 * MINUTE_MS), unattendedIdleMs: ms(MINUTE_MS), // Bounds one Sandbox -> Session notification; real (not scaled) like the @@ -108,11 +137,11 @@ function buildControlPlaneTimers(divisor: number): ControlPlaneTimers { heartbeatNegotiationMs: ms(1_000), // Scales with the reconnect window it must stay inside. handshakeMs: ms(30_000), - cloneMs: 6 * MINUTE_MS, - restoreMs: 2 * MINUTE_MS, + cloneMs: CLONE_MS, + restoreMs: RESTORE_MS, captureMs: 5 * MINUTE_MS + 10_000, - kiloRuntimeStartMs: 2 * MINUTE_MS, - kiloSessionMs: 2 * MINUTE_MS, + kiloRuntimeStartMs: KILO_RUNTIME_START_MS, + kiloSessionMs: KILO_SESSION_MS, sseSilenceMs: 30_000, healthRequestMs: 5_000, sseReconnectLimit: 6, diff --git a/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts b/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts index bb73f8f69aa..c32bc7cf643 100644 --- a/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts +++ b/services/cloud-agent-next/test/integration/control-plane-session-v2-transport.test.ts @@ -125,7 +125,7 @@ describe('SandboxSessionV2 queued transport recovery', () => { const scaled = resolveControlPlaneTimers({ CONTROL_PLANE_TIMER_DIVISOR: '10' }); expect(scaled.session.sandboxRpcDeadlineMs).toBe(2_000); expect(scaled.session.transportRecoveryMs).toBe(1_500); - expect(scaled.sandbox.routePreparationMs).toBe(12 * 60_000); + expect(scaled.sandbox.routePreparationMs).toBe(CONTROL_PLANE_TIMERS.sandbox.routePreparationMs); expect(resolveControlPlaneTimers({ CONTROL_PLANE_TIMER_DIVISOR: 'invalid' })).toBe( CONTROL_PLANE_TIMERS ); From 9c9404cc5060a24abde00b3392c083f646157b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:45:45 +0000 Subject: [PATCH 2/7] fix(cloud-agent-next): cover the repository create bound in a preparation attempt A repository-backed container allocation creates its sandbox under the wider repository startup bound, which is the create path every preparation attempt with a repository takes. Use that bound in the attempt budget and assert it in the invariant test, so a valid slow repository preparation cannot outrun the attempt deadline while each step stays inside its own bound. --- .../src/shared/control-plane-timers.test.ts | 6 +++++- .../cloud-agent-next/src/shared/control-plane-timers.ts | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts index e4055d0a9ba..a4924f8b46f 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { DEADLINE_MS } from '../sandbox-control/deadlines.js'; import { CONTROL_PLANE_TIMERS, resolveControlPlaneTimers, @@ -38,8 +39,11 @@ describe('control-plane idle timers', () => { describe('control-plane preparation attempt budget', () => { it('outlasts every step of one preparation attempt in sequence', () => { const { sandbox, wrapper } = CONTROL_PLANE_TIMERS; + // A repository-backed container allocation runs its create under the wider + // repository startup bound, so that is the create step this covers. + const createMs = Math.max(sandbox.providerCreateMs, DEADLINE_MS.repositoryStartup); const steps = - sandbox.providerCreateMs + + createMs + sandbox.wrapperFirstConnectMs + sandbox.wrapperHelloMs + wrapper.restoreMs + diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index 12e8b97f4a4..b8d6fa5d79e 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -1,3 +1,5 @@ +import { DEADLINE_MS } from '../sandbox-control/deadlines.js'; + export const CONTROL_PLANE_TIMER_OVERRIDE_ENV = 'CONTROL_PLANE_TIMER_DIVISOR'; const MINUTE_MS = 60_000; @@ -76,6 +78,10 @@ export type ControlPlaneTimers = { // scaled) because they bound provider and container work. const PROVIDER_CREATE_MS = 2 * MINUTE_MS; const PROVIDER_CREATE_RETRY_MS = 10_000; +// A repository-backed container allocation widens its create bound to +// DEADLINE_MS.repositoryStartup (`control-plane/sandbox/sandbox-do.ts`), which +// is the create path every preparation attempt with a repository takes. +const PREPARATION_CREATE_MS = Math.max(PROVIDER_CREATE_MS, DEADLINE_MS.repositoryStartup); const WRAPPER_FIRST_CONNECT_MS = 5 * MINUTE_MS; const WRAPPER_HELLO_MS = 30_000; const RESTORE_MS = 2 * MINUTE_MS; @@ -92,7 +98,7 @@ const KILO_SESSION_MS = 2 * MINUTE_MS; * budget — the failure names a timeout the plan never allowed to finish. */ const ROUTE_PREPARATION_MS = - PROVIDER_CREATE_MS + + PREPARATION_CREATE_MS + WRAPPER_FIRST_CONNECT_MS + WRAPPER_HELLO_MS + RESTORE_MS + From 922606dda58db40e600850e1fc7f6d27850326e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:53:28 +0000 Subject: [PATCH 3/7] fix(cloud-agent-next): keep the preparation timers free of worker-only imports This module is bundled into the wrapper with only src/shared, so it cannot import the worker's other sources. State the repository create bound locally with the source it mirrors named, and let the invariant test compare it with that source instead. --- .../src/shared/control-plane-timers.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index b8d6fa5d79e..47e6a816510 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -1,5 +1,3 @@ -import { DEADLINE_MS } from '../sandbox-control/deadlines.js'; - export const CONTROL_PLANE_TIMER_OVERRIDE_ENV = 'CONTROL_PLANE_TIMER_DIVISOR'; const MINUTE_MS = 60_000; @@ -78,10 +76,13 @@ export type ControlPlaneTimers = { // scaled) because they bound provider and container work. const PROVIDER_CREATE_MS = 2 * MINUTE_MS; const PROVIDER_CREATE_RETRY_MS = 10_000; -// A repository-backed container allocation widens its create bound to -// DEADLINE_MS.repositoryStartup (`control-plane/sandbox/sandbox-do.ts`), which -// is the create path every preparation attempt with a repository takes. -const PREPARATION_CREATE_MS = Math.max(PROVIDER_CREATE_MS, DEADLINE_MS.repositoryStartup); +// A repository-backed container allocation creates its sandbox under the wider +// repository startup bound (`DEADLINE_MS.repositoryStartup` in +// `sandbox-control/deadlines.ts`), which is the create path every preparation +// attempt with a repository takes. It is restated here because this module is +// bundled into the wrapper without the worker's other sources; +// `control-plane-timers.test.ts` asserts the attempt budget against that bound. +const PREPARATION_CREATE_MS = 5 * MINUTE_MS; const WRAPPER_FIRST_CONNECT_MS = 5 * MINUTE_MS; const WRAPPER_HELLO_MS = 30_000; const RESTORE_MS = 2 * MINUTE_MS; From eba464c55c413414b2aa169592079e606ecb5753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:03:32 +0000 Subject: [PATCH 4/7] fix(cloud-agent-next): count one retried create in the preparation budget A transient create failure pauses and creates again inside the same preparation attempt, so the attempt budget carries one retried create as well. Further retries stay bounded by the attempt deadline, which remains what stops a create that keeps failing. --- .../src/shared/control-plane-timers.test.ts | 3 +++ .../cloud-agent-next/src/shared/control-plane-timers.ts | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts index a4924f8b46f..b6784882b76 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts @@ -43,6 +43,9 @@ describe('control-plane preparation attempt budget', () => { // repository startup bound, so that is the create step this covers. const createMs = Math.max(sandbox.providerCreateMs, DEADLINE_MS.repositoryStartup); const steps = + // A transient create failure creates again inside the same attempt. + createMs + + sandbox.providerCreateRetryMs + createMs + sandbox.wrapperFirstConnectMs + sandbox.wrapperHelloMs + diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index 47e6a816510..ff8d8f61fb3 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -97,8 +97,15 @@ const KILO_SESSION_MS = 2 * MINUTE_MS; * the attempt deadline is their sum. A shorter deadline fails a slow but valid * preparation as `preparation_timeout` while each step stayed inside its own * budget — the failure names a timeout the plan never allowed to finish. + * + * A transient create failure pauses and creates again inside the same attempt + * (`control-plane/sandbox/allocation.ts`), so one retried create is counted + * too. Further retries stay bounded by this deadline, which is what stops a + * create that keeps failing. */ const ROUTE_PREPARATION_MS = + PREPARATION_CREATE_MS + + PROVIDER_CREATE_RETRY_MS + PREPARATION_CREATE_MS + WRAPPER_FIRST_CONNECT_MS + WRAPPER_HELLO_MS + From fa3b6d1621c65b4e3bd6bbdb4f317e8e806b241c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:35:29 +0000 Subject: [PATCH 5/7] fix(cloud-agent-next): count the workspace save and step retries in a preparation attempt --- .../cloud-agent-next/docs/control-plane.md | 15 ++++---- .../src/shared/control-plane-timers.test.ts | 7 ++-- .../src/shared/control-plane-timers.ts | 36 ++++++++++++------- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/services/cloud-agent-next/docs/control-plane.md b/services/cloud-agent-next/docs/control-plane.md index 6d946a51e87..6462cf2f862 100644 --- a/services/cloud-agent-next/docs/control-plane.md +++ b/services/cloud-agent-next/docs/control-plane.md @@ -114,7 +114,7 @@ terminal. Nothing else. | Outcome (status, reason, `lastMessageId`) | Every `accepted` message up to and including `lastMessageId` takes the status. If `lastMessageId` is unknown, every `accepted` message does. | | Stop | All `queued` and `accepted` → `cancelled`; send `abort` (best effort). | | Cancel one queued message | That message → `cancelled`. | -| Backstop alarm | `queued` older than 16.5 minutes → `failed` (`preparation_timeout`). `accepted` older than 125 minutes → `failed` (`no_outcome`). | +| Backstop alarm | `queued` older than its queued backstop → `failed` (`preparation_timeout`). `accepted` older than its accepted backstop → `failed` (`no_outcome`). | Stop still forwards a best-effort abort to an existing ready Kilo route when no message is queued or accepted and the wrapper has no active turn. It does not @@ -123,7 +123,7 @@ rewrite terminal message outcomes or prepare/wake a stopped sandbox. Late outcomes and notifications for terminal messages are ignored. The backstop exists only for lost notifications; the Sandbox DO and the wrapper settle every normal case earlier. Both values are derived from the owners' timers so the backstop does not end work within its owner's bound: -queued = reconnect window (90 s) + one preparation attempt (12 min) + 3 min, which covers the +queued = reconnect window (90 s) + one preparation attempt (the sum of its step bounds, below) + 3 min, which covers the longest normal wait (a message sent while the socket is down, then a fresh attempt on a new sandbox); accepted = turn hard cap (120 min) + 5 min. The hard cap bounds one native execution, not an accepted message, so a batch whose follow-up waits behind a serialized command spans several @@ -311,7 +311,8 @@ so a later report changes nothing. `preparing` → `ready` → `failed`. Each preparation attempt has a deadline that outlasts its own steps in sequence (sandbox create, wrapper first connect and hello, snapshot restore with -the clone fallback, Kilo runtime start, session import). An attempt +the clone fallback, the workspace save, Kilo runtime start, session import), including the one +retry each of the last two steps may take. An attempt starts when a route enters `preparing`: the first `prepare`, a `prepare` after `failed`, or a re-prepare after a wrapper restart. A repeated `prepare`, a reconnect or a new allocation inside the attempt keeps its deadline. @@ -855,7 +856,7 @@ them through one development-only override. | Owner | Timer | Value | On expiry | |---|---|---|---| -| Session DO | Queued backstop | 16.5 min from send (reconnect + attempt + 3 min) | `failed` (`preparation_timeout`) | +| Session DO | Queued backstop | From send: reconnect window + one preparation attempt + 3 min | `failed` (`preparation_timeout`) | | Session DO | Accepted backstop | 125 min from accept | `failed` (`no_outcome`) | | Session DO | Sandbox transport pass / best-effort abort | 2 s including RPC retries, not scaled | Retain queued intent / return from abort | | Session DO | Queued transport recovery | 15 s after exhaustion; development-only scaling | Consume once; passive status then bounded prepare/deliver, no self-rearm | @@ -865,7 +866,7 @@ them through one development-only override. | Sandbox DO | Unbound wrapper hello | 30 s from socket admission, not scaled | Close candidate only; attachment deadline survives hibernation and participates in the existing alarm even while stopped | | Sandbox DO | Heartbeat | 45 s | Treat the socket as lost | | Sandbox DO | Reconnect | 90 s from the last frame received | Stop the sandbox | -| Sandbox DO | Route preparation | 12 min per preparation attempt | Route `failed` (`preparation_timeout`) | +| Sandbox DO | Route preparation | Sum of one preparation attempt's step bounds (see Preparation) | Route `failed` (`preparation_timeout`) | | Sandbox DO | Session notification | 2 s from enqueue including queue wait and RPC retries, not scaled | Drop expired queued work before RPC; bounded best-effort loss, existing Session backstop | | Sandbox DO | Idle | 10 min without activity | Stop the sandbox | | Sandbox DO | Provider lease | Existing lease length, renewed while active | Provider may stop an inactive sandbox | @@ -915,7 +916,7 @@ them through one development-only override. | Native execution over 120 min | Native session supervisor | Bounded tree abort | Accepted fail (`execution_limit`), or routed error without a current turn | | Idle 10 min, question pending | Sandbox DO | Stop the sandbox | Accepted fail (`sandbox_stopped`); a later answer is a new message | | Sandbox gone | Provider via Sandbox DO | New allocation on next `prepare` | Accepted fail (`sandbox_lost`); queued re-prepare | -| Notification lost | Session DO backstop | — | Fail at 16.5 or 125 min | +| Notification lost | Session DO backstop | — | Fail at the queued or accepted backstop | | User Stop | Session DO | Abort the Kilo session only | Queued and accepted `cancelled` | | Auto-commit or condense fails | Wrapper | None | Warning event; turn `completed` | | Provider stop not confirmed | Sandbox DO | Logged; next `prepare` may create again | None; worktree deletion reports incomplete | @@ -1362,7 +1363,7 @@ counts. overflow drops newest, loss diagnostics stay bounded/internal and expired queue heads make no RPC. Heartbeats, alarms, stop and reconnect continue while notifications are held. Release retires references; reconstruction replays no notification backlog. Public events, reports, callbacks - and the 16.5/125-minute lost-notification backstops are unchanged. + and the queued and accepted lost-notification backstops are unchanged. 22. **Permanent versus transient creation failure.** Balance lost after send preflight fails queued work promptly with `billing_blocked`, not preparation timeout. Proven local invalid configuration fails promptly with `invalid_configuration`. The safe stream reason, durable route/message, diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts index b6784882b76..f485e6e6149 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.test.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.test.ts @@ -51,8 +51,11 @@ describe('control-plane preparation attempt budget', () => { sandbox.wrapperHelloMs + wrapper.restoreMs + wrapper.cloneMs + - wrapper.kiloRuntimeStartMs + - wrapper.kiloSessionMs; + wrapper.captureMs + + // Kilo runtime start and the Kilo session each retry once inside the + // attempt, so each can consume two of its own bounds. + 2 * wrapper.kiloRuntimeStartMs + + 2 * wrapper.kiloSessionMs; // A shorter attempt deadline would fail a slow but valid preparation as // `preparation_timeout` while each step stayed inside its own bound. diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index ff8d8f61fb3..6ac0b81b47b 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -87,21 +87,30 @@ const WRAPPER_FIRST_CONNECT_MS = 5 * MINUTE_MS; const WRAPPER_HELLO_MS = 30_000; const RESTORE_MS = 2 * MINUTE_MS; const CLONE_MS = 6 * MINUTE_MS; +// The attempt's own last workspace step: the Sandbox DO saves the prepared +// workspace for a repository-backed route, and preparation waits for it. +const CAPTURE_MS = 5 * MINUTE_MS + 10_000; const KILO_RUNTIME_START_MS = 2 * MINUTE_MS; const KILO_SESSION_MS = 2 * MINUTE_MS; /** * One preparation attempt runs these steps in order: sandbox create, wrapper - * first connect and hello, snapshot restore (falling back to a clone), Kilo - * runtime start, then the session import. Every step keeps its own bound, so - * the attempt deadline is their sum. A shorter deadline fails a slow but valid - * preparation as `preparation_timeout` while each step stayed inside its own - * budget — the failure names a timeout the plan never allowed to finish. + * first connect and hello, snapshot restore (falling back to a clone), the + * workspace save, Kilo runtime start, then the session import. Every step keeps + * its own bound, so the attempt deadline is their sum. A shorter deadline fails + * a slow but valid preparation as `preparation_timeout` while each step stayed + * inside its own budget — the failure names a timeout the plan never allowed to + * finish. * + * Two steps are retried once inside the same attempt, so each counts twice. * A transient create failure pauses and creates again inside the same attempt * (`control-plane/sandbox/allocation.ts`), so one retried create is counted * too. Further retries stay bounded by this deadline, which is what stops a * create that keeps failing. + * + * Setup commands have no term here: each command carries its own inactivity and + * hard timeout (`wrapper/src/control-plane/prepare.ts`), while their number is + * the route's configuration, so this deadline is their aggregate bound. */ const ROUTE_PREPARATION_MS = PREPARATION_CREATE_MS + @@ -111,17 +120,18 @@ const ROUTE_PREPARATION_MS = WRAPPER_HELLO_MS + RESTORE_MS + CLONE_MS + - KILO_RUNTIME_START_MS + - KILO_SESSION_MS; + CAPTURE_MS + + 2 * KILO_RUNTIME_START_MS + + 2 * KILO_SESSION_MS; // The divisor shortens only internal deadlines whose partner also scales, so // their orderings hold (the 90-s reconnect window stays longer than the 30-s // max backoff). Timers that bound external work stay real: provider create and -// container boot, the prep attempt that must outlast create+connect+clone+ -// runtime+session, the pause before a failed create is retried, clone/runtime/ -// session bounds, SSE silence (Kilo heartbeats every 10-15 s), health request, -// the real-SSE reconnect window, credential grants and the provider stop -// ladder. +// container boot, the prep attempt that must outlast create+connect+restore+ +// clone+save+runtime+session, the pause before a failed create is retried, +// restore/clone/save/runtime/session bounds, SSE silence (Kilo heartbeats every +// 10-15 s), health request, the real-SSE reconnect window, credential grants +// and the provider stop ladder. function buildControlPlaneTimers(divisor: number): ControlPlaneTimers { const ms = (value: number): number => Math.max(1, Math.round(value / divisor)); const sandbox = { @@ -153,7 +163,7 @@ function buildControlPlaneTimers(divisor: number): ControlPlaneTimers { handshakeMs: ms(30_000), cloneMs: CLONE_MS, restoreMs: RESTORE_MS, - captureMs: 5 * MINUTE_MS + 10_000, + captureMs: CAPTURE_MS, kiloRuntimeStartMs: KILO_RUNTIME_START_MS, kiloSessionMs: KILO_SESSION_MS, sseSilenceMs: 30_000, From d608195516c8dca19b7f9bbd16323b7a7961161a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:44:28 +0000 Subject: [PATCH 6/7] docs(cloud-agent-next): state that setup work is bounded by the attempt deadline --- services/cloud-agent-next/docs/control-plane.md | 4 +++- .../src/shared/control-plane-timers.ts | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/services/cloud-agent-next/docs/control-plane.md b/services/cloud-agent-next/docs/control-plane.md index 6462cf2f862..16225b78efb 100644 --- a/services/cloud-agent-next/docs/control-plane.md +++ b/services/cloud-agent-next/docs/control-plane.md @@ -315,7 +315,9 @@ the clone fallback, the workspace save, Kilo runtime start, session import), inc retry each of the last two steps may take. An attempt starts when a route enters `preparing`: the first `prepare`, a `prepare` after `failed`, or a re-prepare after a wrapper restart. A repeated `prepare`, a reconnect or a new allocation inside the -attempt keeps its deadline. +attempt keeps its deadline. Setup commands keep their own per-command limits, and how many run is +configuration, so the attempt deadline is their aggregate stop: a preparation whose setup work +together outlasts the attempt fails as `preparation_timeout`. `prepare` returns the route view that the Sandbox DO would notify: `ready` only when the route is ready and the wrapper is connected, `reconnecting` when the route is ready but the socket is down, diff --git a/services/cloud-agent-next/src/shared/control-plane-timers.ts b/services/cloud-agent-next/src/shared/control-plane-timers.ts index 6ac0b81b47b..b8651492cc7 100644 --- a/services/cloud-agent-next/src/shared/control-plane-timers.ts +++ b/services/cloud-agent-next/src/shared/control-plane-timers.ts @@ -108,9 +108,13 @@ const KILO_SESSION_MS = 2 * MINUTE_MS; * too. Further retries stay bounded by this deadline, which is what stops a * create that keeps failing. * - * Setup commands have no term here: each command carries its own inactivity and - * hard timeout (`wrapper/src/control-plane/prepare.ts`), while their number is - * the route's configuration, so this deadline is their aggregate bound. + * Setup commands have no term here. Each command carries its own inactivity and + * hard timeout (`wrapper/src/control-plane/prepare.ts`), while how many run and + * how long each takes are the route's configuration (bounded per command, not in + * total). This deadline is therefore their aggregate stop by design: a route + * whose setup work together outlasts the attempt fails as + * `preparation_timeout`, because a bound that grew with the configured command + * count would also stretch how long a stuck preparation can hold a session. */ const ROUTE_PREPARATION_MS = PREPARATION_CREATE_MS + From 5098ec20ad5276f7e57992cd02365336eb17ef73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Igor=20=C5=A0=C4=87eki=C4=87?= <4062236+iscekic@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:48:00 +0000 Subject: [PATCH 7/7] chore: re-trigger review