diff --git a/k8s/helm/commonly/values-staging.yaml b/k8s/helm/commonly/values-staging.yaml index 0df2880a8e..2e47ad00e8 100644 --- a/k8s/helm/commonly/values-staging.yaml +++ b/k8s/helm/commonly/values-staging.yaml @@ -204,6 +204,15 @@ backend: # and accepting a staging consent would have sent a staging authorization code to prod. # Caught on the staging Atlassian walk 2026-10-23. backendUrl: "https://api-staging.commonly.me" + # ADR-023 W2 hosted runtime, staging worker (added 2026-11-01). Setting this + # renders the hosted-runtime ExternalSecret (templates/secrets/hosted-runtime.yaml), + # which pulls commonly-staging-hosted-runtime-admin-token from staging SM. With the + # URL and token both set the backend is CONFIGURED, so provision no longer returns + # 503 unconfigured; until the worker itself is deployed it returns 502 + # hosted_runtime_unreachable instead. The worker is the separate + # `commonly-agent-runtime-staging` Cloudflare env (wrangler.toml); until it is live + # the first-user-walk treats both not-ready codes as a skip, not a failure. + hostedRuntimeUrl: "https://commonly-agent-runtime-staging.commonlyai.workers.dev" # Staging backend runs on the SPOT pool at a fraction of prod's requests, so it fits # the nodes we already have (spot-pool sits at single-digit % CPU) without growing a # pool. Prod backend stays on dev-pool because it holds session state a spot diff --git a/scripts/first-user-walk-cleanup.mjs b/scripts/first-user-walk-cleanup.mjs index 1b5e25ef67..7d04b4d27f 100644 --- a/scripts/first-user-walk-cleanup.mjs +++ b/scripts/first-user-walk-cleanup.mjs @@ -58,7 +58,7 @@ export const recoverRunPods = async ({ api, token, userId, podName, podIds }) => // `deleteAccount: false` is the shared staging sign-in account: the run removes // only the pods it recorded, never pods that account owns from elsewhere, and // never the account itself. -export const cleanupRun = async ({ api, token, password, email, userId, agentName, podIds = [], onBotIds = () => {}, deleteAccount = true }) => { +export const cleanupRun = async ({ api, token, password, email, userId, agentName, podIds = [], onBotIds = () => {}, deleteAccount = true, hostedProvisioned = true }) => { const survivors = []; const botIds = new Set(); @@ -69,9 +69,18 @@ export const cleanupRun = async ({ api, token, password, email, userId, agentNam }; // Stop the runtime before the seat goes: uninstalling the seat does not stop - // the hosted worker. 404 means there was no hosted install to stop. + // the hosted worker. 404 means there was no hosted install to stop. A not-ready + // deprovision (503 unconfigured / 502 unreachable by code) is exempt from + // survivor-tracking ONLY when provisioning did NOT succeed (hostedProvisioned + // false — skipped or failed, so nothing was created to leak). When provision + // succeeded, the backend revokes the runtime token only after a SUCCESSFUL stop, + // so a failed deprovision leaves the worker + token live — a real survivor, + // staging or not. Default hostedProvisioned=true keeps the conservative behaviour + // for any caller that does not pass the provisioning outcome. const stop = await api('POST', '/api/hosted/deprovision', { token, body: { agentName } }); - if (stop.status >= 300 && stop.status !== 404) survivors.push(`hosted:${agentName} (deprovision ${stop.status})`); + const stopNotReady = !hostedProvisioned + && (stop.data?.code === 'hosted_runtime_unconfigured' || stop.data?.code === 'hosted_runtime_unreachable'); + if (stop.status >= 300 && stop.status !== 404 && !stopNotReady) survivors.push(`hosted:${agentName} (deprovision ${stop.status})`); let owned = []; if (deleteAccount) { diff --git a/scripts/first-user-walk.mjs b/scripts/first-user-walk.mjs index 14e43a08e3..84076564d4 100644 --- a/scripts/first-user-walk.mjs +++ b/scripts/first-user-walk.mjs @@ -59,6 +59,13 @@ const NOT_COVERED = 'Bring your own connect step is not implemented yet'; const expect = (cond, msg) => { if (!cond) throw new Error(msg); }; +// A step can decline to run rather than fail. runStep records a SkipStep as +// result 'skipped' and does not rethrow, so the run continues and stays +// incomplete rather than going red. Used when the hosted runtime worker is not +// ready on this instance — staging before its worker is deployed, where +// provision returns 503 hosted_runtime_unconfigured or 502 hosted_runtime_unreachable. +class SkipStep extends Error { constructor(message) { super(message); this.skip = true; } } + /** * One walk run. Every dependency is injectable so the production path can be * exercised under test without the network or the clock. @@ -98,6 +105,10 @@ export const runWalk = async ({ // Null until chooseWalkMode accepts the run, so a refused run is never reported as signup. let walkMode = null; let cleanupRan = false; + // True only once provision returned 200: a worker + runtime token were actually + // created, so cleanup must treat a FAILED deprovision as a survivor even on staging. + // False when provision was skipped or failed — nothing was created to leak. + let hostedProvisioned = false; // Signup returns no user id, so the manifest carries the exact username instead: // SRE finds the account by that name, not by a prefix. Never the email. const manifest = { mode: null, userId: null, username: null, podIds: [], unresolvedPodNames: [], agentName: null, botIds: [], survivors: [], status: 'running' }; @@ -143,6 +154,12 @@ export const runWalk = async ({ log(`PASS ${name} (${now() - started} ms)`); return value; } catch (caught) { + if (caught instanceof SkipStep) { + const reason = redact(caught.message); + results.push({ name, result: 'skipped', ms: now() - started, reason }); + log(`SKIP ${name} (${now() - started} ms): ${reason}`); + return undefined; + } const reason = redact(caught.message); results.push({ name, result: 'fail', ms: now() - started, reason }); error(`FAIL ${name} (${now() - started} ms): ${reason}`); @@ -150,6 +167,14 @@ export const runWalk = async ({ } }; + // Record a downstream step as skipped without running it — used when an + // earlier skipped step means this one cannot run (hosted-reply needs the + // provision that hosted-install skipped). + const skipStep = (name, reason) => { + results.push({ name, result: 'skipped', ms: 0, reason }); + log(`SKIP ${name}: ${reason}`); + }; + // Per-run manifest: ids only, outside the repo, for SRE's admin purge of what // the walk's own token cannot remove (Scout, the hosted agent's User). It holds // no password, token or email. It is named by the run stamp, not the username. @@ -198,6 +223,11 @@ export const runWalk = async ({ agentName, podIds: manifest.podIds, deleteAccount: walkMode.deleteAccount, + // A not-ready deprovision (503/502 by code) is exempt from survivor-tracking + // ONLY when provisioning did not succeed — nothing was created to leak. If + // provision returned 200, a failed deprovision IS a leak (the backend revokes + // the token only after a successful stop), so it stays a survivor, staging or not. + hostedProvisioned, // Persist each pod's bot ids before that pod is deleted, so a throw // later in the loop still leaves the ids SRE needs. onBotIds: (podId, bots) => { @@ -324,6 +354,14 @@ export const runWalk = async ({ return created; }); + // The not-ready skip is scoped to the STAGING run (mode 'signin', the shared + // account on api-staging), which is the only place the worker may legitimately + // be undeployed during rollout. On prod (signin-prod) and on signup the worker + // is expected to be live, so a not-ready provision there is a genuine outage and + // must FAIL — never be hidden as a skip. This walk is advisory; staging-smoke, + // not it, gates the deploy. + const hostedOptional = walkMode.mode === 'signin'; + let hostedSkipped = false; await runStep('hosted-install', async () => { manifest.agentName = agentName; writeManifest(); @@ -333,10 +371,22 @@ export const runWalk = async ({ }); expect(r.status < 400, `install ${r.status}: ${JSON.stringify(r.data)}`); const p = await api('POST', '/api/hosted/provision', { token, body: { agentName } }); + // Not-ready comes back in two shapes: 503 hosted_runtime_unconfigured (URL or + // admin token unset) or 502 hosted_runtime_unreachable (configured, but the + // worker is not deployed yet — the backend maps every worker fetch/non-ok to + // 502, see hostedRuntimeService). Keyed on the error CODE so a 503/502 from a + // proxy without that code, or a genuine 500 (token_mint_failed), still fails. + const notReady = p.status >= 400 + && (p.data?.code === 'hosted_runtime_unconfigured' || p.data?.code === 'hosted_runtime_unreachable'); + if (hostedOptional && notReady) { hostedSkipped = true; throw new SkipStep(`no staging worker (${p.data.code})`); } expect(p.status === 200 && p.data?.provisioned, `provision ${p.status}: ${JSON.stringify(p.data)}`); + // Provision succeeded: a worker + token now exist, so a failed deprovision in + // cleanup is a real leak (survivor), staging or not. + hostedProvisioned = true; }); - await runStep('hosted-reply', async () => { + if (hostedSkipped) skipStep('hosted-reply', 'no staging worker (hosted runtime not ready)'); + else await runStep('hosted-reply', async () => { const m = await api('POST', `/api/messages/${pod._id}`, { token, body: { content: `@${agentName} walk check: reply with one short sentence containing KINGFISHER.` }, diff --git a/scripts/first-user-walk.test.mjs b/scripts/first-user-walk.test.mjs index 931835ff20..a60366163b 100644 --- a/scripts/first-user-walk.test.mjs +++ b/scripts/first-user-walk.test.mjs @@ -20,8 +20,13 @@ const SHARED_EMAIL = 'walk-shared@commonly.test'; // password the password the fake account accepts // echoRegister a refused register echoes the request body back, as a bad error would // seeded the account already exists (the shared staging sign-in account) +// hostedProvision status of POST /api/hosted/provision (and the matching +// deprovision). 503 (unconfigured) and 502 (unreachable) are +// "no worker yet"; on a STAGING run the hosted steps skip, off +// staging they fail. Any other 5xx has no not-ready code -> fail. const fakeBackend = ({ preview = 200, create = 'ok', password = PASSWORD, echoRegister = false, seeded = false, + hostedProvision = 200, hostedDeprovision = undefined, } = {}) => { const state = { accountExists: seeded, pods: [], lastAgent: null, calls: [], loginEmails: [] }; const reply = (status, data) => ({ status, json: async () => data }); @@ -69,8 +74,25 @@ const fakeBackend = ({ } } if (method === 'POST' && p === '/api/registry/install') return reply(200, {}); - if (method === 'POST' && p === '/api/hosted/provision') return reply(200, { provisioned: true }); - if (method === 'POST' && p === '/api/hosted/deprovision') return reply(404, {}); + if (method === 'POST' && p === '/api/hosted/provision') { + // The backend's real shapes: 200 provisioned; 503 hosted_runtime_unconfigured + // (URL/token unset); 502 hosted_runtime_unreachable (worker not deployed). Any + // other 5xx is a genuine error with NO not-ready code, which must still fail. + if (hostedProvision === 200) return reply(200, { provisioned: true }); + if (hostedProvision === 503) return reply(503, { code: 'hosted_runtime_unconfigured', message: 'not ready' }); + if (hostedProvision === 502) return reply(502, { code: 'hosted_runtime_unreachable', message: 'not ready' }); + return reply(hostedProvision, { message: 'Hosted runtime request failed' }); + } + if (method === 'POST' && p === '/api/hosted/deprovision') { + // Default: mirror provision (unconfigured/unreachable backend answers both the + // same). hostedDeprovision overrides it to model a SUCCESSFUL provision whose + // deprovision then fails (worker + token leak), which must be a survivor. + const d = hostedDeprovision ?? (hostedProvision === 200 ? 404 : hostedProvision); + if (d === 404) return reply(404, {}); + if (d === 503) return reply(503, { code: 'hosted_runtime_unconfigured' }); + if (d === 502) return reply(502, { code: 'hosted_runtime_unreachable' }); + return reply(d, {}); + } if (method === 'DELETE' && p.startsWith('/api/registry/agents/')) return reply(404, {}); if (method === 'POST' && /^\/api\/messages\//.test(p)) { state.lastAgent = (body.content.match(/@(\S+) walk check/) || [])[1] || null; @@ -135,6 +157,78 @@ test('a clean run writes the machine-readable signal: every covered step passes, assert.equal(state.pods.length, 0); }); +// STAGING run only: both not-ready shapes skip — 503 (URL/token unset) and 502 +// (configured but the worker is not deployed, the state values-staging.yaml creates +// before the operator runs `wrangler deploy --env staging`). The run must stay +// incomplete, and the matching not-ready deprovision must NOT redden cleanup. +for (const status of [503, 502]) { + test(`staging: no hosted worker (provision ${status}) skips the hosted steps, keeps cleanup clean, overall incomplete`, async () => { + const dirsFor = dirs(); + const { fetchImpl, state } = fakeBackend({ seeded: true, hostedProvision: status }); + const { exitCode, result } = await run(fetchImpl, dirsFor, { + base: 'https://api-staging.commonly.me', + env: { COMMONLY_WALK_EMAIL: SHARED_EMAIL }, + }); + + assert.equal(result.mode, 'signin'); + assert.equal(exitCode, 2); + assert.equal(result.overall, 'incomplete'); + + const byName = Object.fromEntries(result.steps.map((s) => [s.name, s.result])); + assert.equal(byName['hosted-install'], 'skipped'); + assert.equal(byName['hosted-reply'], 'skipped'); + assert.match(result.steps.find((s) => s.name === 'hosted-install').reason, /no staging worker/); + assert.ok(result.steps.every((s) => s.result !== 'fail')); + // The not-ready deprovision did NOT become a survivor; cleanup stays clean, + // and the shared staging account is preserved. + assert.equal(result.cleanup.ok, true); + assert.equal(state.accountExists, true); + }); +} + +test('staging: provision SUCCEEDS but deprovision 502 is a SURVIVOR — a live worker/token is never reported clean', async () => { + const dirsFor = dirs(); + const { fetchImpl } = fakeBackend({ seeded: true, hostedProvision: 200, hostedDeprovision: 502 }); + const { exitCode, result } = await run(fetchImpl, dirsFor, { + base: 'https://api-staging.commonly.me', + env: { COMMONLY_WALK_EMAIL: SHARED_EMAIL }, + }); + + assert.equal(result.mode, 'signin'); + // Provision succeeded, so the hosted steps ran and passed; but the failed stop + // leaks the worker + token, so cleanup is NOT clean and the run fails — the + // staging exemption must never cover a failed stop after a successful provision. + assert.equal(result.steps.find((s) => s.name === 'hosted-install').result, 'pass'); + assert.equal(result.cleanup.ok, false); + assert.ok(result.cleanup.survivors.some((s) => /deprovision 502/.test(s))); + assert.equal(exitCode, 1); + assert.equal(result.overall, 'fail'); +}); + +test('PROD: a not-ready hosted provision (502) FAILS — the staging skip is never applied off staging', async () => { + const dirsFor = dirs(); + const { fetchImpl } = fakeBackend({ seeded: true, hostedProvision: 502 }); + const { exitCode, result } = await run(fetchImpl, dirsFor, { + base: 'https://api.commonly.me', + env: { COMMONLY_WALK_EMAIL: 'walk-test@commonly.test' }, + }); + + assert.equal(result.mode, 'signin-prod'); + assert.equal(exitCode, 1); + assert.equal(result.overall, 'fail'); + assert.equal(result.steps.find((s) => s.name === 'hosted-install').result, 'fail'); +}); + +test('a real provision error (500, no not-ready code) still FAILS — the skip never masks a defect', async () => { + const dirsFor = dirs(); + const { fetchImpl } = fakeBackend({ hostedProvision: 500 }); + const { exitCode, result } = await run(fetchImpl, dirsFor); + + assert.equal(exitCode, 1); + assert.equal(result.overall, 'fail'); + assert.equal(result.steps.find((s) => s.name === 'hosted-install').result, 'fail'); +}); + test('a forced step failure writes overall fail, names the step, and exits 1', async () => { const dirsFor = dirs(); const { fetchImpl } = fakeBackend({ preview: 500 }); diff --git a/workers/agent-runtime/wrangler.toml b/workers/agent-runtime/wrangler.toml index d4fb657030..ea4e56e718 100644 --- a/workers/agent-runtime/wrangler.toml +++ b/workers/agent-runtime/wrangler.toml @@ -20,3 +20,29 @@ new_sqlite_classes = ["AgentRuntimeDO"] COMMONLY_API_URL = "https://api.commonly.me" # Provider decision 2026-08-29 (Sam): DeepSeek. Key is a secret (DEEPSEEK_API_KEY). MODEL_PROVIDER = "deepseek" + +# Staging env (added 2026-11-01): a separate worker serving the staging Commonly +# instance at api-staging.commonly.me. A named Cloudflare env inherits neither +# bindings, vars NOR secrets, so the AGENT binding and vars are repeated here and +# BOTH secrets must be set per-env; the top-level [[migrations]] applies to every env. +# Deploy is operator/CI-side (see the header): `wrangler deploy --env staging`, then +# two `wrangler secret put ... --env staging` piped from staging Secret Manager so +# nothing is typed by hand: +# RUNTIME_ADMIN_TOKEN <- commonly-staging-hosted-runtime-admin-token (gates provision) +# DEEPSEEK_API_KEY <- commonly-staging-deepseek-api-key — a SEPARATE, spend-capped +# staging DeepSeek key, NOT prod's key (same fresh-keys rule as +# the admin token: a staging leak must not spend on prod's key). +# MODEL_PROVIDER=deepseek, so a turn throws "DEEPSEEK_API_KEY +# unset" without it and hosted-reply fails even once provision +# succeeds. The worker calls DeepSeek directly (no base-URL env), +# so staging LiteLLM's virtual key is not an option here. +[env.staging] +name = "commonly-agent-runtime-staging" + +[[env.staging.durable_objects.bindings]] +name = "AGENT" +class_name = "AgentRuntimeDO" + +[env.staging.vars] +COMMONLY_API_URL = "https://api-staging.commonly.me" +MODEL_PROVIDER = "deepseek"