feat(agent-runtime): staging hosted-runtime worker env + walk skips hosted steps until live

k8s/helm/commonly/values-staging.yaml

@@ -204,6 +204,15 @@ backend:
     # and accepting a staging consent would have sent a staging authorization code to prod.
     # Caught on the staging Atlassian walk 2026-10-23.
     backendUrl: "https://api-staging.commonly.me"
+    # ADR-023 W2 hosted runtime, staging worker (added 2026-11-01). Setting this
+    # renders the hosted-runtime ExternalSecret (templates/secrets/hosted-runtime.yaml),
+    # which pulls commonly-staging-hosted-runtime-admin-token from staging SM. With the
+    # URL and token both set the backend is CONFIGURED, so provision no longer returns
+    # 503 unconfigured; until the worker itself is deployed it returns 502
+    # hosted_runtime_unreachable instead. The worker is the separate
+    # `commonly-agent-runtime-staging` Cloudflare env (wrangler.toml); until it is live
+    # the first-user-walk treats both not-ready codes as a skip, not a failure.
+    hostedRuntimeUrl: "https://commonly-agent-runtime-staging.commonlyai.workers.dev"
   # Staging backend runs on the SPOT pool at a fraction of prod's requests, so it fits
   # the nodes we already have (spot-pool sits at single-digit % CPU) without growing a
   # pool. Prod backend stays on dev-pool because it holds session state a spot

scripts/first-user-walk-cleanup.mjs

@@ -58,7 +58,7 @@ export const recoverRunPods = async ({ api, token, userId, podName, podIds }) =>
 // `deleteAccount: false` is the shared staging sign-in account: the run removes
 // only the pods it recorded, never pods that account owns from elsewhere, and
 // never the account itself.
-export const cleanupRun = async ({ api, token, password, email, userId, agentName, podIds = [], onBotIds = () => {}, deleteAccount = true }) => {
+export const cleanupRun = async ({ api, token, password, email, userId, agentName, podIds = [], onBotIds = () => {}, deleteAccount = true, hostedProvisioned = true }) => {
   const survivors = [];
   const botIds = new Set();
 
@@ -69,9 +69,18 @@ export const cleanupRun = async ({ api, token, password, email, userId, agentNam
   };
 
   // Stop the runtime before the seat goes: uninstalling the seat does not stop
-  // the hosted worker. 404 means there was no hosted install to stop.
+  // the hosted worker. 404 means there was no hosted install to stop. A not-ready
+  // deprovision (503 unconfigured / 502 unreachable by code) is exempt from
+  // survivor-tracking ONLY when provisioning did NOT succeed (hostedProvisioned
+  // false — skipped or failed, so nothing was created to leak). When provision
+  // succeeded, the backend revokes the runtime token only after a SUCCESSFUL stop,
+  // so a failed deprovision leaves the worker + token live — a real survivor,
+  // staging or not. Default hostedProvisioned=true keeps the conservative behaviour
+  // for any caller that does not pass the provisioning outcome.
   const stop = await api('POST', '/api/hosted/deprovision', { token, body: { agentName } });
-  if (stop.status >= 300 && stop.status !== 404) survivors.push(`hosted:${agentName} (deprovision ${stop.status})`);
+  const stopNotReady = !hostedProvisioned
+    && (stop.data?.code === 'hosted_runtime_unconfigured' || stop.data?.code === 'hosted_runtime_unreachable');
+  if (stop.status >= 300 && stop.status !== 404 && !stopNotReady) survivors.push(`hosted:${agentName} (deprovision ${stop.status})`);
 
   let owned = [];
   if (deleteAccount) {

scripts/first-user-walk.mjs

@@ -59,6 +59,13 @@ const NOT_COVERED = 'Bring your own connect step is not implemented yet';
 
 const expect = (cond, msg) => { if (!cond) throw new Error(msg); };
 
+// A step can decline to run rather than fail. runStep records a SkipStep as
+// result 'skipped' and does not rethrow, so the run continues and stays
+// incomplete rather than going red. Used when the hosted runtime worker is not
+// ready on this instance — staging before its worker is deployed, where
+// provision returns 503 hosted_runtime_unconfigured or 502 hosted_runtime_unreachable.
+class SkipStep extends Error { constructor(message) { super(message); this.skip = true; } }
+
 /**
  * One walk run. Every dependency is injectable so the production path can be
  * exercised under test without the network or the clock.
@@ -98,6 +105,10 @@ export const runWalk = async ({
   // Null until chooseWalkMode accepts the run, so a refused run is never reported as signup.
   let walkMode = null;
   let cleanupRan = false;
+  // True only once provision returned 200: a worker + runtime token were actually
+  // created, so cleanup must treat a FAILED deprovision as a survivor even on staging.
+  // False when provision was skipped or failed — nothing was created to leak.
+  let hostedProvisioned = false;
   // Signup returns no user id, so the manifest carries the exact username instead:
   // SRE finds the account by that name, not by a prefix. Never the email.
   const manifest = { mode: null, userId: null, username: null, podIds: [], unresolvedPodNames: [], agentName: null, botIds: [], survivors: [], status: 'running' };
@@ -143,13 +154,27 @@ export const runWalk = async ({
       log(`PASS ${name} (${now() - started} ms)`);
       return value;
     } catch (caught) {
+      if (caught instanceof SkipStep) {
+        const reason = redact(caught.message);
+        results.push({ name, result: 'skipped', ms: now() - started, reason });
+        log(`SKIP ${name} (${now() - started} ms): ${reason}`);
+        return undefined;
+      }
       const reason = redact(caught.message);
       results.push({ name, result: 'fail', ms: now() - started, reason });
       error(`FAIL ${name} (${now() - started} ms): ${reason}`);
       throw caught;
     }
   };
 
+  // Record a downstream step as skipped without running it — used when an
+  // earlier skipped step means this one cannot run (hosted-reply needs the
+  // provision that hosted-install skipped).
+  const skipStep = (name, reason) => {
+    results.push({ name, result: 'skipped', ms: 0, reason });
+    log(`SKIP ${name}: ${reason}`);
+  };
+
   // Per-run manifest: ids only, outside the repo, for SRE's admin purge of what
   // the walk's own token cannot remove (Scout, the hosted agent's User). It holds
   // no password, token or email. It is named by the run stamp, not the username.
@@ -198,6 +223,11 @@ export const runWalk = async ({
         agentName,
         podIds: manifest.podIds,
         deleteAccount: walkMode.deleteAccount,
+        // A not-ready deprovision (503/502 by code) is exempt from survivor-tracking
+        // ONLY when provisioning did not succeed — nothing was created to leak. If
+        // provision returned 200, a failed deprovision IS a leak (the backend revokes
+        // the token only after a successful stop), so it stays a survivor, staging or not.
+        hostedProvisioned,
         // Persist each pod's bot ids before that pod is deleted, so a throw
         // later in the loop still leaves the ids SRE needs.
         onBotIds: (podId, bots) => {
@@ -324,6 +354,14 @@ export const runWalk = async ({
         return created;
       });
 
+      // The not-ready skip is scoped to the STAGING run (mode 'signin', the shared
+      // account on api-staging), which is the only place the worker may legitimately
+      // be undeployed during rollout. On prod (signin-prod) and on signup the worker
+      // is expected to be live, so a not-ready provision there is a genuine outage and
+      // must FAIL — never be hidden as a skip. This walk is advisory; staging-smoke,
+      // not it, gates the deploy.
+      const hostedOptional = walkMode.mode === 'signin';
+      let hostedSkipped = false;
       await runStep('hosted-install', async () => {
         manifest.agentName = agentName;
         writeManifest();
@@ -333,10 +371,22 @@ export const runWalk = async ({
         });
         expect(r.status < 400, `install ${r.status}: ${JSON.stringify(r.data)}`);
         const p = await api('POST', '/api/hosted/provision', { token, body: { agentName } });
+        // Not-ready comes back in two shapes: 503 hosted_runtime_unconfigured (URL or
+        // admin token unset) or 502 hosted_runtime_unreachable (configured, but the
+        // worker is not deployed yet — the backend maps every worker fetch/non-ok to
+        // 502, see hostedRuntimeService). Keyed on the error CODE so a 503/502 from a
+        // proxy without that code, or a genuine 500 (token_mint_failed), still fails.
+        const notReady = p.status >= 400
+          && (p.data?.code === 'hosted_runtime_unconfigured' || p.data?.code === 'hosted_runtime_unreachable');
+        if (hostedOptional && notReady) { hostedSkipped = true; throw new SkipStep(`no staging worker (${p.data.code})`); }
         expect(p.status === 200 && p.data?.provisioned, `provision ${p.status}: ${JSON.stringify(p.data)}`);
+        // Provision succeeded: a worker + token now exist, so a failed deprovision in
+        // cleanup is a real leak (survivor), staging or not.
+        hostedProvisioned = true;
       });
 
-      await runStep('hosted-reply', async () => {
+      if (hostedSkipped) skipStep('hosted-reply', 'no staging worker (hosted runtime not ready)');
+      else await runStep('hosted-reply', async () => {
         const m = await api('POST', `/api/messages/${pod._id}`, {
           token,
           body: { content: `@${agentName} walk check: reply with one short sentence containing KINGFISHER.` },

scripts/first-user-walk.test.mjs

@@ -20,8 +20,13 @@ const SHARED_EMAIL = 'walk-shared@commonly.test';
 //   password      the password the fake account accepts
 //   echoRegister  a refused register echoes the request body back, as a bad error would
 //   seeded        the account already exists (the shared staging sign-in account)
+//   hostedProvision  status of POST /api/hosted/provision (and the matching
+//                    deprovision). 503 (unconfigured) and 502 (unreachable) are
+//                    "no worker yet"; on a STAGING run the hosted steps skip, off
+//                    staging they fail. Any other 5xx has no not-ready code -> fail.
 const fakeBackend = ({
   preview = 200, create = 'ok', password = PASSWORD, echoRegister = false, seeded = false,
+  hostedProvision = 200, hostedDeprovision = undefined,
 } = {}) => {
   const state = { accountExists: seeded, pods: [], lastAgent: null, calls: [], loginEmails: [] };
   const reply = (status, data) => ({ status, json: async () => data });
@@ -69,8 +74,25 @@ const fakeBackend = ({
       }
     }
     if (method === 'POST' && p === '/api/registry/install') return reply(200, {});
-    if (method === 'POST' && p === '/api/hosted/provision') return reply(200, { provisioned: true });
-    if (method === 'POST' && p === '/api/hosted/deprovision') return reply(404, {});
+    if (method === 'POST' && p === '/api/hosted/provision') {
+      // The backend's real shapes: 200 provisioned; 503 hosted_runtime_unconfigured
+      // (URL/token unset); 502 hosted_runtime_unreachable (worker not deployed). Any
+      // other 5xx is a genuine error with NO not-ready code, which must still fail.
+      if (hostedProvision === 200) return reply(200, { provisioned: true });
+      if (hostedProvision === 503) return reply(503, { code: 'hosted_runtime_unconfigured', message: 'not ready' });
+      if (hostedProvision === 502) return reply(502, { code: 'hosted_runtime_unreachable', message: 'not ready' });
+      return reply(hostedProvision, { message: 'Hosted runtime request failed' });
+    }
+    if (method === 'POST' && p === '/api/hosted/deprovision') {
+      // Default: mirror provision (unconfigured/unreachable backend answers both the
+      // same). hostedDeprovision overrides it to model a SUCCESSFUL provision whose
+      // deprovision then fails (worker + token leak), which must be a survivor.
+      const d = hostedDeprovision ?? (hostedProvision === 200 ? 404 : hostedProvision);
+      if (d === 404) return reply(404, {});
+      if (d === 503) return reply(503, { code: 'hosted_runtime_unconfigured' });
+      if (d === 502) return reply(502, { code: 'hosted_runtime_unreachable' });
+      return reply(d, {});
+    }
     if (method === 'DELETE' && p.startsWith('/api/registry/agents/')) return reply(404, {});
     if (method === 'POST' && /^\/api\/messages\//.test(p)) {
       state.lastAgent = (body.content.match(/@(\S+) walk check/) || [])[1] || null;
@@ -135,6 +157,78 @@ test('a clean run writes the machine-readable signal: every covered step passes,
   assert.equal(state.pods.length, 0);
 });
 
+// STAGING run only: both not-ready shapes skip — 503 (URL/token unset) and 502
+// (configured but the worker is not deployed, the state values-staging.yaml creates
+// before the operator runs `wrangler deploy --env staging`). The run must stay
+// incomplete, and the matching not-ready deprovision must NOT redden cleanup.
+for (const status of [503, 502]) {
+  test(`staging: no hosted worker (provision ${status}) skips the hosted steps, keeps cleanup clean, overall incomplete`, async () => {
+    const dirsFor = dirs();
+    const { fetchImpl, state } = fakeBackend({ seeded: true, hostedProvision: status });
+    const { exitCode, result } = await run(fetchImpl, dirsFor, {
+      base: 'https://api-staging.commonly.me',
+      env: { COMMONLY_WALK_EMAIL: SHARED_EMAIL },
+    });
+
+    assert.equal(result.mode, 'signin');
+    assert.equal(exitCode, 2);
+    assert.equal(result.overall, 'incomplete');
+
+    const byName = Object.fromEntries(result.steps.map((s) => [s.name, s.result]));
+    assert.equal(byName['hosted-install'], 'skipped');
+    assert.equal(byName['hosted-reply'], 'skipped');
+    assert.match(result.steps.find((s) => s.name === 'hosted-install').reason, /no staging worker/);
+    assert.ok(result.steps.every((s) => s.result !== 'fail'));
+    // The not-ready deprovision did NOT become a survivor; cleanup stays clean,
+    // and the shared staging account is preserved.
+    assert.equal(result.cleanup.ok, true);
+    assert.equal(state.accountExists, true);
+  });
+}
+
+test('staging: provision SUCCEEDS but deprovision 502 is a SURVIVOR — a live worker/token is never reported clean', async () => {
+  const dirsFor = dirs();
+  const { fetchImpl } = fakeBackend({ seeded: true, hostedProvision: 200, hostedDeprovision: 502 });
+  const { exitCode, result } = await run(fetchImpl, dirsFor, {
+    base: 'https://api-staging.commonly.me',
+    env: { COMMONLY_WALK_EMAIL: SHARED_EMAIL },
+  });
+
+  assert.equal(result.mode, 'signin');
+  // Provision succeeded, so the hosted steps ran and passed; but the failed stop
+  // leaks the worker + token, so cleanup is NOT clean and the run fails — the
+  // staging exemption must never cover a failed stop after a successful provision.
+  assert.equal(result.steps.find((s) => s.name === 'hosted-install').result, 'pass');
+  assert.equal(result.cleanup.ok, false);
+  assert.ok(result.cleanup.survivors.some((s) => /deprovision 502/.test(s)));
+  assert.equal(exitCode, 1);
+  assert.equal(result.overall, 'fail');
+});
+
+test('PROD: a not-ready hosted provision (502) FAILS — the staging skip is never applied off staging', async () => {
+  const dirsFor = dirs();
+  const { fetchImpl } = fakeBackend({ seeded: true, hostedProvision: 502 });
+  const { exitCode, result } = await run(fetchImpl, dirsFor, {
+    base: 'https://api.commonly.me',
+    env: { COMMONLY_WALK_EMAIL: 'walk-test@commonly.test' },
+  });
+
+  assert.equal(result.mode, 'signin-prod');
+  assert.equal(exitCode, 1);
+  assert.equal(result.overall, 'fail');
+  assert.equal(result.steps.find((s) => s.name === 'hosted-install').result, 'fail');
+});
+
+test('a real provision error (500, no not-ready code) still FAILS — the skip never masks a defect', async () => {
+  const dirsFor = dirs();
+  const { fetchImpl } = fakeBackend({ hostedProvision: 500 });
+  const { exitCode, result } = await run(fetchImpl, dirsFor);
+
+  assert.equal(exitCode, 1);
+  assert.equal(result.overall, 'fail');
+  assert.equal(result.steps.find((s) => s.name === 'hosted-install').result, 'fail');
+});
+
 test('a forced step failure writes overall fail, names the step, and exits 1', async () => {
   const dirsFor = dirs();
   const { fetchImpl } = fakeBackend({ preview: 500 });

workers/agent-runtime/wrangler.toml

@@ -20,3 +20,29 @@ new_sqlite_classes = ["AgentRuntimeDO"]
 COMMONLY_API_URL = "https://api.commonly.me"
 # Provider decision 2026-08-29 (Sam): DeepSeek. Key is a secret (DEEPSEEK_API_KEY).
 MODEL_PROVIDER = "deepseek"
+
+# Staging env (added 2026-11-01): a separate worker serving the staging Commonly
+# instance at api-staging.commonly.me. A named Cloudflare env inherits neither
+# bindings, vars NOR secrets, so the AGENT binding and vars are repeated here and
+# BOTH secrets must be set per-env; the top-level [[migrations]] applies to every env.
+# Deploy is operator/CI-side (see the header): `wrangler deploy --env staging`, then
+# two `wrangler secret put ... --env staging` piped from staging Secret Manager so
+# nothing is typed by hand:
+#   RUNTIME_ADMIN_TOKEN  <- commonly-staging-hosted-runtime-admin-token (gates provision)
+#   DEEPSEEK_API_KEY     <- commonly-staging-deepseek-api-key — a SEPARATE, spend-capped
+#                           staging DeepSeek key, NOT prod's key (same fresh-keys rule as
+#                           the admin token: a staging leak must not spend on prod's key).
+#                           MODEL_PROVIDER=deepseek, so a turn throws "DEEPSEEK_API_KEY
+#                           unset" without it and hosted-reply fails even once provision
+#                           succeeds. The worker calls DeepSeek directly (no base-URL env),
+#                           so staging LiteLLM's virtual key is not an option here.
+[env.staging]
+name = "commonly-agent-runtime-staging"
+
+[[env.staging.durable_objects.bindings]]
+name = "AGENT"
+class_name = "AgentRuntimeDO"
+
+[env.staging.vars]
+COMMONLY_API_URL = "https://api-staging.commonly.me"
+MODEL_PROVIDER = "deepseek"