From 86566fb828523ea6294d27caa8ca623d2d906335 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 1 Nov 2026 01:06:20 -0700
Subject: [PATCH 1/6] ci(deploy): auto-roll-back a failed deploy + staging
 rehearsal input (D227)

Part of the D227 fast lane: the heavy test matrix is moving off the merge path,
so a bad image can reach a deploy and the post-deploy verify becomes the safety
net. Add an `if: failure()` step to deploy-dev and deploy-staging that rolls the
release back to the previous helm revision (CUR -> CUR-1) so the environment
self-heals; the job still ends red. On prod that red makes the auto-deploy
dispatcher write auto-deploy.hold and stop re-dispatching.

Adds a `force_bad_tag` workflow_dispatch input to deploy-staging that deploys a
nonexistent backend image tag, so the rollout fails and the rollback step can be
rehearsed on staging before this gates prod.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA
---
 .github/workflows/deploy-dev.yml     | 21 +++++++++++++++++
 .github/workflows/deploy-staging.yml | 34 +++++++++++++++++++++++++++-
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index b3bb1a813c..e59c921670 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -393,6 +393,27 @@ jobs:
           echo "::error title=Chat unavailable on the new backend::$POD has pg/status=$STATUS and pg/messages=$ROUTE (401 means mounted, 404 unmounted). Chat history fails until the pod restarts: kubectl rollout restart deploy/backend -n $NAMESPACE"
           exit 1
 
+      # D227 (Sam via Cofounder, 2026-11-01): the heavy test matrix is leaving the merge
+      # path, so a bad image can reach prod and this post-deploy verify is the net. If any
+      # deploy/verify step above failed, undo this release in-cluster by rolling back to the
+      # previous helm revision, so prod self-heals without waiting for a human. The job still
+      # ends red, which makes the auto-deploy dispatcher write auto-deploy.hold and stop
+      # re-dispatching (Mac-side). Rolls back revision CUR -> CUR-1; a no-op at revision 1 or
+      # if the cluster is unreachable (helm history empty).
+      - name: Roll back on a failed deploy
+        if: failure()
+        run: |
+          set -uo pipefail
+          CUR=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null | jq -r '.[-1].revision // 0')
+          PREV=$(( CUR - 1 ))
+          if [ "$PREV" -lt 1 ]; then
+            echo "::warning title=No rollback target::commonly-dev is at revision ${CUR:-0}; nothing to roll back to."
+            exit 0
+          fi
+          echo "Rolling commonly-dev back: revision $CUR -> $PREV"
+          helm rollback commonly-dev "$PREV" -n "$NAMESPACE" --wait --timeout 8m
+          echo "::error title=Deploy rolled back::commonly-dev rolled back to revision $PREV after a failed deploy. Investigate the bad image before the next deploy."
+
       # The stranger journey walks signup -> Scout's welcome -> a real question
       # -> the connect card against the release just deployed
       # (docs/runbooks/alerting.md#stranger-journey). It runs as a Job from the
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 3142d301c2..a54856a0e5 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -25,6 +25,11 @@ on:
         description: 'Git ref / SHA to build + deploy to staging (defaults to the triggering ref)'
         required: false
         default: ''
+      force_bad_tag:
+        description: 'REHEARSAL: deploy a nonexistent backend image tag so the rollout fails and the rollback step fires (staging only)'
+        type: boolean
+        required: false
+        default: false
   push:
     branches: [ main ]
 
@@ -157,12 +162,21 @@ jobs:
       - name: Helm upgrade commonly-staging
         env:
           TAG: ${{ steps.tag.outputs.tag }}
+          FORCE_BAD: ${{ inputs.force_bad_tag }}
         run: |
+          # Rollback rehearsal (force_bad_tag): point the backend image at a tag that does not
+          # exist, so the pod never pulls, the rollout verify below fails, and the rollback step
+          # at the end of the job fires. Staging only; a normal run leaves BTAG = the real tag.
+          BTAG="$TAG"
+          if [ "$FORCE_BAD" = "true" ]; then
+            BTAG="rollback-rehearsal-nonexistent"
+            echo "::warning title=Rollback rehearsal::forcing a nonexistent backend image tag ($BTAG) to exercise the rollback step"
+          fi
           helm upgrade --install "$RELEASE" k8s/helm/commonly -n "$NAMESPACE" --create-namespace \
             -f k8s/helm/commonly/values.yaml \
             -f k8s/helm/commonly/values-staging.yaml \
             -f "$RUNNER_TEMP/values-private.yaml" \
-            --set backend.image.tag="$TAG" \
+            --set backend.image.tag="$BTAG" \
             --set frontend.image.tag="$TAG-staging" \
             --set agents.commonlyBot.image.tag="$TAG" \
             --timeout 10m \
@@ -300,3 +314,21 @@ jobs:
             -f context="staging-smoke" \
             -f description="$DESC" >/dev/null
           [ "$STATE" = "success" ]
+
+      # D227 (Sam via Cofounder, 2026-11-01): mirror of the prod rollback net. If any step
+      # above failed (rollout or smoke), roll this staging release back to the previous helm
+      # revision so staging self-heals. The `force_bad_tag` rehearsal input drives a failed
+      # rollout specifically to exercise this path. The job still ends red.
+      - name: Roll back on a failed deploy
+        if: failure()
+        run: |
+          set -uo pipefail
+          CUR=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null | jq -r '.[-1].revision // 0')
+          PREV=$(( CUR - 1 ))
+          if [ "$PREV" -lt 1 ]; then
+            echo "::warning title=No rollback target::$RELEASE is at revision ${CUR:-0}; nothing to roll back to."
+            exit 0
+          fi
+          echo "Rolling $RELEASE back: revision $CUR -> $PREV"
+          helm rollback "$RELEASE" "$PREV" -n "$NAMESPACE" --wait --timeout 8m
+          echo "::error title=Deploy rolled back::$RELEASE rolled back to revision $PREV after a failed deploy. Investigate before the next deploy."

From cf4764a8fd9d35522bc2fb6715f3f31371140553 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 1 Nov 2026 01:12:49 -0700
Subject: [PATCH 2/6] ci(deploy): roll back to last successfully-deployed
 revision, not blindly CUR-1 (D227)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Target the newest revision below current that Helm records as superseded/deployed,
skipping failed/pending revisions a retry may leave — a blind CUR-1 could land on
another bad image (Cofounder review). Also guard `helm history` with `|| echo '[]'`
so the step's safe "no rollback target -> exit 0" path is reached under `bash -e`
when the failure happened before GKE creds (nothing was deployed to roll back).

Target-selection logic unit-tested locally against synthetic histories (normal,
failed-CUR-1, multiple-failed, pending-in-middle, no-prior-good) — all pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA
---
 .github/workflows/deploy-dev.yml     | 18 +++++++++++-------
 .github/workflows/deploy-staging.yml | 18 +++++++++++-------
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index e59c921670..131bac34bd 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -404,15 +404,19 @@ jobs:
         if: failure()
         run: |
           set -uo pipefail
-          CUR=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null | jq -r '.[-1].revision // 0')
-          PREV=$(( CUR - 1 ))
-          if [ "$PREV" -lt 1 ]; then
-            echo "::warning title=No rollback target::commonly-dev is at revision ${CUR:-0}; nothing to roll back to."
+          HIST=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
+          CUR=$(printf '%s' "$HIST" | jq -r '.[-1].revision // 0')
+          # Target the newest revision BELOW current that Helm recorded as a real deploy
+          # (status superseded or deployed), skipping failed/pending revisions a retry may
+          # have left — a blind CUR-1 could land on another bad image (Cofounder, 2026-11-01).
+          TARGET=$(printf '%s' "$HIST" | jq -r --argjson cur "${CUR:-0}" '[.[] | select((.revision < $cur) and (.status=="superseded" or .status=="deployed"))] | (sort_by(.revision) | last | .revision) // empty')
+          if [ -z "$TARGET" ]; then
+            echo "::warning title=No rollback target::commonly-dev has no prior superseded/deployed revision below ${CUR:-0}; nothing to roll back to."
             exit 0
           fi
-          echo "Rolling commonly-dev back: revision $CUR -> $PREV"
-          helm rollback commonly-dev "$PREV" -n "$NAMESPACE" --wait --timeout 8m
-          echo "::error title=Deploy rolled back::commonly-dev rolled back to revision $PREV after a failed deploy. Investigate the bad image before the next deploy."
+          echo "Rolling commonly-dev back: revision $CUR -> $TARGET (last successfully-deployed)"
+          helm rollback commonly-dev "$TARGET" -n "$NAMESPACE" --wait --timeout 8m
+          echo "::error title=Deploy rolled back::commonly-dev rolled back to revision $TARGET after a failed deploy. Investigate the bad image before the next deploy."
 
       # The stranger journey walks signup -> Scout's welcome -> a real question
       # -> the connect card against the release just deployed
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index a54856a0e5..8b976f4602 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -323,12 +323,16 @@ jobs:
         if: failure()
         run: |
           set -uo pipefail
-          CUR=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null | jq -r '.[-1].revision // 0')
-          PREV=$(( CUR - 1 ))
-          if [ "$PREV" -lt 1 ]; then
-            echo "::warning title=No rollback target::$RELEASE is at revision ${CUR:-0}; nothing to roll back to."
+          HIST=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
+          CUR=$(printf '%s' "$HIST" | jq -r '.[-1].revision // 0')
+          # Target the newest revision BELOW current that Helm recorded as a real deploy
+          # (status superseded or deployed), skipping failed/pending revisions a retry may
+          # have left — a blind CUR-1 could land on another bad image (Cofounder, 2026-11-01).
+          TARGET=$(printf '%s' "$HIST" | jq -r --argjson cur "${CUR:-0}" '[.[] | select((.revision < $cur) and (.status=="superseded" or .status=="deployed"))] | (sort_by(.revision) | last | .revision) // empty')
+          if [ -z "$TARGET" ]; then
+            echo "::warning title=No rollback target::$RELEASE has no prior superseded/deployed revision below ${CUR:-0}; nothing to roll back to."
             exit 0
           fi
-          echo "Rolling $RELEASE back: revision $CUR -> $PREV"
-          helm rollback "$RELEASE" "$PREV" -n "$NAMESPACE" --wait --timeout 8m
-          echo "::error title=Deploy rolled back::$RELEASE rolled back to revision $PREV after a failed deploy. Investigate before the next deploy."
+          echo "Rolling $RELEASE back: revision $CUR -> $TARGET (last successfully-deployed)"
+          helm rollback "$RELEASE" "$TARGET" -n "$NAMESPACE" --wait --timeout 8m
+          echo "::error title=Deploy rolled back::$RELEASE rolled back to revision $TARGET after a failed deploy. Investigate before the next deploy."

From 33e62f84b78f9bc506edb90f49189edf97b21e3d Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 1 Nov 2026 01:20:03 -0700
Subject: [PATCH 3/6] ci(deploy): gate rollback on deploy/verify step outcomes,
 not bare failure() (D227 review)

From the adversarial review of this PR:
- MEDIUM: staging's bare `if: failure()` would roll back a healthy release on a
  post-deploy infra flake (setup-node / npm / playwright-install, which run after a
  successful helm upgrade). Gate staging rollback on helm-upgrade OR rollout-verify failing.
- LOW: make the prod rollback robust against future step inserts by gating on helm/rollout/
  chat step outcomes rather than relying on step ordering + the helm-history no-op. A
  pre-deploy failure (checkout/auth/build) now cleanly SKIPS the rollback.
- LOW(doc): correct the "CUR -> CUR-1" comment (code targets the newest prior good revision).

Adds id: helm / id: chat (deploy-dev) and id: helm / id: rollout (deploy-staging); deploy-dev
rollout already had id: rollout. force_bad_tag rehearsal still fires via rollout-verify failure.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA
---
 .github/workflows/deploy-dev.yml     | 10 +++++++---
 .github/workflows/deploy-staging.yml | 13 ++++++++-----
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index 131bac34bd..8d1aad0abf 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -282,6 +282,7 @@ jobs:
       # incident silently reverts every deploy since, which by then included
       # two auth fixes. Rollback safety depends on this job telling the truth.
       - name: Helm upgrade commonly-dev
+        id: helm
         env:
           TAG: ${{ steps.tag.outputs.tag }}
           GATEWAY_ENABLED: ${{ steps.gateway.outputs.enabled }}
@@ -357,6 +358,7 @@ jobs:
       # (TASK-168), ask the newest backend pod from the inside. Unauthenticated,
       # a mounted /api/pg/messages answers 401 and an unmounted one 404.
       - name: Verify the new backend serves chat
+        id: chat
         # Deliberately not `always()` / `!cancelled()`: if an earlier step failed
         # (image push, helm upgrade, GKE credentials) the cluster is unreachable
         # or unchanged, and this step would then report "Chat unavailable on the
@@ -398,10 +400,12 @@ jobs:
       # deploy/verify step above failed, undo this release in-cluster by rolling back to the
       # previous helm revision, so prod self-heals without waiting for a human. The job still
       # ends red, which makes the auto-deploy dispatcher write auto-deploy.hold and stop
-      # re-dispatching (Mac-side). Rolls back revision CUR -> CUR-1; a no-op at revision 1 or
-      # if the cluster is unreachable (helm history empty).
+      # re-dispatching (Mac-side). Rolls back to the newest prior good revision; a no-op at
+      # revision 1 or if the cluster is unreachable (helm history empty). Gated on the helm
+      # upgrade or a verify step failing (not bare failure()), so a pre-deploy failure (checkout,
+      # auth, build/push) never demotes a healthy release.
       - name: Roll back on a failed deploy
-        if: failure()
+        if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure' || steps.chat.outcome == 'failure') }}
         run: |
           set -uo pipefail
           HIST=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 8b976f4602..b16b474d81 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -160,6 +160,7 @@ jobs:
       # Namespace and collide with the one helm creates here. Tracked against
       # the staging chart PR.
       - name: Helm upgrade commonly-staging
+        id: helm
         env:
           TAG: ${{ steps.tag.outputs.tag }}
           FORCE_BAD: ${{ inputs.force_bad_tag }}
@@ -184,6 +185,7 @@ jobs:
             --qps 100
 
       - name: Verify rollout of the deployed workloads
+        id: rollout
         run: |
           set -uo pipefail
           FAILED=""
@@ -315,12 +317,13 @@ jobs:
             -f description="$DESC" >/dev/null
           [ "$STATE" = "success" ]
 
-      # D227 (Sam via Cofounder, 2026-11-01): mirror of the prod rollback net. If any step
-      # above failed (rollout or smoke), roll this staging release back to the previous helm
-      # revision so staging self-heals. The `force_bad_tag` rehearsal input drives a failed
-      # rollout specifically to exercise this path. The job still ends red.
+      # D227 (Sam via Cofounder, 2026-11-01): mirror of the prod rollback net. Roll this staging
+      # release back to the newest prior good revision when the helm upgrade or the rollout verify
+      # fails — NOT on a post-deploy infra flake (setup-node / npm / playwright-install), which
+      # would otherwise self-heal a healthy release. The `force_bad_tag` rehearsal input drives a
+      # failed rollout to exercise this path. The job still ends red.
       - name: Roll back on a failed deploy
-        if: failure()
+        if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure') }}
         run: |
           set -uo pipefail
           HIST=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')

From f32736044baf2bba35e18c512485919953cf9619 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 1 Nov 2026 01:38:06 -0700
Subject: [PATCH 4/6] =?UTF-8?q?ci(deploy):=20harden=20auto-rollback=20per?=
 =?UTF-8?q?=20CodeRabbit=20=E2=80=94=20recorded=20good=20revision,=20scope?=
 =?UTF-8?q?d=20workloads,=20no=20--wait,=20rehearsal-safe?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Addresses the verified CodeRabbit findings on the rollback nets in deploy-dev.yml
and deploy-staging.yml:

- Record the pre-upgrade `deployed` revision (steps.prerev) and roll back to THAT,
  not to revision arithmetic. Skip rollback when the upgrade created no new revision
  (CUR==PRE: a chart/values preflight failure leaves the live release unchanged), so
  a healthy deploy is never demoted one step too far. Replaces the CUR-1/superseded
  heuristic. (findings: select-verified-target, no-revision-created, roll-back-this-run)

- Scope the rollout verify AND the post-rollback verify to the workloads this job
  image-tags (backend/+worker, frontend, clawdbot-gateway, commonly-bot), derived from
  the cluster and intersected with that set. An unrelated unhealthy workload (litellm,
  redis, cloud-codex, ops-runner, cloudflared) no longer fails the deploy or trips the
  rollback; a parked workload still drops out. The code now matches the step's own
  stated intent. (finding: exclude-unrelated-rollout-failures)

- Drop `--wait` from `helm rollback` (the upgrade deliberately avoids it): a release-wide
  wait can hang on an unrelated workload and record the rollback itself as failed, leaving
  the bad revision marked deployed. Apply, then verify only the managed workloads.
  (finding: do-not-wait-on-unrelated-resources)

- Rehearsal safety: force_bad_tag no longer posts a production-gating staging-smoke=failure
  for the checked-out SHA (which would make deploy-dev reject the SHA until a real staging
  run succeeds). The rollout failure already drives the rollback step. (finding: rehearsal-status)

- Bump both jobs' timeout-minutes 30 -> 45 so an 8m rollout wait plus an 8m rollback has
  headroom and GitHub cannot cancel the job mid-rollback. (finding: reserve-time-for-rollback)

actionlint clean (only pre-existing info-level shellcheck notes on untouched lines).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA
---
 .github/workflows/deploy-dev.yml     | 87 ++++++++++++++++++++++------
 .github/workflows/deploy-staging.yml | 81 +++++++++++++++++++++-----
 2 files changed, 133 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index 8d1aad0abf..a5fd56391c 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -111,7 +111,11 @@ jobs:
     needs: require_staging_smoke
     runs-on: ubuntu-latest
     environment: dev
-    timeout-minutes: 30
+    # 45, not 30: a bad image makes `kubectl rollout status` wait its full 8m per
+    # stuck workload before the rollout step fails, and the rollback step then needs
+    # its own window. 30m could see GitHub cancel the job mid-rollback (cancellation
+    # does not satisfy failure(), so the rollback would not re-run). 45m reserves room.
+    timeout-minutes: 45
 
     steps:
       - name: Checkout (with submodules for _external/clawdbot)
@@ -261,6 +265,20 @@ jobs:
           cluster_name: ${{ env.CLUSTER }}
           location: ${{ env.REGION }}
 
+      # Record the revision Helm currently has as `deployed` — the last known-good
+      # release — BEFORE the upgrade, so the rollback step can return to exactly this
+      # revision instead of inferring one by arithmetic after a failed upgrade (a
+      # failed retry can leave a bad image at CUR-1). 0 == no prior deploy (first
+      # install): nothing to roll back to.
+      - name: Record the current good revision (for rollback)
+        id: prerev
+        run: |
+          set -uo pipefail
+          HIST=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
+          PRE=$(printf '%s' "$HIST" | jq -r '[.[] | select(.status=="deployed")] | (sort_by(.revision) | last | .revision) // 0')
+          echo "prerev=$PRE" >> "$GITHUB_OUTPUT"
+          echo "Pre-upgrade good (deployed) revision: $PRE"
+
       # `--wait` is deliberately NOT used here. It waits for EVERY resource in
       # the release, so any long-unhealthy auxiliary workload fails a deploy
       # that actually succeeded. On 2026-08-04 that produced five consecutive
@@ -328,13 +346,21 @@ jobs:
         id: rollout
         run: |
           set -uo pipefail
+          # Derived from the cluster AND intersected with the workloads this job sets
+          # image tags for — backend (+ its worker), frontend, clawdbot-gateway,
+          # commonly-bot. A parked workload (enabled:false removes the Deployment, as
+          # #1051 did to the gateway) simply isn't present and drops out; an unrelated
+          # unhealthy workload in the same namespace (litellm, redis, cloud-codex,
+          # ops-runner, cloudflared) is a real problem but NOT this job's failure, and
+          # must not fail the deploy or trip the rollback below. A scaled-to-zero managed
+          # workload passes instantly (0 of 0 updated) — parked is not stuck.
+          MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
           FAILED=""
-          DEPLOYS=$(kubectl get deploy -n "$NAMESPACE" -o name)
-          if [ -z "$DEPLOYS" ]; then
-            echo "::error title=Rollout failed::no deployments found in $NAMESPACE — nothing to verify is itself a failure"
-            exit 1
-          fi
-          for d in $DEPLOYS; do
+          CHECKED=""
+          for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
+            n=${d##*/}
+            case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac
+            CHECKED="$CHECKED $n"
             echo "::group::rollout status $d"
             if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then
               echo "$d: rolled out"
@@ -344,11 +370,15 @@ jobs:
             fi
             echo "::endgroup::"
           done
+          if [ -z "$CHECKED" ]; then
+            echo "::error title=Rollout failed::none of the image-tagged workloads (backend/frontend/clawdbot-gateway/commonly-bot) found in $NAMESPACE — nothing to verify is itself a failure"
+            exit 1
+          fi
           if [ -n "$FAILED" ]; then
             echo "Deploy failed for:$FAILED"
             exit 1
           fi
-          echo "All four deployed workloads are live."
+          echo "Deployed workloads live:$CHECKED"
 
       # A pod can be Ready and still unable to serve chat. The backend decides
       # once at boot whether Postgres is available, and /ready answers 200 in
@@ -400,27 +430,46 @@ jobs:
       # deploy/verify step above failed, undo this release in-cluster by rolling back to the
       # previous helm revision, so prod self-heals without waiting for a human. The job still
       # ends red, which makes the auto-deploy dispatcher write auto-deploy.hold and stop
-      # re-dispatching (Mac-side). Rolls back to the newest prior good revision; a no-op at
-      # revision 1 or if the cluster is unreachable (helm history empty). Gated on the helm
+      # re-dispatching (Mac-side). Rolls back to the revision recorded as good before this
+      # upgrade (steps.prerev); a no-op at first install, if the cluster is unreachable, or
+      # if the upgrade created no new revision (live release unchanged). Gated on the helm
       # upgrade or a verify step failing (not bare failure()), so a pre-deploy failure (checkout,
       # auth, build/push) never demotes a healthy release.
       - name: Roll back on a failed deploy
         if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure' || steps.chat.outcome == 'failure') }}
+        env:
+          PRE: ${{ steps.prerev.outputs.prerev }}
         run: |
           set -uo pipefail
           HIST=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
           CUR=$(printf '%s' "$HIST" | jq -r '.[-1].revision // 0')
-          # Target the newest revision BELOW current that Helm recorded as a real deploy
-          # (status superseded or deployed), skipping failed/pending revisions a retry may
-          # have left — a blind CUR-1 could land on another bad image (Cofounder, 2026-11-01).
-          TARGET=$(printf '%s' "$HIST" | jq -r --argjson cur "${CUR:-0}" '[.[] | select((.revision < $cur) and (.status=="superseded" or .status=="deployed"))] | (sort_by(.revision) | last | .revision) // empty')
-          if [ -z "$TARGET" ]; then
-            echo "::warning title=No rollback target::commonly-dev has no prior superseded/deployed revision below ${CUR:-0}; nothing to roll back to."
+          # Roll back only to the revision recorded as `deployed` BEFORE this upgrade
+          # (steps.prerev). Two no-ops, each a release we must NOT demote:
+          #   - no recorded good revision (first install) -> nothing to roll back to;
+          #   - CUR == PRE -> the upgrade failed before Helm created a revision (chart
+          #     load / values parse): the live release is unchanged, so leave it.
+          if [ "${CUR:-0}" = "0" ] || [ "${PRE:-0}" = "0" ] || [ -z "${PRE:-}" ]; then
+            echo "::warning title=No rollback target::commonly-dev has no pre-upgrade good revision (CUR=${CUR:-0} PRE=${PRE:-0}); nothing to roll back to."
             exit 0
           fi
-          echo "Rolling commonly-dev back: revision $CUR -> $TARGET (last successfully-deployed)"
-          helm rollback commonly-dev "$TARGET" -n "$NAMESPACE" --wait --timeout 8m
-          echo "::error title=Deploy rolled back::commonly-dev rolled back to revision $TARGET after a failed deploy. Investigate the bad image before the next deploy."
+          if [ "$CUR" = "$PRE" ]; then
+            echo "::warning title=No rollback needed::the upgrade created no new revision (CUR==PRE==$PRE); the live release is unchanged."
+            exit 0
+          fi
+          echo "Rolling commonly-dev back: revision $CUR -> $PRE (last successfully-deployed)"
+          # No `--wait`: like the upgrade above, a release-wide wait can be blocked by an
+          # unrelated unhealthy workload and would then record the rollback itself as
+          # failed, leaving the bad revision marked deployed. Apply, then verify only the
+          # workloads this job manages.
+          helm rollback commonly-dev "$PRE" -n "$NAMESPACE" --timeout 8m
+          MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
+          for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
+            n=${d##*/}
+            case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac
+            kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m \
+              || echo "::warning title=Rollback not yet ready::$d did not report ready within 8m after rollback to $PRE; check the cluster."
+          done
+          echo "::error title=Deploy rolled back::commonly-dev rolled back to revision $PRE after a failed deploy. Investigate the bad image before the next deploy."
 
       # The stranger journey walks signup -> Scout's welcome -> a real question
       # -> the connect card against the release just deployed
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index b16b474d81..d979d093f7 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -54,7 +54,10 @@ jobs:
     name: Build + push + helm upgrade (staging)
     runs-on: ubuntu-latest
     environment: staging
-    timeout-minutes: 30
+    # 45, not 30: rollout verification can wait its full 8m per stuck workload and the
+    # rollback step needs its own window after; 30m risks GitHub cancelling mid-rollback
+    # (cancellation does not satisfy failure(), so rollback would not re-run).
+    timeout-minutes: 45
 
     steps:
       - name: Checkout
@@ -154,6 +157,18 @@ jobs:
           cluster_name: ${{ env.CLUSTER }}
           location: ${{ env.REGION }}
 
+      # Record the revision Helm currently has as `deployed` (last known-good) BEFORE the
+      # upgrade, so the rollback step returns to exactly this revision instead of inferring
+      # one by arithmetic. 0 == no prior deploy (first install).
+      - name: Record the current good revision (for rollback)
+        id: prerev
+        run: |
+          set -uo pipefail
+          HIST=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
+          PRE=$(printf '%s' "$HIST" | jq -r '[.[] | select(.status=="deployed")] | (sort_by(.revision) | last | .revision) // 0')
+          echo "prerev=$PRE" >> "$GITHUB_OUTPUT"
+          echo "Pre-upgrade good (deployed) revision: $PRE"
+
       # --create-namespace: commonly-staging does not exist before the first
       # install. NOTE: values-staging.yaml should set createNamespace:false
       # (as values-local.yaml does) so the chart does not ALSO declare the
@@ -188,13 +203,17 @@ jobs:
         id: rollout
         run: |
           set -uo pipefail
+          # Scoped to the workloads this job image-tags (backend/+worker, frontend,
+          # commonly-bot — clawdbot-gateway when the staging chart ships it), derived from
+          # the cluster. An unrelated unhealthy workload in the namespace is not this job's
+          # failure and must not trip the rollback below; a parked workload drops out.
+          MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
           FAILED=""
-          DEPLOYS=$(kubectl get deploy -n "$NAMESPACE" -o name)
-          if [ -z "$DEPLOYS" ]; then
-            echo "::error title=Rollout failed::no deployments in $NAMESPACE"
-            exit 1
-          fi
-          for d in $DEPLOYS; do
+          CHECKED=""
+          for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
+            n=${d##*/}
+            case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac
+            CHECKED="$CHECKED $n"
             echo "::group::rollout status $d"
             if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then
               echo "$d: rolled out"
@@ -204,7 +223,12 @@ jobs:
             fi
             echo "::endgroup::"
           done
+          if [ -z "$CHECKED" ]; then
+            echo "::error title=Rollout failed::none of the image-tagged workloads found in $NAMESPACE"
+            exit 1
+          fi
           if [ -n "$FAILED" ]; then echo "Deploy failed for:$FAILED"; exit 1; fi
+          echo "Deployed workloads live:$CHECKED"
 
       - name: Set up Node for the Tier-1 smoke
         uses: actions/setup-node@v4
@@ -254,8 +278,18 @@ jobs:
           SHA: ${{ steps.tag.outputs.sha }}
           JOB_STATUS: ${{ job.status }}
           GH_TOKEN: ${{ github.token }}
+          FORCE_BAD: ${{ inputs.force_bad_tag }}
         run: |
           set -uo pipefail
+          # Rollback rehearsal (force_bad_tag): the rollout was failed deliberately, so this
+          # SHA's backend image was never actually tested. Posting staging-smoke=failure here
+          # would make deploy-dev.yml reject the SHA until a real staging run succeeds — a
+          # production-gating status from a drill. Post nothing; the rollout failure above
+          # already drives the rollback step (its gate reads steps.rollout.outcome).
+          if [ "${FORCE_BAD:-}" = "true" ]; then
+            echo "rollback rehearsal: not posting a production-gating staging-smoke status for $SHA ($TAG)"
+            exit 0
+          fi
           # A cancelled job deployed nothing, so it has nothing truthful to say about
           # this SHA. The step condition already skips a cancelled RUN; this covers a
           # job cancelled while the run is not, where cancelled() is false and
@@ -324,18 +358,33 @@ jobs:
       # failed rollout to exercise this path. The job still ends red.
       - name: Roll back on a failed deploy
         if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure') }}
+        env:
+          PRE: ${{ steps.prerev.outputs.prerev }}
         run: |
           set -uo pipefail
           HIST=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null || echo '[]')
           CUR=$(printf '%s' "$HIST" | jq -r '.[-1].revision // 0')
-          # Target the newest revision BELOW current that Helm recorded as a real deploy
-          # (status superseded or deployed), skipping failed/pending revisions a retry may
-          # have left — a blind CUR-1 could land on another bad image (Cofounder, 2026-11-01).
-          TARGET=$(printf '%s' "$HIST" | jq -r --argjson cur "${CUR:-0}" '[.[] | select((.revision < $cur) and (.status=="superseded" or .status=="deployed"))] | (sort_by(.revision) | last | .revision) // empty')
-          if [ -z "$TARGET" ]; then
-            echo "::warning title=No rollback target::$RELEASE has no prior superseded/deployed revision below ${CUR:-0}; nothing to roll back to."
+          # Roll back only to the revision recorded as `deployed` before this upgrade
+          # (steps.prerev). No-op when there is no recorded good revision, or when CUR==PRE
+          # (the upgrade failed before Helm created a revision — the live release is
+          # unchanged, so a CUR-1 would demote a healthy deploy one step too far).
+          if [ "${CUR:-0}" = "0" ] || [ "${PRE:-0}" = "0" ] || [ -z "${PRE:-}" ]; then
+            echo "::warning title=No rollback target::$RELEASE has no pre-upgrade good revision (CUR=${CUR:-0} PRE=${PRE:-0}); nothing to roll back to."
             exit 0
           fi
-          echo "Rolling $RELEASE back: revision $CUR -> $TARGET (last successfully-deployed)"
-          helm rollback "$RELEASE" "$TARGET" -n "$NAMESPACE" --wait --timeout 8m
-          echo "::error title=Deploy rolled back::$RELEASE rolled back to revision $TARGET after a failed deploy. Investigate before the next deploy."
+          if [ "$CUR" = "$PRE" ]; then
+            echo "::warning title=No rollback needed::the upgrade created no new revision (CUR==PRE==$PRE); the live release is unchanged."
+            exit 0
+          fi
+          echo "Rolling $RELEASE back: revision $CUR -> $PRE (last successfully-deployed)"
+          # No `--wait` (see the upgrade above): a release-wide wait can hang on an unrelated
+          # workload and record the rollback as failed. Apply, then verify only managed workloads.
+          helm rollback "$RELEASE" "$PRE" -n "$NAMESPACE" --timeout 8m
+          MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
+          for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
+            n=${d##*/}
+            case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac
+            kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m \
+              || echo "::warning title=Rollback not yet ready::$d did not report ready within 8m after rollback to $PRE; check the cluster."
+          done
+          echo "::error title=Deploy rolled back::$RELEASE rolled back to revision $PRE after a failed deploy. Investigate before the next deploy."

From 1807c80f0dbe22a44e87cc940feffde395fab9f2 Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 1 Nov 2026 01:06:11 -0800
Subject: [PATCH 5/6] ci(deploy): short-circuit rollout verify on first stuck
 workload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Bounds the rollback window CodeRabbit flagged as "mitigated, not fully fixed": the
rollout verify loop waited up to 8m PER workload across the 5-item managed set, so a
bad image tag (which fails every managed workload to pull) could burn ~40m before the
rollback step — risking the 45m job cap cancelling the job before rollback runs, on
exactly the failure the net exists to catch. Now the loop exits on the first stuck
workload (the deploy is already failing), bounding verification to ~8m + 8m rollback.
Managed-set presence is still checked up front. Both deploy-dev and deploy-staging.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA
---
 .github/workflows/deploy-dev.yml     | 35 ++++++++++++++--------------
 .github/workflows/deploy-staging.yml | 31 +++++++++++++-----------
 2 files changed, 35 insertions(+), 31 deletions(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index a5fd56391c..4fefea19b3 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -355,30 +355,31 @@ jobs:
           # must not fail the deploy or trip the rollback below. A scaled-to-zero managed
           # workload passes instantly (0 of 0 updated) — parked is not stuck.
           MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
-          FAILED=""
-          CHECKED=""
+          TARGETS=""
           for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
             n=${d##*/}
-            case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac
-            CHECKED="$CHECKED $n"
+            case "$MANAGED" in *" $n "*) TARGETS="$TARGETS $d" ;; esac
+          done
+          if [ -z "$TARGETS" ]; then
+            echo "::error title=Rollout failed::none of the image-tagged workloads (backend/frontend/clawdbot-gateway/commonly-bot) found in $NAMESPACE — nothing to verify is itself a failure"
+            exit 1
+          fi
+          # Short-circuit on the FIRST stuck workload. The loop runs sequentially and each
+          # kubectl rollout status waits up to 8m; a bad image tag fails ALL managed
+          # workloads, so continuing would burn ~8m x N (~40m for 5) and could hit the 45m
+          # job cap before the rollback step runs. One failure already means the deploy is
+          # bad — stop and let the failure() rollback fire.
+          for d in $TARGETS; do
             echo "::group::rollout status $d"
             if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then
-              echo "$d: rolled out"
+              echo "$d: rolled out"; echo "::endgroup::"
             else
-              echo "::error title=Rollout failed::$d did not become ready within 8m"
-              FAILED="$FAILED $d"
+              echo "::endgroup::"
+              echo "::error title=Rollout failed::$d did not become ready within 8m — stopping verification; the deploy is failing and the rollback step will run"
+              exit 1
             fi
-            echo "::endgroup::"
           done
-          if [ -z "$CHECKED" ]; then
-            echo "::error title=Rollout failed::none of the image-tagged workloads (backend/frontend/clawdbot-gateway/commonly-bot) found in $NAMESPACE — nothing to verify is itself a failure"
-            exit 1
-          fi
-          if [ -n "$FAILED" ]; then
-            echo "Deploy failed for:$FAILED"
-            exit 1
-          fi
-          echo "Deployed workloads live:$CHECKED"
+          echo "Deployed workloads live:$TARGETS"
 
       # A pod can be Ready and still unable to serve chat. The backend decides
       # once at boot whether Postgres is available, and /ready answers 200 in
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index d979d093f7..2e358f735a 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -208,27 +208,30 @@ jobs:
           # the cluster. An unrelated unhealthy workload in the namespace is not this job's
           # failure and must not trip the rollback below; a parked workload drops out.
           MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
-          FAILED=""
-          CHECKED=""
+          TARGETS=""
           for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
             n=${d##*/}
-            case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac
-            CHECKED="$CHECKED $n"
+            case "$MANAGED" in *" $n "*) TARGETS="$TARGETS $d" ;; esac
+          done
+          if [ -z "$TARGETS" ]; then
+            echo "::error title=Rollout failed::none of the image-tagged workloads found in $NAMESPACE"
+            exit 1
+          fi
+          # Short-circuit on the first stuck workload (see deploy-dev.yml): a bad tag fails
+          # all managed workloads, and waiting 8m x N could exceed the 45m job cap before
+          # rollback. One failure = the deploy is bad; stop and let failure() rollback run.
+          # The force_bad_tag rehearsal hits this on the first workload.
+          for d in $TARGETS; do
             echo "::group::rollout status $d"
             if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then
-              echo "$d: rolled out"
+              echo "$d: rolled out"; echo "::endgroup::"
             else
-              echo "::error title=Rollout failed::$d did not become ready within 8m"
-              FAILED="$FAILED $d"
+              echo "::endgroup::"
+              echo "::error title=Rollout failed::$d did not become ready within 8m — stopping; rollback will run"
+              exit 1
             fi
-            echo "::endgroup::"
           done
-          if [ -z "$CHECKED" ]; then
-            echo "::error title=Rollout failed::none of the image-tagged workloads found in $NAMESPACE"
-            exit 1
-          fi
-          if [ -n "$FAILED" ]; then echo "Deploy failed for:$FAILED"; exit 1; fi
-          echo "Deployed workloads live:$CHECKED"
+          echo "Deployed workloads live:$TARGETS"
 
       - name: Set up Node for the Tier-1 smoke
         uses: actions/setup-node@v4

From 4ceb3c6dece6074187f1bfca389b02f27b3db30e Mon Sep 17 00:00:00 2001
From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com>
Date: Sun, 1 Nov 2026 01:21:52 -0800
Subject: [PATCH 6/6] ci(deploy): separate the deployment lookup from the
 rollout check (no rollback on a lookup blip)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CodeRabbit flagged that the short-circuit refactor could demote a healthy release: a
transient `kubectl get deploy` failure after a good helm upgrade yields an empty list,
which the in-step guard turned into a rollout failure — and the rollback gate keys on
steps.rollout.outcome, so a lookup blip would roll back prod.

Fix: move the lookup into its own `deploylist` step BEFORE `rollout`, with a 3x retry.
That step is NOT in the rollback gate, so a lookup error (cluster/API unreachable) or a
confirmed-empty managed set fails the JOB for a human while the rollout step stays
skipped (outcome != 'failure') — no rollback. Only a real `kubectl rollout status`
failure in the rollout step is eligible for rollback. Both deploy-dev and deploy-staging.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015AKgpNr8tFMMj8aFvFPUdA
---
 .github/workflows/deploy-dev.yml     | 57 +++++++++++++++++++---------
 .github/workflows/deploy-staging.yml | 37 +++++++++++++-----
 2 files changed, 68 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
index 4fefea19b3..a8c1e119a8 100644
--- a/.github/workflows/deploy-dev.yml
+++ b/.github/workflows/deploy-dev.yml
@@ -342,33 +342,56 @@ jobs:
       # moment it is disabled — the gate follows the release, not a memory of
       # it. A deployment scaled to zero passes instantly (0 of 0 updated),
       # which is correct: parked is not stuck.
-      - name: Verify rollout of the deployed workloads
-        id: rollout
+      # The deployment LOOKUP is its own step, deliberately BEFORE the rollout check and
+      # NOT referenced by the rollback gate. A transient `kubectl get deploy` failure
+      # after a good helm upgrade must not be read as a stuck rollout: if it were in the
+      # rollout step, the empty result would fail that step and the gate would demote a
+      # healthy release over an API blip. Here a lookup error (or a confirmed-empty
+      # managed set) fails the JOB for a human, while the rollout step stays skipped
+      # (outcome != 'failure'), so no rollback fires. Retry rides out a blip.
+      - name: List the workloads to verify (lookup, not a rollout check)
+        id: deploylist
         run: |
           set -uo pipefail
-          # Derived from the cluster AND intersected with the workloads this job sets
-          # image tags for — backend (+ its worker), frontend, clawdbot-gateway,
-          # commonly-bot. A parked workload (enabled:false removes the Deployment, as
-          # #1051 did to the gateway) simply isn't present and drops out; an unrelated
-          # unhealthy workload in the same namespace (litellm, redis, cloud-codex,
-          # ops-runner, cloudflared) is a real problem but NOT this job's failure, and
-          # must not fail the deploy or trip the rollback below. A scaled-to-zero managed
-          # workload passes instantly (0 of 0 updated) — parked is not stuck.
           MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
+          ALL=""
+          for attempt in 1 2 3; do
+            if ALL=$(kubectl get deploy -n "$NAMESPACE" -o name) && [ -n "$ALL" ]; then break; fi
+            echo "deploy lookup attempt $attempt returned nothing/failed; retrying in 5s"; ALL=""; sleep 5
+          done
+          if [ -z "$ALL" ]; then
+            echo "::error title=Deploy lookup failed::could not list deployments in $NAMESPACE after retries — cluster/API unreachable. NOT a rollout failure; the running release is left in place (no rollback)."
+            exit 1
+          fi
+          # Intersect the cluster's deployments with the workloads this job image-tags —
+          # backend (+ worker), frontend, clawdbot-gateway, commonly-bot. A parked workload
+          # (enabled:false removes the Deployment, as #1051 did to the gateway) drops out;
+          # an unrelated unhealthy workload (litellm, redis, cloud-codex, ops-runner,
+          # cloudflared) is never this job's failure. A scaled-to-zero managed workload
+          # passes rollout instantly (0 of 0 updated) — parked is not stuck.
           TARGETS=""
-          for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
+          for d in $ALL; do
             n=${d##*/}
             case "$MANAGED" in *" $n "*) TARGETS="$TARGETS $d" ;; esac
           done
           if [ -z "$TARGETS" ]; then
-            echo "::error title=Rollout failed::none of the image-tagged workloads (backend/frontend/clawdbot-gateway/commonly-bot) found in $NAMESPACE — nothing to verify is itself a failure"
+            echo "::error title=No managed workloads::none of backend/frontend/clawdbot-gateway/commonly-bot present in $NAMESPACE (cluster reachable). A human should look; not auto-rolling back."
             exit 1
           fi
-          # Short-circuit on the FIRST stuck workload. The loop runs sequentially and each
-          # kubectl rollout status waits up to 8m; a bad image tag fails ALL managed
-          # workloads, so continuing would burn ~8m x N (~40m for 5) and could hit the 45m
-          # job cap before the rollback step runs. One failure already means the deploy is
-          # bad — stop and let the failure() rollback fire.
+          echo "targets=$TARGETS" >> "$GITHUB_OUTPUT"
+          echo "will verify:$TARGETS"
+
+      - name: Verify rollout of the deployed workloads
+        id: rollout
+        env:
+          TARGETS: ${{ steps.deploylist.outputs.targets }}
+        run: |
+          set -uo pipefail
+          # Short-circuit on the FIRST stuck workload. Each kubectl rollout status waits up
+          # to 8m; a bad image tag fails ALL managed workloads, so continuing would burn
+          # ~8m x N (~40m for 5) and could hit the 45m job cap before the rollback step
+          # runs. One failure already means the deploy is bad — stop and let failure()
+          # rollback fire. Only THIS step's failure is eligible for rollback.
           for d in $TARGETS; do
             echo "::group::rollout status $d"
             if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then
diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
index 2e358f735a..5b6d5ffc7a 100644
--- a/.github/workflows/deploy-staging.yml
+++ b/.github/workflows/deploy-staging.yml
@@ -199,28 +199,47 @@ jobs:
             --burst-limit 200 \
             --qps 100
 
-      - name: Verify rollout of the deployed workloads
-        id: rollout
+      # Deployment LOOKUP in its own step before the rollout check, NOT in the rollback
+      # gate (see deploy-dev.yml): a transient `kubectl get deploy` failure after a good
+      # upgrade must fail the job for a human, not be read as a stuck rollout that demotes
+      # a healthy staging release. Retry rides out a blip.
+      - name: List the workloads to verify (lookup, not a rollout check)
+        id: deploylist
         run: |
           set -uo pipefail
-          # Scoped to the workloads this job image-tags (backend/+worker, frontend,
-          # commonly-bot — clawdbot-gateway when the staging chart ships it), derived from
-          # the cluster. An unrelated unhealthy workload in the namespace is not this job's
-          # failure and must not trip the rollback below; a parked workload drops out.
           MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot "
+          ALL=""
+          for attempt in 1 2 3; do
+            if ALL=$(kubectl get deploy -n "$NAMESPACE" -o name) && [ -n "$ALL" ]; then break; fi
+            echo "deploy lookup attempt $attempt returned nothing/failed; retrying in 5s"; ALL=""; sleep 5
+          done
+          if [ -z "$ALL" ]; then
+            echo "::error title=Deploy lookup failed::could not list deployments in $NAMESPACE after retries — cluster/API unreachable. NOT a rollout failure (no rollback)."
+            exit 1
+          fi
           TARGETS=""
-          for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do
+          for d in $ALL; do
             n=${d##*/}
             case "$MANAGED" in *" $n "*) TARGETS="$TARGETS $d" ;; esac
           done
           if [ -z "$TARGETS" ]; then
-            echo "::error title=Rollout failed::none of the image-tagged workloads found in $NAMESPACE"
+            echo "::error title=No managed workloads::none of the image-tagged workloads present in $NAMESPACE (cluster reachable); not auto-rolling back."
             exit 1
           fi
+          echo "targets=$TARGETS" >> "$GITHUB_OUTPUT"
+          echo "will verify:$TARGETS"
+
+      - name: Verify rollout of the deployed workloads
+        id: rollout
+        env:
+          TARGETS: ${{ steps.deploylist.outputs.targets }}
+        run: |
+          set -uo pipefail
           # Short-circuit on the first stuck workload (see deploy-dev.yml): a bad tag fails
           # all managed workloads, and waiting 8m x N could exceed the 45m job cap before
           # rollback. One failure = the deploy is bad; stop and let failure() rollback run.
-          # The force_bad_tag rehearsal hits this on the first workload.
+          # The force_bad_tag rehearsal hits this on the first workload. Only THIS step's
+          # failure is eligible for rollback.
           for d in $TARGETS; do
             echo "::group::rollout status $d"
             if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then