diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index b3bb1a813..a8c1e119a 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -111,7 +111,11 @@ jobs: needs: require_staging_smoke runs-on: ubuntu-latest environment: dev - timeout-minutes: 30 + # 45, not 30: a bad image makes `kubectl rollout status` wait its full 8m per + # stuck workload before the rollout step fails, and the rollback step then needs + # its own window. 30m could see GitHub cancel the job mid-rollback (cancellation + # does not satisfy failure(), so the rollback would not re-run). 45m reserves room. + timeout-minutes: 45 steps: - name: Checkout (with submodules for _external/clawdbot) @@ -261,6 +265,20 @@ jobs: cluster_name: ${{ env.CLUSTER }} location: ${{ env.REGION }} + # Record the revision Helm currently has as `deployed` — the last known-good + # release — BEFORE the upgrade, so the rollback step can return to exactly this + # revision instead of inferring one by arithmetic after a failed upgrade (a + # failed retry can leave a bad image at CUR-1). 0 == no prior deploy (first + # install): nothing to roll back to. + - name: Record the current good revision (for rollback) + id: prerev + run: | + set -uo pipefail + HIST=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null || echo '[]') + PRE=$(printf '%s' "$HIST" | jq -r '[.[] | select(.status=="deployed")] | (sort_by(.revision) | last | .revision) // 0') + echo "prerev=$PRE" >> "$GITHUB_OUTPUT" + echo "Pre-upgrade good (deployed) revision: $PRE" + # `--wait` is deliberately NOT used here. It waits for EVERY resource in # the release, so any long-unhealthy auxiliary workload fails a deploy # that actually succeeded. On 2026-08-04 that produced five consecutive @@ -282,6 +300,7 @@ jobs: # incident silently reverts every deploy since, which by then included # two auth fixes. Rollback safety depends on this job telling the truth. - name: Helm upgrade commonly-dev + id: helm env: TAG: ${{ steps.tag.outputs.tag }} GATEWAY_ENABLED: ${{ steps.gateway.outputs.enabled }} @@ -323,31 +342,67 @@ jobs: # moment it is disabled — the gate follows the release, not a memory of # it. A deployment scaled to zero passes instantly (0 of 0 updated), # which is correct: parked is not stuck. - - name: Verify rollout of the deployed workloads - id: rollout + # The deployment LOOKUP is its own step, deliberately BEFORE the rollout check and + # NOT referenced by the rollback gate. A transient `kubectl get deploy` failure + # after a good helm upgrade must not be read as a stuck rollout: if it were in the + # rollout step, the empty result would fail that step and the gate would demote a + # healthy release over an API blip. Here a lookup error (or a confirmed-empty + # managed set) fails the JOB for a human, while the rollout step stays skipped + # (outcome != 'failure'), so no rollback fires. Retry rides out a blip. + - name: List the workloads to verify (lookup, not a rollout check) + id: deploylist run: | set -uo pipefail - FAILED="" - DEPLOYS=$(kubectl get deploy -n "$NAMESPACE" -o name) - if [ -z "$DEPLOYS" ]; then - echo "::error title=Rollout failed::no deployments found in $NAMESPACE — nothing to verify is itself a failure" + MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot " + ALL="" + for attempt in 1 2 3; do + if ALL=$(kubectl get deploy -n "$NAMESPACE" -o name) && [ -n "$ALL" ]; then break; fi + echo "deploy lookup attempt $attempt returned nothing/failed; retrying in 5s"; ALL=""; sleep 5 + done + if [ -z "$ALL" ]; then + echo "::error title=Deploy lookup failed::could not list deployments in $NAMESPACE after retries — cluster/API unreachable. NOT a rollout failure; the running release is left in place (no rollback)." exit 1 fi - for d in $DEPLOYS; do + # Intersect the cluster's deployments with the workloads this job image-tags — + # backend (+ worker), frontend, clawdbot-gateway, commonly-bot. A parked workload + # (enabled:false removes the Deployment, as #1051 did to the gateway) drops out; + # an unrelated unhealthy workload (litellm, redis, cloud-codex, ops-runner, + # cloudflared) is never this job's failure. A scaled-to-zero managed workload + # passes rollout instantly (0 of 0 updated) — parked is not stuck. + TARGETS="" + for d in $ALL; do + n=${d##*/} + case "$MANAGED" in *" $n "*) TARGETS="$TARGETS $d" ;; esac + done + if [ -z "$TARGETS" ]; then + echo "::error title=No managed workloads::none of backend/frontend/clawdbot-gateway/commonly-bot present in $NAMESPACE (cluster reachable). A human should look; not auto-rolling back." + exit 1 + fi + echo "targets=$TARGETS" >> "$GITHUB_OUTPUT" + echo "will verify:$TARGETS" + + - name: Verify rollout of the deployed workloads + id: rollout + env: + TARGETS: ${{ steps.deploylist.outputs.targets }} + run: | + set -uo pipefail + # Short-circuit on the FIRST stuck workload. Each kubectl rollout status waits up + # to 8m; a bad image tag fails ALL managed workloads, so continuing would burn + # ~8m x N (~40m for 5) and could hit the 45m job cap before the rollback step + # runs. One failure already means the deploy is bad — stop and let failure() + # rollback fire. Only THIS step's failure is eligible for rollback. + for d in $TARGETS; do echo "::group::rollout status $d" if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then - echo "$d: rolled out" + echo "$d: rolled out"; echo "::endgroup::" else - echo "::error title=Rollout failed::$d did not become ready within 8m" - FAILED="$FAILED $d" + echo "::endgroup::" + echo "::error title=Rollout failed::$d did not become ready within 8m — stopping verification; the deploy is failing and the rollback step will run" + exit 1 fi - echo "::endgroup::" done - if [ -n "$FAILED" ]; then - echo "Deploy failed for:$FAILED" - exit 1 - fi - echo "All four deployed workloads are live." + echo "Deployed workloads live:$TARGETS" # A pod can be Ready and still unable to serve chat. The backend decides # once at boot whether Postgres is available, and /ready answers 200 in @@ -357,6 +412,7 @@ jobs: # (TASK-168), ask the newest backend pod from the inside. Unauthenticated, # a mounted /api/pg/messages answers 401 and an unmounted one 404. - name: Verify the new backend serves chat + id: chat # Deliberately not `always()` / `!cancelled()`: if an earlier step failed # (image push, helm upgrade, GKE credentials) the cluster is unreachable # or unchanged, and this step would then report "Chat unavailable on the @@ -393,6 +449,52 @@ jobs: echo "::error title=Chat unavailable on the new backend::$POD has pg/status=$STATUS and pg/messages=$ROUTE (401 means mounted, 404 unmounted). Chat history fails until the pod restarts: kubectl rollout restart deploy/backend -n $NAMESPACE" exit 1 + # D227 (Sam via Cofounder, 2026-11-01): the heavy test matrix is leaving the merge + # path, so a bad image can reach prod and this post-deploy verify is the net. If any + # deploy/verify step above failed, undo this release in-cluster by rolling back to the + # previous helm revision, so prod self-heals without waiting for a human. The job still + # ends red, which makes the auto-deploy dispatcher write auto-deploy.hold and stop + # re-dispatching (Mac-side). Rolls back to the revision recorded as good before this + # upgrade (steps.prerev); a no-op at first install, if the cluster is unreachable, or + # if the upgrade created no new revision (live release unchanged). Gated on the helm + # upgrade or a verify step failing (not bare failure()), so a pre-deploy failure (checkout, + # auth, build/push) never demotes a healthy release. + - name: Roll back on a failed deploy + if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure' || steps.chat.outcome == 'failure') }} + env: + PRE: ${{ steps.prerev.outputs.prerev }} + run: | + set -uo pipefail + HIST=$(helm history commonly-dev -n "$NAMESPACE" -o json 2>/dev/null || echo '[]') + CUR=$(printf '%s' "$HIST" | jq -r '.[-1].revision // 0') + # Roll back only to the revision recorded as `deployed` BEFORE this upgrade + # (steps.prerev). Two no-ops, each a release we must NOT demote: + # - no recorded good revision (first install) -> nothing to roll back to; + # - CUR == PRE -> the upgrade failed before Helm created a revision (chart + # load / values parse): the live release is unchanged, so leave it. + if [ "${CUR:-0}" = "0" ] || [ "${PRE:-0}" = "0" ] || [ -z "${PRE:-}" ]; then + echo "::warning title=No rollback target::commonly-dev has no pre-upgrade good revision (CUR=${CUR:-0} PRE=${PRE:-0}); nothing to roll back to." + exit 0 + fi + if [ "$CUR" = "$PRE" ]; then + echo "::warning title=No rollback needed::the upgrade created no new revision (CUR==PRE==$PRE); the live release is unchanged." + exit 0 + fi + echo "Rolling commonly-dev back: revision $CUR -> $PRE (last successfully-deployed)" + # No `--wait`: like the upgrade above, a release-wide wait can be blocked by an + # unrelated unhealthy workload and would then record the rollback itself as + # failed, leaving the bad revision marked deployed. Apply, then verify only the + # workloads this job manages. + helm rollback commonly-dev "$PRE" -n "$NAMESPACE" --timeout 8m + MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot " + for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do + n=${d##*/} + case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac + kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m \ + || echo "::warning title=Rollback not yet ready::$d did not report ready within 8m after rollback to $PRE; check the cluster." + done + echo "::error title=Deploy rolled back::commonly-dev rolled back to revision $PRE after a failed deploy. Investigate the bad image before the next deploy." + # The stranger journey walks signup -> Scout's welcome -> a real question # -> the connect card against the release just deployed # (docs/runbooks/alerting.md#stranger-journey). It runs as a Job from the diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 3142d301c..5b6d5ffc7 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -25,6 +25,11 @@ on: description: 'Git ref / SHA to build + deploy to staging (defaults to the triggering ref)' required: false default: '' + force_bad_tag: + description: 'REHEARSAL: deploy a nonexistent backend image tag so the rollout fails and the rollback step fires (staging only)' + type: boolean + required: false + default: false push: branches: [ main ] @@ -49,7 +54,10 @@ jobs: name: Build + push + helm upgrade (staging) runs-on: ubuntu-latest environment: staging - timeout-minutes: 30 + # 45, not 30: rollout verification can wait its full 8m per stuck workload and the + # rollback step needs its own window after; 30m risks GitHub cancelling mid-rollback + # (cancellation does not satisfy failure(), so rollback would not re-run). + timeout-minutes: 45 steps: - name: Checkout @@ -149,46 +157,100 @@ jobs: cluster_name: ${{ env.CLUSTER }} location: ${{ env.REGION }} + # Record the revision Helm currently has as `deployed` (last known-good) BEFORE the + # upgrade, so the rollback step returns to exactly this revision instead of inferring + # one by arithmetic. 0 == no prior deploy (first install). + - name: Record the current good revision (for rollback) + id: prerev + run: | + set -uo pipefail + HIST=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null || echo '[]') + PRE=$(printf '%s' "$HIST" | jq -r '[.[] | select(.status=="deployed")] | (sort_by(.revision) | last | .revision) // 0') + echo "prerev=$PRE" >> "$GITHUB_OUTPUT" + echo "Pre-upgrade good (deployed) revision: $PRE" + # --create-namespace: commonly-staging does not exist before the first # install. NOTE: values-staging.yaml should set createNamespace:false # (as values-local.yaml does) so the chart does not ALSO declare the # Namespace and collide with the one helm creates here. Tracked against # the staging chart PR. - name: Helm upgrade commonly-staging + id: helm env: TAG: ${{ steps.tag.outputs.tag }} + FORCE_BAD: ${{ inputs.force_bad_tag }} run: | + # Rollback rehearsal (force_bad_tag): point the backend image at a tag that does not + # exist, so the pod never pulls, the rollout verify below fails, and the rollback step + # at the end of the job fires. Staging only; a normal run leaves BTAG = the real tag. + BTAG="$TAG" + if [ "$FORCE_BAD" = "true" ]; then + BTAG="rollback-rehearsal-nonexistent" + echo "::warning title=Rollback rehearsal::forcing a nonexistent backend image tag ($BTAG) to exercise the rollback step" + fi helm upgrade --install "$RELEASE" k8s/helm/commonly -n "$NAMESPACE" --create-namespace \ -f k8s/helm/commonly/values.yaml \ -f k8s/helm/commonly/values-staging.yaml \ -f "$RUNNER_TEMP/values-private.yaml" \ - --set backend.image.tag="$TAG" \ + --set backend.image.tag="$BTAG" \ --set frontend.image.tag="$TAG-staging" \ --set agents.commonlyBot.image.tag="$TAG" \ --timeout 10m \ --burst-limit 200 \ --qps 100 - - name: Verify rollout of the deployed workloads + # Deployment LOOKUP in its own step before the rollout check, NOT in the rollback + # gate (see deploy-dev.yml): a transient `kubectl get deploy` failure after a good + # upgrade must fail the job for a human, not be read as a stuck rollout that demotes + # a healthy staging release. Retry rides out a blip. + - name: List the workloads to verify (lookup, not a rollout check) + id: deploylist run: | set -uo pipefail - FAILED="" - DEPLOYS=$(kubectl get deploy -n "$NAMESPACE" -o name) - if [ -z "$DEPLOYS" ]; then - echo "::error title=Rollout failed::no deployments in $NAMESPACE" + MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot " + ALL="" + for attempt in 1 2 3; do + if ALL=$(kubectl get deploy -n "$NAMESPACE" -o name) && [ -n "$ALL" ]; then break; fi + echo "deploy lookup attempt $attempt returned nothing/failed; retrying in 5s"; ALL=""; sleep 5 + done + if [ -z "$ALL" ]; then + echo "::error title=Deploy lookup failed::could not list deployments in $NAMESPACE after retries — cluster/API unreachable. NOT a rollout failure (no rollback)." exit 1 fi - for d in $DEPLOYS; do + TARGETS="" + for d in $ALL; do + n=${d##*/} + case "$MANAGED" in *" $n "*) TARGETS="$TARGETS $d" ;; esac + done + if [ -z "$TARGETS" ]; then + echo "::error title=No managed workloads::none of the image-tagged workloads present in $NAMESPACE (cluster reachable); not auto-rolling back." + exit 1 + fi + echo "targets=$TARGETS" >> "$GITHUB_OUTPUT" + echo "will verify:$TARGETS" + + - name: Verify rollout of the deployed workloads + id: rollout + env: + TARGETS: ${{ steps.deploylist.outputs.targets }} + run: | + set -uo pipefail + # Short-circuit on the first stuck workload (see deploy-dev.yml): a bad tag fails + # all managed workloads, and waiting 8m x N could exceed the 45m job cap before + # rollback. One failure = the deploy is bad; stop and let failure() rollback run. + # The force_bad_tag rehearsal hits this on the first workload. Only THIS step's + # failure is eligible for rollback. + for d in $TARGETS; do echo "::group::rollout status $d" if kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m; then - echo "$d: rolled out" + echo "$d: rolled out"; echo "::endgroup::" else - echo "::error title=Rollout failed::$d did not become ready within 8m" - FAILED="$FAILED $d" + echo "::endgroup::" + echo "::error title=Rollout failed::$d did not become ready within 8m — stopping; rollback will run" + exit 1 fi - echo "::endgroup::" done - if [ -n "$FAILED" ]; then echo "Deploy failed for:$FAILED"; exit 1; fi + echo "Deployed workloads live:$TARGETS" - name: Set up Node for the Tier-1 smoke uses: actions/setup-node@v4 @@ -238,8 +300,18 @@ jobs: SHA: ${{ steps.tag.outputs.sha }} JOB_STATUS: ${{ job.status }} GH_TOKEN: ${{ github.token }} + FORCE_BAD: ${{ inputs.force_bad_tag }} run: | set -uo pipefail + # Rollback rehearsal (force_bad_tag): the rollout was failed deliberately, so this + # SHA's backend image was never actually tested. Posting staging-smoke=failure here + # would make deploy-dev.yml reject the SHA until a real staging run succeeds — a + # production-gating status from a drill. Post nothing; the rollout failure above + # already drives the rollback step (its gate reads steps.rollout.outcome). + if [ "${FORCE_BAD:-}" = "true" ]; then + echo "rollback rehearsal: not posting a production-gating staging-smoke status for $SHA ($TAG)" + exit 0 + fi # A cancelled job deployed nothing, so it has nothing truthful to say about # this SHA. The step condition already skips a cancelled RUN; this covers a # job cancelled while the run is not, where cancelled() is false and @@ -300,3 +372,41 @@ jobs: -f context="staging-smoke" \ -f description="$DESC" >/dev/null [ "$STATE" = "success" ] + + # D227 (Sam via Cofounder, 2026-11-01): mirror of the prod rollback net. Roll this staging + # release back to the newest prior good revision when the helm upgrade or the rollout verify + # fails — NOT on a post-deploy infra flake (setup-node / npm / playwright-install), which + # would otherwise self-heal a healthy release. The `force_bad_tag` rehearsal input drives a + # failed rollout to exercise this path. The job still ends red. + - name: Roll back on a failed deploy + if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure') }} + env: + PRE: ${{ steps.prerev.outputs.prerev }} + run: | + set -uo pipefail + HIST=$(helm history "$RELEASE" -n "$NAMESPACE" -o json 2>/dev/null || echo '[]') + CUR=$(printf '%s' "$HIST" | jq -r '.[-1].revision // 0') + # Roll back only to the revision recorded as `deployed` before this upgrade + # (steps.prerev). No-op when there is no recorded good revision, or when CUR==PRE + # (the upgrade failed before Helm created a revision — the live release is + # unchanged, so a CUR-1 would demote a healthy deploy one step too far). + if [ "${CUR:-0}" = "0" ] || [ "${PRE:-0}" = "0" ] || [ -z "${PRE:-}" ]; then + echo "::warning title=No rollback target::$RELEASE has no pre-upgrade good revision (CUR=${CUR:-0} PRE=${PRE:-0}); nothing to roll back to." + exit 0 + fi + if [ "$CUR" = "$PRE" ]; then + echo "::warning title=No rollback needed::the upgrade created no new revision (CUR==PRE==$PRE); the live release is unchanged." + exit 0 + fi + echo "Rolling $RELEASE back: revision $CUR -> $PRE (last successfully-deployed)" + # No `--wait` (see the upgrade above): a release-wide wait can hang on an unrelated + # workload and record the rollback as failed. Apply, then verify only managed workloads. + helm rollback "$RELEASE" "$PRE" -n "$NAMESPACE" --timeout 8m + MANAGED=" backend backend-worker frontend clawdbot-gateway commonly-bot " + for d in $(kubectl get deploy -n "$NAMESPACE" -o name); do + n=${d##*/} + case "$MANAGED" in *" $n "*) : ;; *) continue ;; esac + kubectl rollout status "$d" -n "$NAMESPACE" --timeout=8m \ + || echo "::warning title=Rollback not yet ready::$d did not report ready within 8m after rollback to $PRE; check the cluster." + done + echo "::error title=Deploy rolled back::$RELEASE rolled back to revision $PRE after a failed deploy. Investigate before the next deploy."