diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index b0ff472b0..106738774 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -449,6 +449,36 @@ jobs: echo "::error title=Chat unavailable on the new backend::$POD has pg/status=$STATUS and pg/messages=$ROUTE (401 means mounted, 404 unmounted). Chat history fails until the pod restarts: kubectl rollout restart deploy/backend -n $NAMESPACE" exit 1 + # A real hosted-agent turn through the live injection guard, for an agent named after a + # guard keyword — the class that 400'd a design partner every turn until #3285. Exit 3 = + # the guard blocked the turn, so fail the job and let the rollback step below fire; exit 2 + # = inconclusive and exit 1 = kubectl/node failed, both warn only and never roll back. + # ~30s and 4 model turns, sharing the hosted cap with any hand run, so the + # concurrency:deploy-dev group (no cancel-in-progress) keeps deploys serial — a hand run + # during a deploy is the case to avoid. The script comes from the dispatched checkout, so + # it matches this release even while ops-runner is still rolling to the new image. The + # deploy service account needs pods/exec on litellm and ops-runner (it already execs the + # backend pod above). See docs/runbooks/guard-name-canary.md. + - name: Guard-name canary + id: canary + run: | + set -uo pipefail + KW=$(kubectl exec -i -n "$NAMESPACE" deploy/litellm -c litellm -- \ + /app/.venv/bin/python3 -I - < scripts/guard-keywords-dump.py) || KW='' + # Actions runs this under `bash -e`; `|| CODE=$?` keeps a non-zero + # exit from ending the step before it is read. + CODE=0 + kubectl exec -i -n "$NAMESPACE" deploy/ops-runner -c ops-runner -- \ + env NODE_PATH=/app/node_modules CANARY_RUN=1 CANARY_GUARD_KEYWORDS="$KW" \ + node - < scripts/guard-name-canary.js || CODE=$? + if [ "$CODE" = 3 ]; then + echo "::error title=Guard-name canary blocked::the injection guard stopped a turn for an agent named after a guard keyword; see the check lines above" + exit 1 + fi + if [ "$CODE" != 0 ]; then + echo "::warning title=Guard-name canary inconclusive::exit $CODE; read the check lines above" + fi + # Record THIS release as rollout-verified (and, for dev, chat-verified), so the rollback # step below returns to a revision that actually passed verification — never merely "last # deployed". With auto-deploy, a deploy cancelled mid-verify by a newer push leaves an @@ -484,7 +514,7 @@ jobs: # upgrade or a verify step failing (not bare failure()), so a pre-deploy failure (checkout, # auth, build/push) never demotes a healthy release. - name: Roll back on a failed deploy - if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure' || steps.chat.outcome == 'failure') }} + if: ${{ failure() && (steps.helm.outcome == 'failure' || steps.rollout.outcome == 'failure' || steps.chat.outcome == 'failure' || steps.canary.outcome == 'failure') }} env: PRE: ${{ steps.prerev.outputs.prerev }} run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a837d232a..61f313f77 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -120,6 +120,11 @@ jobs: node-version: 20 - name: Filter regex regression test run: node --test scripts/changes-filter.test.mjs + # Guard the deploy-dev canary exit-code routing: the test reads the canary step + # and the rollback condition out of deploy-dev.yml, so a silent edit that turned a + # guard block (exit 3) into a green deploy, or dropped the rollback clause, fails CI. + - name: Deploy-dev canary wiring regression test + run: node --test scripts/guard-canary-wiring.test.mjs # Heavy jobs (backend, frontend, service tests) route to Latchkey runners on # a push to main when vars.LATCHKEY_HEAVY is 'on' AND the push touched a heavy diff --git a/scripts/guard-canary-wiring.test.mjs b/scripts/guard-canary-wiring.test.mjs new file mode 100644 index 000000000..a830cb4f2 --- /dev/null +++ b/scripts/guard-canary-wiring.test.mjs @@ -0,0 +1,63 @@ +// Regression test for the guard-name canary's exit-code routing in +// .github/workflows/deploy-dev.yml. The canary step maps its script's exit code to a +// deploy outcome: 3 = the live injection guard blocked the turn -> fail the step so the +// rollback step fires; 1 (kubectl/node error) and 2 (inconclusive) = warn only, step +// succeeds, no rollback; 0 = pass. A silent edit — 3 changed to 2, the sole `exit 1` +// dropped, or the rollback clause removed — would turn a real guard block into a green +// deploy, which is exactly the regression this guards. It reads the logic straight out of +// the workflow (no duplication), the same approach as changes-filter.test.mjs. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const yml = readFileSync(join(ROOT, '.github/workflows/deploy-dev.yml'), 'utf8'); + +// Extract one step's block: from its `- name: ` line to the next step's `- name:`. +function stepBlock(text, name) { + const lines = text.split('\n'); + const start = lines.findIndex((l) => l.trim() === `- name: ${name}`); + assert.ok(start >= 0, `step "${name}" not found in deploy-dev.yml`); + let end = lines.length; + for (let i = start + 1; i < lines.length; i += 1) { + if (/^\s*- name: /.test(lines[i])) { end = i; break; } + } + return lines.slice(start, end).join('\n'); +} + +const canary = stepBlock(yml, 'Guard-name canary'); + +test('the canary step carries id: canary (the rollback condition references it)', () => { + assert.match(canary, /^\s*id:\s*canary\s*$/m); +}); + +test('only exit code 3 hard-fails the canary step', () => { + // 3 -> the step exits nonzero (fails) so failure() holds and rollback fires. + assert.match(canary, /if\s*\[\s*"\$CODE"\s*=\s*3\s*\]\s*;\s*then/); + // The CODE=3 branch is the ONLY path that exits nonzero. Any other nonzero exit + // would fail the step for an inconclusive/error code and trip a spurious rollback. + const nonzeroExits = (canary.match(/^\s*exit\s+[1-9][0-9]*\s*$/gm) || []).map((s) => s.trim()); + assert.deepEqual(nonzeroExits, ['exit 1'], + 'canary must have exactly one nonzero exit (inside the CODE=3 branch)'); +}); + +test('exit codes 1 and 2 are warn-only (no rollback)', () => { + // A single `if [ "$CODE" != 0 ]` warn branch catches 1 and 2 without exiting nonzero. + assert.match(canary, /if\s*\[\s*"\$CODE"\s*!=\s*0\s*\]\s*;\s*then/); + assert.match(canary, /::warning title=Guard-name canary inconclusive/); +}); + +test('the rollback step fires on a canary failure', () => { + const rb = stepBlock(yml, 'Roll back on a failed deploy'); + assert.match(rb, /failure\(\)\s*&&/); + assert.match(rb, /steps\.canary\.outcome\s*==\s*'failure'/); +}); + +test('a guard-blocked release is never recorded as rollout-verified', () => { + // The record step is `if: success()`, so a failed canary skips it and the next + // rollback returns to the last genuinely-verified revision, not this blocked one. + const rec = stepBlock(yml, 'Record this release as rollout-verified (rollback target)'); + assert.match(rec, /^\s*if:\s*success\(\)\s*$/m); +});