diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e22864..7ba2ef1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,9 @@ jobs: - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh + - name: Bridge pairing rollout safety + run: tests/bridge_pairing_rollout.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index 9b3efbb0..fcdef01d 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -53,6 +53,32 @@ platform rather than templated here: external ingress/service mesh, KEDA-style queue-depth autoscaling, and cloud-IAM secret delivery (the env hooks below cover all of them). +**Pairing-fence rollbacks.** Do not use a direct `helm rollback` from a chart +revision containing the bridge pairing fence to an older revision. Helm runs +rollback hooks from the target revision, so a pre-fence target cannot stop its +own old and new API replicas from overlapping. Use the chart's fail-closed +helper instead: + +```bash +helm/codeapi/scripts/safe-pairing-rollback.sh RELEASE REVISION NAMESPACE +``` + +The helper records an out-of-band rollback epoch, deletes the API HPA, scales +the live fenced API deployment to zero, verifies that the Deployment and every +matching pod have converged to zero, and only then invokes `helm rollback`. +When a fenced revision is deployed again, the epoch forces one fresh cleanup of +legacy pairing codes even if the original migration window has expired. This +causes an API outage by design. If rollback fails, the helper repeats the drain +after re-discovering every API Deployment and explicitly deletes any remaining +API pods, so a partially applied rollback cannot leave a mixed-version API +running. The operator running it needs permission to read/scale Deployments, +delete HPAs and pods, and create or update the rollback ConfigMap. +Pass the intended cluster context to both `kubectl` and `helm` before invoking +the helper; it rejects forwarded kubeconfig, context, identity, API-server, and +namespace flags and Helm-specific target environment overrides so the drain and +rollback cannot target different clusters. Termination signals during Helm +also trigger a final recovery drain before the helper exits. + **Execution profile.** By default this chart leaves `CODEAPI_EXECUTION_PROFILE` unset. Its bundled HTTP/stateless configuration is inferred as the AWS-free `default` profile and retains the existing diff --git a/helm/codeapi/scripts/safe-pairing-rollback.sh b/helm/codeapi/scripts/safe-pairing-rollback.sh new file mode 100755 index 00000000..c70bddc7 --- /dev/null +++ b/helm/codeapi/scripts/safe-pairing-rollback.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "usage: $0 RELEASE REVISION [NAMESPACE] [helm rollback flags...]" >&2 + exit 64 +} + +release=${1:-} +revision=${2:-} +namespace=${3:-default} +if [[ -z "$release" || ! "$revision" =~ ^[1-9][0-9]*$ ]]; then + usage +fi +shift $(( $# >= 3 ? 3 : $# )) + +if [[ ! "$release" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "invalid Helm release name: $release" >&2 + exit 64 +fi +if [[ ! "$namespace" =~ ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ]]; then + echo "invalid Kubernetes namespace: $namespace" >&2 + exit 64 +fi +for flag in "$@"; do + case "$flag" in + -n|-n?*|--namespace|--namespace=*|--kube-context|--kube-context=*|\ + --kubeconfig|--kubeconfig=*|--kube-apiserver|--kube-apiserver=*|\ + --kube-ca-file|--kube-ca-file=*|--kube-token|--kube-token=*|\ + --kube-tls-server-name|--kube-tls-server-name=*|\ + --kube-as-user|--kube-as-user=*|--kube-as-group|--kube-as-group=*|\ + --kube-insecure-skip-tls-verify|--kube-insecure-skip-tls-verify=*) + echo "refusing target-changing Helm rollback flag: $flag" >&2 + exit 64 + ;; + esac +done +for variable in \ + HELM_KUBEAPISERVER \ + HELM_KUBEASGROUPS \ + HELM_KUBEASUSER \ + HELM_KUBECAFILE \ + HELM_KUBECONTEXT \ + HELM_KUBEINSECURE_SKIP_TLS_VERIFY \ + HELM_KUBETLS_SERVER_NAME \ + HELM_KUBETOKEN \ + HELM_NAMESPACE; do + if [[ -n ${!variable:-} ]]; then + echo "refusing Helm target override from environment: $variable" >&2 + exit 64 + fi +done + +timeout=${CODEAPI_ROLLBACK_TIMEOUT:-10m} +selector="app.kubernetes.io/instance=${release},app.kubernetes.io/component=api" + +discover_api_deployments() { + local output + output=$(kubectl --namespace "$namespace" get deployment \ + --selector "$selector" --output name) || return + deployments=() + if [[ -n "$output" ]]; then + mapfile -t deployments <<< "$output" + fi +} + +list_api_pods() { + local output + output=$(kubectl --namespace "$namespace" get pod \ + --selector "$selector" --output name) || return + pods=() + if [[ -n "$output" ]]; then + mapfile -t pods <<< "$output" + fi +} + +discover_api_deployments +if (( ${#deployments[@]} != 1 )); then + echo "expected exactly one Code API deployment for $selector" >&2 + exit 1 +fi +deployment=${deployments[0]} + +fence=$(kubectl --namespace "$namespace" get "$deployment" \ + --output 'jsonpath={.spec.template.metadata.annotations.codeapi\.librechat\.ai/pairing-fence-version}') +if [[ -z "$fence" ]]; then + echo "refusing rollback: the live API deployment has no pairing fence" >&2 + exit 1 +fi + +deployment_name=${deployment#*/} +rollback_config_map=${deployment_name%-api}-pairing-rollback +rollback_epoch="$(date +%s)-${RANDOM}-${RANDOM}" + +echo "Recording pairing rollback epoch $rollback_epoch..." >&2 +kubectl --namespace "$namespace" create configmap "$rollback_config_map" \ + --from-literal="epoch=$rollback_epoch" --dry-run=client --output yaml | \ + kubectl --namespace "$namespace" apply --filename - + +drain_api() { + local pod_action=${1:-wait} + local replica_state desired current ready available updated + + # Helm may have partially installed a target with a different fullname. + # Resolve every matching API Deployment on each drain attempt. + discover_api_deployments + if (( ${#deployments[@]} == 0 )) && [[ "$pod_action" != delete ]]; then + echo "refusing rollback: no API deployment matched $selector" >&2 + return 1 + fi + + echo "Deleting API autoscalers before the rollback fence is lowered..." >&2 + kubectl --namespace "$namespace" delete horizontalpodautoscaler \ + --selector "$selector" --ignore-not-found --wait=true + + echo "Scaling the fenced API deployment to zero..." >&2 + for deployment in "${deployments[@]}"; do + kubectl --namespace "$namespace" scale "$deployment" --replicas=0 + kubectl --namespace "$namespace" rollout status "$deployment" \ + --timeout "$timeout" + done + + list_api_pods + if (( ${#pods[@]} > 0 )); then + if [[ "$pod_action" == delete ]]; then + kubectl --namespace "$namespace" delete pod \ + --selector "$selector" --wait=true --timeout "$timeout" + else + kubectl --namespace "$namespace" wait "${pods[@]}" \ + --for=delete --timeout "$timeout" + fi + fi + + # Relist immediately before Helm can lower the fence. This catches a new + # matching pod that appeared after the first snapshot. + discover_api_deployments + for deployment in "${deployments[@]}"; do + replica_state=$(kubectl --namespace "$namespace" get "$deployment" \ + --output 'jsonpath={.spec.replicas},{.status.replicas},{.status.readyReplicas},{.status.availableReplicas},{.status.updatedReplicas}') || return + IFS=, read -r desired current ready available updated <<< "$replica_state" + if [[ ${desired:-0} != 0 || ${current:-0} != 0 || ${ready:-0} != 0 || + ${available:-0} != 0 || ${updated:-0} != 0 ]]; then + echo "refusing rollback: API deployment did not converge to zero replicas" >&2 + return 1 + fi + done + list_api_pods + if (( ${#pods[@]} > 0 )); then + echo "refusing rollback: API pods appeared after the drain" >&2 + return 1 + fi +} + +drain_api wait + +echo "All fenced API pods are gone; starting Helm rollback..." >&2 +rollback_pid= +recover_interrupted_rollback() { + local exit_status=$1 + trap - HUP INT TERM + if [[ -n "$rollback_pid" ]]; then + kill -TERM "$rollback_pid" 2>/dev/null || true + wait "$rollback_pid" 2>/dev/null || true + fi + echo "Helm rollback interrupted; restoring the fail-closed API drain..." >&2 + set -e + drain_api delete + exit "$exit_status" +} +trap 'recover_interrupted_rollback 129' HUP +trap 'recover_interrupted_rollback 130' INT +trap 'recover_interrupted_rollback 143' TERM + +helm rollback "$release" "$revision" \ + --namespace "$namespace" --wait --wait-for-jobs --timeout "$timeout" "$@" & +rollback_pid=$! +set +e +wait "$rollback_pid" +rollback_status=$? +set -e +rollback_pid= +trap - HUP INT TERM + +if (( rollback_status == 0 )); then + exit 0 +else + echo "Helm rollback failed; restoring the fail-closed API drain..." >&2 + drain_api delete + exit "$rollback_status" +fi diff --git a/helm/codeapi/templates/api-deployment.yaml b/helm/codeapi/templates/api-deployment.yaml index bf5f91e0..69985db4 100644 --- a/helm/codeapi/templates/api-deployment.yaml +++ b/helm/codeapi/templates/api-deployment.yaml @@ -17,11 +17,15 @@ spec: {{- if not .Values.api.autoscaling.enabled }} replicas: {{ .Values.api.replicaCount }} {{- end }} + strategy: + {{- toYaml .Values.api.strategy | nindent 4 }} selector: matchLabels: {{- include "codeapi.api.selectorLabels" . | nindent 6 }} template: metadata: + annotations: + codeapi.librechat.ai/pairing-fence-version: "1" labels: {{- include "codeapi.api.selectorLabels" . | nindent 8 }} spec: @@ -55,6 +59,12 @@ spec: secretKeyRef: name: {{ include "codeapi.fullname" . }}-secrets key: redis-password + - name: CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH + valueFrom: + configMapKeyRef: + name: {{ include "codeapi.fullname" . }}-pairing-rollback + key: epoch + optional: true # Service URLs - name: FILE_SERVER_URL value: "http://{{ include "codeapi.fullname" . }}-file-server:{{ .Values.fileServer.service.port }}" diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 75abaaf7..ee997ce8 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -68,10 +68,20 @@ api: enabled: true replicaCount: 2 # Start with 2 API pods + # Pairing revocation relies on every serving replica honoring the Redis + # generation fence. Recreate prevents a pre-fence binary from redeeming an + # already-revoked code during the first rollout of paired bridge workers. + # Roll back to pre-fence revisions only with scripts/safe-pairing-rollback.sh. + strategy: + type: Recreate + rollingUpdate: null + image: repository: codeapi-api tag: latest - pullPolicy: IfNotPresent # Use Always in production + # Recreate is a security fence only if replacement pods cannot reuse a + # cached pre-fence image behind the mutable default tag. + pullPolicy: Always # Resource limits resources: diff --git a/service/Dockerfile.api b/service/Dockerfile.api index f1fdf9c8..2921401e 100644 --- a/service/Dockerfile.api +++ b/service/Dockerfile.api @@ -21,6 +21,7 @@ COPY service/src ./src COPY packages/code/src /packages/code/src COPY service/scripts ./scripts COPY shared /shared +COPY packages/code /packages/code COPY service/tsconfig.json ./ RUN bun build ./src/api-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' RUN bun build ./scripts/rehydrate-session-cache.ts --minify --outdir .build-migrations --target bun --external '@opentelemetry/*' @@ -49,6 +50,7 @@ COPY --from=install /temp/dev/node_modules node_modules COPY service/src ./src COPY packages/code/src /packages/code/src COPY shared /shared +COPY packages/code /packages/code COPY service/tsconfig.json ./ EXPOSE 3112 9230 CMD ["bun", "run", "--watch", "src/api-server.ts"] diff --git a/service/src/bridge/pairing.test.ts b/service/src/bridge/pairing.test.ts index 310ad7d9..e709a4a7 100644 --- a/service/src/bridge/pairing.test.ts +++ b/service/src/bridge/pairing.test.ts @@ -63,7 +63,7 @@ describe('RedisBridgePairingStore', () => { originalAuthorization.identityId, ); await expect( - pairings.authorize(requestFor(issued.credential, 'overlap-bound-proof')), + pairings.authorize(requestFor(issued.credential, 'superseded-bound-proof')), ).resolves.toMatchObject({ workerId: 'vm-bound', identityId: originalAuthorization.identityId, @@ -127,6 +127,27 @@ describe('RedisBridgePairingStore', () => { ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); }); + test('preserves a pairing code after public-key validation fails', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-public-key-retry'); + + await expect( + pairings.redeem({ + workerId: 'vm-public-key-retry', + code: pairing.code, + publicKey: 'not-a-public-key', + }), + ).rejects.toMatchObject({ code: 'PUBLIC_KEY_INVALID' }); + + await expect( + pairings.redeem({ + workerId: 'vm-public-key-retry', + code: pairing.code, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-public-key-retry' }); + }); + test('only the newest pairing code can rebind a worker identity', async () => { const identity = createBridgeIdentity(); const older = await pairings.issue('vm-1', { @@ -172,16 +193,15 @@ describe('RedisBridgePairingStore', () => { }); let paused = false; redis.eval = (async (script: string, ...args: unknown[]) => { - const result = await (originalEval as (...evalArgs: unknown[]) => Promise)( - script, - ...args, - ); - if (!paused && script.includes('return pairing')) { + if (!paused && script.includes('if pairing ~= ARGV[1]')) { paused = true; firstRedeemed(); await releaseFirstPromise; } - return result; + return await (originalEval as (...evalArgs: unknown[]) => Promise)( + script, + ...args, + ); }) as typeof redis.eval; try { @@ -338,7 +358,556 @@ describe('RedisBridgePairingStore', () => { ).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); }); - test('rotation keeps the prior same-identity credential usable for recovery', async () => { + test('revocation invalidates an unredeemed pairing code', async () => { + const identity = createBridgeIdentity(); + const pairing = await pairings.issue('vm-1'); + + await pairings.revoke('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: pairing.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + }); + + test('revocation removes pairing codes issued by a pre-fence replica', async () => { + const legacyCode = 'legacy-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ workerId: 'vm-1', expiresAt: new Date(Date.now() + 60_000).toISOString() }), + 'EX', + 60, + ); + + await pairings.revoke('vm-1'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('retries legacy cleanup after a transient scan failure', async () => { + const legacyCode = 'retryable-legacy-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-scan-retry', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const scan = redis.scan.bind(redis); + let failScan = true; + redis.scan = (async (...args: Parameters) => { + if (failScan) { + failScan = false; + throw new Error('transient scan failure'); + } + return scan(...args); + }) as Redis['scan']; + + await expect(pairings.revoke('vm-scan-retry')).rejects.toThrow( + 'transient scan failure', + ); + redis.scan = scan; + await pairings.revoke('vm-scan-retry'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('retries a claimed cleanup after the migration deadline', async () => { + const legacyCode = 'post-deadline-retry-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-post-deadline-retry', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:migration:legacy-pairing-scan-until', + String(Date.now() + 15), + ); + const scan = redis.scan.bind(redis); + let failScan = true; + redis.scan = (async (...args: Parameters) => { + if (failScan) { + failScan = false; + await new Promise((resolve) => setTimeout(resolve, 30)); + throw new Error('scan failed after deadline'); + } + return scan(...args); + }) as Redis['scan']; + + await expect(pairings.revoke('vm-post-deadline-retry')).rejects.toThrow( + 'scan failed after deadline', + ); + await pairings.revoke('vm-post-deadline-retry'); + + redis.scan = scan; + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('renews the cleanup claim while a shared-keyspace scan is in flight', async () => { + const store = new RedisBridgePairingStore(redis, 600, 300, 30); + const legacyCode = 'renewed-claim-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-renewed-claim', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const scan = redis.scan.bind(redis); + let scanCalls = 0; + let renewCalls = 0; + let markScanStarted = () => {}; + const scanStarted = new Promise((resolve) => { + markScanStarted = resolve; + }); + redis.scan = (async (...args: Parameters) => { + scanCalls += 1; + if (scanCalls === 1) { + markScanStarted(); + await new Promise((resolve) => setTimeout(resolve, 80)); + } + return scan(...args); + }) as Redis['scan']; + const originalEval = redis.eval.bind(redis); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes("redis.call('PEXPIRE'")) renewCalls += 1; + return await (originalEval as (...evalArgs: unknown[]) => Promise)( + script, + ...args, + ); + }) as typeof redis.eval; + + try { + const first = store.revoke('vm-renewed-claim'); + await scanStarted; + await new Promise((resolve) => setTimeout(resolve, 45)); + expect(renewCalls).toBeGreaterThan(0); + await first; + + expect(scanCalls).toBe(1); + await expect(redis.get(legacyKey)).resolves.toBeNull(); + } finally { + redis.scan = scan; + redis.eval = originalEval as typeof redis.eval; + } + }); + + test('rescans an ambiguous marker written by the preceding build', async () => { + const legacyCode = 'ambiguous-predecessor-marker-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const workerId = 'vm-ambiguous-predecessor-marker'; + await redis.set( + legacyKey, + JSON.stringify({ + workerId, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + `codeapi:bridge:v1:migration:legacy-pairing-scanned:${workerId}`, + 'predecessor-random-token', + 'PX', + 60_000, + ); + + await pairings.revoke(workerId); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('starts the advertised pairing lifetime after legacy cleanup', async () => { + const store = new RedisBridgePairingStore(redis, 60); + const scan = redis.scan.bind(redis); + const originalNow = Date.now; + let now = originalNow(); + Date.now = () => now; + redis.scan = (async (...args: Parameters) => { + const result = await scan(...args); + now += 10; + return result; + }) as Redis['scan']; + + try { + const pairing = await store.issue('vm-post-cleanup-expiry'); + expect(Date.parse(pairing.expiresAt) - Date.now()).toBe(60_000); + } finally { + Date.now = originalNow; + redis.scan = scan; + } + }); + + test('waits for a failed in-progress cleanup and confirms legacy removal itself', async () => { + const legacyCode = 'overlapping-legacy-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-overlapping-cleanup', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const scan = redis.scan.bind(redis); + let releaseFirstScan = () => {}; + const firstScanGate = new Promise((resolve) => { + releaseFirstScan = resolve; + }); + let markFirstScanStarted = () => {}; + const firstScanStarted = new Promise((resolve) => { + markFirstScanStarted = resolve; + }); + let scanCalls = 0; + redis.scan = (async (...args: Parameters) => { + scanCalls += 1; + if (scanCalls === 1) { + markFirstScanStarted(); + await firstScanGate; + throw new Error('interrupted claimed scan'); + } + return scan(...args); + }) as Redis['scan']; + + const interrupted = pairings.revoke('vm-overlapping-cleanup'); + await firstScanStarted; + const overlapping = pairings.revoke('vm-overlapping-cleanup'); + let overlappingSettled = false; + void overlapping.then( + () => { + overlappingSettled = true; + }, + () => { + overlappingSettled = true; + }, + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(overlappingSettled).toBe(false); + releaseFirstScan(); + + await expect(interrupted).rejects.toThrow('interrupted claimed scan'); + await expect(overlapping).resolves.toBeUndefined(); + redis.scan = scan; + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('legacy cleanup does not delete generation-fenced pairings', async () => { + const fencedCode = 'concurrent-generation-pairing'; + const fencedKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(fencedCode) + .digest('hex')}`; + await redis.set( + fencedKey, + JSON.stringify({ + workerId: 'vm-generation-fenced', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + generation: 'new-generation', + }), + 'EX', + 60, + ); + + await pairings.revoke('vm-generation-fenced'); + + await expect(redis.get(fencedKey)).resolves.not.toBeNull(); + }); + + test('reopens legacy cleanup after a rollback outlives the prior scan window', async () => { + const legacyCode = 'later-rollback-pairing-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const deadlineKey = 'codeapi:bridge:v1:migration:legacy-pairing-scan-until'; + await redis.set(deadlineKey, '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-later-rollback', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:pairing-index:vm-later-rollback', + legacyKey, + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:migration:legacy-pairing-scanned:vm-later-rollback', + 'done', + ); + + await pairings.revoke('vm-later-rollback'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + await expect(redis.get(deadlineKey)).resolves.toBe('0'); + }); + + test('does not restart an expired migration window without rollback evidence', async () => { + const legacyCode = 'unindexed-post-migration-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const deadlineKey = 'codeapi:bridge:v1:migration:legacy-pairing-scan-until'; + await redis.set(deadlineKey, '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-no-rollback-signal', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + + await pairings.revoke('vm-no-rollback-signal'); + + await expect(redis.get(legacyKey)).resolves.not.toBeNull(); + await expect(redis.get(deadlineKey)).resolves.toBe('0'); + }); + + test('restarts legacy cleanup once for an explicit rollback epoch', async () => { + const legacyCode = 'unindexed-rollback-epoch-code'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + const deadlineKey = 'codeapi:bridge:v1:migration:legacy-pairing-scan-until'; + await redis.set(deadlineKey, '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-rollback-epoch', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:migration:legacy-pairing-scanned:vm-rollback-epoch', + 'done', + ); + const rollbackAwarePairings = new RedisBridgePairingStore( + redis, + 600, + 300, + 5_000, + 'rollback-epoch-1', + ); + + await rollbackAwarePairings.revoke('vm-rollback-epoch'); + + await expect(redis.get(legacyKey)).resolves.toBeNull(); + await expect(redis.get(deadlineKey)).resolves.toBe('0'); + const epochHash = createHash('sha256') + .update('rollback-epoch-1') + .digest('hex'); + const epochStateKey = + `codeapi:bridge:v1:migration:legacy-pairing-scanned:` + + `vm-rollback-epoch:${epochHash}`; + await expect(redis.get(epochStateKey)).resolves.toBe('done'); + await expect(redis.pttl(epochStateKey)).resolves.toBe(-1); + }); + + test('cleans a revoked legacy code before rollback-epoch redemption', async () => { + const identity = createBridgeIdentity(); + const workerId = 'vm-rollback-redeem'; + const legacyCode = 'revoked-rollback-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set('codeapi:bridge:v1:migration:legacy-pairing-scan-until', '0'); + await redis.set( + legacyKey, + JSON.stringify({ + workerId, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + const rollbackAwarePairings = new RedisBridgePairingStore( + redis, + 600, + 300, + 5_000, + 'rollback-epoch-redeem', + ); + + await expect( + rollbackAwarePairings.redeem({ + workerId, + code: legacyCode, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + await expect(redis.get(legacyKey)).resolves.toBeNull(); + }); + + test('does not scan for a nonexistent rollback-epoch pairing code', async () => { + const identity = createBridgeIdentity(); + const scan = redis.scan.bind(redis); + let scanCalls = 0; + redis.scan = (async (...args: Parameters) => { + scanCalls += 1; + return scan(...args); + }) as Redis['scan']; + const rollbackAwarePairings = new RedisBridgePairingStore( + redis, + 600, + 300, + 5_000, + 'rollback-epoch-missing-code', + ); + + try { + await expect( + rollbackAwarePairings.redeem({ + workerId: 'attacker-chosen-worker', + code: 'nonexistent-code', + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + expect(scanCalls).toBe(0); + await expect( + redis.keys( + 'codeapi:bridge:v1:migration:legacy-pairing-scanned:attacker-chosen-worker*', + ), + ).resolves.toEqual([]); + } finally { + redis.scan = scan; + } + }); + + test('redeems an unrevoked pairing code issued by a pre-fence replica', async () => { + const identity = createBridgeIdentity(); + const legacyCode = 'unrevoked-legacy-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ workerId: 'vm-1', expiresAt: new Date(Date.now() + 60_000).toISOString() }), + 'EX', + 60, + ); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: legacyCode, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('redeems an indexed legacy code issued after rollback', async () => { + const identity = createBridgeIdentity(); + await pairings.issue('vm-rollback'); + const legacyCode = 'rollback-issued-legacy-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ + workerId: 'vm-rollback', + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }), + 'EX', + 60, + ); + await redis.set( + 'codeapi:bridge:v1:pairing-index:vm-rollback', + legacyKey, + 'EX', + 60, + ); + + await expect( + pairings.redeem({ + workerId: 'vm-rollback', + code: legacyCode, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-rollback' }); + }); + + test('replacement invalidates a pairing code issued by a pre-fence replica', async () => { + const identity = createBridgeIdentity(); + const legacyCode = 'replaced-legacy-pairing'; + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256') + .update(legacyCode) + .digest('hex')}`; + await redis.set( + legacyKey, + JSON.stringify({ workerId: 'vm-1', expiresAt: new Date(Date.now() + 60_000).toISOString() }), + 'EX', + 60, + ); + + await pairings.issue('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: legacyCode, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + }); + + test('issuing a replacement invalidates the prior unredeemed pairing code', async () => { + const identity = createBridgeIdentity(); + const first = await pairings.issue('vm-1'); + const replacement = await pairings.issue('vm-1'); + + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: first.code, + publicKey: identity.publicKey, + }), + ).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + await expect( + pairings.redeem({ + workerId: 'vm-1', + code: replacement.code, + publicKey: identity.publicKey, + }), + ).resolves.toMatchObject({ workerId: 'vm-1' }); + }); + + test('rotation retains the prior same-identity credential for recovery', async () => { const identity = createBridgeIdentity(); const pairing = await pairings.issue('vm-1'); const original = await pairings.redeem({ diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index fd1e8cff..59dfbe60 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -13,6 +13,44 @@ const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; const PROOF_NONCE_TTL_SECONDS = 2 * 60; const PROOF_CLOCK_SKEW_MS = 60_000; +const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; +const LEGACY_SCAN_POLL_INTERVAL_MS = 25; +const LEGACY_SCAN_PENDING = 'pending'; +const LEGACY_SCAN_COMPLETE = 'done'; +const ISSUE_PAIRING_SCRIPT = ` +local previous = redis.call('GET', KEYS[1]) +if previous then + redis.call('DEL', previous) +end +redis.call('SET', KEYS[1], KEYS[3], 'EX', ARGV[3]) +redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[3]) +redis.call('SET', KEYS[3], ARGV[2], 'EX', ARGV[3]) +return 1 +`; +const REDEEM_PAIRING_SCRIPT = ` +local pairing = redis.call('GET', KEYS[1]) +if pairing ~= ARGV[1] then + return 0 +end +local generation = redis.call('GET', KEYS[2]) +if ARGV[2] == '' then + if generation and redis.call('GET', KEYS[5]) ~= KEYS[1] then + redis.call('DEL', KEYS[1]) + return 0 + end +elseif generation ~= ARGV[2] then + redis.call('DEL', KEYS[1]) + return 0 +end +redis.call('DEL', KEYS[1]) +if redis.call('GET', KEYS[5]) == KEYS[1] then + redis.call('DEL', KEYS[5]) +end +redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) +redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) +redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[4]) +return 1 +`; const ROTATE_CREDENTIAL_SCRIPT = ` local activeDigest = redis.call('GET', KEYS[1]) local previous = redis.call('GET', KEYS[2]) @@ -33,42 +71,54 @@ else end return 1 `; -const ISSUE_PAIRING_SCRIPT = ` -local previous = redis.call('GET', KEYS[1]) -if previous then - redis.call('DEL', previous) -end -redis.call('DEL', KEYS[3]) -redis.call('SET', KEYS[1], KEYS[2], 'EX', ARGV[2]) +const REVOKE_PAIRING_SCRIPT = ` +local indexed = redis.call('GET', KEYS[1]) +local credential = redis.call('GET', KEYS[3]) redis.call('SET', KEYS[2], ARGV[1], 'EX', ARGV[2]) +if indexed then + redis.call('DEL', indexed) +end +redis.call('DEL', KEYS[1]) +if credential then + redis.call('DEL', KEYS[3]) + redis.call('DEL', ARGV[3] .. credential) +end +redis.call('DEL', KEYS[4]) return 1 `; -const REDEEM_PAIRING_SCRIPT = ` -local pairing = redis.call('GET', KEYS[1]) -if not pairing then - return nil +const RELEASE_LEGACY_SCAN_CLAIM_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) end -if redis.call('GET', KEYS[2]) ~= KEYS[1] then - redis.call('DEL', KEYS[1]) - return nil +return 0 +`; +const NORMALIZE_LEGACY_SCAN_STATE_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + redis.call('SET', KEYS[1], ARGV[2]) + return 1 end -redis.call('DEL', KEYS[1], KEYS[2]) -redis.call('SET', KEYS[3], ARGV[1], 'EX', ARGV[2]) -return pairing +return 0 `; -const INSTALL_REDEEMED_CREDENTIAL_SCRIPT = ` -if redis.call('GET', KEYS[1]) ~= ARGV[1] then - return 0 +const RENEW_LEGACY_SCAN_CLAIM_SCRIPT = ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('PEXPIRE', KEYS[1], ARGV[2]) end -redis.call('SET', KEYS[2], ARGV[3], 'EX', ARGV[4]) -redis.call('SET', KEYS[3], ARGV[2], 'EX', ARGV[4]) -if ARGV[5] ~= '' then - redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) -else - redis.call('DEL', KEYS[4]) +return 0 +`; +const COMPLETE_LEGACY_SCAN_CLAIM_SCRIPT = ` +if redis.call('GET', KEYS[2]) == ARGV[1] then + local remaining = tonumber(ARGV[3]) + if ARGV[4] == '1' then + redis.call('SET', KEYS[1], ARGV[2]) + elseif remaining > 0 then + redis.call('SET', KEYS[1], ARGV[2], 'PX', remaining) + else + redis.call('DEL', KEYS[1]) + end + redis.call('DEL', KEYS[2]) + return 1 end -redis.call('DEL', KEYS[1]) -return 1 +return 0 `; export type BridgePrincipalType = 'deployment' | 'tenant' | 'user' | 'role' | 'group'; @@ -84,11 +134,13 @@ export interface BridgeWorkerBinding { interface StoredPairing { workerId: string; expiresAt: string; + generation?: string; binding?: BridgeWorkerBinding; } interface StoredCredential { workerId: string; + /** Stable across refreshes; replaced only when the worker is paired again. */ identityId?: string; publicKey: string; expiresAt: string; @@ -146,8 +198,21 @@ function workerPairingIndexKey(workerId: string): string { return `${PREFIX}:pairing-index:${workerId}`; } -function workerRedemptionKey(workerId: string): string { - return `${PREFIX}:redemption:${workerId}`; +function workerPairingGenerationKey(workerId: string): string { + return `${PREFIX}:pairing-generation:${workerId}`; +} + +function legacyPairingScanDeadlineKey(): string { + return `${PREFIX}:migration:legacy-pairing-scan-until`; +} + +function legacyPairingWorkerScanKey( + workerId: string, + rollbackEpoch?: string, +): string { + const epoch = rollbackEpoch?.trim(); + const epochSuffix = epoch ? `:${digest(epoch)}` : ''; + return `${PREFIX}:migration:legacy-pairing-scanned:${workerId}${epochSuffix}`; } function proofNonceKey(credential: string, nonce: string): string { @@ -167,24 +232,38 @@ export class RedisBridgePairingStore { private readonly redis: Redis, private readonly pairingTtlSeconds = DEFAULT_PAIRING_TTL_SECONDS, private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, + private readonly legacyScanClaimTtlMs = LEGACY_SCAN_CLAIM_TTL_MS, + private readonly rollbackEpoch = + process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', ) {} async issue( workerId: string, binding?: BridgeWorkerBinding, ): Promise { + // Pre-index binaries cannot remove a superseded code themselves. During + // the one pairing-TTL migration window, find and delete those records so + // rolling back cannot make a replaced code valid again. + await this.removeLegacyPairings(workerId); const code = randomBytes(24).toString('base64url'); + const generation = randomBytes(24).toString('base64url'); const expiresAt = new Date( Date.now() + this.pairingTtlSeconds * 1000, ).toISOString(); - const pairing: StoredPairing = { workerId, expiresAt, binding }; + const pairing: StoredPairing = { + workerId, + expiresAt, + generation, + binding, + }; const codeKey = pairingKey(code); await this.redis.eval( ISSUE_PAIRING_SCRIPT, 3, workerPairingIndexKey(workerId), + workerPairingGenerationKey(workerId), codeKey, - workerRedemptionKey(workerId), + generation, JSON.stringify(pairing), String(this.pairingTtlSeconds), ); @@ -196,44 +275,78 @@ export class RedisBridgePairingStore { code: string; publicKey: string; }): Promise { + const codeKey = pairingKey(args.code); + const raw = await this.redis.get(codeKey); + if (raw == null) { + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code is invalid or expired', + ); + } + const pairing = JSON.parse(raw) as StoredPairing; + if (pairing.workerId !== args.workerId) { + await this.redis.del(codeKey); + throw new BridgePairingError( + 'PAIRING_INVALID', + 'Pairing code does not authorize this worker', + ); + } if (!validEd25519PublicKey(args.publicKey)) { throw new BridgePairingError( 'PUBLIC_KEY_INVALID', 'Worker public key must be an Ed25519 key', ); } - const codeKey = pairingKey(args.code); - const redemptionId = randomBytes(18).toString('base64url'); - const raw = await this.redis.eval( - REDEEM_PAIRING_SCRIPT, - 3, - codeKey, - workerPairingIndexKey(args.workerId), - workerRedemptionKey(args.workerId), - redemptionId, - String(this.pairingTtlSeconds), - ); - if (typeof raw !== 'string') { + // Validate the supplied code before it can trigger a shared-keyspace scan. + // A rollback epoch means any generation-less code may have survived a + // legacy revoke, so clean the authenticated worker and reject that code. + if ( + pairing.generation == null && + this.rollbackEpoch.trim().length > 0 + ) { + await this.removeLegacyPairings(args.workerId); throw new BridgePairingError( 'PAIRING_INVALID', 'Pairing code is invalid or expired', ); } - const pairing = JSON.parse(raw) as StoredPairing; - if (pairing.workerId !== args.workerId) { + + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date( + Date.now() + this.credentialTtlSeconds * 1000, + ).toISOString(); + const identityId = randomBytes(18).toString('base64url'); + const stored: StoredCredential = { + workerId: args.workerId, + identityId, + publicKey: args.publicKey, + expiresAt, + binding: pairing.binding, + }; + const accepted = await this.redis.eval( + REDEEM_PAIRING_SCRIPT, + 6, + codeKey, + workerPairingGenerationKey(pairing.workerId), + credentialDigestKey(credentialDigest), + workerIdentityKey(args.workerId), + workerPairingIndexKey(args.workerId), + workerStableIdentityKey(args.workerId), + raw, + pairing.generation ?? '', + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + identityId, + ); + if (accepted !== 1) { throw new BridgePairingError( 'PAIRING_INVALID', - 'Pairing code does not authorize this worker', + 'Pairing code is invalid or expired', ); } - return await this.issueCredential( - args.workerId, - args.publicKey, - undefined, - undefined, - pairing.binding, - redemptionId, - ); + return { workerId: args.workerId, credential, expiresAt }; } async authorize(args: { @@ -327,16 +440,198 @@ export class RedisBridgePairingStore { } async revoke(workerId: string): Promise { - const identityKey = workerIdentityKey(workerId); - const credentialDigest = await this.redis.get(identityKey); - if (credentialDigest == null) return; - await this.redis.del( - identityKey, + await this.removeLegacyPairings(workerId); + // Fence redemption and consume the currently indexed code atomically. An + // issue that linearized before this script is always removed; an issue + // that linearizes afterward installs a distinct generation and code. + await this.redis.eval( + REVOKE_PAIRING_SCRIPT, + 4, + workerPairingIndexKey(workerId), + workerPairingGenerationKey(workerId), + workerIdentityKey(workerId), workerStableIdentityKey(workerId), - credentialDigestKey(credentialDigest), + randomBytes(24).toString('base64url'), + String(this.pairingTtlSeconds), + `${PREFIX}:credential:`, ); } + private async removeLegacyPairings(workerId: string): Promise { + const deadlineKey = legacyPairingScanDeadlineKey(); + const now = Date.now(); + const migrationWindowMs = this.pairingTtlSeconds * 1000; + const proposedDeadline = now + migrationWindowMs; + let rawDeadline = await this.redis.get(deadlineKey); + if (rawDeadline == null) { + const initialized = await this.redis.set( + deadlineKey, + String(proposedDeadline), + 'NX', + ); + rawDeadline = initialized === 'OK' + ? String(proposedDeadline) + : await this.redis.get(deadlineKey); + } else if ((await this.redis.pttl(deadlineKey)) > 0) { + // Markers from the preceding build expired and reopened forever. Keep + // their original deadline, but make it durable so normal idle periods + // cannot start another migration window. + await this.redis.persist(deadlineKey); + } + + const indexedKey = await this.redis.get(workerPairingIndexKey(workerId)); + const indexedRaw = indexedKey == null ? null : await this.redis.get(indexedKey); + let rollbackDetected = false; + if (indexedRaw != null) { + try { + rollbackDetected = (JSON.parse(indexedRaw) as StoredPairing).generation == null; + } catch { + rollbackDetected = false; + } + } + + const deadline = Number(rawDeadline); + const stateKey = legacyPairingWorkerScanKey(workerId, this.rollbackEpoch); + const rollbackEpochDetected = this.rollbackEpoch.trim().length > 0; + while (true) { + const state = await this.redis.get(stateKey); + if (state === LEGACY_SCAN_COMPLETE && !rollbackDetected) return; + if (state === LEGACY_SCAN_PENDING) break; + if (state == null) { + if ( + !rollbackDetected && + !rollbackEpochDetected && + (!Number.isFinite(deadline) || Date.now() > deadline) + ) { + return; + } + const initialized = await this.redis.set( + stateKey, + LEGACY_SCAN_PENDING, + 'NX', + ); + if (initialized === 'OK') break; + continue; + } + // Predecessor builds stored an unqualified random token before scanning. + // It cannot prove whether that scan completed, so normalize it to a + // durable retry requirement instead of treating it as success. + const normalized = await this.redis.eval( + NORMALIZE_LEGACY_SCAN_STATE_SCRIPT, + 1, + stateKey, + state, + LEGACY_SCAN_PENDING, + ); + if (normalized === 1) break; + } + + const claimKey = `${stateKey}:claim`; + let scanClaim: { key: string; token: string } | undefined; + while (scanClaim == null) { + const state = await this.redis.get(stateKey); + if (state === LEGACY_SCAN_COMPLETE || state == null) return; + const token = `claim:${randomBytes(24).toString('base64url')}`; + const claimed = await this.redis.set( + claimKey, + token, + 'PX', + Math.max(1, this.legacyScanClaimTtlMs), + 'NX', + ); + if (claimed === 'OK') { + scanClaim = { key: claimKey, token }; + break; + } + await new Promise((resolve) => + setTimeout(resolve, LEGACY_SCAN_POLL_INTERVAL_MS), + ); + } + + let renewalError: unknown; + let renewal = Promise.resolve(); + let renewalInFlight = false; + const renewClaim = async (): Promise => { + const renewed = await this.redis.eval( + RENEW_LEGACY_SCAN_CLAIM_SCRIPT, + 1, + scanClaim.key, + scanClaim.token, + String(Math.max(1, this.legacyScanClaimTtlMs)), + ); + if (renewed !== 1) { + throw new Error('Legacy pairing cleanup claim was lost'); + } + }; + const renewalTimer = setInterval(() => { + if (renewalInFlight || renewalError != null) return; + renewalInFlight = true; + renewal = renewClaim() + .catch((error: unknown) => { + renewalError = error; + }) + .finally(() => { + renewalInFlight = false; + }); + }, Math.max(1, Math.floor(this.legacyScanClaimTtlMs / 3))); + renewalTimer.unref?.(); + + try { + let cursor = '0'; + do { + const [nextCursor, keys] = await this.redis.scan( + cursor, + 'MATCH', + `${PREFIX}:pairing:*`, + 'COUNT', + 100, + ); + if (renewalError != null) throw renewalError; + cursor = nextCursor; + if (keys.length === 0) continue; + const values = await this.redis.mget(...keys); + const matching = keys.filter((_key, index) => { + const raw = values[index]; + if (raw == null) return false; + try { + const pairing = JSON.parse(raw) as Partial; + return pairing.workerId === workerId && pairing.generation == null; + } catch { + return false; + } + }); + if (matching.length > 0) await this.redis.del(...matching); + } while (cursor !== '0'); + clearInterval(renewalTimer); + await renewal; + if (renewalError != null) throw renewalError; + await renewClaim(); + const completed = await this.redis.eval( + COMPLETE_LEGACY_SCAN_CLAIM_SCRIPT, + 2, + stateKey, + scanClaim.key, + scanClaim.token, + LEGACY_SCAN_COMPLETE, + String(deadline - Date.now()), + rollbackEpochDetected ? '1' : '0', + ); + if (completed !== 1) { + await this.removeLegacyPairings(workerId); + } + } catch (error) { + clearInterval(renewalTimer); + await renewal; + await this.redis.eval( + RELEASE_LEGACY_SCAN_CLAIM_SCRIPT, + 1, + scanClaim.key, + scanClaim.token, + ); + throw error; + } + } + async rotate( workerId: string, expectedCredentialId?: string, @@ -359,8 +654,8 @@ export class RedisBridgePairingStore { workerId, previous.publicKey, previousDigest, - previous.identityId ?? null, previous.binding, + previous.identityId ?? null, ); } @@ -368,16 +663,18 @@ export class RedisBridgePairingStore { workerId: string, publicKey: string, previousDigest?: string, - identityId: string | null | undefined = randomBytes(18).toString('base64url'), binding?: BridgeWorkerBinding, - redemptionId?: string, + identityId?: string | null, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); const expiresAt = new Date( Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); - const stableIdentityId = identityId ?? undefined; + const stableIdentityId = + identityId === undefined + ? randomBytes(18).toString('base64url') + : identityId ?? undefined; const stored: StoredCredential = { workerId, ...(stableIdentityId != null ? { identityId: stableIdentityId } : {}), @@ -405,32 +702,31 @@ export class RedisBridgePairingStore { 'Worker credential is invalid or expired', ); } - return { workerId, credential, expiresAt }; - } - if (redemptionId == null) { - throw new BridgePairingError( - 'PAIRING_INVALID', - 'Pairing redemption was not fenced', + } else { + const transaction = this.redis.multi(); + transaction.set( + credentialDigestKey(credentialDigest), + JSON.stringify(stored), + 'EX', + this.credentialTtlSeconds, ); - } - const installed = await this.redis.eval( - INSTALL_REDEEMED_CREDENTIAL_SCRIPT, - 4, - workerRedemptionKey(workerId), - credentialDigestKey(credentialDigest), - workerIdentityKey(workerId), - workerStableIdentityKey(workerId), - redemptionId, - credentialDigest, - JSON.stringify(stored), - String(this.credentialTtlSeconds), - stableIdentityId ?? '', - ); - if (installed !== 1) { - throw new BridgePairingError( - 'PAIRING_INVALID', - 'Pairing code was superseded before credential installation', + transaction.set( + workerIdentityKey(workerId), + credentialDigest, + 'EX', + this.credentialTtlSeconds, ); + if (stableIdentityId != null) { + transaction.set( + workerStableIdentityKey(workerId), + stableIdentityId, + 'EX', + this.credentialTtlSeconds, + ); + } else { + transaction.del(workerStableIdentityKey(workerId)); + } + await transaction.exec(); } return { workerId, credential, expiresAt }; } diff --git a/tests/bridge_pairing_rollout.sh b/tests/bridge_pairing_rollout.sh new file mode 100755 index 00000000..c4246db6 --- /dev/null +++ b/tests/bridge_pairing_rollout.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -euo pipefail + +values=helm/codeapi/values.yaml +deployment=helm/codeapi/templates/api-deployment.yaml +rollback=helm/codeapi/scripts/safe-pairing-rollback.sh + +if ! grep -A 12 '^api:$' "$values" | grep -q '^ strategy:$'; then + echo 'api.strategy must be configured for pairing-safe rollouts' >&2 + exit 1 +fi +if ! grep -A 2 '^ strategy:$' "$values" | grep -q '^ type: Recreate$'; then + echo 'api.strategy.type must default to Recreate while pre-fence replicas may exist' >&2 + exit 1 +fi +if ! grep -A 3 '^ strategy:$' "$values" | grep -q '^ rollingUpdate: null$'; then + echo 'api.strategy must clear rollingUpdate when switching existing deployments to Recreate' >&2 + exit 1 +fi +if ! grep -q 'toYaml .Values.api.strategy' "$deployment"; then + echo 'the API Deployment must render api.strategy' >&2 + exit 1 +fi +if ! grep -q 'codeapi.librechat.ai/pairing-fence-version: "1"' "$deployment"; then + echo 'the first pairing-fence chart upgrade must revise the API pod template' >&2 + exit 1 +fi +if ! grep -A 8 '^ image:$' "$values" | grep -q '^ pullPolicy: Always$'; then + echo 'the fenced API rollout must pull the current image even when the default tag is mutable' >&2 + exit 1 +fi +if [[ ! -x "$rollback" ]]; then + echo 'the pairing-safe rollback helper must be executable' >&2 + exit 1 +fi +bash -n "$rollback" +if "$rollback" codeapi 1 default --kube-context other >/dev/null 2>&1; then + echo 'rollback must reject a Helm context that differs from the kubectl drain' >&2 + exit 1 +fi +if "$rollback" codeapi 1 default --kubeconfig=/tmp/other >/dev/null 2>&1; then + echo 'rollback must reject a Helm kubeconfig that differs from the kubectl drain' >&2 + exit 1 +fi +if HELM_KUBECONTEXT=other "$rollback" codeapi 1 default >/dev/null 2>&1; then + echo 'rollback must reject a Helm context inherited from the environment' >&2 + exit 1 +fi +if ! grep -q 'delete horizontalpodautoscaler' "$rollback" || + ! grep -q 'scale "$deployment" --replicas=0' "$rollback" || + ! grep -q -- '--for=delete' "$rollback" || + ! grep -q 'create configmap "$rollback_config_map"' "$rollback" || + ! grep -q 'replica_state=' "$rollback" || + ! grep -q 'discover_api_deployments' "$rollback" || + ! grep -q 'list_api_pods' "$rollback" || + ! grep -q '^ drain_api delete$' "$rollback" || + ! grep -q 'recover_interrupted_rollback' "$rollback" || + ! grep -q 'helm rollback' "$rollback"; then + echo 'rollback must record an epoch, remove autoscaling, verify the drain, and fail closed' >&2 + exit 1 +fi +if ! grep -q 'CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH' "$deployment" || + ! grep -q 'optional: true' "$deployment"; then + echo 'the API Deployment must consume the optional rollback epoch' >&2 + exit 1 +fi