From dca0e1d31b4d54210e9550e92505fe76e853d4a5 Mon Sep 17 00:00:00 2001 From: abs2023 <93659489+abs2023@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:27:54 -0400 Subject: [PATCH] fix(market-maker): fail verify unless the running image is the one just built /health now reports imageTag. The verify job compares the running task image and that tag to the release, so a rollback can no longer pass as a stable deploy. Co-authored-by: Cursor --- .github/workflows/deploy-col-mar-mm.yml | 42 ++++++++++++++++++++++- market-maker/configs/portfolio.dev.yml | 1 + market-maker/configs/portfolio.local.yml | 1 + market-maker/configs/portfolio.prd.yml | 1 + market-maker/schemas/portfolio.json | 5 +++ market-maker/src/apps/portfolio/config.ts | 5 +++ market-maker/src/core/portfolioHealth.ts | 6 ++++ 7 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-col-mar-mm.yml b/.github/workflows/deploy-col-mar-mm.yml index c415034..27987f8 100644 --- a/.github/workflows/deploy-col-mar-mm.yml +++ b/.github/workflows/deploy-col-mar-mm.yml @@ -286,6 +286,8 @@ jobs: MAKER_APP: portfolio MAKER_ENV: ${{ needs.build.outputs.maker_env }} COMMIT_HASH: ${{ github.sha }} + # Same string as the image tag. portfolio.*.yml publishes it on /health. + IMAGE_TAG: ${{ needs.build.outputs.version }} run: | set -euo pipefail CONFIG="config/${MAKER_ENV}.env" @@ -320,7 +322,7 @@ jobs: set -a && . "$CONFIG" && set +a KEYS=$( { sed -n 's/^[[:space:]]*\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$CONFIG"; \ - printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH; } \ + printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH IMAGE_TAG; } \ | grep -vxE 'ALCHEMY_API_KEY|LIQUIDATOR_PRIVATE_KEY|WEBHOOK_SECRET|PRIVATE_KEY|FUTURES_MM_PRIVATE_KEY|PERPS_MM_PRIVATE_KEY' || true ) # Empty values are dropped so the maker applies its own defaults @@ -410,6 +412,44 @@ jobs: --region "$REGION" echo "โœ… Stable." + # services-stable also succeeds after a rollback to the previous + # image. The ALB is internal, so this job cannot GET /health. + # The running task image and its IMAGE_TAG env are what /health + # reports as imageTag. + EXPECTED_IMAGE="${{ env.GHCR_IMAGE }}:${{ needs.build.outputs.version }}" + EXPECTED_TAG="${{ needs.build.outputs.version }}" + TASK_ARN=$(aws ecs list-tasks \ + --cluster "$CLUSTER" \ + --service-name "$SERVICE" \ + --desired-status RUNNING \ + --region "$REGION" \ + --query 'taskArns[0]' \ + --output text) + if [ -z "$TASK_ARN" ] || [ "$TASK_ARN" = "None" ]; then + echo "::error::No running task for $SERVICE after it reported stable." + exit 1 + fi + read -r ACTUAL_IMAGE TASK_DEF < <(aws ecs describe-tasks \ + --cluster "$CLUSTER" \ + --tasks "$TASK_ARN" \ + --region "$REGION" \ + --query 'tasks[0].[containers[0].image,taskDefinitionArn]' \ + --output text) + ACTUAL_TAG=$(aws ecs describe-task-definition \ + --task-definition "$TASK_DEF" \ + --region "$REGION" \ + --query "taskDefinition.containerDefinitions[0].environment[?name=='IMAGE_TAG'].value | [0]" \ + --output text) + echo "Expected image: $EXPECTED_IMAGE" + echo "Running image: $ACTUAL_IMAGE" + echo "Expected /health imageTag: $EXPECTED_TAG" + echo "Task IMAGE_TAG: $ACTUAL_TAG" + if [ "$ACTUAL_IMAGE" != "$EXPECTED_IMAGE" ] || [ "$ACTUAL_TAG" != "$EXPECTED_TAG" ]; then + echo "::error::Service stabilized on ${ACTUAL_IMAGE} (imageTag ${ACTUAL_TAG}), not ${EXPECTED_IMAGE}." + exit 1 + fi + echo "โœ… Running task is ${ACTUAL_IMAGE}" + cleanup: name: ๐Ÿงน Cleanup runs-on: ubuntu-latest diff --git a/market-maker/configs/portfolio.dev.yml b/market-maker/configs/portfolio.dev.yml index ca6cf40..4a80550 100644 --- a/market-maker/configs/portfolio.dev.yml +++ b/market-maker/configs/portfolio.dev.yml @@ -10,6 +10,7 @@ nodeEnv: development commitHash: ${COMMIT_HASH:-unknown} +imageTag: ${IMAGE_TAG:-unknown} logLevel: ${MAKER_LOG_LEVEL:-debug} dryRun: ${MAKER_DRY_RUN:-false} # Dev iterates fast; leave resting orders on base-sepolia on Ctrl-C so we diff --git a/market-maker/configs/portfolio.local.yml b/market-maker/configs/portfolio.local.yml index 3006b3d..fd20489 100644 --- a/market-maker/configs/portfolio.local.yml +++ b/market-maker/configs/portfolio.local.yml @@ -7,6 +7,7 @@ nodeEnv: development commitHash: ${COMMIT_HASH:-dev} +imageTag: ${IMAGE_TAG:-local} logLevel: debug dryRun: false cancelOrdersOnShutdown: false diff --git a/market-maker/configs/portfolio.prd.yml b/market-maker/configs/portfolio.prd.yml index f110b46..4ee7c27 100644 --- a/market-maker/configs/portfolio.prd.yml +++ b/market-maker/configs/portfolio.prd.yml @@ -10,6 +10,7 @@ nodeEnv: production commitHash: ${COMMIT_HASH:-unknown} +imageTag: ${IMAGE_TAG:-unknown} logLevel: ${MAKER_LOG_LEVEL:-info} dryRun: ${MAKER_DRY_RUN:-false} cancelOrdersOnShutdown: ${MAKER_CANCEL_ORDERS_ON_SHUTDOWN:-true} diff --git a/market-maker/schemas/portfolio.json b/market-maker/schemas/portfolio.json index 7d912bf..aeaf42b 100644 --- a/market-maker/schemas/portfolio.json +++ b/market-maker/schemas/portfolio.json @@ -34,6 +34,11 @@ "default": "unknown", "type": "string" }, + "imageTag": { + "default": "unknown", + "type": "string", + "description": "Image tag this process was deployed as. Reported on /health." + }, "logLevel": { "default": "info", "type": "string" diff --git a/market-maker/src/apps/portfolio/config.ts b/market-maker/src/apps/portfolio/config.ts index 72db045..d2e2c05 100644 --- a/market-maker/src/apps/portfolio/config.ts +++ b/market-maker/src/apps/portfolio/config.ts @@ -158,6 +158,11 @@ export const portfolioRootSchema = Type.Object( { nodeEnv: Type.String({ default: "development" }), commitHash: Type.String({ default: "unknown" }), + imageTag: Type.String({ + default: "unknown", + description: + "Image tag this process was deployed as (for example v1.4.0). Surfaced on /health so deploy verification can reject a rollback.", + }), logLevel: Type.String({ default: "info" }), dryRun: Type.Boolean({ default: false }), cancelOrdersOnShutdown: Type.Boolean({ diff --git a/market-maker/src/core/portfolioHealth.ts b/market-maker/src/core/portfolioHealth.ts index 9ff9af1..27c5c8b 100644 --- a/market-maker/src/core/portfolioHealth.ts +++ b/market-maker/src/core/portfolioHealth.ts @@ -106,6 +106,9 @@ export class PortfolioHealthCheck { const body = JSON.stringify( { app: this.opts.appName, + imageTag: typeof this.opts.configSummary.imageTag === "string" + ? this.opts.configSummary.imageTag + : "unknown", status: this.status, walletAddress: this.walletAddress, lastError: this.lastError, @@ -161,6 +164,9 @@ export class PortfolioHealthCheck { const body = JSON.stringify( { app: this.opts.appName, + imageTag: typeof this.opts.configSummary.imageTag === "string" + ? this.opts.configSummary.imageTag + : "unknown", status: this.status, walletAddress: this.walletAddress, lastError: this.lastError,