From 13a678a41c0def78db85bc4891b9fb0d78f02dcb Mon Sep 17 00:00:00 2001 From: abs2023 <93659489+abs2023@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:12:50 -0400 Subject: [PATCH 1/2] fix(market-maker): bind the health server on IPv4 The load balancer checks the task IPv4 address. listen(port) on the current node image binds IPv6 only, so those checks time out and ECS rolls the task back. Co-authored-by: Cursor --- market-maker/src/core/healthcheck.ts | 2 +- market-maker/src/core/portfolioHealth.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/market-maker/src/core/healthcheck.ts b/market-maker/src/core/healthcheck.ts index bb731bb..6ea325e 100644 --- a/market-maker/src/core/healthcheck.ts +++ b/market-maker/src/core/healthcheck.ts @@ -98,7 +98,7 @@ export class HealthCheck { const logger = this.opts.logger; const port = this.opts.port; - this.server.listen(port, () => { + this.server.listen(port, "0.0.0.0", () => { logger.info( { human: `http://localhost:${port}/health`, diff --git a/market-maker/src/core/portfolioHealth.ts b/market-maker/src/core/portfolioHealth.ts index 2c05bae..9ff9af1 100644 --- a/market-maker/src/core/portfolioHealth.ts +++ b/market-maker/src/core/portfolioHealth.ts @@ -73,7 +73,9 @@ export class PortfolioHealthCheck { } }); const { logger, port } = this.opts; - this.server.listen(port, () => { + // Bind IPv4 explicitly. listen(port) alone binds :: on current + // node:24-alpine, and the ALB health check to the task IPv4 times out. + this.server.listen(port, "0.0.0.0", () => { logger.info( { human: `http://localhost:${port}/health`, From dca0e1d31b4d54210e9550e92505fe76e853d4a5 Mon Sep 17 00:00:00 2001 From: abs2023 <93659489+abs2023@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:27:54 -0400 Subject: [PATCH 2/2] fix(market-maker): fail verify unless the running image is the one just built /health now reports imageTag. The verify job compares the running task image and that tag to the release, so a rollback can no longer pass as a stable deploy. Co-authored-by: Cursor --- .github/workflows/deploy-col-mar-mm.yml | 42 ++++++++++++++++++++++- market-maker/configs/portfolio.dev.yml | 1 + market-maker/configs/portfolio.local.yml | 1 + market-maker/configs/portfolio.prd.yml | 1 + market-maker/schemas/portfolio.json | 5 +++ market-maker/src/apps/portfolio/config.ts | 5 +++ market-maker/src/core/portfolioHealth.ts | 6 ++++ 7 files changed, 60 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-col-mar-mm.yml b/.github/workflows/deploy-col-mar-mm.yml index c415034..27987f8 100644 --- a/.github/workflows/deploy-col-mar-mm.yml +++ b/.github/workflows/deploy-col-mar-mm.yml @@ -286,6 +286,8 @@ jobs: MAKER_APP: portfolio MAKER_ENV: ${{ needs.build.outputs.maker_env }} COMMIT_HASH: ${{ github.sha }} + # Same string as the image tag. portfolio.*.yml publishes it on /health. + IMAGE_TAG: ${{ needs.build.outputs.version }} run: | set -euo pipefail CONFIG="config/${MAKER_ENV}.env" @@ -320,7 +322,7 @@ jobs: set -a && . "$CONFIG" && set +a KEYS=$( { sed -n 's/^[[:space:]]*\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$CONFIG"; \ - printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH; } \ + printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH IMAGE_TAG; } \ | grep -vxE 'ALCHEMY_API_KEY|LIQUIDATOR_PRIVATE_KEY|WEBHOOK_SECRET|PRIVATE_KEY|FUTURES_MM_PRIVATE_KEY|PERPS_MM_PRIVATE_KEY' || true ) # Empty values are dropped so the maker applies its own defaults @@ -410,6 +412,44 @@ jobs: --region "$REGION" echo "โœ… Stable." + # services-stable also succeeds after a rollback to the previous + # image. The ALB is internal, so this job cannot GET /health. + # The running task image and its IMAGE_TAG env are what /health + # reports as imageTag. + EXPECTED_IMAGE="${{ env.GHCR_IMAGE }}:${{ needs.build.outputs.version }}" + EXPECTED_TAG="${{ needs.build.outputs.version }}" + TASK_ARN=$(aws ecs list-tasks \ + --cluster "$CLUSTER" \ + --service-name "$SERVICE" \ + --desired-status RUNNING \ + --region "$REGION" \ + --query 'taskArns[0]' \ + --output text) + if [ -z "$TASK_ARN" ] || [ "$TASK_ARN" = "None" ]; then + echo "::error::No running task for $SERVICE after it reported stable." + exit 1 + fi + read -r ACTUAL_IMAGE TASK_DEF < <(aws ecs describe-tasks \ + --cluster "$CLUSTER" \ + --tasks "$TASK_ARN" \ + --region "$REGION" \ + --query 'tasks[0].[containers[0].image,taskDefinitionArn]' \ + --output text) + ACTUAL_TAG=$(aws ecs describe-task-definition \ + --task-definition "$TASK_DEF" \ + --region "$REGION" \ + --query "taskDefinition.containerDefinitions[0].environment[?name=='IMAGE_TAG'].value | [0]" \ + --output text) + echo "Expected image: $EXPECTED_IMAGE" + echo "Running image: $ACTUAL_IMAGE" + echo "Expected /health imageTag: $EXPECTED_TAG" + echo "Task IMAGE_TAG: $ACTUAL_TAG" + if [ "$ACTUAL_IMAGE" != "$EXPECTED_IMAGE" ] || [ "$ACTUAL_TAG" != "$EXPECTED_TAG" ]; then + echo "::error::Service stabilized on ${ACTUAL_IMAGE} (imageTag ${ACTUAL_TAG}), not ${EXPECTED_IMAGE}." + exit 1 + fi + echo "โœ… Running task is ${ACTUAL_IMAGE}" + cleanup: name: ๐Ÿงน Cleanup runs-on: ubuntu-latest diff --git a/market-maker/configs/portfolio.dev.yml b/market-maker/configs/portfolio.dev.yml index ca6cf40..4a80550 100644 --- a/market-maker/configs/portfolio.dev.yml +++ b/market-maker/configs/portfolio.dev.yml @@ -10,6 +10,7 @@ nodeEnv: development commitHash: ${COMMIT_HASH:-unknown} +imageTag: ${IMAGE_TAG:-unknown} logLevel: ${MAKER_LOG_LEVEL:-debug} dryRun: ${MAKER_DRY_RUN:-false} # Dev iterates fast; leave resting orders on base-sepolia on Ctrl-C so we diff --git a/market-maker/configs/portfolio.local.yml b/market-maker/configs/portfolio.local.yml index 3006b3d..fd20489 100644 --- a/market-maker/configs/portfolio.local.yml +++ b/market-maker/configs/portfolio.local.yml @@ -7,6 +7,7 @@ nodeEnv: development commitHash: ${COMMIT_HASH:-dev} +imageTag: ${IMAGE_TAG:-local} logLevel: debug dryRun: false cancelOrdersOnShutdown: false diff --git a/market-maker/configs/portfolio.prd.yml b/market-maker/configs/portfolio.prd.yml index f110b46..4ee7c27 100644 --- a/market-maker/configs/portfolio.prd.yml +++ b/market-maker/configs/portfolio.prd.yml @@ -10,6 +10,7 @@ nodeEnv: production commitHash: ${COMMIT_HASH:-unknown} +imageTag: ${IMAGE_TAG:-unknown} logLevel: ${MAKER_LOG_LEVEL:-info} dryRun: ${MAKER_DRY_RUN:-false} cancelOrdersOnShutdown: ${MAKER_CANCEL_ORDERS_ON_SHUTDOWN:-true} diff --git a/market-maker/schemas/portfolio.json b/market-maker/schemas/portfolio.json index 7d912bf..aeaf42b 100644 --- a/market-maker/schemas/portfolio.json +++ b/market-maker/schemas/portfolio.json @@ -34,6 +34,11 @@ "default": "unknown", "type": "string" }, + "imageTag": { + "default": "unknown", + "type": "string", + "description": "Image tag this process was deployed as. Reported on /health." + }, "logLevel": { "default": "info", "type": "string" diff --git a/market-maker/src/apps/portfolio/config.ts b/market-maker/src/apps/portfolio/config.ts index 72db045..d2e2c05 100644 --- a/market-maker/src/apps/portfolio/config.ts +++ b/market-maker/src/apps/portfolio/config.ts @@ -158,6 +158,11 @@ export const portfolioRootSchema = Type.Object( { nodeEnv: Type.String({ default: "development" }), commitHash: Type.String({ default: "unknown" }), + imageTag: Type.String({ + default: "unknown", + description: + "Image tag this process was deployed as (for example v1.4.0). Surfaced on /health so deploy verification can reject a rollback.", + }), logLevel: Type.String({ default: "info" }), dryRun: Type.Boolean({ default: false }), cancelOrdersOnShutdown: Type.Boolean({ diff --git a/market-maker/src/core/portfolioHealth.ts b/market-maker/src/core/portfolioHealth.ts index 9ff9af1..27c5c8b 100644 --- a/market-maker/src/core/portfolioHealth.ts +++ b/market-maker/src/core/portfolioHealth.ts @@ -106,6 +106,9 @@ export class PortfolioHealthCheck { const body = JSON.stringify( { app: this.opts.appName, + imageTag: typeof this.opts.configSummary.imageTag === "string" + ? this.opts.configSummary.imageTag + : "unknown", status: this.status, walletAddress: this.walletAddress, lastError: this.lastError, @@ -161,6 +164,9 @@ export class PortfolioHealthCheck { const body = JSON.stringify( { app: this.opts.appName, + imageTag: typeof this.opts.configSummary.imageTag === "string" + ? this.opts.configSummary.imageTag + : "unknown", status: this.status, walletAddress: this.walletAddress, lastError: this.lastError,