Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 41 additions & 1 deletion .github/workflows/deploy-col-mar-mm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,8 @@ jobs:
MAKER_APP: portfolio
MAKER_ENV: ${{ needs.build.outputs.maker_env }}
COMMIT_HASH: ${{ github.sha }}
# Same string as the image tag. portfolio.*.yml publishes it on /health.
IMAGE_TAG: ${{ needs.build.outputs.version }}
run: |
set -euo pipefail
CONFIG="config/${MAKER_ENV}.env"
Expand Down Expand Up @@ -320,7 +322,7 @@ jobs:
set -a && . "$CONFIG" && set +a

KEYS=$( { sed -n 's/^[[:space:]]*\([A-Za-z_][A-Za-z0-9_]*\)=.*/\1/p' "$CONFIG"; \
printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH; } \
printf '%s\n' MAKER_APP MAKER_ENV COMMIT_HASH IMAGE_TAG; } \
| grep -vxE 'ALCHEMY_API_KEY|LIQUIDATOR_PRIVATE_KEY|WEBHOOK_SECRET|PRIVATE_KEY|FUTURES_MM_PRIVATE_KEY|PERPS_MM_PRIVATE_KEY' || true )

# Empty values are dropped so the maker applies its own defaults
Expand Down Expand Up @@ -410,6 +412,44 @@ jobs:
--region "$REGION"
echo "✅ Stable."

# services-stable also succeeds after a rollback to the previous
# image. The ALB is internal, so this job cannot GET /health.
# The running task image and its IMAGE_TAG env are what /health
# reports as imageTag.
EXPECTED_IMAGE="${{ env.GHCR_IMAGE }}:${{ needs.build.outputs.version }}"
EXPECTED_TAG="${{ needs.build.outputs.version }}"
TASK_ARN=$(aws ecs list-tasks \
--cluster "$CLUSTER" \
--service-name "$SERVICE" \
--desired-status RUNNING \
--region "$REGION" \
--query 'taskArns[0]' \
--output text)
if [ -z "$TASK_ARN" ] || [ "$TASK_ARN" = "None" ]; then
echo "::error::No running task for $SERVICE after it reported stable."
exit 1
fi
read -r ACTUAL_IMAGE TASK_DEF < <(aws ecs describe-tasks \
--cluster "$CLUSTER" \
--tasks "$TASK_ARN" \
--region "$REGION" \
--query 'tasks[0].[containers[0].image,taskDefinitionArn]' \
--output text)
ACTUAL_TAG=$(aws ecs describe-task-definition \
--task-definition "$TASK_DEF" \
--region "$REGION" \
--query "taskDefinition.containerDefinitions[0].environment[?name=='IMAGE_TAG'].value | [0]" \
--output text)
echo "Expected image: $EXPECTED_IMAGE"
echo "Running image: $ACTUAL_IMAGE"
echo "Expected /health imageTag: $EXPECTED_TAG"
echo "Task IMAGE_TAG: $ACTUAL_TAG"
if [ "$ACTUAL_IMAGE" != "$EXPECTED_IMAGE" ] || [ "$ACTUAL_TAG" != "$EXPECTED_TAG" ]; then
echo "::error::Service stabilized on ${ACTUAL_IMAGE} (imageTag ${ACTUAL_TAG}), not ${EXPECTED_IMAGE}."
exit 1
fi
echo "✅ Running task is ${ACTUAL_IMAGE}"

cleanup:
name: 🧹 Cleanup
runs-on: ubuntu-latest
Expand Down
1 change: 1 addition & 0 deletions market-maker/configs/portfolio.dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

nodeEnv: development
commitHash: ${COMMIT_HASH:-unknown}
imageTag: ${IMAGE_TAG:-unknown}
logLevel: ${MAKER_LOG_LEVEL:-debug}
dryRun: ${MAKER_DRY_RUN:-false}
# Dev iterates fast; leave resting orders on base-sepolia on Ctrl-C so we
Expand Down
1 change: 1 addition & 0 deletions market-maker/configs/portfolio.local.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

nodeEnv: development
commitHash: ${COMMIT_HASH:-dev}
imageTag: ${IMAGE_TAG:-local}
logLevel: debug
dryRun: false
cancelOrdersOnShutdown: false
Expand Down
1 change: 1 addition & 0 deletions market-maker/configs/portfolio.prd.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

nodeEnv: production
commitHash: ${COMMIT_HASH:-unknown}
imageTag: ${IMAGE_TAG:-unknown}
logLevel: ${MAKER_LOG_LEVEL:-info}
dryRun: ${MAKER_DRY_RUN:-false}
cancelOrdersOnShutdown: ${MAKER_CANCEL_ORDERS_ON_SHUTDOWN:-true}
Expand Down
5 changes: 5 additions & 0 deletions market-maker/schemas/portfolio.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@
"default": "unknown",
"type": "string"
},
"imageTag": {
"default": "unknown",
"type": "string",
"description": "Image tag this process was deployed as. Reported on /health."
},
"logLevel": {
"default": "info",
"type": "string"
Expand Down
5 changes: 5 additions & 0 deletions market-maker/src/apps/portfolio/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,11 @@ export const portfolioRootSchema = Type.Object(
{
nodeEnv: Type.String({ default: "development" }),
commitHash: Type.String({ default: "unknown" }),
imageTag: Type.String({
default: "unknown",
description:
"Image tag this process was deployed as (for example v1.4.0). Surfaced on /health so deploy verification can reject a rollback.",
}),
logLevel: Type.String({ default: "info" }),
dryRun: Type.Boolean({ default: false }),
cancelOrdersOnShutdown: Type.Boolean({
Expand Down
2 changes: 1 addition & 1 deletion market-maker/src/core/healthcheck.ts
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export class HealthCheck {

const logger = this.opts.logger;
const port = this.opts.port;
this.server.listen(port, () => {
this.server.listen(port, "0.0.0.0", () => {
logger.info(
{
human: `http://localhost:${port}/health`,
Expand Down
10 changes: 9 additions & 1 deletion market-maker/src/core/portfolioHealth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,9 @@ export class PortfolioHealthCheck {
}
});
const { logger, port } = this.opts;
this.server.listen(port, () => {
// Bind IPv4 explicitly. listen(port) alone binds :: on current
// node:24-alpine, and the ALB health check to the task IPv4 times out.
this.server.listen(port, "0.0.0.0", () => {
logger.info(
{
human: `http://localhost:${port}/health`,
Expand Down Expand Up @@ -104,6 +106,9 @@ export class PortfolioHealthCheck {
const body = JSON.stringify(
{
app: this.opts.appName,
imageTag: typeof this.opts.configSummary.imageTag === "string"
? this.opts.configSummary.imageTag
: "unknown",
status: this.status,
walletAddress: this.walletAddress,
lastError: this.lastError,
Expand Down Expand Up @@ -159,6 +164,9 @@ export class PortfolioHealthCheck {
const body = JSON.stringify(
{
app: this.opts.appName,
imageTag: typeof this.opts.configSummary.imageTag === "string"
? this.opts.configSummary.imageTag
: "unknown",
status: this.status,
walletAddress: this.walletAddress,
lastError: this.lastError,
Expand Down
Loading