From 04b1f5be968c6e578f512d0e98cc0129a376a69d Mon Sep 17 00:00:00 2001 From: Eric Zhang Date: Mon, 17 Aug 2026 17:16:27 +0800 Subject: [PATCH] feat(ci): add staging deployment workflow --- .github/workflows/deploy-staging.yml | 310 +++++++++++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 .github/workflows/deploy-staging.yml diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..c18917e --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,310 @@ +name: Deploy staging +run-name: Deploy ${{ inputs.ref }} to staging + +on: + workflow_dispatch: + inputs: + ref: + description: Branch, tag, or commit to deploy to staging + required: true + default: main + type: string + +permissions: + contents: read + +concurrency: + group: staging + cancel-in-progress: false + +env: + SOURCE_REF: ${{ inputs.ref }} + AWS_REGION: ap-southeast-1 + ECR_REGISTRY: 401696231252.dkr.ecr.ap-southeast-1.amazonaws.com + API_REPOSITORY: mem9-node-staging-api + WORKER_REPOSITORY: mem9-node-staging-worker + TF_STATE_BUCKET: mem9-node-prod-terraform-state + TF_STATE_KMS_KEY_ARN: arn:aws:kms:ap-southeast-1:401696231252:key/4cd4c7a1-c1a6-47ad-9205-548678b84d86 + TERRAFORM_VERSION: 1.15.4 + TF_WORKING_DIR: infra/terraform + TF_IN_AUTOMATION: 'true' + TF_INPUT: 'false' + +jobs: + deploy: + name: Build and deploy selected source to staging + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + id-token: write + + steps: + - name: Check out the selected commit + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + + - name: Verify selected source + run: | + git fetch --no-tags --depth=1 origin main + if ! git diff --quiet FETCH_HEAD -- infra/terraform; then + echo "The selected ref changes infra/terraform. Merge reviewed Terraform changes to main before deploying." >&2 + exit 1 + fi + echo "RELEASE_SHA=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + + - name: Configure AWS staging credentials + uses: aws-actions/configure-aws-credentials@v6 + with: + role-to-assume: arn:aws:iam::401696231252:role/mem9-node-staging-github-deploy + role-session-name: mem9-node-staging-deploy + aws-region: ${{ env.AWS_REGION }} + mask-aws-account-id: true + + - name: Log in to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and push API image + run: | + if aws ecr describe-images \ + --repository-name "$API_REPOSITORY" \ + --image-ids "imageTag=$RELEASE_SHA" \ + --region "$AWS_REGION" > /dev/null 2>&1; then + echo "API image $RELEASE_SHA already exists; reusing it." + else + docker build \ + --platform linux/amd64 \ + --file apps/api/Dockerfile \ + --tag "$ECR_REGISTRY/$API_REPOSITORY:$RELEASE_SHA" \ + . + docker push "$ECR_REGISTRY/$API_REPOSITORY:$RELEASE_SHA" + fi + + - name: Build and push worker image + run: | + if aws ecr describe-images \ + --repository-name "$WORKER_REPOSITORY" \ + --image-ids "imageTag=$RELEASE_SHA" \ + --region "$AWS_REGION" > /dev/null 2>&1; then + echo "Worker image $RELEASE_SHA already exists; reusing it." + else + docker build \ + --platform linux/amd64 \ + --file apps/worker/Dockerfile \ + --tag "$ECR_REGISTRY/$WORKER_REPOSITORY:$RELEASE_SHA" \ + . + docker push "$ECR_REGISTRY/$WORKER_REPOSITORY:$RELEASE_SHA" + fi + + - name: Lock release images + run: | + API_IMAGE="$ECR_REGISTRY/$API_REPOSITORY:$RELEASE_SHA" + WORKER_IMAGE="$ECR_REGISTRY/$WORKER_REPOSITORY:$RELEASE_SHA" + API_DIGEST=$(aws ecr describe-images \ + --repository-name "$API_REPOSITORY" \ + --image-ids "imageTag=$RELEASE_SHA" \ + --query 'imageDetails[0].imageDigest' \ + --output text \ + --region "$AWS_REGION") + WORKER_DIGEST=$(aws ecr describe-images \ + --repository-name "$WORKER_REPOSITORY" \ + --image-ids "imageTag=$RELEASE_SHA" \ + --query 'imageDetails[0].imageDigest' \ + --output text \ + --region "$AWS_REGION") + + test -n "$API_DIGEST" && test "$API_DIGEST" != "None" + test -n "$WORKER_DIGEST" && test "$WORKER_DIGEST" != "None" + + { + echo "API_IMAGE=$API_IMAGE" + echo "WORKER_IMAGE=$WORKER_IMAGE" + echo "API_DIGEST=$API_DIGEST" + echo "WORKER_DIGEST=$WORKER_DIGEST" + } >> "$GITHUB_ENV" + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ${{ env.TERRAFORM_VERSION }} + terraform_wrapper: false + + - name: Write staging variables + env: + STAGING_TF_VARS: ${{ secrets.STAGING_TF_VARS }} + run: | + test -n "$STAGING_TF_VARS" + printf '%s\n' "$STAGING_TF_VARS" > "$TF_WORKING_DIR/staging.auto.tfvars" + + - name: Initialize staging Terraform + run: | + terraform -chdir="$TF_WORKING_DIR" init \ + -input=false \ + -backend-config="bucket=$TF_STATE_BUCKET" \ + -backend-config="key=mem9-node/staging/terraform.tfstate" \ + -backend-config="region=$AWS_REGION" \ + -backend-config="encrypt=true" \ + -backend-config="kms_key_id=$TF_STATE_KMS_KEY_ARN" \ + -backend-config="use_lockfile=true" + + - name: Plan release with services stopped + run: | + terraform -chdir="$TF_WORKING_DIR" plan \ + -input=false \ + -var="release_id=$RELEASE_SHA" \ + -var="api_image=$API_IMAGE" \ + -var="worker_image=$WORKER_IMAGE" \ + -var="api_desired_count=0" \ + -var="worker_desired_count=0" \ + -out="$RUNNER_TEMP/staging-prepare.tfplan" + + - name: Guard release preparation plan + run: | + PLAN_JSON="$RUNNER_TEMP/staging-prepare.json" + terraform -chdir="$TF_WORKING_DIR" show -json "$RUNNER_TEMP/staging-prepare.tfplan" > "$PLAN_JSON" + + UNSAFE_ADDRESS_COUNT=$(jq '[ + .resource_changes[]? + | select(.change.actions != ["no-op"] and .change.actions != ["read"]) + | select(.address != "aws_ecs_task_definition.api") + | select(.address != "aws_ecs_task_definition.worker") + | select(.address != "aws_ecs_service.api") + | select(.address != "aws_ecs_service.worker") + ] | length' "$PLAN_JSON") + UNSAFE_DESTROY_COUNT=$(jq '[ + .resource_changes[]? + | select(.change.actions | index("delete")) + | select(.address != "aws_ecs_task_definition.api") + | select(.address != "aws_ecs_task_definition.worker") + ] | length' "$PLAN_JSON") + + test "$UNSAFE_ADDRESS_COUNT" = "0" + test "$UNSAFE_DESTROY_COUNT" = "0" + + - name: Apply release preparation + run: terraform -chdir="$TF_WORKING_DIR" apply -input=false "$RUNNER_TEMP/staging-prepare.tfplan" + + - name: Run database migration + run: | + ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name) + API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name) + API_TASK_DEFINITION=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_task_definition_arn) + + SERVICE_JSON=$(aws ecs describe-services \ + --cluster "$ECS_CLUSTER" \ + --services "$API_SERVICE" \ + --region "$AWS_REGION") + NETWORK_CONFIGURATION=$(jq -c '{ + awsvpcConfiguration: { + subnets: .services[0].networkConfiguration.awsvpcConfiguration.subnets, + securityGroups: .services[0].networkConfiguration.awsvpcConfiguration.securityGroups, + assignPublicIp: .services[0].networkConfiguration.awsvpcConfiguration.assignPublicIp + } + }' <<< "$SERVICE_JSON") + OVERRIDES=$(jq -nc '{containerOverrides:[{name:"api",command:["pnpm","migrate"]}]}') + + RUN_TASK_JSON="$RUNNER_TEMP/migration-task.json" + aws ecs run-task \ + --cluster "$ECS_CLUSTER" \ + --task-definition "$API_TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "$NETWORK_CONFIGURATION" \ + --overrides "$OVERRIDES" \ + --region "$AWS_REGION" > "$RUN_TASK_JSON" + + test "$(jq '.failures | length' "$RUN_TASK_JSON")" = "0" + MIGRATION_TASK_ARN=$(jq -r '.tasks[0].taskArn' "$RUN_TASK_JSON") + test -n "$MIGRATION_TASK_ARN" && test "$MIGRATION_TASK_ARN" != "null" + + aws ecs wait tasks-stopped \ + --cluster "$ECS_CLUSTER" \ + --tasks "$MIGRATION_TASK_ARN" \ + --region "$AWS_REGION" + + TASK_JSON=$(aws ecs describe-tasks \ + --cluster "$ECS_CLUSTER" \ + --tasks "$MIGRATION_TASK_ARN" \ + --region "$AWS_REGION") + EXIT_CODE=$(jq -r '.tasks[0].containers[] | select(.name == "api") | .exitCode' <<< "$TASK_JSON") + if test "$EXIT_CODE" != "0"; then + TASK_ID=${MIGRATION_TASK_ARN##*/} + aws logs get-log-events \ + --log-group-name /ecs/mem9-node-staging-api \ + --log-stream-name "ecs/api/$TASK_ID" \ + --limit 100 \ + --query 'events[].message' \ + --output text \ + --region "$AWS_REGION" || true + exit 1 + fi + + - name: Plan active staging services + run: | + terraform -chdir="$TF_WORKING_DIR" plan \ + -input=false \ + -var="release_id=$RELEASE_SHA" \ + -var="api_image=$API_IMAGE" \ + -var="worker_image=$WORKER_IMAGE" \ + -out="$RUNNER_TEMP/staging-active.tfplan" + + - name: Guard service activation plan + run: | + PLAN_JSON="$RUNNER_TEMP/staging-active.json" + terraform -chdir="$TF_WORKING_DIR" show -json "$RUNNER_TEMP/staging-active.tfplan" > "$PLAN_JSON" + + UNSAFE_CHANGE_COUNT=$(jq '[ + .resource_changes[]? + | select(.change.actions != ["no-op"] and .change.actions != ["read"]) + | select(.address != "aws_ecs_service.api") + | select(.address != "aws_ecs_service.worker") + ] | length' "$PLAN_JSON") + CREATE_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("create"))] | length' "$PLAN_JSON") + DESTROY_COUNT=$(jq '[.resource_changes[]? | select(.change.actions | index("delete"))] | length' "$PLAN_JSON") + + test "$UNSAFE_CHANGE_COUNT" = "0" + test "$CREATE_COUNT" = "0" + test "$DESTROY_COUNT" = "0" + + - name: Activate staging services + run: terraform -chdir="$TF_WORKING_DIR" apply -input=false "$RUNNER_TEMP/staging-active.tfplan" + + - name: Wait for ECS services + run: | + ECS_CLUSTER=$(terraform -chdir="$TF_WORKING_DIR" output -raw ecs_cluster_name) + API_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_service_name) + WORKER_SERVICE=$(terraform -chdir="$TF_WORKING_DIR" output -raw worker_service_name) + + aws ecs wait services-stable \ + --cluster "$ECS_CLUSTER" \ + --services "$API_SERVICE" "$WORKER_SERVICE" \ + --region "$AWS_REGION" + + - name: Verify staging API health + run: | + ALB_DNS=$(terraform -chdir="$TF_WORKING_DIR" output -raw api_load_balancer_dns_name) + API_HEALTH_URL="http://$ALB_DNS/health/ready" + + curl \ + --fail-with-body \ + --silent \ + --show-error \ + --retry 20 \ + --retry-delay 5 \ + --retry-all-errors \ + --connect-timeout 5 \ + --max-time 10 \ + "$API_HEALTH_URL" \ + | jq -e '.status == "ready"' + + { + echo "## Staging deployed" + echo + echo "- Source ref: \`$SOURCE_REF\`" + echo "- Release: \`$RELEASE_SHA\`" + echo "- API digest: \`$API_DIGEST\`" + echo "- Worker digest: \`$WORKER_DIGEST\`" + echo "- Migration: **passed**" + echo "- Health: [$API_HEALTH_URL]($API_HEALTH_URL)" + } >> "$GITHUB_STEP_SUMMARY"