server-deploy.yml

name: Deploy Server

# Release only the SHA selected by dispatching main; no arbitrary ref input or
# push trigger. Configure the production environment to allow main only.
on:
  workflow_dispatch:
    inputs:
      publish_only:
        description: Qualify and publish the image for rehearsal or initial bootstrap; do not deploy
        type: boolean
        default: false
      bootstrap:
        description: Create the first API-only Machine, then await protected cutover approval before enabling workers
        type: boolean
        default: false

permissions:
  contents: read

concurrency:
  group: server-deploy-production
  cancel-in-progress: false

jobs:
  validate:
    name: Require a manual main release
    runs-on: ubuntu-latest
    steps:
      - env:
          PUBLISH_ONLY: ${{ inputs.publish_only }}
          BOOTSTRAP: ${{ inputs.bootstrap }}
        run: |
          test "$GITHUB_EVENT_NAME" = workflow_dispatch
          test "$GITHUB_REF" = refs/heads/main
          test "$PUBLISH_ONLY:$BOOTSTRAP" != true:true

  qualify:
    name: Qualify and publish the exact release image
    needs: validate
    uses: ./.github/workflows/server-test.yml
    with:
      publish_image: true
    secrets:
      # Repository/org secret: environment secrets are unavailable to this call.
      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}

  prepared:
    name: Report the qualified image for rehearsal or bootstrap
    needs: qualify
    if: ${{ inputs.publish_only }}
    runs-on: ubuntu-latest
    env:
      RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
      STANDBY_IMAGE: ${{ needs.qualify.outputs.standby_image }}
    steps:
      - name: Record the exact image without accessing production
        run: |
          [[ "$RUNTIME_IMAGE" =~ ^registry\.fly\.io/inline-api@sha256:[a-f0-9]{64}$ ]] || exit 1
          [[ "$STANDBY_IMAGE" =~ ^registry\.fly\.io/inline-api@sha256:[a-f0-9]{64}$ ]] || exit 1
          printf 'Prepared `%s` at `%s` for rehearsal or initial bootstrap. Production preflight, migrations, and deployment were skipped.\n' \
            "$GITHUB_SHA" "$RUNTIME_IMAGE" >> "$GITHUB_STEP_SUMMARY"
          printf 'ARM64 Coolify image: `%s`. No Hetzner deployment was performed.\n' "$STANDBY_IMAGE" >> "$GITHUB_STEP_SUMMARY"

  preflight:
    name: Record a healthy public Machine baseline before DDL
    needs: qualify
    if: ${{ !inputs.publish_only }}
    runs-on: ubuntu-latest
    timeout-minutes: 5
    environment: production
    env:
      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
      BOOTSTRAP: ${{ inputs.bootstrap }}
    outputs:
      baseline: ${{ steps.baseline.outputs.baseline }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.sha }}
      - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
        with:
          version: "0.4.107"
      - name: Validate public Machine compatibility and service readiness
        id: baseline
        run: |
          test -n "$FLY_API_TOKEN"
          flyctl machine list --config server/fly.toml --json > "$RUNNER_TEMP/machines.json"
          filter=scripts/ci/fly-release-machines.jq
          if [[ "$BOOTSTRAP" = true ]]; then filter=scripts/ci/fly-release-machines-bootstrap.jq; fi
          baseline=$(jq -ce -f "$filter" "$RUNNER_TEMP/machines.json")
          printf 'baseline=%s\n' "$baseline" >> "$GITHUB_OUTPUT"

  migrate:
    name: Migrate and verify with the qualified image
    needs: [qualify, preflight]
    if: ${{ !inputs.publish_only }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    environment: production
    env:
      RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
    steps:
      - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
        with:
          version: "0.4.107"
      - name: Pull and verify the qualified image revision
        run: |
          test -n "$FLY_API_TOKEN"
          [[ "$RUNTIME_IMAGE" =~ ^registry\.fly\.io/inline-api@sha256:[a-f0-9]{64}$ ]]
          flyctl auth docker
          docker pull "$RUNTIME_IMAGE"
          test "$(docker image inspect --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$RUNTIME_IMAGE")" = "$GITHUB_SHA"
      - name: Apply and verify the migration ledger
        env:
          # DDL credentials belong only to this step, never to the API Machines.
          DATABASE_DIRECT_URL: ${{ secrets.PRODUCTION_DATABASE_MIGRATION_URL }}
          DATABASE_CONNECTION_MODE: direct
        run: |
          test -n "$DATABASE_DIRECT_URL"
          docker run --rm -e DATABASE_DIRECT_URL -e DATABASE_CONNECTION_MODE \
            "$RUNTIME_IMAGE" bun server/dist/migrate.js
          docker run --rm -e DATABASE_DIRECT_URL -e DATABASE_CONNECTION_MODE \
            "$RUNTIME_IMAGE" bun server/dist/verify-migrations.js

  deploy:
    name: Replace the public Machines and verify readiness
    needs: [qualify, preflight, migrate]
    if: ${{ !inputs.publish_only }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    environment: production
    outputs:
      bootstrap_baseline: ${{ steps.verify.outputs.bootstrap_baseline }}
    env:
      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
      RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
      BASELINE: ${{ needs.preflight.outputs.baseline }}
      BOOTSTRAP: ${{ inputs.bootstrap }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.sha }}
      - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
        with:
          version: "0.4.107"
      - name: Revalidate the recorded baseline and deploy the same digest
        run: |
          test -n "$FLY_API_TOKEN"
          [[ "$RUNTIME_IMAGE" =~ ^registry\.fly\.io/inline-api@sha256:[a-f0-9]{64}$ ]]
          flyctl machine list --config server/fly.toml --json > "$RUNNER_TEMP/machines-before.json"
          filter=scripts/ci/fly-release-machines.jq
          if [[ "$BOOTSTRAP" = true ]]; then filter=scripts/ci/fly-release-machines-bootstrap.jq; fi
          current=$(jq -ce -f "$filter" "$RUNNER_TEMP/machines-before.json")
          test "$current" = "$BASELINE"
          MACHINE_IDS=$(jq -er '.ids' <<< "$BASELINE")
          if [[ "$BOOTSTRAP" = true ]]; then
            API_ID=$(jq -r '.api.id // empty' <<< "$BASELINE")
            if [[ -n "$API_ID" ]]; then
              flyctl deploy --config server/fly.toml --image "$RUNTIME_IMAGE" \
                --only-machines "$API_ID" --strategy bluegreen --ha=false --signal SIGTERM --wait-timeout 5m --yes \
                --env INLINE_PROCESS_ROLE=api
            else
              exclusions=()
              if [[ -n "$MACHINE_IDS" ]]; then exclusions=(--exclude-machines "$MACHINE_IDS"); fi
              flyctl deploy --config server/fly.toml --image "$RUNTIME_IMAGE" \
                "${exclusions[@]}" --strategy immediate --ha=false --wait-timeout 5m --yes \
                --vm-cpu-kind shared --vm-cpus 6 --vm-memory 1536 --env INLINE_PROCESS_ROLE=api
            fi
          else
            flyctl deploy --config server/fly.toml --image "$RUNTIME_IMAGE" \
              --only-machines "$MACHINE_IDS" --strategy bluegreen --ha=false \
              --signal SIGTERM --wait-timeout 5m --yes
          fi
      - name: Verify baseline count, image revision, and service health
        id: verify
        run: |
          BASELINE_COUNT=1
          if [[ "$BOOTSTRAP" != true ]]; then BASELINE_COUNT=$(jq -er '.count' <<< "$BASELINE"); fi
          bash scripts/ci/fly-release-wait.sh "$RUNNER_TEMP/machines-after.json" \
            "$BASELINE_COUNT" "${RUNTIME_IMAGE##*@}" "$GITHUB_SHA"
          current=$(jq -ce -f scripts/ci/fly-release-machines.jq "$RUNNER_TEMP/machines-after.json")
          if [[ "$BOOTSTRAP" = true ]]; then
            after=$(jq -ce -f scripts/ci/fly-release-machines-bootstrap.jq "$RUNNER_TEMP/machines-after.json")
            test "$(jq -c 'del(.api)' <<< "$after")" = "$(jq -c 'del(.api)' <<< "$BASELINE")"
            snapshot=$(jq -ce '.api' <<< "$after")
            printf 'bootstrap_baseline=%s\n' "$snapshot" >> "$GITHUB_OUTPUT"
          fi
          jq -e --argjson count "$BASELINE_COUNT" --arg digest "${RUNTIME_IMAGE##*@}" --arg sha "$GITHUB_SHA" \
            '.count == $count and .digest == $digest and .revision == $sha' <<< "$current" > /dev/null
          printf 'Released `%s` at `%s`; %s public Machine(s) passed Fly service checks.\n' \
            "$GITHUB_SHA" "$RUNTIME_IMAGE" "$BASELINE_COUNT" >> "$GITHUB_STEP_SUMMARY"
          if [[ "$BOOTSTRAP" = true ]]; then
            printf 'API-only bootstrap is ready. Qualify authenticated paths, transfer routing, and stop the old all-role API before approving production-cutover.\n' >> "$GITHUB_STEP_SUMMARY"
          fi

  activate:
    name: Enable workers on the same qualified bootstrap image
    needs: [qualify, deploy]
    if: ${{ inputs.bootstrap }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    # Required-reviewer protection must be configured before bootstrap dispatch.
    # Approval means routing is verified and the old all-role API is stopped.
    environment: production-cutover
    env:
      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
      RUNTIME_IMAGE: ${{ needs.qualify.outputs.runtime_image }}
      BOOTSTRAP_BASELINE: ${{ needs.deploy.outputs.bootstrap_baseline }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.sha }}
      - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
        with:
          version: "0.4.107"
      - name: Recheck the exact API-only Machine
        run: |
          [[ "$RUNTIME_IMAGE" =~ ^registry\.fly\.io/inline-api@sha256:[a-f0-9]{64}$ ]] || exit 1
          flyctl machine list --config server/fly.toml --json > "$RUNNER_TEMP/before-activation.json"
          current=$(jq -ce -f scripts/ci/fly-release-machines.jq "$RUNNER_TEMP/before-activation.json")
          jq -e --argjson expected "$BOOTSTRAP_BASELINE" --arg digest "${RUNTIME_IMAGE##*@}" --arg sha "$GITHUB_SHA" \
            '.count == 1 and .ids == $expected.id and .digest == $digest and .revision == $sha' <<< "$current" > /dev/null
          snapshot=$(jq -c '[.[] | select(.state != "destroyed" and (.config.services // [] | length) > 0)] | .[0] |
            {id, version, digest: .image_ref.digest, revision: .image_ref.labels["org.opencontainers.image.revision"]}' "$RUNNER_TEMP/before-activation.json")
          test "$snapshot" = "$BOOTSTRAP_BASELINE"
          jq -e '[.[] | select(.state != "destroyed" and (.config.services // [] | length) > 0)] | all(.[];
            .config.env.INLINE_PROCESS_ROLE == "api" and .config.env.INLINE_INGRESS_HOST == "api.inline.chat")' \
            "$RUNNER_TEMP/before-activation.json" > /dev/null
      - name: Verify the predecessor is durably stopped
        env:
          FLY_API_TOKEN: ${{ secrets.LEGACY_FLY_READ_TOKEN }}
          PREDECESSOR_FLY_APP: ${{ vars.PREDECESSOR_FLY_APP }}
          PREDECESSOR_MACHINE_ID: ${{ vars.PREDECESSOR_MACHINE_ID }}
        run: |
          test -n "$FLY_API_TOKEN"
          test "$PREDECESSOR_FLY_APP" = inline-fra-standby
          test -n "$PREDECESSOR_MACHINE_ID"
          flyctl machine list --app "$PREDECESSOR_FLY_APP" --json > "$RUNNER_TEMP/predecessor.json"
          jq -e --arg predecessor "$PREDECESSOR_MACHINE_ID" -f scripts/ci/fly-release-machines-legacy.jq \
            "$RUNNER_TEMP/predecessor.json" > /dev/null
      - name: Activate the same qualified digest with workers
        run: |
          MACHINE_ID=$(jq -er '.id' <<< "$BOOTSTRAP_BASELINE")
          flyctl deploy --config server/fly.toml --image "$RUNTIME_IMAGE" \
            --only-machines "$MACHINE_ID" --strategy bluegreen --ha=false --signal SIGTERM --wait-timeout 5m --yes
      - name: Verify the all-role Machine
        run: |
          bash scripts/ci/fly-release-wait.sh "$RUNNER_TEMP/after-activation.json" \
            1 "${RUNTIME_IMAGE##*@}" "$GITHUB_SHA"
          current=$(jq -ce -f scripts/ci/fly-release-machines.jq "$RUNNER_TEMP/after-activation.json")
          jq -e --arg digest "${RUNTIME_IMAGE##*@}" --arg sha "$GITHUB_SHA" \
            '.count == 1 and .digest == $digest and .revision == $sha' <<< "$current" > /dev/null
          jq -e '[.[] | select(.state != "destroyed" and (.config.services // [] | length) > 0)] | all(.[];
            .config.env.INLINE_PROCESS_ROLE == "all" and .config.env.INLINE_INGRESS_HOST == "api.inline.chat")' \
            "$RUNNER_TEMP/after-activation.json" > /dev/null
          printf 'Activated workers for `%s` at `%s`; one all-role public Machine passed service checks.\n' \
            "$GITHUB_SHA" "$RUNTIME_IMAGE" >> "$GITHUB_STEP_SUMMARY"