diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index be10d52..dd16e56 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1105,13 +1105,210 @@ jobs: echo "Non-DAG Bundle Deploy Action validation failed: bundle was not updated" exit 1 + # Deploys DAGs served from a non-default path into a named DAG bundle, which the deploy does not + # create — so the bundle is registered up front with `astro deployment bundle create`. + named-dag-bundle-test: + name: Named DAG Bundle Deploy Test + runs-on: ubuntu-latest + needs: [non-dags-deploy-test, create-test-deployments] + if: ${{ !cancelled() && needs.create-test-deployments.result == 'success' }} + env: + E2E_DEPLOYMENT_ID: ${{ fromJSON(needs.create-test-deployments.outputs.DEPLOYMENTS)['bundle'] }} + DAG_BUNDLE_NAME: e2e-team-a + steps: + - name: Checkout code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Patch deploy-action for manual dispatch + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + - name: Get Astro Environment Info + id: get-astro-env-info + uses: ./.github/workflows/e2e/get_astro_env_info + with: + input_workspace_id: ${{ github.event.inputs.workspace_id }} + input_organization_id: ${{ github.event.inputs.org_id }} + input_astronomer_host: ${{ github.event.inputs.astronomer_host }} + input_astro_api_token: ${{ github.event.inputs.token }} + secret_workspace_id: ${{ secrets.WORKSPACE_ID }} + secret_organization_id: ${{ secrets.ORGANIZATION_ID }} + secret_astronomer_host: ${{ secrets.ASTRONOMER_HOST }} + secret_astro_api_token: ${{ secrets.ASTRO_API_TOKEN }} + + - name: Install Astro CLI + uses: astronomer/setup-astro-cli@v0.0.1 + with: + version: ${{ github.event.inputs.cli_version }} + + - name: Install dependencies + run: sudo apt-get install jq + + - name: Set CLI context + run: astro context switch ${{ steps.get-astro-env-info.outputs.astronomer_host }} + + # must happen before the deploy, and before the git mock is installed, since the CLI reads git metadata + - name: Create the named DAG bundle + run: | + astro workspace switch ${{ steps.get-astro-env-info.outputs.workspace_id }} + astro deployment bundle create --deployment-id ${{ env.E2E_DEPLOYMENT_ID }} --name "$DAG_BUNDLE_NAME" + + - name: Get named DAG bundle before test + id: get-bundle-before + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + bundle_name: e2e-team-a + + - name: Mock git commands + run: | + mv e2e-setup/mocks/configurable-git.sh /usr/local/bin/git + chmod +x /usr/local/bin/git + + # a DAG change elsewhere (here the project's dags/ folder, which the default matcher would treat + # as a DAG change) is not this bundle's change, so this must not deploy + - name: Deploy with only an unrelated DAG changed + uses: ./ + env: + E2E_CHANGED_FILES: e2e-setup/astro-project/dags/exampledag.py + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + deploy-type: dags-only + dags-path: e2e-setup/bundles/team_a + dag-bundle-name: e2e-team-a + no-dags-base-dir: true + wait-time: 15m + + - name: Get named DAG bundle after unrelated change + id: get-bundle-unrelated + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + bundle_name: e2e-team-a + + - name: Validate the unrelated change did not deploy + shell: bash + run: | + before="${{ steps.get-bundle-before.outputs.desired_version }}" + after="${{ steps.get-bundle-unrelated.outputs.desired_version }}" + echo "desired version before: '$before', after: '$after'" + if [[ "$after" != "$before" ]]; then + echo "Named DAG Bundle validation failed: a change outside dags-path deployed the bundle" + exit 1 + fi + echo "Named DAG Bundle validation succeeded: change outside dags-path skipped" + + - name: Re-patch deploy-action before the next invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + # The DAGs live outside the Astro project and root-folder is left empty, so the CLI runs from the + # repo root, which has no project: the deploy only succeeds if --dags-path reaches the CLI. The + # changed file's path has no dags/ in it, so it only counts as a DAG change if dags-path is honored. + - name: Deploy DAGs to the named bundle + uses: ./ + env: + E2E_CHANGED_FILES: e2e-setup/bundles/team_a/team_a_dag.py + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + deploy-type: dags-only + dags-path: e2e-setup/bundles/team_a + dag-bundle-name: e2e-team-a + no-dags-base-dir: true + description: "test-named-dag-bundle-deploy-action" + wait-time: 15m + + - name: Get named DAG bundle after test + id: get-bundle-after + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + bundle_name: e2e-team-a + + - name: Validate Named DAG Bundle Deploy Action + shell: bash + run: | + if [[ "${{ steps.get-bundle-after.outputs.found }}" != "true" ]]; then + echo "Named DAG Bundle validation failed: bundle $DAG_BUNDLE_NAME not found on the deployment" + exit 1 + fi + + before="${{ steps.get-bundle-before.outputs.desired_version }}" + after="${{ steps.get-bundle-after.outputs.desired_version }}" + echo "desired version before: '$before', after: '$after'" + + if [[ "$after" == "" ]]; then + echo "Named DAG Bundle validation failed: DAGs were not deployed to the named bundle" + exit 1 + fi + + if [[ "$after" == "$before" ]]; then + echo "Named DAG Bundle validation failed: bundle version did not change" + exit 1 + fi + + echo "Named DAG Bundle validation succeeded: DAGs landed in $DAG_BUNDLE_NAME" + + - name: Re-patch deploy-action before the dispatch invocation + uses: ./.github/workflows/e2e/patch_action_for_dispatch + + # a workflow_dispatch (or the create event that sets up a deployment preview) has no diff, so a + # dags-only step must deploy its DAGs even though nothing in its path is reported as changed + - name: Deploy DAGs on a workflow dispatch + uses: ./ + env: + DEPLOY_ACTION_DIFF_BEFORE: "" + DEPLOY_ACTION_DIFF_AFTER: "" + E2E_CHANGED_FILES: README.md + with: + deployment-id: ${{ env.E2E_DEPLOYMENT_ID }} + workspace: ${{ steps.get-astro-env-info.outputs.workspace_id }} + deploy-type: dags-only + dags-path: e2e-setup/bundles/team_a + dag-bundle-name: e2e-team-a + no-dags-base-dir: true + wait-time: 15m + + - name: Get named DAG bundle after dispatch + id: get-bundle-dispatch + uses: ./.github/workflows/e2e/get_deployment_bundle + with: + deployment_id: ${{ env.E2E_DEPLOYMENT_ID }} + organization_id: ${{ steps.get-astro-env-info.outputs.organization_id }} + astro_api_token: ${{ steps.get-astro-env-info.outputs.astro_api_token }} + astronomer_host: ${{ steps.get-astro-env-info.outputs.astronomer_host }} + bundle_name: e2e-team-a + + - name: Validate the dispatch deployed the DAGs + shell: bash + run: | + before="${{ steps.get-bundle-after.outputs.desired_version }}" + after="${{ steps.get-bundle-dispatch.outputs.desired_version }}" + echo "desired version before: '$before', after: '$after'" + if [[ -z "$after" || "$after" == "$before" ]]; then + echo "Named DAG Bundle validation failed: a dispatch with no diff did not deploy the DAGs" + exit 1 + fi + echo "Named DAG Bundle validation succeeded: dispatch deployed the DAGs" + # Asserts the skip side of the bundle deploy decision: no change under root-folder means no deploy, # unless skip-unchanged is off, a bundle-trigger-path matched, or the diff cannot be computed. # The git mock reports README.md. bundle-skip-test: name: Bundle Deploy Skip Test runs-on: ubuntu-latest - needs: [non-dags-deploy-test, create-test-deployments] + needs: [named-dag-bundle-test, create-test-deployments] if: ${{ !cancelled() && needs.create-test-deployments.result == 'success' }} env: E2E_CHANGED_FILES: README.md @@ -1445,6 +1642,47 @@ jobs: mount_path: /usr/local/airflow/e2e-valid root_folder: e2e-setup/non-dags accepted: true + - case: "dags-path with non-dags" + deploy_type: non-dags + mount_path: /usr/local/airflow/e2e-invalid + root_folder: e2e-setup/non-dags + dags_path: e2e-setup/bundles/team_a + - case: "dag-bundle-name with image-only" + deploy_type: image-only + dag_bundle_name: e2e-team-a + root_folder: e2e-setup/astro-project + - case: "dags-path with image-only" + deploy_type: image-only + dags_path: e2e-setup/bundles/team_a + root_folder: e2e-setup/astro-project + - case: "dag-bundle-name with image-name" + deploy_type: image-and-dags + dag_bundle_name: e2e-team-a + root_folder: e2e-setup/astro-project + image_name: custom-image:latest + - case: "dags-path with infer" + deploy_type: infer + dags_path: e2e-setup/bundles/team_a + - case: "dag-bundle-name with infer" + deploy_type: infer + dag_bundle_name: e2e-team-a + root_folder: e2e-setup/astro-project + - case: "dags-path that does not exist" + deploy_type: dags-only + dags_path: e2e-setup/bundles/missing + - case: "dags-path outside the repository" + deploy_type: dags-only + dags_path: .. + - case: "dag-bundle-name below the minimum CLI version" + deploy_type: dags-only + dag_bundle_name: e2e-team-a + dags_path: e2e-setup/bundles/team_a + cli_version: 1.43.1 + - case: "control: dags-only with a named bundle" + deploy_type: dags-only + dag_bundle_name: e2e-team-a + dags_path: e2e-setup/bundles/team_a + accepted: true env: E2E_STUB_CLI_VERSION: ${{ matrix.cli_version || '1.45.0' }} steps: @@ -1485,6 +1723,8 @@ jobs: build-secrets: ${{ matrix.build_secrets }} mount-path: ${{ matrix.mount_path }} non-dags-bundle-type: ${{ matrix.non_dags_bundle_type }} + dag-bundle-name: ${{ matrix.dag_bundle_name }} + dags-path: ${{ matrix.dags_path }} - name: Validate the action's verdict on the configuration shell: bash @@ -2015,6 +2255,7 @@ jobs: custom-docker-image-test, dbt-deploy-test, non-dags-deploy-test, + named-dag-bundle-test, bundle-skip-test, deployment-preview-test, remote-deploy-test, diff --git a/README.md b/README.md index 259ac59..de9fa52 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ The following table lists the configuration options for the Deploy to Astro acti | `deploy-type` | `infer` | Specify the type of deploy you would like to do. Use this option to deploy images and/or DAGs, a DBT project, or a non-DAG bundle. Possible options are `infer`, `dags-only`, `image-and-dags`, `image-only`, `dbt` or `non-dags`. `infer` option would infer between DAG only deploy and image and DAG deploy based on updated files. For description on each deploy type checkout [deploy type details](https://github.com/astronomer/deploy-action#deploy-type-details) | | `image-trigger-paths` | `` | Comma- or newline-separated list of paths outside `root-folder` that should also trigger a deploy when changed. Useful in monorepos where files built or copied into the image at deploy time (e.g. generated protos) live outside the Astro project. A change under any of these paths forces an image deploy instead of being skipped. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory (e.g. `proto/`). Only applies to the `infer`, `image-and-dags`, and `image-only` deploy types. For `dbt` and `non-dags` deploys, use `bundle-trigger-paths`. | | `bundle-trigger-paths` | `` | Comma- or newline-separated list of paths outside `root-folder` that should also trigger a bundle deploy when changed. Useful in monorepos where a bundle depends on shared files that live outside its own directory. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory (e.g. `shared/macros/`). Only applies to the `dbt` and `non-dags` deploy types. | -| `skip-unchanged` | `true` | When `true` (default), the action skips the deploy if no changed files fall under `root-folder` (or `image-trigger-paths` / `bundle-trigger-paths`). Set to `false` to disable this diff-based skip and always run the deploy dictated by `deploy-type`. This is the closest replacement for the deprecated `deploy-image: true` behavior. | +| `skip-unchanged` | `true` | When `true` (default), the action skips the deploy if no changed files fall under `root-folder` or `dags-path` (or `image-trigger-paths` / `bundle-trigger-paths`). Set to `false` to disable this diff-based skip and always run the deploy dictated by `deploy-type`. This is the closest replacement for the deprecated `deploy-image: true` behavior. | | `deployment-id` | `false` | Specifies the id of the deployment you to make a preview from or are deploying too. | | `deployment-name` | `false` | Specifies The name of the deployment you want to make preview from or are deploying too. Cannot be used with `deployment-id` | | `description` | | Configure a description for a deploy to Astro. Description will be visible in the Deploy History tab. | @@ -70,6 +70,9 @@ The following table lists the configuration options for the Deploy to Astro acti | `build-secrets` | `` | Newline-delimited list of Docker build secrets, one `docker build --secret` specification per line (e.g. `id=mysecret,src=secrets.txt`). Each line is passed to `astro deploy` as its own `--build-secret` flag. See https://docs.docker.com/build/building/secrets/ for more information. | | `mount-path` | `` | Path to mount the bundle in Airflow, for reference by DAGs. Required with `deploy-type: non-dags`. With `deploy-type: dbt` it is optional and defaults to /usr/local/airflow/dbt/{dbt project name} | | `non-dags-bundle-type` | `` | Free-form label identifying the kind of non-DAG bundle being deployed (e.g. `dbt`), shown in the Deploy History tab. Only applies to the `non-dags` deploy type. Requires Astro CLI 1.44.0 or higher. | +| `dag-bundle-name` | `` | Deploy DAGs to a named DAG bundle on the Deployment instead of the default bundle. Requires Airflow 3, and the bundle must already exist on the Deployment. Use with `deploy-type: dags-only` or `image-and-dags`. Requires Astro CLI 1.44.0 or higher. | +| `dags-path` | `` | Deploy DAGs from this path, relative to `root-folder`, instead of the project's `dags` directory. Useful for serving several DAG bundles out of one repository, one action invocation per bundle. A `dags-only` deploy classifies changes against this path instead of `dags/`, and with Astro CLI 1.45.0 or higher (and `parse` and `pytest` off) it needs no Astro project, so `root-folder` can be left empty. Use with `deploy-type: dags-only` or `image-and-dags`. | +| `no-dags-base-dir` | `false` | If true, exclude the DAGs directory prefix from the uploaded bundle. Use for Airflow 3 Deployments, where the bundle root is already on `sys.path`. Use with `deploy-type` `infer`, `dags-only`, or `image-and-dags`, and not with `image-name`. | | `checkout-submodules` | `false` | Whether to checkout submodules when cloning the repository: `false` to disable (default), `true` to checkout submodules or `recursive` to recursively checkout submodules. Works only when `checkout` is set to `true`. Works only when `checkout` is set to `true`. | | `sparse-checkout` | `` | Comma- or newline-separated list of cone-mode sparse-checkout patterns passed to `actions/checkout`. When set, only the listed paths (plus repo-root files) are checked out, which can dramatically reduce checkout size for large monorepos. Typically set to the same value as `root-folder`. Works only when `checkout` is set to `true`. Leave empty for a full checkout (default). | | `wake-on-deploy` | `false` | If true, the deployment will be woken up from hibernation before deploying. NOTE: This option overrides the deployment's hibernation override spec. | @@ -99,7 +102,7 @@ The following section describe each of the deploy type input value in detail to 3. `dags-only`: In this mode, deploy-action would run through all the file changes: - if there are no file changes in the configured root-folder then it skips deploy - - if all the file changes are in folders except `dags` folder then it skips deploy + - if all the file changes are in folders except `dags` folder (or `dags-path`, when set) then it skips deploy - if the changed files cannot be determined (a manual workflow dispatch, a new branch, or a missing diff base) it would do a dag only deploy - otherwise it would do a dag only deploy @@ -179,6 +182,40 @@ not change how the bundle is handled. > The bundle directory must not sit inside an Astro project. `astro-project/` and `sql/` as siblings works; `astro-project/sql/` > is rejected by the Astro CLI. Keep the bundle a sibling of the Astro project in the repository. +## Named DAG bundles + +On Airflow 3 Deployments, DAGs can be served from several named bundles instead of a single default one. `dag-bundle-name` +targets one of them; the bundle must already exist on the Deployment, since the deploy does not create it. Requires Astro +CLI 1.44.0 or higher. + +```yaml +- name: Deploy team A's DAGs + uses: astronomer/deploy-action@v0.16.0 + with: + deployment-id: + deploy-type: dags-only + dags-path: bundles/team-a + dag-bundle-name: team-a +``` + +A named bundle's DAGs don't have to live in an Astro project. With Astro CLI 1.45.0 or higher, a `dags-only` deploy with +`dags-path` set (and `parse` and `pytest` off) needs no project, so leave `root-folder` empty and point `dags-path` at the +bundle's directory from the repository root. `dags-path` is otherwise resolved from `root-folder`. + +To serve several DAG bundles out of one repository, use one action invocation per bundle, each with its own `dags-path` and +`dag-bundle-name`. With `dags-path` set, a `dags-only` deploy classifies changes against that path instead of `dags/`, so +each bundle only deploys when its own DAGs change. `dag-bundle-name` and `dags-path` cannot be used with `deploy-type: infer` +(the default), which would build and deploy the project's image for any change outside the bundle: use `dags-only`, or +`image-and-dags` to deliberately deploy the image as well. + +`no-dags-base-dir` excludes the DAGs directory prefix from the uploaded bundle, which is what Airflow 3 expects when the +bundle root is already on `sys.path`. + +`dag-bundle-name`, `dags-path`, and `no-dags-base-dir` cannot be combined with `image-name` on deploys that use it, because +a pre-built image carries its own DAGs. None of them can be used with `deploy-type` `image-only`, `dbt`, or `non-dags`, +which deploy no DAGs. + + ## Installing the Astro CLI Astronomer recommends using the [`setup-astro-cli`](https://github.com/astronomer/setup-astro-cli) action to install and manage the Astro CLI in your workflows. This keeps CLI installation decoupled from deployment logic, lets you pin or update CLI versions independently (including via Dependabot), and allows a single installation to be reused across multiple deploy steps. @@ -433,6 +470,50 @@ steps: parse: true ``` +### Deploy several bundles from the same repo + +In the following example we would deploy the image from `astro-project`, two named DAG bundles from `bundles/`, and a +non-DAG bundle of SQL assets, all from the same repository. The bundles sit outside the Astro project, so each step only +deploys when its own files change: a change to team A's DAGs does not redeploy team B's, or rebuild the image. See +[named DAG bundles](https://github.com/astronomer/deploy-action#named-dag-bundles) and +[non-DAG bundle deploys](https://github.com/astronomer/deploy-action#non-dag-bundle-deploys) for the details. + +```yaml +steps: +- name: Install Astro CLI + uses: astronomer/setup-astro-cli@v0.0.1 + with: + version: "1.45.0" # DAG bundles deployed outside an Astro project require CLI >= 1.45.0 +- name: Image Deploy to Astro + uses: astronomer/deploy-action@v0.16.0 + with: + deployment-id: + deploy-type: image-only + root-folder: astro-project +- name: Deploy team A's DAGs + uses: astronomer/deploy-action@v0.16.0 + with: + deployment-id: + deploy-type: dags-only + dags-path: bundles/team-a + dag-bundle-name: team-a +- name: Deploy team B's DAGs + uses: astronomer/deploy-action@v0.16.0 + with: + deployment-id: + deploy-type: dags-only + dags-path: bundles/team-b + dag-bundle-name: team-b +- name: Deploy SQL assets + uses: astronomer/deploy-action@v0.16.0 + with: + deployment-id: + deploy-type: non-dags + root-folder: sql + mount-path: /usr/local/airflow/sql + non-dags-bundle-type: sql +``` + ### Wake on deploy In the following example, the deployment is woken up from hibernation before deploying via the `wake-on-deploy` option. diff --git a/action.yaml b/action.yaml index 4bf500f..61fa67b 100644 --- a/action.yaml +++ b/action.yaml @@ -97,7 +97,7 @@ inputs: required: false default: true description: > - When true (default), the action skips the deploy if no changed files fall under `root-folder` (or + When true (default), the action skips the deploy if no changed files fall under `root-folder` or `dags-path` (or `image-trigger-paths` / `bundle-trigger-paths`). Set to false to disable this diff-based skip and always run the deploy dictated by `deploy-type` on every run. This is the closest replacement for the deprecated `deploy-image: true` behavior. build-secrets: @@ -125,6 +125,28 @@ inputs: Comma- or newline-separated list of paths outside `root-folder` that should also trigger a bundle deploy when changed. Useful in monorepos where a bundle depends on shared files that live outside its own directory. Paths are matched as prefixes against repo-root-relative file paths, so include a trailing slash to scope to a directory (e.g. `shared/macros/`). Only applies to the `dbt` and `non-dags` deploy types. + dag-bundle-name: + required: false + default: "" + description: > + Deploy DAGs to a named DAG bundle on the Deployment instead of the default bundle. Requires Airflow 3, and the + bundle must already exist on the Deployment. Use with `deploy-type: dags-only` or `image-and-dags`. Requires Astro CLI 1.44.0 or higher. + dags-path: + required: false + default: "" + description: > + Deploy DAGs from this path, relative to `root-folder`, instead of the project's `dags` directory. Useful for + serving several DAG bundles out of one repository, one action invocation per bundle. A `dags-only` deploy + classifies changes against this path instead of `dags/`, and with Astro CLI 1.45.0 or higher (and `parse` and + `pytest` off) it needs no Astro project, so `root-folder` can be left empty. Use with `deploy-type: dags-only` or + `image-and-dags`. + no-dags-base-dir: + required: false + default: false + description: > + If true, exclude the DAGs directory prefix from the uploaded bundle. Use for Airflow 3 Deployments, where the + bundle root is already on sys.path. Use with `deploy-type: infer`, `dags-only`, or `image-and-dags`, and not with + `image-name`. checkout-submodules: required: false default: false @@ -221,6 +243,15 @@ runs: fi fi + # Check if the Astro CLI version is less than 1.44.0 for named DAG bundles + if [[ "$DAG_BUNDLE_NAME" != "" && $CLI_VERSION != "" ]]; then + REQUIRED_VERSION="1.44.0" + if ! semver -r ">=${REQUIRED_VERSION}" "${CLI_VERSION}"; then + echo "The dag-bundle-name option requires Astro CLI version $REQUIRED_VERSION or higher" + exit 1 + fi + fi + # skip Astro CLI installation if already installed if command -v astro &> /dev/null; then echo "Astro CLI is already installed" @@ -248,6 +279,8 @@ runs: fi echo ::endgroup:: shell: bash + env: + DAG_BUNDLE_NAME: ${{ inputs.dag-bundle-name }} - name: Determine Deploy Deployment run: | @@ -325,8 +358,48 @@ runs: echo "ERROR: root-folder is required with deploy-type non-dags" exit 1 # terminate and indicate error fi - fi + # the DAG options below only apply to deploys that carry DAGs + for dag_option in "dag-bundle-name:$DAG_BUNDLE_NAME" "dags-path:$DAGS_PATH" "no-dags-base-dir:$NO_DAGS_BASE_DIR"; do + option_name="${dag_option%%:*}" + option_value="${dag_option#*:}" + if [[ -z "$option_value" || "$option_value" == "false" ]]; then + continue + fi + if [[ "${{ inputs.deploy-type }}" == "image-only" || "${{ inputs.deploy-type }}" == "dbt" || "${{ inputs.deploy-type }}" == "non-dags" ]]; then + echo "ERROR: $option_name is only supported with deploy-type infer, dags-only, or image-and-dags" + exit 1 # terminate and indicate error + fi + # the CLI rejects these options alongside a pre-built image, which carries its own DAGs + if [[ "${{ inputs.image-name }}" != "no-custom-image" ]]; then + echo "ERROR: $option_name cannot be used with image-name" + exit 1 # terminate and indicate error + fi + done + # a step deploying one bundle must never fall back to building the project's image, which infer + # does for any change it doesn't classify as a DAG + if [[ ( -n "$DAG_BUNDLE_NAME" || -n "$DAGS_PATH" ) && "${{ inputs.deploy-type }}" == "infer" ]]; then + echo "ERROR: dag-bundle-name and dags-path cannot be used with deploy-type infer (the default); set deploy-type to dags-only, or image-and-dags to also deploy the image" + exit 1 # terminate and indicate error + fi + + # dags-path is resolved from root-folder, the CLI's working directory; changes are matched against + # its repo-relative form, so DAGs outside the Astro project are still picked up + DAGS_DIR="" + if [[ -n "$DAGS_PATH" ]]; then + if ! dags_abs=$(cd "${ROOT_FOLDER:-.}" 2> /dev/null && cd "$DAGS_PATH" 2> /dev/null && pwd); then + echo "ERROR: dags-path '$DAGS_PATH' is not a directory" + exit 1 # terminate and indicate error + fi + if [[ $dags_abs != "$PWD" && $dags_abs != "$PWD"/* ]]; then + echo "ERROR: dags-path must resolve to a directory inside the repository" + exit 1 # terminate and indicate error + fi + DAGS_DIR=${dags_abs#"$PWD"} + DAGS_DIR=${DAGS_DIR#/} + fi + echo "DAGS_DIR=$DAGS_DIR" >> $GITHUB_OUTPUT + fi # Select workspace if specified if [[ "${{ inputs.workspace }}" != "" ]]; then @@ -468,6 +541,9 @@ runs: BUNDLE_TRIGGER_PATHS: ${{ inputs.bundle-trigger-paths }} MOUNT_PATH: ${{ inputs.mount-path }} NON_DAGS_BUNDLE_TYPE: ${{ inputs.non-dags-bundle-type }} + DAG_BUNDLE_NAME: ${{ inputs.dag-bundle-name }} + DAGS_PATH: ${{ inputs.dags-path }} + NO_DAGS_BASE_DIR: ${{ inputs.no-dags-base-dir }} - name: Determine if DAG Deploy is enabled run: | @@ -590,6 +666,17 @@ runs: DIFF_UNAVAILABLE=false files=() + # a changed file counts as a DAG when it sits under the project's dags directory, or under + # dags-path when the DAGs are served from somewhere else (e.g. one bundle of several in a repo). + # DAGS_DIR is dags-path in repo-relative form, empty when it is the repo root itself. + is_dag_file() { + if [[ -n "$DAGS_PATH" ]]; then + [[ -z "$DAGS_DIR" || $1 == "$DAGS_DIR/"* ]] + else + [[ $1 == *"dags/"* ]] + fi + } + # collect image-trigger-paths (comma- or newline-separated), trimmed and non-empty IMAGE_TRIGGER_PATHS="${{ inputs.image-trigger-paths }}" IMAGE_TRIGGERS=() @@ -629,11 +716,12 @@ runs: fi done - if [[ $file =~ ^"${{ inputs.root-folder }}".* ]]; then - echo $file is part of the input root folder + # dags-path may sit outside root-folder (DAGs deployed independently of the Astro project) + if [[ $file =~ ^"${{ inputs.root-folder }}".* ]] || { [[ -n "$DAGS_PATH" ]] && is_dag_file "$file"; }; then + echo $file is part of the input root folder or dags-path SKIP_IMAGE_OR_DAGS_DEPLOY=false if [[ ${{ inputs.deploy-type }} == 'infer' ]]; then - if [[ $file == *"dags/"* ]]; then + if is_dag_file "$file"; then echo $file is part of dags folder DAGS_ONLY_DEPLOY=true else @@ -641,7 +729,7 @@ runs: break fi elif [[ ${{ inputs.deploy-type }} == 'dags-only' ]]; then - if [[ $file == *"dags/"* ]]; then + if is_dag_file "$file"; then echo $file is part of dags folder DAGS_ONLY_DEPLOY=true fi @@ -708,6 +796,10 @@ runs: echo ::endgroup:: shell: bash id: deploy-type + env: + ROOT_FOLDER: ${{ inputs.root-folder }} + DAGS_PATH: ${{ inputs.dags-path }} + DAGS_DIR: ${{ steps.deployment-preview.outputs.DAGS_DIR }} # If only DAGs changed and dag deploys is enabled, do a DAG-only deploy - name: Setup image or dags deploy options if: ${{ inputs.deploy-type == 'infer' || inputs.deploy-type == 'dags-only' || inputs.deploy-type == 'image-and-dags' || inputs.deploy-type == 'image-only' }} @@ -776,12 +868,33 @@ runs: fi fi + # the DAG options below are meaningless for an image-only deploy, which leaves DAGs untouched + if [[ ${{ inputs.deploy-type }} != "image-only" ]]; then + # add named DAG bundle option + if [[ -n "$DAG_BUNDLE_NAME" ]]; then + options="$options --dag-bundle-name $(printf '%q' "$DAG_BUNDLE_NAME")" + fi + + # add custom dags path option + if [[ -n "$DAGS_PATH" ]]; then + options="$options --dags-path $(printf '%q' "$DAGS_PATH")" + fi + + # add no dags base dir option + if [[ "$NO_DAGS_BASE_DIR" == true ]]; then + options="$options --no-dags-base-dir" + fi + fi + echo "OPTIONS=$options" >> $GITHUB_OUTPUT echo ::endgroup:: shell: bash + id: deploy-options env: BUILD_SECRETS: ${{ inputs.build-secrets }} - id: deploy-options + DAG_BUNDLE_NAME: ${{ inputs.dag-bundle-name }} + DAGS_PATH: ${{ inputs.dags-path }} + NO_DAGS_BASE_DIR: ${{ inputs.no-dags-base-dir }} - name: Determine if Development Mode is enabled if: ${{ inputs.wake-on-deploy == 'true' && inputs.action != 'delete-deployment-preview' }} run: | diff --git a/e2e-setup/bundles/team_a/team_a_dag.py b/e2e-setup/bundles/team_a/team_a_dag.py new file mode 100644 index 0000000..7150e60 --- /dev/null +++ b/e2e-setup/bundles/team_a/team_a_dag.py @@ -0,0 +1,28 @@ +""" +DAG served from a non-default path, for the named DAG bundle e2e test. + +The test deploys this directory with `dags-path` and `dag-bundle-name` from +the repository root, outside any Astro project, so it exercises the custom +DAGs path, the named bundle, and a DAG deploy that needs no project. It is +deliberately trivial: it only has to parse and be uploaded. +""" + +from airflow.decorators import dag, task +from pendulum import datetime + + +@dag( + start_date=datetime(2026, 1, 1), + schedule=None, + catchup=False, + tags=["e2e", "team-a"], +) +def team_a_dag(): + @task + def hello() -> str: + return "hello from team a" + + hello() + + +team_a_dag()