diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 97865f8..9e316c2 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -5,6 +5,7 @@ on: # yamllint disable-line rule:truthy paths: - transform/* - transform/**/* + - automate/dbt/** # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -15,12 +16,60 @@ concurrency: cancel-in-progress: true jobs: + validate-branch: + name: Validate Branch + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + + steps: + - name: Checkout branch + uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set Secure Directory + run: git config --global --add safe.directory $GITHUB_WORKSPACE + + - name: Validate branch naming and sync with main + run: automate/git/validate_branch.sh "${{ github.head_ref }}" "${{ github.base_ref }}" + + detect-changes: + name: Detect dbt-related Changes + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + + outputs: + dbt_changed: ${{ steps.check.outputs.dbt_changed }} + + steps: + - name: Checkout branch + uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check for dbt-related changes + id: check + run: | + git fetch origin ${{ github.base_ref }} + CHANGED=$(git diff --name-only origin/${{ github.base_ref }} HEAD -- transform/ automate/dbt/) + if [ -n "$CHANGED" ]; then + echo "dbt_changed=true" >> $GITHUB_OUTPUT + else + echo "dbt_changed=false" >> $GITHUB_OUTPUT + fi + dbt: name: Pull Request dbt Tests + needs: [validate-branch, detect-changes] + # always(): needed deps are skipped (not failed) on workflow_dispatch since they're PR-only; + # this still blocks on a real validate-branch failure, and requires a detected change on PRs. + if: ${{ always() && needs.validate-branch.result != 'failure' && (github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.dbt_changed == 'true') }} runs-on: ubuntu-latest # most people should use this one - container: datacoves/ci-basic-dbt-snowflake:3.3 + container: datacoves/ci-basic-dbt-snowflake:5.0 defaults: run: @@ -89,6 +138,15 @@ jobs: ##### Real dbt run given that we passed governance checks + # Slim CI: skip rebuilding models unchanged by this PR, clone their prod tables instead. + - name: Clone unchanged incremental models from prod + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} + run: "dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs" + + - name: Clone unchanged snapshot models from prod + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} + run: "dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs" + - name: Run dbt build slim mode if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} run: "dbt build --fail-fast --defer --state logs --select state:modified+" diff --git a/.github/workflows/push-to-main.yml b/.github/workflows/push-to-main.yml index d8fff3e..22ea47b 100644 --- a/.github/workflows/push-to-main.yml +++ b/.github/workflows/push-to-main.yml @@ -25,8 +25,10 @@ jobs: name: Deployment Script runs-on: ubuntu-latest + permissions: + contents: write - container: datacoves/ci-basic-dbt-snowflake:3.3 + container: datacoves/ci-basic-dbt-snowflake:5.0 defaults: run: @@ -68,17 +70,26 @@ jobs: id: prod-manifest run: "../automate/dbt/get_artifacts.sh" + - name: Check for full-refresh deploy marker + id: full-refresh-marker + run: | + if git log -1 --pretty=%B | grep -q "\[deploy:full-refresh\]"; then + echo "flag=--full-refresh" >> $GITHUB_OUTPUT + else + echo "flag=" >> $GITHUB_OUTPUT + fi + # Runs blue green with no deferral - name: Run blue / green deployment if: ${{ steps.prod-manifest.outputs.manifest_found == 'false' }} id: run-blue-green - run: "dbt-coves blue-green" + run: "dbt-coves blue-green ${{ steps.full-refresh-marker.outputs.flag }}" # Runs blue green with deferral - name: Run blue / green deployment with deferral if: ${{ steps.prod-manifest.outputs.manifest_found == 'true' }} id: run-blue-green-defer - run: "dbt-coves blue-green --defer" + run: "dbt-coves blue-green --defer ${{ steps.full-refresh-marker.outputs.flag }}" - name: Drop orphaned relations in db that are no longer in dbt run: "dbt run-operation drop_orphaned_relations --args '{\"dry_run\": false}'" @@ -95,6 +106,30 @@ jobs: - name: Upload dbt artifacts run: "dbt run-operation upload_artifacts" + # push-only: avoids double-bumping (a merge fires both pull_request:closed and push) + # and avoids tagging on an unmerged PR close or manual dispatch. + - name: Configure git user for deployment tagging + if: ${{ github.event_name == 'push' }} + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Set deployment version and tag + if: ${{ github.event_name == 'push' }} + run: "../automate/dbt/set_deployment_version.sh" + + - name: Push version bump and tags to main + if: ${{ github.event_name == 'push' }} + run: | + git push origin HEAD:main + git push origin --tags + + # Safety net for any failure above (not push-gated); drop is IF EXISTS so it's a + # no-op if dbt-coves already cleaned up staging on success. + - name: Drop staging database on blue/green failure + if: failure() + run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{ env.DATACOVES__MAIN__DATABASE }}_STAGING, recreate: False}'" + # Drops the temporary PR database drop-pr-db-on-close: name: Drop PR Database on Close @@ -108,7 +143,7 @@ jobs: # Alternatively, You can define multiple ENV for different workflows. # https://github.com///settings/environments # environment: PR_ENV - container: datacoves/ci-basic-dbt-snowflake:3.2 + container: datacoves/ci-basic-dbt-snowflake:5.0 defaults: run: diff --git a/automate/dbt/set_deployment_version.sh b/automate/dbt/set_deployment_version.sh new file mode 100755 index 0000000..e8c4740 --- /dev/null +++ b/automate/dbt/set_deployment_version.sh @@ -0,0 +1,19 @@ +#! /bin/bash +set -e +cd "$DATACOVES__DBT_HOME" +git fetch --tags +date_version=$(date -u +%Y%m%d) +# Version scheme: YYYYMMDD.N.0, N = next unused sequence number for today. +last_sequence=$( + git tag --list "${date_version}.*" | + sed -E "s/^${date_version}\.([0-9]+)\..*/\1/" | + grep -E '^[0-9]+$' | sort -n | tail -1 +) +if [ -z "$last_sequence" ]; then next_sequence=1; else next_sequence=$((last_sequence + 1)); fi +new_version="${date_version}.${next_sequence}.0" +# Anchored to line start so it only matches `version:`, not `config-version:`. +sed -i "s/^version:.*/version: '${new_version}'/g" dbt_project.yml +git add dbt_project.yml +# [skip ci] prevents this commit from retriggering push-to-main.yml. +git commit -m "Set deployment version to ${new_version} [skip ci]" +git tag -m "[skip ci]" "${new_version}" diff --git a/automate/git/validate_branch.sh b/automate/git/validate_branch.sh new file mode 100755 index 0000000..4aace92 --- /dev/null +++ b/automate/git/validate_branch.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +SOURCE_BRANCH="${1#refs/heads/}" +TARGET_BRANCH="${2#refs/heads/}" + +if [ "$SOURCE_BRANCH" = "$TARGET_BRANCH" ]; then + echo "Source and target branch are the same ($SOURCE_BRANCH); skipping branch validation." + exit 0 +fi + +if [[ "$SOURCE_BRANCH" != feature/* && "$SOURCE_BRANCH" != infra/* ]]; then + echo "ERROR: Source branch '$SOURCE_BRANCH' must start with 'feature/' or 'infra/'." + exit 1 +fi + +if [ "$TARGET_BRANCH" != "main" ] && ! [[ "$SOURCE_BRANCH" == feature/* && "$TARGET_BRANCH" == feature/* ]]; then + echo "ERROR: Target branch must be 'main', or, for a feature/ source branch, another feature/ branch. Got '$TARGET_BRANCH'." + exit 1 +fi + +git fetch origin main +# Require the PR branch to already contain latest main (avoids deploying stale merges). +commits_behind=$(git rev-list --count HEAD..origin/main) +if [ "$commits_behind" -gt 0 ]; then + echo "ERROR: Branch '$SOURCE_BRANCH' is $commits_behind commit(s) behind 'main'. Please merge/pull main into your branch and push again." + exit 1 +fi + +if [[ "$SOURCE_BRANCH" == infra/* ]]; then + changed_transform_files=$(git diff --name-only origin/main HEAD -- transform/) + if [ -n "$changed_transform_files" ]; then + echo "ERROR: infra/ branches cannot modify transform/. dbt model changes must use a feature/ branch." + echo "$changed_transform_files" + exit 1 + fi +fi + +echo "Branch validation passed: '$SOURCE_BRANCH' -> '$TARGET_BRANCH'."