From a7bc860f26c0de952436a9892f1e0a6e2ea9352c Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 10:48:27 -0500 Subject: [PATCH 01/11] Add branch-naming and sync validation gate to PR build --- .github/workflows/pull_request_build.yml | 19 ++++++++++++ automate/git/validate_branch.sh | 38 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100755 automate/git/validate_branch.sh diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 97865f8..a63c6ec 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -15,8 +15,27 @@ concurrency: cancel-in-progress: true jobs: + validate-branch: + name: Validate Branch + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + + steps: + - name: Checkout branch + uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set Secure Directory + run: git config --global --add safe.directory $GITHUB_WORKSPACE + + - name: Validate branch naming and sync with main + run: automate/git/validate_branch.sh "${{ github.head_ref }}" "${{ github.base_ref }}" + dbt: name: Pull Request dbt Tests + needs: validate-branch runs-on: ubuntu-latest # most people should use this one diff --git a/automate/git/validate_branch.sh b/automate/git/validate_branch.sh new file mode 100755 index 0000000..2b4bd20 --- /dev/null +++ b/automate/git/validate_branch.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +SOURCE_BRANCH="${1#refs/heads/}" +TARGET_BRANCH="${2#refs/heads/}" + +if [ "$SOURCE_BRANCH" = "$TARGET_BRANCH" ]; then + echo "Source and target branch are the same ($SOURCE_BRANCH); skipping branch validation." + exit 0 +fi + +if [[ "$SOURCE_BRANCH" != feature/* && "$SOURCE_BRANCH" != infra/* ]]; then + echo "ERROR: Source branch '$SOURCE_BRANCH' must start with 'feature/' or 'infra/'." + exit 1 +fi + +if [ "$TARGET_BRANCH" != "main" ]; then + echo "ERROR: Target branch must be 'main', got '$TARGET_BRANCH'." + exit 1 +fi + +git fetch origin main +commits_behind=$(git rev-list --count HEAD..origin/main) +if [ "$commits_behind" -gt 0 ]; then + echo "ERROR: Branch '$SOURCE_BRANCH' is $commits_behind commit(s) behind 'main'. Please merge/pull main into your branch and push again." + exit 1 +fi + +if [[ "$SOURCE_BRANCH" == infra/* ]]; then + changed_transform_files=$(git diff --name-only origin/main HEAD -- transform/) + if [ -n "$changed_transform_files" ]; then + echo "ERROR: infra/ branches cannot modify transform/. dbt model changes must use a feature/ branch." + echo "$changed_transform_files" + exit 1 + fi +fi + +echo "Branch validation passed: '$SOURCE_BRANCH' -> '$TARGET_BRANCH'." From 5d795b2bb4c57d90c6d99a0a93e84d8485ac3f21 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 10:52:42 -0500 Subject: [PATCH 02/11] Skip PR dbt job when no transform/automate/dbt changes detected --- .github/workflows/pull_request_build.yml | 31 +++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index a63c6ec..802ce08 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -5,6 +5,8 @@ on: # yamllint disable-line rule:truthy paths: - transform/* - transform/**/* + - automate/dbt/* + - automate/dbt/** # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -33,9 +35,36 @@ jobs: - name: Validate branch naming and sync with main run: automate/git/validate_branch.sh "${{ github.head_ref }}" "${{ github.base_ref }}" + detect-changes: + name: Detect dbt-related Changes + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + + outputs: + dbt_changed: ${{ steps.check.outputs.dbt_changed }} + + steps: + - name: Checkout branch + uses: actions/checkout@v3.5.0 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Check for dbt-related changes + id: check + run: | + git fetch origin ${{ github.base_ref }} + CHANGED=$(git diff --name-only origin/${{ github.base_ref }} HEAD -- transform/ automate/dbt/) + if [ -n "$CHANGED" ]; then + echo "dbt_changed=true" >> $GITHUB_OUTPUT + else + echo "dbt_changed=false" >> $GITHUB_OUTPUT + fi + dbt: name: Pull Request dbt Tests - needs: validate-branch + needs: [validate-branch, detect-changes] + if: ${{ always() && needs.validate-branch.result != 'failure' && (github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.dbt_changed == 'true') }} runs-on: ubuntu-latest # most people should use this one From 46d1b7ce281eca19c3befdbb5782733a5480d091 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 10:55:58 -0500 Subject: [PATCH 03/11] Clone unchanged incremental and snapshot models from prod before slim build --- .github/workflows/pull_request_build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 802ce08..ab42d61 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -137,6 +137,14 @@ jobs: ##### Real dbt run given that we passed governance checks + - name: Clone unchanged incremental models from prod + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} + run: "dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs" + + - name: Clone unchanged snapshot models from prod + if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} + run: "dbt clone -s state:modified+,config.materialized:snapshot,state:old --state logs" + - name: Run dbt build slim mode if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} run: "dbt build --fail-fast --defer --state logs --select state:modified+" From 300d54597fc84fea189491448933422ab08d1977 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 10:58:00 -0500 Subject: [PATCH 04/11] Bump PR build container image to datacoves/ci-basic-dbt-snowflake:5.0 --- .github/workflows/pull_request_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index ab42d61..14f99c7 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest # most people should use this one - container: datacoves/ci-basic-dbt-snowflake:3.3 + container: datacoves/ci-basic-dbt-snowflake:5.0 defaults: run: From e47314ffbf2b5ff96d63c831d851b69564e371b0 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 11:00:30 -0500 Subject: [PATCH 05/11] Support [deploy:full-refresh] commit marker for blue/green deploys --- .github/workflows/push-to-main.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-to-main.yml b/.github/workflows/push-to-main.yml index d8fff3e..0603d07 100644 --- a/.github/workflows/push-to-main.yml +++ b/.github/workflows/push-to-main.yml @@ -68,17 +68,26 @@ jobs: id: prod-manifest run: "../automate/dbt/get_artifacts.sh" + - name: Check for full-refresh deploy marker + id: full-refresh-marker + run: | + if git log -1 --pretty=%B | grep -q "\[deploy:full-refresh\]"; then + echo "flag=--full-refresh" >> $GITHUB_OUTPUT + else + echo "flag=" >> $GITHUB_OUTPUT + fi + # Runs blue green with no deferral - name: Run blue / green deployment if: ${{ steps.prod-manifest.outputs.manifest_found == 'false' }} id: run-blue-green - run: "dbt-coves blue-green" + run: "dbt-coves blue-green ${{ steps.full-refresh-marker.outputs.flag }}" # Runs blue green with deferral - name: Run blue / green deployment with deferral if: ${{ steps.prod-manifest.outputs.manifest_found == 'true' }} id: run-blue-green-defer - run: "dbt-coves blue-green --defer" + run: "dbt-coves blue-green --defer ${{ steps.full-refresh-marker.outputs.flag }}" - name: Drop orphaned relations in db that are no longer in dbt run: "dbt run-operation drop_orphaned_relations --args '{\"dry_run\": false}'" From 0870cf286d0d4c2e659ac3d51fa93a76402cd410 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 11:19:40 -0500 Subject: [PATCH 06/11] Bump dbt_project.yml version and tag after successful deploy --- .github/workflows/push-to-main.yml | 18 ++++++++++++++++++ automate/dbt/set_deployment_version.sh | 16 ++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 automate/dbt/set_deployment_version.sh diff --git a/.github/workflows/push-to-main.yml b/.github/workflows/push-to-main.yml index 0603d07..a467f55 100644 --- a/.github/workflows/push-to-main.yml +++ b/.github/workflows/push-to-main.yml @@ -25,6 +25,8 @@ jobs: name: Deployment Script runs-on: ubuntu-latest + permissions: + contents: write container: datacoves/ci-basic-dbt-snowflake:3.3 @@ -104,6 +106,22 @@ jobs: - name: Upload dbt artifacts run: "dbt run-operation upload_artifacts" + - name: Configure git user for deployment tagging + if: ${{ github.event_name == 'push' }} + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + - name: Set deployment version and tag + if: ${{ github.event_name == 'push' }} + run: "../automate/dbt/set_deployment_version.sh" + + - name: Push version bump and tags to main + if: ${{ github.event_name == 'push' }} + run: | + git push origin HEAD:main + git push origin --tags + # Drops the temporary PR database drop-pr-db-on-close: name: Drop PR Database on Close diff --git a/automate/dbt/set_deployment_version.sh b/automate/dbt/set_deployment_version.sh new file mode 100755 index 0000000..e3f30b6 --- /dev/null +++ b/automate/dbt/set_deployment_version.sh @@ -0,0 +1,16 @@ +#! /bin/bash +set -e +cd "$DATACOVES__DBT_HOME" +git fetch --tags +date_version=$(date -u +%Y%m%d) +last_sequence=$( + git tag --list "${date_version}.*" | + sed -E "s/^${date_version}\.([0-9]+)\..*/\1/" | + grep -E '^[0-9]+$' | sort -n | tail -1 +) +if [ -z "$last_sequence" ]; then next_sequence=1; else next_sequence=$((last_sequence + 1)); fi +new_version="${date_version}.${next_sequence}.0" +sed -i "s/^version:.*/version: '${new_version}'/g" dbt_project.yml +git add dbt_project.yml +git commit -m "Set deployment version to ${new_version} [skip ci]" +git tag -m "[skip ci]" "${new_version}" From ad10a3a023be796c1f1f7619e2148e297f3931ca Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 11:27:26 -0500 Subject: [PATCH 07/11] Drop blue/green staging database on deploy failure --- .github/workflows/push-to-main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/push-to-main.yml b/.github/workflows/push-to-main.yml index a467f55..8f54e4e 100644 --- a/.github/workflows/push-to-main.yml +++ b/.github/workflows/push-to-main.yml @@ -122,6 +122,10 @@ jobs: git push origin HEAD:main git push origin --tags + - name: Drop staging database on blue/green failure + if: failure() + run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{ env.DATACOVES__MAIN__DATABASE }}_STAGING, recreate: False}'" + # Drops the temporary PR database drop-pr-db-on-close: name: Drop PR Database on Close From ded066bb899955d4cbd2c9c2f303a164d1f7f070 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 11:37:19 -0500 Subject: [PATCH 08/11] Bump deploy workflow container images to datacoves/ci-basic-dbt-snowflake:5.0 --- .github/workflows/push-to-main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-to-main.yml b/.github/workflows/push-to-main.yml index 8f54e4e..b9ca730 100644 --- a/.github/workflows/push-to-main.yml +++ b/.github/workflows/push-to-main.yml @@ -28,7 +28,7 @@ jobs: permissions: contents: write - container: datacoves/ci-basic-dbt-snowflake:3.3 + container: datacoves/ci-basic-dbt-snowflake:5.0 defaults: run: @@ -139,7 +139,7 @@ jobs: # Alternatively, You can define multiple ENV for different workflows. # https://github.com///settings/environments # environment: PR_ENV - container: datacoves/ci-basic-dbt-snowflake:3.2 + container: datacoves/ci-basic-dbt-snowflake:5.0 defaults: run: From 6ce964db830c4bbb20936a9fc1863969a73be150 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 13:56:54 -0500 Subject: [PATCH 09/11] Add explanatory comments to non-obvious CI logic Documents the compound if: gates, push-only event guards, staging DB cleanup safety net, and version-bump script's tricky bits. --- .github/workflows/pull_request_build.yml | 3 +++ .github/workflows/push-to-main.yml | 4 ++++ automate/dbt/set_deployment_version.sh | 3 +++ automate/git/validate_branch.sh | 1 + 4 files changed, 11 insertions(+) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 14f99c7..60a28a2 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -64,6 +64,8 @@ jobs: dbt: name: Pull Request dbt Tests needs: [validate-branch, detect-changes] + # always(): needed deps are skipped (not failed) on workflow_dispatch since they're PR-only; + # this still blocks on a real validate-branch failure, and requires a detected change on PRs. if: ${{ always() && needs.validate-branch.result != 'failure' && (github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.dbt_changed == 'true') }} runs-on: ubuntu-latest @@ -137,6 +139,7 @@ jobs: ##### Real dbt run given that we passed governance checks + # Slim CI: skip rebuilding models unchanged by this PR, clone their prod tables instead. - name: Clone unchanged incremental models from prod if: ${{ steps.prod_manifest.outputs.manifest_found == 'true' && contains(github.event.pull_request.labels.*.name, 'full-refresh') != true }} run: "dbt clone -s state:modified+,config.materialized:incremental,state:old --state logs" diff --git a/.github/workflows/push-to-main.yml b/.github/workflows/push-to-main.yml index b9ca730..22ea47b 100644 --- a/.github/workflows/push-to-main.yml +++ b/.github/workflows/push-to-main.yml @@ -106,6 +106,8 @@ jobs: - name: Upload dbt artifacts run: "dbt run-operation upload_artifacts" + # push-only: avoids double-bumping (a merge fires both pull_request:closed and push) + # and avoids tagging on an unmerged PR close or manual dispatch. - name: Configure git user for deployment tagging if: ${{ github.event_name == 'push' }} run: | @@ -122,6 +124,8 @@ jobs: git push origin HEAD:main git push origin --tags + # Safety net for any failure above (not push-gated); drop is IF EXISTS so it's a + # no-op if dbt-coves already cleaned up staging on success. - name: Drop staging database on blue/green failure if: failure() run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{ env.DATACOVES__MAIN__DATABASE }}_STAGING, recreate: False}'" diff --git a/automate/dbt/set_deployment_version.sh b/automate/dbt/set_deployment_version.sh index e3f30b6..e8c4740 100755 --- a/automate/dbt/set_deployment_version.sh +++ b/automate/dbt/set_deployment_version.sh @@ -3,6 +3,7 @@ set -e cd "$DATACOVES__DBT_HOME" git fetch --tags date_version=$(date -u +%Y%m%d) +# Version scheme: YYYYMMDD.N.0, N = next unused sequence number for today. last_sequence=$( git tag --list "${date_version}.*" | sed -E "s/^${date_version}\.([0-9]+)\..*/\1/" | @@ -10,7 +11,9 @@ last_sequence=$( ) if [ -z "$last_sequence" ]; then next_sequence=1; else next_sequence=$((last_sequence + 1)); fi new_version="${date_version}.${next_sequence}.0" +# Anchored to line start so it only matches `version:`, not `config-version:`. sed -i "s/^version:.*/version: '${new_version}'/g" dbt_project.yml git add dbt_project.yml +# [skip ci] prevents this commit from retriggering push-to-main.yml. git commit -m "Set deployment version to ${new_version} [skip ci]" git tag -m "[skip ci]" "${new_version}" diff --git a/automate/git/validate_branch.sh b/automate/git/validate_branch.sh index 2b4bd20..1106d36 100755 --- a/automate/git/validate_branch.sh +++ b/automate/git/validate_branch.sh @@ -20,6 +20,7 @@ if [ "$TARGET_BRANCH" != "main" ]; then fi git fetch origin main +# Require the PR branch to already contain latest main (avoids deploying stale merges). commits_behind=$(git rev-list --count HEAD..origin/main) if [ "$commits_behind" -gt 0 ]; then echo "ERROR: Branch '$SOURCE_BRANCH' is $commits_behind commit(s) behind 'main'. Please merge/pull main into your branch and push again." From 2f84c3eb04d49a0dcc5a4f21a1827ab2d6ab3c02 Mon Sep 17 00:00:00 2001 From: Fernando Mercado Date: Tue, 4 Aug 2026 14:30:06 -0500 Subject: [PATCH 10/11] Allow feature branches to target another feature branch in validate_branch.sh --- automate/git/validate_branch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automate/git/validate_branch.sh b/automate/git/validate_branch.sh index 1106d36..4aace92 100755 --- a/automate/git/validate_branch.sh +++ b/automate/git/validate_branch.sh @@ -14,8 +14,8 @@ if [[ "$SOURCE_BRANCH" != feature/* && "$SOURCE_BRANCH" != infra/* ]]; then exit 1 fi -if [ "$TARGET_BRANCH" != "main" ]; then - echo "ERROR: Target branch must be 'main', got '$TARGET_BRANCH'." +if [ "$TARGET_BRANCH" != "main" ] && ! [[ "$SOURCE_BRANCH" == feature/* && "$TARGET_BRANCH" == feature/* ]]; then + echo "ERROR: Target branch must be 'main', or, for a feature/ source branch, another feature/ branch. Got '$TARGET_BRANCH'." exit 1 fi From 9f7adaf32707d90f5aeed490d0302429a46c948a Mon Sep 17 00:00:00 2001 From: fmercadop <90359057+fmercadop@users.noreply.github.com> Date: Wed, 5 Aug 2026 06:46:00 -0700 Subject: [PATCH 11/11] Remove redundant path --- .github/workflows/pull_request_build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull_request_build.yml b/.github/workflows/pull_request_build.yml index 60a28a2..9e316c2 100644 --- a/.github/workflows/pull_request_build.yml +++ b/.github/workflows/pull_request_build.yml @@ -5,7 +5,6 @@ on: # yamllint disable-line rule:truthy paths: - transform/* - transform/**/* - - automate/dbt/* - automate/dbt/** # Allows you to run this workflow manually from the Actions tab