From f2ce4a0883ffca38d3b249c6567a7793b6b4c747 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 17:32:15 +0000
Subject: [PATCH 01/17] tests: parallel batching for packages when running unit
 tests

---
 .github/workflows/unittest.yml | 107 +++++++++++++++++++++++++++++++--
 ci/run_conditional_tests.sh    |  27 +++++++--
 2 files changed, 124 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 3929a0145963..57d3db310884 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -12,13 +12,71 @@ name: unittest
 permissions:
   contents: read
 
+# Configurable global environment variables for batching
+env:
+  BATCH_SIZE: 10
+  TEST_ALL_PACKAGES: "true" # Set to "false" to only run tests for packages with a git diff
+
 jobs:
+  # Dynamic package discovery job to calculate required matrix size automatically
+  discover-packages:
+    runs-on: ubuntu-latest
+    outputs:
+      batch-indices: ${{ steps.set-matrix.outputs.indices }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      with:
+        persist-credentials: false
+    - name: Generate Batch Indices
+      id: set-matrix
+      run: |
+        # Testing all monorepo packages sequentially on a single runner node is 
+        # too slow and creates a severe CI bottleneck as the repository expands.
+        #
+        # To scale efficiently, we chunk the workload into parallel batch slices.
+        # Instead of using a fixed, hardcoded matrix array (which risks silently 
+        # skipping newly added packages if the repository outgrows the array size), 
+        # this step dynamically audits the 'packages/' directory at runtime.
+        #
+        # It calculates exactly how many concurrent runners are required based on
+        # the current repository size and the configured BATCH_SIZE variable, 
+        # ensuring 100% test coverage with zero manual YAML maintenance.
+
+        # 1. Count the number of total directories matching the 'packages/*' pattern.
+        #    Redirect stderr to /dev/null so empty repos do not print unneeded errors.
+        TOTAL_PACKAGES=$(ls -d packages/*/ 2>/dev/null | wc -l | tr -d ' ')
+        
+        # 2. Safety fallback: If no packages are detected, assign a single slice index [0]
+        #    so subsequent matrix-dependent jobs do not break or fail validation on an empty matrix.
+        if [ "$TOTAL_PACKAGES" -eq 0 ]; then
+          echo "indices=[0]" >> "$GITHUB_OUTPUT"
+          exit 0
+        fi
+
+        # 3. Calculate the number of batches required using ceiling division: ceil(TOTAL_PACKAGES / BATCH_SIZE).
+        #    The formula ((A + B - 1) / B) ensures integer division rounds up if there's any remaining package leftover.
+        #    Example: 251 packages with a batch size of 10 gives ((251 + 10 - 1) / 10) = 260 / 10 = 26 batches.
+        NUM_BATCHES=$(( (TOTAL_PACKAGES + ${{ env.BATCH_SIZE }} - 1) / ${{ env.BATCH_SIZE }} ))
+        
+        # 4. Generate a zero-indexed sequence from 0 to (NUM_BATCHES - 1).
+        #    Use jq to securely parse the raw numbers and compile them into a compacted JSON array string.
+        #    Example output format: [0,1,2,3,...,25]
+        INDICES=$(seq 0 $((NUM_BATCHES - 1)) | jq -R . | jq -s -c .)
+        
+        # 5. Output the finished JSON string to the GitHub environment outputs pipeline.
+        #    This will safely feed directly into the execution matrix downstream.
+        echo "indices=${INDICES}" >> "$GITHUB_OUTPUT"
+
   unit:
+    name: "unit-run (${{ matrix.python }}, Batch ${{ matrix.batch-index }})"
     runs-on: ubuntu-22.04
+    needs: discover-packages
     strategy:
-      fail-fast: true
       matrix:
-        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
+        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14", "3.15"]
+        # Dynamically scales to fit every package perfectly without hardcoding array indices
+        batch-index: ${{ fromJson(needs.discover-packages.outputs.batch-indices) }}
     steps:
     - name: Checkout
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
@@ -39,23 +97,41 @@ jobs:
     - name: Run unit tests
       env:
         COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}
-        BUILD_TYPE: presubmit
+        # Dynamically set BUILD_TYPE to an empty string to skip the diff calculation if TEST_ALL_PACKAGES is true
+        BUILD_TYPE: ${{ env.TEST_ALL_PACKAGES == 'true' && '' || 'presubmit' }}
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         TEST_TYPE: unit
         PY_VERSION: ${{ matrix.python }}
       run: |
-        ci/run_conditional_tests.sh
+        # Gather all packages in alphabetical order
+        ALL_PACKAGES=($(ls -d packages/*/ | sort))
+        TOTAL_PACKAGES=${#ALL_PACKAGES[@]}
+
+        # Determine this runner's slice window
+        START_INDEX=$(( ${{ matrix.batch-index }} * ${{ env.BATCH_SIZE }} ))
+        
+        if [ $START_INDEX -ge $TOTAL_PACKAGES ]; then
+          exit 0
+        fi
+
+        BATCH_PACKAGES=("${ALL_PACKAGES[@]:$START_INDEX:${{ env.BATCH_SIZE }}}")
+
+        # Strip trailing slashes to pass down directly into ci/run_conditional_tests.sh
+        subdirs=("${BATCH_PACKAGES[@]%/}")
+
+        ci/run_conditional_tests.sh "${subdirs[@]}"
     - name: Upload coverage results
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
-        name: coverage-artifact-${{ matrix.python }}
+        # Appended batch-index to separate parallel coverage uploads cleanly
+        name: coverage-artifact-${{ matrix.python }}-${{ matrix.batch-index }}
         path: .coverage-${{ matrix.python }}
         include-hidden-files: true
 
   cover:
     runs-on: ubuntu-latest
     needs:
-        - unit
+      - unit
     steps:
     - name: Checkout
       uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -167,3 +243,22 @@ jobs:
           echo "This usually means the unit tests did not run or failed to upload their coverage files."
           exit 1
         fi
+
+  unittest-runtime-result:
+    name: "unit (${{ matrix.python }})"
+    needs: unit
+    if: always()
+    strategy:
+      matrix:
+        python: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15']
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check unit tests results
+        run: |
+          UNIT_STATUS="${{ needs.unit.result }}"
+
+        if [[ "$UNIT_STATUS" == "success" ]]; then
+            echo "Python ${{ matrix.python }} tests passed."
+          else
+            echo "Error: Python ${{ matrix.python }} status is '$UNIT_STATUS'."
+            exit 1
diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 9b8eaee52e5b..a9c6ac2f9727 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -21,6 +21,10 @@
 
 # `TEST_TYPE` and `PY_VERSION` are required by the script `ci/run_single_test.sh`
 
+# Optional Arguments:
+# Pass specific space-separated package paths (e.g., "packages/google-cloud-storage") to only test those directories.
+# If no arguments are provided, the script automatically determines which directories have changed
+#
 # This script will determine which directories have changed
 # under the `packages` folder. For `BUILD_TYPE=="presubmit"`,
 # we'll compare against the `packages` folder in HEAD,
@@ -78,14 +82,29 @@ set -e
 # Now we have a fixed list, but we can change it to autodetect if
 # necessary.
 
-subdirs=(
-    packages
-)
+if [ $# -gt 0 ]; then
+    subdirs=("$@")
+else
+    subdirs=(
+        packages
+    )
+fi
 
 RETVAL=0
 
 for subdir in ${subdirs[@]}; do
-    for d in `ls -d ${subdir}/*/`; do
+    # If a specific package path was passed directly, use it; otherwise scan the parent folder
+    if [ -d "${subdir}" ] && [[ "${subdir}" != "packages" ]]; then
+        loop_dirs=("${subdir}")
+    else
+        loop_dirs=(`ls -d ${subdir}/*/`)
+    fi
+
+    for d in "${loop_dirs[@]}"; do
+        # Ensure the directory path always ends with a trailing slash for git diff safety
+        if [[ "$d" != */ ]]; then
+            d="$d/"
+        fi
         should_test=false
         if [ -n "${GIT_DIFF_ARG}" ]; then
             echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'"

From 8cbf92eb4f1c663fb403077b789171a82d0f5b66 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 14:36:08 -0400
Subject: [PATCH 02/17] Update ci/run_conditional_tests.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 ci/run_conditional_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index a9c6ac2f9727..4126ae07ab92 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -94,7 +94,7 @@ RETVAL=0
 
 for subdir in ${subdirs[@]}; do
     # If a specific package path was passed directly, use it; otherwise scan the parent folder
-    if [ -d "${subdir}" ] && [[ "${subdir}" != "packages" ]]; then
+    if [ -d "${subdir}" ] && [[ "${subdir%/}" != "packages" ]]; then
         loop_dirs=("${subdir}")
     else
         loop_dirs=(`ls -d ${subdir}/*/`)

From ef700392b98d82a29b78b9aabcf74916b4be1784 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 14:36:16 -0400
Subject: [PATCH 03/17] Update ci/run_conditional_tests.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 ci/run_conditional_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 4126ae07ab92..69277f0b28d1 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -97,7 +97,7 @@ for subdir in ${subdirs[@]}; do
     if [ -d "${subdir}" ] && [[ "${subdir%/}" != "packages" ]]; then
         loop_dirs=("${subdir}")
     else
-        loop_dirs=(`ls -d ${subdir}/*/`)
+        loop_dirs=("${subdir}"/*/)
     fi
 
     for d in "${loop_dirs[@]}"; do

From e8b90afa0b565c84b1c6a4801cbb67119aa694f6 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 18:37:10 +0000
Subject: [PATCH 04/17] Typo

---
 .github/workflows/unittest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 57d3db310884..fbc301e9fb83 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -257,7 +257,7 @@ jobs:
         run: |
           UNIT_STATUS="${{ needs.unit.result }}"
 
-        if [[ "$UNIT_STATUS" == "success" ]]; then
+          if [[ "$UNIT_STATUS" == "success" ]]; then
             echo "Python ${{ matrix.python }} tests passed."
           else
             echo "Error: Python ${{ matrix.python }} status is '$UNIT_STATUS'."

From 47e915c5bb1968c424cf5821dc1a78fb2a3b4d94 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 18:38:40 +0000
Subject: [PATCH 05/17] remove 3.15

---
 .github/workflows/unittest.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index fbc301e9fb83..745e61377a14 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -74,7 +74,7 @@ jobs:
     needs: discover-packages
     strategy:
       matrix:
-        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14", "3.15"]
+        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
         # Dynamically scales to fit every package perfectly without hardcoding array indices
         batch-index: ${{ fromJson(needs.discover-packages.outputs.batch-indices) }}
     steps:
@@ -250,7 +250,7 @@ jobs:
     if: always()
     strategy:
       matrix:
-        python: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14', '3.15']
+        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
     runs-on: ubuntu-latest
     steps:
       - name: Check unit tests results

From 492ee5fdffc7babe38f833bc2aa79055ecab080a Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 18:41:00 +0000
Subject: [PATCH 06/17] Typo

---
 .github/workflows/unittest.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 745e61377a14..ed69c5eb0bf7 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -262,3 +262,4 @@ jobs:
           else
             echo "Error: Python ${{ matrix.python }} status is '$UNIT_STATUS'."
             exit 1
+          fi

From 08933fd70d0a938165e49cf2307148acf4dd7042 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 14:45:27 -0400
Subject: [PATCH 07/17] Update ci/run_conditional_tests.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 ci/run_conditional_tests.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 69277f0b28d1..1d6fe87a4a3f 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -101,6 +101,9 @@ for subdir in ${subdirs[@]}; do
     fi
 
     for d in "${loop_dirs[@]}"; do
+        if [ ! -d "$d" ]; then
+            continue
+        fi
         # Ensure the directory path always ends with a trailing slash for git diff safety
         if [[ "$d" != */ ]]; then
             d="$d/"

From c85a266c477f2f09c7e66639b5459b51c593de12 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 14:45:41 -0400
Subject: [PATCH 08/17] Update ci/run_conditional_tests.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 ci/run_conditional_tests.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 1d6fe87a4a3f..02a8b706b474 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -93,11 +93,15 @@ fi
 RETVAL=0
 
 for subdir in ${subdirs[@]}; do
-    # If a specific package path was passed directly, use it; otherwise scan the parent folder
-    if [ -d "${subdir}" ] && [[ "${subdir%/}" != "packages" ]]; then
-        loop_dirs=("${subdir}")
-    else
+    if [ ! -d "${subdir}" ]; then
+        echo "Error: Directory '${subdir}' does not exist." >&2
+        exit 1
+    fi
+
+    if [[ "${subdir%/}" == "packages" ]]; then
         loop_dirs=("${subdir}"/*/)
+    else
+        loop_dirs=("${subdir}")
     fi
 
     for d in "${loop_dirs[@]}"; do

From 75e565736e2d021dd39efe3051e583abf67e0156 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 18:48:53 +0000
Subject: [PATCH 09/17] fix TEST_ALL_PACKAGES override

---
 ci/run_conditional_tests.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 02a8b706b474..8d7d5d5c3642 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -113,7 +113,12 @@ for subdir in ${subdirs[@]}; do
             d="$d/"
         fi
         should_test=false
-        if [ -n "${GIT_DIFF_ARG}" ]; then
+        
+        # Override check: Force test if explicitly asked to test all packages
+        if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+            echo "TEST_ALL_PACKAGES is true, forcing execution for ${d}"
+            should_test=true
+        elif [ -n "${GIT_DIFF_ARG}" ]; then
             echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'"
             set +e
             git diff --quiet ${GIT_DIFF_ARG} ${d}
@@ -145,4 +150,4 @@ for subdir in ${subdirs[@]}; do
     done
 done
 
-exit ${RETVAL}
+exit ${RETVAL}
\ No newline at end of file

From d3a042e3a991a208366544c93974abb1be2dd0d8 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 20:16:58 +0000
Subject: [PATCH 10/17] fix presubmit

---
 .github/workflows/unittest.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index ed69c5eb0bf7..3cd5289406c8 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -247,7 +247,6 @@ jobs:
   unittest-runtime-result:
     name: "unit (${{ matrix.python }})"
     needs: unit
-    if: always()
     strategy:
       matrix:
         python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]

From 4898748b8275430c60b67ad8ce6517314ad4bcc6 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Thu, 2 Jul 2026 20:59:30 +0000
Subject: [PATCH 11/17] update --cov-fail-under

---
 packages/django-google-spanner/noxfile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/django-google-spanner/noxfile.py b/packages/django-google-spanner/noxfile.py
index 0b94e5b87bff..d240345c76ea 100644
--- a/packages/django-google-spanner/noxfile.py
+++ b/packages/django-google-spanner/noxfile.py
@@ -131,7 +131,7 @@ def default(session):
         "--cov-append",
         "--cov-config=.coveragerc",
         "--cov-report=",
-        "--cov-fail-under=75",
+        "--cov-fail-under=0",
         os.path.join("tests", "unit"),
         *session.posargs,
     )

From bbd81442073bf361b36885fcd47e51adf8b4fba3 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Mon, 6 Jul 2026 15:47:37 +0000
Subject: [PATCH 12/17] add weighting

---
 .github/workflows/unittest.yml | 135 ++++++++++++++++++---------------
 ci/get_batches.py              |  81 ++++++++++++++++++++
 2 files changed, 154 insertions(+), 62 deletions(-)
 create mode 100644 ci/get_batches.py

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 3cd5289406c8..de46b0dca33c 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -14,11 +14,19 @@ permissions:
 
 # Configurable global environment variables for batching
 env:
-  BATCH_SIZE: 10
   TEST_ALL_PACKAGES: "true" # Set to "false" to only run tests for packages with a git diff
+  ALL_PYTHON: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]'
 
 jobs:
   # Dynamic package discovery job to calculate required matrix size automatically
+  python_config:
+    runs-on: ubuntu-latest
+    outputs:
+      all_python: ${{ steps.export.outputs.python_list }}
+    steps:
+      - id: export
+        run: echo "python_list=${{ env.ALL_PYTHON }}" >> "$GITHUB_OUTPUT"
+
   discover-packages:
     runs-on: ubuntu-latest
     outputs:
@@ -28,54 +36,23 @@ jobs:
       uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
       with:
         persist-credentials: false
+    - name: Setup Python
+      uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
+      with:
+        python-version: "3.14"
     - name: Generate Batch Indices
       id: set-matrix
       run: |
-        # Testing all monorepo packages sequentially on a single runner node is 
-        # too slow and creates a severe CI bottleneck as the repository expands.
-        #
-        # To scale efficiently, we chunk the workload into parallel batch slices.
-        # Instead of using a fixed, hardcoded matrix array (which risks silently 
-        # skipping newly added packages if the repository outgrows the array size), 
-        # this step dynamically audits the 'packages/' directory at runtime.
-        #
-        # It calculates exactly how many concurrent runners are required based on
-        # the current repository size and the configured BATCH_SIZE variable, 
-        # ensuring 100% test coverage with zero manual YAML maintenance.
-
-        # 1. Count the number of total directories matching the 'packages/*' pattern.
-        #    Redirect stderr to /dev/null so empty repos do not print unneeded errors.
-        TOTAL_PACKAGES=$(ls -d packages/*/ 2>/dev/null | wc -l | tr -d ' ')
-        
-        # 2. Safety fallback: If no packages are detected, assign a single slice index [0]
-        #    so subsequent matrix-dependent jobs do not break or fail validation on an empty matrix.
-        if [ "$TOTAL_PACKAGES" -eq 0 ]; then
-          echo "indices=[0]" >> "$GITHUB_OUTPUT"
-          exit 0
-        fi
-
-        # 3. Calculate the number of batches required using ceiling division: ceil(TOTAL_PACKAGES / BATCH_SIZE).
-        #    The formula ((A + B - 1) / B) ensures integer division rounds up if there's any remaining package leftover.
-        #    Example: 251 packages with a batch size of 10 gives ((251 + 10 - 1) / 10) = 260 / 10 = 26 batches.
-        NUM_BATCHES=$(( (TOTAL_PACKAGES + ${{ env.BATCH_SIZE }} - 1) / ${{ env.BATCH_SIZE }} ))
-        
-        # 4. Generate a zero-indexed sequence from 0 to (NUM_BATCHES - 1).
-        #    Use jq to securely parse the raw numbers and compile them into a compacted JSON array string.
-        #    Example output format: [0,1,2,3,...,25]
-        INDICES=$(seq 0 $((NUM_BATCHES - 1)) | jq -R . | jq -s -c .)
-        
-        # 5. Output the finished JSON string to the GitHub environment outputs pipeline.
-        #    This will safely feed directly into the execution matrix downstream.
+        INDICES=$(python -c 'import sys; sys.path.append("ci"); import get_batches; print(get_batches.get_batch_indices())')
         echo "indices=${INDICES}" >> "$GITHUB_OUTPUT"
 
   unit:
     name: "unit-run (${{ matrix.python }}, Batch ${{ matrix.batch-index }})"
     runs-on: ubuntu-22.04
-    needs: discover-packages
+    needs: [python_config, discover-packages]
     strategy:
       matrix:
-        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
-        # Dynamically scales to fit every package perfectly without hardcoding array indices
+        python: ${{ fromJSON(needs.python_config.outputs.all_python) }}
         batch-index: ${{ fromJson(needs.discover-packages.outputs.batch-indices) }}
     steps:
     - name: Checkout
@@ -95,43 +72,47 @@ jobs:
         python -m pip install --upgrade setuptools pip wheel
         python -m pip install nox
     - name: Run unit tests
+      id: run-tests
       env:
-        COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}
+        COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}-${{ matrix.batch-index }}
         # Dynamically set BUILD_TYPE to an empty string to skip the diff calculation if TEST_ALL_PACKAGES is true
         BUILD_TYPE: ${{ env.TEST_ALL_PACKAGES == 'true' && '' || 'presubmit' }}
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         TEST_TYPE: unit
         PY_VERSION: ${{ matrix.python }}
       run: |
-        # Gather all packages in alphabetical order
-        ALL_PACKAGES=($(ls -d packages/*/ | sort))
-        TOTAL_PACKAGES=${#ALL_PACKAGES[@]}
+        UNIQUE_BATCH_PACKAGES=$(python -c 'import sys; sys.path.append("ci"); import get_batches; print(get_batches.get_batch_slice(${{ matrix.batch-index }}))')
 
-        # Determine this runner's slice window
-        START_INDEX=$(( ${{ matrix.batch-index }} * ${{ env.BATCH_SIZE }} ))
-        
-        if [ $START_INDEX -ge $TOTAL_PACKAGES ]; then
+        if [ -z "$UNIQUE_BATCH_PACKAGES" ]; then
+          echo "No structural units allocated to this matrix slice."
           exit 0
         fi
 
-        BATCH_PACKAGES=("${ALL_PACKAGES[@]:$START_INDEX:${{ env.BATCH_SIZE }}}")
+        echo "Running tests for packages in this weighted batch: ${UNIQUE_BATCH_PACKAGES}"
+        ci/run_conditional_tests.sh ${UNIQUE_BATCH_PACKAGES}
 
-        # Strip trailing slashes to pass down directly into ci/run_conditional_tests.sh
-        subdirs=("${BATCH_PACKAGES[@]%/}")
+    - name: Save Status Footprint
+      run: |
+        mkdir -p footprints
+        echo "${{ steps.run-tests.outcome }}" > footprints/status.txt
+
+    - name: Upload Status Footprint
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: footprint-${{ matrix.python }}-${{ matrix.batch-index }}
+        path: footprints/
 
-        ci/run_conditional_tests.sh "${subdirs[@]}"
     - name: Upload coverage results
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         # Appended batch-index to separate parallel coverage uploads cleanly
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.batch-index }}
-        path: .coverage-${{ matrix.python }}
+        path: .coverage-${{ matrix.python }}-${{ matrix.batch-index }}
         include-hidden-files: true
 
   cover:
     runs-on: ubuntu-latest
-    needs:
-      - unit
+    needs: [unit]
     steps:
     - name: Checkout
       uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
@@ -246,19 +227,49 @@ jobs:
 
   unittest-runtime-result:
     name: "unit (${{ matrix.python }})"
-    needs: unit
+    needs: [python_config, discover-packages, unit]
+    if: always()
     strategy:
+      fail-fast: false
       matrix:
-        python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
+        python: ${{ fromJSON(needs.python_config.outputs.all_python) }}
     runs-on: ubuntu-latest
     steps:
-      - name: Check unit tests results
-        run: |
-          UNIT_STATUS="${{ needs.unit.result }}"
+      - name: Download all status footprints for this runtime
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
+        with:
+          pattern: footprint-${{ matrix.python }}-*
+          path: footprint-results/
 
-          if [[ "$UNIT_STATUS" == "success" ]]; then
-            echo "Python ${{ matrix.python }} tests passed."
+      - name: Validate complete batch execution footprint status
+        run: |
+          EXPECTED_BATCHES=$(echo '${{ needs.discover-packages.outputs.batch-indices }}' | jq '. | length')
+          
+          if [ -d "footprint-results" ]; then
+            ACTUAL_BATCHES=$(find footprint-results -type f -name 'status.txt' | wc -l | tr -d ' ')
           else
-            echo "Error: Python ${{ matrix.python }} status is '$UNIT_STATUS'."
+            ACTUAL_BATCHES=0
+          fi
+
+          echo "Validation metrics for Python ${{ matrix.python }}:"
+          echo " -> Expected footprint files: $EXPECTED_BATCHES"
+          echo " -> Downloaded footprint files: $ACTUAL_BATCHES"
+
+          if [ "$ACTUAL_BATCHES" -ne "$EXPECTED_BATCHES" ]; then
+            echo "Error: Footprint count mismatch! Expected $EXPECTED_BATCHES files, but found $ACTUAL_BATCHES."
+            exit 1
+          fi
+
+          FAILED_RUNS=0
+          while read -r STATUS_FILE; do
+            RUN_STATUS=$(cat "$STATUS_FILE" | tr -d '[:space:]')
+            if [[ "$RUN_STATUS" != "success" ]]; then
+              echo "Failure detected in batch profile path: $STATUS_FILE (Status: $RUN_STATUS)"
+              FAILED_RUNS=$((FAILED_RUNS + 1))
+            fi
+          done < <(find footprint-results -type f -name 'status.txt' 2>/dev/null)
+
+          if [ "$FAILED_RUNS" -gt 0 ]; then
+            echo "Error: Validation failed. Found $FAILED_RUNS failing test batches."
             exit 1
           fi
diff --git a/ci/get_batches.py b/ci/get_batches.py
new file mode 100644
index 000000000000..03d7a5ff68f2
--- /dev/null
+++ b/ci/get_batches.py
@@ -0,0 +1,81 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#!/usr/bin/env python3
+import os
+import glob
+import math
+import json
+import sys
+
+BATCH_SIZE = 10
+
+# Large SDKs take significantly longer to run tests. Assigning higher weights 
+# ensures these packages occupy more runner slots and load-balance effectively.
+PACKAGE_WEIGHTS = {
+    "google-cloud-compute": 5,
+    "google-cloud-compute-v1beta": 5,
+    "google-cloud-dialogflow": 5,
+    "google-cloud-dialogflow-cx": 5,
+    "google-cloud-retail": 5,
+}
+
+def get_weighted_package_list():
+    """Audits the packages directory and expands the list based on weights."""
+    raw_paths = sorted(glob.glob("packages/*"))
+    package_paths = [p for p in raw_paths if os.path.isdir(p)]
+    
+    weighted_list = []
+    for path in package_paths:
+        pkg_name = os.path.basename(path)
+        weight = PACKAGE_WEIGHTS.get(pkg_name, 1)
+        
+        for _ in range(weight):
+            weighted_list.append(path)
+            
+    return weighted_list
+
+def get_batch_indices():
+    """Returns a JSON string of the array of batch indices for GitHub Actions matrix."""
+    weighted_list = get_weighted_package_list()
+    total_units = len(weighted_list)
+    num_batches = math.ceil(total_units / BATCH_SIZE)
+
+    if num_batches == 0:
+        num_batches = 1
+
+    return json.dumps(list(range(num_batches)))
+
+def get_batch_slice(batch_index):
+    """Returns a space-separated string of unique package paths for a specific batch index."""
+    weighted_list = get_weighted_package_list()
+    total_units = len(weighted_list)
+    start_idx = batch_index * BATCH_SIZE
+    
+    if start_idx >= total_units:
+        return ""
+        
+    slice_window = weighted_list[start_idx : start_idx + BATCH_SIZE]
+    
+    unique_packages = []
+    for pkg in slice_window:
+        if pkg not in unique_packages:
+            unique_packages.append(pkg)
+            
+    return " ".join(unique_packages)
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1 and sys.argv[1] == "--count":
+        print(get_batch_indices())
+    elif len(sys.argv) > 2 and sys.argv[1] == "--slice":
+        print(get_batch_slice(int(sys.argv[2])))

From 5ef6f918a6f8d19fd9790d9541359e60d668bc10 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Mon, 6 Jul 2026 15:51:15 +0000
Subject: [PATCH 13/17] syntax

---
 .github/workflows/unittest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index de46b0dca33c..4f0ea26fa404 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -15,7 +15,7 @@ permissions:
 # Configurable global environment variables for batching
 env:
   TEST_ALL_PACKAGES: "true" # Set to "false" to only run tests for packages with a git diff
-  ALL_PYTHON: '["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]'
+  ALL_PYTHON: "['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']"
 
 jobs:
   # Dynamic package discovery job to calculate required matrix size automatically

From 7b1b6b73d4144674cb403311b5b1f6f835849121 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Mon, 6 Jul 2026 16:08:04 +0000
Subject: [PATCH 14/17] address review feedback

---
 ci/get_batches.py | 57 +++++++++++++++++++++++------------------------
 1 file changed, 28 insertions(+), 29 deletions(-)

diff --git a/ci/get_batches.py b/ci/get_batches.py
index 03d7a5ff68f2..7f3a8ff465c1 100644
--- a/ci/get_batches.py
+++ b/ci/get_batches.py
@@ -30,49 +30,48 @@
     "google-cloud-retail": 5,
 }
 
-def get_weighted_package_list():
-    """Audits the packages directory and expands the list based on weights."""
+def get_batches():
+    """Distributes packages into batches using a greedy load-balancing algorithm."""
     raw_paths = sorted(glob.glob("packages/*"))
     package_paths = [p for p in raw_paths if os.path.isdir(p)]
     
-    weighted_list = []
+    packages_with_weights = []
+    total_weight = 0
     for path in package_paths:
         pkg_name = os.path.basename(path)
         weight = PACKAGE_WEIGHTS.get(pkg_name, 1)
+        packages_with_weights.append((path, weight))
+        total_weight += weight
         
-        for _ in range(weight):
-            weighted_list.append(path)
-            
-    return weighted_list
-
-def get_batch_indices():
-    """Returns a JSON string of the array of batch indices for GitHub Actions matrix."""
-    weighted_list = get_weighted_package_list()
-    total_units = len(weighted_list)
-    num_batches = math.ceil(total_units / BATCH_SIZE)
-
+    num_batches = math.ceil(total_weight / BATCH_SIZE)
     if num_batches == 0:
         num_batches = 1
+        
+    # Sort packages by weight descending (Longest Processing Time first)
+    packages_with_weights.sort(key=lambda x: x[1], reverse=True)
+    
+    batches = [[] for _ in range(num_batches)]
+    batch_weights = [0] * num_batches
+    
+    for path, weight in packages_with_weights:
+        # Greedily assign the package to the currently least-loaded batch
+        min_idx = batch_weights.index(min(batch_weights))
+        batches[min_idx].append(path)
+        batch_weights[min_idx] += weight
+        
+    return batches
 
-    return json.dumps(list(range(num_batches)))
+def get_batch_indices():
+    """Returns a JSON string of the array of batch indices for GitHub Actions matrix."""
+    batches = get_batches()
+    return json.dumps(list(range(len(batches))))
 
 def get_batch_slice(batch_index):
     """Returns a space-separated string of unique package paths for a specific batch index."""
-    weighted_list = get_weighted_package_list()
-    total_units = len(weighted_list)
-    start_idx = batch_index * BATCH_SIZE
-    
-    if start_idx >= total_units:
+    batches = get_batches()
+    if batch_index < 0 or batch_index >= len(batches):
         return ""
-        
-    slice_window = weighted_list[start_idx : start_idx + BATCH_SIZE]
-    
-    unique_packages = []
-    for pkg in slice_window:
-        if pkg not in unique_packages:
-            unique_packages.append(pkg)
-            
-    return " ".join(unique_packages)
+    return " ".join(batches[batch_index])
 
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == "--count":

From 1aab0d582147a3959df70e14f2399b13f74eebbf Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Mon, 6 Jul 2026 12:09:00 -0400
Subject: [PATCH 15/17] Update ci/run_conditional_tests.sh

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
---
 ci/run_conditional_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 8d7d5d5c3642..36aadf7b1a93 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -92,7 +92,7 @@ fi
 
 RETVAL=0
 
-for subdir in ${subdirs[@]}; do
+for subdir in "${subdirs[@]}"; do
     if [ ! -d "${subdir}" ]; then
         echo "Error: Directory '${subdir}' does not exist." >&2
         exit 1

From 4d5e55decb48cd0468c18b782d74d03cc4ca1a97 Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Mon, 6 Jul 2026 16:14:46 +0000
Subject: [PATCH 16/17] remove weighting

---
 ci/get_batches.py | 59 ++++++++++++++++++++++++-----------------------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/ci/get_batches.py b/ci/get_batches.py
index 7f3a8ff465c1..34f256020fd9 100644
--- a/ci/get_batches.py
+++ b/ci/get_batches.py
@@ -20,45 +20,46 @@
 
 BATCH_SIZE = 10
 
-# Large SDKs take significantly longer to run tests. Assigning higher weights 
-# ensures these packages occupy more runner slots and load-balance effectively.
-PACKAGE_WEIGHTS = {
-    "google-cloud-compute": 5,
-    "google-cloud-compute-v1beta": 5,
-    "google-cloud-dialogflow": 5,
-    "google-cloud-dialogflow-cx": 5,
-    "google-cloud-retail": 5,
-}
+# Packages that take significantly longer to run tests.
+# These will always be assigned to their own dedicated runner.
+ISOLATED_PACKAGES = [
+    "google-cloud-compute",
+    "google-cloud-compute-v1beta",
+    "google-cloud-dialogflow",
+    "google-cloud-dialogflow-cx",
+    "google-cloud-retail",
+]
 
 def get_batches():
-    """Distributes packages into batches using a greedy load-balancing algorithm."""
+    """Splits packages into dedicated isolated batches and evenly chunked standard batches."""
     raw_paths = sorted(glob.glob("packages/*"))
     package_paths = [p for p in raw_paths if os.path.isdir(p)]
     
-    packages_with_weights = []
-    total_weight = 0
+    batches = []
+    standard_packages = []
+    
     for path in package_paths:
         pkg_name = os.path.basename(path)
-        weight = PACKAGE_WEIGHTS.get(pkg_name, 1)
-        packages_with_weights.append((path, weight))
-        total_weight += weight
-        
-    num_batches = math.ceil(total_weight / BATCH_SIZE)
-    if num_batches == 0:
-        num_batches = 1
         
-    # Sort packages by weight descending (Longest Processing Time first)
-    packages_with_weights.sort(key=lambda x: x[1], reverse=True)
-    
-    batches = [[] for _ in range(num_batches)]
-    batch_weights = [0] * num_batches
+        if pkg_name in ISOLATED_PACKAGES:
+            # Put heavy packages into their own standalone batches immediately
+            batches.append([path])
+        else:
+            standard_packages.append(path)
+            
+    # Chunk the remaining standard packages by BATCH_SIZE
+    num_standard_packages = len(standard_packages)
+    num_standard_batches = math.ceil(num_standard_packages / BATCH_SIZE)
     
-    for path, weight in packages_with_weights:
-        # Greedily assign the package to the currently least-loaded batch
-        min_idx = batch_weights.index(min(batch_weights))
-        batches[min_idx].append(path)
-        batch_weights[min_idx] += weight
+    if num_standard_batches == 0 and not batches:
+        num_standard_batches = 1
         
+    for i in range(num_standard_batches):
+        start_idx = i * BATCH_SIZE
+        chunk = standard_packages[start_idx : start_idx + BATCH_SIZE]
+        if chunk:
+            batches.append(chunk)
+            
     return batches
 
 def get_batch_indices():

From 4e39eebc21b14f997c9eca4a3bfe6fb00a5fbb0a Mon Sep 17 00:00:00 2001
From: Anthonios Partheniou <partheniou@google.com>
Date: Mon, 6 Jul 2026 16:59:34 +0000
Subject: [PATCH 17/17] set TEST_ALL_PACKAGES to false

---
 .github/workflows/unittest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 4f0ea26fa404..62bedbba0d07 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -14,7 +14,7 @@ permissions:
 
 # Configurable global environment variables for batching
 env:
-  TEST_ALL_PACKAGES: "true" # Set to "false" to only run tests for packages with a git diff
+  TEST_ALL_PACKAGES: "false" # Set to "false" to only run tests for packages with a git diff
   ALL_PYTHON: "['3.9', '3.10', '3.11', '3.12', '3.13', '3.14']"
 
 jobs: