ci(unit-tests): assign every test module a tier and run each tier where it belongs (#1958)

.github/workflows/experimental.yml

@@ -203,8 +203,8 @@ jobs:
   # The tests' 120: GraalPy 25.4's tests step reached 26% of the unit tier's
   # tests at its old 25-minute cap (job 115915976024; setup-python took 0.7
   # minutes, the install 3.3). Those tests hold 20-26% of the tier's time, and
-  # the tier less the round-trip sweeps, which this job never runs (see `Run
-  # unit tests`), holds 3.5-4.5 times as much, by per-module times from a local
+  # the tier less the round-trip sweeps, which this job did not run then (see
+  # `Run unit tests`), held 3.5-4.5 times as much, by per-module times from a local
   # 3.14 run and the 3.14, 3.15 and 3.14t legs of run 38581834697: 88-113
   # minutes on GraalPy (estimate). 3.14t, which shares the caps, took 5.8
   # minutes over the whole unit tier in that run.
@@ -242,19 +242,20 @@ jobs:
         id: checkout
         continue-on-error: true
 
-      # #1954: the project pins run once per push, in unit-tests.yml's
-      # `project-tests`, so these legs leave them out, as `test`'s do. The
-      # runner's own `python3` asks tests/_tiers.py, which needs only the
-      # standard library, before setup-python puts the leg's interpreter on
-      # PATH, so GraalPy does not parse the tree. Where it fails, the tests
-      # run the pins too, and `Experimental summary` names this step.
-      - name: List the project pins
-        id: pins
+      # #1958: these legs run the `ALL` test tier alone, as `test`'s legs but
+      # `latest` do, so they leave out every `LATEST` module -- the project
+      # pins (#1954) and the round-trip sweeps (#1732) among them. The runner's
+      # own `python3` asks tests/_tiers.py, which needs only the standard
+      # library, before setup-python puts the leg's interpreter on PATH, so
+      # GraalPy does not parse the tree. Where it fails, the tests run the
+      # `LATEST` tier too, and `Experimental summary` names this step.
+      - name: List the LATEST tier
+        id: tier
         if: ${{ steps.checkout.outcome == 'success' }}
         continue-on-error: true
         run: |
-          pins=$(python3 -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))")
-          echo "paths=$pins" >> "$GITHUB_OUTPUT"
+          latest=$(python3 -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))")
+          echo "paths=$latest" >> "$GITHUB_OUTPUT"
 
       - uses: actions/setup-python@v7
         id: python
@@ -380,7 +381,7 @@ jobs:
 
       # unit-tests.yml `test`'s selection on every leg but `latest`: no
       # experimental version is `latest`, which util/python_versions.py holds to
-      # a supported CPython, so the round-trip sweeps (#1732) are always left out
+      # a supported CPython, so the `LATEST` test tier (#1958) is always left out
       # here. #1740: `tail -F` copies the resource monitor's samples into this
       # step's log as they are taken, and stops with the step.
       #
@@ -461,7 +462,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           WATCHER: ${{ steps.watcher.outcome }}
           WATCHER_PYTHON: ${{ steps.watcher-python.outputs.python-path }}
-          PINS: ${{ steps.pins.outputs.paths }}
+          LATEST_TIER: ${{ steps.tier.outputs.paths }}
         run: |
           tail -n +1 -F "$RUNNER_TEMP/resource-monitor.log" 2>/dev/null &
           monitor=$!
@@ -471,11 +472,10 @@ jobs:
             --ignore=tests/integration \
             --ignore-glob='*_runtime.py' \
             --ignore-glob='*_regression.py'
-          # #1954: the project pins, from `List the project pins`.
-          for pin in $PINS; do
-            set -- "$@" --ignore="$pin"
+          # #1958: the `LATEST` tier, from `List the LATEST tier`.
+          for module in $LATEST_TIER; do
+            set -- "$@" --ignore="$module"
           done
-          set -- "$@" --ignore-glob='*roundtrip*'
           set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m python -m pytest "$@"
           scoped() {
             unit=$1

.github/workflows/unit-tests.yml

@@ -214,40 +214,44 @@ jobs:
         # distributed. The durations table and JUnit file are #1538's per-test
         # timings, uploaded by the last step of this job.
         #
-        # #1732, the owner's ruling: the round-trip sweeps -- every module whose
-        # path holds `roundtrip`, tests/_tiers.py's ROUNDTRIP_SWEEP_GLOB -- run
-        # on `latest` alone, which is also the coverage leg, so coverage keeps
-        # its selection. Every other leg ignores them, as do experimental.yml's.
-        # pytest matches the glob, made absolute, against each path it
-        # collects, with `*` spanning `/` (_pytest/main.py:437-461, pytest
-        # 9.1.1). The modules that import the sweeps' harnesses for their
-        # packet builders still import them.
+        # #1958, the test tiers: tests/_tiers.py's module_tier() puts every test
+        # module in `ALL` or `LATEST`, by the `__tier__` constant a module states
+        # or by the rule where it states none, and `LATEST` runs on `latest`
+        # alone, which is also the coverage leg, so coverage keeps its
+        # selection. Every other leg -- `test-ktlo`'s too, as it runs these
+        # steps -- passes each module latest_tier_paths() prints as an
+        # `--ignore`, as experimental.yml's legs do. That takes in the
+        # round-trip sweeps (#1732), each marked `LATEST`; the hand-picked
+        # round trips are `ALL`.
         #
         # #1954, the owner's option 3: the project pins -- the tests/project
         # modules whose run does not vary by interpreter, tests/_tiers.py's
-        # project_pin_paths() -- run once per push, in `project-tests`, so every
-        # leg passes each as an `--ignore`, as experimental.yml's do. Asked of
-        # tests/_tiers.py here rather than listed, as `integration` asks for its
-        # selection. The pins import no pcapkit, so coverage loses nothing.
+        # project_pin_paths(), each of them `LATEST` -- run once per push, in
+        # `project-tests`, so `latest` passes each as an `--ignore` too. The pins
+        # import no pcapkit, so coverage loses nothing. Both lists are asked of
+        # tests/_tiers.py at run time rather than listed, as `integration` asks
+        # for its selection.
         env:
           COVERAGE: ${{ matrix.coverage }}
-          SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }}
+          LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }}
         run: |
-          if ! pins=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then
-            echo "::${ANNOTATION_LEVEL} title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above."
+          if [ "$LATEST" = "true" ]; then
+            if ! skipped=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then
+              echo "::${ANNOTATION_LEVEL} title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above."
+              exit 1
+            fi
+          elif ! skipped=$(python -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))"); then
+            echo "::${ANNOTATION_LEVEL} title=Could not compute the LATEST tier::tests._tiers.latest_tier_paths() failed -- see the traceback above."
             exit 1
           fi
           set -- -q -n auto --dist load \
             --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \
             --ignore=tests/integration \
             --ignore-glob='*_runtime.py' \
             --ignore-glob='*_regression.py'
-          for pin in $pins; do
-            set -- "$@" --ignore="$pin"
+          for module in $skipped; do
+            set -- "$@" --ignore="$module"
           done
-          if [ "$SWEEPS" != "true" ]; then
-            set -- "$@" --ignore-glob='*roundtrip*'
-          fi
           if [ "$COVERAGE" = "true" ]; then
             # Logs every measured process and its parent, for the check below.
             COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="$PWD/coverage-processes.log" \
@@ -579,24 +583,30 @@ jobs:
       # say so. `--dist load` is pinned explicitly; see the `test` job above
       # for why.
       #
-      # #1732, the owner's ruling: the round-trip sweeps here, the modules
-      # tests/_tiers.py's ROUNDTRIP_RUNTIME_GLOB names, run on `latest` alone, as
-      # `test`'s do. Every other leg adds the glob as an `--ignore-glob`, which
-      # covers a module a directory argument reaches, and drops each path or
-      # node ID the glob matches from the selection: pytest never applies an
-      # ignore to a path it was handed (`Dir.collect` asks `isinitpath` first,
-      # _pytest/main.py:564-584, pytest 9.1.1).
+      # #1958, the test tiers: the fixture tier's `LATEST` modules, the sweeps
+      # over every capture among them, run on `latest` alone, as `test`'s do.
+      # Every other leg asks for the `ALL` tier's part of the selection,
+      # FIXTURE_ALL_TIER_SELECTOR, which drops each `LATEST` path and node ID
+      # and replaces a directory holding a `LATEST` module by its `ALL` ones,
+      # since pytest applies no ignore to a path it was handed (`Dir.collect`
+      # asks `isinitpath` first, _pytest/main.py:564-584, pytest 9.1.1).
       - name: Run full test suite
         id: suite
         if: ${{ steps.install.outcome == 'success' }}
         continue-on-error: ${{ matrix.ktlo == true }}
         shell: bash
         env:
-          SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }}
+          LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }}
         run: |
-          if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then
+          cannot_compute() {
             echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly."
             exit 1
+          }
+          if [ "$LATEST" = "true" ]; then
+            selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))") || cannot_compute
+          else
+            selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths('ALL')))") || cannot_compute
+            echo "Not latest, so the ALL tier alone (#1958)."
           fi
           if [ -z "$selection" ]; then
             echo "::${ANNOTATION_LEVEL} title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments."
@@ -605,17 +615,6 @@ jobs:
           echo "Fixture-dependent selection: $selection"
           # shellcheck disable=SC2086
           set -- $selection
-          if [ "$SWEEPS" != "true" ]; then
-            kept=()
-            for arg in "$@"; do
-              case "${arg%%::*}" in
-                *roundtrip*_runtime.py) ;;
-                *) kept+=("$arg") ;;
-              esac
-            done
-            set -- "${kept[@]}" --ignore-glob='*roundtrip*_runtime.py'
-            echo "Not latest, so without the round-trip sweeps (#1732): $*"
-          fi
           python -m pytest -q -n auto --dist load \
             --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" "$@"
 
@@ -1351,13 +1350,16 @@ jobs:
 
       # See the `integration` job above for why this is asked of
       # tests/_tiers.py directly rather than spelled out as literal flags.
+      # #1958: neither leg here is `latest`, so each runs the `ALL` tier's part
+      # of the selection, as `integration`'s other legs do; the `LATEST`
+      # sweeps over every capture run in `integration` on `latest`.
       - name: Run full test suite
         id: suite
         if: ${{ steps.install.outcome == 'success' }}
         continue-on-error: ${{ matrix.ktlo == true }}
         shell: bash
         run: |
-          if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then
+          if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths('ALL')))"); then
             echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly."
             exit 1
           fi

docs/source/contributing/releasing.rst

@@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s
 (``:494-514``) asserts the two agree, and ``create-release.yml``'s
 ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with
 ``gate-only: true``, which runs the **full** suite rather than the tiered
-subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1840-1910``).
+subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1842-1912``).
 That gate runs whenever a release will -- a moved ``__version__`` is exactly
 what makes ``version_check``'s evidence read "not yet published" -- and every
 publishing job requires it to have succeeded (``:621,808,974,1210``), so a
@@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit.
 This is the **only** edit a person makes to get a release started -- the tag,
 the Release, and every upload are the workflow's job from here. Other
 checks run unconditionally on the same path, though none is a step in *this*
-process: the ``changelog`` job (``unit-tests.yml:1644-1670``) fails outright if
+process: the ``changelog`` job (``unit-tests.yml:1646-1672``) fails outright if
 ``CHANGELOG.md`` has drifted from its source entry under
 :file:`docs/source/changelog/`, and the release-body step warns, without
 failing, if that entry's heading still reads "unreleased"