ci(timing): run the unit legs at -n logical and read latest's JUnit timings (#2571)

.github/workflows/unit-tests.yml

@@ -101,8 +101,9 @@ jobs:
       # legs run the same `ALL` tier as every CPython leg but `latest`.
       #
       # `include` marks one leg `coverage: true`, `latest` (#1661), so coverage
-      # adds no leg to the queue. Uncovered, 3.15's tests step took a median
-      # 8.1 minutes and coverage cost about 1.8x, which fits the job's 45.
+      # adds no leg to the queue. That leg runs slower than the rest: it runs the
+      # `LATEST` tier as well as `ALL`, under coverage. util/ci_timing.py's W3
+      # tracks the gap.
       #
       # #2243, the owner's option B: a test-only pull request runs the
       # `latest` leg alone, the coverage leg, which runs the whole suite with
@@ -166,7 +167,7 @@ jobs:
         if: ${{ matrix.coverage }}
         run: python -m pip install 'coverage>=7.10'
 
-      # Nothing else in the log says what `-n auto` resolved to on this runner.
+      # Nothing else in the log says what `-n logical` resolved to on this runner.
       - name: Report available parallelism
         id: parallelism
         if: ${{ steps.install.outcome == 'success' }}
@@ -183,6 +184,9 @@ jobs:
         # cannot silently change how this suite is distributed. The durations
         # table and JUnit file are #1538's per-test timings.
         #
+        # #2571: `-n logical`, as with psutil (the `test` extra's, CPython only)
+        # `auto` counts physical cores, half the runner's CPUs; on PyPy they agree.
+        #
         # #1958, the test tiers: tests/_tiers.py's module_tier() puts every test
         # module in `ALL`, a curated list each stating `__tier__ = 'ALL'` (#1963),
         # or else in `LATEST`, and `LATEST` runs on `latest`
@@ -215,7 +219,7 @@ jobs:
             echo "::${ANNOTATION_LEVEL} title=Could not compute the LATEST tier::tests._tiers.latest_tier_paths() failed -- see the traceback above."
             exit 1
           fi
-          set -- -q -n auto --dist load \
+          set -- -q -n logical --dist load \
             --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \
             --ignore=tests/integration \
             --ignore-glob='*_runtime.py' \

docs/source/contributing/conventions/process.rst

@@ -428,8 +428,8 @@ thresholds are on :doc:`../workflows`.
    job's shape or selection -- a leg added, split or moved -- dispatch the workflow, so
    the next week compares like with like.
 *  **Monthly, refresh the measured figures** quoted in :file:`unit-tests.yml`'s
-   comments that the report shows to be stale -- the coverage leg's headroom, the
-   ordering cells' timings and their step cap's rationale.
+   comments that the report shows to be stale -- the ordering cells' timings and
+   their step cap's rationale.
 
 
 The PEP Discussion, Kept Current

docs/source/contributing/conventions/test-coverage.rst

@@ -123,10 +123,11 @@ Running the Tests
 
 *  **One module per process, never the whole suite.** Check a change by running the
    modules it touches, each as its own ``python -m pytest <module> -q``, and leave the
-   whole tier to CI, which spreads it over workers with ``-n auto --dist load``. Every
-   purge-then-import leaves the previous generation of :mod:`pcapkit` behind as
-   garbage, and the whole suite in one plain-unittest process runs out of memory --
-   :file:`util/run_unittest_leg.py` records the measurement.
+   whole tier to CI, whose ``test`` job spreads the unit tier over workers with
+   ``-n logical --dist load``. Every purge-then-import leaves the previous generation
+   of :mod:`pcapkit` behind as garbage, and the whole suite in one plain-unittest
+   process runs out of memory -- :file:`util/run_unittest_leg.py` records the
+   measurement.
 *  **Read the summary line, not the exit code.** pytest's last line
    (``N passed, M failed``) and unittest's ``OK`` or ``FAILED (...)`` are the result;
    an exit code read through a pipe or a wrapper is the pipe's or the wrapper's.

tests/project/test_ci_timing.py

@@ -1178,6 +1178,15 @@ def test_the_legs_the_review_reads_exist(self) -> 'None':
         coverage = [entry['python-version'] for entry in found if entry.get('coverage')]
         self.assertEqual([f'Python {version}' for version in coverage], [ci_timing.COVERAGE_JOB])
 
+    def test_the_review_reads_the_coverage_legs_timings(self) -> 'None':
+        """#2571: the coverage leg, ``latest``, alone runs the ``LATEST`` tier, so its file is read too."""
+        coverage = [entry['python-version'] for entry in matrix_legs(matrix('unit-tests.yml', 'test'))
+                    if entry.get('coverage')]
+        self.assertEqual(len(coverage), 1)
+        self.assertIn(coverage[0], ci_timing.JUNIT_LEGS)
+        # Each leg once, or its artifact is read and counted twice.
+        self.assertEqual(len(set(ci_timing.JUNIT_LEGS)), len(ci_timing.JUNIT_LEGS))
+
     def test_the_reusable_workflow_asks_for_no_write_permission(self) -> 'None':
         """Its callers grant ``contents: read``, so a write scope would break them."""
         code = '\n'.join(line.split('#', 1)[0] for line in self.text.splitlines())

tests/project/test_sweeps_on_latest_only_1732.py

@@ -293,6 +293,19 @@ def test_every_other_leg_runs_the_rest(self) -> None:
                 self.assertEqual(argv[:2], ['-m', 'pytest'])
                 self.assertEqual(pytest_args(argv), want)
 
+    def test_every_test_leg_runs_a_worker_per_logical_cpu(self) -> None:
+        """#2571: ``-n logical``, as ``auto`` counts physical cores wherever psutil is installed.
+
+        PyPy's legs share the flag, and with no psutil there resolve it as ``auto`` would.
+
+        """
+        for version, (leg, argv) in self.runs.items():
+            if is_experimental(leg):
+                continue
+            with self.subTest(python=version):
+                args = pytest_args(argv)
+                self.assertEqual(args[args.index('-n') + 1], 'logical')
+
     def test_a_step_that_ignores_the_pins_alone_off_latest_is_caught(self) -> None:
         """The check above fails when a leg off ``latest`` leaves out the pins rather than the tier."""
         script = step(JOB)['run']

tests/project/test_windows_test_leg_2060.py

@@ -15,9 +15,9 @@
   an error.
 * :class:`TestItRunsOnWindows`: every ``run:`` is Git Bash's, none calls a
   Linux-only tool, and the checkout keeps LF line endings.
-* :class:`TestTheTestsStep`: run as bash runs it, the step hands pytest ``test``'s
-  flags and ignores the ``LATEST`` tier, and a tier it cannot list fails the step
-  with a warning.
+* :class:`TestTheTestsStep`: run as bash runs it, the step hands pytest all but
+  ``-n`` of ``test``'s flags and ignores the ``LATEST`` tier, and a tier it cannot
+  list fails the step with a warning.
 * :class:`TestItHoldsLatestBackNowhere`: the weekly tier job does not wait for a Windows
   build before it moves ``latest``.
 * :class:`TestTheReporters`: CI Reports and the CI Timing review read the job as a
@@ -287,14 +287,15 @@ def test_it_runs_the_all_tier_with_tests_flags(self) -> None:
                                 *(f'--ignore={path}' for path in latest)])
 
     def test_its_flags_are_tests(self) -> None:
-        """The flags as unit-tests.yml ``test``'s step writes them."""
-        flags = re.compile(r"set -- -q -n auto --dist load .*?--ignore-glob='\*_regression\.py'", re.S)
+        """``test``'s flags, but for ``-n``, which only ``test``'s step sets to ``logical``."""
+        flags = re.compile(r"set -- -q -n (\w+) (--dist load .*?--ignore-glob='\*_regression\.py')", re.S)
         own = flags.search(step('Run unit tests')['run'])
         tests = [entry for entry in _workflows.steps('unit-tests.yml', 'test') if entry.get('name') == 'Run unit tests']
         theirs = flags.search(tests[0]['run'])
         self.assertIsNotNone(own)
         self.assertIsNotNone(theirs)
-        self.assertEqual(own.group(0).split(), theirs.group(0).split())  # type: ignore[union-attr]
+        self.assertEqual((own.group(1), theirs.group(1)), ('auto', 'logical'))  # type: ignore[union-attr]
+        self.assertEqual(own.group(2).split(), theirs.group(2).split())  # type: ignore[union-attr]
 
     def test_a_tier_it_cannot_list_fails_the_step_with_a_warning(self) -> None:
         result, argv, _ = self.run_step(FAIL_C='1')

util/ci_timing.py

@@ -124,26 +124,27 @@
 TIERS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), '.github', 'python-versions.toml')
 
 
-def _coverage_legs() -> 'tuple[str, tuple[str, ...]]':
-    """The ``latest`` leg's job name, and the other supported legs', from :data:`TIERS`."""
+def _coverage_legs() -> 'tuple[str, str, tuple[str, ...]]':
+    """``latest``, its leg's job name, and the other supported legs', from :data:`TIERS`."""
     with open(TIERS, 'rb') as file:
         data = tomllib.load(file)
     paired = tuple(f'Python {version}' for version, entry in data['versions']['cpython'].items()
                    if entry['tier'] == 'supported' and version != data['latest'])
-    return f"Python {data['latest']}", paired
+    return data['latest'], f"Python {data['latest']}", paired
 
 
-#: The leg running under coverage, ``latest``, and the legs its tests step is paired
-#: against: the rest of the supported tier (#1633), not the KTLO legs, which are
-#: named ``Python 3.x (KTLO)`` and may stop early without failing.
-COVERAGE_JOB, PAIRED_JOBS = _coverage_legs()
+#: ``latest``, the job of its leg, which runs under coverage, and the legs that job's
+#: tests step is paired against: the rest of the supported tier (#1633), not the KTLO
+#: legs, which are named ``Python 3.x (KTLO)`` and may stop early without failing.
+LATEST_VERSION, COVERAGE_JOB, PAIRED_JOBS = _coverage_legs()
 TESTS_STEP = 'Run unit tests'
 #: The leg ms/test is measured on: a supported version not under coverage, kept on
 #: 3.13 -- the newest such leg when W4's baseline was measured -- so that W4 compares
 #: one interpreter with itself week over week.
 MS_PER_TEST_LEG = '3.13'
-#: The JUnit artifacts read, by leg, and from how many push runs.
-JUNIT_LEGS = ('3.13', '3.14')
+#: The JUnit artifacts read, by leg, and from how many push runs. ``latest``'s holds
+#: the ``LATEST`` tier, which no other leg runs (#2571).
+JUNIT_LEGS = tuple(dict.fromkeys(('3.13', '3.14', LATEST_VERSION)))
 JUNIT_ARTIFACT = 'junit-unit-{leg}'
 JUNIT_RUNS = 5