ci: run the corekit plain-unittest ordering leg (#2115)

.github/workflows/unit-tests.yml

@@ -1125,17 +1125,18 @@ jobs:
   # by a root-level module survives. The modules directly in tests/protocols
   # stay in the rest cell, as moving them would split more pairs than it joins.
   #
-  # Deliberately absent from ``leg``:
+  # Every tests/ subdirectory is a leg except those below, and
+  # tests/project/test_unittest_ordering_shards.py fails on a new one left out:
   #
-  #   * tests/corekit -- plain unittest fails its identity subtests, as
-  #     tests/_support.py's purge_modules() docstring documents; pytest is the
-  #     supported runner for exactly that reason, so the leg would be
-  #     permanently red for an accepted characteristic.
   #   * tests/integration -- needs generated captures, and `integration` runs
   #     it under pytest. tests._tiers.is_unit_tier() rejects every one of its
   #     files, so the leg would run the root-level modules alone -- green,
   #     while covering none of it.
   #
+  # tests/corekit was once left out as permanently red; #2115 found its
+  # failures were test modules mixing two pcapkit imports, fixed them, and
+  # added the leg.
+  #
   # The `project` cell keeps the project pins (#1954): whether a module passes
   # in one interpreter, in file order, with no xdist, is a question about the
   # module, so a pin can fail it as readily as any other.
@@ -1162,6 +1163,7 @@ jobs:
         leg:
           - cli
           - const
+          - corekit
           - dumpkit
           - foundation
           - interface

docs/source/contributing/conventions/test-coverage.rst

@@ -150,9 +150,11 @@ Running the Tests
       python util/run_unittest_leg.py protocols/internet
       python util/run_unittest_leg.py protocols --exclude protocols/internet
 
-   The leg list lives in the workflow, not the script; :file:`tests/corekit/` and
-   :file:`tests/integration/` are deliberately not legs, for the reasons recorded
-   there.
+   The leg list lives in the workflow, not the script. Every :file:`tests/`
+   subdirectory is a leg except :file:`tests/integration/`, which holds no unit-tier
+   module and runs under pytest in the ``integration`` job;
+   :file:`tests/project/test_unittest_ordering_shards.py` fails on a subdirectory
+   left out (:issue:`2115`).
 
 Adding a Protocol, Option or Dumper
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

tests/project/test_unittest_ordering_shards.py

@@ -13,7 +13,10 @@
   ``protocols`` leg, so a new ``tests/protocols/<dir>/`` is either inside a
   shard or fails here;
 * every other matrix cell is a bare direct child of ``tests/`` with no
-  ``--exclude``, i.e. runs exactly what it ran before the split.
+  ``--exclude``, i.e. runs exactly what it ran before the split;
+* every ``tests/`` subdirectory holding a test module is a leg, except the
+  :data:`NOT_LEGS` names, each left out for the reason the workflow comment
+  gives (#2115: ``corekit`` was missing, and nothing said so).
 
 The workflow is read with a small hand-rolled scan rather than :mod:`yaml`,
 which comes only with the ``test`` extra of :file:`pyproject.toml`; the same split
@@ -36,6 +39,19 @@
 
 import run_unittest_leg as leg  # noqa: E402  pylint: disable=wrong-import-position
 
+#: ``tests/`` subdirectories deliberately not in the matrix, each for the reason
+#: the ``unittest-ordering`` comment in the workflow records (#2115).
+NOT_LEGS = {
+    # fixture-dependent tier only, run under pytest by ``integration``
+    'integration',
+}
+
+
+def _test_directories() -> 'set[str]':
+    """Every direct child of ``tests/`` that holds a ``test_*.py`` module."""
+    return {path.name for path in leg.TESTS_ROOT.iterdir()
+            if path.is_dir() and next(path.rglob('test_*.py'), None) is not None}
+
 
 def _matrix() -> 'list[tuple[str, list[str]]]':
     """``(leg, exclude)`` for every cell of the ``unittest-ordering`` matrix."""
@@ -77,13 +93,31 @@ def test_every_other_cell_is_a_plain_direct_child(self):
         others = [(name, exclude) for name, exclude in _matrix()
                   if name != 'protocols' and not name.startswith('protocols/')]
         self.assertEqual(sorted(name for name, _ in others),
-                         sorted(['cli', 'const', 'dumpkit', 'foundation', 'interface',
-                                 'project', 'toolkit', 'utilities', 'vendor']))
+                         sorted(_test_directories() - NOT_LEGS - {'protocols'}))
         for name, exclude in others:
             self.assertEqual(exclude, [], name)
             self.assertEqual(leg.leg_modules(name),
                              leg.leg_modules(name, ()), name)
 
+    def test_every_tests_subdirectory_is_a_leg(self):
+        """#2115: a ``tests/`` subdirectory left out of the matrix is never run
+        under plain unittest, and the job stays green while covering less."""
+        covered = {name.split('/', 1)[0] for name, _ in _matrix()}
+        self.assertEqual(sorted(_test_directories() - NOT_LEGS - covered), [],
+                         'a tests/ subdirectory is missing from unittest-ordering')
+
+    def test_every_exception_is_real_and_still_left_out(self):
+        covered = {name.split('/', 1)[0] for name, _ in _matrix()}
+        for name in sorted(NOT_LEGS):
+            with self.subTest(name=name):
+                self.assertIn(name, _test_directories(), 'stale NOT_LEGS entry')
+                self.assertNotIn(name, covered, 'a leg now, so drop it from NOT_LEGS')
+
+    def test_integration_has_no_unit_tier_module(self):
+        """The reason ``integration`` is not a leg: the leg would run only the
+        root-level modules."""
+        self.assertEqual(leg.leg_modules('integration'), ())
+
     def test_no_shard_is_empty(self):
         for name, exclude in _matrix():
             self.assertTrue(leg.leg_modules(name, exclude), name)