diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d1bdbe71ce..98c8092e53 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1125,17 +1125,18 @@ jobs: # by a root-level module survives. The modules directly in tests/protocols # stay in the rest cell, as moving them would split more pairs than it joins. # - # Deliberately absent from ``leg``: + # Every tests/ subdirectory is a leg except those below, and + # tests/project/test_unittest_ordering_shards.py fails on a new one left out: # - # * tests/corekit -- plain unittest fails its identity subtests, as - # tests/_support.py's purge_modules() docstring documents; pytest is the - # supported runner for exactly that reason, so the leg would be - # permanently red for an accepted characteristic. # * tests/integration -- needs generated captures, and `integration` runs # it under pytest. tests._tiers.is_unit_tier() rejects every one of its # files, so the leg would run the root-level modules alone -- green, # while covering none of it. # + # tests/corekit was once left out as permanently red; #2115 found its + # failures were test modules mixing two pcapkit imports, fixed them, and + # added the leg. + # # The `project` cell keeps the project pins (#1954): whether a module passes # in one interpreter, in file order, with no xdist, is a question about the # module, so a pin can fail it as readily as any other. @@ -1162,6 +1163,7 @@ jobs: leg: - cli - const + - corekit - dumpkit - foundation - interface diff --git a/docs/source/contributing/conventions/test-coverage.rst b/docs/source/contributing/conventions/test-coverage.rst index f4904c1e83..fd4ebce718 100644 --- a/docs/source/contributing/conventions/test-coverage.rst +++ b/docs/source/contributing/conventions/test-coverage.rst @@ -150,9 +150,11 @@ Running the Tests python util/run_unittest_leg.py protocols/internet python util/run_unittest_leg.py protocols --exclude protocols/internet - The leg list lives in the workflow, not the script; :file:`tests/corekit/` and - :file:`tests/integration/` are deliberately not legs, for the reasons recorded - there. + The leg list lives in the workflow, not the script. Every :file:`tests/` + subdirectory is a leg except :file:`tests/integration/`, which holds no unit-tier + module and runs under pytest in the ``integration`` job; + :file:`tests/project/test_unittest_ordering_shards.py` fails on a subdirectory + left out (:issue:`2115`). Adding a Protocol, Option or Dumper ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tests/project/test_unittest_ordering_shards.py b/tests/project/test_unittest_ordering_shards.py index 83aa3b505d..7b9d3820b4 100644 --- a/tests/project/test_unittest_ordering_shards.py +++ b/tests/project/test_unittest_ordering_shards.py @@ -13,7 +13,10 @@ ``protocols`` leg, so a new ``tests/protocols//`` is either inside a shard or fails here; * every other matrix cell is a bare direct child of ``tests/`` with no - ``--exclude``, i.e. runs exactly what it ran before the split. + ``--exclude``, i.e. runs exactly what it ran before the split; +* every ``tests/`` subdirectory holding a test module is a leg, except the + :data:`NOT_LEGS` names, each left out for the reason the workflow comment + gives (#2115: ``corekit`` was missing, and nothing said so). The workflow is read with a small hand-rolled scan rather than :mod:`yaml`, which comes only with the ``test`` extra of :file:`pyproject.toml`; the same split @@ -36,6 +39,19 @@ import run_unittest_leg as leg # noqa: E402 pylint: disable=wrong-import-position +#: ``tests/`` subdirectories deliberately not in the matrix, each for the reason +#: the ``unittest-ordering`` comment in the workflow records (#2115). +NOT_LEGS = { + # fixture-dependent tier only, run under pytest by ``integration`` + 'integration', +} + + +def _test_directories() -> 'set[str]': + """Every direct child of ``tests/`` that holds a ``test_*.py`` module.""" + return {path.name for path in leg.TESTS_ROOT.iterdir() + if path.is_dir() and next(path.rglob('test_*.py'), None) is not None} + def _matrix() -> 'list[tuple[str, list[str]]]': """``(leg, exclude)`` for every cell of the ``unittest-ordering`` matrix.""" @@ -77,13 +93,31 @@ def test_every_other_cell_is_a_plain_direct_child(self): others = [(name, exclude) for name, exclude in _matrix() if name != 'protocols' and not name.startswith('protocols/')] self.assertEqual(sorted(name for name, _ in others), - sorted(['cli', 'const', 'dumpkit', 'foundation', 'interface', - 'project', 'toolkit', 'utilities', 'vendor'])) + sorted(_test_directories() - NOT_LEGS - {'protocols'})) for name, exclude in others: self.assertEqual(exclude, [], name) self.assertEqual(leg.leg_modules(name), leg.leg_modules(name, ()), name) + def test_every_tests_subdirectory_is_a_leg(self): + """#2115: a ``tests/`` subdirectory left out of the matrix is never run + under plain unittest, and the job stays green while covering less.""" + covered = {name.split('/', 1)[0] for name, _ in _matrix()} + self.assertEqual(sorted(_test_directories() - NOT_LEGS - covered), [], + 'a tests/ subdirectory is missing from unittest-ordering') + + def test_every_exception_is_real_and_still_left_out(self): + covered = {name.split('/', 1)[0] for name, _ in _matrix()} + for name in sorted(NOT_LEGS): + with self.subTest(name=name): + self.assertIn(name, _test_directories(), 'stale NOT_LEGS entry') + self.assertNotIn(name, covered, 'a leg now, so drop it from NOT_LEGS') + + def test_integration_has_no_unit_tier_module(self): + """The reason ``integration`` is not a leg: the leg would run only the + root-level modules.""" + self.assertEqual(leg.leg_modules('integration'), ()) + def test_no_shard_is_empty(self): for name, exclude in _matrix(): self.assertTrue(leg.leg_modules(name, exclude), name)