From 7f878830717b4e166428cb21c5857453a25c27e4 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 19 Oct 2026 16:37:38 -0400 Subject: [PATCH] ci(unit-tests): run platform-independent project pins once per push (#1954) - tests/_tiers.py: is_project_pin() and project_pin_paths(), the one rule. A tests/project module is a pin unless its AST, or that of the repository code it runs, imports pcapkit or compares the interpreter version. - unit-tests.yml: project-tests runs the pins under pytest on every pull request and push, but not under gate-only, and required-checks needs it; every test and test-ktlo leg ignores them. The docs-only steps stay. - experimental.yml: lists the pins on the runner's python3 and ignores them. - tests/_dependency_gates.py: models project-tests as its own selection. - Tier-guard tests run the steps against the rule; the 1732, queue, 1938 and 1672 tests, the workflows page and its citations follow the new layout. Closes #1954 --- .github/workflows/experimental.yml | 23 +- .github/workflows/unit-tests.yml | 116 +++- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 65 +- tests/_dependency_gates.py | 30 +- tests/_tiers.py | 563 ++++++++++++++++++ .../project/test_ktlo_legs_stay_green_1672.py | 3 +- tests/project/test_project_pins_once_1954.py | 96 +++ tests/project/test_superseded_runs_1938.py | 3 +- .../test_sweeps_on_latest_only_1732.py | 13 +- tests/project/test_unit_tests_queue.py | 34 +- tests/test_tier_guard.py | 241 +++++++- 12 files changed, 1108 insertions(+), 83 deletions(-) create mode 100644 tests/project/test_project_pins_once_1954.py diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 85fcc32815..3709d16dd2 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -242,6 +242,20 @@ jobs: id: checkout continue-on-error: true + # #1954: the project pins run once per push, in unit-tests.yml's + # `project-tests`, so these legs leave them out, as `test`'s do. The + # runner's own `python3` asks tests/_tiers.py, which needs only the + # standard library, before setup-python puts the leg's interpreter on + # PATH, so GraalPy does not parse the tree. Where it fails, the tests + # run the pins too, and `Experimental summary` names this step. + - name: List the project pins + id: pins + if: ${{ steps.checkout.outcome == 'success' }} + continue-on-error: true + run: | + pins=$(python3 -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))") + echo "paths=$pins" >> "$GITHUB_OUTPUT" + - uses: actions/setup-python@v7 id: python continue-on-error: true @@ -447,6 +461,7 @@ jobs: GH_TOKEN: ${{ github.token }} WATCHER: ${{ steps.watcher.outcome }} WATCHER_PYTHON: ${{ steps.watcher-python.outputs.python-path }} + PINS: ${{ steps.pins.outputs.paths }} run: | tail -n +1 -F "$RUNNER_TEMP/resource-monitor.log" 2>/dev/null & monitor=$! @@ -455,8 +470,12 @@ jobs: --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ - --ignore-glob='*_regression.py' \ - --ignore-glob='*roundtrip*' + --ignore-glob='*_regression.py' + # #1954: the project pins, from `List the project pins`. + for pin in $PINS; do + set -- "$@" --ignore="$pin" + done + set -- "$@" --ignore-glob='*roundtrip*' set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m python -m pytest "$@" scoped() { unit=$1 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3ffcf256a8..a88ea1d215 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -222,15 +222,29 @@ jobs: # collects, with `*` spanning `/` (_pytest/main.py:437-461, pytest # 9.1.1). The modules that import the sweeps' harnesses for their # packet builders still import them. + # + # #1954, the owner's option 3: the project pins -- the tests/project + # modules whose run does not vary by interpreter, tests/_tiers.py's + # project_pin_paths() -- run once per push, in `project-tests`, so every + # leg passes each as an `--ignore`, as experimental.yml's do. Asked of + # tests/_tiers.py here rather than listed, as `integration` asks for its + # selection. The pins import no pcapkit, so coverage loses nothing. env: COVERAGE: ${{ matrix.coverage }} SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }} run: | + if ! pins=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then + echo "::${ANNOTATION_LEVEL} title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above." + exit 1 + fi set -- -q -n auto --dist load \ --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ --ignore-glob='*_regression.py' + for pin in $pins; do + set -- "$@" --ignore="$pin" + done if [ "$SWEEPS" != "true" ]; then set -- "$@" --ignore-glob='*roundtrip*' fi @@ -1513,6 +1527,12 @@ jobs: # covering none of tests/integration. Adding it back here would buy that # nothing, not a failure. # + # The `project` cell keeps the project pins (#1954), which `test` leaves to + # `project-tests`: this job asks whether each module passes in one + # interpreter, in file order, with no xdist to isolate it, which is a + # question about the module and not the Python version, so a pin can fail + # it as readily as any other. It runs on main's pushes alone, one cell. + # # Not added to ``required-checks`` below: a brand-new job earns a place in # the required set once its own track record justifies it, which is the # maintainer's call to make separately from landing it. @@ -1721,23 +1741,32 @@ jobs: print(f'code={code}', file=file) PY - # The docs-only replacement for `test`, running every test that reads - # docs/ or a Markdown file, since a docs-only diff can still break those: - # tests/project and the root-level modules, plus the modules elsewhere under - # tests/ named in the second run step. tests/project/test_unit_tests_queue.py - # finds every module outside those that names a docs/ or Markdown path - # and fails until it is listed there, so a new reader cannot go unrun. Runs - # only when `changes` says the diff is docs-only, so it never duplicates - # `test` on a code change. It uses + # #1954, the owner's option 3: the project pins, once per push. They are the + # tests/project modules whose run does not vary by interpreter -- each reads + # the repository's text or drives a util/ script, and none imports pcapkit + # or branches on the Python version, by tests/_tiers.py's + # is_project_pin() -- so one run on `latest` is every leg's answer, and + # `test` and experimental.yml leave them out. On every pull request and + # every push, `required-checks` needs this job to pass; under `gate-only`, + # `gate`'s full suite runs them instead. tests/_dependency_gates.py reads its + # pytest step as the `project-pins` selection. + # + # On a docs-only diff, it is `test`'s replacement too, running every test + # that reads docs/ or a Markdown file, since a docs-only diff can still break + # those: tests/project and the root-level modules, plus the modules + # elsewhere under tests/ named in the last run step. + # tests/project/test_unit_tests_queue.py finds every module outside those + # that names a docs/ or Markdown path and fails until it is listed there, so + # a new reader cannot go unrun. Those steps run only when `changes` says the + # diff is docs-only, so they never duplicate `test` on a code change, and + # they take in the pins, so the pins step is skipped there. They use # util/run_unittest_leg.py's `project` leg -- tests/project plus the # root-level modules, with the extras `unittest-ordering`'s `project` cell - # runs green on main -- rather than a pytest invocation of its own, which - # tests/_dependency_gates.py would then have to model as a new selection - # shape. + # runs green on main. project-tests: - name: Project tests (docs-only) + name: Project tests needs: [changes] - if: ${{ inputs.gate-only != true && needs.changes.outputs.code == 'false' }} + if: ${{ inputs.gate-only != true }} runs-on: ubuntu-latest timeout-minutes: 20 @@ -1754,16 +1783,51 @@ jobs: python-version: ${{ steps.python-versions.outputs.latest }} cache: pip + # `test`'s install line, so a gated pin skips or runs here exactly as it + # did on `latest`. - name: Install package and test dependencies run: | python -m pip install -U pip setuptools wheel python -m pip install -e '.[test,DPKT,crypto,NGAP]' + # See the `test` job above for why this step exists. + - name: Report available parallelism + run: | + nproc + python -c "import os; print('cpu_count', os.cpu_count())" + + # `test`'s pytest flags, with the pins as the selection. A selection that + # cannot be computed, or is empty, fails the step rather than letting a + # bare `pytest` run the whole suite. `ulimit -f` caps any file a test + # writes at 2 GiB (bash counts 1024-byte blocks), as experimental.yml's + # `prlimit --fsize` does. The step cap is several times the run's + # length, under the job's 20. + - name: Run the project pins + if: ${{ needs.changes.outputs.code != 'false' }} + timeout-minutes: 10 + shell: bash + run: | + ulimit -f 2097152 + if ! pins=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then + echo "::error title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above." + exit 1 + fi + if [ -z "$pins" ]; then + echo "::error title=No project pins::tests._tiers.project_pin_paths() returned nothing. Refusing to run pytest with no arguments, which would run the whole suite." + exit 1 + fi + # shellcheck disable=SC2086 + set -- $pins + python -m pytest -q -n auto --dist load \ + --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" "$@" + - name: Run tests/project and the root-level modules under plain unittest + if: ${{ needs.changes.outputs.code == 'false' }} timeout-minutes: 15 run: python util/run_unittest_leg.py project --verbose - name: Run the other tests that read docs/ or Markdown + if: ${{ needs.changes.outputs.code == 'false' }} timeout-minutes: 5 run: python -m unittest -v tests.corekit.test_sentinel_exports_unit tests.utilities.test_config_unit tests.vendor.test_vendor_variables_1726_unit @@ -1902,8 +1966,8 @@ jobs: # gating leg, and folding it in would widen what merging requires -- a # change for the maintainer to decide, not one to slip in here. # `test`, `integration`, `engine-tests` and `pypcap-parity` are the gating - # legs; `changes` and `project-tests` are needed only for the docs-only - # path below. + # legs, and `project-tests` gates every diff too, as the one run of the + # project pins (#1954); `changes` is needed for the docs-only path below. # # `engine-tests`'s `unsupported` and `not-installable` cells are not a gap # here: the cell script exits 0 for a cell that declined exactly as @@ -1924,13 +1988,13 @@ jobs: # `docs/**` and `*.md` files outside the code directories (the `changes` # job's classifier), `test`, `integration`, `engine-tests` and # `pypcap-parity` are skipped by their own `if:`, and `project-tests` runs - # instead. A job-level `if:` rather than workflow-level `paths-ignore`, - # because `paths-ignore` would not run this workflow at all and leave this - # required context pending forever. So `skipped` is accepted for those - # legs **only** when `changes` succeeded and said `code == 'false'`, and then - # `project-tests` must have succeeded. Every other combination keeps the - # strict rule: `changes` failing, its output missing, or any leg skipped - # while `code` is `true` fails this gate. `changes` reports `code: true` for + # the docs readers as well. A job-level `if:` rather than workflow-level + # `paths-ignore`, because `paths-ignore` would not run this workflow at all + # and leave this required context pending forever. So `skipped` is accepted + # for those legs **only** when `changes` succeeded and said + # `code == 'false'`; `project-tests` must have succeeded either way. Every + # other combination keeps the strict rule: `changes` failing, its output + # missing, or any leg skipped while `code` is `true` fails this gate. `changes` reports `code: true` for # every event but this workflow's own `pull_request`, so `push` and # `workflow_call` runs never take this path. required-checks: @@ -1966,11 +2030,13 @@ jobs: failed=1 fi - # Docs-only (code == false): a leg may be skipped, and project-tests - # is what must have passed. Otherwise every leg must have succeeded. + # project-tests runs the project pins on every diff (#1954), and the + # docs readers on a docs-only one, so it must pass on both paths. + # Docs-only (code == false): a leg may be skipped. Otherwise every leg + # must have succeeded. + [ "$PROJECT_TESTS" = "success" ] || { echo "::error title=project-tests did not succeed::result was '$PROJECT_TESTS'; it runs the project pins, and on a docs-only diff the docs readers too, so it has to pass."; failed=1; } if [ "$CODE" = "false" ]; then allowed="success skipped" - [ "$PROJECT_TESTS" = "success" ] || { echo "::error title=project-tests did not succeed::result was '$PROJECT_TESTS'; on a docs-only diff it is the test that has to pass."; failed=1; } else allowed="success" fi diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 347da17bb7..6aebe8970d 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1776-1846``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1840-1910``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1624-1650``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1644-1670``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 3021e817ab..0a626b6385 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -323,12 +323,13 @@ engine, each looping over its Python versions), ``pypcap-parity``, once for this commit from Unit Tests' own ``push`` trigger (and, all but ``unittest-ordering``, which runs on ``main`` pushes only, from its ``pull_request`` trigger), so running any of them again per caller would test -the same commit several times over. It also skips the docs-only path's -``changes`` and ``project-tests``, and ``required-checks`` (see `Required Status +the same commit several times over. It also skips ``changes``, +``project-tests``, whose project pins ``gate``'s full suite runs too (see +`Project Pins`_ below), and ``required-checks`` (see `Required Status Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1612-1618``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1632-1638``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -638,7 +639,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1937`` + - ``unit-tests.yml:2001`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -660,30 +661,31 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1937``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:2001``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1849`` mentions the string without defining it. +``unit-tests.yml:1913`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1848-1858``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1912-1922``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, -``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` -for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1936-1990``). Rulesets match check names literally, with no +``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of +the project pins, and ``changes`` for the docs-only path -- and checks each +result explicitly +(``unit-tests.yml:2000-2056``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where -``project-tests`` -- every test that reads ``docs/`` or Markdown -- must pass -instead. The ``Compat`` legs stay required separately because they come from +``project-tests`` runs every test that reads ``docs/`` or Markdown as well; +``project-tests`` must pass on every diff. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1939``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2003``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -707,7 +709,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:406-423``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:420-437``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -838,7 +840,10 @@ holds ``roundtrip`` (``ROUNDTRIP_SWEEP_GLOB`` in :file:`tests/_tiers.py`), and since pytest applies no ignore to a path it was handed. On 2026-10-14 they took 8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% on the 3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy 88-113 -minutes (estimate). +minutes (estimate). The experimental legs leave out the project pins too, as +``test``'s do (see `Project Pins`_ below); ``List the project pins`` asks for them +on the runner's own ``python3``, before setup-python, so GraalPy does not parse the +tree. GraalPy's leg still loses its runner 18-30 minutes into its tests, with ``The runner has received a shutdown signal`` (`#1740 `__), @@ -905,6 +910,36 @@ workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inf from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a warning: that is expected, blocks nothing, and leaves the caps as they are. +Project Pins +------------ + +The *project pins* run once per push (`#1954 +`__): the modules under +:file:`tests/project` whose run does not vary by interpreter, as they read the +repository's text or drive a ``util/`` script. ``is_project_pin()`` in +:file:`tests/_tiers.py` decides, by the AST of each module and of the repository +code it runs: a module is a pin unless it, a module it takes names from, or a file +it runs by path imports ``pcapkit``, or compares ``sys.version_info`` or +``sys.implementation``. A computed import, or a ``.py`` name computed for a +loader or a child process, counts as importing it, and so does a file that does +not parse. The modules that import ``pcapkit`` stay on every leg. + +``project-tests`` runs the pins under pytest, with ``test``'s flags, on ``latest``, +on every pull request and every push to ``main``, and ``required-checks`` needs it. +Every ``test``, ``test-ktlo`` and ``experimental`` leg passes each pin as an +``--ignore``, ``latest``'s coverage leg included, which measures only ``pcapkit``. +Each asks ``project_pin_paths()`` at run time, as ``integration`` asks for its +selection, so no workflow lists the pins, and :file:`tests/test_tier_guard.py` runs +the steps to check what each does with the answer. Under ``gate-only``, ``gate``'s +full suite runs the pins with everything else. ``unittest-ordering``'s ``project`` +cell keeps them, as it asks how a module runs in one interpreter, in file order and +without xdist, which is a question about the module rather than the Python version. + +A pin's skip where ``tomllib`` is missing, before 3.11, runs on no leg: the scripts +those pins drive read the tier file on the runner's own ``python3``, and +:file:`tests/project/test_python_versions_1660.py` checks those skips in a child +interpreter that cannot import ``tomllib``. + Python Versions --------------- diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index b040fac47a..1d86bfc246 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -652,6 +652,7 @@ class Exclusion(NamedTuple): ), 'HAS_SPHINX': Exclusion( dark={'test': ('sphinx', 'sphinx_autodoc_typehints'), + 'project-tests': ('sphinx', 'sphinx_autodoc_typehints'), 'gate': ('sphinx', 'sphinx_autodoc_typehints')}, reason=( 'The classes it gates (#1775, #1856, #1858-#1863, #1917, #1918 and the enum-value ' @@ -669,11 +670,12 @@ class Exclusion(NamedTuple): 'imports. They run wherever docs/requirements.txt or the docs extra is ' 'installed, a developer checkout included, and skip visibly elsewhere, saying ' 'which condition failed. The directive-name and pin-agreement tests need ' - 'nothing and run on every leg.' + 'nothing and run on every leg, or in project-tests where their module is a ' + 'project pin (#1954).' ), ), 'HAS_PYLINT': Exclusion( - dark={'test': ('pylint',), 'gate': ('pylint',)}, + dark={'project-tests': ('pylint',), 'gate': ('pylint',)}, reason=( 'The method it gates (#1923) looks every pylint pragma name in pcapkit/ up in ' "pylint's own message store, with the plugins `make pylint` loads, so it needs " @@ -683,8 +685,9 @@ class Exclusion(NamedTuple): 'no pyproject.toml extra declares it. The argument that declines HAS_MYPY above ' 'declines it here: lint.yml already runs pylint over the whole package, and an ' 'unknown pragma name shows there as W0012 unknown-option-value, so a pytest leg ' - 'that installed pylint would pay for a second copy of that check. What runs on ' - 'every leg instead is the ungated check in the same module, which holds each ' + 'that installed pylint would pay for a second copy of that check. What runs ' + 'instead is the ungated check in the same module, a project pin that ' + 'project-tests runs (#1954), which holds each ' 'pragma name to the pinned KNOWN_MESSAGES list; the lookup keeps that list ' 'honest wherever pylint is importable, a developer checkout included, and skips ' 'visibly elsewhere.' @@ -777,7 +780,9 @@ class Job(NamedTuple): #: :func:`~tests._tiers.fixture_tier_paths` for the selection, #: ``'ignore'`` when it subtracts ``--ignore`` flags from the whole suite, #: ``'whole-suite'`` when it passes no selection at all, ``'explicit'`` - #: when it names literal test-file paths (see :attr:`paths`). + #: when it names literal test-file paths (see :attr:`paths`), and + #: ``'project-pins'`` when it runs what :func:`~tests._tiers.project_pin_paths` + #: prints (#1954). selection: 'str' #: The literal test-file paths named by an ``'explicit'`` selection, empty #: for every other shape. Compared to :class:`Gate`.\ ``module`` by exact @@ -1572,6 +1577,8 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': selection = 'fixture-tier' elif '--ignore' in run_block: selection = 'ignore' + elif 'project_pin_paths' in run_block: + selection = 'project-pins' else: selection = 'whole-suite' @@ -1583,12 +1590,13 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': """Whether ``job``'s selection would collect the tests ``gate`` guards. - The four selection shapes are answered four ways, and none of them + The five selection shapes are answered five ways, and none of them reimplements the workflow's own list: * ``'whole-suite'`` reaches everything under :file:`tests/`. - * ``'ignore'`` is the unit tier, so :func:`~tests._tiers.is_unit_tier` - answers it -- and + * ``'ignore'`` is the unit tier less the project pins (#1954), so + :func:`~tests._tiers.is_unit_tier` and :func:`~tests._tiers.is_project_pin` + answer it -- and :meth:`~tests.test_tier_guard.WorkflowAgreementTests\ .test_ignore_flags_match_the_fixture_tier_constants` is what keeps that equivalence true. It is the ``latest`` leg's: the @@ -1596,6 +1604,8 @@ def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': changes no answer, since every leg shares the job's one install line. The same holds for ``'fixture'``, whose legs off ``latest`` drop :data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB`'s modules. + * ``'project-pins'`` is :func:`~tests._tiers.is_project_pin`, the job running + exactly what :func:`~tests._tiers.project_pin_paths` prints. * ``'explicit'`` is exact membership in :attr:`Job.paths` -- the literal ``TEST_PATHS`` list a ``matrix.engine`` cell of ``engine-tests`` (#849) runs, at file granularity (that job names whole files, never node IDs). @@ -1619,7 +1629,9 @@ def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': if job.selection == 'whole-suite': return True if job.selection == 'ignore': - return _tiers.is_unit_tier(gate.module) + return _tiers.is_unit_tier(gate.module) and not _tiers.is_project_pin(gate.module) + if job.selection == 'project-pins': + return _tiers.is_project_pin(gate.module) if job.selection == 'explicit': return gate.module in job.paths diff --git a/tests/_tiers.py b/tests/_tiers.py index a7e5aa1085..f9bf4e7aa4 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -48,6 +48,9 @@ in order to still exercise them. * :func:`is_roundtrip_sweep` -- is this unit-tier module one of the round-trip sweeps, which only the ``latest`` leg of the ``test`` job runs (#1732)? +* :func:`is_project_pin` and :func:`project_pin_paths` -- which + :file:`tests/project` modules give one answer on every interpreter, and so run + once per push, in the ``project-tests`` job, rather than on every leg (#1954)? :file:`tests/conftest.py` runs :func:`is_unit_tier` and :func:`audit_module` -- and through it :func:`committed_captures` and :func:`explain` -- at collection @@ -124,6 +127,7 @@ import fnmatch import functools import pathlib +import re import subprocess from typing import TYPE_CHECKING, NamedTuple @@ -135,6 +139,8 @@ 'ROOT', 'TESTS_ROOT', 'SAMPLE_ROOT', 'REGENERATE_SAMPLES_CMD', 'UNIT_TIER_SELECTION', 'FIXTURE_TIER_SUFFIXES', 'FIXTURE_TIER_DIRS', 'FIXTURE_TIER_SELECTOR', 'ROUNDTRIP_SWEEP_GLOB', 'ROUNDTRIP_RUNTIME_GLOB', 'is_roundtrip_sweep', + 'PROJECT_PINS_DIR', 'PROJECT_PINS_SELECTOR', 'PROJECT_PIN_PACKAGE', 'LOADABLE_DIRS', + 'varies_by_interpreter', 'is_project_pin', 'project_pin_paths', 'is_unit_tier', 'committed_captures', 'committed_capture_names', 'guard_unavailable_reason', 'handled_lines', 'sample_path_calls', 'audit_module', 'check_unit_tier_read', 'explain', @@ -188,6 +194,27 @@ #: alone. Every other leg drops them from :func:`fixture_tier_paths`'s selection and #: passes this as an ``--ignore-glob``, since pytest ignores no path it was handed. ROUNDTRIP_RUNTIME_GLOB = '*roundtrip*_runtime.py' +#: The directory whose platform- and version-independent modules, the *project +#: pins*, run once per push in the ``project-tests`` job of +#: :file:`.github/workflows/unit-tests.yml` rather than on every leg (#1954). See +#: :func:`is_project_pin`. +PROJECT_PINS_DIR = 'project' +#: The project pins' selector, verbatim from the workflows: the ``project-tests`` +#: job runs what it prints, and every ``test`` and ``experimental`` leg passes each +#: path as an ``--ignore``. Asked at run time, as :data:`FIXTURE_TIER_SELECTOR` is, +#: so neither workflow can drift from :func:`is_project_pin`. +PROJECT_PINS_SELECTOR = ( + 'python -c "from tests._tiers import project_pin_paths; ' + 'print(\' \'.join(project_pin_paths()))"' +) +#: The package whose behaviour varies by interpreter, so a module that reaches it +#: runs on every leg (#1954). +PROJECT_PIN_PACKAGE = 'pcapkit' +#: The directories, besides :file:`tests/`, whose Python files a test module may load +#: by path -- a ``util/`` script, a generator, :file:`docs/source/conf.py` -- and +#: :func:`varies_by_interpreter` follows. :file:`pcapkit/` is left out on purpose: a test +#: that names a file there reads its text, which is the same on every leg. +LOADABLE_DIRS = ('tests', 'util', 'examples', 'docs') #: Suffixes that make a tracked file worth suggesting as a replacement capture. #: Committedness itself is whatever git says -- this filter only keeps the #: suggestion in :func:`explain` from offering :file:`out.txt` as a capture. @@ -302,6 +329,542 @@ def is_roundtrip_sweep(path: 'pathlib.Path | str') -> 'bool': str(ROOT / ROUNDTRIP_SWEEP_GLOB)) +#: Calls that import the module their first argument names. ``patch`` is +#: :func:`unittest.mock.patch`, which imports its target's module, and ``find_spec`` +#: imports a dotted name's parents. +_IMPORTING_CALLS = frozenset({'import_module', '__import__', 'patch', 'find_spec'}) +#: Of those, the ones whose computed argument may import anything at all. +_COMPUTED_IMPORTS = frozenset({'import_module', '__import__'}) +#: Calls that run a file or a child process, so the files, directories and modules +#: their arguments name run too. +_RUNNING_CALLS = frozenset({ + 'spec_from_file_location', 'run_path', 'SourceFileLoader', 'load_source', 'run', 'Popen', + 'call', 'check_call', 'check_output', 'system', 'create_subprocess_exec', 'create_subprocess_shell', + 'exec', 'eval', +}) +#: Source text that imports :data:`PROJECT_PIN_PACKAGE` in a child interpreter: a +#: ``python -c`` script, or ``-m pcapkit``. Not ``from pcapkit/__init__.py``, a path. +_IMPORT_TEXT = re.compile(rf'(?:\b(?:import|from)|-m)\s+{PROJECT_PIN_PACKAGE}(?:\.\w+)*(?![\w/.])') +#: A string that is wholly a relative path, e.g. ``util/ci_timing.py`` or ``tests/cli``. +_PATH_TEXT = re.compile(r'[\w.-]+(?:/[\w.-]+)*/?\Z') +#: A string that is wholly a dotted module name, e.g. ``tests.project.test_setup``. +_DOTTED_NAME = re.compile(r'[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)+\Z') + +#: What a module asks of the running interpreter when it branches on it: +#: ``sys.version_info``, ``sys.implementation``, ``platform.python_version()``. +_INTERPRETER_FACTS = frozenset({'version_info', 'hexversion', 'implementation', 'python_implementation', + 'python_version', 'python_version_tuple'}) +#: The same, imported bare: ``from sys import version_info``. +_INTERPRETER_NAMES = frozenset({'version_info', 'hexversion'}) + +#: One unit of :func:`varies_by_interpreter`'s search: a file, and the top-level names of it +#: that run -- :data:`None` for the whole file, as when it is the test module itself, +#: is loaded by path, or is imported as a module object. +_Region = 'tuple[pathlib.Path, Optional[frozenset[str]]]' + + +def _is_package(name: 'str') -> 'bool': + return name == PROJECT_PIN_PACKAGE or name.startswith(PROJECT_PIN_PACKAGE + '.') + + +@functools.lru_cache(maxsize=1) +def _loadable_files() -> 'tuple[pathlib.Path, ...]': + """Every Python file a test may run by path: those under :data:`LOADABLE_DIRS`, and the root's.""" + files = set(ROOT.glob('*.py')) + for name in LOADABLE_DIRS: + files.update(path for path in (ROOT / name).rglob('*.py') if '__pycache__' not in path.parts) + return tuple(sorted(files)) + + +@functools.lru_cache(maxsize=None) +def _paths_named(text: 'str') -> 'frozenset[pathlib.Path]': + """The loadable files a path string names: the files whose path ends with it, + e.g. ``conf.py``, or every file under the directory it is, e.g. ``tests/cli``. + + A single word is no directory: ``ROOT / 'util'`` is a prefix being built, and + ``'examples'`` an argument as often as a path. + + """ + relative = text.strip('/') + while relative.startswith('./'): + relative = relative[2:] + if relative.endswith('.py'): + return frozenset(path for path in _loadable_files() + if ('/' + path.relative_to(ROOT).as_posix()).endswith('/' + relative)) + directory = ROOT / relative + if '/' not in relative or not directory.is_dir(): + return frozenset() + return frozenset(path for path in _loadable_files() if directory in path.parents) + + +def _module_file(name: 'str') -> 'Optional[pathlib.Path]': + """The file of the repository module ``name``, e.g. ``tests._support``, if there is one. + + A ``util/`` script counts as a module too, as the tests that put :file:`util/` + on :data:`sys.path` import it: ``import run_unittest_leg``. + + """ + for root in (ROOT, ROOT / 'util'): + base = root.joinpath(*name.split('.')) + for candidate in (base.with_suffix('.py'), base / '__init__.py'): + if candidate.is_file(): + return candidate + return None + + +@functools.lru_cache(maxsize=None) +def _parse_module(path: 'pathlib.Path') -> 'Optional[ast.Module]': + try: + return ast.parse(path.read_text(encoding='utf-8'), filename=str(path)) + except (OSError, UnicodeDecodeError, SyntaxError, ValueError): + return None + + +def _called(call: 'ast.Call') -> 'Optional[str]': + """The bare name a call is made through, e.g. ``run`` for ``subprocess.run(...)``.""" + func = call.func + return func.id if isinstance(func, ast.Name) else func.attr if isinstance(func, ast.Attribute) else None + + +def _defined_names(node: 'ast.stmt') -> 'set[str]': + """The top-level names a module-level statement binds, in a block it holds too: + ``if sys.version_info >= (3, 11): import tomllib`` binds ``tomllib``.""" + if isinstance(node, (ast.If, ast.Try, ast.With, ast.For, ast.While)) or type(node).__name__ == 'TryStar': + return {name for field in ('body', 'orelse', 'finalbody', 'handlers') + for child in getattr(node, field, ()) for name in _defined_names(child)} + if isinstance(node, ast.ExceptHandler): + return {name for child in node.body for name in _defined_names(child)} + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + return {node.name} + if isinstance(node, (ast.Import, ast.ImportFrom)): + return {(alias.asname or alias.name).partition('.')[0] for alias in node.names} + targets = [] # type: list[ast.expr] + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + targets = [node.target] + return {name.id for target in targets for name in ast.walk(target) if isinstance(name, ast.Name)} + + +@functools.lru_cache(maxsize=None) +def _import_time_nodes(tree: 'ast.Module') -> 'tuple[ast.AST, ...]': + """Every import and call of ``tree`` outside a function body, which run on import.""" + inside = set() # type: set[int] + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + for statement in node.body if isinstance(node.body, list) else [node.body]: + inside.update(id(child) for child in ast.walk(statement)) + return tuple(node for node in ast.walk(tree) + if isinstance(node, (ast.Import, ast.ImportFrom, ast.Call)) and id(node) not in inside) + + +def _executed_nodes(tree: 'ast.Module', + names: 'Optional[frozenset[str]]') -> 'tuple[list[ast.AST], list[ast.AST]]': + """The nodes of ``tree`` that run when ``names`` of it are used, as two lists. + + The first is what runs on import -- every import and every call outside a + function -- and the second the definitions of ``names``, with the other + top-level names those use. For :data:`None`, the whole module is the second. + + """ + if names is None: + return [], [tree] + on_import = list(_import_time_nodes(tree)) + nodes = [] # type: list[ast.AST] + wanted, todo = set(), list(names) # type: set[str], list[str] + while todo: + name = todo.pop() + if name in wanted: + continue + wanted.add(name) + for statement in tree.body: + if name in _defined_names(statement): + nodes.append(statement) + todo.extend(node.id for node in ast.walk(statement) + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)) + return on_import, nodes + + +def _compares_interpreter(node: 'ast.AST') -> 'bool': + """Whether ``node`` compares the running interpreter, e.g. ``sys.version_info < (3, 12)``.""" + if not isinstance(node, ast.Compare): + return False + for operand in [node.left] + list(node.comparators): + for part in ast.walk(operand): + if isinstance(part, ast.Name) and part.id in _INTERPRETER_NAMES: + return True + if (isinstance(part, ast.Attribute) and part.attr in _INTERPRETER_FACTS + and isinstance(part.value, ast.Name) and part.value.id in ('sys', 'platform')): + return True + return False + + +#: The key :func:`_bindings` files a ``self.name`` or ``cls.name`` attribute under. +_SELF = -1 + + +@functools.lru_cache(maxsize=None) +def _scopes(tree: 'ast.Module') -> 'tuple[dict[int, int], dict[int, int]]': + """Each node's scope in ``tree``, and each scope's enclosing one, by ``id``. + + A scope is the module, a function or a lambda; a class body belongs to the + scope around it, as its names are reached through ``self`` or ``cls``. + + """ + scope_of = {id(tree): id(tree)} # type: dict[int, int] + parent = {} # type: dict[int, int] + todo = [(tree, id(tree))] # type: list[tuple[ast.AST, int]] + while todo: # not recursion: a long ``'a' + 'b' + ...`` nests as deep as it is long + node, scope = todo.pop() + for child in ast.iter_child_nodes(node): + scope_of[id(child)] = scope + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + parent[id(child)] = scope + todo.append((child, id(child))) + else: + todo.append((child, scope)) + return scope_of, parent + + +@functools.lru_cache(maxsize=None) +def _bindings(tree: 'ast.Module') -> 'dict[tuple[int, str], tuple[ast.AST, ...]]': + """What each name of each scope of ``tree``, and each ``self.`` attribute, is bound to. + + Keyed by ``(scope, name)``, ``(_SELF, attribute)`` for an attribute of ``self`` + or ``cls``. An assignment binds its target to its value, a ``for`` loop or a + comprehension its target to what it iterates, and a function's parameter to + its default and to each argument the module passes it -- every one, since + which call runs is not known. + + """ + scope_of = _scopes(tree)[0] + found = {} # type: dict[tuple[int, str], list[ast.AST]] + + def bind(target: 'ast.AST', value: 'ast.AST') -> None: + for node in ast.walk(target): + if isinstance(node, ast.Name): + found.setdefault((scope_of[id(node)], node.id), []).append(value) + elif (isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) + and node.value.id in ('self', 'cls')): + found.setdefault((_SELF, node.attr), []).append(value) + + functions = {} # type: dict[str, list[tuple[int, ast.arguments]]] + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + bind(target, node.value) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign, ast.NamedExpr)) and node.value is not None: + bind(node.target, node.value) + elif isinstance(node, (ast.For, ast.AsyncFor, ast.comprehension)): + bind(node.target, node.iter) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + functions.setdefault(node.name, []).append((id(node), node.args)) + for arg, default in zip(reversed(node.args.args), reversed(node.args.defaults)): + found.setdefault((id(node), arg.arg), []).append(default) + for call in ast.walk(tree): + if not isinstance(call, ast.Call): + continue + for scope, arguments in functions.get(_called(call) or '', ()): + params = [arg.arg for arg in arguments.posonlyargs + arguments.args] + if params[:1] in (['self'], ['cls']) and isinstance(call.func, ast.Attribute): + params = params[1:] + for param, value in zip(params, call.args): + found.setdefault((scope, param), []).append(value) + if arguments.vararg is not None: + found.setdefault((scope, arguments.vararg.arg), []).extend(call.args[len(params):]) + for keyword in call.keywords: + if keyword.arg is not None: + found.setdefault((scope, keyword.arg), []).append(keyword.value) + return {key: tuple(values) for key, values in found.items()} + + +def _bound_values(tree: 'ast.Module', node: 'ast.AST') -> 'tuple[ast.AST, ...]': + """What the name or ``self.`` attribute ``node`` may hold, from its own scope outwards.""" + bindings = _bindings(tree) + if isinstance(node, ast.Attribute): + return bindings.get((_SELF, node.attr), ()) + assert isinstance(node, ast.Name) + scope_of, parent = _scopes(tree) + scope = scope_of.get(id(node), id(tree)) + while True: + values = bindings.get((scope, node.id)) + if values is not None or scope == id(tree): + return values or () + scope = parent[scope] + + +def _imported_from(tree: 'ast.Module', path: 'pathlib.Path', name: 'str') -> 'Optional[tuple[pathlib.Path, str]]': + """Where ``tree`` imports ``name`` from, as the module's file and the name there.""" + for node in tree.body: + if isinstance(node, ast.ImportFrom): + for module, taken in _imports(node, path): + found = _module_file(module) + if found is not None and taken is not None: + for alias in node.names: + if (alias.asname or alias.name) == name and alias.name in taken: + return found, alias.name + return None + + +def _flowing_strings(expressions: 'list[ast.AST]', path: 'pathlib.Path') -> 'tuple[set[str], bool]': + """The strings that may flow into ``expressions``, and whether a ``.py`` name is computed. + + Followed through :func:`_bindings`, and through a name imported from another + repository module to its bindings there. + + """ + strings = set() # type: set[str] + seen = set() # type: set[tuple[pathlib.Path, int]] + origins = set() # type: set[tuple[pathlib.Path, str]] + todo = [(expression, path) for expression in expressions] + while todo: + node, where = todo.pop() + if (where, id(node)) in seen: + continue + seen.add((where, id(node))) + if isinstance(node, ast.Constant): + if isinstance(node.value, str): + strings.add(node.value) + elif isinstance(node, ast.JoinedStr): + for before, after in zip(node.values, node.values[1:]): + # ``f'{name}.py'``: a file whose name is computed. + if (isinstance(before, ast.FormattedValue) and isinstance(after, ast.Constant) + and str(after.value).startswith('.py')): + return strings, True + todo.extend((value, where) for value in node.values) + elif isinstance(node, (ast.Name, ast.Attribute)): + if isinstance(node, ast.Attribute): + todo.append((node.value, where)) + if not (isinstance(node.value, ast.Name) and node.value.id in ('self', 'cls')): + continue # ``sys.executable``: nothing this module binds + tree = _parse_module(where) + if tree is None: + continue + values = _bound_values(tree, node) + todo.extend((value, where) for value in values) + origin = _imported_from(tree, where, node.id) if isinstance(node, ast.Name) and not values else None + if origin is not None and origin not in origins: + origins.add(origin) + todo.append((ast.Name(id=origin[1], ctx=ast.Load()), origin[0])) + elif isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + # ``ROOT / 'util' / 'ci_timing.py'`` names ``util/ci_timing.py``, not ``util``. + parts = [] # type: list[ast.expr] + while isinstance(node, ast.BinOp) and isinstance(node.op, ast.Div): + parts.append(node.right) + node = node.left + parts.append(node) + joined = [] # type: list[str] + for part in reversed([None] + parts): # type: ignore[list-item] + if isinstance(part, ast.Constant) and isinstance(part.value, str): + joined.append(part.value) + continue + if joined: + strings.add('/'.join(joined)) + joined = [] + if part is not None: + todo.append((part, where)) + elif isinstance(node, ast.Call): + todo.extend((arg, where) for arg in node.args) + todo.extend((keyword.value, where) for keyword in node.keywords) + if isinstance(node.func, ast.Attribute): + todo.append((node.func.value, where)) + elif not isinstance(node, (ast.Lambda, ast.FunctionDef, ast.AsyncFunctionDef)): + todo.extend((child, where) for child in ast.iter_child_nodes(node)) + return strings, False + + +def _attributes_used(tree: 'ast.Module', name: 'str') -> 'Optional[frozenset[str]]': + """The attributes ``tree`` reads off the module it binds to ``name``, or :data:`None` + when it uses that module some other way, e.g. passes it on.""" + return _name_uses(tree).get(name, frozenset()) + + +@functools.lru_cache(maxsize=None) +def _name_uses(tree: 'ast.Module') -> 'dict[str, Optional[frozenset[str]]]': + """For each name ``tree`` reads, the attributes it reads off it, :data:`None` once it is + used bare.""" + values = {id(node.value) for node in ast.walk(tree) if isinstance(node, ast.Attribute)} + used = {} # type: dict[str, Optional[set[str]]] + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + found = used.setdefault(node.value.id, set()) + if found is not None: + found.add(node.attr) + elif isinstance(node, ast.Name) and id(node) not in values: + used[node.id] = None + return {name: None if attrs is None else frozenset(attrs) for name, attrs in used.items()} + + +def _imports(node: 'ast.AST', path: 'pathlib.Path', + tree: 'Optional[ast.Module]' = None) -> 'list[tuple[str, Optional[frozenset[str]]]]': + """The modules an ``import`` statement runs, and the names it takes from each. + + A module bound to a name of its own -- ``from tests.project import + test_release_gates as gates`` -- gives the attributes ``tree`` reads off it. + + """ + if isinstance(node, ast.Import): + return [(alias.name, _attributes_used(tree, alias.asname) if tree is not None and alias.asname else None) + for alias in node.names] + assert isinstance(node, ast.ImportFrom) + module = node.module or '' + if node.level: + try: + package = path.parent.relative_to(ROOT).parts + except ValueError: # a module outside the repository, as a test writes one + package = () + package = package[:len(package) - node.level + 1] + module = '.'.join(package + ((module,) if module else ())) + found = [] # type: list[tuple[str, Optional[frozenset[str]]]] + names = set() # type: set[str] + for alias in node.names: + # ``from tests.project import test_release_gates`` imports a module, not a name. + if alias.name != '*' and _module_file(f'{module}.{alias.name}') is not None: + taken = _attributes_used(tree, alias.asname or alias.name) if tree is not None else None + found.append((f'{module}.{alias.name}', taken)) + else: + names.add(alias.name) + if names: + found.append((module, None if '*' in names else frozenset(names))) + return found + + +@functools.lru_cache(maxsize=None) +def _region_reach(path: 'pathlib.Path', names: 'Optional[frozenset[str]]') -> 'tuple[bool, frozenset[_Region]]': + """Whether the region of ``path`` that ``names`` runs varies by interpreter itself, + and the regions of other repository files it runs. + + By the AST alone, and erring towards "varies": an unparsable file does, and so + does an import whose target is computed, or a file run whose name is. A + comparison counts in what the region defines, not in what merely runs on + import: a ``skipIf`` on a class nothing asked for decides nothing here. + + """ + tree = _parse_module(path) + if tree is None: + return True, frozenset() + modules = [] # type: list[tuple[str, Optional[frozenset[str]]]] + files = set() # type: set[pathlib.Path] + on_import, definitions = _executed_nodes(tree, names) + for root in on_import + definitions: + for node in ast.walk(root): + if isinstance(node, (ast.Import, ast.ImportFrom)): + modules.extend(_imports(node, path, tree)) + elif isinstance(node, ast.Call) and _called(node) in _IMPORTING_CALLS: + target = node.args[0] if node.args else None + if isinstance(target, ast.Constant) and isinstance(target.value, str): + modules.append((target.value, None)) + elif _called(node) in _COMPUTED_IMPORTS: + return True, frozenset() + elif isinstance(node, ast.Call) and _called(node) in _RUNNING_CALLS: + strings, computed = _flowing_strings(list(node.args) + [keyword.value for keyword in node.keywords], + path) + if computed or any(_IMPORT_TEXT.search(text) for text in strings): + return True, frozenset() + for text in strings: + if _DOTTED_NAME.match(text) and _module_file(text) is not None: + modules.append((text, None)) + elif _PATH_TEXT.match(text): + files.update(_paths_named(text)) + elif _compares_interpreter(node) and root in definitions: + return True, frozenset() + elif isinstance(node, (ast.List, ast.Tuple)): + # ``[sys.executable, '-m', 'pcapkit']`` + values = [element.value if isinstance(element, ast.Constant) else None for element in node.elts] + if any(first == '-m' and isinstance(second, str) and _is_package(second) + for first, second in zip(values, values[1:])): + return True, frozenset() + + regions = {(file, None) for file in files} # type: set[_Region] + for module, taken in modules: + if _is_package(module): + return True, frozenset() + found = _module_file(module) + if found is not None: + regions.add((found, taken)) + regions.discard((path, names)) + return False, frozenset(regions) + + +@functools.lru_cache(maxsize=None) +def varies_by_interpreter(path: 'pathlib.Path | str') -> 'bool': + """Whether running the module at ``path`` may give another answer on another interpreter (#1954). + + By the AST of the module and of the repository code it runs. It may when any + of that imports :data:`PROJECT_PIN_PACKAGE` -- an ``import`` at any depth; + :func:`importlib.import_module`, :func:`__import__`, :func:`unittest.mock.patch` + or :func:`importlib.util.find_spec` naming it; a child interpreter's ``import + pcapkit`` or ``-m pcapkit`` -- or compares ``sys.version_info``, + ``sys.implementation`` or the like. The module runs whole, and so does a module + it imports as an object, unless it only reads attributes off it, which run as + names taken from it do: the module's imports and the calls it makes on import, + and the definitions of those names. A file or directory runs whole when a + string naming it flows into a call that runs one, e.g. + ``spec_from_file_location`` or ``subprocess.run``: a ``util/`` script, a + generator, :file:`docs/source/conf.py`, a test directory a child ``pytest`` + collects. A computed import, a computed ``.py`` name, or a file that cannot be + parsed counts as importing the package. + + Args: + path: Path to a module, absolute or relative to :data:`ROOT`. + + """ + start = pathlib.Path(path) + if not start.is_absolute(): + start = ROOT / start + seen = set() # type: set[_Region] + todo = [(start.resolve(), None)] # type: list[_Region] + while todo: + region = todo.pop() + if region in seen: + continue + seen.add(region) + reaches, regions = _region_reach(*region) + if reaches: + return True + todo.extend(regions - seen) + return False + + +def is_project_pin(path: 'pathlib.Path | str') -> 'bool': + """Whether ``path`` names a project pin, which runs once per push (#1954). + + A project pin is a unit-tier test module under + :file:`tests/`:data:`PROJECT_PINS_DIR` whose run does not + :func:`vary by interpreter `: it reads the repository's + text or drives a ``util/`` script, so the ``project-tests`` job runs it once, on + ``latest``. Any other module stays on every leg of ``test`` and + ``experimental``. + + Args: + path: Path to a module, absolute or relative to :data:`ROOT`. + + """ + candidate = pathlib.Path(path) + if not candidate.is_absolute(): + candidate = ROOT / candidate + candidate = candidate.resolve() + try: + relative = candidate.relative_to(TESTS_ROOT / PROJECT_PINS_DIR) + except ValueError: + return False + return (relative.name.startswith('test_') and relative.suffix == '.py' and candidate.is_file() + and is_unit_tier(candidate) and not varies_by_interpreter(candidate)) + + +@functools.lru_cache(maxsize=1) +def project_pin_paths() -> 'tuple[str, ...]': + """Every :func:`project pin `, relative to :data:`ROOT`, sorted. + + What :data:`PROJECT_PINS_SELECTOR` prints: the ``project-tests`` job's + selection, and the ``--ignore`` list of every ``test`` and ``experimental`` leg. + + """ + return tuple(sorted(path.relative_to(ROOT).as_posix() + for path in (TESTS_ROOT / PROJECT_PINS_DIR).rglob('test_*.py') + if is_project_pin(path))) + + def _git(*args: 'str') -> 'Optional[str]': """Run :program:`git` in :data:`ROOT` and return its standard output. diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 815fc0eadc..63cad7df20 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -130,7 +130,8 @@ #: without a cap of their own -- checkout, the parallelism report, the resource #: monitor, the uploads and the summaries. Everything but the install and the tests #: took at most 0.32 minutes, setup-python included, on every uncovered leg of runs -#: 38569321543, 38569429406 and 38569522442. +#: 38569321543, 38569429406 and 38569522442. #1954's ``List the project pins`` runs +#: on the runner's CPython, which lists them in seconds (measured locally on 3.14). UNCAPPED_MINUTES = 3 #: Those steps' cap on every other leg: the largest a step's ``timeout-minutes`` may be #: (docs.github.com, workflow-syntax, ``jobs..steps[*].timeout-minutes``). diff --git a/tests/project/test_project_pins_once_1954.py b/tests/project/test_project_pins_once_1954.py new file mode 100644 index 0000000000..383c5dd588 --- /dev/null +++ b/tests/project/test_project_pins_once_1954.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +"""#1954: the project pins run once per push, in ``project-tests``. + +The owner's option 3, paraphrased: run the :file:`tests/project` modules whose run +does not vary by interpreter once per push, on pull requests and on pushes to +``main`` alike, and leave them out of every matrix leg; the modules that import +``pcapkit`` stay on every leg. :func:`tests._tiers.is_project_pin` decides which, +and :file:`tests/test_tier_guard.py` checks that rule and runs the steps that use +it. This module pins the rest: the job runs on every diff, on ``latest``, and +blocks; the release gate and the ``unittest-ordering`` cell keep every pin; the +workflows page says so. + +""" + +from __future__ import annotations + +import pathlib +import sys +import unittest + +from tests import _tiers +from tests.project.test_python_matrix_tiers_1633 import job + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' +#: The job, and its step that runs the pins. +JOB = ('unit-tests.yml', 'project-tests') +STEP = 'Run the project pins' + +if str(ROOT / 'util') not in sys.path: + sys.path.insert(0, str(ROOT / 'util')) + +import run_unittest_leg # noqa: E402 pylint: disable=wrong-import-position + + +def step(where: 'tuple[str, str]', name: 'str') -> 'dict[str, object]': + found = [entry for entry in job(*where)['steps'] if entry.get('name') == name] + if len(found) != 1: + raise AssertionError(f'expected one {name!r} step in {where}, found {len(found)}') + return found[0] + + +class TestThePinsJob(unittest.TestCase): + """``project-tests`` runs the pins on every diff but ``gate-only``'s, on ``latest``, and gates.""" + + def test_it_runs_on_every_diff_but_not_under_gate_only(self) -> None: + spec = job(*JOB) + self.assertEqual(spec['if'], '${{ inputs.gate-only != true }}') + self.assertEqual(spec['needs'], ['changes']) + self.assertEqual(step(JOB, STEP)['if'], "${{ needs.changes.outputs.code != 'false' }}") + self.assertIn('project-tests', job('unit-tests.yml', 'required-checks')['needs']) + + def test_it_runs_on_latest_with_what_test_installs(self) -> None: + setup = next(entry for entry in job(*JOB)['steps'] if 'setup-python' in entry.get('uses', '')) + self.assertEqual(setup['with']['python-version'], '${{ steps.python-versions.outputs.latest }}') + install = step(JOB, 'Install package and test dependencies')['run'] + self.assertIn(step(('unit-tests.yml', 'test'), 'Install package and test dependencies')['run'] + .split("pip install -e '", 1)[1].split("'", 1)[0], install) + + def test_it_runs_under_the_caps(self) -> None: + """xdist as ``test`` runs it, a 2 GiB file cap, and a step cap under the job's.""" + run = str(step(JOB, STEP)['run']) + self.assertIn('ulimit -f 2097152\n', run) + self.assertIn('python -m pytest -q -n auto --dist load', run) + self.assertIn(_tiers.PROJECT_PINS_SELECTOR, run) + self.assertLess(step(JOB, STEP)['timeout-minutes'], job(*JOB)['timeout-minutes']) + + +class TestEveryPinStillRunsWhereTheMatrixDoesNot(unittest.TestCase): + """The release gate's full suite and ``unittest-ordering``'s ``project`` cell keep every pin.""" + + def test_the_gate_runs_the_whole_suite(self) -> None: + self.assertEqual(step(('unit-tests.yml', 'gate'), 'Run full test suite')['run'], + 'python -m pytest -q -n auto --dist load') + + def test_the_ordering_cell_runs_every_pin(self) -> None: + self.assertIn('project', job('unit-tests.yml', 'unittest-ordering')['strategy']['matrix']['leg']) + modules = set(run_unittest_leg.leg_modules('project')) + pins = {pin[:-len('.py')].replace('/', '.') for pin in _tiers.project_pin_paths()} + self.assertTrue(pins) + self.assertEqual(pins - modules, set()) + + +class TestTheDocsSayIt(unittest.TestCase): + + def test_the_workflows_page_has_the_rule_and_the_job(self) -> None: + text = DOC.read_text(encoding='utf-8') + section = text.split('\nProject Pins\n------------\n', 1)[1].split('\n----', 1)[0] + for phrase in ('``is_project_pin()``', '``project_pin_paths()``', '``project-tests``', '#1954', + '``--ignore``', '``gate``', '``unittest-ordering``'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, section) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_superseded_runs_1938.py b/tests/project/test_superseded_runs_1938.py index 77c65a12c7..5a2fa57f3a 100644 --- a/tests/project/test_superseded_runs_1938.py +++ b/tests/project/test_superseded_runs_1938.py @@ -171,7 +171,8 @@ def test_the_gate_checks_on_latest_with_requests(self) -> None: steps = {entry.get('id'): entry for entry in self.jobs[GATE]['steps']} self.assertEqual(steps['versions']['run'], 'python3 util/python_versions.py --github-output') setup = steps['python'] - self.assertEqual(setup['uses'], job(NAME, 'experimental')['steps'][1]['uses']) + legs = {entry.get('id'): entry for entry in job(NAME, 'experimental')['steps']} + self.assertEqual(setup['uses'], legs['python']['uses']) self.assertEqual(setup['with'], {'python-version': '${{ steps.versions.outputs.latest }}'}) self.assertEqual(steps['requests']['run'], 'python -m pip install requests') diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 9153a1ea29..0ef9eec8f2 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -99,8 +99,10 @@ def stringify(value: 'object') -> 'str': def step_env(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'dict[str, str]': """The step's ``env:`` on ``leg``, ``needs`` read from the tier file.""" # #1938: experimental's tests step reads the token and the watcher's interpreter, unwatched here. + # #1954: and the project pins its own step lists, as that step prints them. context = {'matrix': leg, 'needs': {'python-versions': {'outputs': live_outputs()}}, 'github': {'token': ''}, - 'steps': {'watcher': {'outcome': 'skipped'}, 'watcher-python': {'outputs': {'python-path': ''}}}} + 'steps': {'watcher': {'outcome': 'skipped'}, 'watcher-python': {'outputs': {'python-path': ''}}, + 'pins': {'outputs': {'paths': ' '.join(_tiers.project_pin_paths())}}}} return {key: re.sub(r'\$\{\{.*?\}\}', lambda match: stringify(evaluate_expression(match.group(0), context)), str(value)) for key, value in (step(where).get('env') or {}).items()} @@ -157,13 +159,16 @@ def test_only_latest_runs_the_sweeps(self) -> None: with self.subTest(python=version): self.assertEqual(SWEEP_FLAG in pytest_args(argv), version != self.latest) - def test_latest_keeps_the_whole_unit_tier_under_coverage(self) -> None: - """Coverage is measured over the selection it was before #1732.""" + def test_latest_keeps_the_unit_tier_less_the_pins_under_coverage(self) -> None: + """Coverage is measured over the selection it was before #1732, less the project pins, + which import no pcapkit and run in ``project-tests`` (#1954).""" leg, argv = self.runs[self.latest] self.assertIs(leg.get('coverage'), True) self.assertEqual(argv[:4], ['-m', 'coverage', 'run', '--rcfile=.github/coverage.toml']) flags = [arg for arg in pytest_args(argv) if arg.startswith('--ignore')] - self.assertEqual(flags, list(TIER_FLAGS)) + pins = [f'--ignore={pin}' for pin in _tiers.project_pin_paths()] + self.assertTrue(pins) + self.assertEqual(flags, list(TIER_FLAGS) + pins) def test_every_other_leg_runs_the_rest(self) -> None: """The same selection as ``latest``, the sweeps' glob last, and no coverage.""" diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index 656f744bf2..7653e2bed4 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -7,6 +7,7 @@ classified the diff as docs-only. Its shell step is executed below against every combination that matters, rather than read, because the one way it goes wrong -- a skip accepted on a code change -- is a green gate over untested code. + It needs ``project-tests`` on every diff, as the project pins' one run (#1954). * The ``changes`` classifier must call anything outside ``docs/`` and ``*.md`` code, and a Markdown file under a code directory code too. * ``unittest-ordering`` stays off pull requests and out of the required set, @@ -106,15 +107,21 @@ def gate(self, code: 'str', legs: 'str', project: 'str', changes: 'str' = 'succe capture_output=True, text=True).returncode def test_needs_every_leg_and_the_docs_only_helpers(self) -> None: - """The gate reads every leg plus ``changes`` and ``project-tests``.""" + """The gate reads every leg plus ``changes`` and ``project-tests``, the pins' one run (#1954).""" self.assertEqual(required_needs(), set(LEGS) | {'changes', 'project-tests'}) def test_code_change_requires_every_leg_to_succeed(self) -> None: """On a code change nothing but ``success`` passes.""" - self.assertEqual(self.gate('true', 'success', 'skipped'), 0) + self.assertEqual(self.gate('true', 'success', 'success'), 0) for result in ('skipped', 'failure', 'cancelled'): with self.subTest(result=result): - self.assertNotEqual(self.gate('true', result, 'skipped'), 0) + self.assertNotEqual(self.gate('true', result, 'success'), 0) + + def test_code_change_still_needs_project_tests(self) -> None: + """#1954: ``project-tests`` is the one run of the project pins, so a code change needs it too.""" + for result in ('skipped', 'failure', 'cancelled'): + with self.subTest(result=result): + self.assertNotEqual(self.gate('true', 'success', result), 0) def test_docs_only_accepts_skipped_legs_when_project_tests_pass(self) -> None: """A docs-only diff may skip the legs when ``project-tests`` passed.""" @@ -135,8 +142,8 @@ def test_every_leg_is_checked_on_its_own(self) -> None: """One failing leg among passing ones fails the gate, on either path.""" for env_name in ('TEST', 'INTEGRATION', 'ENGINE_TESTS', 'PYPCAP_PARITY'): with self.subTest(leg=env_name): - for code, legs, project, odd in (('true', 'success', 'skipped', 'failure'), - ('true', 'success', 'skipped', 'skipped'), + for code, legs, project, odd in (('true', 'success', 'success', 'failure'), + ('true', 'success', 'success', 'skipped'), ('false', 'skipped', 'success', 'failure')): self.assertNotEqual(self.gate(code, legs, project, **{env_name: odd}), 0) @@ -194,9 +201,20 @@ def test_the_four_legs_skip_only_on_a_docs_only_diff(self) -> None: self.assertIn("needs.changes.outputs.code != 'false'", head) self.assertRegex(head, r'(?m)^ needs: \[changes, python-versions\]$') - def test_project_tests_runs_only_on_a_docs_only_diff(self) -> None: - """``project-tests`` never duplicates ``test`` on a code change.""" - self.assertIn("needs.changes.outputs.code == 'false'", header(job('project-tests'))) + def test_project_tests_runs_on_every_diff_and_never_duplicates_test(self) -> None: + """#1954: ``project-tests`` runs on every diff, but not under ``gate-only``, whose ``gate`` + runs the pins in its full suite. Its pins step runs on a code change, and its steps that + stand in for ``test`` only on a docs-only diff, where they take in the pins.""" + head = header(job('project-tests')) + self.assertIn('if: ${{ inputs.gate-only != true }}', head) + self.assertNotIn('needs.changes.outputs.code', head) + for name, condition in (('Run the project pins', "needs.changes.outputs.code != 'false'"), + ('Run tests/project and the root-level modules under plain unittest', + "needs.changes.outputs.code == 'false'"), + ('Run the other tests that read docs/ or Markdown', + "needs.changes.outputs.code == 'false'")): + with self.subTest(step=name): + self.assertIn(f'if: ${{{{ {condition} }}}}', step(job('project-tests'), name)) def test_project_tests_failures_are_not_softened(self) -> None: """No ``continue-on-error`` anywhere in ``project-tests``: a red step must fail it.""" diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index df22e84159..f9ea743e6c 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -35,9 +35,12 @@ from __future__ import annotations import ast +import os import pathlib import re +import shutil import subprocess +import sys import tempfile import textwrap import unittest @@ -267,6 +270,185 @@ def test_integration_job_selects_positively_by_asking_tiers_for_it(self) -> None ) self.assertIn('pytest', run_block) + @unittest.skipIf(shutil.which('bash') is None, 'the steps are bash scripts') + def test_the_legs_leave_out_exactly_the_pins_project_tests_runs(self) -> None: + """#1954: run as bash runs them, ``test`` ignores each project pin and ``project-tests`` runs each. + + Both steps ask :data:`~tests._tiers.PROJECT_PINS_SELECTOR` at run time, so + they cannot drift from :func:`~tests._tiers.is_project_pin`; what can is + how each uses the answer, which is what running them checks. Off + ``latest`` too, where the sweeps' glob is added after the pins. + experimental.yml asks in a step of its own, under the runner's + ``python3``; its legs run in + :file:`tests/project/test_sweeps_on_latest_only_1732.py`. + + """ + if not WORKFLOW.is_file(): + self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') + text = WORKFLOW.read_text(encoding='utf-8') + pins = list(_tiers.project_pin_paths()) + self.assertTrue(pins) + for sweeps in ('true', 'false'): + with self.subTest(sweeps=sweeps): + self.assertEqual(leg_pin_ignores(text, sweeps), pins) + self.assertEqual(project_pins_run(text), pins) + for workflow, job, step in ((WORKFLOW, 'test', 'Run unit tests'), + (WORKFLOW, 'project-tests', 'Run the project pins'), + (EXPERIMENTAL_WORKFLOW, 'experimental', 'List the project pins')): + with self.subTest(job=job): + script = run_script(job_section(workflow.read_text(encoding='utf-8'), job), step) + self.assertIn(_tiers.PROJECT_PINS_SELECTOR.replace('python -c', 'python3 -c') + if job == 'experimental' else _tiers.PROJECT_PINS_SELECTOR, script) + # The experimental legs ignore what that step printed, as `test` does. + tests = step_run_block(job_section(EXPERIMENTAL_WORKFLOW.read_text(encoding='utf-8'), 'experimental'), + 'Run unit tests') + self.assertIn('PINS: ${{ steps.pins.outputs.paths }}', tests) + self.assertIn('for pin in $PINS; do\n set -- "$@" --ignore="$pin"', tests) + + @unittest.skipIf(shutil.which('bash') is None, 'the steps are bash scripts') + def test_a_leg_that_stops_ignoring_the_pins_is_caught(self) -> None: + """The check above, against a ``test`` step that computes the pins and then drops them.""" + if not WORKFLOW.is_file(): + self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') + text = WORKFLOW.read_text(encoding='utf-8') + loop = 'set -- "$@" --ignore="$pin"' + self.assertEqual(text.count(loop), 1, 'the loop this test removes has been reworded') + self.assertEqual(leg_pin_ignores(text.replace(loop, ':'), 'true'), []) + + +#: The experimental legs' workflow (#1732), whose unit-tier step leaves the pins out too. +EXPERIMENTAL_WORKFLOW = _tiers.ROOT / '.github' / 'workflows' / 'experimental.yml' +#: A ``python`` that writes each argument on a line of ``$ARGS_FILE``, and hands a +#: ``-c`` to the real one, which is how the steps ask tests/_tiers.py for the pins. +PYTHON_STUB = ('#!/bin/sh\nif [ "$1" = -c ]; then exec "$REAL_PYTHON" "$@"; fi\n' + 'printf \'%s\\n\' "$@" > "$ARGS_FILE"\n') + + +def run_script(section: 'str', step_name: 'str') -> 'str': + """The dedented ``run: |`` script of the step named ``step_name`` in a job section.""" + match = re.search(r'(?ms)^ run: \|\n(.*)', step_run_block(section, step_name)) + if match is None: + raise AssertionError(f'the step {step_name!r} has no run: | block') + lines = [] + for line in match.group(1).splitlines(): + if line.strip() and not line.startswith(' ' * 10): + break + lines.append(line) + return textwrap.dedent('\n'.join(lines)) + + +def stubbed_argv(script: 'str', env: 'dict[str, str]') -> 'list[str]': + """The arguments ``script`` hands ``python``, run from the root under ``bash -e``.""" + with tempfile.TemporaryDirectory(prefix='pcapkit-pins-') as temp: + stub = pathlib.Path(temp, 'python') + stub.write_text(PYTHON_STUB, encoding='utf-8') + stub.chmod(0o755) + record = pathlib.Path(temp, 'args') + full = dict(os.environ, **env, ARGS_FILE=str(record), RUNNER_TEMP=temp, REAL_PYTHON=sys.executable, + PATH=temp + os.pathsep + os.environ.get('PATH', '')) + full.pop('PYTHONSAFEPATH', None) # the runner sets none, and it would hide tests/ from `-c` + result = subprocess.run(['bash', '-e', '-c', script], env=full, cwd=str(_tiers.ROOT), + capture_output=True, text=True, check=False, timeout=120) + if result.returncode: + raise AssertionError(f'the step failed: {result.stdout[-1000:]}{result.stderr[-1000:]}') + return record.read_text(encoding='utf-8').splitlines() + + +def leg_pin_ignores(text: 'str', sweeps: 'str') -> 'list[str]': + """The ``tests/project`` paths a ``test`` leg's ``Run unit tests`` step ``--ignore``\\ s.""" + script = run_script(job_section(text, 'test'), 'Run unit tests') + argv = stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'SWEEPS': sweeps}) + return [arg.split('=', 1)[1] for arg in argv if arg.startswith(f'--ignore=tests/{_tiers.PROJECT_PINS_DIR}/')] + + +def project_pins_run(text: 'str') -> 'list[str]': + """The modules ``project-tests``'s ``Run the project pins`` step hands pytest.""" + argv = stubbed_argv(run_script(job_section(text, 'project-tests'), 'Run the project pins'), {}) + return [arg for arg in argv if arg.endswith('.py')] + + +class ProjectPinRuleTests(unittest.TestCase): + """#1954: :func:`~tests._tiers.varies_by_interpreter`, the rule behind the project pins.""" + + def varies(self, source: 'str') -> 'bool': + with tempfile.TemporaryDirectory(prefix='pcapkit-pin-rule-') as temp: + return _tiers.varies_by_interpreter(write_module(pathlib.Path(temp), 'test_probe.py', source)) + + def test_each_way_of_reaching_the_package_is_seen(self) -> None: + cases = { + 'import pcapkit': 'import pcapkit\n', + 'nested from-import': 'def test():\n from pcapkit.protocols import Raw\n', + 'import_module, literal': 'import importlib\nimportlib.import_module("pcapkit.foundation")\n', + 'import_module, computed': 'import importlib\ndef f(name):\n importlib.import_module(name)\n', + 'mock.patch target': 'from unittest import mock\n@mock.patch("pcapkit.utilities.logging.logger")\n' + 'def test(_):\n pass\n', + '-m in a child': 'import subprocess, sys\nsubprocess.run([sys.executable, "-m", "pcapkit"])\n', + '-c in a child': 'import subprocess, sys\nSCRIPT = "import pcapkit"\n' + 'subprocess.run([sys.executable, "-c", SCRIPT])\n', + 'version branch': 'import sys, unittest\n@unittest.skipIf(sys.version_info < (3, 12), "")\n' + 'def test():\n pass\n', + 'implementation branch': 'import sys\nGRAAL = sys.implementation.name == "graalpy"\n', + 'a script that imports it, by path': 'import importlib.util\nfrom tests._tiers import ROOT\n' + 'importlib.util.spec_from_file_location(' + '"b", ROOT / "util" / "bump_version.py")\n', + 'a computed script name': 'import importlib.util\ndef load(name):\n' + ' importlib.util.spec_from_file_location(name, f"util/{name}.py")\n', + 'a module that imports it on import': 'from tests.project.test_pyshark_encap_map import ROOT\n', + 'unparsable': 'def broken(:\n', + } + for label, source in cases.items(): + with self.subTest(case=label): + self.assertIs(self.varies(source), True) + + def test_text_and_scripts_that_import_nothing_are_independent(self) -> None: + cases = { + 'reads text': 'from tests._tiers import ROOT\nTEXT = (ROOT / "pcapkit" / "__init__.py").read_text()\n', + 'names it in prose': 'MESSAGE = "run python -c \'import pcapkit\' to check"\n' + 'PATH = "util/bump_version.py"\n', + 'a dotted name, not imported': 'NAME = "pcapkit.foundation.extraction"\n', + 'a script that imports nothing, by path': 'import importlib.util\nfrom tests._tiers import ROOT\n' + 'importlib.util.spec_from_file_location(' + '"c", ROOT / "util" / "ci_timing.py")\n', + 'a helper whose other names would': 'from tests._tiers import ROOT, is_unit_tier\n', + 'a version only assigned': 'import types\nFAKE = types.SimpleNamespace()\nFAKE.version_info = (3, 5)\n', + } + for label, source in cases.items(): + with self.subTest(case=label): + self.assertIs(self.varies(source), False) + + def test_a_pin_is_an_independent_unit_tier_module_under_tests_project(self) -> None: + cases = { + 'tests/project/test_isort_clean.py': True, + 'tests/project/test_workflows_at_a_glance_1732.py': True, + 'tests/project/test_public_api.py': False, # imports pcapkit + 'tests/project/test_release_gates.py': False, # branches on sys.version_info + 'tests/project/test_docs_ambiguous_xrefs_1775_unit.py': False, # loads docs/source/conf.py + 'tests/test_tier_guard.py': False, # not under tests/project + 'tests/project/__init__.py': False, + 'util/ci_timing.py': False, + } + for module, expected in cases.items(): + with self.subTest(module=module): + self.assertIs(_tiers.is_project_pin(module), expected) + self.assertIs(_tiers.is_project_pin(_tiers.ROOT / module), expected) + + def test_the_pins_are_sorted_paths_the_rule_accepts(self) -> None: + pins = _tiers.project_pin_paths() + self.assertTrue(pins) + self.assertEqual(list(pins), sorted(set(pins))) + rest = [path.relative_to(_tiers.ROOT).as_posix() + for path in (_tiers.TESTS_ROOT / _tiers.PROJECT_PINS_DIR).rglob('test_*.py') + if path.relative_to(_tiers.ROOT).as_posix() not in pins] + self.assertTrue(rest) + for module in pins: + with self.subTest(module=module): + self.assertTrue(module.startswith(f'tests/{_tiers.PROJECT_PINS_DIR}/')) + self.assertTrue(_tiers.is_unit_tier(module)) + self.assertFalse(_tiers.varies_by_interpreter(module)) + for module in rest: + with self.subTest(module=module): + self.assertTrue(_tiers.varies_by_interpreter(module)) + class EnclosingScopeTests(unittest.TestCase): """:func:`~tests._tiers._enclosing_scope`, on synthetic sources. @@ -1389,20 +1571,23 @@ def test_job_sections_agrees_with_the_single_job_slice(self) -> None: self.assertEqual(section, job_section(text, name)) def test_each_job_selection_is_one_of_the_three_recognised_shapes(self) -> None: - """A fifth shape has to fail here rather than be guessed at. + """A sixth shape has to fail here rather than be guessed at. - Four are recognised: subtract ``--ignore`` flags from the whole suite + Five are recognised: subtract ``--ignore`` flags from the whole suite (``test``), ask :func:`~tests._tiers.fixture_tier_paths` - (``integration``), pass no selection at all (``gate``), or name + (``integration``), pass no selection at all (``gate``), name literal test-file paths (``engine-tests``, one :class:`Job` per - ``matrix.engine`` cell, #849). A new job selecting some fifth way - would otherwise be classified ``'whole-suite'`` by the fallback and - credited with reaching gates it does not run. + ``matrix.engine`` cell, #849), or run what + :func:`~tests._tiers.project_pin_paths` prints (``project-tests``, + #1954). A new job selecting some sixth way would otherwise be + classified ``'whole-suite'`` by the fallback and credited with + reaching gates it does not run. """ for job in _dependency_gates.pytest_jobs(): with self.subTest(job=job.name): - self.assertIn(job.selection, ('ignore', 'fixture-tier', 'whole-suite', 'explicit')) + self.assertIn(job.selection, ('ignore', 'fixture-tier', 'whole-suite', 'explicit', + 'project-pins')) # engine-tests yields several Job entries sharing that one name -- # one per matrix.engine cell, all ``'explicit'`` -- so this collects @@ -1412,7 +1597,8 @@ def test_each_job_selection_is_one_of_the_three_recognised_shapes(self) -> None: selections.setdefault(job.name, set()).add(job.selection) self.assertEqual(selections, {'test': {'ignore'}, 'integration': {'fixture-tier'}, 'gate': {'whole-suite'}, 'engine-tests': {'explicit'}, - 'pypcap-parity': {'fixture-tier'}}) + 'pypcap-parity': {'fixture-tier'}, + 'project-tests': {'project-pins'}}) def test_the_selection_is_read_off_the_step_that_runs_pytest(self) -> None: """Not off the job, whose comments contradict it. @@ -1446,7 +1632,8 @@ def test_a_job_whose_pytest_step_is_renamed_is_still_classified(self) -> None: selections.setdefault(job.name, set()).add(job.selection) self.assertEqual(selections, {'test': {'ignore'}, 'integration': {'fixture-tier'}, 'gate': {'whole-suite'}, 'engine-tests': {'explicit'}, - 'pypcap-parity': {'fixture-tier'}}) + 'pypcap-parity': {'fixture-tier'}, + 'project-tests': {'project-pins'}}) def test_two_install_lines_in_one_pytest_job_is_refused(self) -> None: """Ambiguity fails loudly instead of the first line winning.""" @@ -1481,10 +1668,13 @@ def test_two_pytest_steps_in_one_job_is_refused(self) -> None: self.assertIn('steps', str(caught.exception)) def test_the_ignore_selection_is_exactly_the_unit_tier(self) -> None: - """Answered by :func:`~tests._tiers.is_unit_tier`, not by a second copy.""" + """Answered by :func:`~tests._tiers.is_unit_tier`, not by a second copy, + less the project pins, which ``project-tests`` runs instead (#1954).""" job = next(job for job in _dependency_gates.pytest_jobs() if job.selection == 'ignore') cases = { 'tests/protocols/internet/test_esp_unit.py': True, + 'tests/project/test_public_api.py': True, + 'tests/project/test_isort_clean.py': False, 'tests/integration/test_cli_subprocess.py': False, 'tests/foundation/engines/test_new_engine_parity_runtime.py': False, } @@ -1493,6 +1683,23 @@ def test_the_ignore_selection_is_exactly_the_unit_tier(self) -> None: gate = _dependency_gates.Gate('HAS_CRYPTO', module, 1, 'Tests', None) self.assertIs(_dependency_gates.job_reaches(job, gate), expected) + def test_the_project_pins_selection_is_exactly_the_pins(self) -> None: + """#1954: ``project-tests`` reaches the pins and nothing else, so a gate moves with its module.""" + job = next(job for job in _dependency_gates.pytest_jobs() if job.selection == 'project-pins') + self.assertEqual(job.name, 'project-tests') + self.assertEqual(job.extras, next(job for job in _dependency_gates.pytest_jobs() + if job.name == 'test').extras) + cases = { + 'tests/project/test_isort_clean.py': True, + 'tests/project/test_public_api.py': False, + 'tests/protocols/internet/test_esp_unit.py': False, + 'tests/integration/test_cli_subprocess.py': False, + } + for module, expected in cases.items(): + with self.subTest(module=module): + gate = _dependency_gates.Gate('HAS_ISORT', module, 1, 'Tests', None) + self.assertIs(_dependency_gates.job_reaches(job, gate), expected) + def test_the_whole_suite_selection_reaches_everything(self) -> None: job = next(job for job in _dependency_gates.pytest_jobs() if job.selection == 'whole-suite') @@ -1745,8 +1952,9 @@ def test_the_isort_gate_is_visible_and_closed_by_the_test_extra(self) -> None: reaching = {job.name for job in _dependency_gates.pytest_jobs() if _dependency_gates.job_reaches(job, gates[0])} self.assertEqual( - reaching, {'test', 'gate'}, - 'the ignore-shape leg and the whole-suite one collect tests/project/; ' + reaching, {'project-tests', 'gate'}, + 'the module is a project pin, so project-tests runs it and the ignore-shape ' + 'leg leaves it out (#1954); the whole-suite one collects tests/project/; ' 'integration and pypcap-parity select by fixture tier and never reach it, ' 'the same reason they are absent from HAS_MYPY above. engine-tests (#849) ' 'no longer reaches it either -- none of its five explicit TEST_PATHS lists ' @@ -2005,7 +2213,7 @@ def test_dropping_isort_from_the_test_extra_is_caught(self) -> None: :meth:`DependencyGateCoverageTests.test_the_isort_gate_is_visible_and_closed_by_the_test_extra` says a gap is *absent*, and an analysis that had quietly stopped resolving ``HAS_ISORT`` at all would satisfy every one of them. So this - takes the requirement back out on a copy and requires the three gaps to + takes the requirement back out on a copy and requires the gaps to reappear -- which is also the state #766 predicted a visible gate would produce before the extra carried isort ("would correctly turn the dependency-gate guard red until isort is on a pytest install line"), and @@ -2028,17 +2236,18 @@ def test_dropping_isort_from_the_test_extra_is_caught(self) -> None: # engine-tests (#849) no longer reaches tests/project/ at all -- see # test_the_isort_gate_is_visible_and_closed_by_the_test_extra above. + # #1954: the module is a project pin, which project-tests runs instead of test. self.assertEqual( sorted(job for (flag, job) in gaps if flag == 'HAS_ISORT'), - ['gate', 'test']) + ['gate', 'project-tests']) - gap = gaps[('HAS_ISORT', 'test')] + gap = gaps[('HAS_ISORT', 'project-tests')] self.assertEqual(gap.missing, ('isort',)) self.assertEqual({gate.module for gate in gap.gates}, {'tests/project/test_isort_clean.py'}) message = _dependency_gates.describe_gap(gap) - self.assertIn("the 'test' job", message) + self.assertIn("the 'project-tests' job", message) self.assertIn('isort', message) self.assertIn('test_isort_clean.py', message)