From d6fc7d5079d5c254c4b39e62c3fd1cdf9b835fee Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 19 Oct 2026 20:54:08 -0400 Subject: [PATCH] ci(unit-tests): assign every test module a tier and run each tier where it belongs (#1958) - tests/_tiers.py: module_tier() gives every test module one test tier, ALL or LATEST, from a module-level __tier__ constant read by AST, or else from the rule: LATEST when its run does not vary by interpreter - Mark each real sweep LATEST and each hand-picked round trip ALL, and record a reasoned marker on each borderline module - unit-tests.yml, experimental.yml: legs off latest ignore every LATEST module, from latest_tier_paths() at run time; integration off latest and pypcap-parity run fixture_tier_paths('ALL'); the *roundtrip* globs are gone. latest keeps ALL + LATEST, the pins stay in project-tests - Tier guard, the #1732 and #1954 pins and _dependency_gates.py check the partition, the marker and the workflows' agreement with the rule - workflows.rst: a Test Tiers section replaces the glob's description Refs #1958 --- .github/workflows/experimental.yml | 36 +-- .github/workflows/unit-tests.yml | 84 ++--- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 118 +++++-- tests/_dependency_gates.py | 29 +- tests/_tiers.py | 259 ++++++++++++--- tests/const/test_const_linktype_209_unit.py | 3 + ...st_const_method_case_sensitive_896_unit.py | 3 + tests/const/test_register_roundtrip_unit.py | 3 + .../corekit/test_decorators_path_1735_unit.py | 4 + tests/corekit/test_field_roundtrip_unit.py | 3 + .../test_fields_copy_dispatch_runtime.py | 4 + .../corekit/test_from_dict_roundtrip_unit.py | 3 + tests/corekit/test_info_roundtrip_unit.py | 3 + tests/corekit/test_schema_roundtrip_unit.py | 3 + tests/corekit/test_sentinel_exports_unit.py | 4 + tests/dumpkit/test_capture_dump_runtime.py | 3 + tests/dumpkit/test_dumper_roundtrip_unit.py | 3 + .../engines/test_new_engine_parity_runtime.py | 4 + .../test_ip_reassembly_roundtrip_unit.py | 3 + ...pv6_cut_non_final_fragment_1567_runtime.py | 4 + .../test_reassembly_payload_roundtrip_unit.py | 3 + .../test_tcp_reassembly_roundtrip_unit.py | 3 + .../registry/test_registry_symmetry_unit.py | 3 + ...test_traceflow_reread_roundtrip_runtime.py | 3 + .../test_traceflow_reread_roundtrip_unit.py | 3 + tests/integration/test_cli_subprocess.py | 4 + tests/integration/test_engine_parity.py | 4 + .../test_reassembly_engine_parity.py | 4 + .../integration/test_traceflow_end_to_end.py | 3 + tests/project/test_all_exports_unit.py | 4 + tests/project/test_all_layer_grouping_unit.py | 4 + tests/project/test_annotation_names.py | 4 + tests/project/test_build_leftovers_1773.py | 4 + tests/project/test_bump_version.py | 3 + .../test_class_guard_annotations_1857_unit.py | 4 + .../test_conda_cryptography_pin_1827.py | 3 + .../test_conda_native_platforms_1822.py | 4 + .../test_conda_platform_pin_refresh_1851.py | 3 + tests/project/test_conda_vendor_1814.py | 3 + ...onst_get_overrides_documented_1921_unit.py | 3 + tests/project/test_conventions_doc_claims.py | 3 + .../test_docs_aenum_formatter_1858_unit.py | 4 + .../test_docs_ambiguous_xrefs_1775_unit.py | 4 + .../test_docs_class_formatter_1918_unit.py | 4 + .../test_docs_class_signature_1860_unit.py | 4 + .../test_docs_enum_signature_1862_unit.py | 4 + .../project/test_docs_enum_value_hook_unit.py | 4 + .../test_docs_internal_refs_1940_unit.py | 4 + .../test_docs_internal_refs_1947_unit.py | 4 + .../test_docs_pcap_module_link_1948_unit.py | 4 + .../test_docs_preserve_defaults_1859_unit.py | 4 + .../test_docs_table_wrapper_ids_1861_unit.py | 4 + .../test_docs_typehints_pin_1856_unit.py | 4 + .../test_docs_viewcode_builtins_1863_unit.py | 4 + .../test_docs_wrapped_overloads_1917_unit.py | 4 + tests/project/test_documentation_claims.py | 4 + .../test_generators_import_their_own_tree.py | 4 + .../project/test_ktlo_legs_stay_green_1672.py | 2 +- tests/project/test_module_isolation.py | 4 + ...test_no_moved_name_re_exports_1744_unit.py | 4 + .../test_pcapng_download_retry_1658_unit.py | 3 + tests/project/test_project_pins_once_1954.py | 11 + tests/project/test_pyshark_encap_map.py | 3 + tests/project/test_release_gates.py | 3 + .../test_release_tags_the_run_sha_1634.py | 4 + tests/project/test_resource_monitor_1740.py | 7 +- .../test_rfc_constants_one_home_1739_unit.py | 4 + tests/project/test_setup.py | 4 + .../test_sweeps_on_latest_only_1732.py | 214 ++++++++----- .../test_application_edge_roundtrip_unit.py | 4 + .../test_ftp_http1_roundtrip_unit.py | 3 + ...test_ftp_http_unassigned_code_1443_unit.py | 3 + .../application/test_http2_roundtrip_unit.py | 3 + .../test_ospf_auth_dict_roundtrip_unit.py | 3 + .../test_ospf_ngap_roundtrip_unit.py | 3 + .../test_ext_chain_limit_rebuild_1619_unit.py | 3 + .../internet/test_hip_roundtrip_unit.py | 3 + .../test_internet_edge_roundtrip_unit.py | 4 + .../internet/test_ipv4_roundtrip_unit.py | 3 + .../test_ipv4_ts_sec_roundtrip_unit.py | 3 + ...test_ipv4_tso_total_length_1547_runtime.py | 4 + .../internet/test_ipv6_core_roundtrip_unit.py | 3 + .../test_ipv6_exthdr_cut_rebuild_unit.py | 3 + .../test_ipv6_options_roundtrip_unit.py | 3 + .../test_ipv6_route_rpl_roundtrip_unit.py | 3 + .../internet/test_ipx_ah_roundtrip_unit.py | 3 + .../internet/test_mh_roundtrip_unit.py | 3 + .../protocols/link/test_arp_roundtrip_unit.py | 3 + .../link/test_ethernet_mac_roundtrip_unit.py | 3 + .../link/test_l2tpv2_length_roundtrip_unit.py | 3 + .../link/test_l2tpv2_roundtrip_unit.py | 3 + .../link/test_link_edge_roundtrip_unit.py | 3 + .../misc/test_misc_short_read_1470_runtime.py | 3 + .../misc/test_pcapng_roundtrip2_unit.py | 3 + .../misc/test_pcapng_roundtrip_unit.py | 3 + .../schema/test_bitfield_coverage_unit.py | 3 + .../schema/test_schema_roundtrip_unit.py | 3 + .../test_capture_roundtrip_runtime.py | 3 + .../test_declared_length_roundtrip_runtime.py | 3 + .../test_dispatch_reachability_unit.py | 3 + .../protocols/test_dispatch_registry_unit.py | 3 + .../test_edge_roundtrip_deadline_1736_unit.py | 4 + .../protocols/test_option_coverage_runtime.py | 4 + .../test_option_generator_1337_unit.py | 4 + ...est_option_generator_httpv2_stream_unit.py | 4 + ...st_option_generator_jumbogram_1687_unit.py | 4 + .../test_option_generator_tcp_base_unit.py | 4 + tests/protocols/test_option_roundtrip_unit.py | 3 + .../test_short_read_roundtrip_unit.py | 3 + ...est_to_dict_payload_roundtrip_1447_unit.py | 3 + .../transport/test_mptcp_roundtrip_unit.py | 3 + .../transport/test_sctp_roundtrip_unit.py | 3 + .../test_tcp_ns_flag_roundtrip_unit.py | 3 + .../test_tcp_options_roundtrip_unit.py | 3 + .../transport/test_tcp_roundtrip_unit.py | 3 + .../test_transport_edge_roundtrip_unit.py | 4 + tests/test_tier_guard.py | 302 ++++++++++++++---- tests/test_tier_guard_xdist.py | 3 + .../vendor/test_crawler_reachability_unit.py | 4 + tests/vendor/test_ftp_return_code_unit.py | 4 + tests/vendor/test_ipx_packet_unit.py | 4 + tests/vendor/test_ipx_socket_unit.py | 4 + tests/vendor/test_request_prompt_unit.py | 4 + tests/vendor/test_user_agent_unit.py | 4 + tests/vendor/test_vendor_dest_path_unit.py | 4 + tests/vendor/test_vendor_exit_code_unit.py | 4 + ...t_vendor_ipv6_extension_header_925_unit.py | 4 + .../test_vendor_mh_lla_code_1848_unit.py | 4 + tests/vendor/test_vendor_missing_body_unit.py | 4 + tests/vendor/test_vendor_ngap_unit.py | 4 + ...test_vendor_reference_formats_1664_unit.py | 4 + .../test_vendor_reg_apptype_generator_unit.py | 3 + .../test_vendor_snapshot_restore_unit.py | 4 + .../vendor/test_vendor_variables_1726_unit.py | 4 + tests/vendor/test_vendor_wrap_comment_unit.py | 4 + 136 files changed, 1216 insertions(+), 286 deletions(-) diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 3709d16dd2..27ab9a7207 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -203,8 +203,8 @@ jobs: # The tests' 120: GraalPy 25.4's tests step reached 26% of the unit tier's # tests at its old 25-minute cap (job 115915976024; setup-python took 0.7 # minutes, the install 3.3). Those tests hold 20-26% of the tier's time, and - # the tier less the round-trip sweeps, which this job never runs (see `Run - # unit tests`), holds 3.5-4.5 times as much, by per-module times from a local + # the tier less the round-trip sweeps, which this job did not run then (see + # `Run unit tests`), held 3.5-4.5 times as much, by per-module times from a local # 3.14 run and the 3.14, 3.15 and 3.14t legs of run 38581834697: 88-113 # minutes on GraalPy (estimate). 3.14t, which shares the caps, took 5.8 # minutes over the whole unit tier in that run. @@ -242,19 +242,20 @@ jobs: id: checkout continue-on-error: true - # #1954: the project pins run once per push, in unit-tests.yml's - # `project-tests`, so these legs leave them out, as `test`'s do. The - # runner's own `python3` asks tests/_tiers.py, which needs only the - # standard library, before setup-python puts the leg's interpreter on - # PATH, so GraalPy does not parse the tree. Where it fails, the tests - # run the pins too, and `Experimental summary` names this step. - - name: List the project pins - id: pins + # #1958: these legs run the `ALL` test tier alone, as `test`'s legs but + # `latest` do, so they leave out every `LATEST` module -- the project + # pins (#1954) and the round-trip sweeps (#1732) among them. The runner's + # own `python3` asks tests/_tiers.py, which needs only the standard + # library, before setup-python puts the leg's interpreter on PATH, so + # GraalPy does not parse the tree. Where it fails, the tests run the + # `LATEST` tier too, and `Experimental summary` names this step. + - name: List the LATEST tier + id: tier if: ${{ steps.checkout.outcome == 'success' }} continue-on-error: true run: | - pins=$(python3 -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))") - echo "paths=$pins" >> "$GITHUB_OUTPUT" + latest=$(python3 -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))") + echo "paths=$latest" >> "$GITHUB_OUTPUT" - uses: actions/setup-python@v7 id: python @@ -380,7 +381,7 @@ jobs: # unit-tests.yml `test`'s selection on every leg but `latest`: no # experimental version is `latest`, which util/python_versions.py holds to - # a supported CPython, so the round-trip sweeps (#1732) are always left out + # a supported CPython, so the `LATEST` test tier (#1958) is always left out # here. #1740: `tail -F` copies the resource monitor's samples into this # step's log as they are taken, and stops with the step. # @@ -461,7 +462,7 @@ jobs: GH_TOKEN: ${{ github.token }} WATCHER: ${{ steps.watcher.outcome }} WATCHER_PYTHON: ${{ steps.watcher-python.outputs.python-path }} - PINS: ${{ steps.pins.outputs.paths }} + LATEST_TIER: ${{ steps.tier.outputs.paths }} run: | tail -n +1 -F "$RUNNER_TEMP/resource-monitor.log" 2>/dev/null & monitor=$! @@ -471,11 +472,10 @@ jobs: --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ --ignore-glob='*_regression.py' - # #1954: the project pins, from `List the project pins`. - for pin in $PINS; do - set -- "$@" --ignore="$pin" + # #1958: the `LATEST` tier, from `List the LATEST tier`. + for module in $LATEST_TIER; do + set -- "$@" --ignore="$module" done - set -- "$@" --ignore-glob='*roundtrip*' set -- prlimit --fsize=2147483648 timeout --signal=INT --kill-after=3m 110m python -m pytest "$@" scoped() { unit=$1 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a88ea1d215..fdca8b697f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -214,27 +214,34 @@ jobs: # distributed. The durations table and JUnit file are #1538's per-test # timings, uploaded by the last step of this job. # - # #1732, the owner's ruling: the round-trip sweeps -- every module whose - # path holds `roundtrip`, tests/_tiers.py's ROUNDTRIP_SWEEP_GLOB -- run - # on `latest` alone, which is also the coverage leg, so coverage keeps - # its selection. Every other leg ignores them, as do experimental.yml's. - # pytest matches the glob, made absolute, against each path it - # collects, with `*` spanning `/` (_pytest/main.py:437-461, pytest - # 9.1.1). The modules that import the sweeps' harnesses for their - # packet builders still import them. + # #1958, the test tiers: tests/_tiers.py's module_tier() puts every test + # module in `ALL` or `LATEST`, by the `__tier__` constant a module states + # or by the rule where it states none, and `LATEST` runs on `latest` + # alone, which is also the coverage leg, so coverage keeps its + # selection. Every other leg -- `test-ktlo`'s too, as it runs these + # steps -- passes each module latest_tier_paths() prints as an + # `--ignore`, as experimental.yml's legs do. That takes in the + # round-trip sweeps (#1732), each marked `LATEST`; the hand-picked + # round trips are `ALL`. # # #1954, the owner's option 3: the project pins -- the tests/project # modules whose run does not vary by interpreter, tests/_tiers.py's - # project_pin_paths() -- run once per push, in `project-tests`, so every - # leg passes each as an `--ignore`, as experimental.yml's do. Asked of - # tests/_tiers.py here rather than listed, as `integration` asks for its - # selection. The pins import no pcapkit, so coverage loses nothing. + # project_pin_paths(), each of them `LATEST` -- run once per push, in + # `project-tests`, so `latest` passes each as an `--ignore` too. The pins + # import no pcapkit, so coverage loses nothing. Both lists are asked of + # tests/_tiers.py at run time rather than listed, as `integration` asks + # for its selection. env: COVERAGE: ${{ matrix.coverage }} - SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }} + LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }} run: | - if ! pins=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then - echo "::${ANNOTATION_LEVEL} title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above." + if [ "$LATEST" = "true" ]; then + if ! skipped=$(python -c "from tests._tiers import project_pin_paths; print(' '.join(project_pin_paths()))"); then + echo "::${ANNOTATION_LEVEL} title=Could not compute the project pins::tests._tiers.project_pin_paths() failed -- see the traceback above." + exit 1 + fi + elif ! skipped=$(python -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))"); then + echo "::${ANNOTATION_LEVEL} title=Could not compute the LATEST tier::tests._tiers.latest_tier_paths() failed -- see the traceback above." exit 1 fi set -- -q -n auto --dist load \ @@ -242,12 +249,9 @@ jobs: --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ --ignore-glob='*_regression.py' - for pin in $pins; do - set -- "$@" --ignore="$pin" + for module in $skipped; do + set -- "$@" --ignore="$module" done - if [ "$SWEEPS" != "true" ]; then - set -- "$@" --ignore-glob='*roundtrip*' - fi if [ "$COVERAGE" = "true" ]; then # Logs every measured process and its parent, for the check below. COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="$PWD/coverage-processes.log" \ @@ -579,24 +583,30 @@ jobs: # say so. `--dist load` is pinned explicitly; see the `test` job above # for why. # - # #1732, the owner's ruling: the round-trip sweeps here, the modules - # tests/_tiers.py's ROUNDTRIP_RUNTIME_GLOB names, run on `latest` alone, as - # `test`'s do. Every other leg adds the glob as an `--ignore-glob`, which - # covers a module a directory argument reaches, and drops each path or - # node ID the glob matches from the selection: pytest never applies an - # ignore to a path it was handed (`Dir.collect` asks `isinitpath` first, - # _pytest/main.py:564-584, pytest 9.1.1). + # #1958, the test tiers: the fixture tier's `LATEST` modules, the sweeps + # over every capture among them, run on `latest` alone, as `test`'s do. + # Every other leg asks for the `ALL` tier's part of the selection, + # FIXTURE_ALL_TIER_SELECTOR, which drops each `LATEST` path and node ID + # and replaces a directory holding a `LATEST` module by its `ALL` ones, + # since pytest applies no ignore to a path it was handed (`Dir.collect` + # asks `isinitpath` first, _pytest/main.py:564-584, pytest 9.1.1). - name: Run full test suite id: suite if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash env: - SWEEPS: ${{ matrix.python-version == needs.python-versions.outputs.latest }} + LATEST: ${{ matrix.python-version == needs.python-versions.outputs.latest }} run: | - if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then + cannot_compute() { echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." exit 1 + } + if [ "$LATEST" = "true" ]; then + selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))") || cannot_compute + else + selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths('ALL')))") || cannot_compute + echo "Not latest, so the ALL tier alone (#1958)." fi if [ -z "$selection" ]; then echo "::${ANNOTATION_LEVEL} title=Fixture-tier selection is empty::tests._tiers.fixture_tier_paths() returned nothing, which cannot be right -- tests/integration alone should always be part of it. Refusing to run pytest with no arguments." @@ -605,17 +615,6 @@ jobs: echo "Fixture-dependent selection: $selection" # shellcheck disable=SC2086 set -- $selection - if [ "$SWEEPS" != "true" ]; then - kept=() - for arg in "$@"; do - case "${arg%%::*}" in - *roundtrip*_runtime.py) ;; - *) kept+=("$arg") ;; - esac - done - set -- "${kept[@]}" --ignore-glob='*roundtrip*_runtime.py' - echo "Not latest, so without the round-trip sweeps (#1732): $*" - fi python -m pytest -q -n auto --dist load \ --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" "$@" @@ -1351,13 +1350,16 @@ jobs: # See the `integration` job above for why this is asked of # tests/_tiers.py directly rather than spelled out as literal flags. + # #1958: neither leg here is `latest`, so each runs the `ALL` tier's part + # of the selection, as `integration`'s other legs do; the `LATEST` + # sweeps over every capture run in `integration` on `latest`. - name: Run full test suite id: suite if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash run: | - if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths()))"); then + if ! selection=$(python -c "from tests._tiers import fixture_tier_paths; print(' '.join(fixture_tier_paths('ALL')))"); then echo "::${ANNOTATION_LEVEL} title=Could not compute the fixture-tier selection::tests._tiers.fixture_tier_paths() failed -- see the traceback above. Refusing to fall back to a bare 'pytest -q', which would silently re-run the entire suite instead of failing loudly." exit 1 fi diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 6aebe8970d..a7ba176d86 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1840-1910``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1842-1912``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1644-1670``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1646-1672``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 0a626b6385..08da005dea 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -329,7 +329,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1632-1638``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1634-1640``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -639,7 +639,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:2001`` + - ``unit-tests.yml:2003`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -661,22 +661,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:2001``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:2003``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1913`` mentions the string without defining it. +``unit-tests.yml:1915`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1912-1922``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1914-1924``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:2000-2056``). Rulesets match check names literally, with no +(``unit-tests.yml:2002-2058``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -685,7 +685,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2003``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2005``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -709,7 +709,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:420-437``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:424-441``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -829,21 +829,15 @@ and a ``Compat`` leg. The tests' 120 minutes are sized for GraalPy, the slowest of the legs that share them: its leg reached 26% of the unit tier in 25 minutes (job 115915976024), where -3.14t's ran the whole tier in 5.8 (run 38581834697). By the owner's ruling the -round-trip sweeps run on ``latest`` alone: every leg but ``latest`` adds -``--ignore-glob='*roundtrip*'``, which leaves out each unit-tier module whose path -holds ``roundtrip`` (``ROUNDTRIP_SWEEP_GLOB`` in :file:`tests/_tiers.py`), and -``latest``, the coverage leg, runs them as before. The subset covers -``integration`` too: off ``latest`` its legs leave out the -``*roundtrip*_runtime.py`` modules (``ROUNDTRIP_RUNTIME_GLOB``), dropping them from -``fixture_tier_paths()``'s selection and adding the glob as an ``--ignore-glob``, -since pytest applies no ignore to a path it was handed. On 2026-10-14 they took -8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% on the -3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy 88-113 -minutes (estimate). The experimental legs leave out the project pins too, as -``test``'s do (see `Project Pins`_ below); ``List the project pins`` asks for them -on the runner's own ``python3``, before setup-python, so GraalPy does not parse the -tree. +3.14t's ran the whole tier in 5.8 (run 38581834697). The experimental legs run the +``ALL`` test tier alone, as every leg but ``latest`` does (see `Test Tiers`_ below): +``List the LATEST tier`` asks ``latest_tier_paths()`` on the runner's own ``python3``, +before setup-python, so GraalPy does not parse the tree, and ``Run unit tests`` +passes each module it printed as an ``--ignore``. The +round-trip sweeps of `#1732 `__ +took 8% of the unit tier's time in a serial 3.14 run (86 of 1,031 s), and 10-11% on +the 3.14, 3.15 and 3.14t legs of run 38581834697, so the rest should take GraalPy +88-113 minutes (estimate). GraalPy's leg still loses its runner 18-30 minutes into its tests, with ``The runner has received a shutdown signal`` (`#1740 `__), @@ -910,6 +904,76 @@ workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inf from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a warning: that is expected, blocks nothing, and leaves the caps as they are. +Test Tiers +---------- + +Every test module runs in one of two *test tiers* (`#1958 +`__): ``ALL``, which runs on every +leg, and ``LATEST``, which runs on ``latest`` alone. They are orthogonal to the unit +and fixture-dependent tiers, which decide what a module may read and so which job +collects it; a test tier decides on which of that job's legs it runs. + +``module_tier()`` in :file:`tests/_tiers.py` decides, in one place, for every +``tests/**/test_*.py``: + +* **The marker.** A module may state its tier outright with a module-level + ``__tier__ = 'ALL'`` or ``__tier__ = 'LATEST'``, read by AST, under a ``#:`` + comment that gives the reason. It wins over the rule. It is a module constant + rather than a pytest marker, so ``unittest-ordering``'s plain ``unittest`` runs + read the same module. A value that is not one of the two string literals, or a + marker assigned twice or anywhere but at the module's top level, fails the tier + guard. +* **The rule.** Otherwise a module is ``LATEST`` when its run does not vary by + interpreter, by the same AST test as the project pins below: it reads the + repository's text, a workflow or the docs, or drives a ``util/`` script, and + neither imports ``pcapkit`` nor compares the Python version. Everything that + exercises ``pcapkit`` is ``ALL``. + +The marker says what the AST cannot see. Each real sweep -- a round trip over every +case of a table, every registry entry or every capture -- is marked ``LATEST``, as +the owner ruled on #1732 and #1958; the hand-picked round trips, whatever their +names, run on ``ALL``. A module that imports ``pcapkit`` only to read its text or +structure, or that drives a docs hook or a generator other legs already run, is +marked ``LATEST``; one that the rule would send to ``latest`` but that depends on +the interpreter, such as the tier guard itself, which :file:`tests/conftest.py` runs at +collection on every leg, is marked ``ALL``. + +Which legs run which tier: + +.. list-table:: + :header-rows: 1 + + * - Leg + - Tiers + * - ``test`` on ``latest``, the coverage leg + - ``ALL`` and ``LATEST``, less the project pins + * - ``project-tests`` + - the project pins + * - ``test`` off ``latest``, ``test-ktlo``, ``experimental`` + - ``ALL`` + * - ``integration`` on ``latest`` + - ``ALL`` and ``LATEST`` + * - ``integration`` off ``latest``, ``integration-ktlo``, ``pypcap-parity``, + ``pypcap-parity-ktlo`` + - ``ALL`` + * - ``gate``, ``unittest-ordering`` + - ``ALL`` and ``LATEST`` + +Each leg asks :file:`tests/_tiers.py` at run time, so no workflow lists a tier: a +``test`` leg off ``latest`` passes each module ``latest_tier_paths()`` prints as an +``--ignore``, and an ``integration`` or ``pypcap-parity`` leg off ``latest`` runs +``fixture_tier_paths('ALL')``, which drops each ``LATEST`` path and node ID and +replaces a directory holding a ``LATEST`` module by its ``ALL`` ones, since pytest +applies no ignore to a path it was handed. :file:`tests/test_tier_guard.py` checks +that every module resolves to exactly one tier and runs the steps to check what each +does with the answer, and :file:`tests/project/test_sweeps_on_latest_only_1732.py` +does the same for every leg the tier file gives. + +The project pins stay in a job of their own rather than joining the ``latest`` leg: +``project-tests`` runs in parallel with the coverage leg, the slowest of ``test``'s, +rather than lengthening it under coverage, and it is also the job that runs the docs +readers on a docs-only diff. + Project Pins ------------ @@ -922,13 +986,17 @@ code it runs: a module is a pin unless it, a module it takes names from, or a fi it runs by path imports ``pcapkit``, or compares ``sys.version_info`` or ``sys.implementation``. A computed import, or a ``.py`` name computed for a loader or a child process, counts as importing it, and so does a file that does -not parse. The modules that import ``pcapkit`` stay on every leg. +not parse. Every pin is in the ``LATEST`` test tier (see `Test Tiers`_ above), +and a ``tests/project`` module marked ``__tier__ = 'ALL'`` is no pin. The modules +that import ``pcapkit`` are no pins either; those the tiers send to ``latest`` run +on ``test``'s ``latest`` leg, under coverage. ``project-tests`` runs the pins under pytest, with ``test``'s flags, on ``latest``, on every pull request and every push to ``main``, and ``required-checks`` needs it. Every ``test``, ``test-ktlo`` and ``experimental`` leg passes each pin as an ``--ignore``, ``latest``'s coverage leg included, which measures only ``pcapkit``. -Each asks ``project_pin_paths()`` at run time, as ``integration`` asks for its +``latest`` asks ``project_pin_paths()`` at run time, and every other leg +``latest_tier_paths()``, which holds each pin, as ``integration`` asks for its selection, so no workflow lists the pins, and :file:`tests/test_tier_guard.py` runs the steps to check what each does with the answer. Under ``gate-only``, ``gate``'s full suite runs the pins with everything else. ``unittest-ordering``'s ``project`` diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index 1d86bfc246..daa003565b 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -789,6 +789,11 @@ class Job(NamedTuple): #: string equality -- unlike ``'fixture-tier'``, there is no directory or #: node-ID form here, because that is not the shape ``TEST_PATHS`` takes. paths: 'tuple[str, ...]' = () + #: The test tier a ``'fixture-tier'`` selection is limited to on every leg + #: (#1958): ``'ALL'`` for a job none of whose legs is ``latest``, which asks + #: :func:`~tests._tiers.fixture_tier_paths` for that tier alone, :data:`None` + #: for one whose ``latest`` leg asks for every tier, and for every other shape. + tier: 'Optional[str]' = None class Gap(NamedTuple): @@ -1573,8 +1578,14 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': ) run_block = running[0] + tier = None # type: Optional[str] if 'fixture_tier_paths' in run_block: selection = 'fixture-tier' + # #1958: a step that asks for every tier on some leg reaches what that + # leg runs; one that only ever asks for one tier reaches that tier. + asked = set(re.findall(r"join\(fixture_tier_paths\(([^)]*)\)\)", run_block)) + if '' not in asked and len(asked) == 1: + tier = ast.literal_eval(asked.pop()) elif '--ignore' in run_block: selection = 'ignore' elif 'project_pin_paths' in run_block: @@ -1583,7 +1594,7 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': selection = 'whole-suite' jobs.append(Job(name, tuple(extra.strip() for extra in installs[0].split(',')), - selection)) + selection, tier=tier)) return tuple(jobs) @@ -1600,18 +1611,20 @@ def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': :meth:`~tests.test_tier_guard.WorkflowAgreementTests\ .test_ignore_flags_match_the_fixture_tier_constants` is what keeps that equivalence true. It is the ``latest`` leg's: the - others also skip :data:`~tests._tiers.ROUNDTRIP_SWEEP_GLOB` (#1732), which - changes no answer, since every leg shares the job's one install line. The - same holds for ``'fixture'``, whose legs off ``latest`` drop - :data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB`'s modules. + others also leave out the ``LATEST`` test tier, + :func:`~tests._tiers.latest_tier_paths` (#1958), which changes no answer, + since every leg shares the job's one install line. The same holds for a + ``'fixture-tier'`` job with a ``latest`` leg, whose other legs ask for the + ``ALL`` tier alone. * ``'project-pins'`` is :func:`~tests._tiers.is_project_pin`, the job running exactly what :func:`~tests._tiers.project_pin_paths` prints. * ``'explicit'`` is exact membership in :attr:`Job.paths` -- the literal ``TEST_PATHS`` list a ``matrix.engine`` cell of ``engine-tests`` (#849) runs, at file granularity (that job names whole files, never node IDs). * ``'fixture-tier'`` is whatever - :func:`~tests._tiers.fixture_tier_paths` returns, which is what the job - itself runs. Its node-ID entries are honoured at method granularity: a + :func:`~tests._tiers.fixture_tier_paths` returns for the job's + :attr:`Job.tier`, which is what the job itself runs. Its node-ID entries + are honoured at method granularity: a selection naming one method of a module does not reach a gate on a *different* method of it. @@ -1635,7 +1648,7 @@ def job_reaches(job: 'Job', gate: 'Gate') -> 'bool': if job.selection == 'explicit': return gate.module in job.paths - for entry in _tiers.fixture_tier_paths(): + for entry in _tiers.fixture_tier_paths(job.tier): module, _, scope = entry.partition('::') if not scope: candidate = _tiers.ROOT / module diff --git a/tests/_tiers.py b/tests/_tiers.py index f9bf4e7aa4..5bf864cd3d 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -46,8 +46,11 @@ which of their test methods) skip a generated capture rather than reading it for real, and the complete pytest selection the fixture-dependent tier needs in order to still exercise them. -* :func:`is_roundtrip_sweep` -- is this unit-tier module one of the round-trip - sweeps, which only the ``latest`` leg of the ``test`` job runs (#1732)? +* :func:`module_tier` -- which *test tier* does this module run in (#1958): + ``ALL``, on every leg, or ``LATEST``, on ``latest`` alone? A ``__tier__`` + constant in the module (:func:`declared_tier`) answers first, and + :func:`rule_tier` otherwise. :func:`latest_tier_paths` lists the ``LATEST`` + modules, which every leg but ``latest`` leaves out. * :func:`is_project_pin` and :func:`project_pin_paths` -- which :file:`tests/project` modules give one answer on every interpreter, and so run once per push, in the ``project-tests`` job, rather than on every leg (#1954)? @@ -124,7 +127,6 @@ from __future__ import annotations import ast -import fnmatch import functools import pathlib import re @@ -138,7 +140,8 @@ 'GeneratedFixtureInUnitTierError', 'TierGuardWarning', 'SampleCall', 'ROOT', 'TESTS_ROOT', 'SAMPLE_ROOT', 'REGENERATE_SAMPLES_CMD', 'UNIT_TIER_SELECTION', 'FIXTURE_TIER_SUFFIXES', 'FIXTURE_TIER_DIRS', 'FIXTURE_TIER_SELECTOR', - 'ROUNDTRIP_SWEEP_GLOB', 'ROUNDTRIP_RUNTIME_GLOB', 'is_roundtrip_sweep', + 'TIER_ALL', 'TIER_LATEST', 'TIERS', 'TIER_MARKER', 'LATEST_TIER_SELECTOR', 'FIXTURE_ALL_TIER_SELECTOR', + 'InvalidTierError', 'suite_modules', 'declared_tier', 'rule_tier', 'module_tier', 'latest_tier_paths', 'PROJECT_PINS_DIR', 'PROJECT_PINS_SELECTOR', 'PROJECT_PIN_PACKAGE', 'LOADABLE_DIRS', 'varies_by_interpreter', 'is_project_pin', 'project_pin_paths', 'is_unit_tier', 'committed_captures', 'committed_capture_names', @@ -183,17 +186,34 @@ 'python -c "from tests._tiers import fixture_tier_paths; ' 'print(\' \'.join(fixture_tier_paths()))"' ) -#: The round-trip sweeps (#1732), as the ``--ignore-glob`` that the ``test`` job -#: of :file:`.github/workflows/unit-tests.yml` adds to :data:`UNIT_TIER_SELECTION` -#: on every leg but ``latest``: a module whose path under :data:`ROOT` holds -#: ``roundtrip``. A selection within the unit tier, not a tier of its own: the -#: modules stay unit-tier, and ``latest`` runs them, under coverage. -ROUNDTRIP_SWEEP_GLOB = '*roundtrip*' -#: The round-trip sweeps of the fixture tier (#1732): the ``*_runtime.py`` modules -#: whose name holds ``roundtrip``, which the ``integration`` job runs on ``latest`` -#: alone. Every other leg drops them from :func:`fixture_tier_paths`'s selection and -#: passes this as an ``--ignore-glob``, since pytest ignores no path it was handed. -ROUNDTRIP_RUNTIME_GLOB = '*roundtrip*_runtime.py' +#: The test tiers (#1958), which say on which legs a module runs. Orthogonal to the +#: unit and fixture-dependent tiers above, which say what a module may read: ``ALL`` +#: modules run on every leg of every job that collects them, ``LATEST`` ones on the +#: ``latest`` leg alone. See :func:`module_tier`. +TIER_ALL = 'ALL' +#: See :data:`TIER_ALL`. +TIER_LATEST = 'LATEST' +#: Every test tier, the values a :data:`TIER_MARKER` may take. +TIERS = (TIER_ALL, TIER_LATEST) +#: The module-level constant that states a module's test tier outright, e.g. +#: ``__tier__ = 'LATEST'``, read by AST in :func:`declared_tier`. A constant rather +#: than a pytest marker, so plain :mod:`unittest` runs see the same module. +TIER_MARKER = '__tier__' +#: The ``LATEST`` tier's selector, verbatim from the workflows: every leg of ``test`` +#: but ``latest``, and every ``experimental`` leg, passes each path it prints as an +#: ``--ignore``. Asked at run time, as :data:`FIXTURE_TIER_SELECTOR` is, so no +#: workflow lists the tier. See :func:`latest_tier_paths`. +LATEST_TIER_SELECTOR = ( + 'python -c "from tests._tiers import latest_tier_paths; ' + 'print(\' \'.join(latest_tier_paths()))"' +) +#: The fixture-dependent tier's ``ALL``-tier selection, verbatim from the workflows: +#: what ``integration`` runs on every leg but ``latest``, and ``pypcap-parity`` on each +#: of its legs. See :func:`fixture_tier_paths`. +FIXTURE_ALL_TIER_SELECTOR = ( + 'python -c "from tests._tiers import fixture_tier_paths; ' + 'print(\' \'.join(fixture_tier_paths(\'ALL\')))"' +) #: The directory whose platform- and version-independent modules, the *project #: pins*, run once per push in the ``project-tests`` job of #: :file:`.github/workflows/unit-tests.yml` rather than on every leg (#1954). See @@ -259,6 +279,15 @@ class TierGuardWarning(UserWarning): """ +class InvalidTierError(ValueError): + """A test module's :data:`TIER_MARKER` cannot be read as one of :data:`TIERS`. + + Raised rather than ignored: a marker that silently fell back to the rule would + put the module on legs its author ruled it off, or the other way round. + + """ + + class SampleCall(NamedTuple): """One ``sample_path(...)`` call found in a module's source.""" @@ -308,27 +337,6 @@ def is_unit_tier(path: 'pathlib.Path | str') -> 'bool': return FIXTURE_TIER_DIRS.isdisjoint(relative.parts[:-1]) -def is_roundtrip_sweep(path: 'pathlib.Path | str') -> 'bool': - """Whether ``path`` names a unit-tier round-trip sweep (#1732). - - The executable statement of :data:`ROUNDTRIP_SWEEP_GLOB`, matched the way - pytest matches an ``--ignore-glob`` run from :data:`ROOT`: the glob made - absolute, against the absolute path, with ``*`` spanning ``/``. - - Args: - path: Path to a module, absolute or relative to :data:`ROOT`. - - Returns: - :data:`True` for a unit-tier module the glob matches. - - """ - candidate = pathlib.Path(path) - if not candidate.is_absolute(): - candidate = ROOT / candidate - return is_unit_tier(candidate) and fnmatch.fnmatch(str(candidate.resolve()), - str(ROOT / ROUNDTRIP_SWEEP_GLOB)) - - #: Calls that import the module their first argument names. ``patch`` is #: :func:`unittest.mock.patch`, which imports its target's module, and ``find_spec`` #: imports a dotted name's parents. @@ -826,6 +834,143 @@ def varies_by_interpreter(path: 'pathlib.Path | str') -> 'bool': return False +def suite_modules() -> 'tuple[str, ...]': + """Every test module under :file:`tests/`, relative to :data:`ROOT`, sorted. + + What pytest collects and :file:`util/run_unittest_leg.py` discovers: the + ``test_*.py`` files, ``python_files`` in :file:`pyproject.toml`. + + """ + return tuple(sorted(path.relative_to(ROOT).as_posix() for path in TESTS_ROOT.rglob('test_*.py') + if '__pycache__' not in path.parts)) + + +def _test_module(path: 'pathlib.Path | str') -> 'pathlib.Path': + """``path`` resolved, once it is known to be a test module under :file:`tests/`.""" + candidate = pathlib.Path(path) + if not candidate.is_absolute(): + candidate = ROOT / candidate + candidate = candidate.resolve() + try: + candidate.relative_to(TESTS_ROOT) + except ValueError: + raise ValueError(f'{path} is not under {TESTS_ROOT}, so it has no test tier') from None + if not (candidate.name.startswith('test_') and candidate.suffix == '.py' and candidate.is_file()): + raise ValueError(f'{path} is not a test module (tests/**/test_*.py), so it has no test tier') + return candidate + + +@functools.lru_cache(maxsize=None) +def _declared_tier(candidate: 'pathlib.Path') -> 'Optional[str]': + tree = _parse_module(candidate) + if tree is None: + return None + where = _display_path(candidate) + top = [node for node in tree.body + if (isinstance(node, ast.Assign) and any(isinstance(target, ast.Name) and target.id == TIER_MARKER + for target in node.targets)) + or (isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) + and node.target.id == TIER_MARKER)] + stores = [node for node in ast.walk(tree) + if isinstance(node, ast.Name) and node.id == TIER_MARKER and isinstance(node.ctx, ast.Store)] + if len(stores) != len(top) or len(top) > 1: + raise InvalidTierError( + f'{where} assigns {TIER_MARKER} {len(stores)} times, {len(top)} of them at module level; ' + f'state the tier once, as a module-level {TIER_MARKER} = {TIER_LATEST!r} or {TIER_ALL!r}' + ) + if not top: + return None + value = top[0].value + if not (isinstance(value, ast.Constant) and isinstance(value.value, str) and value.value in TIERS): + shown = (ast.get_source_segment(candidate.read_text(encoding='utf-8'), value) or ast.dump(value) + if value is not None else 'nothing') + raise InvalidTierError( + f'{where}:{top[0].lineno}: {TIER_MARKER} is set to {shown}, which is not a test tier; ' + f'it must be one of the string literals {", ".join(map(repr, TIERS))}' + ) + return value.value + + +def declared_tier(path: 'pathlib.Path | str') -> 'Optional[str]': + """The test tier the module at ``path`` states with :data:`TIER_MARKER`, if it does. + + Read by AST, never by importing the module: one module-level statement, + ``__tier__ = 'ALL'`` or ``__tier__ = 'LATEST'``, the value a string literal. + A module that does not parse states nothing here, and :func:`rule_tier` + answers for it. + + Args: + path: Path to a test module, absolute or relative to :data:`ROOT`. + + Raises: + ValueError: ``path`` is not a test module under :file:`tests/`. + InvalidTierError: The marker is not one of :data:`TIERS`, not a string + literal, assigned more than once, or assigned anywhere but at the + module's top level. + + """ + return _declared_tier(_test_module(path)) + + +def rule_tier(path: 'pathlib.Path | str') -> 'str': + """The test tier the rule gives the module at ``path`` (#1958). + + :data:`TIER_LATEST` when its run does not :func:`vary by interpreter + ` -- it reads the repository's text, a workflow or the + docs, or drives a ``util/`` script, and imports no ``pcapkit`` -- since one run + on ``latest`` is then every leg's answer. :data:`TIER_ALL` for everything + else, which is everything that exercises ``pcapkit``. What the AST cannot + see -- that a module is a sweep over every case of a table or every capture, + that one which imports ``pcapkit`` still answers a question about the + repository, or that one which imports nothing still depends on the + interpreter -- is stated with :data:`TIER_MARKER` instead; see + :func:`module_tier`. + + Args: + path: Path to a test module, absolute or relative to :data:`ROOT`. + + Raises: + ValueError: ``path`` is not a test module under :file:`tests/`. + + """ + return TIER_ALL if varies_by_interpreter(_test_module(path)) else TIER_LATEST + + +def module_tier(path: 'pathlib.Path | str') -> 'str': + """The test tier of the module at ``path``: :data:`TIER_ALL` or :data:`TIER_LATEST` (#1958). + + Its :func:`declared_tier` where it states one, its :func:`rule_tier` + otherwise, so every test module has exactly one. ``ALL`` modules run on every + leg; ``LATEST`` ones on ``latest`` alone, which is also the coverage leg, the + ``gate`` job's interpreter, and every ``unittest-ordering`` cell's. Within a + tier, the unit and fixture-dependent tiers still decide which job collects a + module. + + Args: + path: Path to a test module, absolute or relative to :data:`ROOT`. + + Raises: + ValueError: ``path`` is not a test module under :file:`tests/`. + InvalidTierError: Its marker is malformed; see :func:`declared_tier`. + + """ + candidate = _test_module(path) + return _declared_tier(candidate) or rule_tier(candidate) + + +@functools.lru_cache(maxsize=1) +def latest_tier_paths() -> 'tuple[str, ...]': + """Every :data:`TIER_LATEST` test module, relative to :data:`ROOT`, sorted. + + What :data:`LATEST_TIER_SELECTOR` prints: the modules every leg but ``latest`` + passes as an ``--ignore``. Unit-tier and fixture-dependent modules alike; the + unit tier's own flags ignore the latter already, and :func:`fixture_tier_paths` + leaves them out of its ``ALL`` selection. + + """ + return tuple(path for path in suite_modules() if module_tier(path) == TIER_LATEST) + + def is_project_pin(path: 'pathlib.Path | str') -> 'bool': """Whether ``path`` names a project pin, which runs once per push (#1954). @@ -833,8 +978,10 @@ def is_project_pin(path: 'pathlib.Path | str') -> 'bool': :file:`tests/`:data:`PROJECT_PINS_DIR` whose run does not :func:`vary by interpreter `: it reads the repository's text or drives a ``util/`` script, so the ``project-tests`` job runs it once, on - ``latest``. Any other module stays on every leg of ``test`` and - ``experimental``. + ``latest``. Every pin is in the :data:`TIER_LATEST` test tier (#1958): a module + that marks itself :data:`TIER_ALL` is no pin, and one that marks itself + ``LATEST`` while it imports ``pcapkit`` is no pin either, and runs on the + ``latest`` leg of ``test``, under coverage. Args: path: Path to a module, absolute or relative to :data:`ROOT`. @@ -849,7 +996,8 @@ def is_project_pin(path: 'pathlib.Path | str') -> 'bool': except ValueError: return False return (relative.name.startswith('test_') and relative.suffix == '.py' and candidate.is_file() - and is_unit_tier(candidate) and not varies_by_interpreter(candidate)) + and is_unit_tier(candidate) and not varies_by_interpreter(candidate) + and module_tier(candidate) == TIER_LATEST) @functools.lru_cache(maxsize=1) @@ -857,7 +1005,8 @@ def project_pin_paths() -> 'tuple[str, ...]': """Every :func:`project pin `, relative to :data:`ROOT`, sorted. What :data:`PROJECT_PINS_SELECTOR` prints: the ``project-tests`` job's - selection, and the ``--ignore`` list of every ``test`` and ``experimental`` leg. + selection, and the ``--ignore`` list of the ``latest`` leg of ``test``. Each is + in :func:`latest_tier_paths` too, which every other leg leaves out. """ return tuple(sorted(path.relative_to(ROOT).as_posix() @@ -1431,8 +1580,8 @@ def skip_idiom_test_ids() -> 'tuple[str, ...]': return tuple(sorted(set(ids))) -@functools.lru_cache(maxsize=1) -def fixture_tier_paths() -> 'tuple[str, ...]': +@functools.lru_cache(maxsize=None) +def fixture_tier_paths(tier: 'Optional[str]' = None) -> 'tuple[str, ...]': """The complete pytest selection the fixture-dependent tier needs to run. Three things go into it: @@ -1463,6 +1612,15 @@ def fixture_tier_paths() -> 'tuple[str, ...]': ask a single function for its selection instead of re-deriving it -- see :data:`FIXTURE_TIER_SELECTOR`. + With ``tier``, only that test tier's part of it (#1958): ``'ALL'`` is what + every leg but ``latest`` runs, :data:`FIXTURE_ALL_TIER_SELECTOR`. Each entry + whose module is in another tier is dropped, and a directory argument holding + such a module is replaced by its modules of ``tier``, since pytest applies no + ``--ignore`` to a path it was handed. + + Args: + tier: One of :data:`TIERS`, or :data:`None` for every tier. + Returns: Paths and/or node IDs relative to :data:`ROOT`, POSIX-separated, sorted, and deduplicated -- ready to hand to ``pytest`` as positional @@ -1470,7 +1628,12 @@ def fixture_tier_paths() -> 'tuple[str, ...]': the directory arguments do not depend on git, so they are always present even when :func:`skip_idiom_test_ids` cannot answer. + Raises: + ValueError: ``tier`` is not one of :data:`TIERS`. + """ + if tier is not None and tier not in TIERS: + raise ValueError(f'{tier!r} is not a test tier; expected one of {", ".join(map(repr, TIERS))}') paths = {f'tests/{name}' for name in FIXTURE_TIER_DIRS} for path in sorted(TESTS_ROOT.rglob('*.py')): @@ -1482,4 +1645,16 @@ def fixture_tier_paths() -> 'tuple[str, ...]': paths.update(skip_idiom_test_ids()) + if tier is not None: + kept = set() # type: set[str] + for entry in paths: + module = entry.partition('::')[0] + if (ROOT / module).is_dir(): + members = [path for path in suite_modules() if path.startswith(module.rstrip('/') + '/')] + within = [path for path in members if module_tier(path) == tier] + kept.update([entry] if within == members else within) + elif module_tier(module) == tier: + kept.add(entry) + paths = kept + return tuple(sorted(paths)) diff --git a/tests/const/test_const_linktype_209_unit.py b/tests/const/test_const_linktype_209_unit.py index c772ec9215..959613417e 100644 --- a/tests/const/test_const_linktype_209_unit.py +++ b/tests/const/test_const_linktype_209_unit.py @@ -72,6 +72,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises the link-type lookups. +__tier__ = 'ALL' + #: The crawler dependencies :class:`LinkTypeGeneratorLegacyOrderingTests` needs to #: parse its fixture. Spelled exactly as #: :data:`tests.vendor.test_request_prompt_unit.VENDOR_DEPS` so this file reaches diff --git a/tests/const/test_const_method_case_sensitive_896_unit.py b/tests/const/test_const_method_case_sensitive_896_unit.py index 4d3f77b16b..c745b15c43 100644 --- a/tests/const/test_const_method_case_sensitive_896_unit.py +++ b/tests/const/test_const_method_case_sensitive_896_unit.py @@ -75,6 +75,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises ``Method.get`` as well as its regeneration pin. +__tier__ = 'ALL' + def _reconstruct_fixture_csv() -> 'str': """Rebuild the IANA-shaped CSV that reproduces the committed generated diff --git a/tests/const/test_register_roundtrip_unit.py b/tests/const/test_register_roundtrip_unit.py index b14256ffef..df75b8a32c 100644 --- a/tests/const/test_register_roundtrip_unit.py +++ b/tests/const/test_register_roundtrip_unit.py @@ -24,6 +24,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + class TestOptionTypeRegister(unittest.TestCase): """GitHub issues #1298 and #1300.""" diff --git a/tests/corekit/test_decorators_path_1735_unit.py b/tests/corekit/test_decorators_path_1735_unit.py index e9a8b784b4..f7390fa750 100644 --- a/tests/corekit/test_decorators_path_1735_unit.py +++ b/tests/corekit/test_decorators_path_1735_unit.py @@ -16,6 +16,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): checks where the decorators live and that their old path is +#: gone, a question about the tree. +__tier__ = 'LATEST' + #: Where the decorators live. NEW_PATH = 'pcapkit.corekit.decorators' diff --git a/tests/corekit/test_field_roundtrip_unit.py b/tests/corekit/test_field_roundtrip_unit.py index 366ae210e2..156e6ef060 100644 --- a/tests/corekit/test_field_roundtrip_unit.py +++ b/tests/corekit/test_field_roundtrip_unit.py @@ -40,6 +40,9 @@ if TYPE_CHECKING: from typing import Any, Callable +#: Test tier (#1958): a sweep over every corekit field class. +__tier__ = 'LATEST' + class FieldCase(NamedTuple): """One field and the inputs it is round-tripped on.""" diff --git a/tests/corekit/test_fields_copy_dispatch_runtime.py b/tests/corekit/test_fields_copy_dispatch_runtime.py index c2e871dca2..068c425acc 100644 --- a/tests/corekit/test_fields_copy_dispatch_runtime.py +++ b/tests/corekit/test_fields_copy_dispatch_runtime.py @@ -39,6 +39,10 @@ from tests._support import reimport_once_per_class, sample_path +#: Test tier (#1958): a sweep over every record of every classic-PCAP capture; its +#: ``_unit`` sibling pins the mechanism on every leg. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/corekit/test_from_dict_roundtrip_unit.py b/tests/corekit/test_from_dict_roundtrip_unit.py index 58b26bbf28..4f74e81fa0 100644 --- a/tests/corekit/test_from_dict_roundtrip_unit.py +++ b/tests/corekit/test_from_dict_roundtrip_unit.py @@ -28,6 +28,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: PCAP-NG blocks, little-endian, one per block schema that claims a type code. #: Each nests its body in :attr:`PCAPNG.block #: `, which is the diff --git a/tests/corekit/test_info_roundtrip_unit.py b/tests/corekit/test_info_roundtrip_unit.py index 6399f25efe..374e389f79 100644 --- a/tests/corekit/test_info_roundtrip_unit.py +++ b/tests/corekit/test_info_roundtrip_unit.py @@ -49,6 +49,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): a sweep over every ``Info`` class. +__tier__ = 'LATEST' + #: Packages whose modules define :class:`Info` subclasses but are not imported by #: ``import pcapkit`` alone. PACKAGES = ('pcapkit.protocols.data', 'pcapkit.foundation.reassembly.data', diff --git a/tests/corekit/test_schema_roundtrip_unit.py b/tests/corekit/test_schema_roundtrip_unit.py index 9c15b18e36..811bd3740e 100644 --- a/tests/corekit/test_schema_roundtrip_unit.py +++ b/tests/corekit/test_schema_roundtrip_unit.py @@ -51,6 +51,9 @@ if TYPE_CHECKING: from typing import Any, Callable, Optional +#: Test tier (#1958): a sweep over every protocol schema. +__tier__ = 'LATEST' + #: Package prefix stripped from a schema's module to make its label. PREFIX = 'pcapkit.protocols.schema.' diff --git a/tests/corekit/test_sentinel_exports_unit.py b/tests/corekit/test_sentinel_exports_unit.py index 9ff353ce2c..92ef46b013 100644 --- a/tests/corekit/test_sentinel_exports_unit.py +++ b/tests/corekit/test_sentinel_exports_unit.py @@ -133,6 +133,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises the sentinels' exports and behaviour as well as the +#: conventions page. +__tier__ = 'ALL' + # ``pcapkit`` is deliberately not imported at module load. GitHub issue #1332: the # plain-unittest ordering legs leave a freshly re-imported ``pcapkit`` in # ``sys.modules``, so a sentinel bound here would be the *old* import's object and diff --git a/tests/dumpkit/test_capture_dump_runtime.py b/tests/dumpkit/test_capture_dump_runtime.py index 041752784d..5d426cbe84 100644 --- a/tests/dumpkit/test_capture_dump_runtime.py +++ b/tests/dumpkit/test_capture_dump_runtime.py @@ -40,6 +40,9 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): a sweep of every dump format over every sample capture. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/dumpkit/test_dumper_roundtrip_unit.py b/tests/dumpkit/test_dumper_roundtrip_unit.py index 43607d3079..31ffd63c60 100644 --- a/tests/dumpkit/test_dumper_roundtrip_unit.py +++ b/tests/dumpkit/test_dumper_roundtrip_unit.py @@ -39,6 +39,9 @@ if TYPE_CHECKING: from typing import Iterator +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_new_engine_parity_runtime.py b/tests/foundation/engines/test_new_engine_parity_runtime.py index e692700df9..a535dd207b 100644 --- a/tests/foundation/engines/test_new_engine_parity_runtime.py +++ b/tests/foundation/engines/test_new_engine_parity_runtime.py @@ -28,6 +28,10 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): its engines install below Python 3.12 only, so the parity legs, +#: none of them latest, are where it runs. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/reassembly/test_ip_reassembly_roundtrip_unit.py b/tests/foundation/reassembly/test_ip_reassembly_roundtrip_unit.py index b101f077d5..51ea9e0a2b 100644 --- a/tests/foundation/reassembly/test_ip_reassembly_roundtrip_unit.py +++ b/tests/foundation/reassembly/test_ip_reassembly_roundtrip_unit.py @@ -19,6 +19,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: A 2000-octet payload with no zero octet in it, so a zero-filled hole shows. FULL = (bytes(range(1, 256)) * 8)[:2000] diff --git a/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py b/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py index 76764ef91e..137ad3829e 100644 --- a/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py +++ b/tests/foundation/reassembly/test_ipv6_cut_non_final_fragment_1567_runtime.py @@ -27,6 +27,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): one capture through each engine, whose releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) ENGINES = ('default',) + tuple(name for name in ('dpkt', 'scapy') if importlib.util.find_spec(name) is not None) diff --git a/tests/foundation/reassembly/test_reassembly_payload_roundtrip_unit.py b/tests/foundation/reassembly/test_reassembly_payload_roundtrip_unit.py index 17ef0a84dd..4435fa4dd6 100644 --- a/tests/foundation/reassembly/test_reassembly_payload_roundtrip_unit.py +++ b/tests/foundation/reassembly/test_reassembly_payload_roundtrip_unit.py @@ -42,6 +42,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): a sweep over every payload in every arrival order. +__tier__ = 'LATEST' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py b/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py index 563d4600d7..317bf7bd7c 100644 --- a/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py +++ b/tests/foundation/reassembly/test_tcp_reassembly_roundtrip_unit.py @@ -23,6 +23,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + BUFID = (ip_address('192.0.2.1'), 12345, ip_address('198.51.100.2'), 443) diff --git a/tests/foundation/registry/test_registry_symmetry_unit.py b/tests/foundation/registry/test_registry_symmetry_unit.py index 6624c26146..992e4b2ee4 100644 --- a/tests/foundation/registry/test_registry_symmetry_unit.py +++ b/tests/foundation/registry/test_registry_symmetry_unit.py @@ -59,6 +59,9 @@ if TYPE_CHECKING: from typing import Any, Callable +#: Test tier (#1958): a sweep that overrides and restores every public registrar. +__tier__ = 'LATEST' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/foundation/traceflow/test_traceflow_reread_roundtrip_runtime.py b/tests/foundation/traceflow/test_traceflow_reread_roundtrip_runtime.py index 7281b53a19..e8d08e9fb2 100644 --- a/tests/foundation/traceflow/test_traceflow_reread_roundtrip_runtime.py +++ b/tests/foundation/traceflow/test_traceflow_reread_roundtrip_runtime.py @@ -32,6 +32,9 @@ from tests.foundation import _roundtrip as harness from tests.foundation._roundtrip import Gap, Outcome, read_pcap +#: Test tier (#1958): a sweep that traces and rereads every sample capture. +__tier__ = 'LATEST' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/foundation/traceflow/test_traceflow_reread_roundtrip_unit.py b/tests/foundation/traceflow/test_traceflow_reread_roundtrip_unit.py index 1c44890514..8ecfc6669c 100644 --- a/tests/foundation/traceflow/test_traceflow_reread_roundtrip_unit.py +++ b/tests/foundation/traceflow/test_traceflow_reread_roundtrip_unit.py @@ -44,6 +44,9 @@ if TYPE_CHECKING: from typing import Callable +#: Test tier (#1958): a sweep over every trace-and-reread case of its table. +__tier__ = 'LATEST' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/integration/test_cli_subprocess.py b/tests/integration/test_cli_subprocess.py index 6178e902b8..221b267cb7 100644 --- a/tests/integration/test_cli_subprocess.py +++ b/tests/integration/test_cli_subprocess.py @@ -24,6 +24,10 @@ from tests._tiers import ROOT from tests.integration._helpers import HAS_EMOJI, HAS_RUNTIME, EndToEndTestCase +#: Test tier (#1958): runs ``python -m pcapkit`` and the console script, whose lookup +#: varies by interpreter. +__tier__ = 'ALL' + #: How long a single CLI run is allowed to take. The captures used here are two #: to twenty-six frames, so this is a hang guard rather than a budget. TIMEOUT = 120 diff --git a/tests/integration/test_engine_parity.py b/tests/integration/test_engine_parity.py index aac5867f30..f5c522cfba 100644 --- a/tests/integration/test_engine_parity.py +++ b/tests/integration/test_engine_parity.py @@ -22,6 +22,10 @@ from tests._support import sample_path from tests.integration._helpers import HAS_DPKT, HAS_RUNTIME, HAS_SCAPY, EndToEndTestCase +#: Test tier (#1958): compares the engines, whose releases vary by interpreter, on +#: hand-picked captures. +__tier__ = 'ALL' + #: Captures every engine is run over, with the frame count each must report. #: All are small: the point is what each engine reports, not throughput. PARITY_CAPTURES = { diff --git a/tests/integration/test_reassembly_engine_parity.py b/tests/integration/test_reassembly_engine_parity.py index 478ea65fdb..dee8901d75 100644 --- a/tests/integration/test_reassembly_engine_parity.py +++ b/tests/integration/test_reassembly_engine_parity.py @@ -29,6 +29,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): compares the engines, whose releases vary by interpreter, on +#: hand-picked captures. +__tier__ = 'ALL' + #: Next Header value of the IPv6 Fragment header (:rfc:`8200#section-4.5`). NH_IPV6_FRAG = 44 #: Next Header value of UDP, which is what ``ipv6.pcap``'s fragments carry. diff --git a/tests/integration/test_traceflow_end_to_end.py b/tests/integration/test_traceflow_end_to_end.py index 7e6652c07d..8c7860f16e 100644 --- a/tests/integration/test_traceflow_end_to_end.py +++ b/tests/integration/test_traceflow_end_to_end.py @@ -15,6 +15,9 @@ from tests._support import sample_path from tests.integration._helpers import HAS_RUNTIME, EndToEndTestCase, read_json +#: Test tier (#1958): hand-picked traces of the sample captures, not a sweep. +__tier__ = 'ALL' + #: The two connections of :file:`tcp.pcap`: concurrent SSH sessions, one over #: IPv4 and one over IPv6. Tracing keys a flow on the canonical *pair* of #: endpoints, so a connection is one flow rather than one per direction; the diff --git a/tests/project/test_all_exports_unit.py b/tests/project/test_all_exports_unit.py index d1c269bef5..b3f5bf8030 100644 --- a/tests/project/test_all_exports_unit.py +++ b/tests/project/test_all_exports_unit.py @@ -16,6 +16,10 @@ import os import unittest +#: Test tier (#1958): checks the names ``pcapkit.all`` exports, a question about the +#: tree; test_public_api.py imports the package on every leg. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/project/test_all_layer_grouping_unit.py b/tests/project/test_all_layer_grouping_unit.py index 920b84d9ca..4a0ea1056d 100644 --- a/tests/project/test_all_layer_grouping_unit.py +++ b/tests/project/test_all_layer_grouping_unit.py @@ -18,6 +18,10 @@ import tokenize import unittest +#: Test tier (#1958): checks how pcapkit/all.py groups its names by layer, a question +#: about the tree. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/project/test_annotation_names.py b/tests/project/test_annotation_names.py index 76d1f37cc3..546bc9b9bd 100644 --- a/tests/project/test_annotation_names.py +++ b/tests/project/test_annotation_names.py @@ -49,6 +49,10 @@ if TYPE_CHECKING: from typing import Iterator, Optional +#: Test tier (#1958): checks the AST of the package's sources, the same text on every +#: leg. +__tier__ = 'LATEST' + #: Package directory whose annotations are checked. PACKAGE = ROOT / 'pcapkit' diff --git a/tests/project/test_build_leftovers_1773.py b/tests/project/test_build_leftovers_1773.py index 673554b465..1ff5b3aa3e 100644 --- a/tests/project/test_build_leftovers_1773.py +++ b/tests/project/test_build_leftovers_1773.py @@ -30,6 +30,10 @@ import unittest from unittest import mock +#: Test tier (#1958): checks what a stub build through setup.py leaves behind; every +#: leg's install runs setup.py for real. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/project/test_bump_version.py b/tests/project/test_bump_version.py index b4ac857c9e..0c7325e58e 100644 --- a/tests/project/test_bump_version.py +++ b/tests/project/test_bump_version.py @@ -54,6 +54,9 @@ import unittest from unittest import mock +#: Test tier (#1958): drives util/bump_version.py, which runs on one interpreter. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: A minimal document that ``cffconvert --validate`` accepts, carrying the two diff --git a/tests/project/test_class_guard_annotations_1857_unit.py b/tests/project/test_class_guard_annotations_1857_unit.py index 2348b3399d..315b758bd2 100644 --- a/tests/project/test_class_guard_annotations_1857_unit.py +++ b/tests/project/test_class_guard_annotations_1857_unit.py @@ -56,6 +56,10 @@ class body nor a class-level guarded import is in scope. :file:`docs/source/conf from types import ModuleType from typing import Any, Iterator, Optional +#: Test tier (#1958): resolves annotations as the docs build does, which runs on one +#: interpreter. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/project/test_conda_cryptography_pin_1827.py b/tests/project/test_conda_cryptography_pin_1827.py index f4740c8ce2..9985cd5211 100644 --- a/tests/project/test_conda_cryptography_pin_1827.py +++ b/tests/project/test_conda_cryptography_pin_1827.py @@ -35,6 +35,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives util/conda_vendor.py, which runs on one interpreter. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] REQUIREMENTS = ROOT / 'conda' / 'requirements.txt' #: The Intel macOS line :func:`stand_in`'s pin writes. diff --git a/tests/project/test_conda_native_platforms_1822.py b/tests/project/test_conda_native_platforms_1822.py index 3880573860..14db045e07 100644 --- a/tests/project/test_conda_native_platforms_1822.py +++ b/tests/project/test_conda_native_platforms_1822.py @@ -49,6 +49,10 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): checks the conda jobs' matrices and drives their scripts +#: offline. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOWS = ROOT / '.github' / 'workflows' FIXTURES = pathlib.Path(__file__).resolve().parent / 'python_versions_fixtures' diff --git a/tests/project/test_conda_platform_pin_refresh_1851.py b/tests/project/test_conda_platform_pin_refresh_1851.py index 155f29e61a..c62a55c32e 100644 --- a/tests/project/test_conda_platform_pin_refresh_1851.py +++ b/tests/project/test_conda_platform_pin_refresh_1851.py @@ -60,6 +60,9 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): drives util/conda_vendor.py, which runs on one interpreter. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW = ROOT / '.github' / 'workflows' / 'cron-conda.yml' #: The conda Pythons the stand-ins are built for; the matcher takes any list. diff --git a/tests/project/test_conda_vendor_1814.py b/tests/project/test_conda_vendor_1814.py index 5252e3455b..3a79776725 100644 --- a/tests/project/test_conda_vendor_1814.py +++ b/tests/project/test_conda_vendor_1814.py @@ -48,6 +48,9 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): drives util/conda_vendor.py, which runs on one interpreter. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] SCRIPT = ROOT / 'util' / 'conda_vendor.py' NO_TOMLLIB = 'tomllib, which reads pyproject.toml and .github/python-versions.toml, needs Python 3.11+' diff --git a/tests/project/test_const_get_overrides_documented_1921_unit.py b/tests/project/test_const_get_overrides_documented_1921_unit.py index e24f59e08d..cf2c9efe26 100644 --- a/tests/project/test_const_get_overrides_documented_1921_unit.py +++ b/tests/project/test_const_get_overrides_documented_1921_unit.py @@ -28,6 +28,9 @@ if TYPE_CHECKING: from typing import Any, Iterator, Optional +#: Test tier (#1958): checks the constant modules' docstrings against the docs pages. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: The directive that hides a member from autodoc. diff --git a/tests/project/test_conventions_doc_claims.py b/tests/project/test_conventions_doc_claims.py index ceac7b0eb0..e3f436a2ab 100644 --- a/tests/project/test_conventions_doc_claims.py +++ b/tests/project/test_conventions_doc_claims.py @@ -84,6 +84,9 @@ import aenum +#: Test tier (#1958): checks the conventions pages' claims against the tree. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: The directory GitHub issue #918 part 2 split the single-page diff --git a/tests/project/test_docs_aenum_formatter_1858_unit.py b/tests/project/test_docs_aenum_formatter_1858_unit.py index 884eea65e8..5845455c82 100644 --- a/tests/project/test_docs_aenum_formatter_1858_unit.py +++ b/tests/project/test_docs_aenum_formatter_1858_unit.py @@ -29,6 +29,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + @unittest.skipUnless(HAS_SPHINX, SPHINX_SKIP_REASON) class AenumFormatterTests(unittest.TestCase): diff --git a/tests/project/test_docs_ambiguous_xrefs_1775_unit.py b/tests/project/test_docs_ambiguous_xrefs_1775_unit.py index f23b342a33..de9c9f39a4 100644 --- a/tests/project/test_docs_ambiguous_xrefs_1775_unit.py +++ b/tests/project/test_docs_ambiguous_xrefs_1775_unit.py @@ -46,6 +46,10 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: The documentation tree, and the configuration whose hooks are tested. diff --git a/tests/project/test_docs_class_formatter_1918_unit.py b/tests/project/test_docs_class_formatter_1918_unit.py index 23ad29515d..ad9ad4f541 100644 --- a/tests/project/test_docs_class_formatter_1918_unit.py +++ b/tests/project/test_docs_class_formatter_1918_unit.py @@ -31,6 +31,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None HAS_DPKT = importlib.util.find_spec('dpkt') is not None HAS_PYSHARK = importlib.util.find_spec('pyshark') is not None diff --git a/tests/project/test_docs_class_signature_1860_unit.py b/tests/project/test_docs_class_signature_1860_unit.py index 8492715ec2..b8a4fe0501 100644 --- a/tests/project/test_docs_class_signature_1860_unit.py +++ b/tests/project/test_docs_class_signature_1860_unit.py @@ -36,6 +36,10 @@ class docstrings lost the types of the parameters they list. if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The event the signature hook answers. EVENT = 'autodoc-process-signature' diff --git a/tests/project/test_docs_enum_signature_1862_unit.py b/tests/project/test_docs_enum_signature_1862_unit.py index 1c50bbd0a0..cb8305ba14 100644 --- a/tests/project/test_docs_enum_signature_1862_unit.py +++ b/tests/project/test_docs_enum_signature_1862_unit.py @@ -30,6 +30,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The memberless enumerations that rendered the functional-API form. MEMBERLESS = ( 'pcapkit.const.reg.apptype.apptype.AppType', diff --git a/tests/project/test_docs_enum_value_hook_unit.py b/tests/project/test_docs_enum_value_hook_unit.py index 8356d00013..d51d0d00f6 100644 --- a/tests/project/test_docs_enum_value_hook_unit.py +++ b/tests/project/test_docs_enum_value_hook_unit.py @@ -29,6 +29,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The member the hook skips. HOOK = '_generate_next_value_' diff --git a/tests/project/test_docs_internal_refs_1940_unit.py b/tests/project/test_docs_internal_refs_1940_unit.py index b5b90aaf19..302b69dd73 100644 --- a/tests/project/test_docs_internal_refs_1940_unit.py +++ b/tests/project/test_docs_internal_refs_1940_unit.py @@ -40,6 +40,10 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): checks the docs' cross-reference roles, partly through +#: docs/source/conf.py. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] DOCS = ROOT / 'docs' / 'source' diff --git a/tests/project/test_docs_internal_refs_1947_unit.py b/tests/project/test_docs_internal_refs_1947_unit.py index f791faa743..f8c9b89397 100644 --- a/tests/project/test_docs_internal_refs_1947_unit.py +++ b/tests/project/test_docs_internal_refs_1947_unit.py @@ -28,6 +28,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): checks the references in the vendor templates and the generated +#: constant modules. +__tier__ = 'LATEST' + TRANSPORT_PROTOCOL = ':class:`~pcapkit.const.reg.apptype.apptype.TransportProtocol`' APPTYPE_GET = ':meth:`AppType.get() `' diff --git a/tests/project/test_docs_pcap_module_link_1948_unit.py b/tests/project/test_docs_pcap_module_link_1948_unit.py index bb07ca5a23..53f608b200 100644 --- a/tests/project/test_docs_pcap_module_link_1948_unit.py +++ b/tests/project/test_docs_pcap_module_link_1948_unit.py @@ -47,6 +47,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): checks the docs' links to the pcap module note, partly through +#: docs/source/conf.py. +__tier__ = 'LATEST' + #: The label ``|pcap|_`` links to. LABEL = 'pcap-module' diff --git a/tests/project/test_docs_preserve_defaults_1859_unit.py b/tests/project/test_docs_preserve_defaults_1859_unit.py index 794fae34a9..fb790efe34 100644 --- a/tests/project/test_docs_preserve_defaults_1859_unit.py +++ b/tests/project/test_docs_preserve_defaults_1859_unit.py @@ -31,6 +31,10 @@ if TYPE_CHECKING: from typing import Any, Callable +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The event the alias hook answers. EVENT = 'autodoc-before-process-signature' diff --git a/tests/project/test_docs_table_wrapper_ids_1861_unit.py b/tests/project/test_docs_table_wrapper_ids_1861_unit.py index 999d77b624..ebf39f8123 100644 --- a/tests/project/test_docs_table_wrapper_ids_1861_unit.py +++ b/tests/project/test_docs_table_wrapper_ids_1861_unit.py @@ -26,6 +26,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The theme whose transform copies the ids. HAS_FURO = importlib.util.find_spec('furo') is not None diff --git a/tests/project/test_docs_typehints_pin_1856_unit.py b/tests/project/test_docs_typehints_pin_1856_unit.py index 61821abe18..e870e3679e 100644 --- a/tests/project/test_docs_typehints_pin_1856_unit.py +++ b/tests/project/test_docs_typehints_pin_1856_unit.py @@ -37,6 +37,10 @@ class signatures fell back to a bare ``_VT``, which matched another module's if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): checks docs/source/conf.py against the docs build's pins, which +#: run on one interpreter. +__tier__ = 'LATEST' + class PinAgreementTests(unittest.TestCase): """The ``docs`` extra installs the release :file:`docs/requirements.txt` pins.""" diff --git a/tests/project/test_docs_viewcode_builtins_1863_unit.py b/tests/project/test_docs_viewcode_builtins_1863_unit.py index fdfbb70bc9..d820094156 100644 --- a/tests/project/test_docs_viewcode_builtins_1863_unit.py +++ b/tests/project/test_docs_viewcode_builtins_1863_unit.py @@ -26,6 +26,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The event the hook answers. EVENT = 'viewcode-follow-imported' diff --git a/tests/project/test_docs_wrapped_overloads_1917_unit.py b/tests/project/test_docs_wrapped_overloads_1917_unit.py index f42f82e21f..244c71670f 100644 --- a/tests/project/test_docs_wrapped_overloads_1917_unit.py +++ b/tests/project/test_docs_wrapped_overloads_1917_unit.py @@ -30,6 +30,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a hook of docs/source/conf.py, which the docs build runs +#: on one interpreter. +__tier__ = 'LATEST' + #: The event the hook answers. EVENT = 'autodoc-before-process-signature' diff --git a/tests/project/test_documentation_claims.py b/tests/project/test_documentation_claims.py index 3871974cdd..f249a34d46 100644 --- a/tests/project/test_documentation_claims.py +++ b/tests/project/test_documentation_claims.py @@ -43,6 +43,10 @@ import textwrap import unittest +#: Test tier (#1958): checks what the docs claim about the package against its code, a +#: question about the tree. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/project/test_generators_import_their_own_tree.py b/tests/project/test_generators_import_their_own_tree.py index a4a0f8db4f..7bc7c6cda1 100644 --- a/tests/project/test_generators_import_their_own_tree.py +++ b/tests/project/test_generators_import_their_own_tree.py @@ -34,6 +34,10 @@ import unittest from unittest import mock +#: Test tier (#1958): checks that each generator imports its own tree; make_samples.py +#: runs the generators on every integration leg. +__tier__ = 'LATEST' + #: Repository root. ROOT = pathlib.Path(__file__).resolve().parents[2] #: Directory holding the generators. diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 63cad7df20..e8c6ae8199 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -130,7 +130,7 @@ #: without a cap of their own -- checkout, the parallelism report, the resource #: monitor, the uploads and the summaries. Everything but the install and the tests #: took at most 0.32 minutes, setup-python included, on every uncovered leg of runs -#: 38569321543, 38569429406 and 38569522442. #1954's ``List the project pins`` runs +#: 38569321543, 38569429406 and 38569522442. #1958's ``List the LATEST tier`` runs #: on the runner's CPython, which lists them in seconds (measured locally on 3.14). UNCAPPED_MINUTES = 3 #: Those steps' cap on every other leg: the largest a step's ``timeout-minutes`` may be diff --git a/tests/project/test_module_isolation.py b/tests/project/test_module_isolation.py index 02e64ab678..92ee1e0df5 100644 --- a/tests/project/test_module_isolation.py +++ b/tests/project/test_module_isolation.py @@ -95,6 +95,10 @@ from tests._tiers import ROOT +#: Test tier (#1958): checks that modules purge and reimport cleanly, which is the +#: interpreter's import system. +__tier__ = 'ALL' + #: Selections that failed before #660 was fixed, each as (label, files). The #: polluting file comes first in every one, because that is the order that broke. POLLUTING_ORDERS = [ diff --git a/tests/project/test_no_moved_name_re_exports_1744_unit.py b/tests/project/test_no_moved_name_re_exports_1744_unit.py index 15964bd30c..b07c0ff4a9 100644 --- a/tests/project/test_no_moved_name_re_exports_1744_unit.py +++ b/tests/project/test_no_moved_name_re_exports_1744_unit.py @@ -37,6 +37,10 @@ import re import unittest +#: Test tier (#1958): checks that each moved name's old path is gone, a question about +#: the tree. +__tier__ = 'LATEST' + #: Repository root. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/project/test_pcapng_download_retry_1658_unit.py b/tests/project/test_pcapng_download_retry_1658_unit.py index 9d4506991e..060d79414c 100644 --- a/tests/project/test_pcapng_download_retry_1658_unit.py +++ b/tests/project/test_pcapng_download_retry_1658_unit.py @@ -40,6 +40,9 @@ from tests._support import PCAPNG_GENERATOR, scale_timeout +#: Test tier (#1958): drives the PCAP-NG generator's download retry with HTTP mocked. +__tier__ = 'LATEST' + #: URL the stubbed getter is asked for. URL = 'https://example.invalid/many_interfaces.pcapng.1' #: Fake current time, a whole second so an HTTP-date maps to it exactly. diff --git a/tests/project/test_project_pins_once_1954.py b/tests/project/test_project_pins_once_1954.py index 383c5dd588..984e040fe8 100644 --- a/tests/project/test_project_pins_once_1954.py +++ b/tests/project/test_project_pins_once_1954.py @@ -80,6 +80,17 @@ def test_the_ordering_cell_runs_every_pin(self) -> None: self.assertTrue(pins) self.assertEqual(pins - modules, set()) + def test_every_pin_is_in_the_latest_tier(self) -> None: + """#1958: the pins are ``LATEST`` modules, so the legs off ``latest`` leave them out with the rest of + the tier, and ``latest`` leaves out the pins alone.""" + pins = set(_tiers.project_pin_paths()) + self.assertTrue(pins) + self.assertEqual({_tiers.module_tier(pin) for pin in pins}, {_tiers.TIER_LATEST}) + self.assertLess(pins, set(_tiers.latest_tier_paths())) + run = str(step(('unit-tests.yml', 'test'), 'Run unit tests')['run']) + self.assertIn(_tiers.PROJECT_PINS_SELECTOR, run) + self.assertIn(_tiers.LATEST_TIER_SELECTOR, run) + class TestTheDocsSayIt(unittest.TestCase): diff --git a/tests/project/test_pyshark_encap_map.py b/tests/project/test_pyshark_encap_map.py index 7edf902389..0a150c62c4 100644 --- a/tests/project/test_pyshark_encap_map.py +++ b/tests/project/test_pyshark_encap_map.py @@ -65,6 +65,9 @@ from pcapkit.const.reg.linktype import LinkType as Enum_LinkType +#: Test tier (#1958): drives util/pyshark_encap_map.py, which runs on one interpreter. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/project/test_release_gates.py b/tests/project/test_release_gates.py index 287da7eddd..593d0e0ebb 100644 --- a/tests/project/test_release_gates.py +++ b/tests/project/test_release_gates.py @@ -229,6 +229,9 @@ if TYPE_CHECKING: from typing import Optional +#: Test tier (#1958): checks the release and conda workflows' gates. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: The workflow #641 is about, and the one every test above diff --git a/tests/project/test_release_tags_the_run_sha_1634.py b/tests/project/test_release_tags_the_run_sha_1634.py index 1f4c3c55f4..d4e0aafb4a 100644 --- a/tests/project/test_release_tags_the_run_sha_1634.py +++ b/tests/project/test_release_tags_the_run_sha_1634.py @@ -75,6 +75,10 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): drives the release workflow's tag job against temporary +#: repositories. +__tier__ = 'LATEST' + ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW = ROOT / '.github' / 'workflows' / 'create-release.yml' CONDA_WORKFLOW = ROOT / '.github' / 'workflows' / 'cron-conda.yml' diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py index 2828e1be76..63fba6d93a 100644 --- a/tests/project/test_resource_monitor_1740.py +++ b/tests/project/test_resource_monitor_1740.py @@ -234,6 +234,9 @@ def test_the_stop_step_passes_on_a_leg_the_monitor_never_started(self) -> None: #: Round 2's containment, as the issue's research sized it (estimates, until a run shows them); #: #1753 adds the exit on a spent heap, which tests/project/test_graalpy_worker_heap_1753.py pins. HEAP, GRAAL_WORKERS, SCOPE_MEMORY = '--vm.Xmx2g --vm.XX:+ExitOnOutOfMemoryError', '3', '12G' +#: A ``LATEST``-tier module, as ``List the LATEST tier`` would hand the tests step (#1958): +#: a stand-in naming no real module, so this pin runs no test module by path. +SWEEP = 'tests/protocols/test_a_latest_sweep_unit.py' FSIZE = 2 * 1024 ** 3 #: Round 3: the probe is the scope itself, every flag included, under a throwaway unit. PROBE = 'if probe=$(scoped "pcapkit-probe-$$.scope" true 2>&1); then' @@ -341,7 +344,7 @@ def setUp(self) -> None: self.vmstat.write_text('oom_kill 7\n', encoding='utf-8') self.argv = self.tmp / 'argv' self.env = dict(os.environ, RUNNER_TEMP=str(self.temp), GITHUB_WORKSPACE=str(ROOT), WORKERS='3', - GRAAL_PYTHON_VM_ARGS=HEAP, + GRAAL_PYTHON_VM_ARGS=HEAP, LATEST_TIER=SWEEP, LEG='Python graalpy-25.4 (experimental, non-blocking)', GITHUB_STEP_SUMMARY=str(self.tmp / 'summary.md'), PATH=str(self.bindir) + os.pathsep + os.environ.get('PATH', '')) @@ -439,7 +442,7 @@ def test_the_uncapped_tests_run_the_capped_argv(self) -> None: argv = argvs['capped'] self.assertEqual(argv[:6], ['-m', 'pytest', '-q', '-n', '3', '--dist']) for arg in (f'--junitxml={self.temp}/junit.xml', '--ignore=tests/integration', '--ignore-glob=*_runtime.py', - '--ignore-glob=*_regression.py', '--ignore-glob=*roundtrip*', '--durations=30'): + '--ignore-glob=*_regression.py', f'--ignore={SWEEP}', '--durations=30'): with self.subTest(arg=arg): self.assertIn(arg, argv) # Only the capped path took the tests through sudo: the probe and the run. diff --git a/tests/project/test_rfc_constants_one_home_1739_unit.py b/tests/project/test_rfc_constants_one_home_1739_unit.py index d9a0de8fc3..48ae3302b3 100644 --- a/tests/project/test_rfc_constants_one_home_1739_unit.py +++ b/tests/project/test_rfc_constants_one_home_1739_unit.py @@ -39,6 +39,10 @@ import sys import unittest +#: Test tier (#1958): checks that each RFC constant has one home, a question about the +#: tree. +__tier__ = 'LATEST' + #: Repository root. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/project/test_setup.py b/tests/project/test_setup.py index 455d739f6e..de5372122b 100644 --- a/tests/project/test_setup.py +++ b/tests/project/test_setup.py @@ -8,6 +8,10 @@ import unittest from unittest import mock +#: Test tier (#1958): loads setup.py under the running interpreter, as every leg's +#: install does. +__tier__ = 'ALL' + ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 0ef9eec8f2..1c694fd683 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -1,31 +1,37 @@ # -*- coding: utf-8 -*- -"""#1732: the round-trip sweeps run on ``latest`` alone. - -The owner's ruling, paraphrased: run a subset, and the round-trip sweeps need run -only on the latest version. So ``test``'s ``Run unit tests`` step adds -:data:`~tests._tiers.ROUNDTRIP_SWEEP_GLOB` as one more ``--ignore-glob`` on every -leg whose version is not ``latest`` -- the KTLO and older supported legs -- as +"""#1732 and #1958: the ``LATEST`` test tier runs on ``latest`` alone. + +The owner's ruling on #1732, paraphrased: run a subset, and the round-trip sweeps +need run only on the latest version. #1958 makes that subset a test tier rather +than a name glob: :func:`tests._tiers.module_tier` puts each module in ``ALL`` or +``LATEST``, by the module's own ``__tier__`` or by the rule, and the owner's +rulings there, paraphrased, keep the hand-picked round trips on every leg and send +only the real sweeps, with the modules whose answer no interpreter changes, to +``latest``. So ``test``'s ``Run unit tests`` step passes each module +:func:`~tests._tiers.latest_tier_paths` prints as an ``--ignore`` on every leg whose +version is not ``latest`` -- the KTLO and older supported legs -- as ``experimental``'s does on each of its legs, and ``latest``, the coverage leg, keeps -the whole unit tier. The owner later ruled that the subset covers ``integration`` -too: off ``latest``, its ``Run full test suite`` drops the modules -:data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB` names, :class:`TestTheIntegrationSubset`. +the whole unit tier less the project pins (#1954). ``integration`` does the same +with the fixture tier: off ``latest``, its ``Run full test suite`` asks for +:func:`~tests._tiers.fixture_tier_paths`'s ``ALL`` part, and ``pypcap-parity``, none +of whose legs is ``latest``, asks for it on each, :class:`TestTheIntegrationSubset`. :class:`TestTheStepSelectsByLeg` runs the step's script on each leg the tier file gives either job, with a ``python`` on ``PATH`` that records its arguments, so what is checked is the selection bash builds, not what the text appears to say. -:class:`TestTheGlobIsTheSweeps` asks pytest itself what the glob ignores. +:class:`TestTheIgnoresAreTheLatestTier` asks pytest itself what those ignores leave +out. -The owner's second ruling, option 2: nothing waits on the experimental legs, which -with the caps GraalPy needs run for up to two hours. A job of unit-tests.yml would -still hold its run in progress, and with it every ``workflow_run`` consumer of -"Unit Tests", so they are a workflow of their own, experimental.yml: -:class:`TestTheExperimentalLegsAreAWorkflowOfTheirOwn`. +The owner's second ruling on #1732, option 2: nothing waits on the experimental +legs, which with the caps GraalPy needs run for up to two hours. A job of +unit-tests.yml would still hold its run in progress, and with it every +``workflow_run`` consumer of "Unit Tests", so they are a workflow of their own, +experimental.yml: :class:`TestTheExperimentalLegsAreAWorkflowOfTheirOwn`. """ from __future__ import annotations -import fnmatch import os import pathlib import re @@ -42,19 +48,19 @@ from tests.project.test_release_gates import NULL, evaluate_expression if TYPE_CHECKING: - from typing import Any + from typing import Any, Optional + +#: Test tier (#1958): checks the workflows' tier selections. +__tier__ = 'LATEST' #: The jobs and step that run the unit tier: ``test``, and the experimental legs' own (#1732). JOB = ('unit-tests.yml', 'test') EXPERIMENTAL = ('experimental.yml', 'experimental') STEP = 'Run unit tests' -#: The job and step that run the fixture tier, which the subset covers too (#1732). +#: The jobs and step that run the fixture tier, which the tiers cover too (#1732, #1958). INTEGRATION = ('unit-tests.yml', 'integration') +PARITY = ('unit-tests.yml', 'pypcap-parity') INTEGRATION_STEP = 'Run full test suite' -#: The fixture tier's round-trip sweeps, by path, as the owner's ruling counted them. -RUNTIME_SWEEPS = ('tests/foundation/traceflow/test_traceflow_reread_roundtrip_runtime.py', - 'tests/protocols/test_capture_roundtrip_runtime.py', - 'tests/protocols/test_declared_length_roundtrip_runtime.py') #: Every workflow that waits on "Unit Tests" completing, with ``workflow_run``. CONSUMERS = ('coverage-comment.yml', 'deploy-pages.yml', 'cron-vendor.yml', 'cron-conda.yml') #: Each job's ``timeout-minutes``, and its capped steps', by ``id``: ``test``'s are @@ -64,23 +70,21 @@ #: The jobs whose result reaches a gate: ``required-checks`` and what it needs, and #: ``unittest-ordering``, which runs on main's pushes and gates publishing. GATING = ('required-checks', 'unittest-ordering') -#: The flag that leaves the sweeps out, as pytest receives it once bash has run. -SWEEP_FLAG = f'--ignore-glob={_tiers.ROUNDTRIP_SWEEP_GLOB}' #: The unit tier's own flags, which every leg passes. TIER_FLAGS = ('--ignore=tests/integration', '--ignore-glob=*_runtime.py', '--ignore-glob=*_regression.py') #: A ``python`` that writes each argument on a line of ``$ARGS_FILE``, and hands a -#: ``-c`` to the real one, which is how ``integration`` asks tests/_tiers.py. +#: ``-c`` to the real one, which is how the steps ask tests/_tiers.py. STUB = ('#!/bin/sh\nif [ "$1" = -c ]; then exec "$REAL_PYTHON" "$@"; fi\n' 'printf \'%s\\n\' "$@" > "$ARGS_FILE"\n') -#: A directory holding round-trip sweeps and modules that only import their -#: harnesses' packet builders, for :class:`TestTheGlobIsTheSweeps` to collect. +#: A directory holding sweeps, hand-picked round trips, and modules that only import +#: the sweeps' harnesses' packet builders, for :class:`TestTheIgnoresAreTheLatestTier`. PROBE = 'tests/protocols/internet' #: ``$RUNNER_TEMP``, which only names the JUnit file here, so one value for every leg. RUNNER_TEMP = '/home/runner/work/_temp' def step(where: 'tuple[str, str]' = JOB) -> 'dict[str, Any]': - name = INTEGRATION_STEP if where == INTEGRATION else STEP + name = STEP if where in (JOB, EXPERIMENTAL) else INTEGRATION_STEP found = [entry for entry in job(*where)['steps'] if entry.get('name') == name] if len(found) != 1: raise AssertionError(f'expected one {name!r} step in {where}, found {len(found)}') @@ -99,17 +103,17 @@ def stringify(value: 'object') -> 'str': def step_env(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'dict[str, str]': """The step's ``env:`` on ``leg``, ``needs`` read from the tier file.""" # #1938: experimental's tests step reads the token and the watcher's interpreter, unwatched here. - # #1954: and the project pins its own step lists, as that step prints them. + # #1958: and the LATEST tier its own step lists, as that step prints it. context = {'matrix': leg, 'needs': {'python-versions': {'outputs': live_outputs()}}, 'github': {'token': ''}, 'steps': {'watcher': {'outcome': 'skipped'}, 'watcher-python': {'outputs': {'python-path': ''}}, - 'pins': {'outputs': {'paths': ' '.join(_tiers.project_pin_paths())}}}} + 'tier': {'outputs': {'paths': ' '.join(_tiers.latest_tier_paths())}}}} return {key: re.sub(r'\$\{\{.*?\}\}', lambda match: stringify(evaluate_expression(match.group(0), context)), str(value)) for key, value in (step(where).get('env') or {}).items()} -def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'list[str]': - """The arguments the step of job ``where`` hands ``python`` on ``leg``.""" +def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB, script: 'Optional[str]' = None) -> 'list[str]': + """The arguments the step of job ``where`` hands ``python`` on ``leg``; ``script`` replaces its ``run:``.""" with tempfile.TemporaryDirectory() as temp: stub = pathlib.Path(temp, 'python') stub.write_text(STUB, encoding='utf-8') @@ -125,11 +129,13 @@ def run_step(leg: 'dict[str, Any]', where: 'tuple[str, str]' = JOB) -> 'list[str record = pathlib.Path(temp, 'args') env = dict(os.environ, **step_env(leg, where), ARGS_FILE=str(record), RUNNER_TEMP=str(runner_temp), REAL_PYTHON=sys.executable, PATH=temp + os.pathsep + os.environ.get('PATH', '')) - # From the root, as the runner runs it: ``integration`` imports tests/_tiers.py. - result = subprocess.run(['bash', '-e', '-c', step(where)['run']], env=env, cwd=ROOT, # nosec B603 B607 - capture_output=True, text=True, check=False, timeout=60) + env.setdefault('ANNOTATION_LEVEL', 'error') + env.pop('PYTHONSAFEPATH', None) # the runner sets none, and it would hide tests/ from `-c` + # From the root, as the runner runs it: the steps import tests/_tiers.py. + result = subprocess.run(['bash', '-e', '-c', script or step(where)['run']], env=env, # nosec B603 B607 + cwd=ROOT, capture_output=True, text=True, check=False, timeout=120) if result.returncode: - raise AssertionError(f'the step failed on {leg}: {result.stderr}') + raise AssertionError(f'the step failed on {leg}: {result.stdout[-2000:]}{result.stderr[-2000:]}') return [arg.replace(str(runner_temp), RUNNER_TEMP) for arg in record.read_text(encoding='utf-8').splitlines()] @@ -141,6 +147,11 @@ def pytest_args(argv: 'list[str]') -> 'list[str]': raise AssertionError(f'no `-m pytest` in {argv}') +def ignored(argv: 'list[str]') -> 'list[str]': + """The modules the arguments ``--ignore``, less the unit tier's own flags.""" + return [arg.split('=', 1)[1] for arg in pytest_args(argv) if arg.startswith('--ignore=') and arg not in TIER_FLAGS] + + @unittest.skipIf(shutil.which('bash') is None, 'the step is a bash script') class TestTheStepSelectsByLeg(unittest.TestCase): """Each leg of ``test`` and ``experimental``, from the tier file, through the step's own script.""" @@ -152,16 +163,20 @@ def setUpClass(cls) -> None: for where in (JOB, EXPERIMENTAL) for leg in legs(*where)} assert len(cls.runs) == len(legs(*JOB)) + len(legs(*EXPERIMENTAL)), 'a version runs in both jobs' - def test_only_latest_runs_the_sweeps(self) -> None: + def test_only_latest_runs_the_latest_tier(self) -> None: + """Every other leg ignores each ``LATEST`` module; ``latest`` ignores the project pins alone.""" self.assertIn(self.latest, self.runs) self.assertTrue(any(is_experimental(leg) for leg, _ in self.runs.values())) + tier, pins = list(_tiers.latest_tier_paths()), list(_tiers.project_pin_paths()) + self.assertLess(set(pins), set(tier)) for version, (_, argv) in self.runs.items(): with self.subTest(python=version): - self.assertEqual(SWEEP_FLAG in pytest_args(argv), version != self.latest) + self.assertEqual(ignored(argv), pins if version == self.latest else tier) + self.assertFalse([arg for arg in argv if 'roundtrip' in arg and arg.startswith('--ignore-glob')]) def test_latest_keeps_the_unit_tier_less_the_pins_under_coverage(self) -> None: - """Coverage is measured over the selection it was before #1732, less the project pins, - which import no pcapkit and run in ``project-tests`` (#1954).""" + """Coverage is measured over the whole unit tier, less the project pins, which + import no pcapkit and run in ``project-tests`` (#1954).""" leg, argv = self.runs[self.latest] self.assertIs(leg.get('coverage'), True) self.assertEqual(argv[:4], ['-m', 'coverage', 'run', '--rcfile=.github/coverage.toml']) @@ -171,8 +186,10 @@ def test_latest_keeps_the_unit_tier_less_the_pins_under_coverage(self) -> None: self.assertEqual(flags, list(TIER_FLAGS) + pins) def test_every_other_leg_runs_the_rest(self) -> None: - """The same selection as ``latest``, the sweeps' glob last, and no coverage.""" - expected = pytest_args(self.runs[self.latest][1]) + [SWEEP_FLAG] + """The same selection as ``latest``, the ``LATEST`` tier ignored in place of the pins, and no coverage.""" + pins = {f'--ignore={pin}' for pin in _tiers.project_pin_paths()} + expected = ([arg for arg in pytest_args(self.runs[self.latest][1]) if arg not in pins] + + [f'--ignore={module}' for module in _tiers.latest_tier_paths()]) for version, (leg, argv) in self.runs.items(): if version == self.latest: continue @@ -187,9 +204,18 @@ def test_every_other_leg_runs_the_rest(self) -> None: self.assertEqual(argv[:2], ['-m', 'pytest']) self.assertEqual(pytest_args(argv), want) + def test_a_step_that_ignores_the_pins_alone_off_latest_is_caught(self) -> None: + """The check above fails when a leg off ``latest`` leaves out the pins rather than the tier.""" + script = step(JOB)['run'] + latest = _tiers.LATEST_TIER_SELECTOR.split('"', 1)[1] + self.assertEqual(script.count(latest), 1) + doctored = script.replace(latest, _tiers.PROJECT_PINS_SELECTOR.split('"', 1)[1]) + leg = next(leg for leg in legs(*JOB) if leg['python-version'] != self.latest) + self.assertNotEqual(ignored(run_step(leg, JOB, doctored)), list(_tiers.latest_tier_paths())) + -class TestTheGlobIsTheSweeps(unittest.TestCase): - """What the glob leaves out, by pytest's own collection.""" +class TestTheIgnoresAreTheLatestTier(unittest.TestCase): + """What the ``LATEST`` ignores leave out, by pytest's own collection.""" def collect(self, *extra: 'str') -> 'set[str]': result = subprocess.run([sys.executable, '-m', 'pytest', '--collect-only', '-q', '-p', 'no:cacheprovider', @@ -198,41 +224,45 @@ def collect(self, *extra: 'str') -> 'set[str]': self.assertEqual(result.returncode, 0, result.stdout[-2000:] + result.stderr[-2000:]) return {line.split('::', 1)[0] for line in result.stdout.splitlines() if '::' in line} - def test_pytest_ignores_exactly_the_sweeps(self) -> None: - """Run from the root, as CI runs it: the modules the glob drops are :func:`~tests._tiers.is_roundtrip_sweep`'s.""" + def test_pytest_leaves_out_exactly_the_latest_modules(self) -> None: + """Run from the root, as CI runs it: the modules the ignores drop are the ``LATEST`` ones.""" everything = self.collect() - rest = self.collect(SWEEP_FLAG) - sweeps = {module for module in everything if _tiers.is_roundtrip_sweep(module)} - self.assertTrue(sweeps) - self.assertEqual(everything - rest, sweeps) - # A module that imports a sweep's harness for its packet builders is no sweep. + rest = self.collect(*(f'--ignore={module}' for module in _tiers.latest_tier_paths())) + latest = {module for module in everything if _tiers.module_tier(module) == _tiers.TIER_LATEST} + self.assertTrue(latest) + self.assertEqual(everything - rest, latest) + # A hand-picked round trip stays, and so does a module that imports a sweep's harness. + self.assertTrue([module for module in rest if 'roundtrip' in module]) builders = [module for module in rest if 'tests.foundation import _roundtrip' in (ROOT / module).read_text(encoding='utf-8')] - self.assertTrue(builders, f'no module under {PROBE} imports the harness without being a sweep') - - def test_the_sweeps_are_the_modules_named_for_a_round_trip(self) -> None: - """Every unit-tier test module whose name holds ``roundtrip``. - - That takes in every module that builds a table on the #1202 protocol-edge - or corekit harness. The foundation harness's registry module is named for - symmetry, so it runs on every leg; it took 1.8 s on 3.14 (measured). - - """ - modules = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / 'tests').rglob('test_*.py')) - sweeps = [module for module in modules if _tiers.is_roundtrip_sweep(module)] - self.assertEqual(sweeps, [module for module in modules - if 'roundtrip' in module.rsplit('/', 1)[1] and _tiers.is_unit_tier(module)]) + self.assertTrue(builders, f'no module under {PROBE} imports the harness without being LATEST') + + def test_the_tier_is_no_name_rule(self) -> None: + """#1958: a name says nothing about the tier; ``LATEST`` sweeps go by other names too.""" + modules = _tiers.suite_modules() + named = [module for module in modules if 'roundtrip' in module.rsplit('/', 1)[1]] + latest = set(_tiers.latest_tier_paths()) + self.assertTrue([module for module in named if module not in latest]) + self.assertTrue([module for module in named if module in latest]) + self.assertTrue([module for module in latest if module not in named and 'tests/project/' not in module]) + + def test_every_module_on_a_sweep_harness_is_latest(self) -> None: + """The #1202 edge-case and corekit tables build every case they list, so their users are sweeps.""" + modules = _tiers.suite_modules() for harness in ('tests/protocols/_edge_roundtrip.py', 'tests/corekit/_roundtrip.py'): users = [module for module in modules if _tiers.is_unit_tier(module) and re.search( rf'^from {re.escape(harness[:-3].replace("/", "."))} import .*\b(?:Case|KnownFailureTable)\b', (ROOT / module).read_text(encoding='utf-8'), re.M)] with self.subTest(harness=harness): self.assertTrue(users) - self.assertEqual([module for module in users if module not in sweeps], []) + self.assertEqual([module for module in users if _tiers.module_tier(module) != _tiers.TIER_LATEST], + []) +@unittest.skipIf(shutil.which('bash') is None, 'the step is a bash script') class TestTheIntegrationSubset(unittest.TestCase): - """#1732: ``integration`` runs the fixture tier's round-trip sweeps on ``latest`` alone. + """#1958: ``integration`` runs the fixture tier's ``LATEST`` modules on ``latest`` alone, and + ``pypcap-parity``, with no ``latest`` leg, never does. Each leg from the tier file, through the step's own script, with tests/_tiers.py really asked for the selection. @@ -243,6 +273,7 @@ class TestTheIntegrationSubset(unittest.TestCase): def setUpClass(cls) -> None: cls.latest = live_outputs()['latest'] cls.selection = list(_tiers.fixture_tier_paths()) + cls.all = list(_tiers.fixture_tier_paths(_tiers.TIER_ALL)) cls.runs = {leg['python-version']: pytest_args(run_step(leg, INTEGRATION)) for leg in legs(*INTEGRATION)} @staticmethod @@ -250,28 +281,37 @@ def flags() -> 'list[str]': return ['-q', '-n', 'auto', '--dist', 'load', '--durations=30', '--durations-min=1.0', f'--junitxml={RUNNER_TEMP}/junit.xml'] - def test_the_glob_names_the_three_runtime_sweeps(self) -> None: - """On disk and in the selection: the three modules the ruling counted.""" - glob = _tiers.ROUNDTRIP_RUNTIME_GLOB - on_disk = sorted(path.relative_to(ROOT).as_posix() for path in (ROOT / 'tests').rglob('test_*.py') - if fnmatch.fnmatch(path.relative_to(ROOT).as_posix(), glob)) - self.assertEqual(on_disk, list(RUNTIME_SWEEPS)) - self.assertEqual(sorted({arg.split('::', 1)[0] for arg in self.selection - if fnmatch.fnmatch(arg.split('::', 1)[0], glob)}), list(RUNTIME_SWEEPS)) + def test_the_all_selection_leaves_out_the_latest_modules(self) -> None: + latest = [module for module in _tiers.latest_tier_paths() if not _tiers.is_unit_tier(module)] + self.assertTrue(latest) + modules = [{entry.split('::', 1)[0] for entry in entries} for entries in (self.selection, self.all)] + self.assertEqual(sorted(modules[0] - modules[1]), latest) def test_only_latest_runs_them(self) -> None: - """``latest`` passes the whole selection; every other leg drops them and adds the glob.""" + """``latest`` passes the whole selection; every other leg its ``ALL`` part.""" self.assertIn(self.latest, self.runs) self.assertGreater(len(self.runs), 1) - flag = f'--ignore-glob={_tiers.ROUNDTRIP_RUNTIME_GLOB}' - rest = [arg for arg in self.selection if arg.split('::', 1)[0] not in RUNTIME_SWEEPS] - self.assertLess(len(rest), len(self.selection)) + self.assertLess(len(self.all), len(self.selection)) for version, args in self.runs.items(): with self.subTest(python=version): - if version == self.latest: - self.assertEqual(args, self.flags() + self.selection) - else: - self.assertEqual(args, self.flags() + rest + [flag]) + self.assertEqual(args, self.flags() + (self.selection if version == self.latest else self.all)) + + def test_a_leg_that_asks_for_every_tier_off_latest_is_caught(self) -> None: + script = step(INTEGRATION)['run'] + self.assertEqual(script.count("fixture_tier_paths('ALL')"), 1) + leg = next(leg for leg in legs(*INTEGRATION) if leg['python-version'] != self.latest) + args = pytest_args(run_step(leg, INTEGRATION, script.replace("fixture_tier_paths('ALL')", + 'fixture_tier_paths()'))) + self.assertNotEqual(args, self.flags() + self.all) + + def test_the_parity_legs_run_the_all_tier(self) -> None: + """None of ``pypcap-parity``'s legs is ``latest``, so each asks for the ``ALL`` part alone.""" + found = legs(*PARITY) + legs('unit-tests.yml', 'pypcap-parity-ktlo') + self.assertTrue(found) + self.assertNotIn(self.latest, [leg['python-version'] for leg in found]) + self.assertIn(_tiers.FIXTURE_ALL_TIER_SELECTOR, step(PARITY)['run']) + self.assertEqual(pytest_args(run_step(legs(*PARITY)[0], PARITY)), + ['-q', '-n', 'auto', '--dist', 'load'] + self.all) def triggers(spec: 'dict[str, Any]') -> 'dict[str, Any]': @@ -411,9 +451,13 @@ def test_no_required_context_names_a_leg(self) -> None: class TestTheDocsSayIt(unittest.TestCase): - def test_the_workflows_page_names_the_glob_and_the_leg(self) -> None: + def test_the_workflows_page_states_the_tiers(self) -> None: page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') - self.assertIn(f"every leg but ``latest`` adds ``--ignore-glob='{_tiers.ROUNDTRIP_SWEEP_GLOB}'``", page) + for phrase in ('``module_tier()``', "``__tier__ = 'LATEST'``", '``latest_tier_paths()``', "``fixture_tier_paths('ALL')``", + '``ALL``', '``LATEST``', '#1958'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, page) + self.assertNotIn('roundtrip*', page) if __name__ == '__main__': diff --git a/tests/protocols/application/test_application_edge_roundtrip_unit.py b/tests/protocols/application/test_application_edge_roundtrip_unit.py index 71ea8b7d11..46345e6fd2 100644 --- a/tests/protocols/application/test_application_edge_roundtrip_unit.py +++ b/tests/protocols/application/test_application_edge_roundtrip_unit.py @@ -20,6 +20,10 @@ from tests.protocols import _edge_roundtrip as edge from tests.protocols._edge_roundtrip import Case, MakeCase +#: Test tier (#1958): the #1202 edge-case table for the application layer, every case +#: of it. +__tier__ = 'LATEST' + FTP = 'pcapkit.protocols.application.ftp:FTP' HTTP1 = 'pcapkit.protocols.application.httpv1:HTTP' HTTP2 = 'pcapkit.protocols.application.httpv2:HTTP' diff --git a/tests/protocols/application/test_ftp_http1_roundtrip_unit.py b/tests/protocols/application/test_ftp_http1_roundtrip_unit.py index 5a539fef94..80c0c02097 100644 --- a/tests/protocols/application/test_ftp_http1_roundtrip_unit.py +++ b/tests/protocols/application/test_ftp_http1_roundtrip_unit.py @@ -22,6 +22,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/application/test_ftp_http_unassigned_code_1443_unit.py b/tests/protocols/application/test_ftp_http_unassigned_code_1443_unit.py index 64f23f598c..3c7f1365ae 100644 --- a/tests/protocols/application/test_ftp_http_unassigned_code_1443_unit.py +++ b/tests/protocols/application/test_ftp_http_unassigned_code_1443_unit.py @@ -21,6 +21,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): a sweep that rebuilds every three-digit HTTP status code. +__tier__ = 'LATEST' + class TestFTPReturnCodeEveryValue(unittest.TestCase): """Pin FTP replies across the whole three-digit range.""" diff --git a/tests/protocols/application/test_http2_roundtrip_unit.py b/tests/protocols/application/test_http2_roundtrip_unit.py index 64811cb613..7cb0e1b64b 100644 --- a/tests/protocols/application/test_http2_roundtrip_unit.py +++ b/tests/protocols/application/test_http2_roundtrip_unit.py @@ -24,6 +24,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' SETTINGS_ACK = bytes.fromhex('000000040100000000') diff --git a/tests/protocols/application/test_ospf_auth_dict_roundtrip_unit.py b/tests/protocols/application/test_ospf_auth_dict_roundtrip_unit.py index 31847ab72e..ddff12303d 100644 --- a/tests/protocols/application/test_ospf_auth_dict_roundtrip_unit.py +++ b/tests/protocols/application/test_ospf_auth_dict_roundtrip_unit.py @@ -20,6 +20,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/application/test_ospf_ngap_roundtrip_unit.py b/tests/protocols/application/test_ospf_ngap_roundtrip_unit.py index ee16b6e468..6774cef5f6 100644 --- a/tests/protocols/application/test_ospf_ngap_roundtrip_unit.py +++ b/tests/protocols/application/test_ospf_ngap_roundtrip_unit.py @@ -23,6 +23,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + HAS_PYCRATE = importlib.util.find_spec('pycrate_asn1dir') is not None #: Cryptographic authentication: Key ID 1, Auth Data Len 16, sequence 7. diff --git a/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py b/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py index c8141a1dba..8636353850 100644 --- a/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py +++ b/tests/protocols/internet/test_ext_chain_limit_rebuild_1619_unit.py @@ -27,6 +27,9 @@ from tests._support import reimport_once_per_class from tests.foundation import _roundtrip as wire +#: Test tier (#1958): hand-picked chain lengths for #1619, not a sweep. +__tier__ = 'ALL' + #: The TCP segment every chain ends in. SEGMENT = wire.tcp(b'payload', seq=1) diff --git a/tests/protocols/internet/test_hip_roundtrip_unit.py b/tests/protocols/internet/test_hip_roundtrip_unit.py index 25e477f67f..16d47282c7 100644 --- a/tests/protocols/internet/test_hip_roundtrip_unit.py +++ b/tests/protocols/internet/test_hip_roundtrip_unit.py @@ -25,6 +25,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + def packet(params: bytes = b'', controls: bytes = b'\x00\x00') -> bytes: """Return a HIPv2 I1 packet carrying ``params`` (already 8-octet aligned).""" diff --git a/tests/protocols/internet/test_internet_edge_roundtrip_unit.py b/tests/protocols/internet/test_internet_edge_roundtrip_unit.py index 79e0dfdd72..696c496643 100644 --- a/tests/protocols/internet/test_internet_edge_roundtrip_unit.py +++ b/tests/protocols/internet/test_internet_edge_roundtrip_unit.py @@ -20,6 +20,10 @@ from tests.protocols import _edge_roundtrip as edge from tests.protocols._edge_roundtrip import Case, MakeCase, Reject +#: Test tier (#1958): the #1202 edge-case table for the internet layer, every case of +#: it. +__tier__ = 'LATEST' + IPV4 = 'pcapkit.protocols.internet.ipv4:IPv4' IPX = 'pcapkit.protocols.internet.ipx:IPX' IPV6 = 'pcapkit.protocols.internet.ipv6:IPv6' diff --git a/tests/protocols/internet/test_ipv4_roundtrip_unit.py b/tests/protocols/internet/test_ipv4_roundtrip_unit.py index a4aa99ba4a..923aa1ac21 100644 --- a/tests/protocols/internet/test_ipv4_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv4_roundtrip_unit.py @@ -31,6 +31,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + def header(options: 'str' = '', flags: 'str' = '0000') -> 'bytes': """An IPv4 header from 10.0.0.1 to 10.0.0.2, protocol 253, with ``options`` in hex.""" diff --git a/tests/protocols/internet/test_ipv4_ts_sec_roundtrip_unit.py b/tests/protocols/internet/test_ipv4_ts_sec_roundtrip_unit.py index a275a1a78d..f3048b9bde 100644 --- a/tests/protocols/internet/test_ipv4_ts_sec_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv4_ts_sec_roundtrip_unit.py @@ -22,6 +22,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + def header(options: 'str') -> 'bytes': """An IPv4 header from 10.0.0.1 to 10.0.0.2, protocol 253, with ``options`` in hex.""" diff --git a/tests/protocols/internet/test_ipv4_tso_total_length_1547_runtime.py b/tests/protocols/internet/test_ipv4_tso_total_length_1547_runtime.py index ab127708d6..935e8af81b 100644 --- a/tests/protocols/internet/test_ipv4_tso_total_length_1547_runtime.py +++ b/tests/protocols/internet/test_ipv4_tso_total_length_1547_runtime.py @@ -34,6 +34,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): one capture through each engine, whose releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) ENGINES = tuple(name for name in ('dpkt', 'scapy') if importlib.util.find_spec(name) is not None) diff --git a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py index 2077f7e3f6..6afa2f4ad3 100644 --- a/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv6_core_roundtrip_unit.py @@ -23,6 +23,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + def ipv6(nxt: int, payload: bytes) -> bytes: """Return an IPv6 header with next header ``nxt``, followed by ``payload``.""" diff --git a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py index f1f169a6d8..ba3124476d 100644 --- a/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py +++ b/tests/protocols/internet/test_ipv6_exthdr_cut_rebuild_unit.py @@ -24,6 +24,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): a sweep that rebuilds every cut of each extension header. +__tier__ = 'LATEST' + HOP, TCP, DST, AH, NONXT, HIP = 0, 6, 60, 51, 59, 139 #: A TCP header and one octet of data. diff --git a/tests/protocols/internet/test_ipv6_options_roundtrip_unit.py b/tests/protocols/internet/test_ipv6_options_roundtrip_unit.py index aebedbe704..d0864ac1b0 100644 --- a/tests/protocols/internet/test_ipv6_options_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv6_options_roundtrip_unit.py @@ -33,6 +33,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: The protocol classes sharing the option code, by module and class name. PROTOCOLS = ( ('pcapkit.protocols.internet.hopopt', 'HOPOPT'), diff --git a/tests/protocols/internet/test_ipv6_route_rpl_roundtrip_unit.py b/tests/protocols/internet/test_ipv6_route_rpl_roundtrip_unit.py index 7731430764..fe5ec5caef 100644 --- a/tests/protocols/internet/test_ipv6_route_rpl_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipv6_route_rpl_roundtrip_unit.py @@ -27,6 +27,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): a sweep over every cut and every compression of an RPL route. +__tier__ = 'LATEST' + SRC = ipaddress.IPv6Address('2001:db8::1').packed DST = ipaddress.IPv6Address('2001:db8:aa:bb:cc:dd:ee:ff').packed diff --git a/tests/protocols/internet/test_ipx_ah_roundtrip_unit.py b/tests/protocols/internet/test_ipx_ah_roundtrip_unit.py index fec4962a67..dc9dda11a0 100644 --- a/tests/protocols/internet/test_ipx_ah_roundtrip_unit.py +++ b/tests/protocols/internet/test_ipx_ah_roundtrip_unit.py @@ -25,6 +25,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: Checksum, length, hop count, type, destination and source, then payload. IPX_PACKETS = { 'all-zero': bytes.fromhex('0000' '001e') + bytes(26), diff --git a/tests/protocols/internet/test_mh_roundtrip_unit.py b/tests/protocols/internet/test_mh_roundtrip_unit.py index dbde5fe1a9..6fb11dd3b0 100644 --- a/tests/protocols/internet/test_mh_roundtrip_unit.py +++ b/tests/protocols/internet/test_mh_roundtrip_unit.py @@ -24,6 +24,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: Mobility Header octets, parsed as an IPv6 extension header (next header 59). FRAMES = { # BRR carrying a PadN of Option Length 0, then a PadN of Option Length 4 diff --git a/tests/protocols/link/test_arp_roundtrip_unit.py b/tests/protocols/link/test_arp_roundtrip_unit.py index 938dcdfb7d..c13deb3e9c 100644 --- a/tests/protocols/link/test_arp_roundtrip_unit.py +++ b/tests/protocols/link/test_arp_roundtrip_unit.py @@ -28,6 +28,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: ``(module, class)`` for every ARP-family class. CLASSES = ( ('pcapkit.protocols.link.arp', 'ARP'), diff --git a/tests/protocols/link/test_ethernet_mac_roundtrip_unit.py b/tests/protocols/link/test_ethernet_mac_roundtrip_unit.py index ed337c3573..82811ba6d8 100644 --- a/tests/protocols/link/test_ethernet_mac_roundtrip_unit.py +++ b/tests/protocols/link/test_ethernet_mac_roundtrip_unit.py @@ -23,6 +23,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: Destination MAC, source MAC, EtherType (IPv4) and a short payload. FRAMES = { 'nontrivial': bytes.fromhex('0123456789ab' 'fedcba987654' '0800') + b'payload', diff --git a/tests/protocols/link/test_l2tpv2_length_roundtrip_unit.py b/tests/protocols/link/test_l2tpv2_length_roundtrip_unit.py index 2c981cb697..aa0fc49c94 100644 --- a/tests/protocols/link/test_l2tpv2_length_roundtrip_unit.py +++ b/tests/protocols/link/test_l2tpv2_length_roundtrip_unit.py @@ -23,6 +23,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: A PPP frame header, as the L2TPv2 payload. PAYLOAD = bytes.fromhex('ff030021') diff --git a/tests/protocols/link/test_l2tpv2_roundtrip_unit.py b/tests/protocols/link/test_l2tpv2_roundtrip_unit.py index 62b4c3ecd2..0e43057576 100644 --- a/tests/protocols/link/test_l2tpv2_roundtrip_unit.py +++ b/tests/protocols/link/test_l2tpv2_roundtrip_unit.py @@ -24,6 +24,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: A PPP frame header and the first IPv4 octet, as the L2TPv2 payload. PAYLOAD = bytes.fromhex('ff03002145') diff --git a/tests/protocols/link/test_link_edge_roundtrip_unit.py b/tests/protocols/link/test_link_edge_roundtrip_unit.py index 9de6fd1471..489b721a01 100644 --- a/tests/protocols/link/test_link_edge_roundtrip_unit.py +++ b/tests/protocols/link/test_link_edge_roundtrip_unit.py @@ -18,6 +18,9 @@ from tests.protocols import _edge_roundtrip as edge from tests.protocols._edge_roundtrip import Case, MakeCase, Reject +#: Test tier (#1958): the #1202 edge-case table for the link layer, every case of it. +__tier__ = 'LATEST' + ETHERNET = 'pcapkit.protocols.link.ethernet:Ethernet' C_TAG = 'pcapkit.protocols.link.c_tag:C_Tag' S_TAG = 'pcapkit.protocols.link.s_tag:S_Tag' diff --git a/tests/protocols/misc/test_misc_short_read_1470_runtime.py b/tests/protocols/misc/test_misc_short_read_1470_runtime.py index c81dc5e999..252d21d1e8 100644 --- a/tests/protocols/misc/test_misc_short_read_1470_runtime.py +++ b/tests/protocols/misc/test_misc_short_read_1470_runtime.py @@ -33,6 +33,9 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): a sweep that cuts every generated capture at every offset. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('aenum', 'chardet') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/misc/test_pcapng_roundtrip2_unit.py b/tests/protocols/misc/test_pcapng_roundtrip2_unit.py index 1cf4a766e9..619fa40fec 100644 --- a/tests/protocols/misc/test_pcapng_roundtrip2_unit.py +++ b/tests/protocols/misc/test_pcapng_roundtrip2_unit.py @@ -26,6 +26,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + H = bytes.fromhex diff --git a/tests/protocols/misc/test_pcapng_roundtrip_unit.py b/tests/protocols/misc/test_pcapng_roundtrip_unit.py index 131f5ee31b..ea97b17eaa 100644 --- a/tests/protocols/misc/test_pcapng_roundtrip_unit.py +++ b/tests/protocols/misc/test_pcapng_roundtrip_unit.py @@ -32,6 +32,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: A little-endian Enhanced Packet Block on interface 0, timestamp 0, carrying #: the four octets ``abcd`` -- the repro of #1270. EPB = bytes.fromhex('06000000 24000000 00000000 00000000 00000000' diff --git a/tests/protocols/schema/test_bitfield_coverage_unit.py b/tests/protocols/schema/test_bitfield_coverage_unit.py index 0d418004b6..4de86add31 100644 --- a/tests/protocols/schema/test_bitfield_coverage_unit.py +++ b/tests/protocols/schema/test_bitfield_coverage_unit.py @@ -38,6 +38,9 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): a sweep over every bit field of the protocol schemas. +__tier__ = 'LATEST' + #: Bit fields allowed to leave bits unnamed, as ``schema.field`` (relative to #: :mod:`pcapkit.protocols.schema`) -> the ``(start, length)`` bit ranges left #: uncovered. Every known site has been fixed, so this is empty, and it must diff --git a/tests/protocols/schema/test_schema_roundtrip_unit.py b/tests/protocols/schema/test_schema_roundtrip_unit.py index d7b90239f0..e02ab6ebaa 100644 --- a/tests/protocols/schema/test_schema_roundtrip_unit.py +++ b/tests/protocols/schema/test_schema_roundtrip_unit.py @@ -24,6 +24,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: TCP header, Data Offset 6: NOP, EOOL, then two octets of padding. TCP_HEADER = bytes.fromhex('0050c3500000000100000000' '6002ffff00000000' '01000000') #: IPv4 header, IHL 6: NOP, EOOL, then two octets of padding. diff --git a/tests/protocols/test_capture_roundtrip_runtime.py b/tests/protocols/test_capture_roundtrip_runtime.py index a23b567d1c..cae3444d7f 100644 --- a/tests/protocols/test_capture_roundtrip_runtime.py +++ b/tests/protocols/test_capture_roundtrip_runtime.py @@ -38,6 +38,9 @@ if TYPE_CHECKING: from typing import Any, Iterator, Optional +#: Test tier (#1958): a sweep over every layer of every sample capture. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_declared_length_roundtrip_runtime.py b/tests/protocols/test_declared_length_roundtrip_runtime.py index 24eef94440..fac0d56830 100644 --- a/tests/protocols/test_declared_length_roundtrip_runtime.py +++ b/tests/protocols/test_declared_length_roundtrip_runtime.py @@ -23,6 +23,9 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): hand-picked cases on one truncated fixture, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_dispatch_reachability_unit.py b/tests/protocols/test_dispatch_reachability_unit.py index fac497e440..ebdbe4a69e 100644 --- a/tests/protocols/test_dispatch_reachability_unit.py +++ b/tests/protocols/test_dispatch_reachability_unit.py @@ -41,6 +41,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): a sweep over every indexed class, under its code. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_dispatch_registry_unit.py b/tests/protocols/test_dispatch_registry_unit.py index cf60d7f792..8a2966c885 100644 --- a/tests/protocols/test_dispatch_registry_unit.py +++ b/tests/protocols/test_dispatch_registry_unit.py @@ -40,6 +40,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): a sweep over every ``__proto__`` registry entry. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_edge_roundtrip_deadline_1736_unit.py b/tests/protocols/test_edge_roundtrip_deadline_1736_unit.py index f62cd92a0d..e720c7703a 100644 --- a/tests/protocols/test_edge_roundtrip_deadline_1736_unit.py +++ b/tests/protocols/test_edge_roundtrip_deadline_1736_unit.py @@ -27,6 +27,10 @@ if TYPE_CHECKING: from typing import Iterator +#: Test tier (#1958): tests the deadline of the edge-case harness, which only the +#: edge-case sweeps use. +__tier__ = 'LATEST' + ETHERNET = 'pcapkit.protocols.link.ethernet:Ethernet' MACS = bytes.fromhex('0123456789ab') + bytes.fromhex('fedcba987654') diff --git a/tests/protocols/test_option_coverage_runtime.py b/tests/protocols/test_option_coverage_runtime.py index 7480c1527b..6c511312e3 100644 --- a/tests/protocols/test_option_coverage_runtime.py +++ b/tests/protocols/test_option_coverage_runtime.py @@ -41,6 +41,10 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): a sweep that extracts and dumps every ``options-*.pcap`` +#: fixture. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_option_generator_1337_unit.py b/tests/protocols/test_option_generator_1337_unit.py index 654c1798dd..6d870ac3b4 100644 --- a/tests/protocols/test_option_generator_1337_unit.py +++ b/tests/protocols/test_option_generator_1337_unit.py @@ -27,6 +27,10 @@ from tests._support import time_limit from tests._tiers import ROOT +#: Test tier (#1958): tests examples/generators/options.py, which make_samples.py runs +#: on every integration leg. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_option_generator_httpv2_stream_unit.py b/tests/protocols/test_option_generator_httpv2_stream_unit.py index 07cdbbf44c..212653a5a2 100644 --- a/tests/protocols/test_option_generator_httpv2_stream_unit.py +++ b/tests/protocols/test_option_generator_httpv2_stream_unit.py @@ -31,6 +31,10 @@ from tests._support import time_limit from tests._tiers import ROOT +#: Test tier (#1958): tests examples/generators/options.py, which make_samples.py runs +#: on every integration leg. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_option_generator_jumbogram_1687_unit.py b/tests/protocols/test_option_generator_jumbogram_1687_unit.py index 9c9a7c47b8..d4b7907c21 100644 --- a/tests/protocols/test_option_generator_jumbogram_1687_unit.py +++ b/tests/protocols/test_option_generator_jumbogram_1687_unit.py @@ -35,6 +35,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): tests examples/generators/options.py, which make_samples.py runs +#: on every integration leg. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) HAS_SCAPY = importlib.util.find_spec('scapy') is not None diff --git a/tests/protocols/test_option_generator_tcp_base_unit.py b/tests/protocols/test_option_generator_tcp_base_unit.py index 5341db2047..feeb5f66ce 100644 --- a/tests/protocols/test_option_generator_tcp_base_unit.py +++ b/tests/protocols/test_option_generator_tcp_base_unit.py @@ -87,6 +87,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): tests examples/generators/options.py, which make_samples.py runs +#: on every integration leg. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_option_roundtrip_unit.py b/tests/protocols/test_option_roundtrip_unit.py index 8ce6103bc0..bc529e1da7 100644 --- a/tests/protocols/test_option_roundtrip_unit.py +++ b/tests/protocols/test_option_roundtrip_unit.py @@ -92,6 +92,9 @@ if TYPE_CHECKING: from typing import Any, Optional +#: Test tier (#1958): a sweep over every registered option code. +__tier__ = 'LATEST' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_short_read_roundtrip_unit.py b/tests/protocols/test_short_read_roundtrip_unit.py index f508820d55..3ea9ce238a 100644 --- a/tests/protocols/test_short_read_roundtrip_unit.py +++ b/tests/protocols/test_short_read_roundtrip_unit.py @@ -26,6 +26,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): a sweep that cuts every protocol family at every offset. +__tier__ = 'LATEST' + ETHERNET = 'link.ethernet:Ethernet' IPV6 = 'internet.ipv6:IPv6' diff --git a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py index 0ce7c0463f..edc9cfd4e0 100644 --- a/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py +++ b/tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py @@ -23,6 +23,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + MAC = bytes.fromhex('0123456789ab' 'fedcba987654') SRC4, DST4 = bytes([192, 0, 2, 1]), bytes([192, 0, 2, 2]) SRC6 = bytes.fromhex('20010db8' + '00' * 11 + '01') diff --git a/tests/protocols/transport/test_mptcp_roundtrip_unit.py b/tests/protocols/transport/test_mptcp_roundtrip_unit.py index fd1f11ab06..d84dca8914 100644 --- a/tests/protocols/transport/test_mptcp_roundtrip_unit.py +++ b/tests/protocols/transport/test_mptcp_roundtrip_unit.py @@ -29,6 +29,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: TCP header without its offset/flags octets: ports 1234 and 80, sequence 1, #: acknowledgement 2; the window, checksum and urgent pointer follow them. HEAD = bytes.fromhex('04d2' '0050' '00000001' '00000002') diff --git a/tests/protocols/transport/test_sctp_roundtrip_unit.py b/tests/protocols/transport/test_sctp_roundtrip_unit.py index 455f9d0da8..aa24077283 100644 --- a/tests/protocols/transport/test_sctp_roundtrip_unit.py +++ b/tests/protocols/transport/test_sctp_roundtrip_unit.py @@ -16,6 +16,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/transport/test_tcp_ns_flag_roundtrip_unit.py b/tests/protocols/transport/test_tcp_ns_flag_roundtrip_unit.py index c3acd07994..fe55a462f0 100644 --- a/tests/protocols/transport/test_tcp_ns_flag_roundtrip_unit.py +++ b/tests/protocols/transport/test_tcp_ns_flag_roundtrip_unit.py @@ -16,6 +16,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/transport/test_tcp_options_roundtrip_unit.py b/tests/protocols/transport/test_tcp_options_roundtrip_unit.py index 9109a61e69..e3e3331fe2 100644 --- a/tests/protocols/transport/test_tcp_options_roundtrip_unit.py +++ b/tests/protocols/transport/test_tcp_options_roundtrip_unit.py @@ -16,6 +16,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/transport/test_tcp_roundtrip_unit.py b/tests/protocols/transport/test_tcp_roundtrip_unit.py index 48ac84a870..0ba1b7183b 100644 --- a/tests/protocols/transport/test_tcp_roundtrip_unit.py +++ b/tests/protocols/transport/test_tcp_roundtrip_unit.py @@ -28,6 +28,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): hand-picked round-trip cases, not a sweep. +__tier__ = 'ALL' + #: TCP header without its offset/flags octets: ports 1234 and 80, sequence 1, #: acknowledgement 2; the window, checksum and urgent pointer follow them. HEAD = bytes.fromhex('04d2' '0050' '00000001' '00000002') diff --git a/tests/protocols/transport/test_transport_edge_roundtrip_unit.py b/tests/protocols/transport/test_transport_edge_roundtrip_unit.py index e23a676a26..38deca58f4 100644 --- a/tests/protocols/transport/test_transport_edge_roundtrip_unit.py +++ b/tests/protocols/transport/test_transport_edge_roundtrip_unit.py @@ -20,6 +20,10 @@ from tests.protocols import _edge_roundtrip as edge from tests.protocols._edge_roundtrip import Case, MakeCase, Reject +#: Test tier (#1958): the #1202 edge-case table for the transport layer, every case of +#: it. +__tier__ = 'LATEST' + TCP = 'pcapkit.protocols.transport.tcp:TCP' UDP = 'pcapkit.protocols.transport.udp:UDP' SCTP = 'pcapkit.protocols.transport.sctp:SCTP' diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index f9ea743e6c..96d172e0f8 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -28,6 +28,11 @@ and never says so -- #729 and #738 were that defect, and #745 is the ticket for stopping the third instance. +The test tiers of #1958 are checked here too (:class:`TestTierTests`): every test +module resolves to exactly one, ``ALL`` or ``LATEST``, a malformed ``__tier__`` +fails, and :class:`WorkflowAgreementTests` runs the steps that leave the ``LATEST`` +tier out. + This module is itself unit-tier, so it reads no capture at all: the violating modules it needs are written into a temporary directory and audited by path. @@ -48,6 +53,10 @@ from tests import _dependency_gates, _tiers +#: Test tier (#1958): tests/conftest.py runs this guard at collection on every leg, +#: with that leg's own ast module. +__tier__ = 'ALL' + #: The unit-tier workflow job, for :class:`WorkflowAgreementTests`. Absent from a #: source distribution, which is why that test skips rather than fails. WORKFLOW = _tiers.ROOT / '.github' / 'workflows' / 'unit-tests.yml' @@ -191,10 +200,10 @@ def test_ignore_flags_match_the_fixture_tier_constants(self) -> None: The point of failure this catches: somebody adds a fourth fixture-dependent naming convention to the workflow and the guard goes on classifying those modules as unit-tier, flagging their perfectly - legal capture reads. The globs that are not a tier's are - :data:`~tests._tiers.ROUNDTRIP_SWEEP_GLOB` and - :data:`~tests._tiers.ROUNDTRIP_RUNTIME_GLOB`, which only the legs other - than ``latest`` add (#1732), each a constant of its own. + legal capture reads. No other glob is left: the round-trip sweeps + (#1732) are left out by test tier (#1958), module by module, rather + than by a name glob, so a glob here is a tier's or it is unaccounted + for. """ if not WORKFLOW.is_file(): @@ -204,34 +213,9 @@ def test_ignore_flags_match_the_fixture_tier_constants(self) -> None: globs = set(re.findall(r"--ignore-glob='([^']+)'", text)) directories = set(re.findall(r'--ignore=tests/(\S+)', text)) - self.assertEqual(globs, {f'*{suffix}' for suffix in _tiers.FIXTURE_TIER_SUFFIXES} - | {_tiers.ROUNDTRIP_SWEEP_GLOB, _tiers.ROUNDTRIP_RUNTIME_GLOB}) + self.assertEqual(globs, {f'*{suffix}' for suffix in _tiers.FIXTURE_TIER_SUFFIXES}) self.assertEqual(directories, set(_tiers.FIXTURE_TIER_DIRS)) - - def test_a_roundtrip_sweep_stays_in_the_unit_tier(self) -> None: - """#1732: the sweeps are a selection within the unit tier, not a tier. - - So the tier guard keeps auditing them, and ``latest`` runs them; a module - outside the unit tier is no sweep even when its name says round trip. - - """ - cases = { - 'tests/protocols/test_option_roundtrip_unit.py': True, - 'tests/protocols/misc/test_pcapng_roundtrip2_unit.py': True, - 'tests/protocols/test_to_dict_payload_roundtrip_1447_unit.py': True, - 'tests/protocols/test_capture_roundtrip_runtime.py': False, - 'tests/integration/test_roundtrip_end_to_end.py': False, - 'tests/foundation/registry/test_registry_symmetry_unit.py': False, - 'tests/corekit/test_multidict.py': False, - 'examples/roundtrip/test_example.py': False, - } - for relative, expected in cases.items(): - with self.subTest(module=relative): - self.assertIs(_tiers.is_roundtrip_sweep(relative), expected) - if expected: - self.assertTrue(_tiers.is_unit_tier(relative)) - self.assertIs(_tiers.is_roundtrip_sweep(_tiers.ROOT / 'tests/protocols/test_option_roundtrip_unit.py'), - True) + self.assertNotIn('roundtrip', text) def test_integration_job_selects_positively_by_asking_tiers_for_it(self) -> None: """The ``integration`` job's selection runs the other direction, so it @@ -271,15 +255,16 @@ def test_integration_job_selects_positively_by_asking_tiers_for_it(self) -> None self.assertIn('pytest', run_block) @unittest.skipIf(shutil.which('bash') is None, 'the steps are bash scripts') - def test_the_legs_leave_out_exactly_the_pins_project_tests_runs(self) -> None: - """#1954: run as bash runs them, ``test`` ignores each project pin and ``project-tests`` runs each. - - Both steps ask :data:`~tests._tiers.PROJECT_PINS_SELECTOR` at run time, so - they cannot drift from :func:`~tests._tiers.is_project_pin`; what can is - how each uses the answer, which is what running them checks. Off - ``latest`` too, where the sweeps' glob is added after the pins. - experimental.yml asks in a step of its own, under the runner's - ``python3``; its legs run in + def test_the_legs_leave_out_exactly_the_latest_tier_or_the_pins(self) -> None: + """#1958: run as bash runs them, the ``test`` legs but ``latest`` ignore each ``LATEST`` module. + + ``latest`` ignores each project pin (#1954), which ``project-tests`` runs, + and runs the rest of the ``LATEST`` tier itself. Every step asks + :data:`~tests._tiers.LATEST_TIER_SELECTOR` or + :data:`~tests._tiers.PROJECT_PINS_SELECTOR` at run time, so none can drift + from :func:`~tests._tiers.module_tier`; what can is how each uses the + answer, which is what running them checks. experimental.yml asks in a step + of its own, under the runner's ``python3``; its legs run in :file:`tests/project/test_sweeps_on_latest_only_1732.py`. """ @@ -287,36 +272,55 @@ def test_the_legs_leave_out_exactly_the_pins_project_tests_runs(self) -> None: self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') text = WORKFLOW.read_text(encoding='utf-8') pins = list(_tiers.project_pin_paths()) + latest = list(_tiers.latest_tier_paths()) self.assertTrue(pins) - for sweeps in ('true', 'false'): - with self.subTest(sweeps=sweeps): - self.assertEqual(leg_pin_ignores(text, sweeps), pins) + self.assertLess(len(pins), len(latest)) + self.assertEqual(leg_ignores(text, 'true'), pins) + self.assertEqual(leg_ignores(text, 'false'), latest) self.assertEqual(project_pins_run(text), pins) - for workflow, job, step in ((WORKFLOW, 'test', 'Run unit tests'), - (WORKFLOW, 'project-tests', 'Run the project pins'), - (EXPERIMENTAL_WORKFLOW, 'experimental', 'List the project pins')): - with self.subTest(job=job): + for workflow, job, step, selector in ( + (WORKFLOW, 'test', 'Run unit tests', _tiers.PROJECT_PINS_SELECTOR), + (WORKFLOW, 'test', 'Run unit tests', _tiers.LATEST_TIER_SELECTOR), + (WORKFLOW, 'project-tests', 'Run the project pins', _tiers.PROJECT_PINS_SELECTOR), + (EXPERIMENTAL_WORKFLOW, 'experimental', 'List the LATEST tier', + _tiers.LATEST_TIER_SELECTOR.replace('python -c', 'python3 -c'))): + with self.subTest(job=job, selector=selector): script = run_script(job_section(workflow.read_text(encoding='utf-8'), job), step) - self.assertIn(_tiers.PROJECT_PINS_SELECTOR.replace('python -c', 'python3 -c') - if job == 'experimental' else _tiers.PROJECT_PINS_SELECTOR, script) - # The experimental legs ignore what that step printed, as `test` does. + self.assertIn(selector, script) + # The experimental legs ignore what that step printed, as `test`'s legs but `latest` do. tests = step_run_block(job_section(EXPERIMENTAL_WORKFLOW.read_text(encoding='utf-8'), 'experimental'), 'Run unit tests') - self.assertIn('PINS: ${{ steps.pins.outputs.paths }}', tests) - self.assertIn('for pin in $PINS; do\n set -- "$@" --ignore="$pin"', tests) + self.assertIn('LATEST_TIER: ${{ steps.tier.outputs.paths }}', tests) + self.assertIn('for module in $LATEST_TIER; do\n set -- "$@" --ignore="$module"', tests) @unittest.skipIf(shutil.which('bash') is None, 'the steps are bash scripts') - def test_a_leg_that_stops_ignoring_the_pins_is_caught(self) -> None: - """The check above, against a ``test`` step that computes the pins and then drops them.""" + def test_a_leg_that_stops_ignoring_the_latest_tier_is_caught(self) -> None: + """The check above, against a ``test`` step that computes the list and then drops it.""" if not WORKFLOW.is_file(): self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') text = WORKFLOW.read_text(encoding='utf-8') - loop = 'set -- "$@" --ignore="$pin"' + loop = 'set -- "$@" --ignore="$module"' self.assertEqual(text.count(loop), 1, 'the loop this test removes has been reworded') - self.assertEqual(leg_pin_ignores(text.replace(loop, ':'), 'true'), []) + for latest in ('true', 'false'): + with self.subTest(latest=latest): + self.assertEqual(leg_ignores(text.replace(loop, ':'), latest), []) + + @unittest.skipIf(shutil.which('bash') is None, 'the steps are bash scripts') + def test_a_leg_that_asks_for_the_wrong_list_is_caught(self) -> None: + """Off ``latest``, the pins alone are not the ``LATEST`` tier; on it, the tier is not the pins.""" + if not WORKFLOW.is_file(): + self.skipTest(f'{WORKFLOW} is not present, e.g. in a source distribution') + text = WORKFLOW.read_text(encoding='utf-8') + latest = _tiers.LATEST_TIER_SELECTOR.split('"', 1)[1] + pins = _tiers.PROJECT_PINS_SELECTOR.split('"', 1)[1] + self.assertEqual(text.count(latest), 1) + self.assertEqual(text.count(pins), 2) # the `test` step's and `project-tests`'s + self.assertNotEqual(leg_ignores(text.replace(latest, pins), 'false'), list(_tiers.latest_tier_paths())) + swapped = text.replace(pins, latest, 1) + self.assertNotEqual(leg_ignores(swapped, 'true'), list(_tiers.project_pin_paths())) -#: The experimental legs' workflow (#1732), whose unit-tier step leaves the pins out too. +#: The experimental legs' workflow (#1732), whose unit-tier step leaves the ``LATEST`` tier out too. EXPERIMENTAL_WORKFLOW = _tiers.ROOT / '.github' / 'workflows' / 'experimental.yml' #: A ``python`` that writes each argument on a line of ``$ARGS_FILE``, and hands a #: ``-c`` to the real one, which is how the steps ask tests/_tiers.py for the pins. @@ -354,11 +358,12 @@ def stubbed_argv(script: 'str', env: 'dict[str, str]') -> 'list[str]': return record.read_text(encoding='utf-8').splitlines() -def leg_pin_ignores(text: 'str', sweeps: 'str') -> 'list[str]': - """The ``tests/project`` paths a ``test`` leg's ``Run unit tests`` step ``--ignore``\\ s.""" +def leg_ignores(text: 'str', latest: 'str') -> 'list[str]': + """The modules a ``test`` leg's ``Run unit tests`` step ``--ignore``\\ s, ``latest`` or not.""" script = run_script(job_section(text, 'test'), 'Run unit tests') - argv = stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'SWEEPS': sweeps}) - return [arg.split('=', 1)[1] for arg in argv if arg.startswith(f'--ignore=tests/{_tiers.PROJECT_PINS_DIR}/')] + argv = stubbed_argv(script, {'ANNOTATION_LEVEL': 'error', 'COVERAGE': 'false', 'LATEST': latest}) + return [arg.split('=', 1)[1] for arg in argv + if arg.startswith('--ignore=') and arg != f'--ignore=tests/{next(iter(_tiers.FIXTURE_TIER_DIRS))}'] def project_pins_run(text: 'str') -> 'list[str]': @@ -447,7 +452,159 @@ def test_the_pins_are_sorted_paths_the_rule_accepts(self) -> None: self.assertFalse(_tiers.varies_by_interpreter(module)) for module in rest: with self.subTest(module=module): - self.assertTrue(_tiers.varies_by_interpreter(module)) + self.assertTrue(_tiers.varies_by_interpreter(module) or _tiers.module_tier(module) == _tiers.TIER_ALL) + # #1958: every pin is in the LATEST tier, which the other legs leave out. + self.assertEqual(set(pins) - set(_tiers.latest_tier_paths()), set()) + + def test_a_module_that_marks_itself_all_is_no_pin(self) -> None: + """#1958: the marker wins here too, and a ``LATEST`` marker does not make a pin of a module that varies.""" + cases = { + 'text alone': ('TEXT = "pcapkit"\n', True), + 'text, marked ALL': ("TEXT = 'pcapkit'\n__tier__ = 'ALL'\n", False), + 'text, marked LATEST': ("TEXT = 'pcapkit'\n__tier__ = 'LATEST'\n", True), + 'pcapkit, marked LATEST': ("import pcapkit\n__tier__ = 'LATEST'\n", False), + } + for label, (source, expected) in cases.items(): + with self.subTest(case=label), tempfile.TemporaryDirectory(prefix='pcapkit-pin-tier-') as temp: + root = pathlib.Path(temp).resolve() + (root / _tiers.PROJECT_PINS_DIR).mkdir() + path = write_module(root / _tiers.PROJECT_PINS_DIR, 'test_probe.py', source) + with unittest.mock.patch.object(_tiers, 'TESTS_ROOT', root): + self.assertIs(_tiers.is_project_pin(path), expected) + + +class TestTierTests(unittest.TestCase): + """#1958: every test module has exactly one test tier, and the marker is checked.""" + + def tier_of(self, source: 'str', name: 'str' = 'test_probe.py') -> 'str': + """The tier of a module holding ``source``, under a temporary :file:`tests/`.""" + with tempfile.TemporaryDirectory(prefix='pcapkit-test-tier-') as temp: + path = write_module(pathlib.Path(temp), name, source) + with unittest.mock.patch.object(_tiers, 'TESTS_ROOT', pathlib.Path(temp).resolve()): + return _tiers.module_tier(path) + + def test_every_module_resolves_to_exactly_one_tier(self) -> None: + """The two tiers partition the test modules, and the ``LATEST`` list is the one tier.""" + modules = _tiers.suite_modules() + self.assertEqual(list(modules), sorted(set(modules))) + tiers = {} # type: dict[str, list[str]] + for module in modules: + with self.subTest(module=module): + tier = _tiers.module_tier(module) + self.assertIn(tier, _tiers.TIERS) + self.assertEqual(_tiers.module_tier(_tiers.ROOT / module), tier) + tiers.setdefault(tier, []).append(module) + self.assertEqual(set(tiers), set(_tiers.TIERS)) + self.assertEqual(list(_tiers.latest_tier_paths()), tiers[_tiers.TIER_LATEST]) + self.assertEqual(sorted(tiers[_tiers.TIER_ALL] + tiers[_tiers.TIER_LATEST]), list(modules)) + found = sorted(path.relative_to(_tiers.ROOT).as_posix() for path in _tiers.TESTS_ROOT.rglob('test_*.py')) + self.assertEqual(list(modules), found) + + def test_a_marker_wins_over_the_rule(self) -> None: + cases = { + 'the rule, exercising pcapkit': ('import pcapkit\n', _tiers.TIER_ALL), + 'the rule, reading text': ('TEXT = "pcapkit"\n', _tiers.TIER_LATEST), + 'a sweep, marked': ("import pcapkit\n__tier__ = 'LATEST'\n", _tiers.TIER_LATEST), + 'an interpreter-bound guard, marked': ("import ast\n__tier__ = 'ALL'\n", _tiers.TIER_ALL), + 'annotated': ("import pcapkit\n__tier__: str = 'LATEST'\n", _tiers.TIER_LATEST), + } + for label, (source, expected) in cases.items(): + with self.subTest(case=label): + self.assertEqual(self.tier_of(source), expected) + + def test_an_invalid_marker_fails(self) -> None: + cases = { + 'not a tier': "__tier__ = 'PERIODIC'\n", + 'lower case': "__tier__ = 'latest'\n", + 'not a string': '__tier__ = 1\n', + 'not a literal': "TIER = 'ALL'\n__tier__ = TIER\n", + 'assigned twice': "__tier__ = 'ALL'\n__tier__ = 'LATEST'\n", + 'assigned conditionally': "import sys\nif sys.version_info < (3, 12):\n __tier__ = 'ALL'\n", + 'assigned in a function': "def test():\n __tier__ = 'ALL'\n", + 'annotated without a value': '__tier__: str\n', + } + for label, source in cases.items(): + with self.subTest(case=label): + with self.assertRaises(_tiers.InvalidTierError) as caught: + self.tier_of(source) + self.assertIn('test_probe.py', str(caught.exception)) + self.assertIn(_tiers.TIER_MARKER, str(caught.exception)) + + def test_only_a_test_module_has_a_tier(self) -> None: + for path in ('tests/_support.py', 'tests/_tiers.py', 'util/ci_timing.py', 'tests/project', + 'tests/project/test_no_such_module.py'): + with self.subTest(path=path): + with self.assertRaises(ValueError): + _tiers.module_tier(path) + + def test_every_marker_states_a_reason(self) -> None: + """A ``__tier__`` the rule would also give says why it is there, as one that overrules it does.""" + for module in _tiers.suite_modules(): + if _tiers.declared_tier(module) is None: + continue + lines = (_tiers.ROOT / module).read_text(encoding='utf-8').splitlines() + index = next(number for number, line in enumerate(lines) if line.startswith(_tiers.TIER_MARKER)) + with self.subTest(module=module): + self.assertTrue(lines[index - 1].startswith('#: '), lines[index - 1]) + comment = [] + for line in reversed(lines[:index]): + if not line.startswith('#: '): + break + comment.insert(0, line) + self.assertTrue(comment[0].startswith('#: Test tier (#1958): '), comment[0]) + + def test_the_sweeps_and_the_pins_are_latest_and_the_hand_picked_round_trips_are_not(self) -> None: + """The owner's rulings on #1958, paraphrased: hand-picked round trips run on every leg, sweeps on ``latest``.""" + for module, tier in { + 'tests/protocols/test_option_roundtrip_unit.py': _tiers.TIER_LATEST, + 'tests/protocols/test_capture_roundtrip_runtime.py': _tiers.TIER_LATEST, + 'tests/protocols/test_dispatch_reachability_unit.py': _tiers.TIER_LATEST, + 'tests/protocols/transport/test_tcp_ns_flag_roundtrip_unit.py': _tiers.TIER_ALL, + 'tests/corekit/test_from_dict_roundtrip_unit.py': _tiers.TIER_ALL, + 'tests/project/test_isort_clean.py': _tiers.TIER_LATEST, + 'tests/project/test_public_api.py': _tiers.TIER_ALL, + 'tests/test_tier_guard.py': _tiers.TIER_ALL, + }.items(): + with self.subTest(module=module): + self.assertEqual(_tiers.module_tier(module), tier) + self.assertEqual(set(_tiers.project_pin_paths()) - set(_tiers.latest_tier_paths()), set()) + + def test_the_fixture_selection_splits_by_tier(self) -> None: + """``fixture_tier_paths('ALL')`` is the whole selection less its ``LATEST`` entries.""" + everything = _tiers.fixture_tier_paths() + selected = _tiers.fixture_tier_paths(_tiers.TIER_ALL) + latest = set(_tiers.latest_tier_paths()) + self.assertEqual(list(selected), sorted(set(selected))) + dropped = [entry for entry in everything if entry not in selected] + self.assertTrue(dropped) + for entry in dropped: + with self.subTest(entry=entry): + module = entry.partition('::')[0] + self.assertTrue(module in latest or (_tiers.ROOT / module).is_dir()) + for entry in selected: + with self.subTest(entry=entry): + module = entry.partition('::')[0] + self.assertNotIn(module, latest) + self.assertTrue(entry in everything or any(module.startswith(other.rstrip('/') + '/') + for other in everything)) + with self.assertRaises(ValueError): + _tiers.fixture_tier_paths('NIGHTLY') + + def test_a_directory_holding_a_latest_module_is_split(self) -> None: + """pytest ignores no path it was handed, so the ``ALL`` selection names a mixed directory's modules.""" + integration = f'tests/{next(iter(_tiers.FIXTURE_TIER_DIRS))}' + members = [path for path in _tiers.suite_modules() if path.startswith(integration + '/')] + self.assertIn(integration, _tiers.fixture_tier_paths(_tiers.TIER_ALL)) + marked = members[0] + _tiers.fixture_tier_paths.cache_clear() + self.addCleanup(_tiers.fixture_tier_paths.cache_clear) + real = _tiers.module_tier + with unittest.mock.patch.object(_tiers, 'module_tier', + lambda path: _tiers.TIER_LATEST if path == marked else real(path)): + selected = _tiers.fixture_tier_paths(_tiers.TIER_ALL) + self.assertNotIn(integration, selected) + self.assertNotIn(marked, selected) + self.assertEqual([entry for entry in selected if entry.startswith(integration + '/')], members[1:]) class EnclosingScopeTests(unittest.TestCase): @@ -1767,6 +1924,27 @@ def test_a_directory_entry_reaches_what_is_under_it(self) -> None: self.assertTrue(_dependency_gates.job_reaches(job, inside)) self.assertFalse(_dependency_gates.job_reaches(job, outside)) + def test_a_fixture_tier_job_with_no_latest_leg_reaches_the_all_tier_alone(self) -> None: + """#1958: ``pypcap-parity`` asks for the ``ALL`` tier on each leg; ``integration``, on ``latest``, for both.""" + jobs = {job.name: job for job in _dependency_gates.pytest_jobs() if job.selection == 'fixture-tier'} + self.assertEqual({name: job.tier for name, job in jobs.items()}, + {'integration': None, 'pypcap-parity': _tiers.TIER_ALL}) + sweep = _dependency_gates.Gate('HAS_DPKT', 'tests/protocols/test_capture_roundtrip_runtime.py', + 1, 'CaptureRoundTripTests', None) + engines = _dependency_gates.Gate('HAS_PYPCAP', 'tests/foundation/engines/test_new_engine_parity_runtime.py', + 1, 'NewEngineParityTests', None) + self.assertEqual(_tiers.module_tier(sweep.module), _tiers.TIER_LATEST) + self.assertTrue(_dependency_gates.job_reaches(jobs['integration'], sweep)) + self.assertFalse(_dependency_gates.job_reaches(jobs['pypcap-parity'], sweep)) + self.assertTrue(_dependency_gates.job_reaches(jobs['pypcap-parity'], engines)) + + # A parity step that asks for every tier is read as reaching every tier. + doctored = doctored_workflow(self, "fixture_tier_paths('ALL')))\"); then", + "fixture_tier_paths()))\"); then") + parity = next(job for job in _dependency_gates.pytest_jobs(doctored) if job.name == 'pypcap-parity') + self.assertIsNone(parity.tier) + self.assertTrue(_dependency_gates.job_reaches(parity, sweep)) + class DependencyGateCoverageTests(unittest.TestCase): """The assertion #745 asked for, and the checks that keep it honest.""" diff --git a/tests/test_tier_guard_xdist.py b/tests/test_tier_guard_xdist.py index 98e2c9e5d7..b742f5364c 100644 --- a/tests/test_tier_guard_xdist.py +++ b/tests/test_tier_guard_xdist.py @@ -70,6 +70,9 @@ if TYPE_CHECKING: from typing import ContextManager +#: Test tier (#1958): checks the guard in xdist workers on each leg's interpreter. +__tier__ = 'ALL' + #: Capture name the probe module in :class:`XdistSubprocessTests` reads. Not #: tracked by git -- see :func:`tests._tiers.committed_captures` -- so it is a #: tier violation regardless of what ``make samples`` has built locally. diff --git a/tests/vendor/test_crawler_reachability_unit.py b/tests/vendor/test_crawler_reachability_unit.py index 52f55c955a..9ecfa68b61 100644 --- a/tests/vendor/test_crawler_reachability_unit.py +++ b/tests/vendor/test_crawler_reachability_unit.py @@ -75,6 +75,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): checks which crawlers the package reaches, a question about the +#: tree. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_ftp_return_code_unit.py b/tests/vendor/test_ftp_return_code_unit.py index 86278e4202..6445018c7f 100644 --- a/tests/vendor/test_ftp_return_code_unit.py +++ b/tests/vendor/test_ftp_return_code_unit.py @@ -44,6 +44,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_ipx_packet_unit.py b/tests/vendor/test_ipx_packet_unit.py index e5699543c1..6ccd7492ef 100644 --- a/tests/vendor/test_ipx_packet_unit.py +++ b/tests/vendor/test_ipx_packet_unit.py @@ -40,6 +40,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): renders a constant module from recorded data and compares it +#: with the committed file, which the vendor scripts regenerate on one interpreter. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_ipx_socket_unit.py b/tests/vendor/test_ipx_socket_unit.py index 47fbc5f26b..4e27dd739c 100644 --- a/tests/vendor/test_ipx_socket_unit.py +++ b/tests/vendor/test_ipx_socket_unit.py @@ -34,6 +34,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): renders a constant module from recorded data and compares it +#: with the committed file, which the vendor scripts regenerate on one interpreter. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_request_prompt_unit.py b/tests/vendor/test_request_prompt_unit.py index ac9e42eae9..039ba91ec9 100644 --- a/tests/vendor/test_request_prompt_unit.py +++ b/tests/vendor/test_request_prompt_unit.py @@ -88,6 +88,10 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py index 16beba92b7..9fa7286825 100644 --- a/tests/vendor/test_user_agent_unit.py +++ b/tests/vendor/test_user_agent_unit.py @@ -50,6 +50,10 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_dest_path_unit.py b/tests/vendor/test_vendor_dest_path_unit.py index 19f7891e50..493e666da3 100644 --- a/tests/vendor/test_vendor_dest_path_unit.py +++ b/tests/vendor/test_vendor_dest_path_unit.py @@ -123,6 +123,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_exit_code_unit.py b/tests/vendor/test_vendor_exit_code_unit.py index 2674f30249..50c596253e 100644 --- a/tests/vendor/test_vendor_exit_code_unit.py +++ b/tests/vendor/test_vendor_exit_code_unit.py @@ -63,6 +63,10 @@ import warnings from unittest import mock +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Same gate :file:`tests/vendor/test_crawler_reachability_unit.py` and #: :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for the diff --git a/tests/vendor/test_vendor_ipv6_extension_header_925_unit.py b/tests/vendor/test_vendor_ipv6_extension_header_925_unit.py index 06e13e58a0..c721f2907e 100644 --- a/tests/vendor/test_vendor_ipv6_extension_header_925_unit.py +++ b/tests/vendor/test_vendor_ipv6_extension_header_925_unit.py @@ -70,6 +70,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): renders a constant module from recorded data and compares it +#: with the committed file, which the vendor scripts regenerate on one interpreter. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_mh_lla_code_1848_unit.py b/tests/vendor/test_vendor_mh_lla_code_1848_unit.py index 98419ed3e4..cf3185adcf 100644 --- a/tests/vendor/test_vendor_mh_lla_code_1848_unit.py +++ b/tests/vendor/test_vendor_mh_lla_code_1848_unit.py @@ -22,6 +22,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): renders a constant module from recorded data and compares it +#: with the committed file, which the vendor scripts regenerate on one interpreter. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_missing_body_unit.py b/tests/vendor/test_vendor_missing_body_unit.py index ec4c45e9b3..dcc29f87f8 100644 --- a/tests/vendor/test_vendor_missing_body_unit.py +++ b/tests/vendor/test_vendor_missing_body_unit.py @@ -76,6 +76,10 @@ import pathlib import unittest +#: Test tier (#1958): checks the vendor templates' ``_missing_`` bodies against the +#: generated lookups. +__tier__ = 'LATEST' + #: Root of the vendor crawler package, as a path rather than an import, so this #: module reads the same tree it is checked out in regardless of which #: ``pcapkit`` an editable install happens to resolve. diff --git a/tests/vendor/test_vendor_ngap_unit.py b/tests/vendor/test_vendor_ngap_unit.py index e6844eab7c..fc01df9840 100644 --- a/tests/vendor/test_vendor_ngap_unit.py +++ b/tests/vendor/test_vendor_ngap_unit.py @@ -31,6 +31,10 @@ import unittest from unittest import mock +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + HAS_PYCRATE = importlib.util.find_spec('pycrate_asn1dir') is not None diff --git a/tests/vendor/test_vendor_reference_formats_1664_unit.py b/tests/vendor/test_vendor_reference_formats_1664_unit.py index 4fc2649e2b..a71736eec7 100644 --- a/tests/vendor/test_vendor_reference_formats_1664_unit.py +++ b/tests/vendor/test_vendor_reference_formats_1664_unit.py @@ -30,6 +30,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): renders constant modules from recorded data and compares them +#: with the committed files, which the vendor scripts regenerate on one interpreter. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_reg_apptype_generator_unit.py b/tests/vendor/test_vendor_reg_apptype_generator_unit.py index 4ba8c225d4..95e0fc38a0 100644 --- a/tests/vendor/test_vendor_reg_apptype_generator_unit.py +++ b/tests/vendor/test_vendor_reg_apptype_generator_unit.py @@ -123,6 +123,9 @@ import importlib.util import unittest +#: Test tier (#1958): checks the shape of the AppType generator's source. +__tier__ = 'LATEST' + __all__ = ['AppTypeGeneratorShapeTests'] #: Whether :mod:`mypy` is importable, for the one test that runs it. A visible diff --git a/tests/vendor/test_vendor_snapshot_restore_unit.py b/tests/vendor/test_vendor_snapshot_restore_unit.py index 211f606a09..7f84928109 100644 --- a/tests/vendor/test_vendor_snapshot_restore_unit.py +++ b/tests/vendor/test_vendor_snapshot_restore_unit.py @@ -174,6 +174,10 @@ import unittest from unittest import mock +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Same gate the rest of :file:`tests/vendor/` uses. Importing #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for its #: ``vendor_module`` default-target-list fallback), which imports every diff --git a/tests/vendor/test_vendor_variables_1726_unit.py b/tests/vendor/test_vendor_variables_1726_unit.py index 55980b44a0..4bcb1f0774 100644 --- a/tests/vendor/test_vendor_variables_1726_unit.py +++ b/tests/vendor/test_vendor_variables_1726_unit.py @@ -30,6 +30,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported +#: interpreter. +__tier__ = 'ALL' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_wrap_comment_unit.py b/tests/vendor/test_vendor_wrap_comment_unit.py index d419bf8716..1a1c0c7d8f 100644 --- a/tests/vendor/test_vendor_wrap_comment_unit.py +++ b/tests/vendor/test_vendor_wrap_comment_unit.py @@ -29,6 +29,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): checks the vendor scripts' comment wrapper and the comments of +#: the committed constant modules. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2]