ci(compat): compile the 3.8-floor util scripts on the 3.8 leg (#2206)

.github/python-versions.toml

@@ -68,3 +68,12 @@ workflow = "cron-conda.yml"
 job = "conda-dist"
 where = "Conda environment creation and activation: with.python-version"
 value = "3.15"
+
+# The util/ scripts held to requires-python's 3.8 compile on the leg whose
+# parser is 3.8's (#2206): ast's feature_version is best-effort on a newer
+# interpreter. Raising the floor moves this with it.
+[[exemptions]]
+workflow = "python-compatibility.yml"
+job = "compatibility"
+where = "Compile the 3.8-floor util scripts: if"
+value = "3.8"

.github/workflows/python-compatibility.yml

@@ -114,6 +114,32 @@ jobs:
             exit 1
           fi
 
+      # #2206: the util/ scripts held to requires-python's 3.8, compiled by 3.8's
+      # own parser. Their tests' `ast.parse(..., feature_version=(3, 8))` is
+      # best-effort on a newer interpreter, so this leg is the check. The list is
+      # the scripts those tests parse at the floor, as
+      # tests/project/test_compat_util_floor_2206.py pins; the version, an
+      # exemption in .github/python-versions.toml.
+      - name: Compile the 3.8-floor util scripts
+        id: compile-util
+        if: ${{ steps.install.outcome == 'success' && matrix.python-version == '3.8' }}
+        continue-on-error: ${{ matrix.ktlo == true }}
+        shell: bash
+        env:
+          LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}
+        run: |
+          log="$RUNNER_TEMP/compileall-util.log"
+          if ! python -m compileall -q \
+              util/_actions.py util/release_conda.py util/release_published.py \
+              util/release_reset_build.py util/release_status.py util/release_tags.py \
+              util/release_tiers.py util/release_upload.py > "$log" 2>&1; then
+            cat "$log"
+            count=$(awk '/^\*\*\* Error compiling /{n++} END{print n+0}' "$log")
+            first=$(awk '/^  File "/{sub(/^  /, ""); print; exit}' "$log")
+            echo "::${ANNOTATION_LEVEL} title=util/ does not compile on ${LEG//,/%2C}::$count scripts fail to compile, the first at ${first:-an unreported place}; see this step's log."
+            exit 1
+          fi
+
       # #1672: the KTLO leg's report -- the same step ends every KTLO matrix
       # job; see unit-tests.yml's `test` job for how it works.
       - name: KTLO summary

docs/source/contributing/workflows.rst

@@ -1440,6 +1440,16 @@ pin runs each on the cases the shell it replaced handled and requires the same
 status, output and calls.
 :file:`util/_actions.py` is standard library only, and keeps to 3.8's syntax,
 since the scripts on ``/usr/bin/python3`` and on every leg's interpreter import it.
+Their project tests parse these scripts and the helper with ``ast.parse(...,
+feature_version=(3, 8))``, which is best-effort on a newer interpreter, so the
+authoritative check is ``Compile the 3.8-floor util scripts`` in
+``python-compatibility.yml``'s ``compatibility`` job: on the 3.8 leg alone, it runs
+``python -m compileall -q`` over the scripts those tests hold to 3.8, at the job's
+``ANNOTATION_LEVEL``, beside ``Verify package``'s compile of :mod:`pcapkit`
+(:issue:`2206`). A script held to a later floor, as :file:`util/python_versions.py`
+is to 3.11 for ``tomllib``, is left out, and
+:file:`tests/project/test_compat_util_floor_2206.py` fails when the step's list and
+the tests' differ.
 
 The scripts a person runs as well, or alone, sit beside the CI-only ones in
 :file:`util/` rather than in a directory of their own (:pr:`2050`).

tests/project/test_actions_helper_2047.py

@@ -114,6 +114,12 @@ def setUp(self) -> None:
         self.source = HELPER.read_text(encoding='utf-8')
 
     def test_it_parses_as_python_3_8(self) -> None:
+        """Best-effort on a newer interpreter, as ``feature_version`` is (#2206).
+
+        The 3.8 compat leg's ``Compile the 3.8-floor util scripts`` is the
+        authoritative check.
+
+        """
         ast.parse(self.source, filename=str(HELPER), feature_version=FLOOR)
 
     def test_it_imports_only_what_3_8_has(self) -> None:

tests/project/test_compat_util_floor_2206.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+"""#2206: ``Compat Python 3.8 (KTLO)`` compiles the :file:`util/` scripts held to 3.8.
+
+The project tests that hold a :file:`util/` script to ``requires-python``'s floor
+parse it with ``ast.parse(..., feature_version=FLOOR)``, which CPython documents
+as best-effort: on 3.14 it accepts parenthesized context managers, relaxed
+decorators, ``a[*b]`` and quotes reused inside an f-string, none of which 3.8
+compiles. The owner ruled for a real 3.8 compile beside ``Verify package``'s
+``compileall`` of :mod:`pcapkit`. Pinned here, of the step ``Compile the
+3.8-floor util scripts`` in :file:`.github/workflows/python-compatibility.yml`'s
+``compatibility`` job:
+
+* it runs on the floor's leg alone, once the install succeeded, and is masked
+  and reported there as ``Verify package`` is;
+* it compiles exactly the scripts the floor tests parse at
+  :data:`~tests.project.test_requires_python_floor_2051.FLOOR`, found by running
+  those tests with :func:`ast.parse` recorded, so a test that adds a script, drops
+  one or raises one's floor fails here until the step's list follows;
+* a script that does not compile fails it with one annotation at
+  ``ANNOTATION_LEVEL`` naming the first error.
+
+The list stays inline in the step, as plain commands do (see
+:file:`docs/source/contributing/workflows.rst`, "Workflow Scripts"), and this
+module is what keeps it in step with the tests.
+
+"""
+
+from __future__ import annotations
+
+import ast
+import functools
+import importlib
+import inspect
+import os
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from typing import TYPE_CHECKING
+from unittest import mock
+
+from tests._workflows import job
+from tests.project.test_python_matrix_tiers_1633 import legs
+from tests.project.test_release_gates import evaluate_condition
+from tests.project.test_requires_python_floor_2051 import FLOOR
+
+if TYPE_CHECKING:
+    from typing import Any
+
+#: Test tier (#1958): reads the workflow and runs the tree's own floor tests.
+__tier__ = 'LATEST'
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+UTIL = ROOT / 'util'
+WORKFLOW = 'python-compatibility.yml'
+JOB = 'compatibility'
+STEP = 'Compile the 3.8-floor util scripts'
+#: The floor as the matrix spells the leg's version.
+FLOOR_VERSION = '.'.join(map(str, FLOOR))
+#: A :file:`util/` script named in a step's ``run:``.
+UTIL_SCRIPT = re.compile(r'\butil/[\w-]+\.py\b')
+#: The leg the step is for, as its annotations name it.
+LEG = f'Python {FLOOR_VERSION} (KTLO, non-blocking)'
+
+
+def step() -> 'dict[str, Any]':
+    """The compile step of the ``compatibility`` job."""
+    found = [spec for spec in job(WORKFLOW, JOB)['steps'] if spec.get('name') == STEP]
+    if len(found) != 1:
+        raise AssertionError(f'{WORKFLOW} {JOB} holds {len(found)} steps named {STEP!r}')
+    return found[0]
+
+
+@functools.lru_cache(maxsize=None)
+def floor_scripts() -> 'frozenset[str]':
+    """Each :file:`util/` script a ``tests/project`` test parses with ``feature_version=FLOOR``.
+
+    Every test whose source passes ``feature_version`` runs once with
+    :func:`ast.parse` recorded, and each source parsed at :data:`FLOOR` is matched
+    to the :file:`util/` file holding that text.
+
+    """
+    by_text = {path.read_text(encoding='utf-8'): f'util/{path.name}' for path in UTIL.glob('*.py')}
+    parsed = []  # type: list[str]
+    real = ast.parse
+
+    def record(source: 'Any', *args: 'Any', **kwargs: 'Any') -> 'Any':
+        if kwargs.get('feature_version') == FLOOR:
+            parsed.append(source)
+        return real(source, *args, **kwargs)
+
+    for path in sorted((ROOT / 'tests' / 'project').glob('test_*.py')):
+        if path.name == pathlib.Path(__file__).name or 'feature_version=' not in path.read_text(encoding='utf-8'):
+            continue
+        module = importlib.import_module(f'tests.project.{path.stem}')
+        for case in iterate(unittest.defaultTestLoader.loadTestsFromModule(module)):
+            method = getattr(type(case), case._testMethodName)  # pylint: disable=protected-access
+            if 'feature_version=' not in inspect.getsource(method):
+                continue
+            result = unittest.TestResult()
+            with mock.patch.object(ast, 'parse', record):
+                unittest.TestSuite([case]).run(result)
+            if not result.wasSuccessful() or result.skipped:
+                raise AssertionError(f'{case.id()} did not pass, so what it parses is unknown: '
+                                     f'{result.failures + result.errors + result.skipped}')
+    unknown = [source[:80] for source in parsed if source not in by_text]
+    if unknown:
+        raise AssertionError(f'a floor test parses text no util/ script holds: {unknown}')
+    return frozenset(by_text[source] for source in parsed)
+
+
+def iterate(suite: 'unittest.TestSuite') -> 'list[unittest.TestCase]':
+    """Every test case in ``suite``, nested suites flattened."""
+    found = []  # type: list[unittest.TestCase]
+    for item in suite:
+        found.extend(iterate(item) if isinstance(item, unittest.TestSuite) else [item])
+    return found
+
+
+def run_step(cwd: 'pathlib.Path', tmp: 'pathlib.Path') -> 'subprocess.CompletedProcess[str]':
+    """The step's ``run:`` in ``cwd``, as the runner's bash runs it, with this interpreter as ``python``."""
+    bindir = tmp / 'bin'
+    bindir.mkdir()
+    (bindir / 'python').symlink_to(sys.executable)
+    env = dict(os.environ, PATH=f'{bindir}{os.pathsep}{os.environ["PATH"]}', RUNNER_TEMP=str(tmp),
+               ANNOTATION_LEVEL='warning', LEG=LEG)
+    return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', step()['run']],
+                          cwd=cwd, env=env, capture_output=True, text=True, check=False, timeout=60)
+
+
+class TestTheFloorLegCompilesTheFloorScripts(unittest.TestCase):
+    """The 3.8 leg's compile covers what the floor tests can only best-effort check."""
+
+    def test_it_compiles_exactly_the_scripts_the_floor_tests_parse(self) -> None:
+        compiled = sorted(set(UTIL_SCRIPT.findall(step()['run'])))
+        self.assertIn('util/_actions.py', floor_scripts(), 'the recording sees the helper TestStandardLibraryOnly '
+                                                           'parses from setUp')
+        self.assertNotIn('util/python_versions.py', floor_scripts(), 'a script held to a higher floor is left out')
+        self.assertEqual(compiled, sorted(floor_scripts()))
+
+    def test_it_runs_on_the_floor_leg_alone(self) -> None:
+        condition = str(step()['if'])
+        for outcome in ('success', 'failure'):
+            ran = [leg['python-version'] for leg in legs(WORKFLOW, JOB)
+                   if evaluate_condition(condition, {'matrix': leg, 'steps': {'install': {'outcome': outcome}}})]
+            with self.subTest(install=outcome):
+                self.assertEqual(ran, [FLOOR_VERSION] if outcome == 'success' else [])
+
+    def test_it_is_masked_and_reported_as_verify_package_is(self) -> None:
+        verify = next(spec for spec in job(WORKFLOW, JOB)['steps'] if spec.get('name') == 'Verify package')
+        compile_util = step()
+        self.assertIn('id', compile_util, 'KTLO summary reads only steps with an id')
+        for key in ('continue-on-error', 'shell', 'env'):
+            with self.subTest(key=key):
+                self.assertEqual(compile_util.get(key), verify.get(key))
+        self.assertIn('python -m compileall -q', compile_util['run'])
+
+
+class TestAScriptThatDoesNotCompileFailsTheStep(unittest.TestCase):
+    """The step's script, run against a copy of :file:`util/`."""
+
+    def setUp(self) -> None:
+        self.tmp = pathlib.Path(tempfile.mkdtemp())
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.tree = self.tmp / 'tree'
+        shutil.copytree(UTIL, self.tree / 'util', ignore=shutil.ignore_patterns('__pycache__'))
+
+    def test_the_tree_compiles_quietly(self) -> None:
+        result = run_step(self.tree, self.tmp)
+        self.assertEqual((result.returncode, result.stdout, result.stderr), (0, '', ''))
+
+    def test_a_syntax_error_fails_with_one_annotation(self) -> None:
+        target = self.tree / 'util' / 'release_tags.py'
+        lines = target.read_text(encoding='utf-8').count('\n')
+        with target.open('a', encoding='utf-8') as file:
+            file.write('def (:\n')
+        result = run_step(self.tree, self.tmp)
+        self.assertEqual(result.returncode, 1)
+        annotations = [line for line in result.stdout.splitlines() if line.startswith('::')]
+        self.assertEqual(annotations, [
+            f'::warning title=util/ does not compile on {LEG.replace(",", "%2C")}::1 scripts fail to compile, '
+            f'the first at File "util/release_tags.py", line {lines + 1}; see this step\'s log.',
+        ])
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/project/test_ktlo_legs_stay_green_1672.py

@@ -164,6 +164,9 @@
 '''
 #: A ``util/`` script a ``run:`` invokes, whose own annotations are the step's.
 UTIL_SCRIPT = re.compile(r'\butil/[\w-]+\.py\b')
+#: A ``compileall`` command line, continuation lines included: the scripts it names
+#: are compiled, not run, so their annotations are not the step's (#2206).
+COMPILEALL = re.compile(r'-m compileall\b(?:[^\n]*\\\n)*[^\n]*')
 #: ``experimental``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``:
 #: setup-python, the install, the watcher's setup-python and requests (#1938) and the
 #: tests. #1732: the tests' is GraalPy's unit tier less the round-trip sweeps, 88-113
@@ -367,11 +370,12 @@ def commands(script: 'str') -> 'set[str]':
 
     A script's ``::{level}``, where it reads ``ANNOTATION_LEVEL`` into ``level`` as
     :file:`util/regenerate_samples.py` does, is the shell's ``${ANNOTATION_LEVEL}``;
-    so is a ``level`` it passes to util/_actions.py's annotations (#2047).
+    so is a ``level`` it passes to util/_actions.py's annotations (#2047). A script
+    a :data:`COMPILEALL` command line names is compiled, not run (#2206).
 
     """
     found = set(COMMAND.findall(script)) | set(HELPER_CALL.findall(script))
-    for path in set(UTIL_SCRIPT.findall(script)):
+    for path in set(UTIL_SCRIPT.findall(COMPILEALL.sub('', script))):
         text = (ROOT / path).read_text(encoding='utf-8')
         levelled = "level = os.environ['ANNOTATION_LEVEL']" in text
         written = set(COMMAND.findall(text)) | helper_levels(text)
@@ -911,6 +915,17 @@ def test_a_helper_call_in_the_shell_is_read(self) -> None:
             with self.subTest(script=script):
                 self.assertEqual(commands(script), found)
 
+    def test_a_script_compileall_names_is_not_read_as_run(self) -> None:
+        """#2206: compiling a script that writes ``::error`` writes nothing; running it still does."""
+        self.assertIn('error', commands('python3 util/release_tags.py'))
+        for script in ('python -m compileall -q util/release_tags.py',
+                       'if ! python -m compileall -q \\\n'
+                       '    util/release_tags.py \\\n'
+                       '    util/release_upload.py; then'):
+            with self.subTest(script=script):
+                self.assertEqual(commands(script), set())
+        self.assertIn('error', commands('python -m compileall -q util\npython3 util/release_tags.py'))
+
     def test_a_literal_command_is_still_read(self) -> None:
         for script in ('echo "::error::m"', "printf '::error::%s\\n' m"):
             with self.subTest(script=script):

tests/project/test_release_conda_scripts_1967.py

@@ -563,6 +563,12 @@ def test_every_leg_is_mapped_and_every_mapping_is_a_leg(self) -> None:
         self.assertEqual(set(release_conda.PLATFORMS), legs)
 
     def test_the_scripts_parse_as_python_3_8(self) -> None:
+        """Best-effort on a newer interpreter, as ``feature_version`` is (#2206).
+
+        The 3.8 compat leg's ``Compile the 3.8-floor util scripts`` is the
+        authoritative check.
+
+        """
         ast.parse((UTIL / 'release_conda.py').read_text(encoding='utf-8'), feature_version=FLOOR)
 
 

tests/project/test_release_reset_build_1967.py

@@ -464,7 +464,13 @@ def test_its_git_never_pulls_merges_rebases_resets_or_forces(self) -> None:
                                     'push'})
 
     def test_it_parses_as_python_3_8(self) -> None:
-        """``tag`` has no setup-python, so it runs on the image's ``python3``."""
+        """``tag`` has no setup-python, so it runs on the image's ``python3``.
+
+        ``feature_version`` is best-effort on a newer interpreter, so the 3.8
+        compat leg's ``Compile the 3.8-floor util scripts`` is the authoritative
+        check (#2206).
+
+        """
         ast.parse(SCRIPT.read_text(encoding='utf-8'), feature_version=FLOOR)
 
 

tests/project/test_release_scripts_1967.py

@@ -519,7 +519,13 @@ def test_each_job_checks_the_whole_tree_out_first(self) -> None:
                                   and 'working-directory' in step])
 
     def test_the_scripts_parse_as_python_3_8(self) -> None:
-        """The steps run them on the image's ``python3``, or on a KTLO leg's."""
+        """The steps run them on the image's ``python3``, or on a KTLO leg's.
+
+        ``feature_version`` is best-effort on a newer interpreter, so the 3.8
+        compat leg's ``Compile the 3.8-floor util scripts`` is the authoritative
+        check (#2206).
+
+        """
         for name in ('release_tags.py', 'release_upload.py'):
             with self.subTest(script=name):
                 ast.parse((UTIL / name).read_text(encoding='utf-8'), feature_version=FLOOR)

tests/project/test_release_status_scripts_1967.py

@@ -617,7 +617,13 @@ def test_no_step_holds_the_former_shell(self) -> None:
                         self.assertNotIn(line, step.get('run', ''))
 
     def test_the_scripts_need_only_util_and_python_3_8(self) -> None:
-        """The sparse checkout holds :file:`util/`, and the image's ``python3`` nothing installed."""
+        """The sparse checkout holds :file:`util/`, and the image's ``python3`` nothing installed.
+
+        ``feature_version`` is best-effort on a newer interpreter, so the 3.8
+        compat leg's ``Compile the 3.8-floor util scripts`` is the authoritative
+        check (#2206).
+
+        """
         allowed = {'__future__', 'argparse', 'os', 'subprocess', 'sys', 'typing', '_actions', 'util'}
         for name in ('release_status.py',):
             with self.subTest(script=name):

tests/project/test_release_version_check_scripts_1967.py

@@ -820,6 +820,13 @@ def test_no_step_holds_the_former_shell(self) -> None:
                         self.assertNotIn(line, step.get('run', ''))
 
     def test_the_scripts_need_only_util_and_python_3_8(self) -> None:
+        """Each imports only :file:`util/` and the standard library, and parses at the floor.
+
+        ``feature_version`` is best-effort on a newer interpreter, so the 3.8
+        compat leg's ``Compile the 3.8-floor util scripts`` is the authoritative
+        check (#2206).
+
+        """
         allowed = {'__future__', 'argparse', 'os', 're', 'subprocess', 'sys', 'typing', '_actions', 'util'}
         for name in ('release_tiers.py', 'release_published.py'):
             with self.subTest(script=name):