diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index c265a0ea2d..ec99eda154 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -9,7 +9,8 @@ name: Experimental Tests # `needs:` one in another workflow. # # Its check names, `Python (experimental)`, are the ones the legs had -# in unit-tests.yml; no required context of ruleset 23497679 names one. +# in unit-tests.yml, and `windows`'s is `Python on Windows +# (experimental)` (#2060); no required context of ruleset 23497679 names one. # # The same events and `changes` job as unit-tests.yml, so a docs-only pull # request skips the legs here as it skips `test` there. @@ -606,3 +607,132 @@ jobs: ${{ runner.temp }}/job-summary.name if-no-files-found: ignore retention-days: 7 + + # #2060: the unit tier on Windows before the win-64 conda package is + # published, so its first release has had a test run. One leg, on `latest`, + # the newest CPython the conda builds target. Non-blocking by `experimental`'s + # mechanics: every step that can fail is `continue-on-error`, the long ones + # have caps under the job's, `Experimental summary` reports each failure as a + # warning, and the job-level guard covers a lost runner. None of + # `experimental`'s Linux containment runs here: no scope or `prlimit`, nor + # util/cap_file_size.py, which needs `resource`; no resource monitor, which + # reads /proc; no superseded watcher, which signals a process group. So the + # tests' cap is the step's own. Every `run:` is Git Bash's. + windows: + name: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental) + if: >- + ${{ !cancelled() && needs.changes.result == 'success' && needs.python-versions.result == 'success' + && needs.current.outputs.current != 'false' && needs.changes.outputs.code != 'false' }} + needs: [current, changes, python-versions] + runs-on: windows-latest + timeout-minutes: 80 + continue-on-error: true + env: + ANNOTATION_LEVEL: warning + + steps: + # The image's Git checks text out with CRLF, which `* text=auto` leaves to + # it; LF keeps the tree the tests read the one the other legs read. + - name: Keep LF line endings + id: eol + continue-on-error: true + shell: bash + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + id: checkout + continue-on-error: true + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + id: python + if: ${{ steps.checkout.outcome == 'success' }} + continue-on-error: true + timeout-minutes: 5 + with: + python-version: ${{ needs.python-versions.outputs.latest }} + cache: pip + + - name: Install package and test dependencies + id: install + if: ${{ steps.python.outcome == 'success' }} + continue-on-error: true + timeout-minutes: 10 + shell: bash + env: + LEG: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental, non-blocking) + run: python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]' + + - name: Report available parallelism + id: parallelism + if: ${{ steps.install.outcome == 'success' }} + continue-on-error: true + shell: bash + run: python -c "import os; print('cpu_count', os.cpu_count())" + + # `experimental`'s selection, the `ALL` tier (#1958), asked of + # tests/_tiers.py on the leg's own CPython, as `test` asks it. + - name: Run unit tests + id: tests + if: ${{ steps.install.outcome == 'success' }} + continue-on-error: true + timeout-minutes: 60 + shell: bash + run: | + if ! skipped=$(python -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))"); then + echo "::${ANNOTATION_LEVEL} title=Could not compute the LATEST tier::tests._tiers.latest_tier_paths() failed -- see the traceback above." + exit 1 + fi + set -- -q -n auto --dist load \ + --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ + --ignore=tests/integration \ + --ignore-glob='*_runtime.py' \ + --ignore-glob='*_regression.py' + for module in $skipped; do + set -- "$@" --ignore="$module" + done + python -m pytest "$@" + + - name: Upload test timings + if: ${{ !cancelled() && steps.install.outcome == 'success' }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: junit-experimental-windows-${{ needs.python-versions.outputs.latest }} + path: ${{ runner.temp }}/junit.xml + if-no-files-found: warn + retention-days: 14 + + # `experimental`'s, on setup-python's `python`, or the image's where that + # failed, since Windows has no /usr/bin/python3, as cron-conda.yml's + # `Windows summary` runs it (#2060). + - name: Experimental summary + if: ${{ !cancelled() }} + continue-on-error: true + shell: bash + env: + LEG: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental, non-blocking) + STEPS: ${{ toJSON(steps) }} + JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md + run: python util/leg_summary.py experimental + + - name: Name the job summary + if: ${{ always() }} + continue-on-error: true + shell: bash + env: + JOB_NAME: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental) + run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi + + - name: Upload the job summary + if: ${{ always() }} + continue-on-error: true + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: job-summary-${{ github.job }}-${{ strategy.job-index }} + path: | + ${{ runner.temp }}/job-summary.md + ${{ runner.temp }}/job-summary.name + if-no-files-found: ignore + retention-days: 7 diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index b48723940f..5864dd5f11 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -1030,6 +1030,30 @@ workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inf from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a warning: that is expected, blocks nothing, and leaves the caps as they are. +**The Windows leg.** :file:`experimental.yml`'s ``windows`` job runs the ``ALL`` tier +on ``windows-latest`` under ``latest``, the newest CPython the conda builds target, as +``Python on Windows (experimental)`` (`#2060 +`__), so the first published +``win-64`` conda package has had a test run. It needs ``current``, ``changes`` and +``python-versions`` and runs on ``experimental``'s condition, and it is non-blocking by +the same mechanics: every step continues on error, setup-python, the install and ``Run +unit tests`` are capped at 5, 10 and 60 minutes within its 80, the job carries +``continue-on-error: true``, and ``Experimental summary`` reports each failed step as a +warning, on setup-python's ``python`` or the image's, since Windows has no +``/usr/bin/python3``. Every ``run:`` is Git Bash's (``shell: bash``). ``Keep LF line +endings`` sets ``core.autocrlf false`` and ``core.eol lf`` before the checkout, so the +tests read the tree the other legs read, not the CRLF one the image's Git writes under +``* text=auto``. ``Run unit tests`` asks ``latest_tier_paths()`` on the leg's CPython, +as ``test`` does, and passes ``test``'s flags. None of the Linux containment above runs +there: Windows has no cgroup scope, ``prlimit`` or ``RLIMIT_FSIZE``, so no +:file:`util/cap_file_size.py`; no ``/proc`` for the resource monitor; and no process +group for :file:`util/superseded.py`'s watcher, so the step's cap bounds the tests, and +a run superseded once the leg has started runs it to its end. Python Versions' weekly +job moves ``latest`` only to a branch setup-python builds for every label a job +installs it on, but counts no job with ``continue-on-error: true``, so this leg +never holds ``latest`` back for the blocking jobs. +:file:`tests/project/test_windows_test_leg_2060.py` pins the job. + Test Tiers ---------- @@ -1103,7 +1127,8 @@ Which legs run which tier: - ``ALL`` and ``LATEST``, less the project pins * - ``project-tests`` - the project pins - * - ``test`` off ``latest``, PyPy's among them, ``test-ktlo``, ``experimental`` + * - ``test`` off ``latest``, PyPy's among them, ``test-ktlo``, ``experimental``, + ``windows`` - ``ALL`` * - ``integration`` on ``latest`` - ``ALL`` and ``LATEST`` @@ -1198,8 +1223,9 @@ a lower hard limit, and execs the command, so the step's exit status and inherits the cap. A write past it raises ``OSError: [Errno 27] File too large`` in Python, since CPython ignores ``SIGXFSZ``, so the test errors; any other program is killed by the signal, whose default action the script restores before it execs. -``experimental.yml``'s legs get the same 2 GiB through ``prlimit --fsize`` (see -above). +``experimental.yml``'s ``experimental`` legs get the same 2 GiB through +``prlimit --fsize`` (see above); its ``windows`` leg has no cap, as Windows has no +``RLIMIT_FSIZE``. :file:`tests/project/test_file_size_caps_1969.py` fails when a command in ``unit-tests.yml`` runs pytest, unittest or :file:`util/run_unittest_leg.py` diff --git a/tests/project/test_pip_install_1967.py b/tests/project/test_pip_install_1967.py index 302b987d0f..8f511f65ee 100644 --- a/tests/project/test_pip_install_1967.py +++ b/tests/project/test_pip_install_1967.py @@ -13,8 +13,8 @@ its guards and environment, and every other ``pip install`` in ``unit-tests.yml`` to a listed step that keeps its shell. * :class:`TestTheOtherWorkflowsRunTheScript` does the same for the install of - ``experimental.yml``'s ``experimental`` and ``python-compatibility.yml``'s - ``compatibility``. + ``experimental.yml``'s ``experimental`` and ``windows`` (#2060) and + ``python-compatibility.yml``'s ``compatibility``. """ @@ -91,6 +91,17 @@ 'env': {'LEG': 'Python ${{ matrix.python-version }} (experimental, non-blocking)'}, 'run': "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'", }, 'warning'), + # #2060: the Windows leg's, the same install on `latest`. + ('experimental.yml', 'windows'): ({ + 'name': 'Install package and test dependencies', + 'id': 'install', + 'if': "${{ steps.python.outcome == 'success' }}", + 'continue-on-error': True, + 'timeout-minutes': 10, + 'shell': 'bash', + 'env': {'LEG': 'Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental, non-blocking)'}, + 'run': "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'", + }, 'warning'), ('python-compatibility.yml', 'compatibility'): ({ 'name': 'Install package', 'id': 'install', diff --git a/tests/project/test_resource_monitor_1740.py b/tests/project/test_resource_monitor_1740.py index 9ebdf87df0..79179e16e8 100644 --- a/tests/project/test_resource_monitor_1740.py +++ b/tests/project/test_resource_monitor_1740.py @@ -18,6 +18,7 @@ from __future__ import annotations +import json import os import pathlib import re @@ -566,7 +567,10 @@ def test_no_reader_takes_the_artifact_for_a_junit_one(self) -> None: self.assertTrue(ci_timing.JUNIT_ARTIFACT.startswith('junit-')) self.assertEqual([path.name for path in sorted((ROOT / 'util').glob('*.py')) if 'resource-monitor' in path.read_text(encoding='utf-8')], []) - self.assertEqual(list(workflow(EXPERIMENTAL[0])['jobs']), ['current', 'python-versions', 'changes', 'experimental']) + # #2060's `windows` runs no monitor and downloads nothing. + jobs = workflow(EXPERIMENTAL[0])['jobs'] + self.assertEqual(list(jobs), ['current', 'python-versions', 'changes', 'experimental', 'windows']) + self.assertNotIn('resource-monitor', json.dumps(jobs['windows'])) class TestTheDocsSayIt(unittest.TestCase): diff --git a/tests/project/test_superseded_runs_1938.py b/tests/project/test_superseded_runs_1938.py index 966b2bd65a..0e0ca9f73b 100644 --- a/tests/project/test_superseded_runs_1938.py +++ b/tests/project/test_superseded_runs_1938.py @@ -209,7 +209,8 @@ def test_the_watcher_runs_on_a_cpython_of_its_own_with_requests(self) -> None: def test_every_other_job_needs_it_and_skips_on_its_output(self) -> None: others = [job_id for job_id in self.jobs if job_id != GATE] - self.assertEqual(sorted(others), ['changes', 'experimental', 'python-versions']) + # #2060: `windows`, the Windows leg, skips with the rest. + self.assertEqual(sorted(others), ['changes', 'experimental', 'python-versions', 'windows']) for job_id in others: body = self.jobs[job_id] needs = body['needs'] if isinstance(body['needs'], list) else [body['needs']] @@ -237,7 +238,7 @@ def run_if(self, job_id: 'str', current: 'str', code: 'str' = 'true', *, gate: ' return runs def test_a_superseded_commit_skips_every_job_and_anything_else_runs_them(self) -> None: - for job_id in ('python-versions', 'changes', 'experimental'): + for job_id in ('python-versions', 'changes', 'experimental', 'windows'): with self.subTest(job=job_id): self.assertFalse(self.run_if(job_id, 'false')) self.assertTrue(self.run_if(job_id, 'true')) @@ -248,16 +249,17 @@ def test_a_superseded_commit_skips_every_job_and_anything_else_runs_them(self) - self.assertTrue(self.run_if(job_id, '', gate='cancelled')) # A cancelled run stays cancelled. self.assertFalse(self.run_if(job_id, 'true', cancelled=True)) - # A docs-only pull request still skips the legs. - self.assertFalse(self.run_if('experimental', 'true', code='false')) - # And the legs still need the jobs they read: `!cancelled()` drops only the implicit `success()`. - for job_id in ('changes', 'python-versions'): - for result in ('failure', 'skipped', 'cancelled'): - with self.subTest(needs=job_id, result=result): - self.assertFalse(self.run_if('experimental', 'true', **{ - 'versions' if job_id == 'python-versions' else 'changes': result})) - # Superseded: `changes` and `python-versions` skip, and the legs with them. - self.assertFalse(self.run_if('experimental', 'false', versions='skipped', changes='skipped')) + for legs in ('experimental', 'windows'): + # A docs-only pull request still skips the legs. + self.assertFalse(self.run_if(legs, 'true', code='false')) + # And the legs still need the jobs they read: `!cancelled()` drops only the implicit `success()`. + for job_id in ('changes', 'python-versions'): + for result in ('failure', 'skipped', 'cancelled'): + with self.subTest(legs=legs, needs=job_id, result=result): + self.assertFalse(self.run_if(legs, 'true', **{ + 'versions' if job_id == 'python-versions' else 'changes': result})) + # Superseded: `changes` and `python-versions` skip, and the legs with them. + self.assertFalse(self.run_if(legs, 'false', versions='skipped', changes='skipped')) def test_the_legs_and_the_tests_step_read_with_the_token(self) -> None: self.assertEqual(set(job(NAME, 'experimental')['permissions'].values()), {'read'}) diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 3dd90b45e7..100c8f66c0 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -530,7 +530,8 @@ def test_a_failure_never_turns_the_run_red(self) -> None: # #2030: the last step but the two that name and upload the job's summary. self.assertEqual([entry['name'] for entry in spec['steps'][-3:]], ['Experimental summary', 'Name the job summary', 'Upload the job summary']) - expected = {'experimental.yml': ['experimental'], + # #2060: the Windows leg's job, as tests/project/test_windows_test_leg_2060.py pins. + expected = {'experimental.yml': ['experimental', 'windows'], 'unit-tests.yml': ['test-ktlo', 'integration-ktlo', 'pypcap-parity-ktlo']} for name in ('unit-tests.yml', 'experimental.yml'): with self.subTest(workflow=name): diff --git a/tests/project/test_unit_tests_push_reports_2030.py b/tests/project/test_unit_tests_push_reports_2030.py index 989786219d..fd864db0e6 100644 --- a/tests/project/test_unit_tests_push_reports_2030.py +++ b/tests/project/test_unit_tests_push_reports_2030.py @@ -94,7 +94,8 @@ ('unit-tests.yml', 'test'), ('unit-tests.yml', 'test-ktlo'), ('unit-tests.yml', 'integration'), ('unit-tests.yml', 'integration-ktlo'), ('unit-tests.yml', 'engine-tests'), ('unit-tests.yml', 'pypcap-parity'), ('unit-tests.yml', 'pypcap-parity-ktlo'), - ('experimental.yml', 'experimental'), ('python-compatibility.yml', 'compatibility'), + ('experimental.yml', 'experimental'), ('experimental.yml', 'windows'), + ('python-compatibility.yml', 'compatibility'), ) #: The steps that copy the summary through util/_actions.py's JOB_SUMMARY_COPY. COPYING = ('Report coverage', 'KTLO summary', 'Experimental summary') @@ -424,10 +425,11 @@ def test_each_job_names_and_uploads_it(self) -> None: for step in (save, upload): self.assertEqual(step['if'], '${{ always() }}') self.assertIs(step['continue-on-error'], True) - # The name the job's check run carries, on every leg. - for leg in legs(name, job_id): - self.assertEqual(interpolate(save['env']['JOB_NAME'], {'matrix': leg}), - interpolate(job['name'], {'matrix': leg})) + # The name the job's check run carries, on every leg; #2060's `windows` has one. + for leg in (legs(name, job_id) if 'strategy' in job else [{}]): + context = {'matrix': leg, 'needs': NEEDS} + self.assertEqual(interpolate(save['env']['JOB_NAME'], context), + interpolate(job['name'], context)) @unittest.skipIf(not BASH, 'needs bash') def test_the_name_is_written_beside_a_summary_alone(self) -> None: diff --git a/tests/project/test_windows_test_leg_2060.py b/tests/project/test_windows_test_leg_2060.py new file mode 100644 index 0000000000..1c7c8bf004 --- /dev/null +++ b/tests/project/test_windows_test_leg_2060.py @@ -0,0 +1,370 @@ +# -*- coding: utf-8 -*- +"""#2060: a non-blocking Windows unit-test leg, ahead of the win-64 conda package's release. + +The plan for #2060 asks that a Windows unit-test leg run before phase 2 publishes, +so the first win-64 package has had a test run. :file:`.github/workflows/experimental.yml` +gained ``windows``, a job of its own beside ``experimental``, whose Linux +containment (cgroup scope, ``prlimit``, resource monitor, superseded watcher) +cannot run on Windows. + +* :class:`TestTheLeg`: one leg on ``windows-latest``, on ``latest``, the newest + CPython the conda builds target, named ``Python on Windows (experimental)``, + gated as ``experimental`` is, and needed by nothing. +* :class:`TestItNeverBlocks`: every step that runs is ``continue-on-error``, so no + failure fails the job; the long steps have caps under the job's; ``Experimental + summary`` runs after any failure and warns once per failed step; no step writes + an error. +* :class:`TestItRunsOnWindows`: every ``run:`` is Git Bash's, none calls a + Linux-only tool, and the checkout keeps LF line endings. +* :class:`TestTheTestsStep`: run as bash runs it, the step hands pytest ``test``'s + flags and ignores the ``LATEST`` tier, and a tier it cannot list fails the step + with a warning. +* :class:`TestItHoldsLatestBackNowhere`: the weekly tier job does not wait for a Windows + build before it moves ``latest``. +* :class:`TestTheReporters`: CI Reports and the CI Timing review read the job as a + non-blocking experimental leg. + +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING + +from tests import _workflows +from tests.project.test_ktlo_legs_stay_green_1672 import (EXPERIMENTAL_REASON, JOB_SUMMARY_STEPS, KTLO_COMMANDS, + SUMMARY_COPY, commands, interpolate, label, scenarios) +from tests.project.test_python_versions_1660 import live_outputs, prose, resolve +from tests.project.test_release_gates import evaluate_condition +from util import check_suite_report, ci_timing + +if TYPE_CHECKING: + from typing import Any + +#: Test tier (#1958): reads the workflow and docs, and runs a step's shell, a question about the tree. +__tier__ = 'LATEST' + +ROOT = pathlib.Path(__file__).resolve().parents[2] +NAME = 'experimental.yml' +JOB = 'windows' +WHERE = (NAME, JOB) +LATEST = '${{ needs.python-versions.outputs.latest }}' +JOB_NAME = f'Python {LATEST} on Windows (experimental)' +LEG = f'Python {LATEST} on Windows (experimental, non-blocking)' +SUMMARY = 'Experimental summary' +#: The summary's command: setup-python's ``python``, or the image's, not ``/usr/bin/python3``. +SUMMARY_RUN = 'python util/leg_summary.py experimental' +#: The steps with a cap of their own, by ``id``, and the job's. +CAPS = {'python': 5, 'install': 10, 'tests': 60} +JOB_CAP = 80 +#: The minutes left for the uncapped steps: the LF step, checkout, the parallelism +#: report, the uploads and the summaries, each seconds on ``experimental``'s legs. +UNCAPPED_MINUTES = 5 +#: What a Windows runner lacks: Linux tools, and the scripts that need them. +LINUX_ONLY = re.compile(r'(?&2; exit 1; fi + exec "$REAL_PYTHON" "$@" + fi + printf '%s\\n' "$@" > "$ARGS_FILE" + ''') + + +def spec() -> 'dict[str, Any]': + return _workflows.job(*WHERE) + + +def step(name: 'str') -> 'dict[str, Any]': + [found] = [entry for entry in spec()['steps'] if label(entry) == name] + return found + + +def uses(action: 'str') -> 'dict[str, Any]': + """The one step of the job that runs ``action``, at whatever pin.""" + [found] = [entry for entry in spec()['steps'] if str(entry.get('uses', '')).startswith(f'{action}@')] + return found + + +def bash(script: 'str', env: 'dict[str, str]') -> 'subprocess.CompletedProcess[str]': + """``script`` as ``shell: bash`` runs it, from the root.""" + full = dict(os.environ, **env) + full.pop('PYTHONSAFEPATH', None) # the runner sets none, and it would hide tests/ from `-c` + return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', script], # nosec: B603 B607 + env=full, cwd=ROOT, capture_output=True, text=True, check=False, timeout=120) + + +class TestTheLeg(unittest.TestCase): + """One leg, on ``windows-latest`` and ``latest``, gated as ``experimental`` is.""" + + def test_the_job_exists_on_windows(self) -> None: + jobs = _workflows.load(NAME)['jobs'] + self.assertIn(JOB, jobs) + self.assertEqual(spec()['runs-on'], 'windows-latest') + self.assertEqual(spec()['name'], JOB_NAME) + self.assertNotIn('strategy', spec()) + + def test_it_runs_the_newest_cpython_the_conda_builds_target(self) -> None: + setup = uses('actions/setup-python') + self.assertEqual(setup['with'], {'python-version': LATEST, 'cache': 'pip'}) + version = resolve(NAME, JOB, LATEST) + conda = json.loads(live_outputs()['conda']) + self.assertEqual(version, live_outputs()['latest']) + self.assertEqual(version, conda[-1]) + self.assertEqual(interpolate(spec()['name'], {'needs': {'python-versions': {'outputs': {'latest': version}}}}), + f'Python {version} on Windows (experimental)') + + def test_it_is_gated_as_experimental_is(self) -> None: + legs = _workflows.job(NAME, 'experimental') + for key in ('needs', 'if', 'continue-on-error'): + with self.subTest(key=key): + self.assertEqual(spec()[key], legs[key]) + self.assertIs(spec()['continue-on-error'], True) + self.assertEqual(spec()['env'], {'ANNOTATION_LEVEL': 'warning'}) + + def test_nothing_needs_it(self) -> None: + for path in sorted(_workflows.WORKFLOWS.glob('*.yml')): + for job_id, body in (_workflows.load(path.name).get('jobs') or {}).items(): + needs = body.get('needs', []) + with self.subTest(workflow=path.name, job=job_id): + self.assertNotIn(JOB, [needs] if isinstance(needs, str) else needs) + + +class TestItNeverBlocks(unittest.TestCase): + """The KTLO mechanics (#1672), as ``experimental`` has them (#1660).""" + + def setUp(self) -> None: + self.spec = spec() + [self.summary] = [entry for entry in self.spec['steps'] if label(entry) == SUMMARY] + self.position = self.spec['steps'].index(self.summary) + + def test_every_step_continues_on_error(self) -> None: + for entry in self.spec['steps']: + with self.subTest(step=label(entry)): + self.assertIs(entry.get('continue-on-error'), True) + + def test_no_step_can_fail_the_leg(self) -> None: + runs = scenarios(*WHERE, self.spec, {}) + self.assertGreater(len(runs), 3) + self.assertEqual(len(runs), len(self.spec['steps'])) + for index, conclusion, _ in runs: + with self.subTest(step=label(self.spec['steps'][index])): + self.assertEqual(conclusion, 'success') + + def test_the_summary_sees_every_failure(self) -> None: + """Last but the job summary's two steps, after any failure, and every step it reports has an ``id``.""" + self.assertEqual([label(entry) for entry in self.spec['steps'][self.position + 1:]], list(JOB_SUMMARY_STEPS)) + self.assertEqual(self.summary['if'], '${{ !cancelled() }}') + self.assertEqual(self.summary['shell'], 'bash') + self.assertEqual(self.summary['env'], dict({'LEG': LEG, 'STEPS': '${{ toJSON(steps) }}'}, **SUMMARY_COPY)) + self.assertEqual(self.summary['run'], SUMMARY_RUN) + for index, _, record in scenarios(*WHERE, self.spec, {}): + entry = self.spec['steps'][index] + if entry is self.summary or label(entry) in JOB_SUMMARY_STEPS: + continue + with self.subTest(step=label(entry)): + self.assertNotEqual(record[self.position].outcome, 'skipped') + if 'uses' not in entry or 'upload-artifact' not in entry['uses']: + self.assertIn('id', entry) + for entry in self.spec['steps'][self.position + 1:]: + self.assertEqual(entry['if'], '${{ always() }}') + + def test_the_job_summary_carries_the_check_name(self) -> None: + self.assertEqual(step('Name the job summary')['env'], {'JOB_NAME': JOB_NAME}) + self.assertEqual(step('Install package and test dependencies')['env'], {'LEG': LEG}) + + def test_no_step_writes_an_error(self) -> None: + for entry in self.spec['steps']: + with self.subTest(step=label(entry)): + self.assertLessEqual(commands(entry.get('run', '')), KTLO_COMMANDS) + + def test_each_long_step_ends_inside_the_job(self) -> None: + caps = {entry.get('id'): entry['timeout-minutes'] for entry in self.spec['steps'] if 'timeout-minutes' in entry} + self.assertEqual(caps, CAPS) + self.assertEqual(self.spec['timeout-minutes'], JOB_CAP) + self.assertLess(caps['install'], caps['tests']) + self.assertLessEqual(sum(caps.values()) + UNCAPPED_MINUTES, JOB_CAP) + + @unittest.skipIf(shutil.which('bash') is None, 'the summary is a bash step') + def test_the_summary_warns_once_per_failed_step(self) -> None: + name = 'Python 3.15 on Windows (experimental, non-blocking)' + outcomes = (('checkout', 'success'), ('install', 'failure'), ('tests', 'skipped')) + context = {key: {'outputs': {}, 'outcome': outcome, + 'conclusion': 'success' if outcome == 'failure' else outcome} for key, outcome in outcomes} + with tempfile.TemporaryDirectory(prefix='pcapkit-2060-') as tmp: + page = pathlib.Path(tmp, 'summary.md') + page.write_text('', encoding='utf-8') + run = self.summary['run'].replace('python ', f'{sys.executable} ', 1) + result = bash(run, {'LEG': name, 'STEPS': json.dumps(context), 'GITHUB_STEP_SUMMARY': str(page)}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), [ + f"::warning title=install failed on {name.replace(',', '%2C')}::{EXPERIMENTAL_REASON}; " + "see the install step's log."]) + self.assertEqual(page.read_text(encoding='utf-8'), + f'- {name}: step `install` failed; non-blocking, so this job still succeeds.\n') + + +class TestItRunsOnWindows(unittest.TestCase): + """Git Bash for every ``run:``, nothing Linux-only, and the tree checked out with LF.""" + + def test_every_run_is_bash(self) -> None: + """A Windows step's default shell is PowerShell.""" + found = [entry for entry in spec()['steps'] if 'run' in entry] + self.assertTrue(found) + for entry in found: + with self.subTest(step=label(entry)): + self.assertEqual(entry.get('shell'), 'bash') + + def test_no_step_calls_a_linux_only_tool(self) -> None: + for entry in spec()['steps']: + with self.subTest(step=label(entry)): + self.assertIsNone(LINUX_ONLY.search(str(entry.get('run', '')))) + + def test_the_check_sees_the_linux_legs_tools(self) -> None: + """So the check above is not vacuous: ``experimental``'s steps use several.""" + found = {match.group(0) for entry in _workflows.steps(NAME, 'experimental') + for match in LINUX_ONLY.finditer(str(entry.get('run', '')))} + self.assertLessEqual({'sudo', 'prlimit', '/usr/bin/python3', '/proc/', 'superseded.py', 'nproc'}, found) + + def test_line_endings_are_set_before_the_checkout(self) -> None: + steps = spec()['steps'] + lf = step('Keep LF line endings') + checkout = uses('actions/checkout') + self.assertLess(steps.index(lf), steps.index(checkout)) + self.assertEqual(lf['run'].splitlines(), ['git config --global core.autocrlf false', + 'git config --global core.eol lf']) + + def test_every_later_step_waits_on_the_one_before(self) -> None: + """A failed checkout, setup-python or install skips what needs it.""" + for found, needs in ((uses('actions/setup-python'), 'checkout'), + (step('Install package and test dependencies'), 'python'), + (step('Run unit tests'), 'install'), (step('Upload test timings'), 'install')): + with self.subTest(step=label(found)): + condition = found['if'] + self.assertIn(f"steps.{needs}.outcome == 'success'", condition) + failed = {needs: {'outcome': 'failure'}} + self.assertFalse(evaluate_condition(condition.replace('!cancelled() && ', ''), {'steps': failed})) + + +@unittest.skipIf(shutil.which('bash') is None or shutil.which('sh') is None, 'the step is a bash script') +class TestTheTestsStep(unittest.TestCase): + """``Run unit tests``, run as bash runs it against a stand-in ``python``.""" + + def run_step(self, **env: 'str') -> 'tuple[subprocess.CompletedProcess[str], list[str], str]': + with tempfile.TemporaryDirectory(prefix='pcapkit-2060-') as tmp: + stub = pathlib.Path(tmp, 'python') + stub.write_text(PYTHON_STUB, encoding='utf-8') + stub.chmod(0o755) + record = pathlib.Path(tmp, 'args') + result = bash(step('Run unit tests')['run'], { + 'ANNOTATION_LEVEL': 'warning', 'ARGS_FILE': str(record), 'RUNNER_TEMP': tmp, + 'REAL_PYTHON': sys.executable, 'PATH': tmp + os.pathsep + os.environ.get('PATH', ''), **env}) + argv = record.read_text(encoding='utf-8').splitlines() if record.exists() else [] + return result, argv, tmp + + def test_it_runs_the_all_tier_with_tests_flags(self) -> None: + from tests import _tiers # pylint: disable=import-outside-toplevel + result, argv, tmp = self.run_step() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + latest = list(_tiers.latest_tier_paths()) + self.assertTrue(latest) + self.assertEqual(argv, ['-m', 'pytest', '-q', '-n', 'auto', '--dist', 'load', '--durations=30', + '--durations-min=1.0', f'--junitxml={tmp}/junit.xml', '--ignore=tests/integration', + '--ignore-glob=*_runtime.py', '--ignore-glob=*_regression.py', + *(f'--ignore={path}' for path in latest)]) + + def test_its_flags_are_tests(self) -> None: + """The flags as unit-tests.yml ``test``'s step writes them.""" + flags = re.compile(r"set -- -q -n auto --dist load .*?--ignore-glob='\*_regression\.py'", re.S) + own = flags.search(step('Run unit tests')['run']) + tests = [entry for entry in _workflows.steps('unit-tests.yml', 'test') if entry.get('name') == 'Run unit tests'] + theirs = flags.search(tests[0]['run']) + self.assertIsNotNone(own) + self.assertIsNotNone(theirs) + self.assertEqual(own.group(0).split(), theirs.group(0).split()) # type: ignore[union-attr] + + def test_a_tier_it_cannot_list_fails_the_step_with_a_warning(self) -> None: + result, argv, _ = self.run_step(FAIL_C='1') + self.assertEqual(result.returncode, 1) + self.assertEqual(argv, []) + self.assertIn('::warning title=Could not compute the LATEST tier::', result.stdout) + + +class TestItHoldsLatestBackNowhere(unittest.TestCase): + """The weekly job moves ``latest`` without waiting for a Windows build of it. + + util/update_python_versions.py moves ``latest`` only to a branch with a stable + build on every label a job installs it on, so the Windows leg would hold every + blocking job's ``latest`` back on a non-blocking leg; a non-blocking job's label + does not count. + + """ + + def labels(self, drop_guard: 'bool' = False) -> 'list[str]': + from tests.project.test_python_versions_update_1660 import updater # pylint: disable=import-outside-toplevel + module = updater() + workflows = module.load_workflows(ROOT) + if drop_guard: + del workflows[NAME]['jobs'][JOB]['continue-on-error'] + return module.workflow_labels(workflows)['latest'] + + def test_windows_latest_is_no_latest_label(self) -> None: + self.assertNotIn('windows-latest', self.labels()) + self.assertIn('ubuntu-latest', self.labels()) + + def test_a_blocking_job_there_would_be_one(self) -> None: + """So the check above is the guard's doing, not a job the script cannot see.""" + self.assertIn('windows-latest', self.labels(drop_guard=True)) + + +class TestTheReporters(unittest.TestCase): + """CI Reports and the CI Timing review read it as an experimental, non-blocking leg.""" + + def setUp(self) -> None: + self.name = f'Python {live_outputs()["latest"]} on Windows (experimental)' + + def test_ci_reports_reads_it_as_a_non_blocking_leg(self) -> None: + workflow = check_suite_report.WORKFLOWS[_workflows.load(NAME)['name']] + self.assertTrue(workflow.legs.match(self.name)) + self.assertTrue(check_suite_report.NONBLOCKING_JOB.fullmatch(self.name)) + self.assertTrue(check_suite_report.NONBLOCKING_NOTE.search(LEG)) + + def test_ci_timing_finds_its_caps(self) -> None: + specs = ci_timing.parse_workflow((_workflows.WORKFLOWS / NAME).read_text(encoding='utf-8')) + found = ci_timing.match_job(specs, self.name) + assert found is not None + self.assertEqual((found['id'], found['timeout'], ci_timing.step_cap(found, ci_timing.TESTS_STEP)), + (JOB, JOB_CAP, CAPS['tests'])) + self.assertTrue(ci_timing.non_blocking(NAME, self.name)) + # The Linux legs still match their own job. + linux = ci_timing.match_job(specs, 'Python 3.14t (experimental)') + assert linux is not None + self.assertEqual(linux['id'], 'experimental') + + def test_the_workflows_page_describes_it(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + for phrase in ("``windows`` job runs the ``ALL`` tier on ``windows-latest``", + '``Python on Windows (experimental)``', '``Keep LF line endings``', + 'capped at 5, 10 and 60 minutes within its 80', + ':file:`tests/project/test_windows_test_leg_2060.py`'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, page) + self.assertEqual(sorted(map(int, re.findall(r'capped at (\d+), (\d+) and (\d+) minutes within its (\d+)', + page)[0])), sorted([*CAPS.values(), JOB_CAP])) + + +if __name__ == '__main__': + unittest.main() diff --git a/util/cap_file_size.py b/util/cap_file_size.py index 1261520497..f61848d22a 100644 --- a/util/cap_file_size.py +++ b/util/cap_file_size.py @@ -21,7 +21,8 @@ ignored across ``exec``. :data:`LIMIT` is the one copy of the number; :file:`.github/workflows/experimental.yml` -gives its legs the same cap through ``prlimit --fsize``. +gives its ``experimental`` legs the same cap through ``prlimit --fsize``. Its +``windows`` leg has none: :mod:`resource` is POSIX-only (#2060). """ diff --git a/util/leg_summary.py b/util/leg_summary.py index 995ec877a9..930e2ace96 100644 --- a/util/leg_summary.py +++ b/util/leg_summary.py @@ -13,7 +13,9 @@ :file:`.github/workflows/cron-conda.yml`'s ``Windows summary`` runs it as ``python util/leg_summary.py conda-windows``, since a Windows image has no -``/usr/bin/python3`` (#2060). +``/usr/bin/python3`` (#2060), and :file:`.github/workflows/experimental.yml`'s +``windows`` job runs its ``Experimental summary`` as ``python util/leg_summary.py +experimental`` for the same reason. with ``LEG``, the leg's display name, and ``STEPS``, ``${{ toJSON(steps) }}``, in the environment. ``STEPS`` holds each earlier step with an ``id``, and its diff --git a/util/pip_install.py b/util/pip_install.py index e2d9448d96..d0f915a76e 100644 --- a/util/pip_install.py +++ b/util/pip_install.py @@ -4,7 +4,7 @@ The install step of each :file:`.github/workflows/unit-tests.yml` job that installs the package -- ``test``, ``integration``, ``pypcap-parity`` (and so their KTLO twins), ``unittest-ordering``, ``project-tests`` and ``gate`` -- and of the -``experimental`` job of :file:`.github/workflows/experimental.yml` and the +``experimental`` and ``windows`` jobs of :file:`.github/workflows/experimental.yml` and the ``compatibility`` job of :file:`.github/workflows/python-compatibility.yml` runs, from the checkout's root and on the leg's own interpreter:: diff --git a/util/update_python_versions.py b/util/update_python_versions.py index cb10460797..478ced9ebf 100644 --- a/util/update_python_versions.py +++ b/util/update_python_versions.py @@ -20,7 +20,9 @@ The labels whose builds count are read from the workflows, not written here: the label the Python matrix jobs default to (``matrix.os || 'ubuntu-latest'``), the labels of an ``os`` axis, which run the supported tier, and the ``runs-on`` of -each job that installs ``latest``. A ``-latest`` label counts as its image and +each job that installs ``latest``, but a non-blocking one (``continue-on-error: +true``), such as experimental.yml's Windows leg (#2060), which nothing gating waits +on, so it holds ``latest`` back nowhere. A ``-latest`` label counts as its image and every newer one in the table, since GitHub moves the label to the newest image. The rules are the owner's, from #1633 and #1660: @@ -562,7 +564,8 @@ def workflow_labels(workflows: 'dict[str, Any]') -> 'dict[str, list[str]]': ``default``, plus each label of a literal ``os`` axis (the conda jobs). ``latest`` The ``runs-on`` of every job whose setup-python reads ``latest``, or the - label its ``matrix.os || '...'`` falls back to. + label its ``matrix.os || '...'`` falls back to, but a job with + ``continue-on-error: true``, which fails no run (#2060). """ default, axis, latest = set(), set(), set() # type: set[str], set[str], set[str] @@ -576,8 +579,11 @@ def workflow_labels(workflows: 'dict[str, Any]') -> 'dict[str, list[str]]': if output is None and 'matrix.python-version' not in text: continue # a pin the tier file exempts if output == 'latest': - # #1938: experimental's legs set up `latest` beside their own version, on the - # label `matrix.os || '...'` falls back to, as `default` reads it. + # #2060: a job whose failure fails no run holds `latest` back nowhere, since + # nothing gating waits on it: experimental.yml's `windows`, and the watcher + # experimental's legs set up beside their own version (#1938). + if job.get('continue-on-error') is True: + continue match = DEFAULT_LABEL.fullmatch(runs_on) if match: latest.add(match.group(1))