ci: add a non-blocking Windows test leg to Experimental Tests (#2060)

.github/workflows/experimental.yml

@@ -9,7 +9,8 @@ name: Experimental Tests
 # `needs:` one in another workflow.
 #
 # Its check names, `Python <version> (experimental)`, are the ones the legs had
-# in unit-tests.yml; no required context of ruleset 23497679 names one.
+# in unit-tests.yml, and `windows`'s is `Python <latest> on Windows
+# (experimental)` (#2060); no required context of ruleset 23497679 names one.
 #
 # The same events and `changes` job as unit-tests.yml, so a docs-only pull
 # request skips the legs here as it skips `test` there.
@@ -606,3 +607,132 @@ jobs:
             ${{ runner.temp }}/job-summary.name
           if-no-files-found: ignore
           retention-days: 7
+
+  # #2060: the unit tier on Windows before the win-64 conda package is
+  # published, so its first release has had a test run. One leg, on `latest`,
+  # the newest CPython the conda builds target. Non-blocking by `experimental`'s
+  # mechanics: every step that can fail is `continue-on-error`, the long ones
+  # have caps under the job's, `Experimental summary` reports each failure as a
+  # warning, and the job-level guard covers a lost runner. None of
+  # `experimental`'s Linux containment runs here: no scope or `prlimit`, nor
+  # util/cap_file_size.py, which needs `resource`; no resource monitor, which
+  # reads /proc; no superseded watcher, which signals a process group. So the
+  # tests' cap is the step's own. Every `run:` is Git Bash's.
+  windows:
+    name: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental)
+    if: >-
+      ${{ !cancelled() && needs.changes.result == 'success' && needs.python-versions.result == 'success'
+      && needs.current.outputs.current != 'false' && needs.changes.outputs.code != 'false' }}
+    needs: [current, changes, python-versions]
+    runs-on: windows-latest
+    timeout-minutes: 80
+    continue-on-error: true
+    env:
+      ANNOTATION_LEVEL: warning
+
+    steps:
+      # The image's Git checks text out with CRLF, which `* text=auto` leaves to
+      # it; LF keeps the tree the tests read the one the other legs read.
+      - name: Keep LF line endings
+        id: eol
+        continue-on-error: true
+        shell: bash
+        run: |
+          git config --global core.autocrlf false
+          git config --global core.eol lf
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        id: checkout
+        continue-on-error: true
+
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        id: python
+        if: ${{ steps.checkout.outcome == 'success' }}
+        continue-on-error: true
+        timeout-minutes: 5
+        with:
+          python-version: ${{ needs.python-versions.outputs.latest }}
+          cache: pip
+
+      - name: Install package and test dependencies
+        id: install
+        if: ${{ steps.python.outcome == 'success' }}
+        continue-on-error: true
+        timeout-minutes: 10
+        shell: bash
+        env:
+          LEG: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental, non-blocking)
+        run: python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'
+
+      - name: Report available parallelism
+        id: parallelism
+        if: ${{ steps.install.outcome == 'success' }}
+        continue-on-error: true
+        shell: bash
+        run: python -c "import os; print('cpu_count', os.cpu_count())"
+
+      # `experimental`'s selection, the `ALL` tier (#1958), asked of
+      # tests/_tiers.py on the leg's own CPython, as `test` asks it.
+      - name: Run unit tests
+        id: tests
+        if: ${{ steps.install.outcome == 'success' }}
+        continue-on-error: true
+        timeout-minutes: 60
+        shell: bash
+        run: |
+          if ! skipped=$(python -c "from tests._tiers import latest_tier_paths; print(' '.join(latest_tier_paths()))"); then
+            echo "::${ANNOTATION_LEVEL} title=Could not compute the LATEST tier::tests._tiers.latest_tier_paths() failed -- see the traceback above."
+            exit 1
+          fi
+          set -- -q -n auto --dist load \
+            --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \
+            --ignore=tests/integration \
+            --ignore-glob='*_runtime.py' \
+            --ignore-glob='*_regression.py'
+          for module in $skipped; do
+            set -- "$@" --ignore="$module"
+          done
+          python -m pytest "$@"
+
+      - name: Upload test timings
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
+        continue-on-error: true
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: junit-experimental-windows-${{ needs.python-versions.outputs.latest }}
+          path: ${{ runner.temp }}/junit.xml
+          if-no-files-found: warn
+          retention-days: 14
+
+      # `experimental`'s, on setup-python's `python`, or the image's where that
+      # failed, since Windows has no /usr/bin/python3, as cron-conda.yml's
+      # `Windows summary` runs it (#2060).
+      - name: Experimental summary
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+        shell: bash
+        env:
+          LEG: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental, non-blocking)
+          STEPS: ${{ toJSON(steps) }}
+          JOB_SUMMARY_COPY: ${{ runner.temp }}/job-summary.md
+        run: python util/leg_summary.py experimental
+
+      - name: Name the job summary
+        if: ${{ always() }}
+        continue-on-error: true
+        shell: bash
+        env:
+          JOB_NAME: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental)
+        run: if [ -s "$RUNNER_TEMP/job-summary.md" ]; then printf '%s\n' "$JOB_NAME" > "$RUNNER_TEMP/job-summary.name"; fi
+
+      - name: Upload the job summary
+        if: ${{ always() }}
+        continue-on-error: true
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: job-summary-${{ github.job }}-${{ strategy.job-index }}
+          path: |
+            ${{ runner.temp }}/job-summary.md
+            ${{ runner.temp }}/job-summary.name
+          if-no-files-found: ignore
+          retention-days: 7

docs/source/contributing/workflows.rst

@@ -1030,6 +1030,30 @@ workers rather than 4, GraalPy's tests should take about 117-151 minutes (an inf
 from the 88-113 above, not measured), so its leg may end on the 110-minute timeout as a
 warning: that is expected, blocks nothing, and leaves the caps as they are.
 
+**The Windows leg.** :file:`experimental.yml`'s ``windows`` job runs the ``ALL`` tier
+on ``windows-latest`` under ``latest``, the newest CPython the conda builds target, as
+``Python <latest> on Windows (experimental)`` (`#2060
+<https://github.com/JarryShaw/PyPCAPKit/issues/2060>`__), so the first published
+``win-64`` conda package has had a test run. It needs ``current``, ``changes`` and
+``python-versions`` and runs on ``experimental``'s condition, and it is non-blocking by
+the same mechanics: every step continues on error, setup-python, the install and ``Run
+unit tests`` are capped at 5, 10 and 60 minutes within its 80, the job carries
+``continue-on-error: true``, and ``Experimental summary`` reports each failed step as a
+warning, on setup-python's ``python`` or the image's, since Windows has no
+``/usr/bin/python3``. Every ``run:`` is Git Bash's (``shell: bash``). ``Keep LF line
+endings`` sets ``core.autocrlf false`` and ``core.eol lf`` before the checkout, so the
+tests read the tree the other legs read, not the CRLF one the image's Git writes under
+``* text=auto``. ``Run unit tests`` asks ``latest_tier_paths()`` on the leg's CPython,
+as ``test`` does, and passes ``test``'s flags. None of the Linux containment above runs
+there: Windows has no cgroup scope, ``prlimit`` or ``RLIMIT_FSIZE``, so no
+:file:`util/cap_file_size.py`; no ``/proc`` for the resource monitor; and no process
+group for :file:`util/superseded.py`'s watcher, so the step's cap bounds the tests, and
+a run superseded once the leg has started runs it to its end. Python Versions' weekly
+job moves ``latest`` only to a branch setup-python builds for every label a job
+installs it on, but counts no job with ``continue-on-error: true``, so this leg
+never holds ``latest`` back for the blocking jobs.
+:file:`tests/project/test_windows_test_leg_2060.py` pins the job.
+
 Test Tiers
 ----------
 
@@ -1103,7 +1127,8 @@ Which legs run which tier:
      - ``ALL`` and ``LATEST``, less the project pins
    * - ``project-tests``
      - the project pins
-   * - ``test`` off ``latest``, PyPy's among them, ``test-ktlo``, ``experimental``
+   * - ``test`` off ``latest``, PyPy's among them, ``test-ktlo``, ``experimental``,
+       ``windows``
      - ``ALL``
    * - ``integration`` on ``latest``
      - ``ALL`` and ``LATEST``
@@ -1198,8 +1223,9 @@ a lower hard limit, and execs the command, so the step's exit status and
 inherits the cap. A write past it raises ``OSError: [Errno 27] File too large`` in
 Python, since CPython ignores ``SIGXFSZ``, so the test errors; any other program is
 killed by the signal, whose default action the script restores before it execs.
-``experimental.yml``'s legs get the same 2 GiB through ``prlimit --fsize`` (see
-above).
+``experimental.yml``'s ``experimental`` legs get the same 2 GiB through
+``prlimit --fsize`` (see above); its ``windows`` leg has no cap, as Windows has no
+``RLIMIT_FSIZE``.
 
 :file:`tests/project/test_file_size_caps_1969.py` fails when a command in
 ``unit-tests.yml`` runs pytest, unittest or :file:`util/run_unittest_leg.py`

tests/project/test_pip_install_1967.py

@@ -13,8 +13,8 @@
   its guards and environment, and every other ``pip install`` in
   ``unit-tests.yml`` to a listed step that keeps its shell.
 * :class:`TestTheOtherWorkflowsRunTheScript` does the same for the install of
-  ``experimental.yml``'s ``experimental`` and ``python-compatibility.yml``'s
-  ``compatibility``.
+  ``experimental.yml``'s ``experimental`` and ``windows`` (#2060) and
+  ``python-compatibility.yml``'s ``compatibility``.
 
 """
 
@@ -91,6 +91,17 @@
         'env': {'LEG': 'Python ${{ matrix.python-version }} (experimental, non-blocking)'},
         'run': "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'",
     }, 'warning'),
+    # #2060: the Windows leg's, the same install on `latest`.
+    ('experimental.yml', 'windows'): ({
+        'name': 'Install package and test dependencies',
+        'id': 'install',
+        'if': "${{ steps.python.outcome == 'success' }}",
+        'continue-on-error': True,
+        'timeout-minutes': 10,
+        'shell': 'bash',
+        'env': {'LEG': 'Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental, non-blocking)'},
+        'run': "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'",
+    }, 'warning'),
     ('python-compatibility.yml', 'compatibility'): ({
         'name': 'Install package',
         'id': 'install',

tests/project/test_resource_monitor_1740.py

@@ -18,6 +18,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 import pathlib
 import re
@@ -566,7 +567,10 @@ def test_no_reader_takes_the_artifact_for_a_junit_one(self) -> None:
         self.assertTrue(ci_timing.JUNIT_ARTIFACT.startswith('junit-'))
         self.assertEqual([path.name for path in sorted((ROOT / 'util').glob('*.py'))
                           if 'resource-monitor' in path.read_text(encoding='utf-8')], [])
-        self.assertEqual(list(workflow(EXPERIMENTAL[0])['jobs']), ['current', 'python-versions', 'changes', 'experimental'])
+        # #2060's `windows` runs no monitor and downloads nothing.
+        jobs = workflow(EXPERIMENTAL[0])['jobs']
+        self.assertEqual(list(jobs), ['current', 'python-versions', 'changes', 'experimental', 'windows'])
+        self.assertNotIn('resource-monitor', json.dumps(jobs['windows']))
 
 
 class TestTheDocsSayIt(unittest.TestCase):

tests/project/test_superseded_runs_1938.py

@@ -209,7 +209,8 @@ def test_the_watcher_runs_on_a_cpython_of_its_own_with_requests(self) -> None:
 
     def test_every_other_job_needs_it_and_skips_on_its_output(self) -> None:
         others = [job_id for job_id in self.jobs if job_id != GATE]
-        self.assertEqual(sorted(others), ['changes', 'experimental', 'python-versions'])
+        # #2060: `windows`, the Windows leg, skips with the rest.
+        self.assertEqual(sorted(others), ['changes', 'experimental', 'python-versions', 'windows'])
         for job_id in others:
             body = self.jobs[job_id]
             needs = body['needs'] if isinstance(body['needs'], list) else [body['needs']]
@@ -237,7 +238,7 @@ def run_if(self, job_id: 'str', current: 'str', code: 'str' = 'true', *, gate: '
         return runs
 
     def test_a_superseded_commit_skips_every_job_and_anything_else_runs_them(self) -> None:
-        for job_id in ('python-versions', 'changes', 'experimental'):
+        for job_id in ('python-versions', 'changes', 'experimental', 'windows'):
             with self.subTest(job=job_id):
                 self.assertFalse(self.run_if(job_id, 'false'))
                 self.assertTrue(self.run_if(job_id, 'true'))
@@ -248,16 +249,17 @@ def test_a_superseded_commit_skips_every_job_and_anything_else_runs_them(self) -
                 self.assertTrue(self.run_if(job_id, '', gate='cancelled'))
                 # A cancelled run stays cancelled.
                 self.assertFalse(self.run_if(job_id, 'true', cancelled=True))
-        # A docs-only pull request still skips the legs.
-        self.assertFalse(self.run_if('experimental', 'true', code='false'))
-        # And the legs still need the jobs they read: `!cancelled()` drops only the implicit `success()`.
-        for job_id in ('changes', 'python-versions'):
-            for result in ('failure', 'skipped', 'cancelled'):
-                with self.subTest(needs=job_id, result=result):
-                    self.assertFalse(self.run_if('experimental', 'true', **{
-                        'versions' if job_id == 'python-versions' else 'changes': result}))
-        # Superseded: `changes` and `python-versions` skip, and the legs with them.
-        self.assertFalse(self.run_if('experimental', 'false', versions='skipped', changes='skipped'))
+        for legs in ('experimental', 'windows'):
+            # A docs-only pull request still skips the legs.
+            self.assertFalse(self.run_if(legs, 'true', code='false'))
+            # And the legs still need the jobs they read: `!cancelled()` drops only the implicit `success()`.
+            for job_id in ('changes', 'python-versions'):
+                for result in ('failure', 'skipped', 'cancelled'):
+                    with self.subTest(legs=legs, needs=job_id, result=result):
+                        self.assertFalse(self.run_if(legs, 'true', **{
+                            'versions' if job_id == 'python-versions' else 'changes': result}))
+            # Superseded: `changes` and `python-versions` skip, and the legs with them.
+            self.assertFalse(self.run_if(legs, 'false', versions='skipped', changes='skipped'))
 
     def test_the_legs_and_the_tests_step_read_with_the_token(self) -> None:
         self.assertEqual(set(job(NAME, 'experimental')['permissions'].values()), {'read'})

tests/project/test_sweeps_on_latest_only_1732.py

@@ -530,7 +530,8 @@ def test_a_failure_never_turns_the_run_red(self) -> None:
         # #2030: the last step but the two that name and upload the job's summary.
         self.assertEqual([entry['name'] for entry in spec['steps'][-3:]],
                          ['Experimental summary', 'Name the job summary', 'Upload the job summary'])
-        expected = {'experimental.yml': ['experimental'],
+        # #2060: the Windows leg's job, as tests/project/test_windows_test_leg_2060.py pins.
+        expected = {'experimental.yml': ['experimental', 'windows'],
                     'unit-tests.yml': ['test-ktlo', 'integration-ktlo', 'pypcap-parity-ktlo']}
         for name in ('unit-tests.yml', 'experimental.yml'):
             with self.subTest(workflow=name):

tests/project/test_unit_tests_push_reports_2030.py

@@ -94,7 +94,8 @@
     ('unit-tests.yml', 'test'), ('unit-tests.yml', 'test-ktlo'), ('unit-tests.yml', 'integration'),
     ('unit-tests.yml', 'integration-ktlo'), ('unit-tests.yml', 'engine-tests'),
     ('unit-tests.yml', 'pypcap-parity'), ('unit-tests.yml', 'pypcap-parity-ktlo'),
-    ('experimental.yml', 'experimental'), ('python-compatibility.yml', 'compatibility'),
+    ('experimental.yml', 'experimental'), ('experimental.yml', 'windows'),
+    ('python-compatibility.yml', 'compatibility'),
 )
 #: The steps that copy the summary through util/_actions.py's JOB_SUMMARY_COPY.
 COPYING = ('Report coverage', 'KTLO summary', 'Experimental summary')
@@ -424,10 +425,11 @@ def test_each_job_names_and_uploads_it(self) -> None:
                 for step in (save, upload):
                     self.assertEqual(step['if'], '${{ always() }}')
                     self.assertIs(step['continue-on-error'], True)
-                # The name the job's check run carries, on every leg.
-                for leg in legs(name, job_id):
-                    self.assertEqual(interpolate(save['env']['JOB_NAME'], {'matrix': leg}),
-                                     interpolate(job['name'], {'matrix': leg}))
+                # The name the job's check run carries, on every leg; #2060's `windows` has one.
+                for leg in (legs(name, job_id) if 'strategy' in job else [{}]):
+                    context = {'matrix': leg, 'needs': NEEDS}
+                    self.assertEqual(interpolate(save['env']['JOB_NAME'], context),
+                                     interpolate(job['name'], context))
 
     @unittest.skipIf(not BASH, 'needs bash')
     def test_the_name_is_written_beside_a_summary_alone(self) -> None: