ci: let docs-only and test-only pull requests skip the heavy matrices (#2243)

.github/workflows/codeql-analysis.yml

@@ -3,8 +3,19 @@ name: "CodeQL"
 on:
   push:
     branches: [main]
+  # #2243, the owner's option B: a docs-only or test-only pull request skips
+  # the scan. util/classify_changes.py's docs and tests, by docs.github.com's
+  # "Avoiding unnecessary scans of pull requests"; the config leaves tests/ out
+  # of the database anyway (#2130), so such a diff scans as its base does. A
+  # path filter rather than the `changes` job, as no ruleset requires a check
+  # of this workflow, so a run that never starts holds nothing pending. A
+  # push and the schedule have no filter.
   pull_request:
     branches: [main]
+    paths-ignore:
+      - 'docs/**'
+      - 'tests/**'
+      - '**.md'
   schedule:
     - cron: '0 2 * * 6'
 

.github/workflows/deploy-pages.yml

@@ -15,8 +15,15 @@ on:
   # Vendor Update (#1634). workflows.rst gives the evidence.
   schedule:
     - cron: '0 2 * * 6' # Saturdays at 02:00 UTC
+  # #2243, the owner's option B: a test-only pull request, one of tests/ files
+  # alone, skips the docs build, which reads none of them; one that touches
+  # docs too still builds. A path filter rather than unit-tests.yml's
+  # `changes` job, as no ruleset requires a check of this workflow, so a run
+  # that never starts holds nothing pending.
   pull_request:
     branches: [main]
+    paths-ignore:
+      - 'tests/**'
   # A build-only run of the dispatched ref: `head`'s test gate and the Sphinx
   # build, whose site is kept as the `pages-html` artifact. Nothing is
   # published to gh-pages and no report is posted.

.github/workflows/experimental.yml

@@ -13,7 +13,8 @@ name: Experimental Tests
 # (experimental)` (#2060); no required context of ruleset 23497679 names one.
 #
 # The same events and `changes` job as unit-tests.yml, so a docs-only pull
-# request skips the legs here as it skips `test` there.
+# request skips the legs here as it skips `test` there, and a test-only one as
+# it skips the KTLO jobs there (#2243).
 on:
   push:
     branches: [main]
@@ -123,6 +124,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       code: ${{ steps.classify.outputs.code }}
+      test-only: ${{ steps.classify.outputs.test-only }}
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -163,7 +165,8 @@ jobs:
     # `changes` or `python-versions` still skips the legs.
     if: >-
       ${{ !cancelled() && needs.changes.result == 'success' && needs.python-versions.result == 'success'
-      && needs.current.outputs.current != 'false' && needs.changes.outputs.code != 'false' }}
+      && needs.current.outputs.current != 'false' && needs.changes.outputs.code != 'false'
+      && needs.changes.outputs.test-only != 'true' }}
     needs: [current, changes, python-versions]
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     timeout-minutes: 140
@@ -626,7 +629,8 @@ jobs:
     name: Python ${{ needs.python-versions.outputs.latest }} on Windows (experimental)
     if: >-
       ${{ !cancelled() && needs.changes.result == 'success' && needs.python-versions.result == 'success'
-      && needs.current.outputs.current != 'false' && needs.changes.outputs.code != 'false' }}
+      && needs.current.outputs.current != 'false' && needs.changes.outputs.code != 'false'
+      && needs.changes.outputs.test-only != 'true' }}
     needs: [current, changes, python-versions]
     runs-on: windows-latest
     timeout-minutes: 80

.github/workflows/python-compatibility.yml

@@ -28,6 +28,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       compatibility: ${{ steps.read.outputs.compatibility }}
+      compatibility-supported: ${{ steps.read.outputs.compatibility-supported }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -42,6 +43,30 @@ jobs:
         id: read
         run: python3 util/python_versions.py --github-output
 
+  # #2243, the owner's option B: unit-tests.yml's `changes`, the same
+  # util/classify_changes.py step. This workflow is never called, so its own
+  # pull_request run is every pull_request run, and every other event, a push
+  # to main included, runs everything. `light` is `true` on a docs-only or a
+  # test-only diff, neither of which can change what this workflow checks:
+  # pcapkit/ and util/ compile and import as on the base branch.
+  changes:
+    name: Classify changed files
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    outputs:
+      light: ${{ steps.classify.outputs.code == 'false' || steps.classify.outputs.test-only == 'true' }}
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 2
+
+      - name: Classify the diff
+        id: classify
+        env:
+          OWN_PR: ${{ github.event_name == 'pull_request' }}
+        run: python3 util/classify_changes.py
+
   compatibility:
     # `Compat`, so `gh pr checks` cannot show two `Python 3.N` rows, this and
     # Unit Tests', with nothing to tell them apart.
@@ -52,8 +77,19 @@ jobs:
     # as in unit-tests.yml's `test` job: job-level, a failed leg's check run
     # concludes `failure`, so every commit shows red (#1672). Nothing below
     # `requires-python`'s 3.8 runs (#2051).
+    #
+    # #2243: the supported legs are required contexts, so a docs-only or
+    # test-only pull request still runs them, by name, but they skip every
+    # step that sets up, installs or checks, and pass. A job-level `if:` would
+    # not do: a matrix job it skips reports one check under its unexpanded
+    # name, as Unit Tests' docs-only run 40238795866 did, which matches no
+    # required context, so each would stay pending.
+    # The KTLO legs, required by nothing, are left out of that matrix.
+    # `!cancelled()` replaces the implicit `success()`, so a failed `changes`,
+    # whose `light` is then empty, runs every leg in full.
     name: Compat Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }}
-    needs: [python-versions]
+    needs: [python-versions, changes]
+    if: ${{ !cancelled() && needs.python-versions.result == 'success' }}
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     env:
       ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }}
@@ -63,14 +99,24 @@ jobs:
       # the `os` pins: a KTLO leg carries `ktlo: true`, and `os` where
       # `ubuntu-latest` has no setup-python build for it. A tier change edits
       # ruleset 23497679 too.
-      matrix: ${{ fromJSON(needs.python-versions.outputs.compatibility) }}
+      matrix: ${{ fromJSON(needs.changes.outputs.light == 'true' && needs.python-versions.outputs.compatibility-supported || needs.python-versions.outputs.compatibility) }}
     steps:
+      - name: Skip on a docs-only or test-only diff
+        if: ${{ needs.changes.outputs.light == 'true' }}
+        env:
+          LEG: Python ${{ matrix.python-version }}
+        run: echo "::notice title=Nothing to check on ${LEG}::A docs-only or test-only diff leaves pcapkit/ and util/ as on the base branch, so this leg passes without checking them (#2243)."
+
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         id: checkout
+        if: ${{ needs.changes.outputs.light != 'true' }}
         continue-on-error: ${{ matrix.ktlo == true }}
 
+      # Every later step waits on this one's `outcome`, so on a light diff,
+      # where it skips, they skip too.
       - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         id: python
+        if: ${{ needs.changes.outputs.light != 'true' }}
         continue-on-error: ${{ matrix.ktlo == true }}
         with:
           python-version: ${{ matrix.python-version }}

.github/workflows/unit-tests.yml

@@ -54,8 +54,10 @@ jobs:
     outputs:
       latest: ${{ steps.read.outputs.latest }}
       test: ${{ steps.read.outputs.test }}
+      test-latest: ${{ steps.read.outputs.test-latest }}
       test-ktlo: ${{ steps.read.outputs.test-ktlo }}
       integration: ${{ steps.read.outputs.integration }}
+      integration-latest: ${{ steps.read.outputs.integration-latest }}
       integration-ktlo: ${{ steps.read.outputs.integration-ktlo }}
       pypcap-parity: ${{ steps.read.outputs.pypcap-parity }}
       pypcap-parity-ktlo: ${{ steps.read.outputs.pypcap-parity-ktlo }}
@@ -101,7 +103,11 @@ jobs:
       # `include` marks one leg `coverage: true`, `latest` (#1661), so coverage
       # adds no leg to the queue. Uncovered, 3.15's tests step took a median
       # 8.1 minutes and coverage cost about 1.8x, which fits the job's 45.
-      matrix: ${{ fromJSON(needs.python-versions.outputs.test) }}
+      #
+      # #2243, the owner's option B: a test-only pull request runs the
+      # `latest` leg alone, the coverage leg, which runs the whole suite with
+      # `integration`'s `latest` leg and `project-tests`. See `required-checks`.
+      matrix: ${{ fromJSON(needs.changes.outputs.test-only == 'true' && needs.python-versions.outputs.test-latest || needs.python-versions.outputs.test) }}
 
     # `&test-steps`: `test-ktlo` runs these same steps, by YAML alias.
     steps: &test-steps
@@ -343,7 +349,7 @@ jobs:
   # leg before any step runs, which only the job-level guard contains.
   test-ktlo:
     name: Python ${{ matrix.python-version }} (KTLO)
-    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }}
+    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' && needs.changes.outputs.test-only != 'true' }}
     needs: [changes, python-versions]
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     timeout-minutes: 45
@@ -367,8 +373,9 @@ jobs:
     strategy:
       fail-fast: false
       # The supported tier of the `test` job above, without its coverage leg;
-      # the KTLO legs are `integration-ktlo`'s (#1738).
-      matrix: ${{ fromJSON(needs.python-versions.outputs.integration) }}
+      # the KTLO legs are `integration-ktlo`'s (#1738). On a test-only pull
+      # request, the `latest` leg alone, as in `test` (#2243).
+      matrix: ${{ fromJSON(needs.changes.outputs.test-only == 'true' && needs.python-versions.outputs.integration-latest || needs.python-versions.outputs.integration) }}
 
     # `&integration-steps`: `integration-ktlo` runs these same steps.
     steps: &integration-steps
@@ -513,7 +520,7 @@ jobs:
   # `integration`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738).
   integration-ktlo:
     name: Integration Python ${{ matrix.python-version }} (KTLO)
-    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }}
+    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' && needs.changes.outputs.test-only != 'true' }}
     needs: [changes, python-versions]
     runs-on: ${{ matrix.os || 'ubuntu-latest' }}
     timeout-minutes: 45
@@ -562,7 +569,7 @@ jobs:
   # run only what `test` runs.
   engine-tests:
     name: Engines (${{ matrix.engine }})
-    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }}
+    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' && needs.changes.outputs.test-only != 'true' }}
     needs: [changes, python-versions]
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -900,7 +907,7 @@ jobs:
   # `pypcap-parity-ktlo`'s (#1738).
   pypcap-parity:
     name: PyPCAP/PyPCAPFile parity Python ${{ matrix.python-version }}
-    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }}
+    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' && needs.changes.outputs.test-only != 'true' }}
     needs: [changes, python-versions]
     runs-on: ubuntu-latest
     timeout-minutes: 45
@@ -1087,7 +1094,7 @@ jobs:
   # Same apt set and cache key; `pypcap-parity`'s supported leg saves it.
   pypcap-parity-ktlo:
     name: PyPCAP/PyPCAPFile parity Python ${{ matrix.python-version }} (KTLO)
-    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }}
+    if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' && needs.changes.outputs.test-only != 'true' }}
     needs: [changes, python-versions]
     runs-on: ubuntu-latest
     timeout-minutes: 45
@@ -1245,8 +1252,13 @@ jobs:
 
   # Which of a pull request's changed files are code, which decides whether
   # the expensive legs run at all. util/classify_changes.py, which
-  # experimental.yml's `changes` runs too (#1967), fails safe: anything it
-  # cannot classify is code.
+  # experimental.yml's and python-compatibility.yml's `changes` run too
+  # (#1967), fails safe: anything it cannot classify is code.
+  #
+  # #2243: `test-only` is `true` on a diff of tests and docs alone. `test` and
+  # `integration` then run their `latest` leg alone, and the KTLO jobs,
+  # `engine-tests` and `pypcap-parity` skip by their own `if:`, as on a
+  # docs-only diff (`code: false`).
   #
   # Only this workflow's own `pull_request` run is classified; every other
   # event reports `code: true`. A `workflow_call` sees the caller's
@@ -1260,6 +1272,7 @@ jobs:
     timeout-minutes: 5
     outputs:
       code: ${{ steps.classify.outputs.code }}
+      test-only: ${{ steps.classify.outputs.test-only }}
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -1448,8 +1461,14 @@ jobs:
   # `if:`, and `project-tests` runs the docs readers as well. A job-level `if:`
   # rather than workflow-level `paths-ignore`, which would leave this required
   # context pending forever. So `skipped` is accepted for those legs **only**
-  # when `changes` succeeded and said `code == 'false'`; `project-tests` must
-  # have succeeded either way, and every other combination fails this gate.
+  # when `changes` succeeded and said `code == 'false'`.
+  #
+  # The test-only path (#2243). On one whose diff touches tests and docs alone,
+  # `test` and `integration` run their `latest` leg alone and must succeed, and
+  # `engine-tests` and `pypcap-parity` are skipped by their own `if:`. So
+  # `skipped` is accepted for those two **only** when `changes` succeeded and
+  # said `code == 'true'` and `test-only == 'true'`. `project-tests` must have
+  # succeeded on every path, and every other combination fails this gate.
   required-checks:
     name: Required checks passed
     needs: [changes, test, integration, engine-tests, pypcap-parity, project-tests]
@@ -1462,48 +1481,60 @@ jobs:
         env:
           CHANGES: ${{ needs.changes.result }}
           CODE: ${{ needs.changes.outputs.code }}
+          TEST_ONLY: ${{ needs.changes.outputs.test-only }}
           TEST: ${{ needs.test.result }}
           INTEGRATION: ${{ needs.integration.result }}
           ENGINE_TESTS: ${{ needs.engine-tests.result }}
           PYPCAP_PARITY: ${{ needs.pypcap-parity.result }}
           PROJECT_TESTS: ${{ needs.project-tests.result }}
         run: |
           set -eu
-          echo "changes: $CHANGES (code: '$CODE')"
+          echo "changes: $CHANGES (code: '$CODE', test-only: '$TEST_ONLY')"
           echo "test: $TEST"
           echo "integration: $INTEGRATION"
           echo "engine-tests: $ENGINE_TESTS"
           echo "pypcap-parity: $PYPCAP_PARITY"
           echo "project-tests: $PROJECT_TESTS"
 
           failed=0
-          if [ "$CHANGES" != "success" ] || { [ "$CODE" != "true" ] && [ "$CODE" != "false" ]; }; then
-            echo "::error title=changes did not classify the diff::result was '$CHANGES' and code was '$CODE'; without a classification nothing may be skipped."
+          # A docs-only diff is never test-only too, so that pair is garbled.
+          if [ "$CHANGES" != "success" ] || { [ "$CODE" != "true" ] && [ "$CODE" != "false" ]; } \
+              || { [ "$TEST_ONLY" != "true" ] && [ "$TEST_ONLY" != "false" ]; } \
+              || { [ "$CODE" = "false" ] && [ "$TEST_ONLY" = "true" ]; }; then
+            echo "::error title=changes did not classify the diff::result was '$CHANGES', code was '$CODE' and test-only was '$TEST_ONLY'; without a classification nothing may be skipped."
             CODE=true
+            TEST_ONLY=false
             failed=1
           fi
 
           # project-tests runs the project pins on every diff (#1954), and the
-          # docs readers on a docs-only one, so it must pass on both paths.
-          # Docs-only (code == false): a leg may be skipped. Otherwise every leg
-          # must have succeeded.
+          # docs readers on a docs-only one, so it must pass on every path.
+          # Docs-only (code == false): any leg may be skipped. Test-only
+          # (test-only == true): engine-tests and pypcap-parity may be. Otherwise
+          # every leg must have succeeded.
           [ "$PROJECT_TESTS" = "success" ] || { echo "::error title=project-tests did not succeed::result was '$PROJECT_TESTS'; it runs the project pins, and on a docs-only diff the docs readers too, so it has to pass."; failed=1; }
           if [ "$CODE" = "false" ]; then
-            allowed="success skipped"
+            may_skip="test integration engine-tests pypcap-parity"
+          elif [ "$TEST_ONLY" = "true" ]; then
+            may_skip="engine-tests pypcap-parity"
           else
-            allowed="success"
+            may_skip=""
           fi
           for pair in "test=$TEST" "integration=$INTEGRATION" "engine-tests=$ENGINE_TESTS" "pypcap-parity=$PYPCAP_PARITY"; do
             job="${pair%%=*}"
             result="${pair#*=}"
+            case " $may_skip " in
+              *" $job "*) allowed="success skipped" ;;
+              *) allowed="success" ;;
+            esac
             case " $allowed " in
               *" $result "*) ;;
-              *) echo "::error title=$job did not succeed::result was '$result'; accepted here: $allowed (code: '$CODE')."; failed=1 ;;
+              *) echo "::error title=$job did not succeed::result was '$result'; accepted here: $allowed (code: '$CODE', test-only: '$TEST_ONLY')."; failed=1 ;;
             esac
           done
 
           if [ "$failed" -ne 0 ]; then
-            echo "::error title=Required checks did not all succeed::See the per-job lines above. A skipped or cancelled dependency is only a pass on a docs-only diff; see this job's own comment."
+            echo "::error title=Required checks did not all succeed::See the per-job lines above. A skipped dependency is a pass only on a docs-only or test-only diff, and a cancelled one never; see this job's own comment."
             exit 1
           fi
           echo "All required jobs reported an accepted result."