diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
new file mode 100644
index 0000000000..a6a7a03293
--- /dev/null
+++ b/.github/workflows/benchmark.yml
@@ -0,0 +1,288 @@
+name: Engine Benchmark
+
+# #2125: times every extraction engine on every interpreter the engines are
+# tested on, so the docs' Engine Support speed table no longer goes stale
+# between hand runs (#2109). Natively, through examples/benchmark/native.sh
+# rather than run.sh: a hosted runner is already a fresh VM per job, so
+# containers would add a build per version and no isolation, and a native run
+# makes macOS and Windows runners candidates too. One job measures every
+# version, since the per-version table's absolute times compare only within
+# one machine: by default `ubuntu-26.04-arm`, the fastest runner that
+# measures every environment and engine. What a native run does not pin --
+# the interpreter's patch release, tshark and libpcap -- is recorded in the
+# tables; examples/benchmark/README.md's "On GitHub Actions" says more, and
+# gives the runner figures.
+#
+# The `discussion` job posts the ratio table to this job's thread in the
+# Discussions category REPORT_CATEGORY names (#2030); it alone holds a write
+# scope, and a report it cannot post is a warning. Nothing here pushes, and the
+# docs stay hand-updated from the report.
+on:
+ schedule:
+ # Wednesdays 03:17 UTC: off the hour, where GitHub delays scheduled runs
+ # most, and midweek, clear of the Saturday crons (#1967), Monday's Python
+ # Versions and CI Timing, and Project Status at 00:37.
+ - cron: '17 3 * * 3'
+ workflow_dispatch:
+ inputs:
+ runner:
+ description: Runner to measure on; dispatch once per runner to compare them.
+ type: choice
+ default: ubuntu-26.04-arm
+ options:
+ - ubuntu-26.04-arm
+ - ubuntu-latest
+ - macos-latest
+ - windows-latest
+ pythons:
+ description: Comma-separated Python versions to measure (empty = every version the engines are tested on).
+ type: string
+ default: ''
+ engines:
+ description: Comma-separated engines, including default (empty = all).
+ type: string
+ default: ''
+ rounds:
+ description: Timed extractions per engine per pass (empty = benchmark.py's default).
+ type: string
+ default: ''
+ repeats:
+ description: Passes over the whole engine set (empty = benchmark.py's default).
+ type: string
+ default: ''
+
+permissions:
+ contents: read
+
+# One run per runner at a time; dispatches on different runners still run side
+# by side, which is how two runners are compared.
+concurrency:
+ group: benchmark-${{ inputs.runner || 'ubuntu-26.04-arm' }}
+ cancel-in-progress: false
+
+# #2030: one Discussions category holds every report with no pull request to
+# comment on, one thread per job.
+env:
+ REPORT_CATEGORY: CI reports
+
+jobs:
+ # #1660: the interpreters are the engine tiers of .github/python-versions.toml,
+ # as unit-tests.yml's `engine-tests` installs them.
+ python-versions:
+ name: Read the Python versions
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ outputs:
+ engine-supported: ${{ steps.read.outputs.engine-supported }}
+ engine-ktlo: ${{ steps.read.outputs.engine-ktlo }}
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ sparse-checkout: |
+ .github/python-versions.toml
+ util/_actions.py
+ util/python_versions.py
+ sparse-checkout-cone-mode: false
+ persist-credentials: false
+
+ - name: Read .github/python-versions.toml
+ id: read
+ run: python3 util/python_versions.py --github-output
+
+ benchmark:
+ name: Benchmark on ${{ matrix.os }}
+ needs: [ python-versions ]
+ # One leg, on the dispatch's runner, or `ubuntu-26.04-arm` on the schedule,
+ # which has no inputs; spelt as the other matrix jobs spell their runner.
+ runs-on: ${{ matrix.os || 'ubuntu-26.04-arm' }}
+ # Unmeasured on a hosted runner; the first dispatch measures it. Locally a
+ # full matrix took about two hours (the README's "How long it takes"),
+ # nearly all of it a tshark per pyshark extraction, so this leaves room for
+ # a slower runner under the six-hour job limit.
+ timeout-minutes: 300
+ strategy:
+ matrix:
+ os: [ "${{ inputs.runner || 'ubuntu-26.04-arm' }}" ]
+ defaults:
+ run:
+ shell: bash
+ outputs:
+ status: ${{ steps.measure.outputs.status }}
+ env:
+ # Read by util/apt_packages.py install: libpcap's headers for pypcap's
+ # build, and tshark for pyshark.
+ APT_PACKAGES: libpcap-dev tshark
+ APT_SET: benchmark
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ # As in `engine-tests`: each interpreter is then reachable, and the KTLO
+ # one gets a step of its own so that losing it cannot fail the job.
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ if: ${{ needs.python-versions.outputs.engine-ktlo != '' }}
+ continue-on-error: true
+ with:
+ python-version: ${{ needs.python-versions.outputs.engine-ktlo }}
+
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: ${{ needs.python-versions.outputs.engine-supported }}
+
+ # Without its system package an engine is a row with its reason, so a
+ # failed install warns rather than failing the job.
+ - name: Install libpcap headers and tshark
+ if: runner.os == 'Linux'
+ timeout-minutes: 10
+ run: |
+ /usr/bin/python3 util/apt_packages.py install \
+ || echo '::warning title=System packages not installed::pypcap and pyshark may be reported unmeasured.'
+
+ # #2019: tshark's AppArmor profile on Ubuntu 25.04 and later; see
+ # unit-tests.yml's `engine-tests` step of the same name, whose body this is.
+ - name: Let tshark read the checkout past its AppArmor profile
+ if: runner.os == 'Linux'
+ run: |
+ set -eu
+ if [ -f /etc/apparmor.d/tshark ]; then
+ sudo mkdir -p /etc/apparmor.d/local
+ echo "file r ${GITHUB_WORKSPACE}/**," | sudo tee -a /etc/apparmor.d/local/tshark >/dev/null
+ sudo apparmor_parser -r /etc/apparmor.d/tshark
+ fi
+
+ # macOS ships libpcap; Homebrew's `wireshark` formula is the tshark CLI.
+ - name: Install tshark with Homebrew
+ if: runner.os == 'macOS'
+ timeout-minutes: 20
+ env:
+ HOMEBREW_NO_AUTO_UPDATE: '1'
+ HOMEBREW_NO_INSTALL_CLEANUP: '1'
+ run: |
+ brew install wireshark \
+ || echo '::warning title=tshark not installed::pyshark will be reported unmeasured.'
+
+ # Reading a file needs no capture driver. Npcap's free edition cannot
+ # install unattended, so the pcap engines report the missing libpcap.
+ - name: Install tshark with Chocolatey
+ if: runner.os == 'Windows'
+ timeout-minutes: 15
+ run: |
+ if choco install wireshark --yes --no-progress; then
+ echo 'C:\Program Files\Wireshark' >> "$GITHUB_PATH"
+ else
+ echo '::warning title=tshark not installed::pyshark will be reported unmeasured.'
+ fi
+
+ # native.sh's exit status 3 is a table with gaps, each explained in it,
+ # so it warns; anything else but 0 is no table and fails.
+ - name: Run examples/benchmark/native.sh
+ id: measure
+ env:
+ INSTALLED: ${{ needs.python-versions.outputs.engine-ktlo }} ${{ needs.python-versions.outputs.engine-supported }}
+ PYTHONS: ${{ inputs.pythons }}
+ ENGINES: ${{ inputs.engines }}
+ ROUNDS: ${{ inputs.rounds }}
+ REPEATS: ${{ inputs.repeats }}
+ RUNNER_LABEL: ${{ matrix.os }}
+ run: |
+ set -uo pipefail
+ # `engine-supported` is a multi-line output, one version a line, and an
+ # input may say `3.11, 3.12`, so both are split on commas and whitespace.
+ read -ra installed <<< "$(echo "$INSTALLED" | tr -s ',[:space:]' ' ')"
+ read -ra wanted <<< "$(echo "${PYTHONS:-${installed[*]}}" | tr -s ',[:space:]' ' ')"
+ for version in "${wanted[@]}"; do
+ case " ${installed[*]} " in
+ *" $version "*) ;;
+ *) echo "::error title=Unknown Python version::'$version' is not among ${installed[*]}."; exit 2 ;;
+ esac
+ done
+ args=(--pythons "$(IFS=,; echo "${wanted[*]}")" --out out
+ --image "GitHub-hosted $RUNNER_LABEL, image ${ImageOS:-unknown} ${ImageVersion:-unknown}")
+ if [ -n "$ENGINES" ]; then args+=(--engines "$ENGINES"); fi
+ if [ -n "$ROUNDS" ]; then args+=(--rounds "$ROUNDS"); fi
+ if [ -n "$REPEATS" ]; then args+=(--repeats "$REPEATS"); fi
+ code=0
+ examples/benchmark/native.sh "${args[@]}" || code=$?
+ case "$code" in
+ 0) echo 'status=success' >> "$GITHUB_OUTPUT" ;;
+ 3) echo 'status=warning' >> "$GITHUB_OUTPUT"
+ echo '::warning title=Not every Python version measured in full::The tables explain each gap.' ;;
+ *) echo 'status=failure' >> "$GITHUB_OUTPUT"
+ exit "$code" ;;
+ esac
+
+ # The report names the runner, as the tables' Image row does, since a
+ # native run has no image digest to name.
+ - name: Write the report
+ if: ${{ always() && hashFiles('out/table.rst') != '' }}
+ env:
+ RUNNER_LABEL: ${{ matrix.os }}
+ run: |
+ set -eu
+ {
+ echo "## Engine benchmark on $RUNNER_LABEL"
+ echo
+ echo "Measured natively on $RUNNER_OS $RUNNER_ARCH, image ${ImageOS:-unknown} ${ImageVersion:-unknown},"
+ echo "so the absolute times compare only within this run. The docs' tables are updated by hand."
+ echo
+ echo '```rst'
+ cat out/table.rst
+ echo '```'
+ } > out/report.md
+ {
+ cat out/report.md
+ echo
+ echo '```rst'
+ cat out/table-versions.rst
+ echo '```'
+ } >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Upload the results
+ if: ${{ always() }}
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: benchmark-out
+ path: out/
+ if-no-files-found: ignore
+ retention-days: 90
+
+ # #2030: posts the ratio table, with its absolute-time footnote, on
+ # `always()` like ci-timing.yml's, so a run with gaps is posted too.
+ # util/discussion_report.py writes a warning rather than failing, including
+ # for a run that wrote no report; `continue-on-error` covers the rest.
+ discussion:
+ name: Post the engine benchmark report
+ needs: [ benchmark ]
+ if: ${{ always() && needs.benchmark.result != 'skipped' }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ continue-on-error: true
+ permissions:
+ contents: read
+ discussions: write
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Download the report
+ continue-on-error: true
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: benchmark-out
+ path: report
+
+ - name: Post the report to Discussions
+ continue-on-error: true
+ env:
+ GH_TOKEN: ${{ github.token }}
+ STATUS: ${{ needs.benchmark.outputs.status || needs.benchmark.result }}
+ # `scheduled` on dispatch too: a manual run is a deliberate report.
+ run: |
+ python3 util/discussion_report.py --category "$REPORT_CATEGORY" \
+ --title 'benchmark: weekly engine benchmark report' --report report/report.md \
+ --mode scheduled --status "$STATUS"
diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst
index b10477ffd9..15066257ac 100644
--- a/docs/source/contributing/workflows.rst
+++ b/docs/source/contributing/workflows.rst
@@ -128,6 +128,13 @@ At a Glance
- Mon 05:41
- --
- ``workflow_dispatch``
+ * - Engine Benchmark
+ - :file:`benchmark.yml`
+ - --
+ - --
+ - Wed 03:17
+ - --
+ - ``workflow_dispatch``
.. note::
@@ -157,6 +164,7 @@ than starting a new one.
PR["pull_request: main"]
SAT["schedule: Saturday"]
MON["schedule: Monday"]
+ WED["schedule: Wednesday"]
DISPATCH["workflow_dispatch"]
DAILY["schedule: daily"]
ISSUES["issues"]
@@ -173,6 +181,7 @@ than starting a new one.
VU["Vendor Update
cron-vendor.yml"]
PV["Python Versions
cron-python-versions.yml"]
CT["CI Timing
ci-timing.yml"]
+ BM["Engine Benchmark
benchmark.yml"]
PS["Project Status
project-status.yml"]
CR["Create Release
create-release.yml"]
@@ -210,6 +219,9 @@ than starting a new one.
MON -->|07:23| CT
DISPATCH --> CT
+ WED -->|03:17| BM
+ DISPATCH --> BM
+
DISPATCH --> PS
DAILY -->|00:37| PS
ISSUES --> PS
@@ -218,7 +230,7 @@ than starting a new one.
TAG --> CR
classDef trig fill:none,stroke-dasharray:2 2
- class PUSH,PR,SAT,MON,DISPATCH,DAILY,ISSUES,PRT,TAG trig
+ class PUSH,PR,SAT,MON,WED,DISPATCH,DAILY,ISSUES,PRT,TAG trig
.. mermaid::
@@ -1301,8 +1313,8 @@ to a script. Each script's docstring says what it does; each has tests under
- ``Post the report to Discussions`` in the ``discussion`` job of
``ci-timing.yml``, ``cron-vendor.yml``, ``cron-conda.yml``,
``cron-python-versions.yml``, ``lint.yml``, ``deploy-pages.yml``,
- ``codeql-analysis.yml``, ``coverage-comment.yml`` and ``ci-reports.yml``
- (see `CI Reports`_).
+ ``codeql-analysis.yml``, ``coverage-comment.yml``, ``ci-reports.yml`` and
+ ``benchmark.yml`` (see `CI Reports`_).
- No.
* - :file:`util/check_suite_report.py`
- ``Write the run report`` in the ``discussion`` job of
@@ -1312,7 +1324,8 @@ to a script. Each script's docstring says what it does; each has tests under
* - :file:`util/apt_packages.py`
- The apt steps of ``unit-tests.yml``'s ``engine-tests`` and ``pypcap-parity``
(and so its KTLO twin): the plan, install and collect steps, as
- ``apt_packages.py plan``, ``install`` and ``collect``. The
+ ``apt_packages.py plan``, ``install`` and ``collect``, and of
+ ``benchmark.yml``'s ``benchmark``, as ``install`` alone. The
``actions/cache`` steps, the seed steps (``Seed apt's archive with this
engine's cached .deb files`` and ``Seed apt's archive with the cached .deb
files``) and the AppArmor step (`#2019
@@ -1469,8 +1482,12 @@ marker. How often depends on the script's ``--mode``:
counts as green.
``ci-timing.yml`` posts as ``scheduled`` to ``ci-timing: weekly CI timing
-report``. ``cron-vendor.yml`` posts to ``cron-vendor: vendor constant update
-report``, as ``push`` on its ``workflow_run`` and ``scheduled`` otherwise, and
+report``, and ``benchmark.yml`` to ``benchmark: weekly engine benchmark report``:
+the ratio table :file:`examples/benchmark/native.sh` writes, with its
+absolute-time footnote and the runner it ran on, reporting ``warning`` where a
+Python version was not measured in full (:issue:`2125`). ``cron-vendor.yml``
+posts to ``cron-vendor: vendor constant update report``, as ``push`` on its
+``workflow_run`` and ``scheduled`` otherwise, and
reports ``warning`` where ``vendor-update`` passed over a failed target or test
gate. A failed or cancelled ``head`` skips ``vendor-update``, so its
``discussion`` job runs on ``always()`` and then posts a line of its own with
diff --git a/examples/benchmark/README.md b/examples/benchmark/README.md
index 3a89c31c8a..353406585f 100644
--- a/examples/benchmark/README.md
+++ b/examples/benchmark/README.md
@@ -314,6 +314,50 @@ Not pinned, deliberately:
has uncommitted changes under `pcapkit/` -- a revision that does not say so is a
claim about code that was not measured.
+## On GitHub Actions
+
+[`.github/workflows/benchmark.yml`](../../.github/workflows/benchmark.yml) runs the same
+measurement weekly and on dispatch, **without Docker** (#2125), through
+[`native.sh`](native.sh): `run.sh` with the interpreters already on the machine in
+place of the images. A hosted runner is already a fresh VM per job, so containers would
+add a build per version and no isolation. `native.sh` builds the virtualenvs the
+`Dockerfile` builds, with `pypcap` where `python-images.txt`'s column says so, runs
+`benchmark.py` in each and `report.py` over them all, with `run.sh`'s exit statuses.
+The workflow measures the engines' interpreters from `.github/python-versions.toml`, all
+in one job, since the per-version table's absolute times compare only within one
+machine, and posts the ratio table to the `benchmark: weekly engine benchmark report`
+thread in the `CI reports` Discussions category.
+
+What a native run does not pin, against the containers:
+
+- **The interpreter's patch release** is setup-python's newest build for that runner.
+ It differs by OS: on macOS and Windows, 3.10 to 3.12 stop at their last binary
+ release (3.12.10, say) where Linux has the newest. The report names every patch.
+- **tshark and libpcap** are the runner's -- apt on Ubuntu, Homebrew on macOS,
+ Chocolatey on Windows -- and the report records their versions. Windows has no
+ libpcap, since Npcap's free edition cannot install unattended, so the `pcap`
+ engines are rows with their reason there.
+- **The Image row** names the runner label and its image version, in place of a base
+ image. Both are public image names, not a machine.
+- **Absolute times** come from whatever shared hardware the job landed on, so they
+ compare only within one run. The ratios are what carry from run to run -- within
+ an OS family. Across families they disagree: on one dispatch per runner, Windows'
+ `pypcapfile` ratio was about 44% above Ubuntu's.
+
+**The default runner is `ubuntu-26.04-arm`**, the fastest runner that
+measures every environment and engine. One dispatch per runner (run 40047570595, on
+#2128) had the `pcapkit` baseline at 0.580 ms per packet there against 0.757 on
+`ubuntu-latest`, about 23% less time per packet, and both measured every environment
+and every engine. `macos-latest` was faster still, at 0.339, but it and Windows could
+measure neither `pcap_ct` nor `pypcap`, so they stay dispatch choices only.
+
+The docs' tables stay hand-updated, from `run.sh` or from a report. To compare runners,
+dispatch once per runner; dropping pyshark makes it minutes rather than hours:
+
+```shell
+gh workflow run benchmark.yml -f runner=ubuntu-latest -f engines=default,dpkt,scapy,pypcap,pcap_ct,pypcapfile
+```
+
## Output
Everything lands in `--out` (default `examples/benchmark/out/`, which is
diff --git a/examples/benchmark/native.sh b/examples/benchmark/native.sh
new file mode 100755
index 0000000000..0771a47878
--- /dev/null
+++ b/examples/benchmark/native.sh
@@ -0,0 +1,301 @@
+#!/usr/bin/env bash
+#
+# run.sh without Docker: measure the interpreters already on this machine, and
+# print the same two tables.
+#
+# examples/benchmark/native.sh --pythons 3.11,3.12
+#
+# For each version it builds the virtualenvs the Dockerfile builds -- `pcap_ct`
+# always, and `pypcap` where python-images.txt says it can be built -- runs
+# benchmark.py in each, as entrypoint.sh does, and then report.py over every
+# document. .github/workflows/benchmark.yml runs it on a hosted runner, which is a
+# fresh VM per job already, so containers there would add a build per version and
+# no isolation (#2125).
+#
+# **What this does not pin**, against run.sh: the interpreter's patch release, and
+# the system's tshark and libpcap. Each is recorded in the report rather than
+# fixed, so the absolute times compare only within one run. The README's "On
+# GitHub Actions" says more. For a table to publish from a machine of your own,
+# run.sh is still the reproducible path.
+#
+# Failures are held to run.sh's standard: a version that cannot be found, built or
+# measured is written to /missing/ and becomes an explained column, and an
+# engine that cannot be installed is an explained row. Exit status is run.sh's:
+# 0 measured in full, 3 tables written with gaps, 1 no table, 2 bad arguments.
+#
+# Written for bash 3.2, which is what macOS ships as /bin/bash, and for Git Bash
+# on Windows, where a virtualenv's interpreter is under Scripts/ and a hosted
+# runner's interpreters are found in its tool cache rather than as `python3.X`.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+readonly SCRIPT_DIR
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+readonly REPO_ROOT
+readonly PINS_FILE="${SCRIPT_DIR}/python-images.txt"
+
+PYTHONS=''
+ENGINES=''
+ROUNDS=''
+REPEATS=''
+OUT_DIR="${SCRIPT_DIR}/out"
+WORK_DIR="${REPO_ROOT}/.bench-venvs"
+IMAGE="${BENCH_IMAGE:-}"
+CAPTURE="${REPO_ROOT}/examples/captures/in.pcap"
+
+usage() {
+ cat <<'USAGE'
+usage: native.sh [options]
+
+Measures each named Python version with the interpreter already on this machine,
+in the virtualenvs the Dockerfile builds, and prints the two tables run.sh prints.
+
+options:
+ --pythons LIST comma-separated Python versions, e.g. 3.11,3.12; defaults to
+ every version python-images.txt marks 'default'
+ --engines LIST comma-separated engines, including 'default' (default: all)
+ --rounds N timed extractions per engine per pass (default: benchmark.py's)
+ --repeats N passes over the whole engine set (default: benchmark.py's)
+ --image TEXT what to name as the image in the report, e.g. the runner
+ --out DIR where to write the results (default examples/benchmark/out)
+ --work DIR where to build the virtualenvs (default .bench-venvs)
+ -h, --help this message
+
+exit status: 0 measured in full; 3 tables written, some version not measured in
+full; 1 no table; 2 bad arguments.
+USAGE
+}
+
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --pythons) PYTHONS="${2:?--pythons needs a value}"; shift 2 ;;
+ --engines) ENGINES="${2:?--engines needs a value}"; shift 2 ;;
+ --rounds) ROUNDS="${2:?--rounds needs a value}"; shift 2 ;;
+ --repeats) REPEATS="${2:?--repeats needs a value}"; shift 2 ;;
+ --image) IMAGE="${2:?--image needs a value}"; shift 2 ;;
+ --out) OUT_DIR="${2:?--out needs a value}"; shift 2 ;;
+ --work) WORK_DIR="${2:?--work needs a value}"; shift 2 ;;
+ -h|--help) usage; exit 0 ;;
+ *) echo "native.sh: unknown option '$1'" >&2; usage >&2; exit 2 ;;
+ esac
+done
+
+# The pins file's column for a version, as run.sh's `pin_field` reads it: `$1 ""`
+# makes awk compare strings, so 3.1 is not 3.10.
+pin_field() {
+ awk -v version="$1" -v field="$2" \
+ 'substr($1, 1, 1) != "#" && NF >= 4 && $1 "" == version "" { print $field; exit }' \
+ "${PINS_FILE}"
+}
+
+if [ -z "${PYTHONS}" ]; then
+ VERSIONS="$(awk 'substr($1, 1, 1) != "#" && NF >= 4 && $4 == "default" { printf "%s ", $1 }' "${PINS_FILE}")"
+else
+ VERSIONS="$(echo "${PYTHONS}" | tr -s ',[:space:]' ' ')"
+fi
+if [ -z "${VERSIONS// /}" ]; then
+ echo 'native.sh: no Python versions to measure.' >&2
+ exit 2
+fi
+
+# As in run.sh: a version python-images.txt does not name is a typo, and fatal
+# rather than a column; a repeat is dropped, so it is not measured twice.
+WANTED=''
+for version in ${VERSIONS}; do
+ if [ -z "$(pin_field "${version}" 1)" ]; then
+ echo "native.sh: '${version}' is not in ${PINS_FILE}." >&2
+ echo " Known versions: $(awk 'substr($1, 1, 1) != "#" && NF >= 4 { printf "%s ", $1 }' "${PINS_FILE}")" >&2
+ exit 2
+ fi
+ case " ${WANTED} " in
+ *" ${version} "*) ;;
+ *) WANTED="${WANTED} ${version}" ;;
+ esac
+done
+VERSIONS="${WANTED}"
+if [ -n "${ENGINES}" ]; then
+ case ",${ENGINES}," in
+ *,default,*) ;;
+ *) echo "native.sh: --engines must include 'default'; it is the baseline every ratio uses." >&2
+ exit 2 ;;
+ esac
+fi
+
+BENCH_ARGS=()
+if [ -n "${ENGINES}" ]; then BENCH_ARGS+=(--engines "${ENGINES}"); fi
+if [ -n "${ROUNDS}" ]; then BENCH_ARGS+=(--rounds "${ROUNDS}"); fi
+if [ -n "${REPEATS}" ]; then BENCH_ARGS+=(--repeats "${REPEATS}"); fi
+
+# The interpreter for a series, asked for its own version, as the Dockerfile labels
+# an environment: a `python3.12` that is really something else must not fill the
+# 3.12 column. `python3.X` first; then, on Windows, where setup-python provides no
+# such names, the tool cache and the `py` launcher.
+interpreter() {
+ local version="$1" candidate cache
+ local candidates=()
+ if command -v "python${version}" >/dev/null 2>&1; then
+ candidates+=("$(command -v "python${version}")")
+ fi
+ if [ -n "${RUNNER_TOOL_CACHE:-}" ]; then
+ cache="$(cygpath -u "${RUNNER_TOOL_CACHE}" 2>/dev/null || echo "${RUNNER_TOOL_CACHE}")"
+ for candidate in "${cache}/Python/${version}".*/x64/python.exe "${cache}/Python/${version}".*/arm64/python.exe; do
+ if [ -f "${candidate}" ]; then candidates+=("${candidate}"); fi
+ done
+ fi
+ if command -v py >/dev/null 2>&1; then
+ candidate="$(py "-${version}" -c 'import sys; print(sys.executable)' 2>/dev/null || true)"
+ if [ -n "${candidate}" ]; then candidates+=("${candidate}"); fi
+ fi
+ for candidate in ${candidates[@]+"${candidates[@]}"}; do
+ if [ "$("${candidate}" -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null)" = "${version}" ]; then
+ echo "${candidate}"
+ return 0
+ fi
+ done
+ return 1
+}
+
+# A virtualenv's own interpreter: bin/ everywhere but Windows, which uses Scripts/.
+venv_python() {
+ if [ -d "$1/Scripts" ]; then echo "$1/Scripts/python"; else echo "$1/bin/python"; fi
+}
+
+REVISION='unknown'
+if command -v git >/dev/null 2>&1 && git -C "${REPO_ROOT}" rev-parse --git-dir >/dev/null 2>&1; then
+ REVISION="$(git -C "${REPO_ROOT}" rev-parse --short HEAD)"
+ if ! git -C "${REPO_ROOT}" diff --quiet HEAD -- pcapkit; then
+ REVISION="${REVISION}-dirty"
+ fi
+fi
+export PCAPKIT_REVISION="${REVISION}"
+export BENCH_IMAGE="${IMAGE}"
+
+# A previous run's output is not inherited, for run.sh's reasons.
+mkdir -p "${OUT_DIR}/missing" "${WORK_DIR}"
+rm -f "${OUT_DIR}/missing/"*.txt "${OUT_DIR}"/*.json "${OUT_DIR}"/*.rst "${OUT_DIR}"/*.txt
+
+GAPS=0
+record_missing() {
+ printf '%s\n' "$2" > "${OUT_DIR}/missing/$1.txt"
+ GAPS=$((GAPS + 1))
+ echo "native.sh: $1 will be reported as a gap in the table -- $2" >&2
+}
+
+REPORTER=''
+for version in ${VERSIONS}; do
+ echo >&2
+ echo "==> Python ${version}" >&2
+ if ! python="$(interpreter "${version}")"; then
+ record_missing "${version}" "no Python ${version} interpreter was found on this machine"
+ continue
+ fi
+ "${python}" -c 'import platform, sys; print("==> " + platform.python_version() + " at " + sys.executable)' >&2
+
+ notes="${WORK_DIR}/${version}/notes"
+ rm -rf "${WORK_DIR:?}/${version}"
+ mkdir -p "${notes}/install-failures" "${notes}/not-attempted"
+ environments=''
+
+ # pcap-ct's virtualenv is the one every interpreter holds, so a failure here
+ # costs the whole column.
+ venv="${WORK_DIR}/${version}/pcap_ct"
+ if "${python}" -m venv "${venv}" \
+ && "$(venv_python "${venv}")" -m pip install --quiet --upgrade pip setuptools wheel \
+ && "$(venv_python "${venv}")" -m pip install --quiet \
+ --requirement "${SCRIPT_DIR}/requirements-pcap_ct.txt" \
+ && "$(venv_python "${venv}")" -m pip install --quiet --no-deps "${REPO_ROOT}"; then
+ environments="${environments} pcap_ct"
+ else
+ record_missing "${version}" "the pcap_ct virtualenv could not be built for Python ${version}; see the log"
+ continue
+ fi
+
+ # pypcap compiles, so its install alone may fail, and is then a row with the
+ # pip error, in the Dockerfile's words.
+ if [ "$(pin_field "${version}" 3)" = 'pypcap' ]; then
+ venv="${WORK_DIR}/${version}/pypcap"
+ log="${WORK_DIR}/${version}/pypcap-install.log"
+ if "${python}" -m venv "${venv}" \
+ && "$(venv_python "${venv}")" -m pip install --quiet --upgrade pip setuptools wheel \
+ && "$(venv_python "${venv}")" -m pip install --quiet \
+ --requirement "${SCRIPT_DIR}/requirements-common.txt"; then
+ if ! "$(venv_python "${venv}")" -m pip install \
+ --requirement "${SCRIPT_DIR}/requirements-pypcap.txt" > "${log}" 2>&1; then
+ echo 'WARNING: pypcap failed to install; it will be reported as not measured.' >&2
+ { echo 'pypcap failed to build here, so it could not be measured.'
+ echo 'The tail of the pip output was:'
+ tail -n 12 "${log}"; } > "${notes}/install-failures/pypcap.txt"
+ cat "${log}" >&2
+ fi
+ if "$(venv_python "${venv}")" -m pip install --quiet --no-deps "${REPO_ROOT}"; then
+ environments="${environments} pypcap"
+ fi
+ else
+ echo 'pypcap could not be measured: its virtualenv failed to build; see the log.' \
+ > "${notes}/install-failures/pypcap.txt"
+ fi
+ else
+ { echo 'pypcap is not installable on this interpreter, so it was not built here:'
+ echo 'pypcap 1.3.0 ships a pcap.c pre-generated by Cython 0.29.32, which does'
+ echo 'not compile against the Python 3.12+ C API. The engine is unsupported here'
+ echo 'rather than broken here.'; } > "${notes}/not-attempted/pypcap.txt"
+ fi
+
+ if [ -z "${REPORTER}" ]; then
+ REPORTER="$(venv_python "${WORK_DIR}/${version}/pcap_ct")"
+ fi
+
+ failed=''
+ for provider in ${environments}; do
+ label="${version}-${provider}"
+ venv="${WORK_DIR}/${version}/${provider}"
+ echo "==> measuring in the ${label} environment" >&2
+ if ! BENCH_INSTALL_FAILURES="${notes}/install-failures" BENCH_NOT_ATTEMPTED="${notes}/not-attempted" \
+ "$(venv_python "${venv}")" "${SCRIPT_DIR}/benchmark.py" --capture "${CAPTURE}" \
+ --label "${label}" --out "${OUT_DIR}/${label}.json" ${BENCH_ARGS[@]+"${BENCH_ARGS[@]}"}; then
+ failed="${failed} ${label}"
+ fi
+ "$(venv_python "${venv}")" -m pip freeze > "${OUT_DIR}/${label}.txt" 2>/dev/null || true
+ done
+ if [ -n "${failed}" ]; then
+ record_missing "${version}" \
+ "measuring in${failed} exited non-zero; any environment that finished first is still reported"
+ fi
+done
+
+DOCUMENTS=()
+for path in "${OUT_DIR}"/*.json; do
+ if [ -e "${path}" ]; then DOCUMENTS+=("${path}"); fi
+done
+
+echo >&2
+if [ "${#DOCUMENTS[@]}" -eq 0 ] || [ -z "${REPORTER}" ]; then
+ echo 'native.sh: no environment produced a measurement; there is nothing to report.' >&2
+ exit 1
+fi
+
+REPORT_ARGS=()
+for path in "${OUT_DIR}"/missing/*.txt; do
+ if [ -e "${path}" ]; then
+ REPORT_ARGS+=(--missing "$(basename "${path}" .txt)=$(tr '\n' ' ' < "${path}")")
+ fi
+done
+
+status=0
+"${REPORTER}" "${SCRIPT_DIR}/report.py" "${DOCUMENTS[@]}" \
+ --rst-out "${OUT_DIR}/table.rst" --versions-rst-out "${OUT_DIR}/table-versions.rst" \
+ ${REPORT_ARGS[@]+"${REPORT_ARGS[@]}"} || status=$?
+
+if [ ! -f "${OUT_DIR}/table.rst" ] || [ ! -f "${OUT_DIR}/table-versions.rst" ]; then
+ echo "native.sh: no table was produced; ${OUT_DIR} holds whatever the run got to." >&2
+ exit 1
+fi
+if [ "${status}" -ne 0 ]; then
+ exit 1
+fi
+if [ "${GAPS}" -gt 0 ]; then
+ echo "native.sh: ${GAPS} of the requested Python version(s) could not be measured in full;" >&2
+ echo " the tables were still written, with the reasons in them and in ${OUT_DIR}/missing/." >&2
+ exit 3
+fi
+exit 0
diff --git a/tests/project/test_apt_packages_1967.py b/tests/project/test_apt_packages_1967.py
index 60954d0243..9a257b63ec 100644
--- a/tests/project/test_apt_packages_1967.py
+++ b/tests/project/test_apt_packages_1967.py
@@ -502,6 +502,10 @@ def test_refused(self) -> None:
f'{IMAGE_PYTHON} {SCRIPT} collect'),
)
SITES = {'engine-tests': ENGINE_STEPS, 'pypcap-parity': PARITY_STEPS}
+#: #2125: steps of other workflows that run the script's ``install`` alone, with no
+#: cache to plan or collect for, as (workflow, step name). Each is pinned by its
+#: workflow's own test, as :mod:`tests.project.test_benchmark_workflow_2125`.
+ELSEWHERE = {('benchmark.yml', 'Install libpcap headers and tshark')}
#: Each step's former shell, by the subcommand that replaced it.
FORMER = {'plan': FORMER_PLAN, 'collect': FORMER_COLLECT}
APPARMOR = 'Let tshark read the checkout past its AppArmor profile'
@@ -798,7 +802,8 @@ def test_no_step_keeps_apt_shell(self) -> None:
if not line.lstrip().startswith('#'))
found.extend(f'{path.name} {job} {item.get("name")!r}: {piece}'
for piece in INLINE if piece in run)
- if SCRIPT in run and (path.name, item.get('name')) not in {(WORKFLOW, name) for name in covered}:
+ listed = {(WORKFLOW, name) for name in covered} | ELSEWHERE
+ if SCRIPT in run and (path.name, item.get('name')) not in listed:
found.append(f'{path.name} {job} {item.get("name")!r}: unlisted')
self.assertEqual(found, [])
diff --git a/tests/project/test_benchmark_workflow_2125.py b/tests/project/test_benchmark_workflow_2125.py
new file mode 100644
index 0000000000..79239a81ef
--- /dev/null
+++ b/tests/project/test_benchmark_workflow_2125.py
@@ -0,0 +1,481 @@
+# -*- coding: utf-8 -*-
+"""#2125: :file:`benchmark.yml`, the engine benchmark on GitHub Actions.
+
+The workflow runs :file:`examples/benchmark/native.sh` in one job on one runner,
+over the engines' interpreters from :file:`.github/python-versions.toml`, and posts
+the ratio table to Discussions.
+
+* :class:`TestTriggers`: a weekly ``schedule`` and ``workflow_dispatch`` with the
+ runner, the versions, the engines and the rounds and passes as inputs; no push,
+ pull request or ``workflow_run``.
+* :class:`TestTheCron`: off the hour, and in no hour on no day that another
+ workflow's ``schedule`` fires in.
+* :class:`TestPermissions`: read-only at the top, ``discussions: write`` on the
+ ``discussion`` job alone, and no push.
+* :class:`TestTheJobs`: one leg on the chosen runner with the engine tiers
+ installed, running ``native.sh`` and never Docker; the tshark steps as
+ ``unit-tests.yml``'s; the post to ``: ``.
+* :class:`TestTheMeasureStep` and :class:`TestTheReportStep` run those steps' own
+ ``run:`` scripts under bash, with a stand-in ``native.sh``.
+* :class:`TestNativeScript` runs :file:`examples/benchmark/native.sh` itself on
+ what it refuses and on a version with no interpreter, which need no network.
+
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import pathlib
+import re
+import shutil
+import subprocess # nosec: B404
+import sys
+import tempfile
+import textwrap
+import unittest
+from typing import TYPE_CHECKING
+
+from tests._workflows import WORKFLOWS, job, load, steps
+
+try:
+ import tomllib
+except ImportError: # Python 3.10 and older
+ tomllib = None # type: ignore[assignment]
+
+if TYPE_CHECKING:
+ from typing import Any, Optional
+
+#: Test tier (#1958): reads the workflow files and runs their steps and the tree's
+#: scripts, a question about the tree.
+__tier__ = 'LATEST'
+
+ROOT = pathlib.Path(__file__).resolve().parents[2]
+WORKFLOW = 'benchmark.yml'
+NATIVE = ROOT / 'examples' / 'benchmark' / 'native.sh'
+PINS = ROOT / 'examples' / 'benchmark' / 'python-images.txt'
+VERSIONS_SCRIPT = ROOT / 'util' / 'python_versions.py'
+UPDATER = ROOT / 'util' / 'update_python_versions.py'
+BASH = shutil.which('bash')
+#: How ``shell: bash`` runs a step's script, ahead of the script.
+SHELL = [BASH or 'bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c']
+TITLE = 'benchmark: weekly engine benchmark report'
+INPUTS = {'runner', 'pythons', 'engines', 'rounds', 'repeats'}
+MEASURE = 'Run examples/benchmark/native.sh'
+#: The runner a scheduled run, and a dispatch left at its default, measures on.
+DEFAULT_RUNNER = 'ubuntu-26.04-arm'
+#: A cron's fields, ``minute hour day month weekday``.
+CRON = re.compile(r'^(\d+) (\d+) (\S+) (\S+) (\S+)$')
+#: The stand-in ``native.sh``: logs its arguments, one a line, and exits ``$NATIVE_EXIT``.
+FAKE_NATIVE = textwrap.dedent('''\
+ #!/bin/bash
+ printf '%s\\n' "$@" > "$NATIVE_LOG"
+ exit "${NATIVE_EXIT:-0}"
+''')
+
+
+def step(job_id: 'str', name: 'str') -> 'dict[str, Any]':
+ """Job ``job_id``'s step named ``name``."""
+ found = [item for item in steps(WORKFLOW, job_id) if item.get('name') == name]
+ assert len(found) == 1, f'{job_id} has {len(found)} steps named {name!r}' # nosec: B101
+ return found[0]
+
+
+def weekdays(field: 'str') -> 'set[int]':
+ """The weekdays a cron's weekday field names, 0 for Sunday; ``*`` is every day."""
+ if field == '*':
+ return set(range(7))
+ days = set() # type: set[int]
+ for part in field.split(','):
+ low, _, high = part.partition('-')
+ days.update(day % 7 for day in range(int(low), int(high or low) + 1))
+ return days
+
+
+def slots(cron: 'str') -> 'set[tuple[int, int]]':
+ """Every (weekday, hour) ``cron`` fires in, for the plain crons the workflows use."""
+ match = CRON.match(cron)
+ assert match is not None, f'a cron this test cannot read: {cron!r}' # nosec: B101
+ _, hour, day, month, weekday = match.groups()
+ assert (day, month) == ('*', '*'), f'a cron this test cannot read: {cron!r}' # nosec: B101
+ return {(found, int(hour)) for found in weekdays(weekday)}
+
+
+def github_outputs() -> 'dict[str, str]':
+ """``util/python_versions.py --github-output`` as the runner reads ``$GITHUB_OUTPUT``.
+
+ ``name=value``, or ``name< 'str':
+ """A step's ``env:`` value with each ``needs.python-versions.outputs.X`` substituted."""
+ return re.sub(r'\$\{\{ needs\.python-versions\.outputs\.([\w-]+) \}\}', lambda match: outputs[match.group(1)],
+ template)
+
+
+def pinned_versions() -> 'list[str]':
+ """Every version python-images.txt has a row for."""
+ return [line.split()[0] for line in PINS.read_text(encoding='utf-8').splitlines()
+ if not line.startswith('#') and len(line.split()) >= 4]
+
+
+def crons(spec: 'dict[str, Any]') -> 'list[str]':
+ """A parsed workflow's ``schedule`` crons."""
+ schedule = (spec.get('on') or {}).get('schedule') or []
+ return [str(entry['cron']) for entry in schedule]
+
+
+class TestTriggers(unittest.TestCase):
+
+ def test_a_weekly_schedule_and_a_dispatch_alone(self) -> None:
+ spec = load(WORKFLOW)
+ self.assertEqual(set(spec['on']), {'schedule', 'workflow_dispatch'})
+ self.assertEqual(len(crons(spec)), 1)
+ _, _, day, month, weekday = CRON.match(crons(spec)[0]).groups() # type: ignore[union-attr]
+ self.assertEqual((day, month, len(weekdays(weekday))), ('*', '*', 1))
+
+ def test_the_dispatch_inputs(self) -> None:
+ inputs = load(WORKFLOW)['on']['workflow_dispatch']['inputs']
+ self.assertEqual(set(inputs), INPUTS)
+ runner = inputs['runner']
+ # The fastest runner that measures every engine, on #2128; the others stay choices.
+ self.assertEqual((runner['type'], runner['default']), ('choice', DEFAULT_RUNNER))
+ self.assertEqual(runner['options'][0], DEFAULT_RUNNER)
+ self.assertLessEqual({'ubuntu-latest', 'macos-latest', 'windows-latest'}, set(runner['options']))
+ # Empty is the default: every engine interpreter, benchmark.py's engines and counts.
+ for name in INPUTS - {'runner'}:
+ with self.subTest(input=name):
+ self.assertEqual(inputs[name].get('default'), '')
+
+ def test_a_scheduled_run_falls_back_to_the_default_runner(self) -> None:
+ """A schedule has no inputs, so every read of the runner names the default."""
+ text = (WORKFLOWS / WORKFLOW).read_text(encoding='utf-8')
+ reads = re.findall(r'inputs\.runner\b(?: \|\| \'([\w.-]+)\')?', text)
+ self.assertTrue(reads)
+ self.assertEqual(set(reads), {DEFAULT_RUNNER})
+
+
+class TestTheCron(unittest.TestCase):
+
+ def test_off_the_hour(self) -> None:
+ minute = int(CRON.match(crons(load(WORKFLOW))[0]).group(1)) # type: ignore[union-attr]
+ self.assertNotIn(minute, (0, 15, 30, 45))
+
+ def test_no_other_schedule_shares_its_hour(self) -> None:
+ mine = slots(crons(load(WORKFLOW))[0])
+ for path in sorted(WORKFLOWS.glob('*.yml')):
+ if path.name == WORKFLOW:
+ continue
+ for cron in crons(load(path)):
+ with self.subTest(workflow=path.name, cron=cron):
+ self.assertFalse(mine & slots(cron))
+
+ def test_a_collision_is_caught(self) -> None:
+ self.assertTrue(slots('17 3 * * 3') & slots('37 3 * * *'))
+ self.assertTrue(slots('17 7 * * 1') & slots('23 7 * * 1'))
+ self.assertFalse(slots('17 3 * * 3') & slots('17 3 * * 6'))
+
+
+class TestPermissions(unittest.TestCase):
+
+ def test_read_only_at_the_top(self) -> None:
+ self.assertEqual(load(WORKFLOW)['permissions'], {'contents': 'read'})
+
+ def test_only_the_discussion_job_writes(self) -> None:
+ for job_id, spec in load(WORKFLOW)['jobs'].items():
+ with self.subTest(job=job_id):
+ if job_id == 'discussion':
+ self.assertEqual(spec['permissions'], {'contents': 'read', 'discussions': 'write'})
+ else:
+ self.assertNotIn('permissions', spec)
+
+ def test_no_job_pushes_or_keeps_a_token(self) -> None:
+ for job_id, spec in load(WORKFLOW)['jobs'].items():
+ for item in spec['steps']:
+ with self.subTest(job=job_id, step=item.get('name') or item.get('uses')):
+ self.assertNotRegex(str(item.get('run', '')), r'\bgit (?:push|commit|tag)\b')
+ if str(item.get('uses', '')).startswith('actions/checkout@'):
+ self.assertIs(item['with']['persist-credentials'], False)
+
+
+class TestTheJobs(unittest.TestCase):
+
+ def test_the_jobs_and_their_order(self) -> None:
+ jobs = load(WORKFLOW)['jobs']
+ self.assertEqual(list(jobs), ['python-versions', 'benchmark', 'discussion'])
+ self.assertEqual(jobs['benchmark']['needs'], ['python-versions'])
+ self.assertEqual(jobs['discussion']['needs'], ['benchmark'])
+ # A failed or partial benchmark still posts.
+ self.assertEqual(jobs['discussion']['if'], "${{ always() && needs.benchmark.result != 'skipped' }}")
+
+ def test_one_leg_on_the_chosen_runner(self) -> None:
+ """Every version on one machine, as the per-version table's absolute times require."""
+ benchmark = job(WORKFLOW, 'benchmark')
+ self.assertEqual(benchmark['strategy']['matrix'], {'os': [f"${{{{ inputs.runner || '{DEFAULT_RUNNER}' }}}}"]})
+ self.assertEqual(benchmark['runs-on'], f"${{{{ matrix.os || '{DEFAULT_RUNNER}' }}}}")
+ # Only the measurement moves to it: reading the tiers and posting stay on x64.
+ for job_id in ('python-versions', 'discussion'):
+ with self.subTest(job=job_id):
+ self.assertEqual(job(WORKFLOW, job_id)['runs-on'], 'ubuntu-latest')
+
+ def test_the_engine_tiers_are_installed(self) -> None:
+ setups = [item for item in steps(WORKFLOW, 'benchmark')
+ if str(item.get('uses', '')).startswith('actions/setup-python@')]
+ self.assertEqual([item['with']['python-version'] for item in setups],
+ ['${{ needs.python-versions.outputs.engine-ktlo }}',
+ '${{ needs.python-versions.outputs.engine-supported }}'])
+ self.assertIs(setups[0]['continue-on-error'], True)
+ self.assertEqual(step('python-versions', 'Read .github/python-versions.toml')['run'],
+ 'python3 util/python_versions.py --github-output')
+
+ def test_it_runs_native_sh_and_never_docker(self) -> None:
+ """run.sh and its images are the local path; the workflow measures on the runner."""
+ self.assertIn('examples/benchmark/native.sh "${args[@]}"', str(step('benchmark', MEASURE)['run']))
+ for job_id, spec in load(WORKFLOW)['jobs'].items():
+ for item in spec['steps']:
+ with self.subTest(job=job_id, step=item.get('name') or item.get('uses')):
+ self.assertNotRegex(str(item.get('run', '')), r'\bdocker\b|run\.sh')
+
+ def test_the_post(self) -> None:
+ run = str(step('discussion', 'Post the report to Discussions')['run'])
+ self.assertEqual(' '.join(run.replace('\\\n', ' ').split()),
+ 'python3 util/discussion_report.py --category "$REPORT_CATEGORY" '
+ f"--title '{TITLE}' --report report/report.md --mode scheduled --status \"$STATUS\"")
+ self.assertEqual(load(WORKFLOW)['env']['REPORT_CATEGORY'], 'CI reports')
+ self.assertTrue(TITLE.startswith(f'{pathlib.Path(WORKFLOW).stem}: '))
+ self.assertIs(job(WORKFLOW, 'discussion')['continue-on-error'], True)
+ self.assertEqual(step('discussion', 'Post the report to Discussions')['env']['STATUS'],
+ '${{ needs.benchmark.outputs.status || needs.benchmark.result }}')
+
+ def test_tshark_is_installed_and_let_read_the_checkout_as_in_unit_tests(self) -> None:
+ """#2019's AppArmor step directly follows the apt install, with ``engine-tests``' body."""
+ names = [item.get('name') for item in steps(WORKFLOW, 'benchmark')]
+ install = 'Install libpcap headers and tshark'
+ apparmor = 'Let tshark read the checkout past its AppArmor profile'
+ self.assertEqual(names[names.index(install) + 1], apparmor)
+ self.assertIn('tshark', job(WORKFLOW, 'benchmark')['env']['APT_PACKAGES'].split())
+ self.assertTrue(str(step('benchmark', install)['run']).startswith(
+ '/usr/bin/python3 util/apt_packages.py install'))
+ theirs = [item for item in steps('unit-tests.yml', 'engine-tests') if item.get('name') == apparmor]
+ self.assertEqual(step('benchmark', apparmor)['run'], theirs[0]['run'])
+
+
+class TestTheTiersIgnoreTheRunner(unittest.TestCase):
+ """util/update_python_versions.py reads no tier label from a runner the dispatcher picks.
+
+ Its tiers want the default labels on one Linux architecture, and the benchmark's
+ default is arm64 where every test job's is x64.
+
+ """
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ if tomllib is None: # pragma: no cover
+ raise unittest.SkipTest('tomllib, which util/update_python_versions.py imports, needs Python 3.11+')
+ spec = importlib.util.spec_from_file_location('update_python_versions_2125', UPDATER)
+ assert spec is not None and spec.loader is not None # nosec: B101
+ cls.module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(cls.module) # type: ignore[union-attr]
+
+ def workflows(self) -> 'dict[str, Any]':
+ return {path.name: load(path) for path in sorted(WORKFLOWS.glob('*.yml'))}
+
+ def test_the_benchmark_adds_no_label(self) -> None:
+ with_it = self.module.workflow_labels(self.workflows())
+ without = self.workflows()
+ del without[WORKFLOW]
+ self.assertEqual(with_it, self.module.workflow_labels(without))
+ self.assertNotIn(DEFAULT_RUNNER, with_it['default'])
+
+ def test_a_fixed_arm_runner_would_count(self) -> None:
+ """The rule bites only on an input: the same job on a literal axis adds its label."""
+ workflows = self.workflows()
+ workflows[WORKFLOW]['jobs']['benchmark']['strategy']['matrix']['os'] = [DEFAULT_RUNNER]
+ self.assertIn(DEFAULT_RUNNER, self.module.workflow_labels(workflows)['default'])
+
+
+@unittest.skipIf(not BASH, 'needs bash on PATH')
+class _Scratch(unittest.TestCase):
+
+ def setUp(self) -> None:
+ tmp = tempfile.TemporaryDirectory(prefix='pcapkit-2125-') # pylint: disable=consider-using-with
+ self.addCleanup(tmp.cleanup)
+ self.tmp = pathlib.Path(tmp.name)
+ self.output = self.tmp / 'output'
+ self.output.write_text('', encoding='utf-8')
+ self.summary = self.tmp / 'summary.md'
+
+ def run_step(self, job_id: 'str', name: 'str', **env: 'str') -> 'subprocess.CompletedProcess[str]':
+ """``name``'s ``run:`` as ``shell: bash`` runs it, in the scratch directory."""
+ environ = dict(os.environ, GITHUB_OUTPUT=str(self.output), GITHUB_STEP_SUMMARY=str(self.summary),
+ RUNNER_OS='Linux', RUNNER_ARCH='X64', ImageOS='ubuntu26', ImageVersion='20261020.1', **env)
+ return subprocess.run([*SHELL, str(step(job_id, name)['run'])], env=environ, # nosec: B603
+ cwd=self.tmp, capture_output=True, text=True, check=False, timeout=60)
+
+ def outputs(self) -> 'dict[str, str]':
+ return dict(line.split('=', 1) for line in self.output.read_text(encoding='utf-8').splitlines())
+
+
+class TestTheMeasureStep(_Scratch):
+ """``Run examples/benchmark/native.sh`` against a stand-in script."""
+
+ def setUp(self) -> None:
+ super().setUp()
+ script = self.tmp / 'examples' / 'benchmark' / 'native.sh'
+ script.parent.mkdir(parents=True)
+ script.write_text(FAKE_NATIVE, encoding='utf-8')
+ script.chmod(0o755)
+ self.log = self.tmp / 'native.log'
+
+ def measure(self, code: 'int' = 0, **inputs: 'str') -> 'tuple[int, Optional[list[str]], dict[str, str]]':
+ """The step, with ``INSTALLED`` as the runner renders it from the script's real outputs."""
+ installed = render(step('benchmark', MEASURE)['env']['INSTALLED'], self.versions)
+ env = {'INSTALLED': installed, 'PYTHONS': '', 'ENGINES': '', 'ROUNDS': '', 'REPEATS': '',
+ 'RUNNER_LABEL': 'ubuntu-latest', 'NATIVE_LOG': str(self.log), 'NATIVE_EXIT': str(code)}
+ env.update(inputs)
+ result = self.run_step('benchmark', MEASURE, **env)
+ args = self.log.read_text(encoding='utf-8').splitlines() if self.log.exists() else None
+ return result.returncode, args, self.outputs()
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ super().setUpClass()
+ cls.versions = github_outputs()
+
+ def test_the_outputs_are_the_multi_line_form(self) -> None:
+ """What the step must split: ``engine-supported`` one version a line."""
+ self.assertIn('\n', self.versions['engine-supported'])
+ self.assertIn('\n', render(step('benchmark', MEASURE)['env']['INSTALLED'], self.versions))
+
+ def test_a_scheduled_run_measures_every_interpreter(self) -> None:
+ status, args, outputs = self.measure()
+ self.assertEqual((status, outputs), (0, {'status': 'success'}))
+ every = (self.versions['engine-ktlo'] + ' ' + self.versions['engine-supported']).split()
+ self.assertEqual(args, ['--pythons', ','.join(every), '--out', 'out',
+ '--image', 'GitHub-hosted ubuntu-latest, image ubuntu26 20261020.1'])
+
+ def test_a_spaced_list_is_read_as_a_list(self) -> None:
+ status, args, _ = self.measure(PYTHONS=' 3.12, 3.11 ,3.13')
+ self.assertEqual(status, 0)
+ assert args is not None # nosec: B101
+ self.assertEqual(args[:2], ['--pythons', '3.12,3.11,3.13'])
+
+ def test_every_interpreter_has_a_pins_row(self) -> None:
+ """native.sh refuses a version python-images.txt lacks, so a new tier needs its row first."""
+ every = (self.versions['engine-ktlo'] + ' ' + self.versions['engine-supported']).split()
+ self.assertTrue(every)
+ self.assertLessEqual(set(every), set(pinned_versions()))
+
+ def test_the_dispatch_inputs_reach_the_script(self) -> None:
+ status, args, _ = self.measure(PYTHONS='3.12,3.11', ENGINES='default,dpkt', ROUNDS='50', REPEATS='2')
+ self.assertEqual(status, 0)
+ assert args is not None # nosec: B101
+ self.assertEqual(args[:2], ['--pythons', '3.12,3.11'])
+ self.assertEqual(args[6:], ['--engines', 'default,dpkt', '--rounds', '50', '--repeats', '2'])
+
+ def test_a_version_not_installed_is_refused(self) -> None:
+ status, args, outputs = self.measure(PYTHONS='3.11,3.1')
+ self.assertEqual((status, args, outputs), (2, None, {}))
+
+ def test_gaps_warn_and_no_table_fails(self) -> None:
+ status, _, outputs = self.measure(code=3)
+ self.assertEqual((status, outputs), (0, {'status': 'warning'}))
+ self.output.write_text('', encoding='utf-8')
+ status, _, outputs = self.measure(code=1)
+ self.assertEqual((status, outputs), (1, {'status': 'failure'}))
+
+
+class TestTheReportStep(_Scratch):
+ """``Write the report``: the ratio table for the post, and both tables for the summary."""
+
+ def test_the_report_names_the_runner_and_fences_the_tables(self) -> None:
+ out = self.tmp / 'out'
+ out.mkdir()
+ (out / 'table.rst').write_text('RATIOS\n', encoding='utf-8')
+ (out / 'table-versions.rst').write_text('ABSOLUTES\n', encoding='utf-8')
+ self.assertEqual(self.run_step('benchmark', 'Write the report', RUNNER_LABEL='macos-latest').returncode, 0)
+ report = (out / 'report.md').read_text(encoding='utf-8')
+ self.assertTrue(report.startswith('## Engine benchmark on macos-latest\n'))
+ self.assertIn('Linux X64, image ubuntu26 20261020.1', report)
+ self.assertTrue(report.endswith('```rst\nRATIOS\n```\n'))
+ self.assertNotIn('ABSOLUTES', report)
+ self.assertEqual(self.summary.read_text(encoding='utf-8'), f'{report}\n```rst\nABSOLUTES\n```\n')
+ self.assertEqual(step('benchmark', 'Write the report')['if'],
+ "${{ always() && hashFiles('out/table.rst') != '' }}")
+
+
+class TestNativeScript(_Scratch):
+ """``native.sh`` itself, where it needs no interpreter install: a copy, beside a pins file with a 3.99 row."""
+
+ def setUp(self) -> None:
+ super().setUp()
+ directory = self.tmp / 'examples' / 'benchmark'
+ directory.mkdir(parents=True)
+ shutil.copy(NATIVE, directory / 'native.sh')
+ (directory / 'python-images.txt').write_text(
+ PINS.read_text(encoding='utf-8') + '3.99 python:3.99-slim-bookworm@sha256:00 no-pypcap opt-in\n',
+ encoding='utf-8')
+ self.script = directory / 'native.sh'
+
+ def native(self, *args: 'str') -> 'subprocess.CompletedProcess[str]':
+ command = [SHELL[0], str(self.script), '--out', str(self.tmp / 'out'), '--work', str(self.tmp / 'work'),
+ *args]
+ return subprocess.run(command, cwd=self.tmp, # nosec: B603
+ capture_output=True, text=True, check=False, timeout=60,
+ env=dict(os.environ, PATH=os.defpath, RUNNER_TOOL_CACHE=''))
+
+ def test_it_refuses_bad_arguments(self) -> None:
+ self.assertEqual(self.native('--bogus').returncode, 2)
+ self.assertEqual(self.native('--engines', 'dpkt').returncode, 2)
+
+ def test_a_version_the_pins_lack_is_a_typo(self) -> None:
+ """As in run.sh, and before anything is written: 3.1 is not 3.10."""
+ for version in ('3.1', '3.98'):
+ with self.subTest(version=version):
+ result = self.native('--pythons', f'3.99,{version}')
+ self.assertEqual(result.returncode, 2, result.stderr)
+ self.assertIn(f"native.sh: '{version}' is not in ", result.stderr)
+ self.assertFalse((self.tmp / 'out').exists())
+
+ def test_a_repeat_is_measured_once(self) -> None:
+ result = self.native('--pythons', '3.99, 3.99,3.99')
+ self.assertEqual(result.returncode, 1, result.stderr)
+ self.assertEqual(result.stderr.count('==> Python 3.99'), 1)
+
+ def test_a_version_with_no_interpreter_is_a_gap_and_no_table(self) -> None:
+ result = self.native('--pythons', '3.99')
+ self.assertEqual(result.returncode, 1, result.stderr)
+ self.assertEqual((self.tmp / 'out' / 'missing' / '3.99.txt').read_text(encoding='utf-8'),
+ 'no Python 3.99 interpreter was found on this machine\n')
+ self.assertIn('no environment produced a measurement', result.stderr)
+
+ def test_it_is_executable(self) -> None:
+ self.assertTrue(os.access(NATIVE, os.X_OK))
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py
index 722597335d..0fe805854a 100644
--- a/tests/project/test_python_versions_1660.py
+++ b/tests/project/test_python_versions_1660.py
@@ -836,7 +836,9 @@ def test_every_setup_reads_the_matrix_or_the_file(self) -> None:
if where in exemptions or 'matrix.python-version' in text:
continue
output = script_output(name, job_id, READ.fullmatch(text).group(2)) # type: ignore[union-attr]
- expected = {'engine-tests': ('engine-supported', 'engine-ktlo')}.get(job_id, ('latest',))
+ # #2125: benchmark.yml's `benchmark` installs the engines' interpreters too.
+ expected = {'engine-tests': ('engine-supported', 'engine-ktlo'),
+ 'benchmark': ('engine-supported', 'engine-ktlo')}.get(job_id, ('latest',))
self.assertIn(output, expected)
def test_every_single_interpreter_job_runs_latest(self) -> None:
diff --git a/tests/project/test_tshark_apparmor_2019.py b/tests/project/test_tshark_apparmor_2019.py
index 1773926a16..f472ee6193 100644
--- a/tests/project/test_tshark_apparmor_2019.py
+++ b/tests/project/test_tshark_apparmor_2019.py
@@ -37,6 +37,10 @@
#: An AppArmor file rule in ``file ACCESS FILEGLOB,`` form granting read on a tree
#: (apparmor.d(5), FILE RULE), the form the shipped tshark profile itself uses.
RULE = re.compile(r'^file r (/\S+)/\*\*,$')
+#: #2125: jobs of other workflows that install tshark and run the step, as
+#: (workflow, job). Each is pinned by its workflow's own test, as
+#: :mod:`tests.project.test_benchmark_workflow_2125`.
+ELSEWHERE = {('benchmark.yml', 'benchmark')}
STUBS = {
'sudo': 'exec env "$@"\n',
'apparmor_parser': 'echo "$@" >> "$STUB_LOG"\n',
@@ -79,7 +83,7 @@ def test_every_job_installing_tshark_is_covered(self) -> None:
re.search(r'\butil/apt_packages\.py install\b', run)
and re.search(r'\btshark\b', packages)):
found.add((path.name, job_id))
- self.assertEqual(found, {(WORKFLOW, job_id) for job_id, _ in SITES})
+ self.assertEqual(found, {(WORKFLOW, job_id) for job_id, _ in SITES} | ELSEWHERE)
def test_one_body_everywhere(self) -> None:
"""The jobs run one script, so a fix to it cannot reach only some of them."""
diff --git a/util/apt_packages.py b/util/apt_packages.py
index aaf72cd692..5909fbd3c2 100644
--- a/util/apt_packages.py
+++ b/util/apt_packages.py
@@ -9,6 +9,9 @@
/usr/bin/python3 util/apt_packages.py install [--name NAME]
/usr/bin/python3 util/apt_packages.py collect
+:file:`.github/workflows/benchmark.yml`'s ``benchmark`` job runs ``install`` alone,
+with no cache to plan or collect for (#2125).
+
``APT_PACKAGES``, the job's environment, names the packages, separated by
whitespace, for both ``plan`` and ``install``, so the cache key cannot describe a
set other than the one installed. Each subcommand runs the commands the steps'
diff --git a/util/update_python_versions.py b/util/update_python_versions.py
index 478ced9ebf..0104acc57a 100644
--- a/util/update_python_versions.py
+++ b/util/update_python_versions.py
@@ -567,11 +567,20 @@ def workflow_labels(workflows: 'dict[str, Any]') -> 'dict[str, list[str]]':
label its ``matrix.os || '...'`` falls back to, but a job with
``continue-on-error: true``, which fails no run (#2060).
+ A job whose ``os`` axis is read from the workflow's inputs counts in none of
+ them: it runs where its dispatcher chooses, as :file:`benchmark.yml`'s
+ ``benchmark`` does, and gates nothing (#2125). So a job that gates a run must not
+ take its runner from an input: it would be left out of the tiers' labels.
+
"""
default, axis, latest = set(), set(), set() # type: set[str], set[str], set[str]
for name, spec in workflows.items():
for job_id, job in (spec.get('jobs') or {}).items():
runs_on = str(job.get('runs-on', ''))
+ matrix = (job.get('strategy') or {}).get('matrix')
+ axis_labels = matrix.get('os') if isinstance(matrix, dict) else None
+ if isinstance(axis_labels, list) and any('inputs.' in str(label) for label in axis_labels):
+ continue
for step in setup_steps(job):
text = str((step.get('with') or {}).get('python-version', '')).strip()
read = READ.fullmatch(text)