From 63cb5cfc28f10d65e8af38111f1e3e76981f92ab Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 23 Oct 2026 20:44:35 -0400 Subject: [PATCH] ci(benchmark): run the engine benchmark on Actions and post the report to Discussions (#2125) - benchmark.yml: weekly (Wed 03:17 UTC) and on dispatch, with the runner, Python versions, engines, rounds and repeats as inputs. One job, on ubuntu-26.04-arm by default, installs the engine tiers from python-versions.toml and runs native.sh; `discussion`, the only job with `discussions: write`, posts the ratio table to `benchmark: weekly engine benchmark report` in CI reports. Nothing pushes; the docs stay hand-updated. - examples/benchmark/native.sh: run.sh without Docker, on the interpreters already installed, with run.sh's virtualenvs, version checks, gap notes and exit statuses. run.sh and the images stay the reproducible local path; the README says what a native run does not pin, and why arm is the default. - update_python_versions.py: a job whose runner is a dispatch input adds no tier label, so the arm default leaves the tiers' runners as they were. - workflows.rst: the table row, graph node and CI Reports thread. - The apt, AppArmor and Python-version tests allow benchmark.yml's new steps; test_benchmark_workflow_2125.py pins the workflow and runs its steps. --- .github/workflows/benchmark.yml | 288 +++++++++++ docs/source/contributing/workflows.rst | 29 +- examples/benchmark/README.md | 44 ++ examples/benchmark/native.sh | 301 +++++++++++ tests/project/test_apt_packages_1967.py | 7 +- tests/project/test_benchmark_workflow_2125.py | 481 ++++++++++++++++++ tests/project/test_python_versions_1660.py | 4 +- tests/project/test_tshark_apparmor_2019.py | 6 +- util/apt_packages.py | 3 + util/update_python_versions.py | 9 + 10 files changed, 1163 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100755 examples/benchmark/native.sh create mode 100644 tests/project/test_benchmark_workflow_2125.py diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..a6a7a03293 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,288 @@ +name: Engine Benchmark + +# #2125: times every extraction engine on every interpreter the engines are +# tested on, so the docs' Engine Support speed table no longer goes stale +# between hand runs (#2109). Natively, through examples/benchmark/native.sh +# rather than run.sh: a hosted runner is already a fresh VM per job, so +# containers would add a build per version and no isolation, and a native run +# makes macOS and Windows runners candidates too. One job measures every +# version, since the per-version table's absolute times compare only within +# one machine: by default `ubuntu-26.04-arm`, the fastest runner that +# measures every environment and engine. What a native run does not pin -- +# the interpreter's patch release, tshark and libpcap -- is recorded in the +# tables; examples/benchmark/README.md's "On GitHub Actions" says more, and +# gives the runner figures. +# +# The `discussion` job posts the ratio table to this job's thread in the +# Discussions category REPORT_CATEGORY names (#2030); it alone holds a write +# scope, and a report it cannot post is a warning. Nothing here pushes, and the +# docs stay hand-updated from the report. +on: + schedule: + # Wednesdays 03:17 UTC: off the hour, where GitHub delays scheduled runs + # most, and midweek, clear of the Saturday crons (#1967), Monday's Python + # Versions and CI Timing, and Project Status at 00:37. + - cron: '17 3 * * 3' + workflow_dispatch: + inputs: + runner: + description: Runner to measure on; dispatch once per runner to compare them. + type: choice + default: ubuntu-26.04-arm + options: + - ubuntu-26.04-arm + - ubuntu-latest + - macos-latest + - windows-latest + pythons: + description: Comma-separated Python versions to measure (empty = every version the engines are tested on). + type: string + default: '' + engines: + description: Comma-separated engines, including default (empty = all). + type: string + default: '' + rounds: + description: Timed extractions per engine per pass (empty = benchmark.py's default). + type: string + default: '' + repeats: + description: Passes over the whole engine set (empty = benchmark.py's default). + type: string + default: '' + +permissions: + contents: read + +# One run per runner at a time; dispatches on different runners still run side +# by side, which is how two runners are compared. +concurrency: + group: benchmark-${{ inputs.runner || 'ubuntu-26.04-arm' }} + cancel-in-progress: false + +# #2030: one Discussions category holds every report with no pull request to +# comment on, one thread per job. +env: + REPORT_CATEGORY: CI reports + +jobs: + # #1660: the interpreters are the engine tiers of .github/python-versions.toml, + # as unit-tests.yml's `engine-tests` installs them. + python-versions: + name: Read the Python versions + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + engine-supported: ${{ steps.read.outputs.engine-supported }} + engine-ktlo: ${{ steps.read.outputs.engine-ktlo }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: | + .github/python-versions.toml + util/_actions.py + util/python_versions.py + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Read .github/python-versions.toml + id: read + run: python3 util/python_versions.py --github-output + + benchmark: + name: Benchmark on ${{ matrix.os }} + needs: [ python-versions ] + # One leg, on the dispatch's runner, or `ubuntu-26.04-arm` on the schedule, + # which has no inputs; spelt as the other matrix jobs spell their runner. + runs-on: ${{ matrix.os || 'ubuntu-26.04-arm' }} + # Unmeasured on a hosted runner; the first dispatch measures it. Locally a + # full matrix took about two hours (the README's "How long it takes"), + # nearly all of it a tshark per pyshark extraction, so this leaves room for + # a slower runner under the six-hour job limit. + timeout-minutes: 300 + strategy: + matrix: + os: [ "${{ inputs.runner || 'ubuntu-26.04-arm' }}" ] + defaults: + run: + shell: bash + outputs: + status: ${{ steps.measure.outputs.status }} + env: + # Read by util/apt_packages.py install: libpcap's headers for pypcap's + # build, and tshark for pyshark. + APT_PACKAGES: libpcap-dev tshark + APT_SET: benchmark + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # As in `engine-tests`: each interpreter is then reachable, and the KTLO + # one gets a step of its own so that losing it cannot fail the job. + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + if: ${{ needs.python-versions.outputs.engine-ktlo != '' }} + continue-on-error: true + with: + python-version: ${{ needs.python-versions.outputs.engine-ktlo }} + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ needs.python-versions.outputs.engine-supported }} + + # Without its system package an engine is a row with its reason, so a + # failed install warns rather than failing the job. + - name: Install libpcap headers and tshark + if: runner.os == 'Linux' + timeout-minutes: 10 + run: | + /usr/bin/python3 util/apt_packages.py install \ + || echo '::warning title=System packages not installed::pypcap and pyshark may be reported unmeasured.' + + # #2019: tshark's AppArmor profile on Ubuntu 25.04 and later; see + # unit-tests.yml's `engine-tests` step of the same name, whose body this is. + - name: Let tshark read the checkout past its AppArmor profile + if: runner.os == 'Linux' + run: | + set -eu + if [ -f /etc/apparmor.d/tshark ]; then + sudo mkdir -p /etc/apparmor.d/local + echo "file r ${GITHUB_WORKSPACE}/**," | sudo tee -a /etc/apparmor.d/local/tshark >/dev/null + sudo apparmor_parser -r /etc/apparmor.d/tshark + fi + + # macOS ships libpcap; Homebrew's `wireshark` formula is the tshark CLI. + - name: Install tshark with Homebrew + if: runner.os == 'macOS' + timeout-minutes: 20 + env: + HOMEBREW_NO_AUTO_UPDATE: '1' + HOMEBREW_NO_INSTALL_CLEANUP: '1' + run: | + brew install wireshark \ + || echo '::warning title=tshark not installed::pyshark will be reported unmeasured.' + + # Reading a file needs no capture driver. Npcap's free edition cannot + # install unattended, so the pcap engines report the missing libpcap. + - name: Install tshark with Chocolatey + if: runner.os == 'Windows' + timeout-minutes: 15 + run: | + if choco install wireshark --yes --no-progress; then + echo 'C:\Program Files\Wireshark' >> "$GITHUB_PATH" + else + echo '::warning title=tshark not installed::pyshark will be reported unmeasured.' + fi + + # native.sh's exit status 3 is a table with gaps, each explained in it, + # so it warns; anything else but 0 is no table and fails. + - name: Run examples/benchmark/native.sh + id: measure + env: + INSTALLED: ${{ needs.python-versions.outputs.engine-ktlo }} ${{ needs.python-versions.outputs.engine-supported }} + PYTHONS: ${{ inputs.pythons }} + ENGINES: ${{ inputs.engines }} + ROUNDS: ${{ inputs.rounds }} + REPEATS: ${{ inputs.repeats }} + RUNNER_LABEL: ${{ matrix.os }} + run: | + set -uo pipefail + # `engine-supported` is a multi-line output, one version a line, and an + # input may say `3.11, 3.12`, so both are split on commas and whitespace. + read -ra installed <<< "$(echo "$INSTALLED" | tr -s ',[:space:]' ' ')" + read -ra wanted <<< "$(echo "${PYTHONS:-${installed[*]}}" | tr -s ',[:space:]' ' ')" + for version in "${wanted[@]}"; do + case " ${installed[*]} " in + *" $version "*) ;; + *) echo "::error title=Unknown Python version::'$version' is not among ${installed[*]}."; exit 2 ;; + esac + done + args=(--pythons "$(IFS=,; echo "${wanted[*]}")" --out out + --image "GitHub-hosted $RUNNER_LABEL, image ${ImageOS:-unknown} ${ImageVersion:-unknown}") + if [ -n "$ENGINES" ]; then args+=(--engines "$ENGINES"); fi + if [ -n "$ROUNDS" ]; then args+=(--rounds "$ROUNDS"); fi + if [ -n "$REPEATS" ]; then args+=(--repeats "$REPEATS"); fi + code=0 + examples/benchmark/native.sh "${args[@]}" || code=$? + case "$code" in + 0) echo 'status=success' >> "$GITHUB_OUTPUT" ;; + 3) echo 'status=warning' >> "$GITHUB_OUTPUT" + echo '::warning title=Not every Python version measured in full::The tables explain each gap.' ;; + *) echo 'status=failure' >> "$GITHUB_OUTPUT" + exit "$code" ;; + esac + + # The report names the runner, as the tables' Image row does, since a + # native run has no image digest to name. + - name: Write the report + if: ${{ always() && hashFiles('out/table.rst') != '' }} + env: + RUNNER_LABEL: ${{ matrix.os }} + run: | + set -eu + { + echo "## Engine benchmark on $RUNNER_LABEL" + echo + echo "Measured natively on $RUNNER_OS $RUNNER_ARCH, image ${ImageOS:-unknown} ${ImageVersion:-unknown}," + echo "so the absolute times compare only within this run. The docs' tables are updated by hand." + echo + echo '```rst' + cat out/table.rst + echo '```' + } > out/report.md + { + cat out/report.md + echo + echo '```rst' + cat out/table-versions.rst + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload the results + if: ${{ always() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: benchmark-out + path: out/ + if-no-files-found: ignore + retention-days: 90 + + # #2030: posts the ratio table, with its absolute-time footnote, on + # `always()` like ci-timing.yml's, so a run with gaps is posted too. + # util/discussion_report.py writes a warning rather than failing, including + # for a run that wrote no report; `continue-on-error` covers the rest. + discussion: + name: Post the engine benchmark report + needs: [ benchmark ] + if: ${{ always() && needs.benchmark.result != 'skipped' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + discussions: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Download the report + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: benchmark-out + path: report + + - name: Post the report to Discussions + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + STATUS: ${{ needs.benchmark.outputs.status || needs.benchmark.result }} + # `scheduled` on dispatch too: a manual run is a deliberate report. + run: | + python3 util/discussion_report.py --category "$REPORT_CATEGORY" \ + --title 'benchmark: weekly engine benchmark report' --report report/report.md \ + --mode scheduled --status "$STATUS" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index b10477ffd9..15066257ac 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -128,6 +128,13 @@ At a Glance - Mon 05:41 - -- - ``workflow_dispatch`` + * - Engine Benchmark + - :file:`benchmark.yml` + - -- + - -- + - Wed 03:17 + - -- + - ``workflow_dispatch`` .. note:: @@ -157,6 +164,7 @@ than starting a new one. PR["pull_request: main"] SAT["schedule: Saturday"] MON["schedule: Monday"] + WED["schedule: Wednesday"] DISPATCH["workflow_dispatch"] DAILY["schedule: daily"] ISSUES["issues"] @@ -173,6 +181,7 @@ than starting a new one. VU["Vendor Update
cron-vendor.yml"] PV["Python Versions
cron-python-versions.yml"] CT["CI Timing
ci-timing.yml"] + BM["Engine Benchmark
benchmark.yml"] PS["Project Status
project-status.yml"] CR["Create Release
create-release.yml"] @@ -210,6 +219,9 @@ than starting a new one. MON -->|07:23| CT DISPATCH --> CT + WED -->|03:17| BM + DISPATCH --> BM + DISPATCH --> PS DAILY -->|00:37| PS ISSUES --> PS @@ -218,7 +230,7 @@ than starting a new one. TAG --> CR classDef trig fill:none,stroke-dasharray:2 2 - class PUSH,PR,SAT,MON,DISPATCH,DAILY,ISSUES,PRT,TAG trig + class PUSH,PR,SAT,MON,WED,DISPATCH,DAILY,ISSUES,PRT,TAG trig .. mermaid:: @@ -1301,8 +1313,8 @@ to a script. Each script's docstring says what it does; each has tests under - ``Post the report to Discussions`` in the ``discussion`` job of ``ci-timing.yml``, ``cron-vendor.yml``, ``cron-conda.yml``, ``cron-python-versions.yml``, ``lint.yml``, ``deploy-pages.yml``, - ``codeql-analysis.yml``, ``coverage-comment.yml`` and ``ci-reports.yml`` - (see `CI Reports`_). + ``codeql-analysis.yml``, ``coverage-comment.yml``, ``ci-reports.yml`` and + ``benchmark.yml`` (see `CI Reports`_). - No. * - :file:`util/check_suite_report.py` - ``Write the run report`` in the ``discussion`` job of @@ -1312,7 +1324,8 @@ to a script. Each script's docstring says what it does; each has tests under * - :file:`util/apt_packages.py` - The apt steps of ``unit-tests.yml``'s ``engine-tests`` and ``pypcap-parity`` (and so its KTLO twin): the plan, install and collect steps, as - ``apt_packages.py plan``, ``install`` and ``collect``. The + ``apt_packages.py plan``, ``install`` and ``collect``, and of + ``benchmark.yml``'s ``benchmark``, as ``install`` alone. The ``actions/cache`` steps, the seed steps (``Seed apt's archive with this engine's cached .deb files`` and ``Seed apt's archive with the cached .deb files``) and the AppArmor step (`#2019 @@ -1469,8 +1482,12 @@ marker. How often depends on the script's ``--mode``: counts as green. ``ci-timing.yml`` posts as ``scheduled`` to ``ci-timing: weekly CI timing -report``. ``cron-vendor.yml`` posts to ``cron-vendor: vendor constant update -report``, as ``push`` on its ``workflow_run`` and ``scheduled`` otherwise, and +report``, and ``benchmark.yml`` to ``benchmark: weekly engine benchmark report``: +the ratio table :file:`examples/benchmark/native.sh` writes, with its +absolute-time footnote and the runner it ran on, reporting ``warning`` where a +Python version was not measured in full (:issue:`2125`). ``cron-vendor.yml`` +posts to ``cron-vendor: vendor constant update report``, as ``push`` on its +``workflow_run`` and ``scheduled`` otherwise, and reports ``warning`` where ``vendor-update`` passed over a failed target or test gate. A failed or cancelled ``head`` skips ``vendor-update``, so its ``discussion`` job runs on ``always()`` and then posts a line of its own with diff --git a/examples/benchmark/README.md b/examples/benchmark/README.md index 3a89c31c8a..353406585f 100644 --- a/examples/benchmark/README.md +++ b/examples/benchmark/README.md @@ -314,6 +314,50 @@ Not pinned, deliberately: has uncommitted changes under `pcapkit/` -- a revision that does not say so is a claim about code that was not measured. +## On GitHub Actions + +[`.github/workflows/benchmark.yml`](../../.github/workflows/benchmark.yml) runs the same +measurement weekly and on dispatch, **without Docker** (#2125), through +[`native.sh`](native.sh): `run.sh` with the interpreters already on the machine in +place of the images. A hosted runner is already a fresh VM per job, so containers would +add a build per version and no isolation. `native.sh` builds the virtualenvs the +`Dockerfile` builds, with `pypcap` where `python-images.txt`'s column says so, runs +`benchmark.py` in each and `report.py` over them all, with `run.sh`'s exit statuses. +The workflow measures the engines' interpreters from `.github/python-versions.toml`, all +in one job, since the per-version table's absolute times compare only within one +machine, and posts the ratio table to the `benchmark: weekly engine benchmark report` +thread in the `CI reports` Discussions category. + +What a native run does not pin, against the containers: + +- **The interpreter's patch release** is setup-python's newest build for that runner. + It differs by OS: on macOS and Windows, 3.10 to 3.12 stop at their last binary + release (3.12.10, say) where Linux has the newest. The report names every patch. +- **tshark and libpcap** are the runner's -- apt on Ubuntu, Homebrew on macOS, + Chocolatey on Windows -- and the report records their versions. Windows has no + libpcap, since Npcap's free edition cannot install unattended, so the `pcap` + engines are rows with their reason there. +- **The Image row** names the runner label and its image version, in place of a base + image. Both are public image names, not a machine. +- **Absolute times** come from whatever shared hardware the job landed on, so they + compare only within one run. The ratios are what carry from run to run -- within + an OS family. Across families they disagree: on one dispatch per runner, Windows' + `pypcapfile` ratio was about 44% above Ubuntu's. + +**The default runner is `ubuntu-26.04-arm`**, the fastest runner that +measures every environment and engine. One dispatch per runner (run 40047570595, on +#2128) had the `pcapkit` baseline at 0.580 ms per packet there against 0.757 on +`ubuntu-latest`, about 23% less time per packet, and both measured every environment +and every engine. `macos-latest` was faster still, at 0.339, but it and Windows could +measure neither `pcap_ct` nor `pypcap`, so they stay dispatch choices only. + +The docs' tables stay hand-updated, from `run.sh` or from a report. To compare runners, +dispatch once per runner; dropping pyshark makes it minutes rather than hours: + +```shell +gh workflow run benchmark.yml -f runner=ubuntu-latest -f engines=default,dpkt,scapy,pypcap,pcap_ct,pypcapfile +``` + ## Output Everything lands in `--out` (default `examples/benchmark/out/`, which is diff --git a/examples/benchmark/native.sh b/examples/benchmark/native.sh new file mode 100755 index 0000000000..0771a47878 --- /dev/null +++ b/examples/benchmark/native.sh @@ -0,0 +1,301 @@ +#!/usr/bin/env bash +# +# run.sh without Docker: measure the interpreters already on this machine, and +# print the same two tables. +# +# examples/benchmark/native.sh --pythons 3.11,3.12 +# +# For each version it builds the virtualenvs the Dockerfile builds -- `pcap_ct` +# always, and `pypcap` where python-images.txt says it can be built -- runs +# benchmark.py in each, as entrypoint.sh does, and then report.py over every +# document. .github/workflows/benchmark.yml runs it on a hosted runner, which is a +# fresh VM per job already, so containers there would add a build per version and +# no isolation (#2125). +# +# **What this does not pin**, against run.sh: the interpreter's patch release, and +# the system's tshark and libpcap. Each is recorded in the report rather than +# fixed, so the absolute times compare only within one run. The README's "On +# GitHub Actions" says more. For a table to publish from a machine of your own, +# run.sh is still the reproducible path. +# +# Failures are held to run.sh's standard: a version that cannot be found, built or +# measured is written to /missing/ and becomes an explained column, and an +# engine that cannot be installed is an explained row. Exit status is run.sh's: +# 0 measured in full, 3 tables written with gaps, 1 no table, 2 bad arguments. +# +# Written for bash 3.2, which is what macOS ships as /bin/bash, and for Git Bash +# on Windows, where a virtualenv's interpreter is under Scripts/ and a hosted +# runner's interpreters are found in its tool cache rather than as `python3.X`. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +readonly REPO_ROOT +readonly PINS_FILE="${SCRIPT_DIR}/python-images.txt" + +PYTHONS='' +ENGINES='' +ROUNDS='' +REPEATS='' +OUT_DIR="${SCRIPT_DIR}/out" +WORK_DIR="${REPO_ROOT}/.bench-venvs" +IMAGE="${BENCH_IMAGE:-}" +CAPTURE="${REPO_ROOT}/examples/captures/in.pcap" + +usage() { + cat <<'USAGE' +usage: native.sh [options] + +Measures each named Python version with the interpreter already on this machine, +in the virtualenvs the Dockerfile builds, and prints the two tables run.sh prints. + +options: + --pythons LIST comma-separated Python versions, e.g. 3.11,3.12; defaults to + every version python-images.txt marks 'default' + --engines LIST comma-separated engines, including 'default' (default: all) + --rounds N timed extractions per engine per pass (default: benchmark.py's) + --repeats N passes over the whole engine set (default: benchmark.py's) + --image TEXT what to name as the image in the report, e.g. the runner + --out DIR where to write the results (default examples/benchmark/out) + --work DIR where to build the virtualenvs (default .bench-venvs) + -h, --help this message + +exit status: 0 measured in full; 3 tables written, some version not measured in +full; 1 no table; 2 bad arguments. +USAGE +} + +while [ $# -gt 0 ]; do + case "$1" in + --pythons) PYTHONS="${2:?--pythons needs a value}"; shift 2 ;; + --engines) ENGINES="${2:?--engines needs a value}"; shift 2 ;; + --rounds) ROUNDS="${2:?--rounds needs a value}"; shift 2 ;; + --repeats) REPEATS="${2:?--repeats needs a value}"; shift 2 ;; + --image) IMAGE="${2:?--image needs a value}"; shift 2 ;; + --out) OUT_DIR="${2:?--out needs a value}"; shift 2 ;; + --work) WORK_DIR="${2:?--work needs a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "native.sh: unknown option '$1'" >&2; usage >&2; exit 2 ;; + esac +done + +# The pins file's column for a version, as run.sh's `pin_field` reads it: `$1 ""` +# makes awk compare strings, so 3.1 is not 3.10. +pin_field() { + awk -v version="$1" -v field="$2" \ + 'substr($1, 1, 1) != "#" && NF >= 4 && $1 "" == version "" { print $field; exit }' \ + "${PINS_FILE}" +} + +if [ -z "${PYTHONS}" ]; then + VERSIONS="$(awk 'substr($1, 1, 1) != "#" && NF >= 4 && $4 == "default" { printf "%s ", $1 }' "${PINS_FILE}")" +else + VERSIONS="$(echo "${PYTHONS}" | tr -s ',[:space:]' ' ')" +fi +if [ -z "${VERSIONS// /}" ]; then + echo 'native.sh: no Python versions to measure.' >&2 + exit 2 +fi + +# As in run.sh: a version python-images.txt does not name is a typo, and fatal +# rather than a column; a repeat is dropped, so it is not measured twice. +WANTED='' +for version in ${VERSIONS}; do + if [ -z "$(pin_field "${version}" 1)" ]; then + echo "native.sh: '${version}' is not in ${PINS_FILE}." >&2 + echo " Known versions: $(awk 'substr($1, 1, 1) != "#" && NF >= 4 { printf "%s ", $1 }' "${PINS_FILE}")" >&2 + exit 2 + fi + case " ${WANTED} " in + *" ${version} "*) ;; + *) WANTED="${WANTED} ${version}" ;; + esac +done +VERSIONS="${WANTED}" +if [ -n "${ENGINES}" ]; then + case ",${ENGINES}," in + *,default,*) ;; + *) echo "native.sh: --engines must include 'default'; it is the baseline every ratio uses." >&2 + exit 2 ;; + esac +fi + +BENCH_ARGS=() +if [ -n "${ENGINES}" ]; then BENCH_ARGS+=(--engines "${ENGINES}"); fi +if [ -n "${ROUNDS}" ]; then BENCH_ARGS+=(--rounds "${ROUNDS}"); fi +if [ -n "${REPEATS}" ]; then BENCH_ARGS+=(--repeats "${REPEATS}"); fi + +# The interpreter for a series, asked for its own version, as the Dockerfile labels +# an environment: a `python3.12` that is really something else must not fill the +# 3.12 column. `python3.X` first; then, on Windows, where setup-python provides no +# such names, the tool cache and the `py` launcher. +interpreter() { + local version="$1" candidate cache + local candidates=() + if command -v "python${version}" >/dev/null 2>&1; then + candidates+=("$(command -v "python${version}")") + fi + if [ -n "${RUNNER_TOOL_CACHE:-}" ]; then + cache="$(cygpath -u "${RUNNER_TOOL_CACHE}" 2>/dev/null || echo "${RUNNER_TOOL_CACHE}")" + for candidate in "${cache}/Python/${version}".*/x64/python.exe "${cache}/Python/${version}".*/arm64/python.exe; do + if [ -f "${candidate}" ]; then candidates+=("${candidate}"); fi + done + fi + if command -v py >/dev/null 2>&1; then + candidate="$(py "-${version}" -c 'import sys; print(sys.executable)' 2>/dev/null || true)" + if [ -n "${candidate}" ]; then candidates+=("${candidate}"); fi + fi + for candidate in ${candidates[@]+"${candidates[@]}"}; do + if [ "$("${candidate}" -c 'import sys; print("%d.%d" % sys.version_info[:2])' 2>/dev/null)" = "${version}" ]; then + echo "${candidate}" + return 0 + fi + done + return 1 +} + +# A virtualenv's own interpreter: bin/ everywhere but Windows, which uses Scripts/. +venv_python() { + if [ -d "$1/Scripts" ]; then echo "$1/Scripts/python"; else echo "$1/bin/python"; fi +} + +REVISION='unknown' +if command -v git >/dev/null 2>&1 && git -C "${REPO_ROOT}" rev-parse --git-dir >/dev/null 2>&1; then + REVISION="$(git -C "${REPO_ROOT}" rev-parse --short HEAD)" + if ! git -C "${REPO_ROOT}" diff --quiet HEAD -- pcapkit; then + REVISION="${REVISION}-dirty" + fi +fi +export PCAPKIT_REVISION="${REVISION}" +export BENCH_IMAGE="${IMAGE}" + +# A previous run's output is not inherited, for run.sh's reasons. +mkdir -p "${OUT_DIR}/missing" "${WORK_DIR}" +rm -f "${OUT_DIR}/missing/"*.txt "${OUT_DIR}"/*.json "${OUT_DIR}"/*.rst "${OUT_DIR}"/*.txt + +GAPS=0 +record_missing() { + printf '%s\n' "$2" > "${OUT_DIR}/missing/$1.txt" + GAPS=$((GAPS + 1)) + echo "native.sh: $1 will be reported as a gap in the table -- $2" >&2 +} + +REPORTER='' +for version in ${VERSIONS}; do + echo >&2 + echo "==> Python ${version}" >&2 + if ! python="$(interpreter "${version}")"; then + record_missing "${version}" "no Python ${version} interpreter was found on this machine" + continue + fi + "${python}" -c 'import platform, sys; print("==> " + platform.python_version() + " at " + sys.executable)' >&2 + + notes="${WORK_DIR}/${version}/notes" + rm -rf "${WORK_DIR:?}/${version}" + mkdir -p "${notes}/install-failures" "${notes}/not-attempted" + environments='' + + # pcap-ct's virtualenv is the one every interpreter holds, so a failure here + # costs the whole column. + venv="${WORK_DIR}/${version}/pcap_ct" + if "${python}" -m venv "${venv}" \ + && "$(venv_python "${venv}")" -m pip install --quiet --upgrade pip setuptools wheel \ + && "$(venv_python "${venv}")" -m pip install --quiet \ + --requirement "${SCRIPT_DIR}/requirements-pcap_ct.txt" \ + && "$(venv_python "${venv}")" -m pip install --quiet --no-deps "${REPO_ROOT}"; then + environments="${environments} pcap_ct" + else + record_missing "${version}" "the pcap_ct virtualenv could not be built for Python ${version}; see the log" + continue + fi + + # pypcap compiles, so its install alone may fail, and is then a row with the + # pip error, in the Dockerfile's words. + if [ "$(pin_field "${version}" 3)" = 'pypcap' ]; then + venv="${WORK_DIR}/${version}/pypcap" + log="${WORK_DIR}/${version}/pypcap-install.log" + if "${python}" -m venv "${venv}" \ + && "$(venv_python "${venv}")" -m pip install --quiet --upgrade pip setuptools wheel \ + && "$(venv_python "${venv}")" -m pip install --quiet \ + --requirement "${SCRIPT_DIR}/requirements-common.txt"; then + if ! "$(venv_python "${venv}")" -m pip install \ + --requirement "${SCRIPT_DIR}/requirements-pypcap.txt" > "${log}" 2>&1; then + echo 'WARNING: pypcap failed to install; it will be reported as not measured.' >&2 + { echo 'pypcap failed to build here, so it could not be measured.' + echo 'The tail of the pip output was:' + tail -n 12 "${log}"; } > "${notes}/install-failures/pypcap.txt" + cat "${log}" >&2 + fi + if "$(venv_python "${venv}")" -m pip install --quiet --no-deps "${REPO_ROOT}"; then + environments="${environments} pypcap" + fi + else + echo 'pypcap could not be measured: its virtualenv failed to build; see the log.' \ + > "${notes}/install-failures/pypcap.txt" + fi + else + { echo 'pypcap is not installable on this interpreter, so it was not built here:' + echo 'pypcap 1.3.0 ships a pcap.c pre-generated by Cython 0.29.32, which does' + echo 'not compile against the Python 3.12+ C API. The engine is unsupported here' + echo 'rather than broken here.'; } > "${notes}/not-attempted/pypcap.txt" + fi + + if [ -z "${REPORTER}" ]; then + REPORTER="$(venv_python "${WORK_DIR}/${version}/pcap_ct")" + fi + + failed='' + for provider in ${environments}; do + label="${version}-${provider}" + venv="${WORK_DIR}/${version}/${provider}" + echo "==> measuring in the ${label} environment" >&2 + if ! BENCH_INSTALL_FAILURES="${notes}/install-failures" BENCH_NOT_ATTEMPTED="${notes}/not-attempted" \ + "$(venv_python "${venv}")" "${SCRIPT_DIR}/benchmark.py" --capture "${CAPTURE}" \ + --label "${label}" --out "${OUT_DIR}/${label}.json" ${BENCH_ARGS[@]+"${BENCH_ARGS[@]}"}; then + failed="${failed} ${label}" + fi + "$(venv_python "${venv}")" -m pip freeze > "${OUT_DIR}/${label}.txt" 2>/dev/null || true + done + if [ -n "${failed}" ]; then + record_missing "${version}" \ + "measuring in${failed} exited non-zero; any environment that finished first is still reported" + fi +done + +DOCUMENTS=() +for path in "${OUT_DIR}"/*.json; do + if [ -e "${path}" ]; then DOCUMENTS+=("${path}"); fi +done + +echo >&2 +if [ "${#DOCUMENTS[@]}" -eq 0 ] || [ -z "${REPORTER}" ]; then + echo 'native.sh: no environment produced a measurement; there is nothing to report.' >&2 + exit 1 +fi + +REPORT_ARGS=() +for path in "${OUT_DIR}"/missing/*.txt; do + if [ -e "${path}" ]; then + REPORT_ARGS+=(--missing "$(basename "${path}" .txt)=$(tr '\n' ' ' < "${path}")") + fi +done + +status=0 +"${REPORTER}" "${SCRIPT_DIR}/report.py" "${DOCUMENTS[@]}" \ + --rst-out "${OUT_DIR}/table.rst" --versions-rst-out "${OUT_DIR}/table-versions.rst" \ + ${REPORT_ARGS[@]+"${REPORT_ARGS[@]}"} || status=$? + +if [ ! -f "${OUT_DIR}/table.rst" ] || [ ! -f "${OUT_DIR}/table-versions.rst" ]; then + echo "native.sh: no table was produced; ${OUT_DIR} holds whatever the run got to." >&2 + exit 1 +fi +if [ "${status}" -ne 0 ]; then + exit 1 +fi +if [ "${GAPS}" -gt 0 ]; then + echo "native.sh: ${GAPS} of the requested Python version(s) could not be measured in full;" >&2 + echo " the tables were still written, with the reasons in them and in ${OUT_DIR}/missing/." >&2 + exit 3 +fi +exit 0 diff --git a/tests/project/test_apt_packages_1967.py b/tests/project/test_apt_packages_1967.py index 60954d0243..9a257b63ec 100644 --- a/tests/project/test_apt_packages_1967.py +++ b/tests/project/test_apt_packages_1967.py @@ -502,6 +502,10 @@ def test_refused(self) -> None: f'{IMAGE_PYTHON} {SCRIPT} collect'), ) SITES = {'engine-tests': ENGINE_STEPS, 'pypcap-parity': PARITY_STEPS} +#: #2125: steps of other workflows that run the script's ``install`` alone, with no +#: cache to plan or collect for, as (workflow, step name). Each is pinned by its +#: workflow's own test, as :mod:`tests.project.test_benchmark_workflow_2125`. +ELSEWHERE = {('benchmark.yml', 'Install libpcap headers and tshark')} #: Each step's former shell, by the subcommand that replaced it. FORMER = {'plan': FORMER_PLAN, 'collect': FORMER_COLLECT} APPARMOR = 'Let tshark read the checkout past its AppArmor profile' @@ -798,7 +802,8 @@ def test_no_step_keeps_apt_shell(self) -> None: if not line.lstrip().startswith('#')) found.extend(f'{path.name} {job} {item.get("name")!r}: {piece}' for piece in INLINE if piece in run) - if SCRIPT in run and (path.name, item.get('name')) not in {(WORKFLOW, name) for name in covered}: + listed = {(WORKFLOW, name) for name in covered} | ELSEWHERE + if SCRIPT in run and (path.name, item.get('name')) not in listed: found.append(f'{path.name} {job} {item.get("name")!r}: unlisted') self.assertEqual(found, []) diff --git a/tests/project/test_benchmark_workflow_2125.py b/tests/project/test_benchmark_workflow_2125.py new file mode 100644 index 0000000000..79239a81ef --- /dev/null +++ b/tests/project/test_benchmark_workflow_2125.py @@ -0,0 +1,481 @@ +# -*- coding: utf-8 -*- +"""#2125: :file:`benchmark.yml`, the engine benchmark on GitHub Actions. + +The workflow runs :file:`examples/benchmark/native.sh` in one job on one runner, +over the engines' interpreters from :file:`.github/python-versions.toml`, and posts +the ratio table to Discussions. + +* :class:`TestTriggers`: a weekly ``schedule`` and ``workflow_dispatch`` with the + runner, the versions, the engines and the rounds and passes as inputs; no push, + pull request or ``workflow_run``. +* :class:`TestTheCron`: off the hour, and in no hour on no day that another + workflow's ``schedule`` fires in. +* :class:`TestPermissions`: read-only at the top, ``discussions: write`` on the + ``discussion`` job alone, and no push. +* :class:`TestTheJobs`: one leg on the chosen runner with the engine tiers + installed, running ``native.sh`` and never Docker; the tshark steps as + ``unit-tests.yml``'s; the post to ``: ``. +* :class:`TestTheMeasureStep` and :class:`TestTheReportStep` run those steps' own + ``run:`` scripts under bash, with a stand-in ``native.sh``. +* :class:`TestNativeScript` runs :file:`examples/benchmark/native.sh` itself on + what it refuses and on a version with no interpreter, which need no network. + +""" + +from __future__ import annotations + +import importlib.util +import os +import pathlib +import re +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING + +from tests._workflows import WORKFLOWS, job, load, steps + +try: + import tomllib +except ImportError: # Python 3.10 and older + tomllib = None # type: ignore[assignment] + +if TYPE_CHECKING: + from typing import Any, Optional + +#: Test tier (#1958): reads the workflow files and runs their steps and the tree's +#: scripts, a question about the tree. +__tier__ = 'LATEST' + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOW = 'benchmark.yml' +NATIVE = ROOT / 'examples' / 'benchmark' / 'native.sh' +PINS = ROOT / 'examples' / 'benchmark' / 'python-images.txt' +VERSIONS_SCRIPT = ROOT / 'util' / 'python_versions.py' +UPDATER = ROOT / 'util' / 'update_python_versions.py' +BASH = shutil.which('bash') +#: How ``shell: bash`` runs a step's script, ahead of the script. +SHELL = [BASH or 'bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c'] +TITLE = 'benchmark: weekly engine benchmark report' +INPUTS = {'runner', 'pythons', 'engines', 'rounds', 'repeats'} +MEASURE = 'Run examples/benchmark/native.sh' +#: The runner a scheduled run, and a dispatch left at its default, measures on. +DEFAULT_RUNNER = 'ubuntu-26.04-arm' +#: A cron's fields, ``minute hour day month weekday``. +CRON = re.compile(r'^(\d+) (\d+) (\S+) (\S+) (\S+)$') +#: The stand-in ``native.sh``: logs its arguments, one a line, and exits ``$NATIVE_EXIT``. +FAKE_NATIVE = textwrap.dedent('''\ + #!/bin/bash + printf '%s\\n' "$@" > "$NATIVE_LOG" + exit "${NATIVE_EXIT:-0}" +''') + + +def step(job_id: 'str', name: 'str') -> 'dict[str, Any]': + """Job ``job_id``'s step named ``name``.""" + found = [item for item in steps(WORKFLOW, job_id) if item.get('name') == name] + assert len(found) == 1, f'{job_id} has {len(found)} steps named {name!r}' # nosec: B101 + return found[0] + + +def weekdays(field: 'str') -> 'set[int]': + """The weekdays a cron's weekday field names, 0 for Sunday; ``*`` is every day.""" + if field == '*': + return set(range(7)) + days = set() # type: set[int] + for part in field.split(','): + low, _, high = part.partition('-') + days.update(day % 7 for day in range(int(low), int(high or low) + 1)) + return days + + +def slots(cron: 'str') -> 'set[tuple[int, int]]': + """Every (weekday, hour) ``cron`` fires in, for the plain crons the workflows use.""" + match = CRON.match(cron) + assert match is not None, f'a cron this test cannot read: {cron!r}' # nosec: B101 + _, hour, day, month, weekday = match.groups() + assert (day, month) == ('*', '*'), f'a cron this test cannot read: {cron!r}' # nosec: B101 + return {(found, int(hour)) for found in weekdays(weekday)} + + +def github_outputs() -> 'dict[str, str]': + """``util/python_versions.py --github-output`` as the runner reads ``$GITHUB_OUTPUT``. + + ``name=value``, or ``name< 'str': + """A step's ``env:`` value with each ``needs.python-versions.outputs.X`` substituted.""" + return re.sub(r'\$\{\{ needs\.python-versions\.outputs\.([\w-]+) \}\}', lambda match: outputs[match.group(1)], + template) + + +def pinned_versions() -> 'list[str]': + """Every version python-images.txt has a row for.""" + return [line.split()[0] for line in PINS.read_text(encoding='utf-8').splitlines() + if not line.startswith('#') and len(line.split()) >= 4] + + +def crons(spec: 'dict[str, Any]') -> 'list[str]': + """A parsed workflow's ``schedule`` crons.""" + schedule = (spec.get('on') or {}).get('schedule') or [] + return [str(entry['cron']) for entry in schedule] + + +class TestTriggers(unittest.TestCase): + + def test_a_weekly_schedule_and_a_dispatch_alone(self) -> None: + spec = load(WORKFLOW) + self.assertEqual(set(spec['on']), {'schedule', 'workflow_dispatch'}) + self.assertEqual(len(crons(spec)), 1) + _, _, day, month, weekday = CRON.match(crons(spec)[0]).groups() # type: ignore[union-attr] + self.assertEqual((day, month, len(weekdays(weekday))), ('*', '*', 1)) + + def test_the_dispatch_inputs(self) -> None: + inputs = load(WORKFLOW)['on']['workflow_dispatch']['inputs'] + self.assertEqual(set(inputs), INPUTS) + runner = inputs['runner'] + # The fastest runner that measures every engine, on #2128; the others stay choices. + self.assertEqual((runner['type'], runner['default']), ('choice', DEFAULT_RUNNER)) + self.assertEqual(runner['options'][0], DEFAULT_RUNNER) + self.assertLessEqual({'ubuntu-latest', 'macos-latest', 'windows-latest'}, set(runner['options'])) + # Empty is the default: every engine interpreter, benchmark.py's engines and counts. + for name in INPUTS - {'runner'}: + with self.subTest(input=name): + self.assertEqual(inputs[name].get('default'), '') + + def test_a_scheduled_run_falls_back_to_the_default_runner(self) -> None: + """A schedule has no inputs, so every read of the runner names the default.""" + text = (WORKFLOWS / WORKFLOW).read_text(encoding='utf-8') + reads = re.findall(r'inputs\.runner\b(?: \|\| \'([\w.-]+)\')?', text) + self.assertTrue(reads) + self.assertEqual(set(reads), {DEFAULT_RUNNER}) + + +class TestTheCron(unittest.TestCase): + + def test_off_the_hour(self) -> None: + minute = int(CRON.match(crons(load(WORKFLOW))[0]).group(1)) # type: ignore[union-attr] + self.assertNotIn(minute, (0, 15, 30, 45)) + + def test_no_other_schedule_shares_its_hour(self) -> None: + mine = slots(crons(load(WORKFLOW))[0]) + for path in sorted(WORKFLOWS.glob('*.yml')): + if path.name == WORKFLOW: + continue + for cron in crons(load(path)): + with self.subTest(workflow=path.name, cron=cron): + self.assertFalse(mine & slots(cron)) + + def test_a_collision_is_caught(self) -> None: + self.assertTrue(slots('17 3 * * 3') & slots('37 3 * * *')) + self.assertTrue(slots('17 7 * * 1') & slots('23 7 * * 1')) + self.assertFalse(slots('17 3 * * 3') & slots('17 3 * * 6')) + + +class TestPermissions(unittest.TestCase): + + def test_read_only_at_the_top(self) -> None: + self.assertEqual(load(WORKFLOW)['permissions'], {'contents': 'read'}) + + def test_only_the_discussion_job_writes(self) -> None: + for job_id, spec in load(WORKFLOW)['jobs'].items(): + with self.subTest(job=job_id): + if job_id == 'discussion': + self.assertEqual(spec['permissions'], {'contents': 'read', 'discussions': 'write'}) + else: + self.assertNotIn('permissions', spec) + + def test_no_job_pushes_or_keeps_a_token(self) -> None: + for job_id, spec in load(WORKFLOW)['jobs'].items(): + for item in spec['steps']: + with self.subTest(job=job_id, step=item.get('name') or item.get('uses')): + self.assertNotRegex(str(item.get('run', '')), r'\bgit (?:push|commit|tag)\b') + if str(item.get('uses', '')).startswith('actions/checkout@'): + self.assertIs(item['with']['persist-credentials'], False) + + +class TestTheJobs(unittest.TestCase): + + def test_the_jobs_and_their_order(self) -> None: + jobs = load(WORKFLOW)['jobs'] + self.assertEqual(list(jobs), ['python-versions', 'benchmark', 'discussion']) + self.assertEqual(jobs['benchmark']['needs'], ['python-versions']) + self.assertEqual(jobs['discussion']['needs'], ['benchmark']) + # A failed or partial benchmark still posts. + self.assertEqual(jobs['discussion']['if'], "${{ always() && needs.benchmark.result != 'skipped' }}") + + def test_one_leg_on_the_chosen_runner(self) -> None: + """Every version on one machine, as the per-version table's absolute times require.""" + benchmark = job(WORKFLOW, 'benchmark') + self.assertEqual(benchmark['strategy']['matrix'], {'os': [f"${{{{ inputs.runner || '{DEFAULT_RUNNER}' }}}}"]}) + self.assertEqual(benchmark['runs-on'], f"${{{{ matrix.os || '{DEFAULT_RUNNER}' }}}}") + # Only the measurement moves to it: reading the tiers and posting stay on x64. + for job_id in ('python-versions', 'discussion'): + with self.subTest(job=job_id): + self.assertEqual(job(WORKFLOW, job_id)['runs-on'], 'ubuntu-latest') + + def test_the_engine_tiers_are_installed(self) -> None: + setups = [item for item in steps(WORKFLOW, 'benchmark') + if str(item.get('uses', '')).startswith('actions/setup-python@')] + self.assertEqual([item['with']['python-version'] for item in setups], + ['${{ needs.python-versions.outputs.engine-ktlo }}', + '${{ needs.python-versions.outputs.engine-supported }}']) + self.assertIs(setups[0]['continue-on-error'], True) + self.assertEqual(step('python-versions', 'Read .github/python-versions.toml')['run'], + 'python3 util/python_versions.py --github-output') + + def test_it_runs_native_sh_and_never_docker(self) -> None: + """run.sh and its images are the local path; the workflow measures on the runner.""" + self.assertIn('examples/benchmark/native.sh "${args[@]}"', str(step('benchmark', MEASURE)['run'])) + for job_id, spec in load(WORKFLOW)['jobs'].items(): + for item in spec['steps']: + with self.subTest(job=job_id, step=item.get('name') or item.get('uses')): + self.assertNotRegex(str(item.get('run', '')), r'\bdocker\b|run\.sh') + + def test_the_post(self) -> None: + run = str(step('discussion', 'Post the report to Discussions')['run']) + self.assertEqual(' '.join(run.replace('\\\n', ' ').split()), + 'python3 util/discussion_report.py --category "$REPORT_CATEGORY" ' + f"--title '{TITLE}' --report report/report.md --mode scheduled --status \"$STATUS\"") + self.assertEqual(load(WORKFLOW)['env']['REPORT_CATEGORY'], 'CI reports') + self.assertTrue(TITLE.startswith(f'{pathlib.Path(WORKFLOW).stem}: ')) + self.assertIs(job(WORKFLOW, 'discussion')['continue-on-error'], True) + self.assertEqual(step('discussion', 'Post the report to Discussions')['env']['STATUS'], + '${{ needs.benchmark.outputs.status || needs.benchmark.result }}') + + def test_tshark_is_installed_and_let_read_the_checkout_as_in_unit_tests(self) -> None: + """#2019's AppArmor step directly follows the apt install, with ``engine-tests``' body.""" + names = [item.get('name') for item in steps(WORKFLOW, 'benchmark')] + install = 'Install libpcap headers and tshark' + apparmor = 'Let tshark read the checkout past its AppArmor profile' + self.assertEqual(names[names.index(install) + 1], apparmor) + self.assertIn('tshark', job(WORKFLOW, 'benchmark')['env']['APT_PACKAGES'].split()) + self.assertTrue(str(step('benchmark', install)['run']).startswith( + '/usr/bin/python3 util/apt_packages.py install')) + theirs = [item for item in steps('unit-tests.yml', 'engine-tests') if item.get('name') == apparmor] + self.assertEqual(step('benchmark', apparmor)['run'], theirs[0]['run']) + + +class TestTheTiersIgnoreTheRunner(unittest.TestCase): + """util/update_python_versions.py reads no tier label from a runner the dispatcher picks. + + Its tiers want the default labels on one Linux architecture, and the benchmark's + default is arm64 where every test job's is x64. + + """ + + @classmethod + def setUpClass(cls) -> None: + if tomllib is None: # pragma: no cover + raise unittest.SkipTest('tomllib, which util/update_python_versions.py imports, needs Python 3.11+') + spec = importlib.util.spec_from_file_location('update_python_versions_2125', UPDATER) + assert spec is not None and spec.loader is not None # nosec: B101 + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) # type: ignore[union-attr] + + def workflows(self) -> 'dict[str, Any]': + return {path.name: load(path) for path in sorted(WORKFLOWS.glob('*.yml'))} + + def test_the_benchmark_adds_no_label(self) -> None: + with_it = self.module.workflow_labels(self.workflows()) + without = self.workflows() + del without[WORKFLOW] + self.assertEqual(with_it, self.module.workflow_labels(without)) + self.assertNotIn(DEFAULT_RUNNER, with_it['default']) + + def test_a_fixed_arm_runner_would_count(self) -> None: + """The rule bites only on an input: the same job on a literal axis adds its label.""" + workflows = self.workflows() + workflows[WORKFLOW]['jobs']['benchmark']['strategy']['matrix']['os'] = [DEFAULT_RUNNER] + self.assertIn(DEFAULT_RUNNER, self.module.workflow_labels(workflows)['default']) + + +@unittest.skipIf(not BASH, 'needs bash on PATH') +class _Scratch(unittest.TestCase): + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-2125-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.output = self.tmp / 'output' + self.output.write_text('', encoding='utf-8') + self.summary = self.tmp / 'summary.md' + + def run_step(self, job_id: 'str', name: 'str', **env: 'str') -> 'subprocess.CompletedProcess[str]': + """``name``'s ``run:`` as ``shell: bash`` runs it, in the scratch directory.""" + environ = dict(os.environ, GITHUB_OUTPUT=str(self.output), GITHUB_STEP_SUMMARY=str(self.summary), + RUNNER_OS='Linux', RUNNER_ARCH='X64', ImageOS='ubuntu26', ImageVersion='20261020.1', **env) + return subprocess.run([*SHELL, str(step(job_id, name)['run'])], env=environ, # nosec: B603 + cwd=self.tmp, capture_output=True, text=True, check=False, timeout=60) + + def outputs(self) -> 'dict[str, str]': + return dict(line.split('=', 1) for line in self.output.read_text(encoding='utf-8').splitlines()) + + +class TestTheMeasureStep(_Scratch): + """``Run examples/benchmark/native.sh`` against a stand-in script.""" + + def setUp(self) -> None: + super().setUp() + script = self.tmp / 'examples' / 'benchmark' / 'native.sh' + script.parent.mkdir(parents=True) + script.write_text(FAKE_NATIVE, encoding='utf-8') + script.chmod(0o755) + self.log = self.tmp / 'native.log' + + def measure(self, code: 'int' = 0, **inputs: 'str') -> 'tuple[int, Optional[list[str]], dict[str, str]]': + """The step, with ``INSTALLED`` as the runner renders it from the script's real outputs.""" + installed = render(step('benchmark', MEASURE)['env']['INSTALLED'], self.versions) + env = {'INSTALLED': installed, 'PYTHONS': '', 'ENGINES': '', 'ROUNDS': '', 'REPEATS': '', + 'RUNNER_LABEL': 'ubuntu-latest', 'NATIVE_LOG': str(self.log), 'NATIVE_EXIT': str(code)} + env.update(inputs) + result = self.run_step('benchmark', MEASURE, **env) + args = self.log.read_text(encoding='utf-8').splitlines() if self.log.exists() else None + return result.returncode, args, self.outputs() + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.versions = github_outputs() + + def test_the_outputs_are_the_multi_line_form(self) -> None: + """What the step must split: ``engine-supported`` one version a line.""" + self.assertIn('\n', self.versions['engine-supported']) + self.assertIn('\n', render(step('benchmark', MEASURE)['env']['INSTALLED'], self.versions)) + + def test_a_scheduled_run_measures_every_interpreter(self) -> None: + status, args, outputs = self.measure() + self.assertEqual((status, outputs), (0, {'status': 'success'})) + every = (self.versions['engine-ktlo'] + ' ' + self.versions['engine-supported']).split() + self.assertEqual(args, ['--pythons', ','.join(every), '--out', 'out', + '--image', 'GitHub-hosted ubuntu-latest, image ubuntu26 20261020.1']) + + def test_a_spaced_list_is_read_as_a_list(self) -> None: + status, args, _ = self.measure(PYTHONS=' 3.12, 3.11 ,3.13') + self.assertEqual(status, 0) + assert args is not None # nosec: B101 + self.assertEqual(args[:2], ['--pythons', '3.12,3.11,3.13']) + + def test_every_interpreter_has_a_pins_row(self) -> None: + """native.sh refuses a version python-images.txt lacks, so a new tier needs its row first.""" + every = (self.versions['engine-ktlo'] + ' ' + self.versions['engine-supported']).split() + self.assertTrue(every) + self.assertLessEqual(set(every), set(pinned_versions())) + + def test_the_dispatch_inputs_reach_the_script(self) -> None: + status, args, _ = self.measure(PYTHONS='3.12,3.11', ENGINES='default,dpkt', ROUNDS='50', REPEATS='2') + self.assertEqual(status, 0) + assert args is not None # nosec: B101 + self.assertEqual(args[:2], ['--pythons', '3.12,3.11']) + self.assertEqual(args[6:], ['--engines', 'default,dpkt', '--rounds', '50', '--repeats', '2']) + + def test_a_version_not_installed_is_refused(self) -> None: + status, args, outputs = self.measure(PYTHONS='3.11,3.1') + self.assertEqual((status, args, outputs), (2, None, {})) + + def test_gaps_warn_and_no_table_fails(self) -> None: + status, _, outputs = self.measure(code=3) + self.assertEqual((status, outputs), (0, {'status': 'warning'})) + self.output.write_text('', encoding='utf-8') + status, _, outputs = self.measure(code=1) + self.assertEqual((status, outputs), (1, {'status': 'failure'})) + + +class TestTheReportStep(_Scratch): + """``Write the report``: the ratio table for the post, and both tables for the summary.""" + + def test_the_report_names_the_runner_and_fences_the_tables(self) -> None: + out = self.tmp / 'out' + out.mkdir() + (out / 'table.rst').write_text('RATIOS\n', encoding='utf-8') + (out / 'table-versions.rst').write_text('ABSOLUTES\n', encoding='utf-8') + self.assertEqual(self.run_step('benchmark', 'Write the report', RUNNER_LABEL='macos-latest').returncode, 0) + report = (out / 'report.md').read_text(encoding='utf-8') + self.assertTrue(report.startswith('## Engine benchmark on macos-latest\n')) + self.assertIn('Linux X64, image ubuntu26 20261020.1', report) + self.assertTrue(report.endswith('```rst\nRATIOS\n```\n')) + self.assertNotIn('ABSOLUTES', report) + self.assertEqual(self.summary.read_text(encoding='utf-8'), f'{report}\n```rst\nABSOLUTES\n```\n') + self.assertEqual(step('benchmark', 'Write the report')['if'], + "${{ always() && hashFiles('out/table.rst') != '' }}") + + +class TestNativeScript(_Scratch): + """``native.sh`` itself, where it needs no interpreter install: a copy, beside a pins file with a 3.99 row.""" + + def setUp(self) -> None: + super().setUp() + directory = self.tmp / 'examples' / 'benchmark' + directory.mkdir(parents=True) + shutil.copy(NATIVE, directory / 'native.sh') + (directory / 'python-images.txt').write_text( + PINS.read_text(encoding='utf-8') + '3.99 python:3.99-slim-bookworm@sha256:00 no-pypcap opt-in\n', + encoding='utf-8') + self.script = directory / 'native.sh' + + def native(self, *args: 'str') -> 'subprocess.CompletedProcess[str]': + command = [SHELL[0], str(self.script), '--out', str(self.tmp / 'out'), '--work', str(self.tmp / 'work'), + *args] + return subprocess.run(command, cwd=self.tmp, # nosec: B603 + capture_output=True, text=True, check=False, timeout=60, + env=dict(os.environ, PATH=os.defpath, RUNNER_TOOL_CACHE='')) + + def test_it_refuses_bad_arguments(self) -> None: + self.assertEqual(self.native('--bogus').returncode, 2) + self.assertEqual(self.native('--engines', 'dpkt').returncode, 2) + + def test_a_version_the_pins_lack_is_a_typo(self) -> None: + """As in run.sh, and before anything is written: 3.1 is not 3.10.""" + for version in ('3.1', '3.98'): + with self.subTest(version=version): + result = self.native('--pythons', f'3.99,{version}') + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(f"native.sh: '{version}' is not in ", result.stderr) + self.assertFalse((self.tmp / 'out').exists()) + + def test_a_repeat_is_measured_once(self) -> None: + result = self.native('--pythons', '3.99, 3.99,3.99') + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stderr.count('==> Python 3.99'), 1) + + def test_a_version_with_no_interpreter_is_a_gap_and_no_table(self) -> None: + result = self.native('--pythons', '3.99') + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual((self.tmp / 'out' / 'missing' / '3.99.txt').read_text(encoding='utf-8'), + 'no Python 3.99 interpreter was found on this machine\n') + self.assertIn('no environment produced a measurement', result.stderr) + + def test_it_is_executable(self) -> None: + self.assertTrue(os.access(NATIVE, os.X_OK)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index 722597335d..0fe805854a 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -836,7 +836,9 @@ def test_every_setup_reads_the_matrix_or_the_file(self) -> None: if where in exemptions or 'matrix.python-version' in text: continue output = script_output(name, job_id, READ.fullmatch(text).group(2)) # type: ignore[union-attr] - expected = {'engine-tests': ('engine-supported', 'engine-ktlo')}.get(job_id, ('latest',)) + # #2125: benchmark.yml's `benchmark` installs the engines' interpreters too. + expected = {'engine-tests': ('engine-supported', 'engine-ktlo'), + 'benchmark': ('engine-supported', 'engine-ktlo')}.get(job_id, ('latest',)) self.assertIn(output, expected) def test_every_single_interpreter_job_runs_latest(self) -> None: diff --git a/tests/project/test_tshark_apparmor_2019.py b/tests/project/test_tshark_apparmor_2019.py index 1773926a16..f472ee6193 100644 --- a/tests/project/test_tshark_apparmor_2019.py +++ b/tests/project/test_tshark_apparmor_2019.py @@ -37,6 +37,10 @@ #: An AppArmor file rule in ``file ACCESS FILEGLOB,`` form granting read on a tree #: (apparmor.d(5), FILE RULE), the form the shipped tshark profile itself uses. RULE = re.compile(r'^file r (/\S+)/\*\*,$') +#: #2125: jobs of other workflows that install tshark and run the step, as +#: (workflow, job). Each is pinned by its workflow's own test, as +#: :mod:`tests.project.test_benchmark_workflow_2125`. +ELSEWHERE = {('benchmark.yml', 'benchmark')} STUBS = { 'sudo': 'exec env "$@"\n', 'apparmor_parser': 'echo "$@" >> "$STUB_LOG"\n', @@ -79,7 +83,7 @@ def test_every_job_installing_tshark_is_covered(self) -> None: re.search(r'\butil/apt_packages\.py install\b', run) and re.search(r'\btshark\b', packages)): found.add((path.name, job_id)) - self.assertEqual(found, {(WORKFLOW, job_id) for job_id, _ in SITES}) + self.assertEqual(found, {(WORKFLOW, job_id) for job_id, _ in SITES} | ELSEWHERE) def test_one_body_everywhere(self) -> None: """The jobs run one script, so a fix to it cannot reach only some of them.""" diff --git a/util/apt_packages.py b/util/apt_packages.py index aaf72cd692..5909fbd3c2 100644 --- a/util/apt_packages.py +++ b/util/apt_packages.py @@ -9,6 +9,9 @@ /usr/bin/python3 util/apt_packages.py install [--name NAME] /usr/bin/python3 util/apt_packages.py collect +:file:`.github/workflows/benchmark.yml`'s ``benchmark`` job runs ``install`` alone, +with no cache to plan or collect for (#2125). + ``APT_PACKAGES``, the job's environment, names the packages, separated by whitespace, for both ``plan`` and ``install``, so the cache key cannot describe a set other than the one installed. Each subcommand runs the commands the steps' diff --git a/util/update_python_versions.py b/util/update_python_versions.py index 478ced9ebf..0104acc57a 100644 --- a/util/update_python_versions.py +++ b/util/update_python_versions.py @@ -567,11 +567,20 @@ def workflow_labels(workflows: 'dict[str, Any]') -> 'dict[str, list[str]]': label its ``matrix.os || '...'`` falls back to, but a job with ``continue-on-error: true``, which fails no run (#2060). + A job whose ``os`` axis is read from the workflow's inputs counts in none of + them: it runs where its dispatcher chooses, as :file:`benchmark.yml`'s + ``benchmark`` does, and gates nothing (#2125). So a job that gates a run must not + take its runner from an input: it would be left out of the tiers' labels. + """ default, axis, latest = set(), set(), set() # type: set[str], set[str], set[str] for name, spec in workflows.items(): for job_id, job in (spec.get('jobs') or {}).items(): runs_on = str(job.get('runs-on', '')) + matrix = (job.get('strategy') or {}).get('matrix') + axis_labels = matrix.get('os') if isinstance(matrix, dict) else None + if isinstance(axis_labels, list) and any('inputs.' in str(label) for label in axis_labels): + continue for step in setup_steps(job): text = str((step.get('with') or {}).get('python-version', '')).strip() read = READ.fullmatch(text)