diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8846cc9ad6..e6215b5b6d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -451,6 +451,10 @@ jobs: timeout-minutes: 30 env: ENGINE: ${{ matrix.engine }} + # Read by the apt plan step; see the note above it. + APT_PACKAGES: ${{ matrix.engine == 'PyShark' && 'tshark' || matrix.engine == 'PCAP_CT' && 'libpcap0.8' || matrix.engine == 'PyPCAP' && 'build-essential libpcap-dev' || '' }} + APT_SET: ${{ matrix.engine }} + APT_CACHE_SAVE: 'true' strategy: fail-fast: false matrix: @@ -497,37 +501,102 @@ jobs: # compile failure against the 3.12+ C API rather than merely "no # compiler was present to try". # - # `timeout-minutes` bounds this step because an apt stall in it used to + # `timeout-minutes` bounds the apt steps because an apt stall used to # eat the whole job: per #974, a PCAP_CT cell fetched its InRelease # metadata and then went silent for 29m52s until the job's own 30-minute # cap killed it, which reports `cancelled` and so fails `Required checks - # passed` -- an unrelated mirror turning a green PR red. Stalls in - # `apt-get update` have had the same all-or-nothing shape. + # passed` -- an unrelated mirror turning a green PR red. Deliberately + # *not* `-o Acquire::Retries=3 -o Acquire::http::Timeout=30`, which #974 + # proposed: both are noble's defaults already (apt 2.8.3: + # `Acquire::Retries` 3 in apt-pkg/acquire-item.cc, `TimeOut(30)` in + # methods/basehttp.cc), and that per-`select()` timeout never fires on a + # mirror that trickles. Hence `timeout(1)` around each apt command: 60s + # update, 180s install (#1306), two attempts, 480s inside the 10-minute + # cap, which stays the hang detector. `dpkg --configure -a` precedes the + # retry. # - # 10 minutes, with one retry inside it. These are whole-step wall times, - # both attempts included, so they bound no single attempt. On 2026-10-07 - # this step failed at 141s (job 112943258387, #1306, PyShark, - # 18:17:51Z-18:20:12Z) and the parity one at 155s (job 112948578234, - # #1307, 18:31:41Z-18:34:16Z, 27.9 MB of archives with - # `azure.archive.ubuntu.com` logging `Ign:` every 30s). Across the runs - # sampled, failures ran 91s to 194s, most at 187-194s: the band the old - # 45s/90s bounds cut off. So 60s update, 180s install (~27 MB at >=0.15 - # MB/s), two attempts, 480s worst case -- only 120s under the 600s step - # cap, and steps past 200s are on record, so the cap rather than that - # margin is the hang detector. `dpkg --configure -a` precedes the retry. + # #1432: a slow mirror, not a dead one, kept failing it. On 2026-10-08 + # `azure.archive.ubuntu.com` served ~75 kB/s (4.2 MB in 57s ahead of the + # 23.7 MB libwireshark17t64) while its indexes came at 4-10 MB/s, so + # `timeout 180` killed *both* attempts, three times (jobs 113323976331, + # 113470461084, 113485425256). tshark's 27.9 MB at that rate is ~380s, + # which no bound leaving two attempts inside the cap reaches, and the + # retry went back to the same mirror. So, around the unchanged install: # - # Deliberately *not* passing `-o Acquire::Retries=3 -o - # Acquire::http::Timeout=30`, which #974 proposed: both are already the - # defaults in noble's apt (2.8.3: `Acquire::Retries` 3 in - # apt-pkg/acquire-item.cc, `TimeOut(30)` in methods/basehttp.cc), so they - # would change nothing while reading like protection. Their being in - # force is also why the #974 stall ran to 29m52s: whatever hung is not - # something apt's per-`select()` data timeout governs, which is why the - # bound here is `timeout(1)` around the whole command. + # * **The .deb files are cached** (`actions/cache`, which the read-only + # token suffices for), so a warm run downloads nothing. The plan step + # runs `apt-get update`, then keys on a digest of the `Inst` lines of + # `apt-get install -s` -- every package and exact version this image + # still lacks -- plus ImageOS, arch and APT_SET. Any new version + # changes the key, so a stale set is never an exact hit. `restore-keys` + # falls back to the newest older set for the same image and APT_SET; + # apt uses a cached file only when its size and hash match the signed + # index and downloads the rest, and only the versions now installed are + # collected into the new entry, by one job per key (APT_CACHE_SAVE). + # ImageVersion is left out: the digest already moves when the image's + # installed set does, and the weekly image rollout would otherwise force + # a cold miss. + # * **On a miss, attempt 2 goes to another mirror.** apt fails over + # within /etc/apt/apt-mirrors.txt (the image's `mirror+file:` list: + # azure, then archive.ubuntu.com, then security.ubuntu.com) only on an + # error, never on slowness, so azure is dropped from it before the + # retry. apt keeps the killed attempt's partial file and resumes it + # with `Range`/`If-Range`. + # + # A cache only supplies bytes apt verifies, and the install runs in full + # on a hit, so a real install failure still fails; a cache-service error + # is only a warning from actions/cache. The 180s bound is unchanged: with + # the above it binds only on a cold miss, and TestAptRetry in + # tests/project/test_unit_tests_queue.py pins it. APT_PACKAGES (the job's + # env, read by the plan step) must name what the install step installs, + # or the key describes the wrong set -- a lost cache, not a wrong install. + # + # `pypcap-parity` runs the same steps; every body but the install's is + # byte-identical to these. + - name: Plan this engine's system packages + id: apt-plan + if: env.APT_PACKAGES != '' + timeout-minutes: 4 + run: | + set -eu + for attempt in 1 2; do + if sudo timeout 60 apt-get update; then + break + fi + echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound." + [ "$attempt" = 1 ] || exit 1 + done + # shellcheck disable=SC2086 + sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES) + plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true) + printf 'To download:\n%s\n' "${plan:-(nothing)}" + if [ -n "$plan" ]; then + prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-" + echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT" + echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" + fi + + - name: Restore this engine's cached .deb files + id: apt-cache + if: steps.apt-plan.outputs.key != '' + uses: actions/cache/restore@v6 + with: + path: ${{ runner.temp }}/apt-debs/*.deb + key: ${{ steps.apt-plan.outputs.key }} + restore-keys: ${{ steps.apt-plan.outputs.restore-prefix }} + + - name: Seed apt's archive with this engine's cached .deb files + if: steps.apt-cache.outputs.cache-matched-key != '' + run: | + set -eu + sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/ + echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files" + - name: Install this engine's system packages timeout-minutes: 10 run: | set -eu + mirrors=/etc/apt/apt-mirrors.txt apt_install() { for attempt in 1 2; do if sudo timeout 60 apt-get update \ @@ -536,6 +605,11 @@ jobs: fi echo "::warning title=apt attempt $attempt of 2 failed (${ENGINE})::apt-get update or install failed or exceeded its per-attempt bound; retrying once." sudo dpkg --configure -a || true + if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \ + && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \ + && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then + echo "::warning title=apt retry on a fallback mirror (${ENGINE})::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download." + fi done return 1 } @@ -548,6 +622,36 @@ jobs: apt_install build-essential libpcap-dev fi + - name: Collect this engine's installed .deb files for the cache + if: >- + env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' + && steps.apt-cache.outputs.cache-hit != 'true' + run: | + set -eu + out="$RUNNER_TEMP/apt-debs" + rm -rf "$out" + mkdir -p "$out" + for deb in /var/cache/apt/archives/*.deb; do + [ -e "$deb" ] || continue + pkg=$(dpkg-deb -f "$deb" Package) + ver=$(dpkg-deb -f "$deb" Version) + arch=$(dpkg-deb -f "$deb" Architecture) + # shellcheck disable=SC2016 + if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then + cp "$deb" "$out/" + fi + done + echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files" + + - name: Save this engine's .deb files to the cache + if: >- + env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' + && steps.apt-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ${{ runner.temp }}/apt-debs/*.deb + key: ${{ steps.apt-plan.outputs.key }} + # The per-cell script: one (Python version, engine) cell, start to # finish, in a venv of its own. Written to a file so each cell runs as a # separate `bash` process -- `set -e` is suspended inside a function @@ -752,6 +856,13 @@ jobs: needs: [changes] runs-on: ubuntu-latest timeout-minutes: 45 + env: + # Read by the apt plan step; see `engine-tests`'s note on it. + APT_PACKAGES: build-essential libpcap-dev tshark + APT_SET: pypcap-parity + # Both cells install the same set under the same key; only one saves it, + # or the other's save fails with `Unable to reserve cache`. + APT_CACHE_SAVE: ${{ matrix.python-version == '3.10' }} strategy: fail-fast: false matrix: @@ -785,17 +896,59 @@ jobs: # postinst otherwise blocks on the "allow non-superusers to capture # packets" prompt. # - # `timeout-minutes` and the retry for the reason given on `engine-tests`'s - # own install step -- see the note there, which this step shares - # wholesale, including the 10-minute cap (under this job's 45) and the - # 60s/180s bounds. It was not the leg that stalled in #974, but it has - # since failed the same way, at 155s, on 2026-10-07: #1307's job - # 112948578234, the second instance cited in that note. + # The apt steps -- plan, restore, seed, install, collect, save -- are + # `engine-tests`'s, for the reasons in the note above its plan step: + # bounds, retry, .deb cache and fallback mirror alike, with APT_SET + # keeping this job's cache entries apart. This job was not the leg that + # stalled in #974, but it has since failed the same way: at 155s on + # 2026-10-07 (#1307's job 112948578234), and on 2026-10-08 with + # `timeout 180` killing both attempts (#1432, jobs 113323976331 and + # 113485425256). + - name: Plan the system packages + id: apt-plan + if: env.APT_PACKAGES != '' + timeout-minutes: 4 + run: | + set -eu + for attempt in 1 2; do + if sudo timeout 60 apt-get update; then + break + fi + echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound." + [ "$attempt" = 1 ] || exit 1 + done + # shellcheck disable=SC2086 + sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES) + plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true) + printf 'To download:\n%s\n' "${plan:-(nothing)}" + if [ -n "$plan" ]; then + prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-" + echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT" + echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" + fi + + - name: Restore the cached .deb files + id: apt-cache + if: steps.apt-plan.outputs.key != '' + uses: actions/cache/restore@v6 + with: + path: ${{ runner.temp }}/apt-debs/*.deb + key: ${{ steps.apt-plan.outputs.key }} + restore-keys: ${{ steps.apt-plan.outputs.restore-prefix }} + + - name: Seed apt's archive with the cached .deb files + if: steps.apt-cache.outputs.cache-matched-key != '' + run: | + set -eu + sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/ + echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files" + - name: Install libpcap headers, a C toolchain, and tshark timeout-minutes: 10 run: | set -eu echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections + mirrors=/etc/apt/apt-mirrors.txt for attempt in 1 2; do if sudo timeout 60 apt-get update \ && sudo DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y --no-install-recommends build-essential libpcap-dev tshark; then @@ -803,9 +956,44 @@ jobs: fi echo "::warning title=apt attempt $attempt of 2 failed::apt-get update or install failed or exceeded its per-attempt bound; retrying once." sudo dpkg --configure -a || true + if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \ + && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \ + && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then + echo "::warning title=apt retry on a fallback mirror::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download." + fi done exit 1 + - name: Collect the installed .deb files for the cache + if: >- + env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' + && steps.apt-cache.outputs.cache-hit != 'true' + run: | + set -eu + out="$RUNNER_TEMP/apt-debs" + rm -rf "$out" + mkdir -p "$out" + for deb in /var/cache/apt/archives/*.deb; do + [ -e "$deb" ] || continue + pkg=$(dpkg-deb -f "$deb" Package) + ver=$(dpkg-deb -f "$deb" Version) + arch=$(dpkg-deb -f "$deb" Architecture) + # shellcheck disable=SC2016 + if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then + cp "$deb" "$out/" + fi + done + echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files" + + - name: Save the .deb files to the cache + if: >- + env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' + && steps.apt-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v6 + with: + path: ${{ runner.temp }}/apt-debs/*.deb + key: ${{ steps.apt-plan.outputs.key }} + - name: Install package, test and generator dependencies run: | python -m pip install -U pip setuptools wheel diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 8812340449..fbcdd0a65c 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -46,7 +46,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:362-376``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1214-1283``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1402-1471``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:392,493,573,674``), so a @@ -56,7 +56,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1074-1095``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1262-1283``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 5b546f18cf..b0932671b8 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -245,7 +245,7 @@ and ``project-tests``, and ``required-checks`` (see `Required Status Checks`_ below), gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1062-1068``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1250-1256``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -389,7 +389,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1371`` + - ``unit-tests.yml:1559`` * - ``Compat Python 3.10`` - job ``compatibility``, matrix leg ``3.10`` - ``python-compatibility.yml:31,41`` @@ -407,20 +407,20 @@ grepping every workflow file for its name: - ``python-compatibility.yml:31,45`` ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1371``). ``Compat Python`` is defined by two in +(``unit-tests.yml:1559``). ``Compat Python`` is defined by two in ``python-compatibility.yml``: the required ``compatibility`` job (``:31``, ``Compat Python ${{ matrix.python-version }}``, expanding to the five required legs at ``:41-45``) and the non-required ``compatibility-nightly`` job (``:69``, ``Compat Python 3.15 (scheduled)``, see below). A comment at -``unit-tests.yml:1286`` mentions the string without defining it. +``unit-tests.yml:1474`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the five ``Compat Python 3.10``-``3.14`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1285-1295``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1473-1483``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1370-1424``). Rulesets match check names literally, with no +(``unit-tests.yml:1558-1612``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the four legs as ``skipped`` only on a docs-only pull request, where @@ -429,7 +429,7 @@ instead. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1373``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1561``), so it is never produced -- and never expected -- on those paths. ``Compat Python 3.10``-``3.14`` are five ordinary matrix legs, not an diff --git a/tests/project/test_workflow_apt_cache.py b/tests/project/test_workflow_apt_cache.py new file mode 100644 index 0000000000..7f8d6a0778 --- /dev/null +++ b/tests/project/test_workflow_apt_cache.py @@ -0,0 +1,340 @@ +# -*- coding: utf-8 -*- +"""Tests for the apt ``.deb`` cache and fallback mirror in :file:`unit-tests.yml` (#1432). + +#1432: ``azure.archive.ubuntu.com`` served ~75 kB/s, so ``timeout 180`` killed +both tshark install attempts, three times in one day. The fix wraps the +unchanged install step (pinned by +:class:`tests.project.test_unit_tests_queue.TestAptRetry`) with a plan step that +computes a cache key, ``actions/cache`` restore/seed/collect/save steps, and a +retry that drops azure from the runner's ``mirror+file:`` list. + +Each property below is exercised by running the step's real ``run:`` script +under ``bash -e`` with ``sudo``/``timeout``/``apt-get`` stubbed, the way +``TestAptRetry`` does, or by reading the step text where only the Actions runner +could execute it: + +* the azure line is dropped before attempt 2 only, and only when another + ``http`` mirror would remain; +* each job's ``APT_PACKAGES`` (which the key is computed from) names exactly + what its install step installs, for every engine in the matrix; +* the key is ``apt-debs----`` of the sorted + ``Inst`` lines, and ``restore-keys`` is that same-image, same-set prefix only; +* only one job per key saves, so ``pypcap-parity``'s two cells do not race to + reserve the same entry. + +""" + +from __future__ import annotations + +import hashlib +import os +import pathlib +import re +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING + +from tests.project.test_unit_tests_queue import job, step_script + +if TYPE_CHECKING: + from typing import Optional + +#: The runner image's mirror list, as configure-apt-sources.sh writes it. +MIRRORS = ('http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\n' + 'https://archive.ubuntu.com/ubuntu/\tpriority:2\n' + 'https://security.ubuntu.com/ubuntu/\tpriority:3\n') +#: The path the install steps edit, replaced by a scratch file in these tests. +MIRRORS_PATH = '/etc/apt/apt-mirrors.txt' + +#: (job, plan step, install step) for both apt sites. +SITES = ( + ('engine-tests', "Plan this engine's system packages", "Install this engine's system packages"), + ('pypcap-parity', 'Plan the system packages', 'Install libpcap headers, a C toolchain, and tshark'), +) + +#: ``apt-get`` logs ``:`` per call, and an +#: install's operands; the first ``$FAIL_UPDATES`` updates fail; ``install -s`` +#: prints ``$STUB_SIM``. +STUBS = { + 'sudo': 'exec env "$@"\n', + 'timeout': 'shift\nexec "$@"\n', + 'apt-get': textwrap.dedent('''\ + azure=0 + if [ -f "${MIRRORS:-}" ]; then azure=$(grep -c azure "$MIRRORS" || true); fi + echo "$1:$azure" >> "$STUB_LOG.apt" + if [ "$1" = update ]; then + n=$(grep -c '^update' "$STUB_LOG.apt") + [ "$n" -gt "${FAIL_UPDATES:-0}" ] + elif [ "$1" = install ] && [ "$2" = -s ]; then + cat "$STUB_SIM" + elif [ "$1" = install ]; then + shift + for arg in "$@"; do + case "$arg" in -*) ;; *) printf '%s ' "$arg" >> "$STUB_LOG.pkgs" ;; esac + done + fi + '''), + 'dpkg': 'exit 0\n', + 'debconf-set-selections': 'cat > /dev/null\n', +} + + +class _Stubbed(unittest.TestCase): + """A scratch directory with the stubs first on ``PATH``.""" + + def setUp(self) -> None: + if shutil.which('bash') is None: # pragma: no cover + self.skipTest('bash is not available') + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1432-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.bin = self.tmp / 'bin' + self.bin.mkdir() + for command, body in STUBS.items(): + path = self.bin / command + path.write_text('#!/bin/bash\n' + body, encoding='utf-8') + path.chmod(0o755) + self.log = self.tmp / 'stub' + self.reset() + + def reset(self) -> None: + """Empty the stub logs, so each subtest reads only its own calls.""" + for suffix in ('.apt', '.pkgs'): + pathlib.Path(f'{self.log}{suffix}').write_text('', encoding='utf-8') + + def run_script(self, script: 'str', env: 'dict[str, str]') -> 'tuple[int, str]': + """Run ``script`` under ``bash -e``; return its status and combined output.""" + full = dict(os.environ, **env, PATH=f'{self.bin}{os.pathsep}{os.environ["PATH"]}', + STUB_LOG=str(self.log)) + proc = subprocess.run(['bash', '-e', '-c', script], env=full, check=False, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + return proc.returncode, proc.stdout + + def apt_calls(self) -> 'list[str]': + """Every stubbed ``apt-get`` call, as ``:``.""" + return pathlib.Path(f'{self.log}.apt').read_text(encoding='utf-8').split() + + def installed(self) -> 'list[str]': + """The operands of every stubbed ``apt-get install``.""" + return pathlib.Path(f'{self.log}.pkgs').read_text(encoding='utf-8').split() + + +class TestFallbackMirror(_Stubbed): + """#1432: azure is dropped before the retry, because apt never fails over on slowness.""" + + def run_install(self, job_name: 'str', name: 'str', mirrors: 'Optional[str]', + fail_updates: 'int') -> 'tuple[int, str, list[str], Optional[str]]': + """Run ``job_name``'s install step against a scratch mirror list.""" + script = step_script(job(job_name), name) + self.assertEqual(script.count(MIRRORS_PATH), 1, + f'{name!r} no longer names {MIRRORS_PATH} exactly once') + self.reset() + fake = self.tmp / 'apt-mirrors.txt' + if fake.exists(): + fake.unlink() + if mirrors is not None: + fake.write_text(mirrors, encoding='utf-8') + status, output = self.run_script( + script.replace(MIRRORS_PATH, str(fake)), + {'ENGINE': 'PyShark', 'MIRRORS': str(fake), 'FAIL_UPDATES': str(fail_updates)}) + after = fake.read_text(encoding='utf-8') if mirrors is not None else None + return status, output, self.apt_calls(), after + + def test_azure_is_dropped_before_attempt_two(self) -> None: + """Attempt 1 sees azure; attempt 2 does not; the other mirrors stay, in order.""" + for job_name, _, name in SITES: + with self.subTest(job=job_name): + status, output, calls, after = self.run_install(job_name, name, MIRRORS, 1) + self.assertEqual(status, 0, output) + self.assertEqual(calls, ['update:1', 'update:0', 'install:0']) + self.assertEqual(after, ''.join(MIRRORS.splitlines(True)[1:])) + self.assertIn('::warning title=apt retry on a fallback mirror', output) + self.assertIn('Retrying against https://archive.ubuntu.com/ubuntu/', output) + + def test_a_clean_first_attempt_leaves_the_list_alone(self) -> None: + """No failure, no edit.""" + for job_name, _, name in SITES: + with self.subTest(job=job_name): + status, output, calls, after = self.run_install(job_name, name, MIRRORS, 0) + self.assertEqual((status, calls, after), (0, ['update:1', 'install:1'], MIRRORS)) + self.assertNotIn('fallback mirror', output) + + def test_an_azure_only_list_is_left_alone(self) -> None: + """Dropping the only mirror would leave apt with none, so the retry keeps it.""" + only = MIRRORS.splitlines(True)[0] + for job_name, _, name in SITES: + with self.subTest(job=job_name): + status, output, calls, after = self.run_install(job_name, name, only, 1) + self.assertEqual((status, calls, after), (0, ['update:1', 'update:1', 'install:1'], only)) + self.assertNotIn('fallback mirror', output) + + def test_the_drop_happens_once_and_a_second_failure_still_fails(self) -> None: + """Two failures: one edit, before attempt 2 only, then a failed step.""" + for job_name, _, name in SITES: + with self.subTest(job=job_name): + status, output, calls, after = self.run_install(job_name, name, MIRRORS, 2) + self.assertNotEqual(status, 0) + self.assertEqual(calls, ['update:1', 'update:0']) + self.assertEqual(output.count('fallback mirror'), 1) + self.assertNotIn('azure', after or '') + + def test_a_missing_list_is_not_an_error(self) -> None: + """An image without the mirror file still gets its plain retry.""" + for job_name, _, name in SITES: + with self.subTest(job=job_name): + status, output, calls, _ = self.run_install(job_name, name, None, 1) + self.assertEqual((status, calls), (0, ['update:0', 'update:0', 'install:0']), output) + + +def matrix_engines() -> 'list[str]': + """``engine-tests``'s matrix, in order.""" + match = re.search(r'(?m)^ engine:\n((?: - \S+\n)+)', job('engine-tests')) + if match is None: + raise AssertionError('engine-tests has no engine matrix') + return re.findall(r'- (\S+)', match.group(1)) + + +def job_env(job_name: 'str', key: 'str') -> 'str': + """The raw value of ``key`` in ``job_name``'s job-level ``env:``.""" + match = re.search(rf'(?m)^ {re.escape(key)}: (.+)$', job(job_name)) + if match is None: + raise AssertionError(f'{job_name} sets no job-level {key}') + return match.group(1) + + +def engine_packages() -> 'dict[str, str]': + """``engine-tests``'s APT_PACKAGES, evaluated for every engine in its matrix.""" + expression = job_env('engine-tests', 'APT_PACKAGES') + if not re.fullmatch(r"\$\{\{ (?:matrix\.engine == '\w+' && '[^']*' \|\| )+'' \}\}", expression): + raise AssertionError(f'unrecognised APT_PACKAGES expression: {expression}') + named = dict(re.findall(r"matrix\.engine == '(\w+)' && '([^']*)'", expression)) + return {engine: named.get(engine, '') for engine in matrix_engines()} + + +class TestPackageListsAgree(_Stubbed): + """The key describes APT_PACKAGES, so it must be what the install step installs.""" + + def test_every_engine_named_in_the_expression_is_in_the_matrix(self) -> None: + """A renamed engine would silently fall through to ``''``.""" + expression = job_env('engine-tests', 'APT_PACKAGES') + named = set(re.findall(r"matrix\.engine == '(\w+)'", expression)) + self.assertTrue(named) + self.assertLessEqual(named, set(matrix_engines())) + + def test_engine_tests_installs_what_it_plans(self) -> None: + """Per engine, the install step's apt-get operands equal APT_PACKAGES.""" + script = step_script(job('engine-tests'), SITES[0][2]) + for engine, packages in engine_packages().items(): + with self.subTest(engine=engine): + self.reset() + status, output = self.run_script(script, {'ENGINE': engine}) + self.assertEqual(status, 0, output) + self.assertEqual(self.installed(), packages.split()) + + def test_pypcap_parity_installs_what_it_plans(self) -> None: + """The parity job's literal list equals its install step's operands.""" + packages = job_env('pypcap-parity', 'APT_PACKAGES') + status, output = self.run_script(step_script(job('pypcap-parity'), SITES[1][2]), {}) + self.assertEqual(status, 0, output) + self.assertEqual(self.installed(), packages.split()) + self.assertTrue(self.installed()) + + +#: ``apt-get install -s`` output: noise, ``Conf`` lines and unsorted ``Inst`` lines. +SIMULATION = ('NOTE: This is only a simulation!\n' + 'Inst tshark (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n' + 'Conf tshark (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n' + 'Inst libwireshark17t64 (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n') + + +class TestCacheKey(_Stubbed): + """The key moves with any version and nothing else; restore-keys stays on one image and set.""" + + def plan(self, job_name: 'str', name: 'str', simulation: 'str') -> 'dict[str, str]': + """Run the plan step against ``simulation``; return what it wrote to GITHUB_OUTPUT.""" + sim = self.tmp / 'sim' + sim.write_text(simulation, encoding='utf-8') + out = self.tmp / 'output' + out.write_text('', encoding='utf-8') + status, output = self.run_script(step_script(job(job_name), name), { + 'STUB_SIM': str(sim), 'GITHUB_OUTPUT': str(out), 'ImageOS': 'ubuntu24', + 'RUNNER_ARCH': 'X64', 'APT_SET': 'SomeSet', 'APT_PACKAGES': 'tshark'}) + self.assertEqual(status, 0, output) + return dict(line.split('=', 1) for line in out.read_text(encoding='utf-8').splitlines()) + + def test_the_key_format(self) -> None: + """``apt-debs----`` then 16 hex of the sorted Inst lines.""" + inst = sorted(line for line in SIMULATION.splitlines() if line.startswith('Inst ')) + digest = hashlib.sha256(('\n'.join(inst) + '\n').encode()).hexdigest()[:16] + for job_name, name, _ in SITES: + with self.subTest(job=job_name): + outputs = self.plan(job_name, name, SIMULATION) + self.assertEqual(outputs, {'restore-prefix': 'apt-debs-ubuntu24-X64-SomeSet-', + 'key': f'apt-debs-ubuntu24-X64-SomeSet-{digest}'}) + + def test_order_does_not_move_the_key_and_a_version_does(self) -> None: + """Sorted, so apt's ordering is irrelevant; any version change is a new key.""" + lines = SIMULATION.splitlines(True) + for job_name, name, _ in SITES: + with self.subTest(job=job_name): + key = self.plan(job_name, name, SIMULATION)['key'] + self.assertEqual(self.plan(job_name, name, ''.join(reversed(lines)))['key'], key) + bumped = SIMULATION.replace('(4.2.2-1.1build3 Ubuntu', '(4.2.2-1.1build4 Ubuntu', 1) + self.assertNotEqual(self.plan(job_name, name, bumped)['key'], key) + + def test_nothing_to_download_means_no_key(self) -> None: + """No Inst lines, no cache steps: nothing is written for them to read.""" + for job_name, name, _ in SITES: + with self.subTest(job=job_name): + self.assertEqual(self.plan(job_name, name, 'NOTE: This is only a simulation!\n'), {}) + + def test_restore_and_save_use_the_plan_and_the_same_path(self) -> None: + """restore-keys is the plan's prefix only; restore and save share key and path.""" + for job_name, _, _ in SITES: + with self.subTest(job=job_name): + text = job(job_name) + restore = re.search(r'(?ms)uses: actions/cache/restore@\S+\n with:\n(.*?)\n\n', text) + save = re.search(r'(?ms)uses: actions/cache/save@\S+\n with:\n(.*?)\n\n', text) + self.assertIsNotNone(restore) + self.assertIsNotNone(save) + restore_with = dict(re.findall(r'(?m)^ ([\w-]+): (.+)$', restore.group(1))) # type: ignore[union-attr] + save_with = dict(re.findall(r'(?m)^ ([\w-]+): (.+)$', save.group(1))) # type: ignore[union-attr] + self.assertEqual(restore_with, { + 'path': '${{ runner.temp }}/apt-debs/*.deb', + 'key': '${{ steps.apt-plan.outputs.key }}', + 'restore-keys': '${{ steps.apt-plan.outputs.restore-prefix }}'}) + self.assertEqual(save_with, {'path': restore_with['path'], 'key': restore_with['key']}) + + +class TestOneSaverPerKey(unittest.TestCase): + """Two jobs saving one key race, and the loser logs ``Unable to reserve cache``.""" + + def test_only_one_parity_cell_saves(self) -> None: + """APT_CACHE_SAVE selects exactly one of the parity matrix's Python versions.""" + value = job_env('pypcap-parity', 'APT_CACHE_SAVE') + match = re.fullmatch(r"\$\{\{ matrix\.python-version == '([\d.]+)' \}\}", value) + self.assertIsNotNone(match, value) + versions = re.findall(r'(?m)^ - "([\d.]+)"$', job('pypcap-parity')) + self.assertIn(match.group(1), versions) # type: ignore[union-attr] + self.assertGreater(len(versions), 1) + + def test_each_engine_job_saves_its_own_key(self) -> None: + """engine-tests has one job per APT_SET, so each saves.""" + self.assertEqual(job_env('engine-tests', 'APT_CACHE_SAVE'), "'true'") + + def test_collect_and_save_are_gated_on_it(self) -> None: + """Both steps that only matter for a save check the flag.""" + for job_name, _, _ in SITES: + with self.subTest(job=job_name): + steps = re.split(r'(?m)^ - ', job(job_name)) + for marker in ('name: Collect ', 'uses: actions/cache/save@'): + blocks = [block for block in steps if marker in block] + self.assertEqual(len(blocks), 1, marker) + self.assertIn("env.APT_CACHE_SAVE == 'true'", blocks[0]) + + +if __name__ == '__main__': + unittest.main()