From 34a58cf1d515cff21f0b6dd63e778d6147b63dc1 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Fri, 23 Oct 2026 11:01:34 -0400 Subject: [PATCH] ci(release): move the conda legs' shell into util/ scripts (#1967) --- .github/workflows/create-release.yml | 157 +---- .github/workflows/cron-conda.yml | 92 +-- docs/source/contributing/workflows.rst | 25 +- tests/project/test_actions_helper_2047.py | 7 +- ...onda_completeness_counts_every_leg_1922.py | 24 +- .../test_conda_intel_cli_upload_2044.py | 18 +- .../test_conda_native_platforms_1822.py | 33 +- .../test_conda_no_windows_steps_1967.py | 41 +- tests/project/test_conda_windows_leg_2060.py | 6 +- .../project/test_ktlo_legs_stay_green_1672.py | 42 +- .../test_release_conda_leg_smoke_1855.py | 24 +- .../test_release_conda_scripts_1967.py | 569 ++++++++++++++++++ tests/project/test_release_gates.py | 50 +- util/release_conda.py | 232 +++++++ 14 files changed, 985 insertions(+), 335 deletions(-) create mode 100644 tests/project/test_release_conda_scripts_1967.py create mode 100644 util/release_conda.py diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 65e3e5fe2b..bc570278eb 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -761,16 +761,7 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" + run: /usr/bin/python3 util/leg_summary.py ktlo conda: name: Conda deployment (release) on ${{ matrix.os }} with Python ${{ matrix.python-version }} @@ -825,60 +816,17 @@ jobs: run: python3 util/release_tags.py check # #888: the per-leg correctness fix. Uncached, per ``version_check``'s - # Anaconda check, so this reads the live state. + # Anaconda check, so this reads the live state. #1967: util/release_conda.py + # holds this step's shell, and the build's and the Intel upload's, which + # cron-conda.yml's ``conda-dist`` runs too; before setup-python, so here + # on the image's ``python3``. - name: Check whether this leg is already on Anaconda id: check_leg env: PCAPKIT_VERSION: ${{ needs.version_check.outputs.PCAPKIT_VERSION }} - run: | - set -euo pipefail - - case "${{ matrix.os }}" in - macos-latest) - target_platform="osx-arm64" - ;; - macos-26-intel) - target_platform="osx-64" - ;; - ubuntu-latest) - target_platform="linux-64" - ;; - *) - target_platform="" - ;; - esac - py_tag="py$(echo "${{ matrix.python-version }}" | tr -d '.')" - - # Same coupling risk, and the same fix, as version_check's checks. - status="$(curl -s --retry 3 --retry-connrefused \ - -o /tmp/anaconda_release.json -w '%{http_code}' \ - "https://api.anaconda.org/release/jarryshaw/pypcapkit/${PCAPKIT_VERSION}" || echo 000)" - - exists=false - if [ "$status" = "200" ]; then - # The `printf` form of version_check's checks, and needed here: this - # runs before setup-python, so `python3` is the image's (3.12 on - # ubuntu-latest, 3.13 on macos-latest), and indented `python3 -c` - # source would fail under `set -e` -- only on the retry this is for. - exists="$(printf '%s\n' \ - "import json" \ - "with open('/tmp/anaconda_release.json') as f:" \ - " data = json.load(f)" \ - "target_platform = '$target_platform'" \ - "py_tag = '$py_tag'" \ - "found = any(" \ - " d.get('attrs', {}).get('subdir') == target_platform" \ - " and d.get('attrs', {}).get('build', '').startswith(py_tag + '_')" \ - " for d in data.get('distributions', [])" \ - ")" \ - "print('true' if found else 'false')" \ - | python3 -)" - fi - - echo "PCAPKIT_LEG_EXISTS=$exists" >> "$GITHUB_OUTPUT" - if [ "$exists" = "true" ]; then - echo "::notice title=conda leg already published::pypcapkit ${PCAPKIT_VERSION} for ${target_platform} / ${py_tag} is already on anaconda.org/jarryshaw (http ${status}); skipping this leg's upload." - fi + PCAPKIT_CONDA_OS: ${{ matrix.os }} + PCAPKIT_PYTHON_VERSION: ${{ matrix.python-version }} + run: python3 util/release_conda.py leg /tmp/anaconda_release.json - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: @@ -935,6 +883,8 @@ jobs: conda install -n base --override-channels -c conda-forge -y conda-build conda build --version + # On conda's base ``python``, which the login shell activates, from this + # directory, as conda-dist's step of the same name. - name: Build conda packages id: conda_build shell: bash -el {0} @@ -944,58 +894,10 @@ jobs: # because ``conda/build`` says so: at a hand-tagged commit it may still # hold the previous version's counter until ``tag``'s reset (#1634). PCAPKIT_BUILD: "0" + PCAPKIT_CONDA_OS: ${{ matrix.os }} + PCAPKIT_PYTHON_VERSION: ${{ matrix.python-version }} working-directory: conda/pypcapkit - run: | - set -euo pipefail - - out_dir="$(mktemp -d -t compilation-XXXXXXXXXX)" - echo "out_dir=$out_dir" >> "$GITHUB_OUTPUT" - - conda build . --package-format .tar.bz2 --no-anaconda-upload --output-folder "$out_dir" --python "${{ matrix.python-version }}" - host_package="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort | sed -n '1p')" - if [ -z "$host_package" ]; then - echo "::error::No conda package was built" - exit 1 - fi - - case "${{ matrix.os }}" in - macos-latest) - target_platform="osx-arm64" - ;; - macos-26-intel) - target_platform="osx-64" - ;; - ubuntu-latest) - target_platform="linux-64" - ;; - *) - target_platform="" - ;; - esac - - echo "host_package=$host_package" >> "$GITHUB_OUTPUT" - - # #1822: never ``conda convert`` across architectures; see the matrix. - host_platform="$(basename "$(dirname "$host_package")")" - if [ "$host_platform" != "$target_platform" ]; then - echo "::error title=${host_platform} package on a ${target_platform:-unmapped} leg (#1822)::${{ matrix.os }} publishes ${target_platform:-nothing}, and conda built ${host_platform} here; a package is published only for the platform it was built on." - exit 1 - fi - - package_paths="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort)" - if [ -z "$package_paths" ]; then - echo "::error::No conda packages were found" - exit 1 - fi - - printf '%s\n' "$package_paths" - package_paths_csv="$(printf '%s\n' "$package_paths" | paste -sd ',' -)" - echo "packages_csv=$package_paths_csv" >> "$GITHUB_OUTPUT" - { - echo 'packages<> "$GITHUB_OUTPUT" + run: python ../../util/release_conda.py build # #1855: cron-conda.yml's ``conda-dist`` step of the same name, which says # what it checks and why it runs setup-python's interpreter, so a release @@ -1025,7 +927,8 @@ jobs: # parser -- from conda-forge's anaconda-client in base. One difference: # `--skip-existing`, so a file anaconda.org already has is skipped where # the action, with no `force:`, fails on it. "Check whether this leg is - # already on Anaconda" still gates both. + # already on Anaconda" still gates both. ``release_conda.py upload`` runs + # on setup-python's ``python3``. - name: Upload conda packages to Anaconda (Intel) if: ${{ matrix.os == 'macos-26-intel' && steps.check_leg.outputs.PCAPKIT_LEG_EXISTS != 'true' }} shell: bash @@ -1035,35 +938,7 @@ jobs: PCAPKIT_CONDA_CHANNEL: jarryshaw PCAPKIT_CONDA_PACKAGES: ${{ steps.conda_build.outputs.packages }} PCAPKIT_CONDA_LABELS: ${{ needs.version_check.outputs.PCAPKIT_CONDA_LABEL }} - run: | - set -euo pipefail - - conda install -n base --override-channels -c conda-forge -y anaconda-client - - # The action's handling of its `labels:` and `packages:` inputs: a - # comma-separated list, each trimmed, and the build step's - # newline-separated paths. - labels=() - IFS=',' read -ra names <<< "$PCAPKIT_CONDA_LABELS" - for name in ${names[@]+"${names[@]}"}; do - name="$(echo "$name" | xargs)" - if [ -n "$name" ]; then - labels+=(--label "$name") - fi - done - packages=() - while IFS= read -r path; do - if [ -n "$path" ]; then - packages+=("$path") - fi - done <<< "$PCAPKIT_CONDA_PACKAGES" - if [ "${#packages[@]}" -eq 0 ]; then - echo "::error::No conda packages to upload" - exit 1 - fi - - "${CONDA}/bin/anaconda" --token "$ANACONDA_API_TOKEN" upload \ - --user "$PCAPKIT_CONDA_CHANNEL" --skip-existing ${labels[@]+"${labels[@]}"} "${packages[@]}" + run: python3 util/release_conda.py upload # ``pypi``'s upload script, for the same reasons (#1634, #1640). Not # gated on ``check_leg`` as the Anaconda upload is: a leg whose Anaconda diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index 690d235975..60ebd28259 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -501,6 +501,9 @@ jobs: conda install -n base --override-channels -c conda-forge -y conda-build conda build --version + # #1967: util/release_conda.py, as create-release.yml's step of the same + # name, on conda's base `python`, which the login shell activates. #1822: + # it fails a package built for another platform than the leg's. - name: Build conda packages id: conda_build continue-on-error: ${{ matrix.os == 'windows-latest' }} @@ -508,63 +511,10 @@ jobs: env: PCAPKIT_VERSION: ${{ needs.conda-update.outputs.PCAPKIT_VERSION }} PCAPKIT_BUILD: ${{ needs.conda-update.outputs.PCAPKIT_BUILD }} + PCAPKIT_CONDA_OS: ${{ matrix.os }} + PCAPKIT_PYTHON_VERSION: ${{ matrix.python-version }} working-directory: conda/pypcapkit - run: | - set -euo pipefail - - out_dir="$(mktemp -d -t compilation-XXXXXXXXXX)" - echo "out_dir=$out_dir" >> "$GITHUB_OUTPUT" - - conda build . --package-format .tar.bz2 --no-anaconda-upload --output-folder "$out_dir" --python "${{ matrix.python-version }}" - host_package="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort | sed -n '1p')" - if [ -z "$host_package" ]; then - echo "::error::No conda package was built" - exit 1 - fi - - case "${{ matrix.os }}" in - macos-latest) - target_platform="osx-arm64" - ;; - macos-26-intel) - target_platform="osx-64" - ;; - ubuntu-latest) - target_platform="linux-64" - ;; - windows-latest) - target_platform="win-64" - ;; - *) - target_platform="" - ;; - esac - - echo "host_package=$host_package" >> "$GITHUB_OUTPUT" - - # #1822: never `conda convert` across architectures. It relabels the - # package and copies its binaries unchanged, so the osx-64 package it - # made from the arm64 build carried arm64 extensions. - host_platform="$(basename "$(dirname "$host_package")")" - if [ "$host_platform" != "$target_platform" ]; then - echo "::error title=${host_platform} package on a ${target_platform:-unmapped} leg (#1822)::${{ matrix.os }} publishes ${target_platform:-nothing}, and conda built ${host_platform} here; a package is published only for the platform it was built on." - exit 1 - fi - - package_paths="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort)" - if [ -z "$package_paths" ]; then - echo "::error::No conda packages were found" - exit 1 - fi - - printf '%s\n' "$package_paths" - package_paths_csv="$(printf '%s\n' "$package_paths" | paste -sd ',' -)" - echo "packages_csv=$package_paths_csv" >> "$GITHUB_OUTPUT" - { - echo 'packages<> "$GITHUB_OUTPUT" + run: python ../../util/release_conda.py build # #1814: the package's only run requirement is Python, so it has only what # it vendors in `pcapkit/_extern`, and package discovery alone ships those @@ -614,35 +564,7 @@ jobs: PCAPKIT_CONDA_CHANNEL: jarryshaw PCAPKIT_CONDA_PACKAGES: ${{ steps.conda_build.outputs.packages }} PCAPKIT_CONDA_LABELS: ${{ needs.conda-update.outputs.PCAPKIT_CONDA_LABEL }} - run: | - set -euo pipefail - - conda install -n base --override-channels -c conda-forge -y anaconda-client - - # The action's handling of its `labels:` and `packages:` inputs: a - # comma-separated list, each trimmed, and the build step's - # newline-separated paths. - labels=() - IFS=',' read -ra names <<< "$PCAPKIT_CONDA_LABELS" - for name in ${names[@]+"${names[@]}"}; do - name="$(echo "$name" | xargs)" - if [ -n "$name" ]; then - labels+=(--label "$name") - fi - done - packages=() - while IFS= read -r path; do - if [ -n "$path" ]; then - packages+=("$path") - fi - done <<< "$PCAPKIT_CONDA_PACKAGES" - if [ "${#packages[@]}" -eq 0 ]; then - echo "::error::No conda packages to upload" - exit 1 - fi - - "${CONDA}/bin/anaconda" --token "$ANACONDA_API_TOKEN" upload \ - --user "$PCAPKIT_CONDA_CHANNEL" --skip-existing ${labels[@]+"${labels[@]}"} "${packages[@]}" + run: python3 util/release_conda.py upload # #1634: the upload below never makes the release. ncipollo/release-action # v1.21.0 creates a missing one, and its tag, at ``commit`` diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 1ab7443d5a..a44622333b 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -1222,9 +1222,10 @@ does; each has tests under :file:`tests/project/` that run it. - By hand * - :file:`util/leg_summary.py` - ``KTLO summary`` in ``unit-tests.yml``'s ``test``, ``integration`` and - ``pypcap-parity`` (and so their KTLO twins) and ``python-compatibility.yml``'s - ``compatibility``, as ``leg_summary.py ktlo``; ``Experimental summary`` in - ``experimental.yml``'s ``experimental``, as ``leg_summary.py experimental``. + ``pypcap-parity`` (and so their KTLO twins), ``python-compatibility.yml``'s + ``compatibility`` and ``create-release.yml``'s ``pypi``, as ``leg_summary.py + ktlo``; ``Experimental summary`` in ``experimental.yml``'s ``experimental``, + as ``leg_summary.py experimental``. - No. * - :file:`util/classify_changes.py` - ``Classify the diff`` in the ``changes`` job of ``unit-tests.yml`` and of @@ -1313,6 +1314,14 @@ does; each has tests under :file:`tests/project/` that run it. ``release_published.py pypi`` and ``anaconda``, on setup-python's ``python3`` (`#888 `__). - No. + * - :file:`util/release_conda.py` + - ``Check whether this leg is already on Anaconda`` in ``create-release.yml``'s + ``conda``, as ``release_conda.py leg``; ``Build conda packages`` and ``Upload + conda packages to Anaconda (Intel)`` in that job and in ``cron-conda.yml``'s + ``conda-dist``, as ``release_conda.py build`` and ``upload`` (`#1822 + `__, `#2044 + `__). + - No. The summaries run on the image's ``/usr/bin/python3`` rather than the leg's interpreter, which may have failed to set up or be too broken to run anything, so @@ -1333,13 +1342,13 @@ uutils' ``timeout`` behaves as GNU's (`#2020 `__), and the bound reaches apt's own children, which a timeout in the script, running as the runner's user, could not signal. Both ``changes`` jobs check out on every event, since the script is in the -tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of -the KTLO summary's shell until its shell moves to scripts too; a project pin runs -that copy and the script on the same steps and requires the same output. -The release scripts run as ``python3``, from a checkout of the whole tree, and are +tree. The release scripts run as ``python3``, from a checkout of the whole tree, and are standard library only: :file:`util/release_tags.py` on the image's own, since ``github`` has no setup-python and ``pypi`` and ``conda`` run the check -before theirs, and :file:`util/release_upload.py` on the leg's, after it. A project +before theirs, and :file:`util/release_upload.py` on the leg's, after it. +:file:`util/release_conda.py` runs ``leg`` on the image's ``python3``, before +setup-python, ``build`` on conda's base ``python`` under the login shell, from +:file:`conda/pypcapkit`, and ``upload`` on setup-python's. A project pin runs each on the cases the shell it replaced handled and requires the same status, output and calls. :file:`util/_actions.py` is standard library only, and keeps to 3.7's syntax, diff --git a/tests/project/test_actions_helper_2047.py b/tests/project/test_actions_helper_2047.py index 17038ca93d..95ac60ec59 100644 --- a/tests/project/test_actions_helper_2047.py +++ b/tests/project/test_actions_helper_2047.py @@ -45,9 +45,9 @@ #: The scripts that import the helper. SCRIPTS = ('apt_packages', 'check_suite_report', 'ci_timing', 'classify_changes', 'codeql_report', 'coverage_report', 'discussion_report', 'leg_summary', 'pip_install', 'project_status', - 'python_versions', 'regenerate_samples', 'release_published', 'release_status', 'release_tags', - 'release_tiers', 'release_upload', 'release_vendor_update', 'superseded', 'update_python_versions', - 'vendor_code_changed', 'vendor_test_gate', 'verdict_covers_checkout') + 'python_versions', 'regenerate_samples', 'release_conda', 'release_published', 'release_status', + 'release_tags', 'release_tiers', 'release_upload', 'release_vendor_update', 'superseded', + 'update_python_versions', 'vendor_code_changed', 'vendor_test_gate', 'verdict_covers_checkout') #: The top-level modules the helper may import, each in Python 3.7's standard #: library: the running interpreter's :data:`sys.stdlib_module_names` would #: pass ``tomllib``, which 3.7 lacks. @@ -66,6 +66,7 @@ 'project_status': (['--help'], {}), 'python_versions': (['latest'], {}), 'regenerate_samples': (['--help'], {}), + 'release_conda': (['--help'], {}), 'release_published': (['--help'], {}), 'release_status': (['--help'], {}), 'release_tags': (['--help'], {}), diff --git a/tests/project/test_conda_completeness_counts_every_leg_1922.py b/tests/project/test_conda_completeness_counts_every_leg_1922.py index ee69e32f61..fff1ed3bba 100644 --- a/tests/project/test_conda_completeness_counts_every_leg_1922.py +++ b/tests/project/test_conda_completeness_counts_every_leg_1922.py @@ -3,7 +3,8 @@ * The platforms it counts, :file:`util/release_published.py`'s ``SUBDIRS`` since #1967, are the ones the ``conda`` job's ``os`` legs map to, read from - that job's matrix and its ``case "${{ matrix.os }}"`` arms. + that job's matrix and :file:`util/release_conda.py`'s ``PLATFORMS``, which its + steps run. * Run as the workflow runs it, with ``curl`` stubbed, a release reads complete only with each of those platforms for each build, so a missing ``osx-arm64`` upload reads incomplete and the ``conda`` gate runs the leg that lacks it. @@ -15,7 +16,6 @@ import json import os import pathlib -import re import shutil import subprocess # nosec: B404 import sys @@ -24,7 +24,7 @@ from typing import TYPE_CHECKING from tests import _workflows -from util import release_published +from util import release_conda, release_published try: import yaml @@ -39,10 +39,6 @@ STEP = 'Check Anaconda publish completeness' BUILDS = ('py313', 'py314') -#: A ``case "${{ matrix.os }}" in`` arm setting ``target_platform``. -ARM = re.compile(r'^\s*([\w.-]+)\)\s*\n\s*target_platform="([^"]*)"', re.MULTILINE) - - def jobs() -> 'dict[str, Any]': if yaml is None: # pragma: no cover raise unittest.SkipTest('PyYAML, from the test extra, is not installed') @@ -54,17 +50,11 @@ def check() -> 'dict[str, Any]': def published() -> 'set[str]': - """The platform each ``conda`` leg publishes, from its matrix and each ``case`` of its steps.""" + """The platform each ``conda`` leg publishes, from its matrix and the script its steps run (#1967).""" spec = jobs()['conda'] - labels = spec['strategy']['matrix']['os'] - found = set() - for step in spec['steps']: - for block in re.findall(r'case "\$\{\{ matrix\.os \}\}" in\n(.*?)\n\s*esac', str(step.get('run', '')), - re.DOTALL): - arms = dict(ARM.findall(block)) - found.add(frozenset(arms[label] for label in labels)) - assert len(found) == 1, found - return set(found.pop()) + runs = [str(step.get('run', '')) for step in spec['steps'] if 'util/release_conda.py' in str(step.get('run', ''))] + assert len(runs) == 3, runs # the leg check, the build and the Intel upload + return {release_conda.PLATFORMS[label] for label in spec['strategy']['matrix']['os']} class TheCheckCountsTheLegs(unittest.TestCase): diff --git a/tests/project/test_conda_intel_cli_upload_2044.py b/tests/project/test_conda_intel_cli_upload_2044.py index a20b72c991..6b3ccab861 100644 --- a/tests/project/test_conda_intel_cli_upload_2044.py +++ b/tests/project/test_conda_intel_cli_upload_2044.py @@ -12,9 +12,10 @@ Each step's ``if:`` is evaluated per matrix leg with :func:`tests.project.test_release_gates.evaluate_condition`. The CLI step's -script is run with a stand-in ``conda`` and ``anaconda``, and the arguments it -passes are compared with those :func:`action_arguments` builds as the action's -"Upload to anaconda.org" step does. +script, :file:`util/release_conda.py` ``upload`` since #1967, is run with a +stand-in ``conda`` and ``anaconda``, and the arguments it passes are compared +with those :func:`action_arguments` builds as the action's "Upload to +anaconda.org" step does. """ @@ -24,6 +25,7 @@ import pathlib import stat import subprocess # nosec: B404 +import sys import tempfile import textwrap import unittest @@ -48,6 +50,9 @@ #: The leg whose upload-package CLI download returns 404. INTEL = 'macos-26-intel' SECRET = '${{ secrets.ANACONDA_TOKEN }}' +#: The CLI step's ``run:`` (#1967), and the script it runs. +CLI_RUN = 'python3 util/release_conda.py upload' +SCRIPT = pathlib.Path(__file__).resolve().parents[2] / 'util' / 'release_conda.py' #: The CLI step's ``env:`` key for each action input it carries. CARRIED = {'token': 'ANACONDA_API_TOKEN', 'channel': 'PCAPKIT_CONDA_CHANNEL', 'packages': 'PCAPKIT_CONDA_PACKAGES', 'labels': 'PCAPKIT_CONDA_LABELS'} @@ -151,8 +156,8 @@ def test_the_token_comes_through_env(self) -> None: entry = step(name, job_id, CLI_STEP) with self.subTest(workflow=name, job=job_id): self.assertEqual(entry['env']['ANACONDA_API_TOKEN'], SECRET) - self.assertNotIn('${{', entry['run']) - self.assertIn('--token "$ANACONDA_API_TOKEN"', entry['run']) + self.assertEqual(entry['run'], CLI_RUN) + self.assertIn("'--token', env('ANACONDA_API_TOKEN')", SCRIPT.read_text(encoding='utf-8')) def test_it_carries_the_actions_inputs(self) -> None: for name, job_id in JOBS: @@ -207,7 +212,8 @@ def run_script(self, entry: 'dict[str, Any]', packages: 'str', labels: 'str', CONDA=str(scratch / 'base'), FAKE_LOG=str(log), ANACONDA_API_TOKEN='TOKEN', ANACONDA_CLI_FORCE_NEW='1', PCAPKIT_CONDA_CHANNEL=str(entry['env']['PCAPKIT_CONDA_CHANNEL']), PCAPKIT_CONDA_PACKAGES=packages, PCAPKIT_CONDA_LABELS=labels) - result = subprocess.run([str(BASH), '-e', '-c', entry['run']], cwd=temp, env=env, # nosec: B603 + self.assertEqual(entry['run'], CLI_RUN) + result = subprocess.run([sys.executable, str(SCRIPT), 'upload'], cwd=temp, env=env, # nosec: B603 capture_output=True, text=True, check=False, timeout=scale_timeout(60)) self.assertEqual(result.returncode, status, result.stdout + result.stderr) blocks = log.read_text(encoding='utf-8').split('\n\n') diff --git a/tests/project/test_conda_native_platforms_1822.py b/tests/project/test_conda_native_platforms_1822.py index 232557e5c0..ee30d78c4e 100644 --- a/tests/project/test_conda_native_platforms_1822.py +++ b/tests/project/test_conda_native_platforms_1822.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- """Each conda package carries binaries for its own platform (#1822). -* Both conda jobs map each runner label of their ``os`` axis to the conda +* Both conda jobs map each runner label of their ``os`` axis, through + :file:`util/release_conda.py`'s ``PLATFORMS`` since #1967, to the conda platform of that runner's own architecture, as GitHub's runner-images table gives it (the snapshot in :file:`python_versions_fixtures`), and between them build exactly :data:`conda_vendor.PLATFORMS`; win-64 only in Conda Update, @@ -36,6 +37,8 @@ from unittest import mock from tests import _workflows +from tests.project.test_release_conda_scripts_1967 import read_outputs +from util import release_conda try: import tomllib @@ -94,18 +97,10 @@ def job(name: 'str', job_id: 'str') -> 'dict[str, Any]': return _workflows.job(WORKFLOWS / name, job_id) -#: A ``case "${{ matrix.os }}" in`` arm setting ``target_platform``. -ARM = re.compile(r'^\s*([\w.-]+)\)\s*\n\s*target_platform="([^"]*)"', re.MULTILINE) - - def mappings(spec: 'dict[str, Any]') -> 'list[dict[str, str]]': - """Each ``case "${{ matrix.os }}"`` of the job's steps, as label to platform.""" - found = [] - for step in spec.get('steps', []): - for block in re.findall(r'case "\$\{\{ matrix\.os \}\}" in\n(.*?)\n\s*esac', str(step.get('run', '')), - re.DOTALL): - found.append(dict(ARM.findall(block))) - return found + """The label-to-platform map each of the job's steps that maps its leg reads (#1967).""" + return [dict(release_conda.PLATFORMS) for step in spec.get('steps', []) + if re.search(r'\butil/release_conda\.py (?:leg|build)\b', str(step.get('run', '')))] class RunnerMappingTests(unittest.TestCase): @@ -173,8 +168,8 @@ def test_no_workflow_runs_conda_convert(self) -> None: def build(self, name: 'str', job_id: 'str', label: 'str', built: 'str') -> 'tuple[int, str, list[str]]': """The job's build step on ``label`` where conda builds ``built``: status, log, packages.""" step = next(step for step in job(name, job_id)['steps'] if step.get('id') == 'conda_build') - script = step['run'].replace('${{ matrix.os }}', label).replace('${{ matrix.python-version }}', '3.14') - self.assertNotIn('${{', script) + self.assertEqual(step['run'], 'python ../../util/release_conda.py build') + self.assertEqual(step['env']['PCAPKIT_CONDA_OS'], '${{ matrix.os }}') with tempfile.TemporaryDirectory() as temp: scratch = pathlib.Path(temp) (scratch / 'bin').mkdir() @@ -184,14 +179,14 @@ def build(self, name: 'str', job_id: 'str', label: 'str', built: 'str') -> 'tupl output = scratch / 'output' output.write_text('', encoding='utf-8') result = subprocess.run( - [BASH, '-e', '-c', script], cwd=temp, # nosec: B603 + [sys.executable, str(ROOT / 'util' / 'release_conda.py'), 'build'], cwd=temp, # nosec: B603 env=dict(os.environ, PATH=f'{scratch / "bin"}{os.pathsep}{os.environ["PATH"]}', TMPDIR=temp, GITHUB_OUTPUT=str(output), FAKE_SUBDIR=built, - PCAPKIT_VERSION='1.0', PCAPKIT_BUILD='0'), + PCAPKIT_VERSION='1.0', PCAPKIT_BUILD='0', PCAPKIT_CONDA_OS=label, + PCAPKIT_PYTHON_VERSION='3.14'), capture_output=True, text=True, check=False, timeout=60) - written = output.read_text(encoding='utf-8') - packages = re.search(r'^packages< None: diff --git a/tests/project/test_conda_no_windows_steps_1967.py b/tests/project/test_conda_no_windows_steps_1967.py index 23dc16fa0d..ee8f2a1102 100644 --- a/tests/project/test_conda_no_windows_steps_1967.py +++ b/tests/project/test_conda_no_windows_steps_1967.py @@ -9,7 +9,9 @@ * :class:`TestNoDeadRunnerCode` pins that no step's ``if:`` or ``continue-on-error`` names a runner label the job's matrix lacks, no ``case`` - arm maps one, and, with no Windows leg, no step is Windows-only. So adding a + arm maps one, :file:`util/release_conda.py`, which both jobs' steps run since + #1967, maps no label that neither job has a leg on, and, with no Windows leg, + no step is Windows-only. So adding a Windows leg back means adding its steps back deliberately, rather than reviving ones nobody has run. #2060 did so for ``conda-dist``, whose Windows steps are therefore live; ``conda`` still has no Windows leg. @@ -28,9 +30,10 @@ from typing import TYPE_CHECKING from tests import _workflows +from util import release_conda if TYPE_CHECKING: - from typing import Any + from typing import Any, Optional #: Test tier (#1958): reads the workflow files, a question about the tree. __tier__ = 'LATEST' @@ -47,11 +50,24 @@ WINDOWS = re.compile(r'windows', re.IGNORECASE) #: An ``if:`` that runs its step on Windows alone. WINDOWS_ONLY = re.compile(r"(?:matrix|runner)\.os\s*==\s*'[^']*windows", re.IGNORECASE) +#: A ``run:`` that maps the leg through :data:`release_conda.PLATFORMS` (#1967). +MAPS = re.compile(r'\butil/release_conda\.py (?:leg|build)\b') -def dead(spec: 'dict[str, Any]') -> 'list[str]': - """What of ``spec``, a job, names a runner its matrix has no leg on.""" +def legs() -> 'set[str]': + """Every runner label a conda job has a leg on.""" + return {label for name, job_id in JOBS for label in _workflows.job(name, job_id)['strategy']['matrix']['os']} + + +def dead(spec: 'dict[str, Any]', every: 'Optional[set[str]]' = None) -> 'list[str]': + """What of ``spec``, a job, names a runner its matrix has no leg on. + + The labels :data:`release_conda.PLATFORMS` maps serve both jobs, so they are + held to ``every``, the labels either has a leg on, by default :func:`legs`. + + """ labels = list(spec['strategy']['matrix']['os']) + every = legs() if every is None else every windows_leg = any(WINDOWS.search(label) for label in labels) found = [] for step in spec['steps']: @@ -67,6 +83,11 @@ def dead(spec: 'dict[str, Any]') -> 'list[str]': for pattern in ARM.findall(block): if pattern != '*' and pattern not in labels: found.append(f'{name}: `case` arm {pattern}, which is no leg') + if MAPS.search(str(step.get('run', ''))): + found.extend(f'{name}: release_conda.py maps {label}, which no conda job has a leg on' + for label in sorted(set(release_conda.PLATFORMS) - every)) + found.extend(f'{name}: release_conda.py maps no platform for {label}' + for label in labels if label not in release_conda.PLATFORMS) return found @@ -127,14 +148,22 @@ def test_dropping_conda_dists_windows_leg_leaves_its_steps_dead(self) -> None: """#2060's steps are live only while the leg is there.""" spec = copy.deepcopy(_workflows.job('cron-conda.yml', 'conda-dist')) spec['strategy']['matrix']['os'].remove('windows-latest') - found = dead(spec) + found = dead(spec, legs() - {'windows-latest'}) for line in ('Upload conda packages to Anaconda: `if:` compares matrix.os with windows-latest, which is no leg', 'Install and Setup: `continue-on-error:` compares matrix.os with windows-latest, which is no leg', 'Windows summary: Windows-only, and no leg is Windows', - 'Build conda packages: `case` arm windows-latest, which is no leg'): + 'Build conda packages: release_conda.py maps windows-latest, which no conda job has a leg on'): with self.subTest(line=line): self.assertIn(line, found) + def test_a_leg_the_script_does_not_map_is_reported(self) -> None: + spec = copy.deepcopy(_workflows.job('create-release.yml', 'conda')) + spec['strategy']['matrix']['os'].append('macos-99') + self.assertEqual(dead(spec, legs() | {'macos-99'}), [ + "Check whether this leg is already on Anaconda: release_conda.py maps no platform for macos-99", + 'Build conda packages: release_conda.py maps no platform for macos-99', + ]) + def test_a_windows_leg_allows_its_own_steps(self) -> None: spec = {'strategy': {'matrix': {'os': ['ubuntu-latest', 'windows-latest']}}, 'steps': [{'name': 'Setup (Windows)', 'if': "${{ matrix.os == 'windows-latest' }}", diff --git a/tests/project/test_conda_windows_leg_2060.py b/tests/project/test_conda_windows_leg_2060.py index a6e0015f58..6a69232683 100644 --- a/tests/project/test_conda_windows_leg_2060.py +++ b/tests/project/test_conda_windows_leg_2060.py @@ -44,6 +44,7 @@ from tests import _workflows from tests.project.test_conda_native_platforms_1822 import DISTRIBUTION, SHIM, elf, load, pe from tests.project.test_release_gates import PUBLISHING_MARKERS, STATUS, evaluate_condition +from util import release_conda if TYPE_CHECKING: from typing import Any @@ -174,8 +175,11 @@ def test_the_bash_steps_name_their_shell(self) -> None: self.assertTrue(str(step.get('shell', '')).startswith('bash')) def test_the_build_step_maps_the_leg_to_win_64(self) -> None: + # #1967: util/release_conda.py maps it, which the step runs with the leg's label. build = next(step for step in spec()['steps'] if step.get('id') == 'conda_build') - self.assertRegex(build['run'], r'\n\s*windows-latest\)\n\s*target_platform="win-64"\n') + self.assertEqual(build['run'], 'python ../../util/release_conda.py build') + self.assertEqual(build['env']['PCAPKIT_CONDA_OS'], '${{ matrix.os }}') + self.assertEqual(release_conda.PLATFORMS[WINDOWS], 'win-64') def test_the_smoke_test_runs_python_exe_on_windows(self) -> None: smoke = next(step for step in spec()['steps'] if step.get('id') == 'smoke') diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index c27a969514..88a49890f6 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -21,8 +21,8 @@ step's failure is tried on every leg instead of read off the text. Expressions go through :mod:`tests.project.test_release_gates`'s evaluator, and the scripts that write annotations are executed. Both summaries run :file:`util/leg_summary.py` -(#1967), and :file:`create-release.yml`'s ``pypi`` keeps an inline copy of its -shell, which is run against the script on the same steps. +(#1967), :file:`create-release.yml`'s ``pypi``'s too, whose former inline copy +of its shell is run against the script on the same steps. #1660 added an experimental tier, free-threaded CPython and GraalPy, as legs of ``test`` only; #1732 moved them to a job of their own, ``experimental``, which @@ -125,7 +125,7 @@ } #: #2030: the steps after a summary, on ``always()`` and masked on every leg, that #: name and upload the job's copy of its summary for util/check_suite_report.py. -#: They end every job with a KTLO or experimental leg but :data:`INLINE_SUMMARY`'s. +#: They end every job with a KTLO or experimental leg but :data:`UNCOPIED_SUMMARY`'s. JOB_SUMMARY_STEPS = ('Name the job summary', 'Upload the job summary') #: The ``env:`` entry by which a summary step copies its lines there too (#2030). SUMMARY_COPY = {'JOB_SUMMARY_COPY': '${{ runner.temp }}/job-summary.md'} @@ -149,8 +149,21 @@ IMAGE_PYTHON = '/usr/bin/python3' SUMMARY_RUN = f'{IMAGE_PYTHON} util/leg_summary.py ktlo' EXPERIMENTAL_SUMMARY_RUN = f'{IMAGE_PYTHON} util/leg_summary.py experimental' -#: The ``KTLO summary`` that keeps its shell inline until #1967's ``create-release.yml`` step. -INLINE_SUMMARY = ('create-release.yml', 'pypi') +#: The ``KTLO summary`` whose job keeps no copy of its summary: no CI report reads +#: Create Release's runs (#2030). +UNCOPIED_SUMMARY = ('create-release.yml', 'pypi') +#: ``pypi``'s ``KTLO summary`` before it ran :data:`SUMMARY_RUN` (#1967). +FORMER_RELEASE_SUMMARY = '''\ +failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") +if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 +fi +while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." + echo "- ${LEG}: step \\`${id}\\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" +done <<< "$failed" +''' #: A ``util/`` script a ``run:`` invokes, whose own annotations are the step's. UTIL_SCRIPT = re.compile(r'\butil/[\w-]+\.py\b') #: ``experimental``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``: @@ -187,12 +200,12 @@ def label(step: 'dict[str, Any]') -> 'str': def summary_tail(where: 'tuple[str, str]') -> 'tuple[str, ...]': """The steps that follow the summary of the job ``where``, ``(workflow, job)`` (#2030).""" - return () if where == INLINE_SUMMARY else JOB_SUMMARY_STEPS + return () if where == UNCOPIED_SUMMARY else JOB_SUMMARY_STEPS def summary_env(where: 'tuple[str, str]', env: 'dict[str, str]') -> 'dict[str, str]': - """The summary step's ``env:`` for the job ``where``, which copies its lines unless inline (#2030).""" - return dict(env) if where == INLINE_SUMMARY else dict(env, **SUMMARY_COPY) + """The summary step's ``env:`` for the job ``where``, which copies its lines unless uncopied (#2030).""" + return dict(env) if where == UNCOPIED_SUMMARY else dict(env, **SUMMARY_COPY) def summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': @@ -437,8 +450,7 @@ def test_every_failure_it_masks_reaches_the_summary(self) -> None: self.assertEqual(summary.get('continue-on-error'), KTLO_GUARD) self.assertEqual(summary.get('shell'), 'bash') self.assertEqual(summary.get('env'), summary_env((name, job_id), SUMMARY_ENV)) - if (name, job_id) != INLINE_SUMMARY: - self.assertEqual(summary['run'], SUMMARY_RUN) + self.assertEqual(summary['run'], SUMMARY_RUN) blocking = BLOCKS_EVERY_LEG.get((name, job_id), ()) for leg in filter(is_ktlo, legs(name, job_id)): for index, _, record in scenarios(name, job_id, spec, leg): @@ -548,12 +560,12 @@ def test_the_summary_is_quiet_when_nothing_failed(self) -> None: self.assertEqual(page, '') self.assertIn(f'{LEG_KTLO}: no step failed.', result.stdout) - def test_the_inline_release_copy_reports_as_the_script_does(self) -> None: - """``pypi``'s ``KTLO summary`` keeps its shell (#1967), so it must not drift from the script.""" + def test_the_former_release_copy_reports_as_the_script_does(self) -> None: + """``pypi``'s ``KTLO summary`` held its own shell until #1967; the script reports as it did.""" if shutil.which('jq') is None: # pragma: no cover - self.skipTest('jq, on every Ubuntu runner image, is needed to run the inline KTLO summary') - inline = summary_step(job(*INLINE_SUMMARY))['run'] - self.assertNotIn('leg_summary.py', inline) + self.skipTest('jq, on every Ubuntu runner image, is needed to run the former KTLO summary') + self.assertEqual(summary_step(job(*UNCOPIED_SUMMARY))['run'], SUMMARY_RUN) + inline = FORMER_RELEASE_SUMMARY for steps in ({'checkout': 'success', 'python': 'success', 'tests': 'success'}, {'checkout': 'success', 'python': 'success', 'install': 'failure', 'tests': 'skipped'}, {'checkout': 'failure', 'python': 'failure', 'install': 'skipped'}): diff --git a/tests/project/test_release_conda_leg_smoke_1855.py b/tests/project/test_release_conda_leg_smoke_1855.py index aaec698440..bd36a4d211 100644 --- a/tests/project/test_release_conda_leg_smoke_1855.py +++ b/tests/project/test_release_conda_leg_smoke_1855.py @@ -18,6 +18,7 @@ import re import stat import subprocess # nosec: B404 +import sys import tempfile import unittest from typing import TYPE_CHECKING @@ -25,6 +26,7 @@ from tests import _workflows from tests._support import scale_timeout from tests.project.test_conda_native_platforms_1822 import BASH, CONDA +from tests.project.test_release_conda_scripts_1967 import read_outputs try: import yaml @@ -34,7 +36,8 @@ if TYPE_CHECKING: from typing import Any -WORKFLOWS = pathlib.Path(__file__).resolve().parents[2] / '.github' / 'workflows' +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / '.github' / 'workflows' #: The conda jobs, as (workflow, job); the first is the one the release mirrors. CRON, RELEASE = ('cron-conda.yml', 'conda-dist'), ('create-release.yml', 'conda') #: The steps that publish what the leg built. @@ -96,9 +99,8 @@ class TestEachBuildStepNamesItsPackage(unittest.TestCase): def outputs(self, workflow: 'str', job_id: 'str') -> 'tuple[str, str]': """The build step's ``$GITHUB_OUTPUT`` on ``ubuntu-latest``, and its log.""" step = next(step for step in steps(workflow, job_id) if step.get('id') == 'conda_build') - script = step['run'].replace('${{ matrix.os }}', 'ubuntu-latest').replace('${{ matrix.python-version }}', - '3.14') - self.assertNotIn('${{', script) + # #1967: util/release_conda.py's, run from the recipe's directory. + self.assertEqual(step['run'], 'python ../../util/release_conda.py build') with tempfile.TemporaryDirectory() as temp: scratch = pathlib.Path(temp) (scratch / 'bin').mkdir() @@ -108,10 +110,11 @@ def outputs(self, workflow: 'str', job_id: 'str') -> 'tuple[str, str]': output = scratch / 'output' output.write_text('', encoding='utf-8') result = subprocess.run( - [BASH, '-e', '-c', script], cwd=temp, # nosec: B603 + [sys.executable, str(ROOT / 'util' / 'release_conda.py'), 'build'], cwd=temp, # nosec: B603 env=dict(os.environ, PATH=f'{scratch / "bin"}{os.pathsep}{os.environ["PATH"]}', TMPDIR=temp, GITHUB_OUTPUT=str(output), FAKE_SUBDIR='linux-64', - PCAPKIT_VERSION='1.0', PCAPKIT_BUILD='0'), + PCAPKIT_VERSION='1.0', PCAPKIT_BUILD='0', PCAPKIT_CONDA_OS='ubuntu-latest', + PCAPKIT_PYTHON_VERSION='3.14'), capture_output=True, text=True, check=False, timeout=scale_timeout(60)) return output.read_text(encoding='utf-8'), result.stdout + result.stderr @@ -119,11 +122,12 @@ def test_host_package_is_the_package_built(self) -> None: for workflow, job_id in (CRON, RELEASE): written, log = self.outputs(workflow, job_id) with self.subTest(workflow=workflow, job=job_id): - host = re.findall(r'^host_package=(.*)$', written, re.MULTILINE) + outputs = read_outputs(written) + host = [value for name, value in outputs if name == 'host_package'] self.assertEqual(len(host), 1, f'no host_package output\n{written}{log}') - packages = re.search(r'^packages<: line N:`` before its own errors is dropped, and the + build's fresh directory is read as ````. The step bodies are kept here + verbatim, and each case table is checked to reach every way its shell ends. +* :class:`TestTheScripts` pins what the cases cannot compare: the step outputs a + run writes, and the errors for a leg on another platform. +* :class:`TestTheStepsRunTheScripts` pins each step to its script and its + ``env:``, both jobs' legs to :data:`release_conda.PLATFORMS`, and the script + to Python 3.7's syntax. + +""" + +from __future__ import annotations + +import ast +import json +import os +import pathlib +import re +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING + +from tests._workflows import job, steps +from util import release_conda + +if TYPE_CHECKING: + from typing import Optional + +#: Test tier (#1958): runs the tree's own scripts and the workflows' former shell. +__tier__ = 'LATEST' + +ROOT = pathlib.Path(__file__).resolve().parents[2] +UTIL = ROOT / 'util' +BASH = shutil.which('bash') or '' +JOBS = (('create-release.yml', 'conda'), ('cron-conda.yml', 'conda-dist')) + + +def read_outputs(text: 'str') -> 'list[tuple[str, str]]': + """``$GITHUB_OUTPUT``'s ``(name, value)`` pairs as the runner reads them, ``name<> "$CALL_LOG" +while [ $# -gt 0 ]; do [ "$1" = -o ] && cp "$CURL_REPLY" "$2"; shift; done +printf '%s' "${CURL_CODE:-200}" +exit "${CURL_FAIL:-0}" +''' + +#: A ``conda`` whose ``build`` writes each of ``$FAKE_BUILT``'s paths under its +#: ``--output-folder`` -- ``/`` a directory, ``@`` a link -- +#: and whose ``install`` and ``build`` fail with ``$INSTALL_FAIL`` and ``$BUILD_FAIL``. +FAKE_CONDA = r'''#!/bin/bash +out="" +for arg in "$@"; do [ "$prev" = --output-folder ] && out="$arg"; prev="$arg"; done +printf 'conda %s\n' "$*" | sed "s|${out:-@@}||g" >> "$CALL_LOG" +case "$1" in + install) exit "${INSTALL_FAIL:-0}" ;; + build) + echo 'conda-build: building' + [ -z "${BUILD_FAIL:-}" ] || { echo 'conda-build: failed' >&2; exit "$BUILD_FAIL"; } + for path in $FAKE_BUILT; do + case "$path" in + */) mkdir -p "$out/$path" ;; + *@*) mkdir -p "$out/$(dirname "${path%@*}")"; ln -s "${path#*@}" "$out/${path%@*}" ;; + *) mkdir -p "$out/$(dirname "$path")"; : > "$out/$path" ;; + esac + done + ;; +esac +''' + +#: An ``anaconda`` logging its arguments a line each, failing with ``$ANACONDA_FAIL``. +FAKE_ANACONDA = r'''#!/bin/sh +{ echo anaconda; printf '%s\n' "$@"; } >> "$CALL_LOG" +exit "${ANACONDA_FAIL:-0}" +''' + +#: "Check whether this leg is already on Anaconda" in ``conda``, before :file:`util/release_conda.py`. +FORMER_LEG = textwrap.dedent(r''' + set -euo pipefail + + case "${{ matrix.os }}" in + macos-latest) + target_platform="osx-arm64" + ;; + macos-26-intel) + target_platform="osx-64" + ;; + ubuntu-latest) + target_platform="linux-64" + ;; + *) + target_platform="" + ;; + esac + py_tag="py$(echo "${{ matrix.python-version }}" | tr -d '.')" + + # Same coupling risk, and the same fix, as version_check's checks. + status="$(curl -s --retry 3 --retry-connrefused \ + -o /tmp/anaconda_release.json -w '%{http_code}' \ + "https://api.anaconda.org/release/jarryshaw/pypcapkit/${PCAPKIT_VERSION}" || echo 000)" + + exists=false + if [ "$status" = "200" ]; then + # The `printf` form of version_check's checks, and needed here: this + # runs before setup-python, so `python3` is the image's (3.12 on + # ubuntu-latest, 3.13 on macos-latest), and indented `python3 -c` + # source would fail under `set -e` -- only on the retry this is for. + exists="$(printf '%s\n' \ + "import json" \ + "with open('/tmp/anaconda_release.json') as f:" \ + " data = json.load(f)" \ + "target_platform = '$target_platform'" \ + "py_tag = '$py_tag'" \ + "found = any(" \ + " d.get('attrs', {}).get('subdir') == target_platform" \ + " and d.get('attrs', {}).get('build', '').startswith(py_tag + '_')" \ + " for d in data.get('distributions', [])" \ + ")" \ + "print('true' if found else 'false')" \ + | python3 -)" + fi + + echo "PCAPKIT_LEG_EXISTS=$exists" >> "$GITHUB_OUTPUT" + if [ "$exists" = "true" ]; then + echo "::notice title=conda leg already published::pypcapkit ${PCAPKIT_VERSION} for ${target_platform} / ${py_tag} is already on anaconda.org/jarryshaw (http ${status}); skipping this leg's upload." + fi +''').lstrip() + +#: "Build conda packages" in ``conda-dist``, before :file:`util/release_conda.py`; +#: ``conda``'s was the same but for the ``windows-latest`` arm (:data:`FORMER_RELEASE_BUILD`). +FORMER_BUILD = textwrap.dedent(r''' + set -euo pipefail + + out_dir="$(mktemp -d -t compilation-XXXXXXXXXX)" + echo "out_dir=$out_dir" >> "$GITHUB_OUTPUT" + + conda build . --package-format .tar.bz2 --no-anaconda-upload --output-folder "$out_dir" --python "${{ matrix.python-version }}" + host_package="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort | sed -n '1p')" + if [ -z "$host_package" ]; then + echo "::error::No conda package was built" + exit 1 + fi + + case "${{ matrix.os }}" in + macos-latest) + target_platform="osx-arm64" + ;; + macos-26-intel) + target_platform="osx-64" + ;; + ubuntu-latest) + target_platform="linux-64" + ;; + windows-latest) + target_platform="win-64" + ;; + *) + target_platform="" + ;; + esac + + echo "host_package=$host_package" >> "$GITHUB_OUTPUT" + + # #1822: never `conda convert` across architectures. It relabels the + # package and copies its binaries unchanged, so the osx-64 package it + # made from the arm64 build carried arm64 extensions. + host_platform="$(basename "$(dirname "$host_package")")" + if [ "$host_platform" != "$target_platform" ]; then + echo "::error title=${host_platform} package on a ${target_platform:-unmapped} leg (#1822)::${{ matrix.os }} publishes ${target_platform:-nothing}, and conda built ${host_platform} here; a package is published only for the platform it was built on." + exit 1 + fi + + package_paths="$(find "$out_dir" -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print | sort)" + if [ -z "$package_paths" ]; then + echo "::error::No conda packages were found" + exit 1 + fi + + printf '%s\n' "$package_paths" + package_paths_csv="$(printf '%s\n' "$package_paths" | paste -sd ',' -)" + echo "packages_csv=$package_paths_csv" >> "$GITHUB_OUTPUT" + { + echo 'packages<> "$GITHUB_OUTPUT" +''').lstrip() +FORMER_RELEASE_BUILD = FORMER_BUILD.replace(' windows-latest)\n target_platform="win-64"\n ;;\n', '') + +#: "Upload conda packages to Anaconda (Intel)" in both jobs, before :file:`util/release_conda.py`. +FORMER_UPLOAD = textwrap.dedent(r''' + set -euo pipefail + + conda install -n base --override-channels -c conda-forge -y anaconda-client + + # The action's handling of its `labels:` and `packages:` inputs: a + # comma-separated list, each trimmed, and the build step's + # newline-separated paths. + labels=() + IFS=',' read -ra names <<< "$PCAPKIT_CONDA_LABELS" + for name in ${names[@]+"${names[@]}"}; do + name="$(echo "$name" | xargs)" + if [ -n "$name" ]; then + labels+=(--label "$name") + fi + done + packages=() + while IFS= read -r path; do + if [ -n "$path" ]; then + packages+=("$path") + fi + done <<< "$PCAPKIT_CONDA_PACKAGES" + if [ "${#packages[@]}" -eq 0 ]; then + echo "::error::No conda packages to upload" + exit 1 + fi + + "${CONDA}/bin/anaconda" --token "$ANACONDA_API_TOKEN" upload \ + --user "$PCAPKIT_CONDA_CHANNEL" --skip-existing ${labels[@]+"${labels[@]}"} "${packages[@]}" +''').lstrip() + + +def reply(*builds: 'tuple[str, str]') -> 'str': + """An Anaconda release reply holding a distribution per ``(subdir, build)``.""" + return json.dumps({'distributions': [{'attrs': {'subdir': subdir, 'build': build}} for subdir, build in builds]}) + + +#: The leg check's cases: ``(name, os, python, reply, extra env)``. +LEG_CASES = ( + ('the leg is published', 'ubuntu-latest', '3.14', reply(('linux-64', 'py314_0')), {}), + ('another platform is', 'macos-latest', '3.14', reply(('linux-64', 'py314_0'), ('osx-64', 'py314_0')), {}), + ('the arm64 leg is published', 'macos-latest', '3.13', reply(('osx-arm64', 'py313_1')), {}), + ('the Intel leg is published', 'macos-26-intel', '3.13', reply(('osx-64', 'py313_0')), {}), + ('a longer tag is another build', 'ubuntu-latest', '3.1', reply(('linux-64', 'py314_0')), {}), + ('an unmapped label matches nothing', 'macos-99', '3.14', reply(('', 'py314_0'), ('osx-64', 'py314_0')), {}), + ('no distributions', 'ubuntu-latest', '3.14', '{}', {}), + ('not found', 'ubuntu-latest', '3.14', reply(('linux-64', 'py314_0')), {'CURL_CODE': '404'}), + ('a 2xx other than 200 is not read', 'ubuntu-latest', '3.14', reply(('linux-64', 'py314_0')), + {'CURL_CODE': '201'}), + ('a 204 with no body', 'ubuntu-latest', '3.14', '', {'CURL_CODE': '204'}), + ('curl fails', 'ubuntu-latest', '3.14', reply(('linux-64', 'py314_0')), {'CURL_CODE': '000', 'CURL_FAIL': '7'}), + ('a 200 that is no JSON', 'ubuntu-latest', '3.14', 'not json', {}), + ('a 200 that is a list', 'ubuntu-latest', '3.14', '[]', {}), + ('no version', 'ubuntu-latest', '3.14', '{}', {'PCAPKIT_VERSION': None}), +) + +#: The build's cases: ``(name, os, $FAKE_BUILT, extra env)``. +BUILD_CASES = ( + ('linux-64 on ubuntu-latest', 'ubuntu-latest', 'linux-64/pypcapkit-1.5.0-py314_0.tar.bz2', {}), + ('osx-arm64 on macos-latest', 'macos-latest', 'osx-arm64/pypcapkit-1.5.0-py314_0.tar.bz2', {}), + ('osx-64 on macos-26-intel', 'macos-26-intel', 'osx-64/pypcapkit-1.5.0-py314_0.tar.bz2', {}), + ('win-64 on windows-latest', 'windows-latest', 'win-64/pypcapkit-1.5.0-py314_0.tar.bz2', {}), + ('every package, sorted, and nothing else', 'ubuntu-latest', + 'linux-64/z.tar.bz2 linux-64/a.conda linux-64/repodata.json linux-64/dir.tar.bz2/ ' + 'linux-64/link.conda@a.conda linux-64/.hidden.conda', {}), + ('osx-arm64 on macos-26-intel', 'macos-26-intel', 'osx-arm64/pypcapkit-1.5.0-py314_0.tar.bz2', {}), + ('an unmapped label', 'macos-99', 'osx-arm64/pypcapkit-1.5.0-py314_0.tar.bz2', {}), + ('nothing built', 'ubuntu-latest', 'linux-64/repodata.json', {}), + ('a failed build', 'ubuntu-latest', 'linux-64/a.tar.bz2', {'BUILD_FAIL': '3'}), + ('no conda', 'ubuntu-latest', '', {'PATH': None}), +) + +PACKAGES = '/out/osx-64/pypcapkit-1.5.0-py314_0.tar.bz2\n/out/osx-64/pypcapkit-1.5.0-py314_1.conda' +#: The upload's cases: ``(name, $PCAPKIT_CONDA_LABELS, $PCAPKIT_CONDA_PACKAGES, extra env)``. +UPLOAD_CASES = ( + ('one label', 'main', PACKAGES, {}), + ('labels trimmed', ' main , dev ', PACKAGES, {}), + ('no label', '', PACKAGES, {}), + ('empty and blank labels', ',main,, ,dev,', PACKAGES, {}), + ('a label holding a run of spaces', 'a b', PACKAGES, {}), + ('only the first line of labels', 'main\ndev', PACKAGES, {}), + ('blank package lines', 'main', '\n' + PACKAGES + '\n\n', {}), + ('a path holding spaces', 'main', '/out/osx-64/a b.conda', {}), + ('no packages', 'main', '\n\n', {}), + ('a failed install', 'main', PACKAGES, {'INSTALL_FAIL': '5'}), + ('a failed upload', 'main', PACKAGES, {'ANACONDA_FAIL': '2'}), + ('no CONDA', 'main', PACKAGES, {'CONDA': None}), + ('no labels', None, PACKAGES, {}), +) + + +class _Stand(unittest.TestCase): + """A scratch directory with stand-in ``curl``, ``conda``, ``anaconda`` and ``python3`` first on ``PATH``.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1967-conda-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.bin = self.tmp / 'bin' + conda_bin = self.tmp / 'conda' / 'bin' + for path, text in ((self.bin / 'curl', FAKE_CURL), (self.bin / 'conda', FAKE_CONDA), + (conda_bin / 'anaconda', FAKE_ANACONDA)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding='utf-8') + path.chmod(0o755) + # The count's ``python3 -``, on this interpreter in both runs. + (self.bin / 'python3').symlink_to(sys.executable) + self.work = self.tmp / 'work' + self.scratch = self.tmp / 'scratch' + self.output = self.tmp / 'output' + self.log = self.tmp / 'calls' + self.reply = self.tmp / 'reply' + self.release = self.tmp / 'release.json' + self.shell = self.tmp / 'step.sh' + self.base = { + 'PATH': f'{self.bin}{os.pathsep}{os.environ.get("PATH", "")}', 'HOME': str(self.tmp), 'LC_ALL': 'C', + 'TMPDIR': str(self.scratch), 'GITHUB_OUTPUT': str(self.output), 'CALL_LOG': str(self.log), + 'CURL_REPLY': str(self.reply), 'CONDA': str(self.tmp / 'conda'), 'PCAPKIT_VERSION': '1.5.0', + 'ANACONDA_API_TOKEN': 'TOKEN', 'ANACONDA_CLI_FORCE_NEW': '1', 'PCAPKIT_CONDA_CHANNEL': 'jarryshaw', + } + + def reset(self) -> None: + """Leave nothing of an earlier run but the stand-ins.""" + for path in (self.output, self.log, self.release): + if path.exists(): + path.unlink() + for path in (self.work, self.scratch): + shutil.rmtree(path, ignore_errors=True) + path.mkdir() + self.output.write_text('', encoding='utf-8') + + def env(self, extra: 'dict[str, Optional[str]]', **variables: 'Optional[str]') -> 'dict[str, str]': + merged = dict(self.base, **variables, **extra) + if 'PATH' in extra and extra['PATH'] is None: # no conda, but the shell's own tools + merged['PATH'] = os.pathsep.join(os.path.dirname(shutil.which(tool) or '') for tool in + ('find', 'sort', 'sed', 'paste', 'mktemp', 'basename', 'dirname')) + return {key: value for key, value in merged.items() if value is not None} + + def state(self, result: 'subprocess.CompletedProcess[str]') -> 'tuple[object, ...]': + """Status, both streams, the outputs and the calls, the build's directory read as ````.""" + outputs = read_outputs(self.output.read_text(encoding='utf-8')) + out_dir = dict(outputs).get('out_dir') + + def plain(text: 'str') -> 'str': + return text.replace(out_dir, '') if out_dir else text + if out_dir: + self.assertTrue(out_dir.startswith(f'{self.scratch}{os.sep}compilation-'), out_dir) + self.assertTrue(os.path.isdir(out_dir)) + calls = self.log.read_text(encoding='utf-8') if self.log.exists() else None + return (result.returncode, plain(result.stdout), plain(result.stderr), + [(name, plain(value)) for name, value in outputs], calls) + + def script(self, run: 'str', env: 'dict[str, str]') -> 'tuple[object, ...]': + """``run``, the step's ``python[3] [../../]util/