From 540890861d503a9332d1811c5f5d29eb1522f980 Mon Sep 17 00:00:00 2001 From: Yury Lysogorskiy Date: Sat, 31 Oct 2026 16:48:46 +0100 Subject: [PATCH] Configure validated PyPI and TestPyPI package publishing --- .github/workflows/publish-to-test-pypi.yml | 48 ---------- .github/workflows/wheels.yml | 100 +++++++++++++++++---- docs/pacemaker/releases.md | 63 +++++++++++++ mkdocs.yml | 1 + tools/check_release.py | 70 +++++++++++++++ 5 files changed, 218 insertions(+), 64 deletions(-) delete mode 100644 .github/workflows/publish-to-test-pypi.yml create mode 100644 docs/pacemaker/releases.md create mode 100644 tools/check_release.py diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml deleted file mode 100644 index 766f35c..0000000 --- a/.github/workflows/publish-to-test-pypi.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: Publish sdist to TestPyPI - -# Manual only. An index filename can be uploaded exactly once -- deleting or -# yanking does not free it -- so this must never fire by itself. It used to run -# `on: push`, which would have burned a version number on every commit. -on: workflow_dispatch - -jobs: - build-n-publish: - name: Build and publish the source distribution to TestPyPI - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - - - name: Set up Python - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - # sdist only, deliberately. `build --wheel` on a bare runner emits a - # linux_x86_64 tag, which PyPI and TestPyPI both reject -- portable wheels - # have to be built inside a manylinux container, which is what - # wheels.yml does with cibuildwheel. Publishing wheels from here is not a - # matter of adding a flag; it cannot work. - - name: Build the source distribution - run: pipx run build --sdist - - # Same guard as wheels.yml: CMakeExtension declares sources=[], so the - # C++ tree only reaches the sdist via MANIFEST.in. An sdist without it - # uploads happily and then fails for whoever builds from source. - - name: Verify the sdist can actually build - run: | - set -euo pipefail - tar tzf dist/*.tar.gz > files.txt - grep -qE 'CMakeLists\.txt$' files.txt || { echo "sdist has no CMakeLists.txt"; exit 1; } - grep -qE 'lib/pybind11/include/pybind11/pybind11\.h$' files.txt || { echo "sdist has no pybind11"; exit 1; } - cpp=$(grep -cE '\.cpp$' files.txt) - echo "sdist carries $cpp .cpp files" - [ "$cpp" -ge 60 ] || { echo "expected >=60 .cpp files, got $cpp"; exit 1; } - - - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - # wheels.yml can also publish this same sdist to TestPyPI. Without - # this, whichever runs second fails on the duplicate filename. - skip-existing: true diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 850ac33..ec42619 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -23,8 +23,24 @@ on: default: none options: [none, testpypi, pypi] +permissions: + contents: read + jobs: + plan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Check package version and publication target + env: + RELEASE_TARGET: ${{ github.event_name == 'workflow_dispatch' && inputs.publish || 'none' }} + run: python tools/check_release.py --target "$RELEASE_TARGET" + sdist: + needs: plan name: sdist runs-on: ubuntu-latest steps: @@ -59,10 +75,11 @@ jobs: - uses: actions/upload-artifact@v7 with: - name: sdist + name: release-sdist path: dist/*.tar.gz wheels: + needs: plan name: wheels (linux x86_64) runs-on: ubuntu-latest steps: @@ -141,29 +158,80 @@ jobs: - uses: actions/upload-artifact@v7 with: - name: wheels-linux-x86_64 + name: release-wheels-linux-x86_64 path: wheelhouse/*.whl - publish: - name: publish to ${{ github.event.inputs.publish }} + validate: needs: [sdist, wheels] - if: github.event_name == 'workflow_dispatch' && github.event.inputs.publish != 'none' runs-on: ubuntu-latest - environment: ${{ github.event.inputs.publish }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - uses: actions/download-artifact@v8 + with: + pattern: release-* + path: dist + merge-multiple: true + - run: python -m pip install twine + - name: Validate distribution metadata and wheel coverage + env: + FULL_MATRIX: ${{ github.event_name == 'workflow_dispatch' }} + run: | + python -m twine check --strict dist/* + if [ "$FULL_MATRIX" = "true" ]; then + python tools/check_release.py --artifacts dist --full-matrix + else + python tools/check_release.py --artifacts dist + fi + + publish-testpypi: + name: Publish to TestPyPI + needs: validate + if: github.event_name == 'workflow_dispatch' && inputs.publish == 'testpypi' + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/project/python-ace/ + concurrency: + group: publish-python-ace-testpypi + cancel-in-progress: false permissions: - id-token: write # trusted publishing; no API token in secrets + id-token: write steps: - uses: actions/download-artifact@v8 with: + pattern: release-* path: dist merge-multiple: true - - - run: ls -l dist/ - - - name: Publish - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + attestations: ${{ secrets.TEST_PYPI_API_TOKEN == '' }} + repository-url: https://test.pypi.org/legacy/ + + publish-pypi: + name: Publish to PyPI + needs: validate + if: github.event_name == 'workflow_dispatch' && inputs.publish == 'pypi' + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/python-ace/ + concurrency: + group: publish-python-ace-pypi + cancel-in-progress: false + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v8 + with: + pattern: release-* + path: dist + merge-multiple: true + - uses: pypa/gh-action-pypi-publish@release/v1 with: - repository-url: >- - ${{ github.event.inputs.publish == 'testpypi' - && 'https://test.pypi.org/legacy/' - || 'https://upload.pypi.org/legacy/' }} + user: ${{ secrets.PYPI_USERNAME || '__token__' }} + password: ${{ secrets.PYPI_PASSWORD }} + attestations: ${{ secrets.PYPI_PASSWORD == '' }} diff --git a/docs/pacemaker/releases.md b/docs/pacemaker/releases.md new file mode 100644 index 0000000..9122527 --- /dev/null +++ b/docs/pacemaker/releases.md @@ -0,0 +1,63 @@ +# Building and publishing python-ace + +The **Build wheels** GitHub Actions workflow (`.github/workflows/wheels.yml`) +builds a source distribution and Linux x86-64 manylinux_2_28 wheels. Pull +requests and normal pushes build Python 3.12; manual runs build Python +3.9, 3.10, 3.11, 3.12, and 3.13. The wheels are installed and smoke-tested, +then all distributions pass metadata, package-content, and version checks. +This workflow currently produces Linux x86-64 wheels; other platforms can +build from the source distribution with a C++ compiler. + +## Build without uploading + +In GitHub Actions, select **Build wheels → Run workflow** and leave `publish` +as `none`. Download `release-sdist` and `release-wheels-linux-x86_64` from the +completed run. This works before allocating a new release version. + +```bash +gh workflow run wheels.yml --repo ICAMS/python-ace --ref master -f publish=none +``` + +## Publish a release + +1. Set a new version in `pyproject.toml` and merge that change into `master`. + Version 0.4.0rc1 already exists on PyPI and cannot be replaced. +2. Run **Build wheels** from `master` with `publish=testpypi` first. +3. After reviewing the TestPyPI artifacts and installation, run it from + `master` with `publish=pypi`. +4. Verify the release on PyPI and install the explicit new version in a clean + environment. Prereleases require an explicit version or pip's `--pre`. + +```bash +gh workflow run wheels.yml --repo ICAMS/python-ace --ref master -f publish=testpypi +# After checking the TestPyPI release: +gh workflow run wheels.yml --repo ICAMS/python-ace --ref master -f publish=pypi +``` + +Publishing is manual. The workflow rejects non-master publication and versions +already present on the selected index before building. It uploads only after +all builds, wheel smoke tests, and distribution checks pass. PyPI and TestPyPI +use separate publishing jobs and GitHub environments (`pypi` and `testpypi`). + +## Authentication + +The workflow uses the repository's existing secrets: + +| Index | Secrets | +| --- | --- | +| PyPI | `PYPI_PASSWORD` containing a project-scoped API token; `PYPI_USERNAME` must be `__token__` or omitted | +| TestPyPI | `TEST_PYPI_API_TOKEN` containing a TestPyPI API token | + +The workflow never prints token values. The existing secrets' validity cannot +be verified without attempting an upload. Tokens must have permission for the +`python-ace` project on the respective index. + +For token-free Trusted Publishing, configure an existing-project publisher on +PyPI (and separately on TestPyPI) with owner `ICAMS`, repository `python-ace`, +workflow filename `wheels.yml`, and the matching environment name `pypi` or +`testpypi`. Remove the corresponding password secret after configuring the +publisher; the workflow then uses OIDC and enables attestations automatically. +The publisher registration is performed in the PyPI/TestPyPI account settings, +not in GitHub Actions. + +See the [PyPI Trusted Publishing guide](https://docs.pypi.org/trusted-publishers/adding-a-publisher/). diff --git a/mkdocs.yml b/mkdocs.yml index 8a7eb78..90422ba 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,6 +27,7 @@ extra_javascript: nav: - Home: index.md - Installation: pacemaker/install.md + - Building and publishing: pacemaker/releases.md - Quick start: pacemaker/quickstart.md - Pacemaker workflow: pacemaker/workflow.md - Extrapolation grade and active learning: pacemaker/active_learning.md diff --git a/tools/check_release.py b/tools/check_release.py new file mode 100644 index 0000000..5554dc0 --- /dev/null +++ b/tools/check_release.py @@ -0,0 +1,70 @@ +"""Check publication eligibility and the distributions built by wheels.yml.""" +import argparse +from email.parser import BytesParser +import os +from pathlib import Path +import tarfile +import tomllib +from urllib.error import HTTPError +from urllib.request import urlopen +import zipfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--target", choices=["none", "testpypi", "pypi"], default="none") + parser.add_argument("--artifacts", type=Path) + parser.add_argument("--full-matrix", action="store_true") + args = parser.parse_args() + project = tomllib.loads(Path("pyproject.toml").read_text())["project"] + name, version = project["name"], project["version"] + if name != "python-ace": + raise SystemExit(f"Unexpected distribution name: {name}") + print(f"Distribution: {name} {version}", flush=True) + if args.target != "none": + if os.environ.get("GITHUB_REF") != "refs/heads/master": + raise SystemExit("Publish from master after merging the release version change.") + host = "test.pypi.org" if args.target == "testpypi" else "pypi.org" + url = f"https://{host}/pypi/{name}/{version}/json" + try: + with urlopen(url, timeout=30): + pass + except HTTPError as error: + if error.code != 404: + raise + else: + raise SystemExit(f"{name} {version} already exists on {host}; choose a new version in pyproject.toml.") + print(f"Version is available on {host}.") + if args.artifacts is None: + return + wheels = sorted(args.artifacts.glob("*.whl")) + sdists = sorted(args.artifacts.glob("*.tar.gz")) + if not wheels or len(sdists) != 1: + raise SystemExit("Expected at least one wheel and exactly one source distribution.") + + def check_metadata(raw, filename): + metadata = BytesParser().parsebytes(raw) + if metadata["Name"] != name or metadata["Version"] != version: + raise SystemExit(f"Distribution metadata disagrees with pyproject.toml: {filename}") + + for wheel in wheels: + with zipfile.ZipFile(wheel) as archive: + entries = [n for n in archive.namelist() if n.endswith(".dist-info/METADATA")] + if len(entries) != 1: + raise SystemExit(f"Expected one metadata file in {wheel}") + check_metadata(archive.read(entries[0]), wheel) + with tarfile.open(sdists[0]) as archive: + entries = [m for m in archive.getmembers() if m.name.endswith("/PKG-INFO") and m.name.count("/") == 1] + if len(entries) != 1: + raise SystemExit("Expected one top-level PKG-INFO in the source distribution.") + check_metadata(archive.extractfile(entries[0]).read(), sdists[0]) + if args.full_matrix: + expected = {"cp39", "cp310", "cp311", "cp312", "cp313"} + actual = [w.name.split("-")[2] for w in wheels] + if set(actual) != expected or len(actual) != len(expected): + raise SystemExit(f"Expected exactly one wheel per interpreter {sorted(expected)}, found {actual}") + print(f"Validated {len(wheels)} wheels and one source distribution for {name} {version}.") + + +if __name__ == "__main__": + main()