From 6cfcc93740cd9a5be64c33f147447b1112cc7ec9 Mon Sep 17 00:00:00 2001 From: Vaios Papaspyros <8146703+bpapaspyros@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:53:36 +0100 Subject: [PATCH 1/4] feat(release): publish builds of main into the edge channel reusable_edge_build.yml builds every routed package on a push to the default branch at +git. and hands it to the publisher for the edge channel. The tag workflow hands releases over for stable. reusable_release_build.yml takes version_suffix and fails when the .deb does not carry it; reusable_release_publish.yml takes channel and leaves queueing to the publish host. The dispatch moves to release/hand_to_publisher.py, shared by both. --- .github/workflows/reusable_edge_build.yml | 165 ++++++++++++++++++ .github/workflows/reusable_release_build.yml | 24 ++- .../workflows/reusable_release_publish.yml | 19 +- .github/workflows/reusable_release_tag.yml | 61 +++---- .github/workflows/self-check.yml | 2 + README.md | 39 ++++- release/edge_packages.py | 87 +++++++++ release/hand_to_publisher.py | 87 +++++++++ release/test_edge_packages.sh | 73 ++++++++ 9 files changed, 501 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/reusable_edge_build.yml create mode 100755 release/edge_packages.py create mode 100755 release/hand_to_publisher.py create mode 100755 release/test_edge_packages.sh diff --git a/.github/workflows/reusable_edge_build.yml b/.github/workflows/reusable_edge_build.yml new file mode 100644 index 0000000..3d71284 --- /dev/null +++ b/.github/workflows/reusable_edge_build.yml @@ -0,0 +1,165 @@ +name: edge build +# Central, org-wide reusable workflow. On a push to the default branch, builds the .debs of every +# package that declares , for each distro in ros_distros on amd64 and arm64, +# at +git.: after the release it follows and before the next one. +# A distro or architecture that does not build is a warning and uploads nothing. +# +# What built is handed to a private publisher repository, which publishes it into the edge +# channel. The caller passes secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach +# the dispatch. +# +# Called by a product repository on a push to its default branch: +# +# on: +# push: +# branches: [main] +# concurrency: +# group: edge-${{ github.ref }} +# cancel-in-progress: true +# jobs: +# edge: +# uses: Duatic/ci-workflows/.github/workflows/reusable_edge_build.yml@v1 +# secrets: inherit + +on: + workflow_call: + inputs: + runner: + description: 'Runner label(s) to run the job on, e.g. "ubuntu-latest" or "self-hosted".' + default: 'ubuntu-latest' + required: false + type: string + publisher: + description: 'Repository in this organization that publishes what was built; empty takes the RELEASE_PUBLISHER organization variable' + default: '' + required: false + type: string + ros_distros: + description: 'JSON list of the ROS distros to build the packages for' + default: '["jazzy", "kilted", "lyrical", "rolling"]' + required: false + type: string + +permissions: + contents: read + +jobs: + plan: + runs-on: ${{ inputs.runner }} + outputs: + build: ${{ steps.plan.outputs.build || '[]' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + # checkout above takes the calling repository. job.workflow_ref names this file's own + # repository and revision, so a caller pinning the workflow pins the scripts with it. + - name: Resolve this workflow's own revision + id: self + env: + WORKFLOW_REF: ${{ job.workflow_ref }} + run: | + [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; } + echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT" + echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + repository: ${{ steps.self.outputs.repo }} + ref: ${{ steps.self.outputs.ref }} + path: .checks + + - name: List the packages to build + id: plan + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + if [ "$EVENT" != push ] || [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then + echo "::error::edge builds come from a push to $DEFAULT_BRANCH, not a $EVENT to $REF" + exit 1 + fi + build="$(python3 .checks/release/edge_packages.py)" + [ "$build" != "[]" ] || echo "no package declares ; nothing to build" + echo "build=$build" >> "$GITHUB_OUTPUT" + + # The tolerance lives in the build workflow: a job calling a reusable workflow cannot itself be + # allowed to fail. + build: + needs: plan + if: needs.plan.outputs.build != '[]' + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.plan.outputs.build) }} + ros_distro: ${{ fromJSON(inputs.ros_distros) }} + uses: Duatic/ci-workflows/.github/workflows/reusable_release_build.yml@v1 + with: + tag: ${{ matrix.package.tag }} + package: ${{ matrix.package.package }} + ros_distro: ${{ matrix.ros_distro }} + version_suffix: ${{ matrix.package.suffix }} + runner: ${{ inputs.runner }} + continue_on_failure: true + + publish: + needs: [plan, build] + if: ${{ !cancelled() && needs.plan.outputs.build != '[]' }} + runs-on: ${{ inputs.runner }} + permissions: + actions: read + steps: + - name: Resolve this workflow's own revision + id: self + env: + WORKFLOW_REF: ${{ job.workflow_ref }} + run: | + [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; } + echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT" + echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + repository: ${{ steps.self.outputs.repo }} + ref: ${{ steps.self.outputs.ref }} + path: .checks + + - name: Check a publisher and the release App are available + id: app + env: + APP_ID: ${{ secrets.RELEASE_APP_ID }} + PUBLISHER: ${{ inputs.publisher || vars.RELEASE_PUBLISHER }} + run: | + if [ -z "$PUBLISHER" ]; then + echo "::warning::not published: no publisher repository named; set the RELEASE_PUBLISHER organization variable" + echo "ok=false" >> "$GITHUB_OUTPUT" + elif [ -z "$APP_ID" ]; then + echo "::warning::not published: RELEASE_APP_ID does not reach this workflow; pass secrets: inherit" + echo "ok=false" >> "$GITHUB_OUTPUT" + else + echo "ok=true" >> "$GITHUB_OUTPUT" + echo "publisher=$PUBLISHER" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/create-github-app-token@v2 + if: steps.app.outputs.ok == 'true' + id: token + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ steps.app.outputs.publisher }} + permission-contents: write + + - name: Hand what built to the publisher + if: steps.app.outputs.ok == 'true' + env: + GH_TOKEN: ${{ github.token }} + DISPATCH_TOKEN: ${{ steps.token.outputs.token }} + BUILD: ${{ needs.plan.outputs.build }} + DISTROS: ${{ inputs.ros_distros }} + CHANNEL: edge + PUBLISHER: ${{ github.repository_owner }}/${{ steps.app.outputs.publisher }} + run: python3 .checks/release/hand_to_publisher.py diff --git a/.github/workflows/reusable_release_build.yml b/.github/workflows/reusable_release_build.yml index 5b30724..96d852e 100644 --- a/.github/workflows/reusable_release_build.yml +++ b/.github/workflows/reusable_release_build.yml @@ -1,7 +1,8 @@ name: release build # Builds the .deb for one package at a tag with duatic_devtools' builder, for amd64 and arm64 unless # architectures says otherwise, and uploads one artifact per architecture. Signing and publishing -# belong to the archive host, not to a runner. +# belong to the archive host, not to a runner. A build of main names its commit as the tag and +# gives version_suffix, which is appended to the version in package.xml. # # Dispatch it here with a repository, a tag and a package. The Ubuntu base to build against is # derived from ros_distro, since each ROS 2 distro has exactly one supported Ubuntu base. @@ -41,6 +42,10 @@ on: description: 'Report a failed build as a warning and upload nothing, rather than failing the run.' default: false type: boolean + version_suffix: + description: 'Appended to the version in package.xml, e.g. "+git20261027153012.ab12cd3" for a build of main' + default: '' + type: string workflow_call: inputs: repository: @@ -76,6 +81,11 @@ on: default: false required: false type: boolean + version_suffix: + description: 'Appended to the version in package.xml, e.g. "+git20261027153012.ab12cd3" for a build of main' + default: '' + required: false + type: string outputs: artifact: description: 'Artifact name without its architecture: one artifact per architecture built, -' @@ -112,11 +122,13 @@ jobs: TAG: ${{ inputs.tag }} ROS_DISTRO_TARGET: ${{ inputs.ros_distro }} ARCH: ${{ matrix.arch }} + SUFFIX: ${{ inputs.version_suffix }} run: | mkdir -p "$RUNNER_TEMP/out" docker run --rm -i --platform "linux/$ARCH" \ -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ -e ROS_DISTRO="$ROS_DISTRO_TARGET" \ + -e DEB_VERSION_SUFFIX="$SUFFIX" \ -v "$PWD/src:/src" \ -v "$PWD/.devtools/packaging/builder:/builder" \ -v "$RUNNER_TEMP/out:/out" \ @@ -125,6 +137,16 @@ jobs: # Fail unless a .deb of this architecture was produced. ls "$RUNNER_TEMP"/out/*_"$ARCH".deb >/dev/null \ || { echo "::error::no ${ARCH} .deb produced"; ls "$RUNNER_TEMP/out"; exit 1; } + # A builder that ignored the suffix would produce the release's version. + if [ -n "$SUFFIX" ]; then + for deb in "$RUNNER_TEMP"/out/*.deb; do + v="$(dpkg-deb -f "$deb" Version)" + case "$v" in + *"$SUFFIX"-*) ;; + *) echo "::error::$(basename "$deb") is version $v, without $SUFFIX"; exit 1 ;; + esac + done + fi echo "artifact=${PACKAGE}-${TAG//\//-}-${ROS_DISTRO_TARGET}" >> "$GITHUB_OUTPUT" # Debug symbols are a .ddeb, which a *.deb glob misses. build_pkg.sh collects them and the diff --git a/.github/workflows/reusable_release_publish.yml b/.github/workflows/reusable_release_publish.yml index 44b273d..2c85fa8 100644 --- a/.github/workflows/reusable_release_publish.yml +++ b/.github/workflows/reusable_release_publish.yml @@ -23,6 +23,11 @@ on: default: 'jazzy' required: false type: string + channel: + description: 'Archive channel to publish into: stable for releases, edge for builds of main' + default: 'stable' + required: false + type: string runner: description: 'Runner label(s) to run the job on, e.g. "ubuntu-latest" or "self-hosted".' default: 'ubuntu-latest' @@ -40,16 +45,10 @@ on: EXPECTED_KEY_FPR: required: true -# aptly's database is single-writer and the routing lists and dist tree are shared, so two -# releases publishing at once corrupt each other rather than queueing. -concurrency: - group: publish-${{ inputs.root }} - cancel-in-progress: false - jobs: publish: runs-on: ${{ inputs.runner }} - timeout-minutes: 15 + timeout-minutes: 30 steps: - uses: actions/download-artifact@v4 with: @@ -64,6 +63,7 @@ jobs: ARTIFACT: ${{ inputs.artifact }} ROOT: ${{ inputs.root }} DISTRO: ${{ inputs.distro }} + CHANNEL: ${{ inputs.channel }} run: | count="$(find deb -name '*.deb' | wc -l)" # One package per call. build_pkg.sh collects every .deb the build produced, and @@ -77,7 +77,7 @@ jobs: version="${rest%%_*}" # These are interpolated into a command run on the host that holds the signing key, so # anything outside this set is refused rather than escaped. - for field in "$package" "$version" "$ROOT" "$DISTRO"; do + for field in "$package" "$version" "$ROOT" "$DISTRO" "$CHANNEL"; do printf '%s' "$field" | grep -qE '^[A-Za-z0-9][A-Za-z0-9._/+~-]*$' \ || { echo "FATAL: refusing unexpected characters in '$field'" >&2; exit 1; } done @@ -112,6 +112,7 @@ jobs: VERSION: ${{ steps.deb.outputs.version }} ROOT: ${{ inputs.root }} DISTRO: ${{ inputs.distro }} + CHANNEL: ${{ inputs.channel }} HOST: ${{ secrets.PUBLISH_HOST }} PUBLISH_USER: ${{ secrets.PUBLISH_USER }} FPR: ${{ secrets.EXPECTED_KEY_FPR }} @@ -127,7 +128,7 @@ jobs: # conservative character set above, and the host must receive literals. # shellcheck disable=SC2029 ssh "$target" \ - "EXPECTED_KEY_FPR='$FPR' DISTRO='$DISTRO' \ + "EXPECTED_KEY_FPR='$FPR' DISTRO='$DISTRO' CHANNEL='$CHANNEL' \ ~/duatic_devtools/packaging/archive/add_and_publish.sh \ '${PACKAGE}:${ROOT}:${VERSION}'" diff --git a/.github/workflows/reusable_release_tag.yml b/.github/workflows/reusable_release_tag.yml index 6b94715..9d6dd95 100644 --- a/.github/workflows/reusable_release_tag.yml +++ b/.github/workflows/reusable_release_tag.yml @@ -7,9 +7,10 @@ name: release tag # distro in ros_distros on amd64 and arm64. A distro or architecture that does not build is a # warning and uploads nothing; a released package without a root is not built, also a warning. # -# What built is handed to a private publisher repository, which publishes it from a self-hosted -# runner: self-hosted runners never take jobs from public repositories. The caller passes -# secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach the dispatch. +# What built is handed to a private publisher repository, which publishes it into the stable +# channel from a self-hosted runner: self-hosted runners never take jobs from public repositories. +# The caller passes secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach the +# dispatch. # # The calling job must grant contents: write; a called workflow cannot raise the token above what # its caller holds. Requires a squash- or merge-committed pull request: a rebase merge's first @@ -182,6 +183,21 @@ jobs: permissions: actions: read steps: + - name: Resolve this workflow's own revision + id: self + env: + WORKFLOW_REF: ${{ job.workflow_ref }} + run: | + [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; } + echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT" + echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + repository: ${{ steps.self.outputs.repo }} + ref: ${{ steps.self.outputs.ref }} + path: .checks + - name: Check a publisher and the release App are available id: app env: @@ -216,41 +232,6 @@ jobs: DISPATCH_TOKEN: ${{ steps.token.outputs.token }} BUILD: ${{ needs.tag.outputs.build }} DISTROS: ${{ inputs.ros_distros }} + CHANNEL: stable PUBLISHER: ${{ github.repository_owner }}/${{ steps.app.outputs.publisher }} - run: | - python3 - <<'PY' - import json, os, urllib.request - api, repo, run = os.environ["GITHUB_API_URL"], os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_RUN_ID"] - - def call(url, token, body=None): - req = urllib.request.Request(url, data=body, headers={ - "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}) - with urllib.request.urlopen(req) as r: - return json.loads(r.read() or b"null") - - built, page = set(), 1 - while True: - got = call(f"{api}/repos/{repo}/actions/runs/{run}/artifacts?per_page=100&page={page}", - os.environ["GH_TOKEN"])["artifacts"] - built |= {a["name"] for a in got} - if len(got) < 100: - break - page += 1 - - # Named as reusable_release_build.yml names them; a leg that failed uploaded nothing. - artifacts = [] - for release in json.loads(os.environ["BUILD"]): - for distro in json.loads(os.environ["DISTROS"]): - for arch in ("amd64", "arm64"): - name = f'{release["package"]}-{release["tag"].replace("/", "-")}-{distro}-{arch}' - if name in built: - artifacts.append({"artifact": name, **release, "distro": distro, "arch": arch}) - if not artifacts: - print("::warning::nothing built, so nothing was handed to the publisher") - raise SystemExit(0) - payload = {"repository": repo, "run_id": run, "artifacts": artifacts} - call(f'{api}/repos/{os.environ["PUBLISHER"]}/dispatches', os.environ["DISPATCH_TOKEN"], - json.dumps({"event_type": "publish", "client_payload": payload}).encode()) - for a in artifacts: - print(f'handed {a["artifact"]} to {os.environ["PUBLISHER"]}') - PY + run: python3 .checks/release/hand_to_publisher.py diff --git a/.github/workflows/self-check.yml b/.github/workflows/self-check.yml index a23337c..bd08ea5 100644 --- a/.github/workflows/self-check.yml +++ b/.github/workflows/self-check.yml @@ -47,5 +47,7 @@ jobs: run: release/test_prepare_release.sh - name: Released packages suite run: release/test_released_packages.sh + - name: Edge packages suite + run: release/test_edge_packages.sh - name: Upstream expansion suite run: upstream/test_expand_repos.sh diff --git a/README.md b/README.md index 2a6f7ab..9a2f353 100644 --- a/README.md +++ b/README.md @@ -268,8 +268,8 @@ for each distro in `ros_distros` (default jazzy, kilted, lyrical and rolling) on arm64. A distro or architecture that does not build is a warning on the run and uploads nothing; a released package without a root is not built, also with a warning. What built is then handed, with a `repository_dispatch`, to the private publisher repository the organization variable -`RELEASE_PUBLISHER` names (or the `publisher` input), which publishes it into the apt archive from a -self-hosted runner: self-hosted runners never take jobs from public repositories. The tag job passes +`RELEASE_PUBLISHER` names (or the `publisher` input), which publishes it into the apt archive's +`stable` channel from a self-hosted runner: self-hosted runners never take jobs from public repositories. The tag job passes `secrets: inherit` so that `RELEASE_APP_ID` and `RELEASE_APP_PRIVATE_KEY` reach the dispatch; without them, or with no publisher named, the run warns that nothing was published. Its `released` output lists `{package, version, tag, root}` for what it tagged, `[]` otherwise. @@ -280,14 +280,38 @@ merge commit, and one GitHub Release is published under it with every package's under its own heading. The same version-sharing requirement `check_release_pr.py` already enforces on the pull request is checked again here, in case a merge ever bypasses that check. +**`reusable_edge_build.yml`** runs on a push to the default branch. It builds every package that +declares ``, like the tag workflow, at `+git.` +(the committer time in UTC, as `YYYYMMDDHHMMSS`), which sorts after the release it follows and +before the next one. What built is handed to the publisher for the archive's `edge` channel, which +keeps the newest builds of each package. `nightly` is not built: once a day the publisher switches +it to what `edge` publishes. It takes the same `runner`, `publisher` and `ros_distros` inputs as the +tag workflow and needs `secrets: inherit` the same way. + +```yaml +# .github/workflows/edge.yml +name: Edge +on: + push: + branches: [main] +concurrency: + group: edge-${{ github.ref }} + cancel-in-progress: true +jobs: + edge: + uses: Duatic/ci-workflows/.github/workflows/reusable_edge_build.yml@v1 + secrets: inherit +``` + **`reusable_release_build.yml`** builds one package into a `.deb` with `duatic_devtools`' builder, once for amd64 on `runner` and once for arm64 on `arm64_runner` (default `ubuntu-24.04-arm`), both native, or only for the architectures listed in `architectures`, e.g. `'["amd64"]'`. Each architecture is uploaded as its own artifact, `-`, holding the `.deb`, its debug symbols and, when the package declares ``, its source tarball. -It takes a tag to check out and the package to build inside it - either -a `/` tag or a repo-wide `v` tag names the package explicitly, so the -build does not need to guess it from the tag's shape. It is not wired into any repository yet: a +It takes a tag to check out and the package to build inside it, and a `version_suffix` appended to +the `package.xml` version for a build of main. The package is named explicitly, whether the tag is +a `/` tag, a repo-wide `v` tag or a commit, so the build does not need to +guess it from the tag's shape. It is not wired into any repository yet: a package's dependencies on other packages in the org resolve only from the builder's local repository, so a build stops at the first such dependency. Signing and publishing belong to the archive host. @@ -295,7 +319,8 @@ archive host. **`reusable_release_publish.yml`** takes one of those artifacts and publishes it into a signed archive root, by delivering it to the publish host and calling `duatic_devtools`' archive tooling there. The signing key is never on the runner. It is `workflow_call` only and reads the artifact from the same -run, so it is called after the build job, once per architecture, rather than dispatched on its own. The package name and +run, so it is called after the build job, once per architecture, rather than dispatched on its own. +`channel` names the archive channel, `stable` by default. The package name and version are read from the `.deb` filename rather than taken as inputs, and both are refused unless they match a conservative character set, because they reach a command run on the host that holds the key. Needs `PUBLISH_HOST`, `PUBLISH_USER`, `PUBLISH_SSH_KEY`, `PUBLISH_HOST_KEY` and @@ -389,6 +414,8 @@ And in `ci.yml`, beside the orchestrator, for either mode: | `package` | build | yes | | The package to build, by its `package.xml` name. | | `repository` | build | no | the caller | `owner/name` of the repository holding the tag, for a dispatch from elsewhere. | | `ros_distro` | build | no | `jazzy` | Distro to build for. The Ubuntu base is not an input; it's derived from this instead. | +| `version_suffix` | build | no | | Appended to the `package.xml` version, starting with `+` or `~`. The build fails if the `.deb` does not carry it. | +| `channel` | publish | no | `stable` | Archive channel to publish into: `stable` or `edge`. | | `runner` | prepare, prepare-repo, tag, repo-tag, build | no | `ubuntu-latest` | Runner label(s), e.g. `self-hosted`. | | Secret | Workflow | Required | Description | diff --git a/release/edge_packages.py b/release/edge_packages.py new file mode 100755 index 0000000..86c7711 --- /dev/null +++ b/release/edge_packages.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +"""List the packages a build of main publishes, as JSON. + +Every package.xml at --head that declares is built, at a version that sorts +after the release it follows and before the next one: + + [{"package": ..., "version": "+git.", "suffix": "+git...", + "tag": "", "root": ...}] + +The commit time is the committer time in UTC, as YYYYMMDDHHMMSS, so later commits on main sort +later. tag is the commit the build checks out, as reusable_release_build.yml takes it. + +Usage: edge_packages.py [--head REF] +""" + +import argparse +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET + + +def git(*args): + r = subprocess.run(["git", *args], capture_output=True, text=True, + env={**os.environ, "TZ": "UTC"}) + if r.returncode: + raise SystemExit(f'git {" ".join(args)}: {r.stderr.strip()}') + return r.stdout + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--head", default="HEAD") + args = ap.parse_args() + + commit = git("rev-parse", f"{args.head}^{{commit}}").strip() + stamp = git("show", "-s", "--format=%cd", "--date=format-local:%Y%m%d%H%M%S", commit).strip() + suffix = f"+git{stamp}.{commit[:7]}" + + packages = [] + for path in git("ls-tree", "-r", "--name-only", commit).split(): + if os.path.basename(path) != "package.xml": + continue + try: + root = ET.fromstring(git("show", f"{commit}:{path}")) + except ET.ParseError: + print(f"::warning::{path} is not valid XML; not built", file=sys.stderr) + continue + export = root.find("export") + archive_root = (export.findtext("duatic_archive_root") or "").strip() if export is not None else "" + if not archive_root: + continue + name = (root.findtext("name") or "").strip() + version = (root.findtext("version") or "").strip() + packages.append({"package": name, "version": version + suffix, "suffix": suffix, + "tag": commit, "root": archive_root}) + print(f"{name} {version}{suffix} -> {archive_root}", file=sys.stderr) + print(json.dumps(sorted(packages, key=lambda p: p["package"]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/release/hand_to_publisher.py b/release/hand_to_publisher.py new file mode 100755 index 0000000..a6e7d2c --- /dev/null +++ b/release/hand_to_publisher.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +"""Hand what this run built to the publisher repository, for one channel. + +Reads its inputs from the environment, as the release workflows set them: + + BUILD JSON list of {package, version, tag, root} that were built + DISTROS JSON list of the ROS distros they were built for + CHANNEL archive channel to publish into: stable for releases, edge for builds of main + PUBLISHER owner/name of the publisher repository + GH_TOKEN reads this run's artifacts + DISPATCH_TOKEN dispatches to the publisher + +Only the artifacts this run uploaded are handed over: a leg that failed uploaded nothing. +""" + +import json +import os +import sys +import urllib.request + + +def call(url, token, body=None): + req = urllib.request.Request(url, data=body, headers={ + "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}) + with urllib.request.urlopen(req) as r: + return json.loads(r.read() or b"null") + + +def main(): + e = os.environ + api, repo, run = e["GITHUB_API_URL"], e["GITHUB_REPOSITORY"], e["GITHUB_RUN_ID"] + + built, page = set(), 1 + while True: + got = call(f"{api}/repos/{repo}/actions/runs/{run}/artifacts?per_page=100&page={page}", + e["GH_TOKEN"])["artifacts"] + built |= {a["name"] for a in got} + if len(got) < 100: + break + page += 1 + + # Named as reusable_release_build.yml names them. + artifacts = [] + for release in json.loads(e["BUILD"]): + for distro in json.loads(e["DISTROS"]): + for arch in ("amd64", "arm64"): + name = f'{release["package"]}-{release["tag"].replace("/", "-")}-{distro}-{arch}' + if name in built: + artifacts.append({"artifact": name, "package": release["package"], + "version": release["version"], "tag": release["tag"], + "root": release["root"], "distro": distro, "arch": arch}) + if not artifacts: + print("::warning::nothing built, so nothing was handed to the publisher") + return 0 + payload = {"repository": repo, "run_id": run, "channel": e["CHANNEL"], "artifacts": artifacts} + call(f'{api}/repos/{e["PUBLISHER"]}/dispatches', e["DISPATCH_TOKEN"], + json.dumps({"event_type": "publish", "client_payload": payload}).encode()) + for a in artifacts: + print(f'handed {a["artifact"]} to {e["PUBLISHER"]} for {e["CHANNEL"]}') + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/release/test_edge_packages.sh b/release/test_edge_packages.sh new file mode 100755 index 0000000..3c91238 --- /dev/null +++ b/release/test_edge_packages.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# Drive edge_packages.py through its cases, in a throwaway repository. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOOL="$HERE/edge_packages.py" +WORK="$(mktemp -d)" +trap 'rm -r "$WORK" 2>/dev/null' EXIT +pass=0; fail=0 + +check() { # check