diff --git a/.github/workflows/reusable_edge_build.yml b/.github/workflows/reusable_edge_build.yml new file mode 100644 index 0000000..668eba4 --- /dev/null +++ b/.github/workflows/reusable_edge_build.yml @@ -0,0 +1,176 @@ +name: edge build +# Central, org-wide reusable workflow. On a push to the default branch, builds the .debs of every +# package that declares , for each distro in ros_distros on amd64 and arm64, +# at +git.: after the release it follows and before the next one. +# A distro or architecture that does not build is a warning and uploads nothing. +# +# What built is handed to a private publisher repository, which publishes it into the edge +# channel. The caller passes secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach +# the dispatch. +# +# Called by a product repository on a push to its default branch: +# +# on: +# push: +# branches: [main] +# concurrency: +# group: edge-${{ github.ref }} +# cancel-in-progress: true +# jobs: +# edge: +# uses: Duatic/ci-workflows/.github/workflows/reusable_edge_build.yml@v1 +# secrets: inherit + +on: + workflow_call: + inputs: + runner: + description: 'Runner label(s) to run the job on, e.g. "ubuntu-latest" or "self-hosted".' + default: 'ubuntu-latest' + required: false + type: string + publisher: + description: 'Repository in this organization that publishes what was built; empty takes the RELEASE_PUBLISHER organization variable' + default: '' + required: false + type: string + ros_distros: + description: 'JSON list of the ROS distros to build the packages for' + default: '["jazzy", "kilted", "lyrical", "rolling"]' + required: false + type: string + +permissions: + contents: read + +jobs: + plan: + runs-on: ${{ inputs.runner }} + outputs: + build: ${{ steps.plan.outputs.build || '[]' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + # checkout above takes the calling repository. job.workflow_ref names this file's own + # repository and revision, so a caller pinning the workflow pins the scripts with it. + - name: Resolve this workflow's own revision + id: self + env: + WORKFLOW_REF: ${{ job.workflow_ref }} + run: | + [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; } + echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT" + echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + repository: ${{ steps.self.outputs.repo }} + ref: ${{ steps.self.outputs.ref }} + path: .checks + + - name: List the packages to build + id: plan + env: + EVENT: ${{ github.event_name }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + if [ "$EVENT" != push ] || [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then + echo "::error::edge builds come from a push to $DEFAULT_BRANCH, not a $EVENT to $REF" + exit 1 + fi + build="$(python3 .checks/release/edge_packages.py)" + [ "$build" != "[]" ] || echo "no package declares ; nothing to build" + echo "build=$build" >> "$GITHUB_OUTPUT" + + # The tolerance lives in the build workflow: a job calling a reusable workflow cannot itself be + # allowed to fail. + build: + needs: plan + if: needs.plan.outputs.build != '[]' + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.plan.outputs.build) }} + ros_distro: ${{ fromJSON(inputs.ros_distros) }} + uses: Duatic/ci-workflows/.github/workflows/reusable_release_build.yml@v1 + with: + tag: ${{ matrix.package.tag }} + package: ${{ matrix.package.package }} + ros_distro: ${{ matrix.ros_distro }} + version_suffix: ${{ matrix.package.suffix }} + runner: ${{ inputs.runner }} + continue_on_failure: true + + publish: + needs: [plan, build] + if: ${{ !cancelled() && needs.plan.outputs.build != '[]' }} + runs-on: ${{ inputs.runner }} + # Reads this run's artifacts with the App: the caller's token need not grant actions: read. + permissions: + contents: read + steps: + - name: Resolve this workflow's own revision + id: self + env: + WORKFLOW_REF: ${{ job.workflow_ref }} + run: | + [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; } + echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT" + echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + repository: ${{ steps.self.outputs.repo }} + ref: ${{ steps.self.outputs.ref }} + path: .checks + + - name: Check a publisher and the release App are available + id: app + env: + APP_ID: ${{ secrets.RELEASE_APP_ID }} + PUBLISHER: ${{ inputs.publisher || vars.RELEASE_PUBLISHER }} + run: | + if [ -z "$PUBLISHER" ]; then + echo "::warning::not published: no publisher repository named; set the RELEASE_PUBLISHER organization variable" + echo "ok=false" >> "$GITHUB_OUTPUT" + elif [ -z "$APP_ID" ]; then + echo "::warning::not published: RELEASE_APP_ID does not reach this workflow; pass secrets: inherit" + echo "ok=false" >> "$GITHUB_OUTPUT" + else + echo "ok=true" >> "$GITHUB_OUTPUT" + echo "publisher=$PUBLISHER" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/create-github-app-token@v2 + if: steps.app.outputs.ok == 'true' + id: token + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ steps.app.outputs.publisher }} + permission-contents: write + + - uses: actions/create-github-app-token@v2 + if: steps.app.outputs.ok == 'true' + id: artifacts + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-actions: read + + - name: Hand what built to the publisher + if: steps.app.outputs.ok == 'true' + env: + GH_TOKEN: ${{ steps.artifacts.outputs.token }} + DISPATCH_TOKEN: ${{ steps.token.outputs.token }} + BUILD: ${{ needs.plan.outputs.build }} + DISTROS: ${{ inputs.ros_distros }} + CHANNEL: edge + PUBLISHER: ${{ github.repository_owner }}/${{ steps.app.outputs.publisher }} + run: python3 .checks/release/hand_to_publisher.py diff --git a/.github/workflows/reusable_release_build.yml b/.github/workflows/reusable_release_build.yml index 5b30724..96d852e 100644 --- a/.github/workflows/reusable_release_build.yml +++ b/.github/workflows/reusable_release_build.yml @@ -1,7 +1,8 @@ name: release build # Builds the .deb for one package at a tag with duatic_devtools' builder, for amd64 and arm64 unless # architectures says otherwise, and uploads one artifact per architecture. Signing and publishing -# belong to the archive host, not to a runner. +# belong to the archive host, not to a runner. A build of main names its commit as the tag and +# gives version_suffix, which is appended to the version in package.xml. # # Dispatch it here with a repository, a tag and a package. The Ubuntu base to build against is # derived from ros_distro, since each ROS 2 distro has exactly one supported Ubuntu base. @@ -41,6 +42,10 @@ on: description: 'Report a failed build as a warning and upload nothing, rather than failing the run.' default: false type: boolean + version_suffix: + description: 'Appended to the version in package.xml, e.g. "+git20261027153012.ab12cd3" for a build of main' + default: '' + type: string workflow_call: inputs: repository: @@ -76,6 +81,11 @@ on: default: false required: false type: boolean + version_suffix: + description: 'Appended to the version in package.xml, e.g. "+git20261027153012.ab12cd3" for a build of main' + default: '' + required: false + type: string outputs: artifact: description: 'Artifact name without its architecture: one artifact per architecture built, -' @@ -112,11 +122,13 @@ jobs: TAG: ${{ inputs.tag }} ROS_DISTRO_TARGET: ${{ inputs.ros_distro }} ARCH: ${{ matrix.arch }} + SUFFIX: ${{ inputs.version_suffix }} run: | mkdir -p "$RUNNER_TEMP/out" docker run --rm -i --platform "linux/$ARCH" \ -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ -e ROS_DISTRO="$ROS_DISTRO_TARGET" \ + -e DEB_VERSION_SUFFIX="$SUFFIX" \ -v "$PWD/src:/src" \ -v "$PWD/.devtools/packaging/builder:/builder" \ -v "$RUNNER_TEMP/out:/out" \ @@ -125,6 +137,16 @@ jobs: # Fail unless a .deb of this architecture was produced. ls "$RUNNER_TEMP"/out/*_"$ARCH".deb >/dev/null \ || { echo "::error::no ${ARCH} .deb produced"; ls "$RUNNER_TEMP/out"; exit 1; } + # A builder that ignored the suffix would produce the release's version. + if [ -n "$SUFFIX" ]; then + for deb in "$RUNNER_TEMP"/out/*.deb; do + v="$(dpkg-deb -f "$deb" Version)" + case "$v" in + *"$SUFFIX"-*) ;; + *) echo "::error::$(basename "$deb") is version $v, without $SUFFIX"; exit 1 ;; + esac + done + fi echo "artifact=${PACKAGE}-${TAG//\//-}-${ROS_DISTRO_TARGET}" >> "$GITHUB_OUTPUT" # Debug symbols are a .ddeb, which a *.deb glob misses. build_pkg.sh collects them and the diff --git a/.github/workflows/reusable_release_publish.yml b/.github/workflows/reusable_release_publish.yml index 44b273d..bf49424 100644 --- a/.github/workflows/reusable_release_publish.yml +++ b/.github/workflows/reusable_release_publish.yml @@ -23,6 +23,11 @@ on: default: 'jazzy' required: false type: string + channel: + description: 'Archive channel to publish into: stable for releases, edge for builds of main' + default: 'stable' + required: false + type: string runner: description: 'Runner label(s) to run the job on, e.g. "ubuntu-latest" or "self-hosted".' default: 'ubuntu-latest' @@ -40,16 +45,10 @@ on: EXPECTED_KEY_FPR: required: true -# aptly's database is single-writer and the routing lists and dist tree are shared, so two -# releases publishing at once corrupt each other rather than queueing. -concurrency: - group: publish-${{ inputs.root }} - cancel-in-progress: false - jobs: publish: runs-on: ${{ inputs.runner }} - timeout-minutes: 15 + timeout-minutes: 30 steps: - uses: actions/download-artifact@v4 with: @@ -64,6 +63,7 @@ jobs: ARTIFACT: ${{ inputs.artifact }} ROOT: ${{ inputs.root }} DISTRO: ${{ inputs.distro }} + CHANNEL: ${{ inputs.channel }} run: | count="$(find deb -name '*.deb' | wc -l)" # One package per call. build_pkg.sh collects every .deb the build produced, and @@ -77,7 +77,7 @@ jobs: version="${rest%%_*}" # These are interpolated into a command run on the host that holds the signing key, so # anything outside this set is refused rather than escaped. - for field in "$package" "$version" "$ROOT" "$DISTRO"; do + for field in "$package" "$version" "$ROOT" "$DISTRO" "$CHANNEL"; do printf '%s' "$field" | grep -qE '^[A-Za-z0-9][A-Za-z0-9._/+~-]*$' \ || { echo "FATAL: refusing unexpected characters in '$field'" >&2; exit 1; } done @@ -90,7 +90,7 @@ jobs: # Self-hosted runner images do not always carry one. - name: Ensure an SSH client is available run: | - command -v ssh >/dev/null && command -v scp >/dev/null && exit 0 + command -v ssh >/dev/null && command -v sftp >/dev/null && exit 0 sudo apt-get update -qq && sudo apt-get install -y -qq openssh-client - name: Configure SSH to the publish host @@ -112,6 +112,7 @@ jobs: VERSION: ${{ steps.deb.outputs.version }} ROOT: ${{ inputs.root }} DISTRO: ${{ inputs.distro }} + CHANNEL: ${{ inputs.channel }} HOST: ${{ secrets.PUBLISH_HOST }} PUBLISH_USER: ${{ secrets.PUBLISH_USER }} FPR: ${{ secrets.EXPECTED_KEY_FPR }} @@ -120,14 +121,23 @@ jobs: # Everything the build produced, not just the .deb the version was read from: the # debug symbols ship as a .ddeb and the archive indexes them alongside, and an open # root publishes a source tarball beside the package that declares one. - scp deb/* "$target:~/archive/out/" + # Each file lands under a hidden name and is renamed into place, so a publish already + # running on the host never takes one half written. + batch="$RUNNER_TEMP/sftp-batch" + : > "$batch" + for f in deb/*; do + b="$(basename "$f")" + printf 'put "%s" "archive/out/.%s.part"\nrename "archive/out/.%s.part" "archive/out/%s"\n' \ + "$f" "$b" "$b" "$b" >> "$batch" + done + sftp -q -b "$batch" "$target" # Named explicitly: a non-interactive ssh reads no profile, so nothing the publish # host exports in a login shell reaches this command. # Expanded here rather than on the host: every value is checked against a # conservative character set above, and the host must receive literals. # shellcheck disable=SC2029 ssh "$target" \ - "EXPECTED_KEY_FPR='$FPR' DISTRO='$DISTRO' \ + "EXPECTED_KEY_FPR='$FPR' DISTRO='$DISTRO' CHANNEL='$CHANNEL' \ ~/duatic_devtools/packaging/archive/add_and_publish.sh \ '${PACKAGE}:${ROOT}:${VERSION}'" diff --git a/.github/workflows/reusable_release_tag.yml b/.github/workflows/reusable_release_tag.yml index 32ecb57..4a2b8cd 100644 --- a/.github/workflows/reusable_release_tag.yml +++ b/.github/workflows/reusable_release_tag.yml @@ -7,9 +7,10 @@ name: release tag # distro in ros_distros on amd64 and arm64. A distro or architecture that does not build is a # warning and uploads nothing; a released package without a root is not built, also a warning. # -# What built is handed to a private publisher repository, which publishes it from a self-hosted -# runner: self-hosted runners never take jobs from public repositories. The caller passes -# secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach the dispatch. +# What built is handed to a private publisher repository, which publishes it into the stable +# channel from a self-hosted runner: self-hosted runners never take jobs from public repositories. +# The caller passes secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach the +# dispatch. # # The calling job must grant contents: write; a called workflow cannot raise the token above what # its caller holds. Requires a squash- or merge-committed pull request: a rebase merge's first @@ -180,8 +181,24 @@ jobs: if: ${{ !cancelled() && needs.tag.outputs.build != '[]' }} runs-on: ${{ inputs.runner }} # Reads this run's artifacts with the App: the caller grants contents: write only. - permissions: {} + permissions: + contents: read steps: + - name: Resolve this workflow's own revision + id: self + env: + WORKFLOW_REF: ${{ job.workflow_ref }} + run: | + [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; } + echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT" + echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@v4 + with: + repository: ${{ steps.self.outputs.repo }} + ref: ${{ steps.self.outputs.ref }} + path: .checks + - name: Check a publisher and the release App are available id: app env: @@ -226,41 +243,6 @@ jobs: DISPATCH_TOKEN: ${{ steps.token.outputs.token }} BUILD: ${{ needs.tag.outputs.build }} DISTROS: ${{ inputs.ros_distros }} + CHANNEL: stable PUBLISHER: ${{ github.repository_owner }}/${{ steps.app.outputs.publisher }} - run: | - python3 - <<'PY' - import json, os, urllib.request - api, repo, run = os.environ["GITHUB_API_URL"], os.environ["GITHUB_REPOSITORY"], os.environ["GITHUB_RUN_ID"] - - def call(url, token, body=None): - req = urllib.request.Request(url, data=body, headers={ - "Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}) - with urllib.request.urlopen(req) as r: - return json.loads(r.read() or b"null") - - built, page = set(), 1 - while True: - got = call(f"{api}/repos/{repo}/actions/runs/{run}/artifacts?per_page=100&page={page}", - os.environ["GH_TOKEN"])["artifacts"] - built |= {a["name"] for a in got} - if len(got) < 100: - break - page += 1 - - # Named as reusable_release_build.yml names them; a leg that failed uploaded nothing. - artifacts = [] - for release in json.loads(os.environ["BUILD"]): - for distro in json.loads(os.environ["DISTROS"]): - for arch in ("amd64", "arm64"): - name = f'{release["package"]}-{release["tag"].replace("/", "-")}-{distro}-{arch}' - if name in built: - artifacts.append({"artifact": name, **release, "distro": distro, "arch": arch}) - if not artifacts: - print("::warning::nothing built, so nothing was handed to the publisher") - raise SystemExit(0) - payload = {"repository": repo, "run_id": run, "artifacts": artifacts} - call(f'{api}/repos/{os.environ["PUBLISHER"]}/dispatches', os.environ["DISPATCH_TOKEN"], - json.dumps({"event_type": "publish", "client_payload": payload}).encode()) - for a in artifacts: - print(f'handed {a["artifact"]} to {os.environ["PUBLISHER"]}') - PY + run: python3 .checks/release/hand_to_publisher.py diff --git a/.github/workflows/self-check.yml b/.github/workflows/self-check.yml index a23337c..bd08ea5 100644 --- a/.github/workflows/self-check.yml +++ b/.github/workflows/self-check.yml @@ -47,5 +47,7 @@ jobs: run: release/test_prepare_release.sh - name: Released packages suite run: release/test_released_packages.sh + - name: Edge packages suite + run: release/test_edge_packages.sh - name: Upstream expansion suite run: upstream/test_expand_repos.sh diff --git a/README.md b/README.md index 6e92447..53a1da2 100644 --- a/README.md +++ b/README.md @@ -268,8 +268,8 @@ for each distro in `ros_distros` (default jazzy, kilted, lyrical and rolling) on arm64. A distro or architecture that does not build is a warning on the run and uploads nothing; a released package without a root is not built, also with a warning. What built is then handed, with a `repository_dispatch`, to the private publisher repository the organization variable -`RELEASE_PUBLISHER` names (or the `publisher` input), which publishes it into the apt archive from a -self-hosted runner: self-hosted runners never take jobs from public repositories. The tag job passes +`RELEASE_PUBLISHER` names (or the `publisher` input), which publishes it into the apt archive's +`stable` channel from a self-hosted runner: self-hosted runners never take jobs from public repositories. The tag job passes `secrets: inherit` so that `RELEASE_APP_ID` and `RELEASE_APP_PRIVATE_KEY` reach the dispatch; without them, or with no publisher named, the run warns that nothing was published. The App reads the run's artifacts, which needs Actions read on the repository, and dispatches, which needs Contents write @@ -282,14 +282,39 @@ merge commit, and one GitHub Release is published under it with every package's under its own heading. The same version-sharing requirement `check_release_pr.py` already enforces on the pull request is checked again here, in case a merge ever bypasses that check. +**`reusable_edge_build.yml`** runs on a push to the default branch. It builds every package that +declares ``, like the tag workflow, at `+git.` +(the committer time in UTC, as `YYYYMMDDHHMMSS`), which sorts after the release it follows and +before the next one. For a squash- or merge-committed +main, that is the order commits landed in. What built is handed to the publisher for the archive's `edge` channel, which +keeps the newest builds of each package. `nightly` is not built: once a day the publisher switches +it to what `edge` publishes. It takes the same `runner`, `publisher` and `ros_distros` inputs as the +tag workflow and needs `secrets: inherit` the same way. + +```yaml +# .github/workflows/edge.yml +name: Edge +on: + push: + branches: [main] +concurrency: + group: edge-${{ github.ref }} + cancel-in-progress: true +jobs: + edge: + uses: Duatic/ci-workflows/.github/workflows/reusable_edge_build.yml@v1 + secrets: inherit +``` + **`reusable_release_build.yml`** builds one package into a `.deb` with `duatic_devtools`' builder, once for amd64 on `runner` and once for arm64 on `arm64_runner` (default `ubuntu-24.04-arm`), both native, or only for the architectures listed in `architectures`, e.g. `'["amd64"]'`. Each architecture is uploaded as its own artifact, `-`, holding the `.deb`, its debug symbols and, when the package declares ``, its source tarball. -It takes a tag to check out and the package to build inside it - either -a `/` tag or a repo-wide `v` tag names the package explicitly, so the -build does not need to guess it from the tag's shape. It is not wired into any repository yet: a +It takes a tag to check out and the package to build inside it, and a `version_suffix` appended to +the `package.xml` version for a build of main. The package is named explicitly, whether the tag is +a `/` tag, a repo-wide `v` tag or a commit, so the build does not need to +guess it from the tag's shape. It is not wired into any repository yet: a package's dependencies on other packages in the org resolve only from the builder's local repository, so a build stops at the first such dependency. Signing and publishing belong to the archive host. @@ -297,7 +322,8 @@ archive host. **`reusable_release_publish.yml`** takes one of those artifacts and publishes it into a signed archive root, by delivering it to the publish host and calling `duatic_devtools`' archive tooling there. The signing key is never on the runner. It is `workflow_call` only and reads the artifact from the same -run, so it is called after the build job, once per architecture, rather than dispatched on its own. The package name and +run, so it is called after the build job, once per architecture, rather than dispatched on its own. +`channel` names the archive channel, `stable` by default. The package name and version are read from the `.deb` filename rather than taken as inputs, and both are refused unless they match a conservative character set, because they reach a command run on the host that holds the key. Needs `PUBLISH_HOST`, `PUBLISH_USER`, `PUBLISH_SSH_KEY`, `PUBLISH_HOST_KEY` and @@ -391,6 +417,8 @@ And in `ci.yml`, beside the orchestrator, for either mode: | `package` | build | yes | | The package to build, by its `package.xml` name. | | `repository` | build | no | the caller | `owner/name` of the repository holding the tag, for a dispatch from elsewhere. | | `ros_distro` | build | no | `jazzy` | Distro to build for. The Ubuntu base is not an input; it's derived from this instead. | +| `version_suffix` | build | no | | Appended to the `package.xml` version, starting with `+` or `~`. The build fails if the `.deb` does not carry it. | +| `channel` | publish | no | `stable` | Archive channel to publish into: `stable` or `edge`. | | `runner` | prepare, prepare-repo, tag, repo-tag, build | no | `ubuntu-latest` | Runner label(s), e.g. `self-hosted`. | | Secret | Workflow | Required | Description | diff --git a/release/edge_packages.py b/release/edge_packages.py new file mode 100755 index 0000000..e713a68 --- /dev/null +++ b/release/edge_packages.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +"""List the packages a build of main publishes, as JSON. + +Every package.xml at --head that declares is built, at a version that sorts +after the release it follows and before the next one: + + [{"package": ..., "version": "+git.", "suffix": "+git...", + "tag": "", "root": ...}] + +The commit time is the committer time in UTC, as YYYYMMDDHHMMSS, so later commits on main sort +later. tag is the commit the build checks out, as reusable_release_build.yml takes it. + +Usage: edge_packages.py [--head REF] +""" + +import argparse +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET + + +def git(*args): + r = subprocess.run( + ["git", *args], capture_output=True, text=True, env={**os.environ, "TZ": "UTC"} + ) + if r.returncode: + raise SystemExit(f'git {" ".join(args)}: {r.stderr.strip()}') + return r.stdout + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--head", default="HEAD") + args = ap.parse_args() + + commit = git("rev-parse", f"{args.head}^{{commit}}").strip() + stamp = git("show", "-s", "--format=%cd", "--date=format-local:%Y%m%d%H%M%S", commit).strip() + suffix = f"+git{stamp}.{commit[:7]}" + + packages = [] + for path in git("ls-tree", "-r", "--name-only", commit).split(): + if os.path.basename(path) != "package.xml": + continue + try: + root = ET.fromstring(git("show", f"{commit}:{path}")) + except ET.ParseError: + print(f"::warning::{path} is not valid XML; not built", file=sys.stderr) + continue + export = root.find("export") + archive_root = ( + (export.findtext("duatic_archive_root") or "").strip() if export is not None else "" + ) + if not archive_root: + continue + name = (root.findtext("name") or "").strip() + version = (root.findtext("version") or "").strip() + packages.append( + { + "package": name, + "version": version + suffix, + "suffix": suffix, + "tag": commit, + "root": archive_root, + } + ) + print(f"{name} {version}{suffix} -> {archive_root}", file=sys.stderr) + print(json.dumps(sorted(packages, key=lambda p: p["package"]))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/release/hand_to_publisher.py b/release/hand_to_publisher.py new file mode 100755 index 0000000..e3a1449 --- /dev/null +++ b/release/hand_to_publisher.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +"""Hand what this run built to the publisher repository, for one channel. + +Reads its inputs from the environment, as the release workflows set them: + + BUILD JSON list of {package, version, tag, root} that were built + DISTROS JSON list of the ROS distros they were built for + CHANNEL archive channel to publish into: stable for releases, edge for builds of main + PUBLISHER owner/name of the publisher repository + GH_TOKEN reads this run's artifacts + DISPATCH_TOKEN dispatches to the publisher + +Only the artifacts this run uploaded are handed over: a leg that failed uploaded nothing. +""" + +import json +import os +import sys +import urllib.request + + +def call(url, token, body=None): + req = urllib.request.Request( + url, + data=body, + headers={"Authorization": f"Bearer {token}", "Accept": "application/vnd.github+json"}, + ) + with urllib.request.urlopen(req) as r: + return json.loads(r.read() or b"null") + + +def main(): + e = os.environ + api, repo, run = e["GITHUB_API_URL"], e["GITHUB_REPOSITORY"], e["GITHUB_RUN_ID"] + + built, page = set(), 1 + while True: + got = call( + f"{api}/repos/{repo}/actions/runs/{run}/artifacts?per_page=100&page={page}", + e["GH_TOKEN"], + )["artifacts"] + built |= {a["name"] for a in got} + if len(got) < 100: + break + page += 1 + + # Named as reusable_release_build.yml names them. + artifacts = [] + for release in json.loads(e["BUILD"]): + for distro in json.loads(e["DISTROS"]): + for arch in ("amd64", "arm64"): + name = f'{release["package"]}-{release["tag"].replace("/", "-")}-{distro}-{arch}' + if name in built: + artifacts.append( + { + "artifact": name, + "package": release["package"], + "version": release["version"], + "tag": release["tag"], + "root": release["root"], + "distro": distro, + "arch": arch, + } + ) + if not artifacts: + print("::warning::nothing built, so nothing was handed to the publisher") + return 0 + payload = {"repository": repo, "run_id": run, "channel": e["CHANNEL"], "artifacts": artifacts} + call( + f'{api}/repos/{e["PUBLISHER"]}/dispatches', + e["DISPATCH_TOKEN"], + json.dumps({"event_type": "publish", "client_payload": payload}).encode(), + ) + for a in artifacts: + print(f'handed {a["artifact"]} to {e["PUBLISHER"]} for {e["CHANNEL"]}') + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/release/test_edge_packages.sh b/release/test_edge_packages.sh new file mode 100755 index 0000000..3c91238 --- /dev/null +++ b/release/test_edge_packages.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# Drive edge_packages.py through its cases, in a throwaway repository. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TOOL="$HERE/edge_packages.py" +WORK="$(mktemp -d)" +trap 'rm -r "$WORK" 2>/dev/null' EXIT +pass=0; fail=0 + +check() { # check