feat(release): publish builds of main into the edge channel

.github/workflows/reusable_edge_build.yml

@@ -0,0 +1,176 @@
+name: edge build
+# Central, org-wide reusable workflow. On a push to the default branch, builds the .debs of every
+# package that declares <duatic_archive_root>, for each distro in ros_distros on amd64 and arm64,
+# at <version>+git<commit time>.<commit>: after the release it follows and before the next one.
+# A distro or architecture that does not build is a warning and uploads nothing.
+#
+# What built is handed to a private publisher repository, which publishes it into the edge
+# channel. The caller passes secrets: inherit so RELEASE_APP_ID and RELEASE_APP_PRIVATE_KEY reach
+# the dispatch.
+#
+# Called by a product repository on a push to its default branch:
+#
+#   on:
+#     push:
+#       branches: [main]
+#   concurrency:
+#     group: edge-${{ github.ref }}
+#     cancel-in-progress: true
+#   jobs:
+#     edge:
+#       uses: Duatic/ci-workflows/.github/workflows/reusable_edge_build.yml@v1
+#       secrets: inherit
+
+on:
+  workflow_call:
+    inputs:
+      runner:
+        description: 'Runner label(s) to run the job on, e.g. "ubuntu-latest" or "self-hosted".'
+        default: 'ubuntu-latest'
+        required: false
+        type: string
+      publisher:
+        description: 'Repository in this organization that publishes what was built; empty takes the RELEASE_PUBLISHER organization variable'
+        default: ''
+        required: false
+        type: string
+      ros_distros:
+        description: 'JSON list of the ROS distros to build the packages for'
+        default: '["jazzy", "kilted", "lyrical", "rolling"]'
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  plan:
+    runs-on: ${{ inputs.runner }}
+    outputs:
+      build: ${{ steps.plan.outputs.build || '[]' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      # checkout above takes the calling repository. job.workflow_ref names this file's own
+      # repository and revision, so a caller pinning the workflow pins the scripts with it.
+      - name: Resolve this workflow's own revision
+        id: self
+        env:
+          WORKFLOW_REF: ${{ job.workflow_ref }}
+        run: |
+          [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; }
+          echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT"
+          echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@v4
+        with:
+          repository: ${{ steps.self.outputs.repo }}
+          ref: ${{ steps.self.outputs.ref }}
+          path: .checks
+
+      - name: List the packages to build
+        id: plan
+        env:
+          EVENT: ${{ github.event_name }}
+          REF: ${{ github.ref }}
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          if [ "$EVENT" != push ] || [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then
+            echo "::error::edge builds come from a push to $DEFAULT_BRANCH, not a $EVENT to $REF"
+            exit 1
+          fi
+          build="$(python3 .checks/release/edge_packages.py)"
+          [ "$build" != "[]" ] || echo "no package declares <duatic_archive_root>; nothing to build"
+          echo "build=$build" >> "$GITHUB_OUTPUT"
+
+  # The tolerance lives in the build workflow: a job calling a reusable workflow cannot itself be
+  # allowed to fail.
+  build:
+    needs: plan
+    if: needs.plan.outputs.build != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        package: ${{ fromJSON(needs.plan.outputs.build) }}
+        ros_distro: ${{ fromJSON(inputs.ros_distros) }}
+    uses: Duatic/ci-workflows/.github/workflows/reusable_release_build.yml@v1
+    with:
+      tag: ${{ matrix.package.tag }}
+      package: ${{ matrix.package.package }}
+      ros_distro: ${{ matrix.ros_distro }}
+      version_suffix: ${{ matrix.package.suffix }}
+      runner: ${{ inputs.runner }}
+      continue_on_failure: true
+
+  publish:
+    needs: [plan, build]
+    if: ${{ !cancelled() && needs.plan.outputs.build != '[]' }}
+    runs-on: ${{ inputs.runner }}
+    # Reads this run's artifacts with the App: the caller's token need not grant actions: read.
+    permissions:
+      contents: read
+    steps:
+      - name: Resolve this workflow's own revision
+        id: self
+        env:
+          WORKFLOW_REF: ${{ job.workflow_ref }}
+        run: |
+          [ -n "$WORKFLOW_REF" ] || { echo "::error::job.workflow_ref is empty"; exit 1; }
+          echo "repo=${WORKFLOW_REF%%/.github/*}" >> "$GITHUB_OUTPUT"
+          echo "ref=${WORKFLOW_REF#*@}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@v4
+        with:
+          repository: ${{ steps.self.outputs.repo }}
+          ref: ${{ steps.self.outputs.ref }}
+          path: .checks
+
+      - name: Check a publisher and the release App are available
+        id: app
+        env:
+          APP_ID: ${{ secrets.RELEASE_APP_ID }}
+          PUBLISHER: ${{ inputs.publisher || vars.RELEASE_PUBLISHER }}
+        run: |
+          if [ -z "$PUBLISHER" ]; then
+            echo "::warning::not published: no publisher repository named; set the RELEASE_PUBLISHER organization variable"
+            echo "ok=false" >> "$GITHUB_OUTPUT"
+          elif [ -z "$APP_ID" ]; then
+            echo "::warning::not published: RELEASE_APP_ID does not reach this workflow; pass secrets: inherit"
+            echo "ok=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "ok=true" >> "$GITHUB_OUTPUT"
+            echo "publisher=$PUBLISHER" >> "$GITHUB_OUTPUT"
+          fi
+
+      - uses: actions/create-github-app-token@v2
+        if: steps.app.outputs.ok == 'true'
+        id: token
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: ${{ steps.app.outputs.publisher }}
+          permission-contents: write
+
+      - uses: actions/create-github-app-token@v2
+        if: steps.app.outputs.ok == 'true'
+        id: artifacts
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: ${{ github.event.repository.name }}
+          permission-actions: read
+
+      - name: Hand what built to the publisher
+        if: steps.app.outputs.ok == 'true'
+        env:
+          GH_TOKEN: ${{ steps.artifacts.outputs.token }}
+          DISPATCH_TOKEN: ${{ steps.token.outputs.token }}
+          BUILD: ${{ needs.plan.outputs.build }}
+          DISTROS: ${{ inputs.ros_distros }}
+          CHANNEL: edge
+          PUBLISHER: ${{ github.repository_owner }}/${{ steps.app.outputs.publisher }}
+        run: python3 .checks/release/hand_to_publisher.py

.github/workflows/reusable_release_build.yml

@@ -1,7 +1,8 @@
 name: release build
 # Builds the .deb for one package at a tag with duatic_devtools' builder, for amd64 and arm64 unless
 # architectures says otherwise, and uploads one artifact per architecture. Signing and publishing
-# belong to the archive host, not to a runner.
+# belong to the archive host, not to a runner. A build of main names its commit as the tag and
+# gives version_suffix, which is appended to the version in package.xml.
 #
 # Dispatch it here with a repository, a tag and a package. The Ubuntu base to build against is
 # derived from ros_distro, since each ROS 2 distro has exactly one supported Ubuntu base.
@@ -41,6 +42,10 @@ on:
         description: 'Report a failed build as a warning and upload nothing, rather than failing the run.'
         default: false
         type: boolean
+      version_suffix:
+        description: 'Appended to the version in package.xml, e.g. "+git20261027153012.ab12cd3" for a build of main'
+        default: ''
+        type: string
   workflow_call:
     inputs:
       repository:
@@ -76,6 +81,11 @@ on:
         default: false
         required: false
         type: boolean
+      version_suffix:
+        description: 'Appended to the version in package.xml, e.g. "+git20261027153012.ab12cd3" for a build of main'
+        default: ''
+        required: false
+        type: string
     outputs:
       artifact:
         description: 'Artifact name without its architecture: one artifact per architecture built, <artifact>-<arch>'
@@ -112,11 +122,13 @@ jobs:
           TAG: ${{ inputs.tag }}
           ROS_DISTRO_TARGET: ${{ inputs.ros_distro }}
           ARCH: ${{ matrix.arch }}
+          SUFFIX: ${{ inputs.version_suffix }}
         run: |
           mkdir -p "$RUNNER_TEMP/out"
           docker run --rm -i --platform "linux/$ARCH" \
             -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \
             -e ROS_DISTRO="$ROS_DISTRO_TARGET" \
+            -e DEB_VERSION_SUFFIX="$SUFFIX" \
             -v "$PWD/src:/src" \
             -v "$PWD/.devtools/packaging/builder:/builder" \
             -v "$RUNNER_TEMP/out:/out" \
@@ -125,6 +137,16 @@ jobs:
           # Fail unless a .deb of this architecture was produced.
           ls "$RUNNER_TEMP"/out/*_"$ARCH".deb >/dev/null \
             || { echo "::error::no ${ARCH} .deb produced"; ls "$RUNNER_TEMP/out"; exit 1; }
+          # A builder that ignored the suffix would produce the release's version.
+          if [ -n "$SUFFIX" ]; then
+            for deb in "$RUNNER_TEMP"/out/*.deb; do
+              v="$(dpkg-deb -f "$deb" Version)"
+              case "$v" in
+                *"$SUFFIX"-*) ;;
+                *) echo "::error::$(basename "$deb") is version $v, without $SUFFIX"; exit 1 ;;
+              esac
+            done
+          fi
           echo "artifact=${PACKAGE}-${TAG//\//-}-${ROS_DISTRO_TARGET}" >> "$GITHUB_OUTPUT"
 
       # Debug symbols are a .ddeb, which a *.deb glob misses. build_pkg.sh collects them and the

.github/workflows/reusable_release_publish.yml

@@ -23,6 +23,11 @@ on:
         default: 'jazzy'
         required: false
         type: string
+      channel:
+        description: 'Archive channel to publish into: stable for releases, edge for builds of main'
+        default: 'stable'
+        required: false
+        type: string
       runner:
         description: 'Runner label(s) to run the job on, e.g. "ubuntu-latest" or "self-hosted".'
         default: 'ubuntu-latest'
@@ -40,16 +45,10 @@ on:
       EXPECTED_KEY_FPR:
         required: true
 
-# aptly's database is single-writer and the routing lists and dist tree are shared, so two
-# releases publishing at once corrupt each other rather than queueing.
-concurrency:
-  group: publish-${{ inputs.root }}
-  cancel-in-progress: false
-
 jobs:
   publish:
     runs-on: ${{ inputs.runner }}
-    timeout-minutes: 15
+    timeout-minutes: 30
     steps:
       - uses: actions/download-artifact@v4
         with:
@@ -64,6 +63,7 @@ jobs:
           ARTIFACT: ${{ inputs.artifact }}
           ROOT: ${{ inputs.root }}
           DISTRO: ${{ inputs.distro }}
+          CHANNEL: ${{ inputs.channel }}
         run: |
           count="$(find deb -name '*.deb' | wc -l)"
           # One package per call. build_pkg.sh collects every .deb the build produced, and
@@ -77,7 +77,7 @@ jobs:
           version="${rest%%_*}"
           # These are interpolated into a command run on the host that holds the signing key, so
           # anything outside this set is refused rather than escaped.
-          for field in "$package" "$version" "$ROOT" "$DISTRO"; do
+          for field in "$package" "$version" "$ROOT" "$DISTRO" "$CHANNEL"; do
             printf '%s' "$field" | grep -qE '^[A-Za-z0-9][A-Za-z0-9._/+~-]*$' \
               || { echo "FATAL: refusing unexpected characters in '$field'" >&2; exit 1; }
           done
@@ -90,7 +90,7 @@ jobs:
       # Self-hosted runner images do not always carry one.
       - name: Ensure an SSH client is available
         run: |
-          command -v ssh >/dev/null && command -v scp >/dev/null && exit 0
+          command -v ssh >/dev/null && command -v sftp >/dev/null && exit 0
           sudo apt-get update -qq && sudo apt-get install -y -qq openssh-client
 
       - name: Configure SSH to the publish host
@@ -112,6 +112,7 @@ jobs:
           VERSION: ${{ steps.deb.outputs.version }}
           ROOT: ${{ inputs.root }}
           DISTRO: ${{ inputs.distro }}
+          CHANNEL: ${{ inputs.channel }}
           HOST: ${{ secrets.PUBLISH_HOST }}
           PUBLISH_USER: ${{ secrets.PUBLISH_USER }}
           FPR: ${{ secrets.EXPECTED_KEY_FPR }}
@@ -120,14 +121,23 @@ jobs:
           # Everything the build produced, not just the .deb the version was read from: the
           # debug symbols ship as a .ddeb and the archive indexes them alongside, and an open
           # root publishes a source tarball beside the package that declares one.
-          scp deb/* "$target:~/archive/out/"
+          # Each file lands under a hidden name and is renamed into place, so a publish already
+          # running on the host never takes one half written.
+          batch="$RUNNER_TEMP/sftp-batch"
+          : > "$batch"
+          for f in deb/*; do
+            b="$(basename "$f")"
+            printf 'put "%s" "archive/out/.%s.part"\nrename "archive/out/.%s.part" "archive/out/%s"\n' \
+              "$f" "$b" "$b" "$b" >> "$batch"
+          done
+          sftp -q -b "$batch" "$target"
           # Named explicitly: a non-interactive ssh reads no profile, so nothing the publish
           # host exports in a login shell reaches this command.
           # Expanded here rather than on the host: every value is checked against a
           # conservative character set above, and the host must receive literals.
           # shellcheck disable=SC2029
           ssh "$target" \
-            "EXPECTED_KEY_FPR='$FPR' DISTRO='$DISTRO' \
+            "EXPECTED_KEY_FPR='$FPR' DISTRO='$DISTRO' CHANNEL='$CHANNEL' \
              ~/duatic_devtools/packaging/archive/add_and_publish.sh \
              '${PACKAGE}:${ROOT}:${VERSION}'"