From f83a926edbba520f88627809ff7e8777957aba50 Mon Sep 17 00:00:00 2001 From: Vaios Papaspyros <8146703+bpapaspyros@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:42:08 +0100 Subject: [PATCH] fix(builder): read an archive suite not published yet as empty --- .github/workflows/ci.yml | 4 ++ packaging/builder/README.md | 2 + packaging/builder/build_pkg.sh | 26 +++++++- packaging/tests/README.md | 1 + packaging/tests/archive_build_test.sh | 93 +++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 1 deletion(-) create mode 100755 packaging/tests/archive_build_test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 634d75a..943971b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,10 @@ jobs: - name: Archive dependencies map to their Debian names run: packaging/tests/archive_rosdep_test.sh + - name: Build against a signed archive with a suite missing + if: matrix.ros_distro == 'jazzy' + run: packaging/tests/archive_build_test.sh + - name: Generate the rosdep mapping for ${{ matrix.ros_distro }} env: ROS_DISTROS: ${{ matrix.ros_distro }} diff --git a/packaging/builder/README.md b/packaging/builder/README.md index e1d08e5..d7a15c8 100644 --- a/packaging/builder/README.md +++ b/packaging/builder/README.md @@ -35,6 +35,8 @@ rosdep to its Debian name, and `mk-build-deps` installs build dependencies from `ARCHIVE_KEYRING`, the archive's public key mounted into the container, which apt checks the archive's signature against. `ARCHIVE_CHANNELS` lists the channels read, by default `stable`; a build of main reads `edge stable`, so a dependency without a build of main comes from its release. +A suite the archive does not publish yet is read as empty, and with none published the build goes +ahead without the archive. Any other failure to reach it stops the build. ```bash -e ARCHIVE_URL= \ diff --git a/packaging/builder/build_pkg.sh b/packaging/builder/build_pkg.sh index 137acb8..2d438ae 100755 --- a/packaging/builder/build_pkg.sh +++ b/packaging/builder/build_pkg.sh @@ -119,8 +119,32 @@ echo "--- $PKG found at ${PKG_DIR:-}" if [ -n "${ARCHIVE_URL:-}" ]; then [ -f "${ARCHIVE_KEYRING:-}" ] \ || { echo "FATAL: ARCHIVE_URL needs ARCHIVE_KEYRING, the archive's public key" >&2; exit 1; } + # A suite not published yet is read as empty; any other failure to reach one stops the build. suites="" - for channel in ${ARCHIVE_CHANNELS:-stable}; do suites="$suites $OS_VERSION-$channel"; done + for channel in ${ARCHIVE_CHANNELS:-stable}; do + suite="$OS_VERSION-$channel" + state="$(python3 - "${ARCHIVE_URL%/}/dists/$suite/InRelease" <<'PY' +import sys, urllib.error, urllib.request +url = sys.argv[1] +try: + urllib.request.urlopen(url, timeout=60).close() + print("published") +except urllib.error.HTTPError as e: + if e.code != 404: + sys.exit(f"FATAL: {url}: {e}") + print("missing") +except urllib.error.URLError as e: + if not isinstance(e.reason, FileNotFoundError): + sys.exit(f"FATAL: {url}: {e.reason}") + print("missing") +PY +)" + if [ "$state" = published ]; then suites="$suites $suite"; else echo "--- $suite is not published yet"; fi + done +fi +if [ -n "${ARCHIVE_URL:-}" ] && [ -z "$suites" ]; then + echo "--- no suite of $ARCHIVE_URL is published yet, building without it" +elif [ -n "${ARCHIVE_URL:-}" ]; then echo "--- archive $ARCHIVE_URL,$suites" install -D -m 644 "$ARCHIVE_KEYRING" /etc/apt/keyrings/duatic-archive.gpg apt_archive=/tmp/archive-apt diff --git a/packaging/tests/README.md b/packaging/tests/README.md index e3077b4..0310255 100644 --- a/packaging/tests/README.md +++ b/packaging/tests/README.md @@ -12,6 +12,7 @@ install_test.sh install from the local flat repository open_roots_test.sh open roots refuse dependencies on gated packages, and carry sources publish_gate_test.sh publish-gate passes only permitted roots and distros archive_rosdep_test.sh archive_rosdep.sh maps the dependencies an archive publishes +archive_build_test.sh build_pkg.sh against a signed archive with a suite missing, in docker ``` ## The fixtures diff --git a/packaging/tests/archive_build_test.sh b/packaging/tests/archive_build_test.sh new file mode 100755 index 0000000..c389aaf --- /dev/null +++ b/packaging/tests/archive_build_test.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Copyright 2026 Duatic AG +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that +# the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions, and +# the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions, and +# the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or +# promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# Build duatic_probe_exec with build_pkg.sh against a signed local archive that publishes +# ros-jazzy-duatic-helper-msgs in noble-stable and has no noble-edge suite. Needs docker. +# +# Usage: archive_build_test.sh + +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +docker run --rm -i \ + -v "$HERE/../builder:/builder:ro" \ + -v "$HERE/probes:/probes:ro" \ + ros:jazzy-ros-base bash -s <<'INNER' +set -euo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq && apt-get install -y -qq --no-install-recommends gnupg apt-utils dpkg-dev >/dev/null + +arch="$(dpkg --print-architecture)" +w=/work +mkdir -p "$w/deb/DEBIAN" "$w/archive/pool" "$w/keys" "$w/archive/dists/noble-stable/main/binary-$arch" +printf 'Package: ros-jazzy-duatic-helper-msgs\nVersion: 1.0.0-0noble\nArchitecture: %s\nMaintainer: t \nDescription: test\n' \ + "$arch" > "$w/deb/DEBIAN/control" +dpkg-deb -b "$w/deb" "$w/archive/pool/helper.deb" >/dev/null + +export GNUPGHOME="$w/gnupg" +mkdir -m 700 "$GNUPGHOME" +gpg --batch --passphrase '' --quick-generate-key "archive " rsa2048 sign 1d 2>/dev/null +gpg --batch --passphrase '' --quick-generate-key "other " rsa2048 sign 1d 2>/dev/null +gpg --export archive > "$w/keys/archive.gpg" +gpg --export other > "$w/keys/other.gpg" +( cd "$w/archive" && apt-ftparchive packages pool > "dists/noble-stable/main/binary-$arch/Packages" ) +( cd "$w/archive/dists/noble-stable" \ + && apt-ftparchive -o APT::FTPArchive::Release::Suite=noble-stable \ + -o APT::FTPArchive::Release::Components=main -o APT::FTPArchive::Release::Architectures="$arch" \ + release . > Release \ + && gpg --batch --yes --local-user archive --clearsign -o InRelease Release ) + +pass=0 fail=0 +# check : build_pkg.sh for duatic_probe_exec with ends +# as expected, with a line of its log, or the Depends of the .deb it built, matching the pattern. +check() { + local name="$1" expect="$2" pattern="$3" got=built; shift 3 + rm -r -f /src /out /etc/apt/sources.list.d/duatic-archive.sources /etc/ros/rosdep/sources.list.d/15-archive.list + cp -a /probes /src + mkdir /out + env "$@" /builder/build_pkg.sh duatic_probe_exec > /tmp/build.log 2>&1 < /dev/null || got=refused + [ "$got" = refused ] || echo "Depends: $(dpkg-deb -f /out/*.deb Depends)" >> /tmp/build.log + if [ "$got" = "$expect" ] && grep -qE "$pattern" /tmp/build.log; then + pass=$((pass + 1)); echo "ok $name" + else + fail=$((fail + 1)); echo "FAIL $name: $got, expected $expect with '$pattern'" + grep -v '^Skip distro' /tmp/build.log | tail -15 + fi +} + +archive=(ARCHIVE_URL="file://$w/archive" ARCHIVE_KEYRING="$w/keys/archive.gpg") +check "a dependency resolves from stable" built "^Depends: .*ros-jazzy-duatic-helper-msgs" "${archive[@]}" +check "a missing edge suite is read as empty" built "noble-edge is not published yet" \ + "${archive[@]}" ARCHIVE_CHANNELS="edge stable" +check "with no suite published, the archive maps nothing" refused "no suite of .* is published yet" \ + "${archive[@]}" ARCHIVE_CHANNELS="edge" +check "a key that did not sign it is refused" refused "NO_PUBKEY|not signed|signature" \ + ARCHIVE_URL="file://$w/archive" ARCHIVE_KEYRING="$w/keys/other.gpg" +check "an archive that cannot be reached is refused" refused "FATAL: http://archive.invalid" \ + ARCHIVE_URL="http://archive.invalid" ARCHIVE_KEYRING="$w/keys/archive.gpg" + +echo "$pass passed, $fail failed" +[ "$fail" = 0 ] +INNER