From 10ec546b4c21d3548e2031ecf0b1853baac460a6 Mon Sep 17 00:00:00 2001
From: Tyler Jang <tyler@trunk.io>
Date: Fri, 30 Oct 2026 21:44:40 +0000
Subject: [PATCH 1/4] feat(cli): release for ppc64le and s390x Linux

Cross-compile trunk-analytics-cli for powerpc64le-unknown-linux-gnu and
s390x-unknown-linux-gnu, smoke-test both under QEMU, and build them on
every PR. Both link glibc dynamically; Cross.toml pins the cross 0.2.5
images so the floor stays at glibc 2.22/2.18 rather than 2.39.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../action.yaml                               | 29 ++++++++++++-
 .github/workflows/pull_request.yml            |  8 ++++
 .github/workflows/release_cli.yml             | 43 +++++++++++++++++--
 CONTRIBUTING.md                               | 29 +++++++++++++
 Cross.toml                                    |  7 +++
 rust-toolchain.toml                           |  2 +
 6 files changed, 113 insertions(+), 5 deletions(-)

diff --git a/.github/actions/build_cli_linux_windows_target/action.yaml b/.github/actions/build_cli_linux_windows_target/action.yaml
index 32bbfd9d..62dd030b 100644
--- a/.github/actions/build_cli_linux_windows_target/action.yaml
+++ b/.github/actions/build_cli_linux_windows_target/action.yaml
@@ -20,6 +20,14 @@ runs:
         sudo apt-get update
         sudo apt-get install -y musl-tools
 
+    # glibc, not musl: cross has no musl image for these, and static glibc breaks NSS DNS.
+    - name: Install cross binutils (glibc targets)
+      shell: bash
+      if: ${{ endsWith(inputs.target, '-unknown-linux-gnu') }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y "binutils-$(echo '${{ inputs.target }}' | sed 's/-unknown//')"
+
     - name: Build CLI ${{ inputs.target }} target
       uses: ./.github/actions/build_cli_target
       with:
@@ -33,16 +41,21 @@ runs:
         if [[ "${{ inputs.target }}" == *"windows-gnu"* ]]; then
           # Windows binaries use .exe extension
           x86_64-w64-mingw32-strip target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli.exe
+        elif [[ "${{ inputs.target }}" == *"-unknown-linux-gnu" ]]; then
+          "$(echo '${{ inputs.target }}' | sed 's/-unknown//')-strip" -s target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
         else
           strip -s target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
         fi
 
+    # UPX cannot pack s390x, and its ppc64le stub is unproven on 64 KiB-page kernels.
     - name: Install upx
       shell: bash
+      if: ${{ !endsWith(inputs.target, '-unknown-linux-gnu') }}
       run: |
         sudo apt-get install -y upx
 
     - name: Optimize CLI binary
+      if: ${{ !endsWith(inputs.target, '-unknown-linux-gnu') }}
       uses: ./.github/actions/optimize_cli_binary
       with:
         target: ${{ inputs.target }}
@@ -50,7 +63,7 @@ runs:
 
     - name: Check dynamic deps (Unix)
       shell: bash
-      if: ${{ inputs.target != 'x86_64-unknown-illumos' && !contains(inputs.target, 'windows') }}
+      if: ${{ inputs.target != 'x86_64-unknown-illumos' && !contains(inputs.target, 'windows') && !endsWith(inputs.target, '-unknown-linux-gnu') }}
       run: |
         # Confirm that the binary is statically linked
         set +e
@@ -61,6 +74,20 @@ runs:
         fi
         exit 0
 
+    - name: Check dynamic deps (glibc)
+      shell: bash
+      if: ${{ endsWith(inputs.target, '-unknown-linux-gnu') }}
+      run: |
+        binary=target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
+        needed=$(readelf -d "$binary" | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p')
+        echo "NEEDED: $(echo $needed)"
+        echo "Minimum glibc: $(readelf --dyn-syms -W "$binary" | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1)"
+        unexpected=$(echo "$needed" | grep -vE '^(libc|libm|libgcc_s|libpthread|libdl|librt|ld64)\.so' || true)
+        if [ -n "$unexpected" ]; then
+          echo "trunk-analytics-cli links more than glibc: $unexpected"
+          exit 1
+        fi
+
     - name: Check dynamic deps (Windows)
       shell: bash
       if: ${{ contains(inputs.target, 'windows-gnu') }}
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index be778c31..2c641df8 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -61,6 +61,14 @@ jobs:
             runs-on: ubuntu-24.04-arm
             target: aarch64-unknown-linux-musl
 
+          - os-name: linux-ppc64le
+            runs-on: ubuntu-latest
+            target: powerpc64le-unknown-linux-gnu
+
+          - os-name: linux-s390x
+            runs-on: ubuntu-latest
+            target: s390x-unknown-linux-gnu
+
           - os-name: x86_64-darwin
             runs-on: macos-latest
             target: x86_64-apple-darwin
diff --git a/.github/workflows/release_cli.yml b/.github/workflows/release_cli.yml
index bef23c7c..36268527 100644
--- a/.github/workflows/release_cli.yml
+++ b/.github/workflows/release_cli.yml
@@ -41,6 +41,19 @@ jobs:
             runs-on: ubuntu-latest
             target: x86_64-unknown-illumos
             artifact-name: x86_64-unknown-illumos
+            skip-cli-tools: true
+
+          - os-name: linux-ppc64le
+            runs-on: ubuntu-latest
+            target: powerpc64le-unknown-linux-gnu
+            artifact-name: powerpc64le-unknown-linux
+            skip-cli-tools: true
+
+          - os-name: linux-s390x
+            runs-on: ubuntu-latest
+            target: s390x-unknown-linux-gnu
+            artifact-name: s390x-unknown-linux
+            skip-cli-tools: true
 
           - os-name: x86_64-darwin
             runs-on: macos-latest
@@ -115,7 +128,7 @@ jobs:
       #########################################################
       - name: Build CLI Tools
         uses: ./.github/actions/build_cli_tools
-        if: matrix.platform.target != 'x86_64-unknown-illumos'
+        if: ${{ !matrix.platform.skip-cli-tools }}
         with:
           target: ${{ matrix.platform.target }}
           os-name: ${{ matrix.platform.os-name }}
@@ -153,7 +166,7 @@ jobs:
 
       - name: Upload Bundle CLI artifact (Unix)
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.platform.target != 'x86_64-unknown-illumos' && !contains(matrix.platform.os-name, 'windows') }}
+        if: ${{ !matrix.platform.skip-cli-tools && !contains(matrix.platform.os-name, 'windows') }}
         with:
           name: ${{ matrix.platform.artifact-name }}-bundle
           path: target/${{ matrix.platform.target }}/release-with-debug/proto-bin-to-json
@@ -169,7 +182,7 @@ jobs:
 
       - name: Upload XCResult CLI artifact (Unix)
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.platform.target != 'x86_64-unknown-illumos' && !contains(matrix.platform.os-name, 'windows') }}
+        if: ${{ !matrix.platform.skip-cli-tools && !contains(matrix.platform.os-name, 'windows') }}
         with:
           name: ${{ matrix.platform.artifact-name }}-xcresult
           path: target/${{ matrix.platform.target }}/release-with-debug/xcresult-to-junit
@@ -185,7 +198,7 @@ jobs:
 
       - name: Upload Codeowners CLI artifact (Unix)
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.platform.target != 'x86_64-unknown-illumos' && !contains(matrix.platform.os-name, 'windows') }}
+        if: ${{ !matrix.platform.skip-cli-tools && !contains(matrix.platform.os-name, 'windows') }}
         with:
           name: ${{ matrix.platform.artifact-name }}-codeowners
           path: target/${{ matrix.platform.target }}/release-with-debug/check-codeowners
@@ -206,6 +219,17 @@ jobs:
           - name: windows
             os: windows-latest
             target: x86_64-pc-windows-gnu
+          # GitHub has no ppc64le or s390x runners, so these run under QEMU user-mode.
+          - name: linux-ppc64le
+            os: ubuntu-latest
+            target: powerpc64le-unknown-linux
+            qemu-triple: powerpc64le-linux-gnu
+            qemu-deb-arch: ppc64el
+          - name: linux-s390x
+            os: ubuntu-latest
+            target: s390x-unknown-linux
+            qemu-triple: s390x-linux-gnu
+            qemu-deb-arch: s390x
     needs:
       - build_release
     steps:
@@ -218,6 +242,17 @@ jobs:
           name: ${{ matrix.type.target }}
           path: ./trunk-analytics-cli
 
+      - name: Set up QEMU user-mode emulation
+        if: ${{ matrix.type.qemu-triple }}
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-user-static binfmt-support \
+            libc6-${{ matrix.type.qemu-deb-arch }}-cross libgcc-s1-${{ matrix.type.qemu-deb-arch }}-cross
+          echo "QEMU_LD_PREFIX=/usr/${{ matrix.type.qemu-triple }}" >> "$GITHUB_ENV"
+          chmod +x trunk-analytics-cli/trunk-analytics-cli
+          QEMU_LD_PREFIX=/usr/${{ matrix.type.qemu-triple }} trunk-analytics-cli/trunk-analytics-cli --version
+
       - name: Setup Run Smoke Tests
         uses: ./.github/actions/setup_run_smoke_tests
 
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 5289cf7c..4537a8d1 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -17,6 +17,35 @@ cargo build
 
 The CLI will be built to `target/debug/trunk-analytics-cli`
 
+### Cross-Compiling for Other Linux Architectures
+
+Release targets your machine cannot build natively (`powerpc64le-unknown-linux-gnu`, `s390x-unknown-linux-gnu`, `x86_64-unknown-illumos`) build in Docker with [cross](https://github.com/cross-rs/cross):
+
+```bash
+cargo install cross --git https://github.com/cross-rs/cross
+cross build -p trunk-analytics-cli --profile=release --target=s390x-unknown-linux-gnu
+```
+
+Build only `-p trunk-analytics-cli`, as CI does; building the workspace links libraries the release binary must not need.
+
+The ppc64le and s390x builds link glibc dynamically, and `Cross.toml` pins their images to old ones so they run on RHEL 8 and SLES 15. Check the floor after changing an image or dependency:
+
+```bash
+readelf --dyn-syms -W target/s390x-unknown-linux-gnu/release/trunk-analytics-cli | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1
+```
+
+Use a fresh `CARGO_TARGET_DIR` when switching a target's image: build scripts compiled in one image may not run in another.
+
+To try a binary, run it under the QEMU that ships in its cross image:
+
+```bash
+docker run --rm -v "$PWD:/repo" -w /repo ghcr.io/cross-rs/s390x-unknown-linux-gnu:0.2.5 \
+  /linux-runner s390x target/s390x-unknown-linux-gnu/release/trunk-analytics-cli \
+  validate --junit-paths cli/test_fixtures/junit0_fail.xml
+```
+
+s390x is big-endian, so it is the target most likely to show a byte-order bug.
+
 ### Package-Specific Build Instructions
 
 For detailed build instructions for each supported package, see their respective README files:
diff --git a/Cross.toml b/Cross.toml
index 09809e19..11a19d7f 100644
--- a/Cross.toml
+++ b/Cross.toml
@@ -2,3 +2,10 @@
 passthrough = [
   "RANLIB=/usr/local/x86_64-unknown-illumos/bin/x86_64-unknown-illumos-ranlib",
 ]
+
+# cross's main images moved to Ubuntu 24.04 (glibc 2.39); RHEL 8/9 and SLES 15 run older.
+[target.powerpc64le-unknown-linux-gnu]
+image = "ghcr.io/cross-rs/powerpc64le-unknown-linux-gnu:0.2.5"
+
+[target.s390x-unknown-linux-gnu]
+image = "ghcr.io/cross-rs/s390x-unknown-linux-gnu:0.2.5"
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index f28b119a..35c338ed 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -13,5 +13,7 @@ targets = [
   "arm-unknown-linux-gnueabihf",
   "wasm32-unknown-unknown",
   "x86_64-unknown-illumos",
+  "powerpc64le-unknown-linux-gnu",
+  "s390x-unknown-linux-gnu",
   "x86_64-pc-windows-gnu",
 ]

From a1acbecc2d8d824791104e3fd1a9458014ac00bb Mon Sep 17 00:00:00 2001
From: Tyler Jang <tyler@trunk.io>
Date: Fri, 30 Oct 2026 21:55:19 +0000
Subject: [PATCH 2/4] fix(cli): enforce the glibc ceiling and drop the
 toolchain targets

cross installs a missing target itself, so rust-toolchain.toml does not
need ppc64le and s390x. The glibc check now fails above 2.28 (RHEL 8)
instead of only logging the floor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../build_cli_linux_windows_target/action.yaml        | 11 +++++++++--
 CONTRIBUTING.md                                       |  8 +-------
 rust-toolchain.toml                                   |  2 --
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/.github/actions/build_cli_linux_windows_target/action.yaml b/.github/actions/build_cli_linux_windows_target/action.yaml
index 62dd030b..aa1a6f6c 100644
--- a/.github/actions/build_cli_linux_windows_target/action.yaml
+++ b/.github/actions/build_cli_linux_windows_target/action.yaml
@@ -47,7 +47,7 @@ runs:
           strip -s target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
         fi
 
-    # UPX cannot pack s390x, and its ppc64le stub is unproven on 64 KiB-page kernels.
+    # UPX cannot pack s390x, and no packed ppc64le binary has been run.
     - name: Install upx
       shell: bash
       if: ${{ !endsWith(inputs.target, '-unknown-linux-gnu') }}
@@ -81,12 +81,19 @@ runs:
         binary=target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
         needed=$(readelf -d "$binary" | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p')
         echo "NEEDED: $(echo $needed)"
-        echo "Minimum glibc: $(readelf --dyn-syms -W "$binary" | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1)"
         unexpected=$(echo "$needed" | grep -vE '^(libc|libm|libgcc_s|libpthread|libdl|librt|ld64)\.so' || true)
         if [ -n "$unexpected" ]; then
           echo "trunk-analytics-cli links more than glibc: $unexpected"
           exit 1
         fi
+        # RHEL 8's glibc; Cross.toml's pinned images keep the build below it.
+        ceiling=2.28
+        floor=$(readelf --dyn-syms -W "$binary" | grep -o 'GLIBC_[0-9.]*' | sed 's/GLIBC_//' | sort -uV | tail -1)
+        echo "Minimum glibc: $floor"
+        if [ "$(printf '%s\n' "$floor" "$ceiling" | sort -V | tail -1)" != "$ceiling" ]; then
+          echo "trunk-analytics-cli needs glibc $floor, newer than $ceiling"
+          exit 1
+        fi
 
     - name: Check dynamic deps (Windows)
       shell: bash
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4537a8d1..2ad6307c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -28,13 +28,7 @@ cross build -p trunk-analytics-cli --profile=release --target=s390x-unknown-linu
 
 Build only `-p trunk-analytics-cli`, as CI does; building the workspace links libraries the release binary must not need.
 
-The ppc64le and s390x builds link glibc dynamically, and `Cross.toml` pins their images to old ones so they run on RHEL 8 and SLES 15. Check the floor after changing an image or dependency:
-
-```bash
-readelf --dyn-syms -W target/s390x-unknown-linux-gnu/release/trunk-analytics-cli | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1
-```
-
-Use a fresh `CARGO_TARGET_DIR` when switching a target's image: build scripts compiled in one image may not run in another.
+The ppc64le and s390x builds link glibc dynamically, and `Cross.toml` pins their images to old ones so they run on RHEL 8; CI fails a build that needs glibc newer than 2.28. Use a fresh `CARGO_TARGET_DIR` when switching a target's image: build scripts compiled in one image may not run in another.
 
 To try a binary, run it under the QEMU that ships in its cross image:
 
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 35c338ed..f28b119a 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -13,7 +13,5 @@ targets = [
   "arm-unknown-linux-gnueabihf",
   "wasm32-unknown-unknown",
   "x86_64-unknown-illumos",
-  "powerpc64le-unknown-linux-gnu",
-  "s390x-unknown-linux-gnu",
   "x86_64-pc-windows-gnu",
 ]

From db029c3d3593bbe2bc47edae9ed061b19bf8bba7 Mon Sep 17 00:00:00 2001
From: Tyler Jang <tyler@trunk.io>
Date: Fri, 30 Oct 2026 22:39:51 +0000
Subject: [PATCH 3/4] fix(cli): link ppc64le through zig so it runs on glibc
 2.34+

Linked in the cross 0.2.5 image, the ppc64le binary dies with SIGILL at
startup on glibc 2.34 and newer (RHEL 9, Ubuntu 24.04). Link it with
zig against glibc 2.28 instead. The cross 0.2.5 release ignores the zig
setting, so the glibc targets build with cross at a pinned commit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/actions/build_cli_target/action.yaml | 2 ++
 CONTRIBUTING.md                              | 2 +-
 Cross.toml                                   | 6 ++++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/actions/build_cli_target/action.yaml b/.github/actions/build_cli_target/action.yaml
index f0407856..4a95fe43 100644
--- a/.github/actions/build_cli_target/action.yaml
+++ b/.github/actions/build_cli_target/action.yaml
@@ -60,6 +60,8 @@ runs:
         # NOTE: DO NOT BUILD WORKSPACE OTHERWISE THE CLI MAY BE LINKED TO UNNECESSARY LIBRARIES
         args: "-p trunk-analytics-cli --profile=${{ inputs.profile }} --target=${{ inputs.target }} ${{ (inputs.force-sentry-dev == 'true' && '--features force-sentry-env-dev') || '' }}"
         use-rust-cache: false
+        # The latest cross release (0.2.5) ignores Cross.toml's `zig`, which ppc64le needs.
+        cross-version: ${{ endsWith(inputs.target, '-unknown-linux-gnu') && 'd5a5ead7397be7d49db1b12ed1a6f3583d42fb98' || '' }}
 
     - name: Build CLI ${{ inputs.target }} target
       shell: bash
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2ad6307c..2cc6bf24 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -28,7 +28,7 @@ cross build -p trunk-analytics-cli --profile=release --target=s390x-unknown-linu
 
 Build only `-p trunk-analytics-cli`, as CI does; building the workspace links libraries the release binary must not need.
 
-The ppc64le and s390x builds link glibc dynamically, and `Cross.toml` pins their images to old ones so they run on RHEL 8; CI fails a build that needs glibc newer than 2.28. Use a fresh `CARGO_TARGET_DIR` when switching a target's image: build scripts compiled in one image may not run in another.
+The ppc64le and s390x builds link glibc dynamically, and `Cross.toml` keeps them at glibc 2.28 or older so they run on RHEL 8 (zig for ppc64le, which the cross 0.2.5 release ignores; an old image for s390x). CI fails a build that needs a newer glibc. Use a fresh `CARGO_TARGET_DIR` when switching a target's image: build scripts compiled in one image may not run in another.
 
 To try a binary, run it under the QEMU that ships in its cross image:
 
diff --git a/Cross.toml b/Cross.toml
index 11a19d7f..16afefd2 100644
--- a/Cross.toml
+++ b/Cross.toml
@@ -3,9 +3,11 @@ passthrough = [
   "RANLIB=/usr/local/x86_64-unknown-illumos/bin/x86_64-unknown-illumos-ranlib",
 ]
 
-# cross's main images moved to Ubuntu 24.04 (glibc 2.39); RHEL 8/9 and SLES 15 run older.
+# cross's main images link against glibc 2.39, newer than RHEL 8/9 and SLES 15. ppc64le links
+# through zig, because a binary linked in the old 0.2.5 image crashes on glibc 2.34 and newer;
+# zig cannot link s390x, whose 0.2.5-image binary runs on every glibc.
 [target.powerpc64le-unknown-linux-gnu]
-image = "ghcr.io/cross-rs/powerpc64le-unknown-linux-gnu:0.2.5"
+zig = "2.28"
 
 [target.s390x-unknown-linux-gnu]
 image = "ghcr.io/cross-rs/s390x-unknown-linux-gnu:0.2.5"

From 45e967dca23b76b85107584e6cf9529c32b50ce7 Mon Sep 17 00:00:00 2001
From: Tyler Jang <tyler@trunk.io>
Date: Fri, 30 Oct 2026 22:58:01 +0000
Subject: [PATCH 4/4] fix(ci): key the Rust build cache per target

The build jobs shared one rust-cache key across every Linux x64 target,
so s390x could restore build scripts compiled for a newer glibc and run
them in its Ubuntu 16.04 image.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/pull_request.yml | 3 +++
 .github/workflows/release_cli.yml  | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 2c641df8..678e70b2 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -108,6 +108,9 @@ jobs:
 
       - name: Setup Rust & Cargo
         uses: ./.github/actions/setup_rust_cargo
+        with:
+          # Targets build in different images; a shared target/ runs one image's build scripts in another.
+          cache-key: ${{ matrix.platform.target }}
 
       - name: Build darwin target
         uses: ./.github/actions/build_cli_macos_target
diff --git a/.github/workflows/release_cli.yml b/.github/workflows/release_cli.yml
index 36268527..80d24030 100644
--- a/.github/workflows/release_cli.yml
+++ b/.github/workflows/release_cli.yml
@@ -87,6 +87,9 @@ jobs:
 
       - name: Setup Rust & Cargo
         uses: ./.github/actions/setup_rust_cargo
+        with:
+          # Targets build in different images; a shared target/ runs one image's build scripts in another.
+          cache-key: ${{ matrix.platform.target }}
 
       - name: Update CLI version
         uses: ./.github/actions/update_version