trunk-merge/pr-1281/e6868de7-3098-4b32-ba22-a36d514361e8

.github/actions/build_cli_linux_windows_target/action.yaml

@@ -20,6 +20,14 @@ runs:
         sudo apt-get update
         sudo apt-get install -y musl-tools
 
+    # glibc, not musl: cross has no musl image for these, and static glibc breaks NSS DNS.
+    - name: Install cross binutils (glibc targets)
+      shell: bash
+      if: ${{ endsWith(inputs.target, '-unknown-linux-gnu') }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y "binutils-$(echo '${{ inputs.target }}' | sed 's/-unknown//')"
+
     - name: Build CLI ${{ inputs.target }} target
       uses: ./.github/actions/build_cli_target
       with:
@@ -33,24 +41,29 @@ runs:
         if [[ "${{ inputs.target }}" == *"windows-gnu"* ]]; then
           # Windows binaries use .exe extension
           x86_64-w64-mingw32-strip target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli.exe
+        elif [[ "${{ inputs.target }}" == *"-unknown-linux-gnu" ]]; then
+          "$(echo '${{ inputs.target }}' | sed 's/-unknown//')-strip" -s target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
         else
           strip -s target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
         fi
 
+    # UPX cannot pack s390x, and no packed ppc64le binary has been run.
     - name: Install upx
       shell: bash
+      if: ${{ !endsWith(inputs.target, '-unknown-linux-gnu') }}
       run: |
         sudo apt-get install -y upx
 
     - name: Optimize CLI binary
+      if: ${{ !endsWith(inputs.target, '-unknown-linux-gnu') }}
       uses: ./.github/actions/optimize_cli_binary
       with:
         target: ${{ inputs.target }}
         profile: ${{ inputs.profile }}
 
     - name: Check dynamic deps (Unix)
       shell: bash
-      if: ${{ inputs.target != 'x86_64-unknown-illumos' && !contains(inputs.target, 'windows') }}
+      if: ${{ inputs.target != 'x86_64-unknown-illumos' && !contains(inputs.target, 'windows') && !endsWith(inputs.target, '-unknown-linux-gnu') }}
       run: |
         # Confirm that the binary is statically linked
         set +e
@@ -61,6 +74,27 @@ runs:
         fi
         exit 0
 
+    - name: Check dynamic deps (glibc)
+      shell: bash
+      if: ${{ endsWith(inputs.target, '-unknown-linux-gnu') }}
+      run: |
+        binary=target/${{ inputs.target }}/${{ inputs.profile }}/trunk-analytics-cli
+        needed=$(readelf -d "$binary" | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p')
+        echo "NEEDED: $(echo $needed)"
+        unexpected=$(echo "$needed" | grep -vE '^(libc|libm|libgcc_s|libpthread|libdl|librt|ld64)\.so' || true)
+        if [ -n "$unexpected" ]; then
+          echo "trunk-analytics-cli links more than glibc: $unexpected"
+          exit 1
+        fi
+        # RHEL 8's glibc; Cross.toml's pinned images keep the build below it.
+        ceiling=2.28
+        floor=$(readelf --dyn-syms -W "$binary" | grep -o 'GLIBC_[0-9.]*' | sed 's/GLIBC_//' | sort -uV | tail -1)
+        echo "Minimum glibc: $floor"
+        if [ "$(printf '%s\n' "$floor" "$ceiling" | sort -V | tail -1)" != "$ceiling" ]; then
+          echo "trunk-analytics-cli needs glibc $floor, newer than $ceiling"
+          exit 1
+        fi
+
     - name: Check dynamic deps (Windows)
       shell: bash
       if: ${{ contains(inputs.target, 'windows-gnu') }}

.github/actions/build_cli_target/action.yaml

@@ -60,6 +60,8 @@ runs:
         # NOTE: DO NOT BUILD WORKSPACE OTHERWISE THE CLI MAY BE LINKED TO UNNECESSARY LIBRARIES
         args: "-p trunk-analytics-cli --profile=${{ inputs.profile }} --target=${{ inputs.target }} ${{ (inputs.force-sentry-dev == 'true' && '--features force-sentry-env-dev') || '' }}"
         use-rust-cache: false
+        # The latest cross release (0.2.5) ignores Cross.toml's `zig`, which ppc64le needs.
+        cross-version: ${{ endsWith(inputs.target, '-unknown-linux-gnu') && 'd5a5ead7397be7d49db1b12ed1a6f3583d42fb98' || '' }}
 
     - name: Build CLI ${{ inputs.target }} target
       shell: bash

.github/workflows/pull_request.yml

@@ -61,6 +61,14 @@ jobs:
             runs-on: ubuntu-24.04-arm
             target: aarch64-unknown-linux-musl
 
+          - os-name: linux-ppc64le
+            runs-on: ubuntu-latest
+            target: powerpc64le-unknown-linux-gnu
+
+          - os-name: linux-s390x
+            runs-on: ubuntu-latest
+            target: s390x-unknown-linux-gnu
+
           - os-name: x86_64-darwin
             runs-on: macos-latest
             target: x86_64-apple-darwin
@@ -100,6 +108,9 @@ jobs:
 
       - name: Setup Rust & Cargo
         uses: ./.github/actions/setup_rust_cargo
+        with:
+          # Targets build in different images; a shared target/ runs one image's build scripts in another.
+          cache-key: ${{ matrix.platform.target }}
 
       - name: Build darwin target
         uses: ./.github/actions/build_cli_macos_target

.github/workflows/release_cli.yml

@@ -41,6 +41,19 @@ jobs:
             runs-on: ubuntu-latest
             target: x86_64-unknown-illumos
             artifact-name: x86_64-unknown-illumos
+            skip-cli-tools: true
+
+          - os-name: linux-ppc64le
+            runs-on: ubuntu-latest
+            target: powerpc64le-unknown-linux-gnu
+            artifact-name: powerpc64le-unknown-linux
+            skip-cli-tools: true
+
+          - os-name: linux-s390x
+            runs-on: ubuntu-latest
+            target: s390x-unknown-linux-gnu
+            artifact-name: s390x-unknown-linux
+            skip-cli-tools: true
 
           - os-name: x86_64-darwin
             runs-on: macos-latest
@@ -74,6 +87,9 @@ jobs:
 
       - name: Setup Rust & Cargo
         uses: ./.github/actions/setup_rust_cargo
+        with:
+          # Targets build in different images; a shared target/ runs one image's build scripts in another.
+          cache-key: ${{ matrix.platform.target }}
 
       - name: Update CLI version
         uses: ./.github/actions/update_version
@@ -115,7 +131,7 @@ jobs:
       #########################################################
       - name: Build CLI Tools
         uses: ./.github/actions/build_cli_tools
-        if: matrix.platform.target != 'x86_64-unknown-illumos'
+        if: ${{ !matrix.platform.skip-cli-tools }}
         with:
           target: ${{ matrix.platform.target }}
           os-name: ${{ matrix.platform.os-name }}
@@ -153,7 +169,7 @@ jobs:
 
       - name: Upload Bundle CLI artifact (Unix)
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.platform.target != 'x86_64-unknown-illumos' && !contains(matrix.platform.os-name, 'windows') }}
+        if: ${{ !matrix.platform.skip-cli-tools && !contains(matrix.platform.os-name, 'windows') }}
         with:
           name: ${{ matrix.platform.artifact-name }}-bundle
           path: target/${{ matrix.platform.target }}/release-with-debug/proto-bin-to-json
@@ -169,7 +185,7 @@ jobs:
 
       - name: Upload XCResult CLI artifact (Unix)
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.platform.target != 'x86_64-unknown-illumos' && !contains(matrix.platform.os-name, 'windows') }}
+        if: ${{ !matrix.platform.skip-cli-tools && !contains(matrix.platform.os-name, 'windows') }}
         with:
           name: ${{ matrix.platform.artifact-name }}-xcresult
           path: target/${{ matrix.platform.target }}/release-with-debug/xcresult-to-junit
@@ -185,7 +201,7 @@ jobs:
 
       - name: Upload Codeowners CLI artifact (Unix)
         uses: actions/upload-artifact@v4
-        if: ${{ matrix.platform.target != 'x86_64-unknown-illumos' && !contains(matrix.platform.os-name, 'windows') }}
+        if: ${{ !matrix.platform.skip-cli-tools && !contains(matrix.platform.os-name, 'windows') }}
         with:
           name: ${{ matrix.platform.artifact-name }}-codeowners
           path: target/${{ matrix.platform.target }}/release-with-debug/check-codeowners
@@ -206,6 +222,17 @@ jobs:
           - name: windows
             os: windows-latest
             target: x86_64-pc-windows-gnu
+          # GitHub has no ppc64le or s390x runners, so these run under QEMU user-mode.
+          - name: linux-ppc64le
+            os: ubuntu-latest
+            target: powerpc64le-unknown-linux
+            qemu-triple: powerpc64le-linux-gnu
+            qemu-deb-arch: ppc64el
+          - name: linux-s390x
+            os: ubuntu-latest
+            target: s390x-unknown-linux
+            qemu-triple: s390x-linux-gnu
+            qemu-deb-arch: s390x
     needs:
       - build_release
     steps:
@@ -218,6 +245,17 @@ jobs:
           name: ${{ matrix.type.target }}
           path: ./trunk-analytics-cli
 
+      - name: Set up QEMU user-mode emulation
+        if: ${{ matrix.type.qemu-triple }}
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y qemu-user-static binfmt-support \
+            libc6-${{ matrix.type.qemu-deb-arch }}-cross libgcc-s1-${{ matrix.type.qemu-deb-arch }}-cross
+          echo "QEMU_LD_PREFIX=/usr/${{ matrix.type.qemu-triple }}" >> "$GITHUB_ENV"
+          chmod +x trunk-analytics-cli/trunk-analytics-cli
+          QEMU_LD_PREFIX=/usr/${{ matrix.type.qemu-triple }} trunk-analytics-cli/trunk-analytics-cli --version
+
       - name: Setup Run Smoke Tests
         uses: ./.github/actions/setup_run_smoke_tests
 

CONTRIBUTING.md

@@ -17,6 +17,29 @@ cargo build
 
 The CLI will be built to `target/debug/trunk-analytics-cli`
 
+### Cross-Compiling for Other Linux Architectures
+
+Release targets your machine cannot build natively (`powerpc64le-unknown-linux-gnu`, `s390x-unknown-linux-gnu`, `x86_64-unknown-illumos`) build in Docker with [cross](https://github.com/cross-rs/cross):
+
+```bash
+cargo install cross --git https://github.com/cross-rs/cross
+cross build -p trunk-analytics-cli --profile=release --target=s390x-unknown-linux-gnu
+```
+
+Build only `-p trunk-analytics-cli`, as CI does; building the workspace links libraries the release binary must not need.
+
+The ppc64le and s390x builds link glibc dynamically, and `Cross.toml` keeps them at glibc 2.28 or older so they run on RHEL 8 (zig for ppc64le, which the cross 0.2.5 release ignores; an old image for s390x). CI fails a build that needs a newer glibc. Use a fresh `CARGO_TARGET_DIR` when switching a target's image: build scripts compiled in one image may not run in another.
+
+To try a binary, run it under the QEMU that ships in its cross image:
+
+```bash
+docker run --rm -v "$PWD:/repo" -w /repo ghcr.io/cross-rs/s390x-unknown-linux-gnu:0.2.5 \
+  /linux-runner s390x target/s390x-unknown-linux-gnu/release/trunk-analytics-cli \
+  validate --junit-paths cli/test_fixtures/junit0_fail.xml
+```
+
+s390x is big-endian, so it is the target most likely to show a byte-order bug.
+
 ### Package-Specific Build Instructions
 
 For detailed build instructions for each supported package, see their respective README files:

Cross.toml

@@ -2,3 +2,12 @@
 passthrough = [
   "RANLIB=/usr/local/x86_64-unknown-illumos/bin/x86_64-unknown-illumos-ranlib",
 ]
+
+# cross's main images link against glibc 2.39, newer than RHEL 8/9 and SLES 15. ppc64le links
+# through zig, because a binary linked in the old 0.2.5 image crashes on glibc 2.34 and newer;
+# zig cannot link s390x, whose 0.2.5-image binary runs on every glibc.
+[target.powerpc64le-unknown-linux-gnu]
+zig = "2.28"
+
+[target.s390x-unknown-linux-gnu]
+image = "ghcr.io/cross-rs/s390x-unknown-linux-gnu:0.2.5"

cli/src/summary_output/github.rs

@@ -10,8 +10,9 @@ use super::{
     ErrorSummary, FailedTest, Outcome, Quarantining, UploadReport,
     json::Summary,
     text::{
-        FAILED_WITHOUT_RUNNING_TESTS, FAILURE_MESSAGE_MAX_LINES, NO_TEST_FAILURES, NO_TESTS_FOUND,
-        QUARANTINE_LOOKUP_FAILED, SOME_TESTS_MAY_NOT_HAVE_RUN, UNAUTHORIZED_HINT,
+        BAZEL_BEP_HINT, FAILED_WITHOUT_RUNNING_TESTS, FAILURE_MESSAGE_MAX_LINES, NO_TEST_FAILURES,
+        NO_TESTS_FOUND, QUARANTINE_LOOKUP_FAILED, SOME_TESTS_MAY_NOT_HAVE_RUN, UNAUTHORIZED_HINT,
+        old_timestamps_message,
     },
 };
 use crate::{
@@ -74,6 +75,12 @@ fn upload_annotations(report: &UploadReport) -> Vec<Annotation> {
         }
         annotations.push(Annotation::warning(error_message(error)));
     }
+    if report.files_with_old_timestamps > 0 {
+        annotations.push(Annotation::warning(format!(
+            "{}, but the test counts include them.\n{BAZEL_BEP_HINT}",
+            old_timestamps_message(report.files_with_old_timestamps)
+        )));
+    }
 
     let exit_code = report.cli_exit_code();
     match report.outcome {

cli/src/summary_output/text.rs

@@ -1,5 +1,6 @@
 use std::path::{Path, PathBuf};
 
+use context::junit::validator::TIMESTAMP_OLD_HOURS;
 use pluralizer::pluralize;
 use superconsole::{
     Hyperlink, Line, Lines, Span, SuperConsole,
@@ -81,6 +82,16 @@ pub(crate) const SOME_TESTS_MAY_NOT_HAVE_RUN: &str = "This may indicate that som
 pub(crate) const QUARANTINE_LOOKUP_FAILED: &str = "We were unable to determine the quarantine status for tests. Any failing tests will be reported as failures";
 pub(crate) const FAILURE_MESSAGE_MAX_LINES: usize = 20;
 pub(crate) const UNAUTHORIZED_HINT: &str = "Hint: You can find it under the settings page at";
+pub(crate) const BAZEL_BEP_HINT: &str = "If Bazel restored these files from its cache, upload with --bazel-bep-path to skip cached results: https://docs.trunk.io/flaky-tests/get-started/frameworks/bazel";
+
+/// Shown whatever `--validation-report` says, since Trunk's ingestion drops these files' results.
+pub(crate) fn old_timestamps_message(files: usize) -> String {
+    format!(
+        "{} {} test results older than {TIMESTAMP_OLD_HOURS} hours. Trunk will drop these results",
+        pluralize("file", files as isize, true),
+        if files == 1 { "has" } else { "have" },
+    )
+}
 
 pub fn render_error(error: &ErrorSummary) -> anyhow::Result<Vec<Line>> {
     let mut lines = Vec::new();
@@ -209,6 +220,12 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re
                 .attribute(Attribute::Bold),
         )?]));
     }
+    if report.files_with_old_timestamps > 0 {
+        output.extend(render_old_timestamps(format!(
+            "{}, but the test counts above include them.",
+            old_timestamps_message(report.files_with_old_timestamps)
+        ))?);
+    }
     output.push(Line::default());
 
     if quarantined > 0 {
@@ -310,6 +327,17 @@ pub fn render_upload(report: &UploadReport, options: &TextOptions) -> anyhow::Re
     Ok(output)
 }
 
+fn render_old_timestamps(message: String) -> anyhow::Result<Vec<Line>> {
+    Ok(vec![
+        Line::from_iter([Span::new_styled_lossy(
+            style(format!("   ⚠️  {message}"))
+                .with(Color::Yellow)
+                .attribute(Attribute::Bold),
+        )]),
+        Line::from_iter([Span::new_unstyled(format!("   {BAZEL_BEP_HINT}"))?]),
+    ])
+}
+
 fn render_groups(groups: &[FailureGroup], options: &TextOptions) -> anyhow::Result<Lines> {
     let mut output = Vec::new();
     for group in groups {
@@ -663,6 +691,7 @@ pub fn render_validate(report: &ValidateReport) -> anyhow::Result<Vec<Line>> {
         files_with_errors,
         warnings,
         errors,
+        files_with_old_timestamps,
     } = report.counts;
     output.push(Line::from_iter([
         Span::new_styled(style(format!("{}", valid_files)).with(Color::Green))?,
@@ -694,6 +723,13 @@ pub fn render_validate(report: &ValidateReport) -> anyhow::Result<Vec<Line>> {
         Span::new_styled(style(format!("{}", errors)).with(Color::Red))?,
         Span::new_unstyled(format!(" {}", pluralize("error", errors as isize, false)))?,
     ]));
+    if files_with_old_timestamps > 0 {
+        output.push(Line::default());
+        output.extend(render_old_timestamps(format!(
+            "{} when they're uploaded.",
+            old_timestamps_message(files_with_old_timestamps)
+        ))?);
+    }
 
     Ok(output)
 }