From 2340b4f0b5868faaeeee611a800df2af438c3308 Mon Sep 17 00:00:00 2001 From: Petr Fedchenkov Date: Thu, 3 Sep 2026 09:58:37 +0300 Subject: [PATCH 1/2] NGSOK-1746 Deploy Maven from the tag push, and PySpark without hadoop-cloud Packages published by a release-triggered run are linked to that release and listed among its assets, so the Maven deploy moves to the tag push and the release event keeps only the tarballs. The PySpark sdist moves to the plain hadoop3 leg, since it packages the assembly jars and was shipping the AWS SDK bundle, and the CI guard now reports a still-running run as running rather than as missing. A new job attaches the CycloneDX SBOM and its grype vulnerability report, as the kyuubi release workflow does. --- .github/workflows/publish.yml | 161 +++++++++++++++++++++------------- 1 file changed, 98 insertions(+), 63 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c4a989381a1e5..2866de008b4dc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,10 @@ name: Publish on: + # Maven deploys from the tag push, not from `release`: a package published by a + # release-triggered run gets listed among that release's assets. + push: + tags: ['v*'] release: types: [created] workflow_dispatch: @@ -25,7 +29,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: false permissions: @@ -34,17 +38,13 @@ permissions: env: JAVA_VERSION: '17' PYTHON_VERSION: '3.10' - # Profiles shared by the Maven deploy and by every binary distribution. - # -Phadoop-cloud is deliberately NOT here: it pulls the AWS SDK bundle into the - # assembly and roughly doubles the tarball, so only the `hadoop3-cloud` - # distribution (and the Maven deploy, which publishes spark-hadoop-cloud) adds it. + # -Phadoop-cloud is added per job: it roughly doubles the tarball, so only the + # `hadoop3-cloud` leg and the Maven deploy take it. SPARK_PROFILES: -Pscala-2.13 -Phadoop-3 -Phive -Phive-thriftserver -Pyarn -Pkubernetes -Pconnect -Pvolcano jobs: - # Everything that can reject the release runs here, once and cheaply, so a bad - # tag fails in a minute instead of two hours into a reactor build. All three - # checks are no-ops when there is no release tag, which is the dispatch path - # used to smoke-test the workflow itself. + # Cheap checks first, so a bad tag fails in a minute rather than hours into the + # reactor. All of them are no-ops when there is no release tag. verify: name: "Verify the release commit" runs-on: ubuntu-24.04 @@ -53,14 +53,12 @@ jobs: contents: read actions: read env: - RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }} + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }} steps: - uses: actions/checkout@v6 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || '' }} - # `git branch --contains` needs the remote-tracking branches, which - # only a full-history checkout provides. - fetch-depth: 0 + fetch-depth: 0 # `git branch --contains` needs remote-tracking branches - uses: actions/setup-java@v5 with: @@ -89,25 +87,28 @@ jobs: run: | if [[ -z "$RELEASE_TAG" ]]; then echo "No release tag: skipping."; exit 0; fi SHA="$(git rev-parse HEAD)" - CONCLUSION="$(gh run list --commit "$SHA" --workflow CI --json conclusion -q '.[0].conclusion')" - if [[ -z "$CONCLUSION" ]]; then + RUN="$(gh run list --commit "$SHA" --workflow CI --limit 1 --json status,conclusion)" + STATUS="$(jq -r '.[0].status // ""' <<< "$RUN")" + CONCLUSION="$(jq -r '.[0].conclusion // ""' <<< "$RUN")" + if [[ -z "$STATUS" ]]; then echo "::error::No CI run found for $SHA. Push to a release/** or develop/** branch first." exit 1 fi + if [[ "$STATUS" != "completed" ]]; then + echo "::error::CI for $SHA is still $STATUS. Wait for it to finish before releasing." + exit 1 + fi if [[ "$CONCLUSION" != "success" ]]; then echo "::error::CI has not passed for $SHA (conclusion: $CONCLUSION)." exit 1 fi echo "CI passed" - # Refuses to publish under a version the tag does not claim, so a tag cut - # from the wrong branch cannot overwrite someone else's release. - name: Verify release tag matches pom version env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [[ -z "$RELEASE_TAG" ]]; then echo "No release tag: skipping."; exit 0; fi - # build/mvn logs to stderr, so -DforceStdout yields a clean value. POM_VERSION="$(./build/mvn -q -N help:evaluate -Dexpression=project.version -DforceStdout)" TAG_VERSION="${RELEASE_TAG#v}" echo "pom version: '$POM_VERSION'" @@ -120,7 +121,7 @@ jobs: maven: name: "Deploy to GitHub Packages (arenadata)" needs: verify - if: success() && (github.event_name == 'release' || inputs.publish_maven) + if: success() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish_maven)) runs-on: ubuntu-24.04 timeout-minutes: 300 permissions: @@ -139,22 +140,15 @@ jobs: java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven - # Authenticates both the `arenadata` entry the reactor - # reads the Arenadata Hive artifacts from (GitHub Packages requires - # credentials even for public repositories) and the identically-named - # entry it deploys to. + # GitHub Packages wants credentials for reads too, so this one server-id + # covers both the `arenadata` and . server-id: arenadata server-username: GITHUB_ACTOR server-password: GITHUB_TOKEN - # deployAtEnd builds the whole reactor first and defers all uploads until - # every module has built successfully, so a failure mid-reactor does not - # leave a partial release. GitHub Packages does not allow re-publishing an - # existing version, so this all-or-nothing behavior is what we want. - # - # Javadoc/scaladoc jars are skipped: attach-scaladocs is bound to `verify`, - # so `deploy` would build them for the whole reactor and roughly double the - # job. Source jars are still published. + # deployAtEnd defers every upload until the whole reactor has built, so a + # failure mid-reactor cannot leave a partial release. Scaladoc is skipped: it + # is bound to `verify` and would roughly double the job. Source jars are kept. - name: Build and deploy artifacts env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -163,8 +157,7 @@ jobs: set -eo pipefail resume_arg=() if [[ -n "$RESUME_FROM" ]]; then - # NOTE: with -rf, deployAtEnd only covers the modules in the partial - # reactor; the skipped ones are not (re-)deployed. + # With -rf, deployAtEnd covers only the modules still in the reactor. resume_arg=(-rf "$RESUME_FROM") fi ./build/mvn $MAVEN_ARGS deploy \ @@ -176,7 +169,7 @@ jobs: assets: name: "Distribution: ${{ matrix.name }}" needs: verify - if: success() && (github.event_name == 'release' || inputs.publish_assets) + if: success() && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_assets)) runs-on: ubuntu-24.04 timeout-minutes: 300 permissions: @@ -186,29 +179,24 @@ jobs: fail-fast: false matrix: include: - # Reproduces the asset set of the previous releases: - # spark--bin-hadoop3.tgz - # spark--bin-hadoop3-cloud.tgz - # pyspark-.tar.gz (built alongside the -cloud distribution) + # The sdist packages assembly/target/scala-2.13/jars, so the leg it is + # built in decides its contents: -Phadoop-cloud would ship the AWS SDK + # inside `pip install pyspark`. Earlier releases built it on the -cloud leg. - name: hadoop3 extra_profiles: '' - make_pip: false + make_pip: true - name: hadoop3-cloud extra_profiles: '-Phadoop-cloud' - make_pip: true + make_pip: false env: - # Passed through the environment (Maven >= 3.9 reads MAVEN_ARGS) rather than - # on the command line: make-distribution.sh records its own argv in the - # shipped RELEASE file, and CI plumbing does not belong in a released - # artifact. -V is left out because the script parses `mvn help:evaluate` - # output to learn the version and the version banner would end up in it. + # In the environment, not on the command line: make-distribution.sh records + # its own argv in the shipped RELEASE file. -V is left out because the script + # parses `mvn help:evaluate` output for the version. MAVEN_ARGS: -B -ntp -e MAVEN_OPTS: -Xmx4g -XX:ReservedCodeCacheSize=1g -XX:MaxMetaspaceSize=2g - RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }} + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }} steps: - # The -cloud distribution keeps the reactor output, dist/, the tarball and - # the PySpark sdist on disk at the same time; the stock runner image does - # not leave enough room for that. + # Each leg holds the reactor output, dist/ and the tarball at the same time. - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \ @@ -224,10 +212,6 @@ jobs: java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven - # Authenticates both the `arenadata` entry the reactor - # reads the Arenadata Hive artifacts from (GitHub Packages requires - # credentials even for public repositories) and the identically-named - # entry it deploys to. server-id: arenadata server-username: GITHUB_ACTOR server-password: GITHUB_TOKEN @@ -240,9 +224,6 @@ jobs: if: matrix.make_pip run: python3 -m pip install --upgrade pip setuptools wheel - # make-distribution.sh already runs Maven with -DskipTests and with the - # javadoc/scaladoc/source/cyclonedx plugins skipped. The profile list is - # exactly the one recorded in the RELEASE file of v4.2.0.1-4.3.0-2. - name: Build distribution env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -251,9 +232,7 @@ jobs: --name ${{ matrix.name }} --tgz ${{ matrix.make_pip && '--pip' || '' }} ${{ env.SPARK_PROFILES }} ${{ matrix.extra_profiles }} - # dist/ is a full uncompressed copy of the tarball and is no longer needed - # once the tarball exists. Dropped here rather than in the upload step so - # the space comes back on the dispatch-without-a-tag path too. + # dist/ is an uncompressed copy of the tarball; reclaim the space now. - name: Drop the uncompressed distribution run: rm -rf dist @@ -266,8 +245,6 @@ jobs: set -eo pipefail shopt -s nullglob - # Resolves exactly one file for the given glob and attaches it to the - # release, then frees the space before the next asset is handled. upload_single() { local description="$1" shift @@ -289,7 +266,65 @@ jobs: upload_single "spark-*-bin-$DIST_NAME.tgz tarball" spark-*-bin-"$DIST_NAME".tgz if [[ "$MAKE_PIP" == "true" ]]; then - # --pip also builds the pyspark-connect and pyspark-client sdists; - # only the classic one has ever been published, so anchor on a digit. + # --pip also builds the pyspark-connect/-client sdists; anchor on a digit. upload_single "PySpark sdist" python/dist/pyspark-[0-9]*.tar.gz fi + + # Built with -Phadoop-cloud so it describes the reactor the Maven job deploys. + sbom: + name: "SBOM and vulnerability report" + needs: verify + if: success() && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_assets)) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: write + packages: read + env: + MAVEN_ARGS: -B -V -e -ntp + MAVEN_OPTS: -Xmx4g -XX:ReservedCodeCacheSize=1g -XX:MaxMetaspaceSize=2g + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || '' }} + + - uses: actions/setup-java@v5 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + cache: maven + server-id: arenadata + server-username: GITHUB_ACTOR + server-password: GITHUB_TOKEN + + # outputDirectory is pinned: makeAggregateBom otherwise writes next to the + # first module of the reactor rather than into the workspace root. + - name: Generate SBOM + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + ./build/mvn $MAVEN_ARGS cyclonedx:makeAggregateBom + $SPARK_PROFILES -Phadoop-cloud + -DoutputFormat=json -DoutputName=bom + -DoutputDirectory="$GITHUB_WORKSPACE/target" + + - name: Vulnerability scan (grype) + uses: anchore/scan-action@v6 + with: + sbom: target/bom.json + fail-build: false + output-format: json + output-file: vulnerability-report.json + + - name: Upload the SBOM and the report to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + ls -l target/bom.json vulnerability-report.json + if [[ -z "$RELEASE_TAG" ]]; then + echo "No release tag: the SBOM and the report are built but not uploaded." + exit 0 + fi + gh release upload "$RELEASE_TAG" target/bom.json vulnerability-report.json --clobber From b5ea645317c2cbdf62db5db3c7d47b9fe8204720 Mon Sep 17 00:00:00 2001 From: Petr Fedchenkov Date: Thu, 3 Sep 2026 14:20:22 +0300 Subject: [PATCH 2/2] NGSOK-1622 Skip the k8s image rebuild in the install step The install lifecycle passes through pre-integration-test, so setup-integration-test-env.sh ran with the default image tag N/A and rebuilt the images through minikube's Docker daemon, which is containerd-backed and fails to boot buildkit. Point that step at the images already built and loaded so the script skips the rebuild. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac415d279f215..554edba0ff66c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,6 +399,8 @@ jobs: ./build/mvn ${{ env.MAVEN_ARGS }} install -DskipTests -pl :spark-kubernetes-integration-tests_2.13 -am ${{ env.SPARK_PROFILES }} -Pkubernetes-integration-tests -Psparkr + -Dspark.kubernetes.test.imageRepo=docker.io/library + -Dspark.kubernetes.test.imageTag=ci-test - name: Run k8s integration tests env: