NGSOK-1746 PySpark sdist without hadoop-cloud, and a correct CI guard

.github/workflows/ci.yml

@@ -399,6 +399,8 @@ jobs:
           ./build/mvn ${{ env.MAVEN_ARGS }} install -DskipTests
           -pl :spark-kubernetes-integration-tests_2.13 -am
           ${{ env.SPARK_PROFILES }} -Pkubernetes-integration-tests -Psparkr
+          -Dspark.kubernetes.test.imageRepo=docker.io/library
+          -Dspark.kubernetes.test.imageTag=ci-test
 
       - name: Run k8s integration tests
         env:

.github/workflows/publish.yml

@@ -1,6 +1,10 @@
 name: Publish
 
 on:
+  # Maven deploys from the tag push, not from `release`: a package published by a
+  # release-triggered run gets listed among that release's assets.
+  push:
+    tags: ['v*']
   release:
     types: [created]
   workflow_dispatch:
@@ -25,7 +29,7 @@ on:
         type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: false
 
 permissions:
@@ -34,17 +38,13 @@ permissions:
 env:
   JAVA_VERSION: '17'
   PYTHON_VERSION: '3.10'
-  # Profiles shared by the Maven deploy and by every binary distribution.
-  # -Phadoop-cloud is deliberately NOT here: it pulls the AWS SDK bundle into the
-  # assembly and roughly doubles the tarball, so only the `hadoop3-cloud`
-  # distribution (and the Maven deploy, which publishes spark-hadoop-cloud) adds it.
+  # -Phadoop-cloud is added per job: it roughly doubles the tarball, so only the
+  # `hadoop3-cloud` leg and the Maven deploy take it.
   SPARK_PROFILES: -Pscala-2.13 -Phadoop-3 -Phive -Phive-thriftserver -Pyarn -Pkubernetes -Pconnect -Pvolcano
 
 jobs:
-  # Everything that can reject the release runs here, once and cheaply, so a bad
-  # tag fails in a minute instead of two hours into a reactor build. All three
-  # checks are no-ops when there is no release tag, which is the dispatch path
-  # used to smoke-test the workflow itself.
+  # Cheap checks first, so a bad tag fails in a minute rather than hours into the
+  # reactor. All of them are no-ops when there is no release tag.
   verify:
     name: "Verify the release commit"
     runs-on: ubuntu-24.04
@@ -53,14 +53,12 @@ jobs:
       contents: read
       actions: read
     env:
-      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }}
+      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }}
     steps:
       - uses: actions/checkout@v6
         with:
           ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || '' }}
-          # `git branch --contains` needs the remote-tracking branches, which
-          # only a full-history checkout provides.
-          fetch-depth: 0
+          fetch-depth: 0  # `git branch --contains` needs remote-tracking branches
 
       - uses: actions/setup-java@v5
         with:
@@ -89,25 +87,28 @@ jobs:
         run: |
           if [[ -z "$RELEASE_TAG" ]]; then echo "No release tag: skipping."; exit 0; fi
           SHA="$(git rev-parse HEAD)"
-          CONCLUSION="$(gh run list --commit "$SHA" --workflow CI --json conclusion -q '.[0].conclusion')"
-          if [[ -z "$CONCLUSION" ]]; then
+          RUN="$(gh run list --commit "$SHA" --workflow CI --limit 1 --json status,conclusion)"
+          STATUS="$(jq -r '.[0].status // ""' <<< "$RUN")"
+          CONCLUSION="$(jq -r '.[0].conclusion // ""' <<< "$RUN")"
+          if [[ -z "$STATUS" ]]; then
             echo "::error::No CI run found for $SHA. Push to a release/** or develop/** branch first."
             exit 1
           fi
+          if [[ "$STATUS" != "completed" ]]; then
+            echo "::error::CI for $SHA is still $STATUS. Wait for it to finish before releasing."
+            exit 1
+          fi
           if [[ "$CONCLUSION" != "success" ]]; then
             echo "::error::CI has not passed for $SHA (conclusion: $CONCLUSION)."
             exit 1
           fi
           echo "CI passed"
 
-      # Refuses to publish under a version the tag does not claim, so a tag cut
-      # from the wrong branch cannot overwrite someone else's release.
       - name: Verify release tag matches pom version
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           if [[ -z "$RELEASE_TAG" ]]; then echo "No release tag: skipping."; exit 0; fi
-          # build/mvn logs to stderr, so -DforceStdout yields a clean value.
           POM_VERSION="$(./build/mvn -q -N help:evaluate -Dexpression=project.version -DforceStdout)"
           TAG_VERSION="${RELEASE_TAG#v}"
           echo "pom version: '$POM_VERSION'"
@@ -120,7 +121,7 @@ jobs:
   maven:
     name: "Deploy to GitHub Packages (arenadata)"
     needs: verify
-    if: success() && (github.event_name == 'release' || inputs.publish_maven)
+    if: success() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish_maven))
     runs-on: ubuntu-24.04
     timeout-minutes: 300
     permissions:
@@ -139,22 +140,15 @@ jobs:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: temurin
           cache: maven
-          # Authenticates both the `arenadata` <repositories> entry the reactor
-          # reads the Arenadata Hive artifacts from (GitHub Packages requires
-          # credentials even for public repositories) and the identically-named
-          # <distributionManagement> entry it deploys to.
+          # GitHub Packages wants credentials for reads too, so this one server-id
+          # covers both the `arenadata` <repositories> and <distributionManagement>.
           server-id: arenadata
           server-username: GITHUB_ACTOR
           server-password: GITHUB_TOKEN
 
-      # deployAtEnd builds the whole reactor first and defers all uploads until
-      # every module has built successfully, so a failure mid-reactor does not
-      # leave a partial release. GitHub Packages does not allow re-publishing an
-      # existing version, so this all-or-nothing behavior is what we want.
-      #
-      # Javadoc/scaladoc jars are skipped: attach-scaladocs is bound to `verify`,
-      # so `deploy` would build them for the whole reactor and roughly double the
-      # job. Source jars are still published.
+      # deployAtEnd defers every upload until the whole reactor has built, so a
+      # failure mid-reactor cannot leave a partial release. Scaladoc is skipped: it
+      # is bound to `verify` and would roughly double the job. Source jars are kept.
       - name: Build and deploy artifacts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -163,8 +157,7 @@ jobs:
           set -eo pipefail
           resume_arg=()
           if [[ -n "$RESUME_FROM" ]]; then
-            # NOTE: with -rf, deployAtEnd only covers the modules in the partial
-            # reactor; the skipped ones are not (re-)deployed.
+            # With -rf, deployAtEnd covers only the modules still in the reactor.
             resume_arg=(-rf "$RESUME_FROM")
           fi
           ./build/mvn $MAVEN_ARGS deploy \
@@ -176,7 +169,7 @@ jobs:
   assets:
     name: "Distribution: ${{ matrix.name }}"
     needs: verify
-    if: success() && (github.event_name == 'release' || inputs.publish_assets)
+    if: success() && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_assets))
     runs-on: ubuntu-24.04
     timeout-minutes: 300
     permissions:
@@ -186,29 +179,24 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Reproduces the asset set of the previous releases:
-          #   spark-<version>-bin-hadoop3.tgz
-          #   spark-<version>-bin-hadoop3-cloud.tgz
-          #   pyspark-<version>.tar.gz   (built alongside the -cloud distribution)
+          # The sdist packages assembly/target/scala-2.13/jars, so the leg it is
+          # built in decides its contents: -Phadoop-cloud would ship the AWS SDK
+          # inside `pip install pyspark`. Earlier releases built it on the -cloud leg.
           - name: hadoop3
             extra_profiles: ''
-            make_pip: false
+            make_pip: true
           - name: hadoop3-cloud
             extra_profiles: '-Phadoop-cloud'
-            make_pip: true
+            make_pip: false
     env:
-      # Passed through the environment (Maven >= 3.9 reads MAVEN_ARGS) rather than
-      # on the command line: make-distribution.sh records its own argv in the
-      # shipped RELEASE file, and CI plumbing does not belong in a released
-      # artifact. -V is left out because the script parses `mvn help:evaluate`
-      # output to learn the version and the version banner would end up in it.
+      # In the environment, not on the command line: make-distribution.sh records
+      # its own argv in the shipped RELEASE file. -V is left out because the script
+      # parses `mvn help:evaluate` output for the version.
       MAVEN_ARGS: -B -ntp -e
       MAVEN_OPTS: -Xmx4g -XX:ReservedCodeCacheSize=1g -XX:MaxMetaspaceSize=2g
-      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }}
+      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }}
     steps:
-      # The -cloud distribution keeps the reactor output, dist/, the tarball and
-      # the PySpark sdist on disk at the same time; the stock runner image does
-      # not leave enough room for that.
+      # Each leg holds the reactor output, dist/ and the tarball at the same time.
       - name: Free disk space
         run: |
           sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \
@@ -224,10 +212,6 @@ jobs:
           java-version: ${{ env.JAVA_VERSION }}
           distribution: temurin
           cache: maven
-          # Authenticates both the `arenadata` <repositories> entry the reactor
-          # reads the Arenadata Hive artifacts from (GitHub Packages requires
-          # credentials even for public repositories) and the identically-named
-          # <distributionManagement> entry it deploys to.
           server-id: arenadata
           server-username: GITHUB_ACTOR
           server-password: GITHUB_TOKEN
@@ -240,9 +224,6 @@ jobs:
         if: matrix.make_pip
         run: python3 -m pip install --upgrade pip setuptools wheel
 
-      # make-distribution.sh already runs Maven with -DskipTests and with the
-      # javadoc/scaladoc/source/cyclonedx plugins skipped. The profile list is
-      # exactly the one recorded in the RELEASE file of v4.2.0.1-4.3.0-2.
       - name: Build distribution
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -251,9 +232,7 @@ jobs:
           --name ${{ matrix.name }} --tgz ${{ matrix.make_pip && '--pip' || '' }}
           ${{ env.SPARK_PROFILES }} ${{ matrix.extra_profiles }}
 
-      # dist/ is a full uncompressed copy of the tarball and is no longer needed
-      # once the tarball exists. Dropped here rather than in the upload step so
-      # the space comes back on the dispatch-without-a-tag path too.
+      # dist/ is an uncompressed copy of the tarball; reclaim the space now.
       - name: Drop the uncompressed distribution
         run: rm -rf dist
 
@@ -266,8 +245,6 @@ jobs:
           set -eo pipefail
           shopt -s nullglob
 
-          # Resolves exactly one file for the given glob and attaches it to the
-          # release, then frees the space before the next asset is handled.
           upload_single() {
             local description="$1"
             shift
@@ -289,7 +266,65 @@ jobs:
           upload_single "spark-*-bin-$DIST_NAME.tgz tarball" spark-*-bin-"$DIST_NAME".tgz
 
           if [[ "$MAKE_PIP" == "true" ]]; then
-            # --pip also builds the pyspark-connect and pyspark-client sdists;
-            # only the classic one has ever been published, so anchor on a digit.
+            # --pip also builds the pyspark-connect/-client sdists; anchor on a digit.
             upload_single "PySpark sdist" python/dist/pyspark-[0-9]*.tar.gz
           fi
+
+  # Built with -Phadoop-cloud so it describes the reactor the Maven job deploys.
+  sbom:
+    name: "SBOM and vulnerability report"
+    needs: verify
+    if: success() && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_assets))
+    runs-on: ubuntu-24.04
+    timeout-minutes: 120
+    permissions:
+      contents: write
+      packages: read
+    env:
+      MAVEN_ARGS: -B -V -e -ntp
+      MAVEN_OPTS: -Xmx4g -XX:ReservedCodeCacheSize=1g -XX:MaxMetaspaceSize=2g
+      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || '' }}
+
+      - uses: actions/setup-java@v5
+        with:
+          java-version: ${{ env.JAVA_VERSION }}
+          distribution: temurin
+          cache: maven
+          server-id: arenadata
+          server-username: GITHUB_ACTOR
+          server-password: GITHUB_TOKEN
+
+      # outputDirectory is pinned: makeAggregateBom otherwise writes next to the
+      # first module of the reactor rather than into the workspace root.
+      - name: Generate SBOM
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: >-
+          ./build/mvn $MAVEN_ARGS cyclonedx:makeAggregateBom
+          $SPARK_PROFILES -Phadoop-cloud
+          -DoutputFormat=json -DoutputName=bom
+          -DoutputDirectory="$GITHUB_WORKSPACE/target"
+
+      - name: Vulnerability scan (grype)
+        uses: anchore/scan-action@v6
+        with:
+          sbom: target/bom.json
+          fail-build: false
+          output-format: json
+          output-file: vulnerability-report.json
+
+      - name: Upload the SBOM and the report to the release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -eo pipefail
+          ls -l target/bom.json vulnerability-report.json
+          if [[ -z "$RELEASE_TAG" ]]; then
+            echo "No release tag: the SBOM and the report are built but not uploaded."
+            exit 0
+          fi
+          gh release upload "$RELEASE_TAG" target/bom.json vulnerability-report.json --clobber