diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ac415d279f215..554edba0ff66c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -399,6 +399,8 @@ jobs: ./build/mvn ${{ env.MAVEN_ARGS }} install -DskipTests -pl :spark-kubernetes-integration-tests_2.13 -am ${{ env.SPARK_PROFILES }} -Pkubernetes-integration-tests -Psparkr + -Dspark.kubernetes.test.imageRepo=docker.io/library + -Dspark.kubernetes.test.imageTag=ci-test - name: Run k8s integration tests env: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c4a989381a1e5..2866de008b4dc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,10 @@ name: Publish on: + # Maven deploys from the tag push, not from `release`: a package published by a + # release-triggered run gets listed among that release's assets. + push: + tags: ['v*'] release: types: [created] workflow_dispatch: @@ -25,7 +29,7 @@ on: type: string concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: false permissions: @@ -34,17 +38,13 @@ permissions: env: JAVA_VERSION: '17' PYTHON_VERSION: '3.10' - # Profiles shared by the Maven deploy and by every binary distribution. - # -Phadoop-cloud is deliberately NOT here: it pulls the AWS SDK bundle into the - # assembly and roughly doubles the tarball, so only the `hadoop3-cloud` - # distribution (and the Maven deploy, which publishes spark-hadoop-cloud) adds it. + # -Phadoop-cloud is added per job: it roughly doubles the tarball, so only the + # `hadoop3-cloud` leg and the Maven deploy take it. SPARK_PROFILES: -Pscala-2.13 -Phadoop-3 -Phive -Phive-thriftserver -Pyarn -Pkubernetes -Pconnect -Pvolcano jobs: - # Everything that can reject the release runs here, once and cheaply, so a bad - # tag fails in a minute instead of two hours into a reactor build. All three - # checks are no-ops when there is no release tag, which is the dispatch path - # used to smoke-test the workflow itself. + # Cheap checks first, so a bad tag fails in a minute rather than hours into the + # reactor. All of them are no-ops when there is no release tag. verify: name: "Verify the release commit" runs-on: ubuntu-24.04 @@ -53,14 +53,12 @@ jobs: contents: read actions: read env: - RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }} + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }} steps: - uses: actions/checkout@v6 with: ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || '' }} - # `git branch --contains` needs the remote-tracking branches, which - # only a full-history checkout provides. - fetch-depth: 0 + fetch-depth: 0 # `git branch --contains` needs remote-tracking branches - uses: actions/setup-java@v5 with: @@ -89,25 +87,28 @@ jobs: run: | if [[ -z "$RELEASE_TAG" ]]; then echo "No release tag: skipping."; exit 0; fi SHA="$(git rev-parse HEAD)" - CONCLUSION="$(gh run list --commit "$SHA" --workflow CI --json conclusion -q '.[0].conclusion')" - if [[ -z "$CONCLUSION" ]]; then + RUN="$(gh run list --commit "$SHA" --workflow CI --limit 1 --json status,conclusion)" + STATUS="$(jq -r '.[0].status // ""' <<< "$RUN")" + CONCLUSION="$(jq -r '.[0].conclusion // ""' <<< "$RUN")" + if [[ -z "$STATUS" ]]; then echo "::error::No CI run found for $SHA. Push to a release/** or develop/** branch first." exit 1 fi + if [[ "$STATUS" != "completed" ]]; then + echo "::error::CI for $SHA is still $STATUS. Wait for it to finish before releasing." + exit 1 + fi if [[ "$CONCLUSION" != "success" ]]; then echo "::error::CI has not passed for $SHA (conclusion: $CONCLUSION)." exit 1 fi echo "CI passed" - # Refuses to publish under a version the tag does not claim, so a tag cut - # from the wrong branch cannot overwrite someone else's release. - name: Verify release tag matches pom version env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | if [[ -z "$RELEASE_TAG" ]]; then echo "No release tag: skipping."; exit 0; fi - # build/mvn logs to stderr, so -DforceStdout yields a clean value. POM_VERSION="$(./build/mvn -q -N help:evaluate -Dexpression=project.version -DforceStdout)" TAG_VERSION="${RELEASE_TAG#v}" echo "pom version: '$POM_VERSION'" @@ -120,7 +121,7 @@ jobs: maven: name: "Deploy to GitHub Packages (arenadata)" needs: verify - if: success() && (github.event_name == 'release' || inputs.publish_maven) + if: success() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish_maven)) runs-on: ubuntu-24.04 timeout-minutes: 300 permissions: @@ -139,22 +140,15 @@ jobs: java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven - # Authenticates both the `arenadata` entry the reactor - # reads the Arenadata Hive artifacts from (GitHub Packages requires - # credentials even for public repositories) and the identically-named - # entry it deploys to. + # GitHub Packages wants credentials for reads too, so this one server-id + # covers both the `arenadata` and . server-id: arenadata server-username: GITHUB_ACTOR server-password: GITHUB_TOKEN - # deployAtEnd builds the whole reactor first and defers all uploads until - # every module has built successfully, so a failure mid-reactor does not - # leave a partial release. GitHub Packages does not allow re-publishing an - # existing version, so this all-or-nothing behavior is what we want. - # - # Javadoc/scaladoc jars are skipped: attach-scaladocs is bound to `verify`, - # so `deploy` would build them for the whole reactor and roughly double the - # job. Source jars are still published. + # deployAtEnd defers every upload until the whole reactor has built, so a + # failure mid-reactor cannot leave a partial release. Scaladoc is skipped: it + # is bound to `verify` and would roughly double the job. Source jars are kept. - name: Build and deploy artifacts env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -163,8 +157,7 @@ jobs: set -eo pipefail resume_arg=() if [[ -n "$RESUME_FROM" ]]; then - # NOTE: with -rf, deployAtEnd only covers the modules in the partial - # reactor; the skipped ones are not (re-)deployed. + # With -rf, deployAtEnd covers only the modules still in the reactor. resume_arg=(-rf "$RESUME_FROM") fi ./build/mvn $MAVEN_ARGS deploy \ @@ -176,7 +169,7 @@ jobs: assets: name: "Distribution: ${{ matrix.name }}" needs: verify - if: success() && (github.event_name == 'release' || inputs.publish_assets) + if: success() && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_assets)) runs-on: ubuntu-24.04 timeout-minutes: 300 permissions: @@ -186,29 +179,24 @@ jobs: fail-fast: false matrix: include: - # Reproduces the asset set of the previous releases: - # spark--bin-hadoop3.tgz - # spark--bin-hadoop3-cloud.tgz - # pyspark-.tar.gz (built alongside the -cloud distribution) + # The sdist packages assembly/target/scala-2.13/jars, so the leg it is + # built in decides its contents: -Phadoop-cloud would ship the AWS SDK + # inside `pip install pyspark`. Earlier releases built it on the -cloud leg. - name: hadoop3 extra_profiles: '' - make_pip: false + make_pip: true - name: hadoop3-cloud extra_profiles: '-Phadoop-cloud' - make_pip: true + make_pip: false env: - # Passed through the environment (Maven >= 3.9 reads MAVEN_ARGS) rather than - # on the command line: make-distribution.sh records its own argv in the - # shipped RELEASE file, and CI plumbing does not belong in a released - # artifact. -V is left out because the script parses `mvn help:evaluate` - # output to learn the version and the version banner would end up in it. + # In the environment, not on the command line: make-distribution.sh records + # its own argv in the shipped RELEASE file. -V is left out because the script + # parses `mvn help:evaluate` output for the version. MAVEN_ARGS: -B -ntp -e MAVEN_OPTS: -Xmx4g -XX:ReservedCodeCacheSize=1g -XX:MaxMetaspaceSize=2g - RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }} + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }} steps: - # The -cloud distribution keeps the reactor output, dist/, the tarball and - # the PySpark sdist on disk at the same time; the stock runner image does - # not leave enough room for that. + # Each leg holds the reactor output, dist/ and the tarball at the same time. - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android \ @@ -224,10 +212,6 @@ jobs: java-version: ${{ env.JAVA_VERSION }} distribution: temurin cache: maven - # Authenticates both the `arenadata` entry the reactor - # reads the Arenadata Hive artifacts from (GitHub Packages requires - # credentials even for public repositories) and the identically-named - # entry it deploys to. server-id: arenadata server-username: GITHUB_ACTOR server-password: GITHUB_TOKEN @@ -240,9 +224,6 @@ jobs: if: matrix.make_pip run: python3 -m pip install --upgrade pip setuptools wheel - # make-distribution.sh already runs Maven with -DskipTests and with the - # javadoc/scaladoc/source/cyclonedx plugins skipped. The profile list is - # exactly the one recorded in the RELEASE file of v4.2.0.1-4.3.0-2. - name: Build distribution env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -251,9 +232,7 @@ jobs: --name ${{ matrix.name }} --tgz ${{ matrix.make_pip && '--pip' || '' }} ${{ env.SPARK_PROFILES }} ${{ matrix.extra_profiles }} - # dist/ is a full uncompressed copy of the tarball and is no longer needed - # once the tarball exists. Dropped here rather than in the upload step so - # the space comes back on the dispatch-without-a-tag path too. + # dist/ is an uncompressed copy of the tarball; reclaim the space now. - name: Drop the uncompressed distribution run: rm -rf dist @@ -266,8 +245,6 @@ jobs: set -eo pipefail shopt -s nullglob - # Resolves exactly one file for the given glob and attaches it to the - # release, then frees the space before the next asset is handled. upload_single() { local description="$1" shift @@ -289,7 +266,65 @@ jobs: upload_single "spark-*-bin-$DIST_NAME.tgz tarball" spark-*-bin-"$DIST_NAME".tgz if [[ "$MAKE_PIP" == "true" ]]; then - # --pip also builds the pyspark-connect and pyspark-client sdists; - # only the classic one has ever been published, so anchor on a digit. + # --pip also builds the pyspark-connect/-client sdists; anchor on a digit. upload_single "PySpark sdist" python/dist/pyspark-[0-9]*.tar.gz fi + + # Built with -Phadoop-cloud so it describes the reactor the Maven job deploys. + sbom: + name: "SBOM and vulnerability report" + needs: verify + if: success() && (github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish_assets)) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: write + packages: read + env: + MAVEN_ARGS: -B -V -e -ntp + MAVEN_OPTS: -Xmx4g -XX:ReservedCodeCacheSize=1g -XX:MaxMetaspaceSize=2g + RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || github.event_name == 'push' && github.ref_name || inputs.release_tag }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_tag || '' }} + + - uses: actions/setup-java@v5 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: temurin + cache: maven + server-id: arenadata + server-username: GITHUB_ACTOR + server-password: GITHUB_TOKEN + + # outputDirectory is pinned: makeAggregateBom otherwise writes next to the + # first module of the reactor rather than into the workspace root. + - name: Generate SBOM + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: >- + ./build/mvn $MAVEN_ARGS cyclonedx:makeAggregateBom + $SPARK_PROFILES -Phadoop-cloud + -DoutputFormat=json -DoutputName=bom + -DoutputDirectory="$GITHUB_WORKSPACE/target" + + - name: Vulnerability scan (grype) + uses: anchore/scan-action@v6 + with: + sbom: target/bom.json + fail-build: false + output-format: json + output-file: vulnerability-report.json + + - name: Upload the SBOM and the report to the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + ls -l target/bom.json vulnerability-report.json + if [[ -z "$RELEASE_TAG" ]]; then + echo "No release tag: the SBOM and the report are built but not uploaded." + exit 0 + fi + gh release upload "$RELEASE_TAG" target/bom.json vulnerability-report.json --clobber