[GH-3297] Drop Spark 3.4 support

.github/workflows/example.yml

@@ -53,10 +53,6 @@ jobs:
             spark-compat: '3.5'
             sedona: 1.8.0
             hadoop: 3.3.4
-          - spark: 3.4.2
-            spark-compat: '3.4'
-            sedona: 1.8.0
-            hadoop: 3.3.4
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:

.github/workflows/java.yml

@@ -81,16 +81,6 @@ jobs:
             scala: 2.12.15
             jdk: '11'
             skipTests: ''
-          - spark: 3.4.0
-            scala: 2.13.8
-            jdk: '11'
-            skipTests: ''
-            skipLibPostalTests: 'true'
-          - spark: 3.4.0
-            scala: 2.12.15
-            jdk: '11'
-            skipTests: ''
-            skipLibPostalTests: 'true'
           - spark: 3.5.8
             scala: 2.12.15
             jdk: '17'

.github/workflows/python.yml

@@ -72,7 +72,7 @@ jobs:
             scala: '2.12.8'
             java: '11'
             python: '3.9'
-          - spark: '3.4.0'
+          - spark: '3.5.0'
             scala: '2.12.8'
             java: '11'
             python: '3.8'
@@ -158,7 +158,6 @@ jobs:
           uv sync
           uv run pytest -v tests/core/test_rdd.py tests/sql/test_dataframe_api.py
       - name: Run Spark Connect tests
-        if: ${{ matrix.spark >= '3.4.0' }}
         run: |
           cd python
           export SPARK_REMOTE=local

.github/workflows/r.yml

@@ -59,7 +59,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        spark: [3.4.0, 3.5.0]
+        spark: [3.5.0]
         hadoop: [3]
         scala: [2.12.15]
         r: [oldrel, release]

docker/build.sh

@@ -22,12 +22,6 @@ SEDONA_VERSION=$2
 BUILD_MODE=$3
 GEOTOOLS_VERSION=${4:-auto}
 
-SEDONA_SPARK_VERSION=${SPARK_VERSION:0:3}
-if [ "${SPARK_VERSION:0:1}" -eq "3" ] && [ "${SPARK_VERSION:2:1}" -le "3" ]; then
-    # 3.0, 3.1, 3.2, 3.3
-    SEDONA_SPARK_VERSION=3.0
-fi
-
 # Function to compare two version numbers
 version_gt() {
   # Compare two version numbers
@@ -79,6 +73,20 @@ if [ "$SEDONA_VERSION" = "latest" ]; then
     fi
     echo "Using latest geotools-wrapper version: $GEOTOOLS_WRAPPER_VERSION"
 
+    # Building master resolves Sedona's Spark profile from -Dspark. An unsupported
+    # value matches no profile, so Maven would fall back to the default Spark
+    # version and produce jars that do not match the Spark image they are
+    # installed into. Only this branch compiles; images for a published Sedona
+    # version take the jar from Maven Central and can still target older Spark.
+    SEDONA_SPARK_VERSION=${SPARK_VERSION:0:3}
+    case "$SEDONA_SPARK_VERSION" in
+        3.5 | 4.0 | 4.1) ;;
+        *)
+            echo "Cannot build Sedona master against Spark ${SPARK_VERSION}. Supported: 3.5, 4.0, 4.1." >&2
+            exit 1
+            ;;
+    esac
+
     # The compilation must take place outside Docker to avoid unnecessary maven packages
     mvn clean install -DskipTests -Dspark="${SEDONA_SPARK_VERSION}" -Dscala=2.13
 else

docs/community/develop.md

@@ -70,8 +70,8 @@ Make sure you reload the `pom.xml` or reload the maven project. The IDE will ask
 
 In a terminal, go to the Sedona root folder. Run `mvn clean install`. All tests will take more than 15 minutes. To only build the project jars, run `mvn clean install -DskipTests`.
 !!!Note
-    `mvn clean install` will compile Sedona with Spark 3.3 and Scala 2.12. If you have a different version of Spark in $SPARK_HOME, make sure to specify that using -Dspark command line arg.
-    For example, to compile sedona with Spark 3.4 and Scala 2.12, use: `mvn clean install -Dspark=3.4 -Dscala=2.12`
+    `mvn clean install` will compile Sedona with Spark 3.5 and Scala 2.12. If you have a different version of Spark in $SPARK_HOME, make sure to specify that using -Dspark command line arg.
+    For example, to compile sedona with Spark 4.0 and Scala 2.13, use: `mvn clean install -Dspark=4.0 -Dscala=2.13`
 
 More details can be found on [Compile Sedona](../setup/compile.md)
 

docs/community/develop.zh.md

@@ -71,7 +71,7 @@ IDE 通常难以理解 Sedona 中较为复杂的项目结构。
 在终端中进入 Sedona 根目录运行 `mvn clean install`。全部测试需要超过 15 分钟。如果只想构建 jar，运行 `mvn clean install -DskipTests`。
 
 !!!Note
-    `mvn clean install` 默认基于 Spark 3.3、Scala 2.12 编译 Sedona。如果 $SPARK_HOME 中是不同版本的 Spark，请通过 `-Dspark` 命令行参数指定。例如基于 Spark 3.4 与 Scala 2.12 编译：`mvn clean install -Dspark=3.4 -Dscala=2.12`。
+    `mvn clean install` 默认基于 Spark 3.5、Scala 2.12 编译 Sedona。如果 $SPARK_HOME 中是不同版本的 Spark，请通过 `-Dspark` 命令行参数指定。例如基于 Spark 4.0 与 Scala 2.13 编译：`mvn clean install -Dspark=4.0 -Dscala=2.13`。
 
 更多细节请参阅 [编译 Sedona](../setup/compile.md)。
 

docs/community/publish.md

@@ -127,7 +127,7 @@ rm -rf $LOCAL_DIR && git clone --depth 1 --branch $TAG $REPO_URL $LOCAL_DIR && c
 MAVEN_PLUGIN_VERSION="2.3.2"
 
 # Define Spark and Scala versions
-declare -a SPARK_VERSIONS=("3.4" "3.5" "4.0" "4.1")
+declare -a SPARK_VERSIONS=("3.5" "4.0" "4.1")
 declare -a SCALA_VERSIONS=("2.12" "2.13")
 
 # Function to get Java version for Spark version
@@ -383,19 +383,6 @@ verify_java_version() {
   echo "✓ Verified: Maven wrapper is using Java $mvn_java_version"
 }
 
-# Compile for Spark 3.4 with Java 11
-JAVA_VERSION=$(get_java_version "3.4")
-MVN_WRAPPER=$(create_mvn_wrapper $JAVA_VERSION)
-verify_java_version $MVN_WRAPPER $JAVA_VERSION
-
-echo "Compiling for Spark 3.4 with Scala 2.12 using Java $JAVA_VERSION..."
-cd apache-sedona-${SEDONA_VERSION}-src && $MVN_WRAPPER clean && $MVN_WRAPPER install -DskipTests -Dspark=3.4 -Dscala=2.12 && cd ..
-cp apache-sedona-${SEDONA_VERSION}-src/spark-shaded/target/sedona-*${SEDONA_VERSION}.jar apache-sedona-${SEDONA_VERSION}-bin/
-
-echo "Compiling for Spark 3.4 with Scala 2.13 using Java $JAVA_VERSION..."
-cd apache-sedona-${SEDONA_VERSION}-src && $MVN_WRAPPER clean && $MVN_WRAPPER install -DskipTests -Dspark=3.4 -Dscala=2.13 && cd ..
-cp apache-sedona-${SEDONA_VERSION}-src/spark-shaded/target/sedona-*${SEDONA_VERSION}.jar apache-sedona-${SEDONA_VERSION}-bin/
-
 # Compile for Spark 3.5 with Java 11
 JAVA_VERSION=$(get_java_version "3.5")
 MVN_WRAPPER=$(create_mvn_wrapper $JAVA_VERSION)

docs/community/publish.zh.md

@@ -125,7 +125,7 @@ rm -rf $LOCAL_DIR && git clone --depth 1 --branch $TAG $REPO_URL $LOCAL_DIR && c
 MAVEN_PLUGIN_VERSION="2.3.2"
 
 # Spark 与 Scala 版本
-declare -a SPARK_VERSIONS=("3.4" "3.5" "4.0" "4.1")
+declare -a SPARK_VERSIONS=("3.5" "4.0" "4.1")
 declare -a SCALA_VERSIONS=("2.12" "2.13")
 
 # 根据 Spark 版本确定所需 Java 版本
@@ -381,19 +381,6 @@ verify_java_version() {
   echo "✓ Verified: Maven wrapper is using Java $mvn_java_version"
 }
 
-# 使用 Java 11 编译 Spark 3.4
-JAVA_VERSION=$(get_java_version "3.4")
-MVN_WRAPPER=$(create_mvn_wrapper $JAVA_VERSION)
-verify_java_version $MVN_WRAPPER $JAVA_VERSION
-
-echo "Compiling for Spark 3.4 with Scala 2.12 using Java $JAVA_VERSION..."
-cd apache-sedona-${SEDONA_VERSION}-src && $MVN_WRAPPER clean && $MVN_WRAPPER install -DskipTests -Dspark=3.4 -Dscala=2.12 && cd ..
-cp apache-sedona-${SEDONA_VERSION}-src/spark-shaded/target/sedona-*${SEDONA_VERSION}.jar apache-sedona-${SEDONA_VERSION}-bin/
-
-echo "Compiling for Spark 3.4 with Scala 2.13 using Java $JAVA_VERSION..."
-cd apache-sedona-${SEDONA_VERSION}-src && $MVN_WRAPPER clean && $MVN_WRAPPER install -DskipTests -Dspark=3.4 -Dscala=2.13 && cd ..
-cp apache-sedona-${SEDONA_VERSION}-src/spark-shaded/target/sedona-*${SEDONA_VERSION}.jar apache-sedona-${SEDONA_VERSION}-bin/
-
 # 使用 Java 11 编译 Spark 3.5
 JAVA_VERSION=$(get_java_version "3.5")
 MVN_WRAPPER=$(create_mvn_wrapper $JAVA_VERSION)

docs/community/snapshot.md

@@ -59,15 +59,15 @@ rm -f pom.xml.*
 mvn -q -B clean release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -Dresume=false -DdryRun=true -Penable-all-submodules -Darguments="-DskipTests"
 mvn -q -B release:clean -Penable-all-submodules
 
-# Spark 3.3 and Scala 2.12
-mvn -q deploy -DskipTests -Dspark=3.3 -Dscala=2.12
+# Spark 3.5 and Scala 2.12
+mvn -q deploy -DskipTests -Dspark=3.5 -Dscala=2.12
 
-# Spark 3.3 and Scala 2.13
-mvn -q deploy -DskipTests -Dspark=3.3 -Dscala=2.13
+# Spark 3.5 and Scala 2.13
+mvn -q deploy -DskipTests -Dspark=3.5 -Dscala=2.13
 
-# Spark 3.4 and Scala 2.12
-mvn -q deploy -DskipTests -Dspark=3.4 -Dscala=2.12
+# Spark 4.0 and Scala 2.13
+mvn -q deploy -DskipTests -Dspark=4.0 -Dscala=2.13
 
-# Spark 3.4 and Scala 2.13
-mvn -q deploy -DskipTests -Dspark=3.4 -Dscala=2.13
+# Spark 4.1 and Scala 2.13
+mvn -q deploy -DskipTests -Dspark=4.1 -Dscala=2.13
 ```

docs/community/snapshot.zh.md

@@ -59,15 +59,15 @@ rm -f pom.xml.*
 mvn -q -B clean release:prepare -Dtag={{ sedona_create_release.current_git_tag }} -DreleaseVersion={{ sedona_create_release.current_version }} -DdevelopmentVersion={{ sedona_create_release.current_snapshot }} -Dresume=false -DdryRun=true -Penable-all-submodules -Darguments="-DskipTests"
 mvn -q -B release:clean -Penable-all-submodules
 
-# Spark 3.3 与 Scala 2.12
-mvn -q deploy -DskipTests -Dspark=3.3 -Dscala=2.12
+# Spark 3.5 与 Scala 2.12
+mvn -q deploy -DskipTests -Dspark=3.5 -Dscala=2.12
 
-# Spark 3.3 与 Scala 2.13
-mvn -q deploy -DskipTests -Dspark=3.3 -Dscala=2.13
+# Spark 3.5 与 Scala 2.13
+mvn -q deploy -DskipTests -Dspark=3.5 -Dscala=2.13
 
-# Spark 3.4 与 Scala 2.12
-mvn -q deploy -DskipTests -Dspark=3.4 -Dscala=2.12
+# Spark 4.0 与 Scala 2.13
+mvn -q deploy -DskipTests -Dspark=4.0 -Dscala=2.13
 
-# Spark 3.4 与 Scala 2.13
-mvn -q deploy -DskipTests -Dspark=3.4 -Dscala=2.13
+# Spark 4.1 与 Scala 2.13
+mvn -q deploy -DskipTests -Dspark=4.1 -Dscala=2.13
 ```

docs/setup/compile.md

@@ -46,28 +46,28 @@ To compile all modules, please make sure you are in the root folder of all modul
 	Geotools jars will be packaged into the produced fat jars.
 
 !!!note
-	By default, this command will compile Sedona with Spark 3.4 and Scala 2.12
+	By default, this command will compile Sedona with Spark 3.5 and Scala 2.12
 
 ### Compile with different targets
 
 User can specify `-Dspark` and `-Dscala` command line options to compile with different targets. Available targets are:
 
-* `-Dspark`: `{major}.{minor}`: For example, specify `-Dspark=3.4` to build for Spark 3.4.
+* `-Dspark`: `{major}.{minor}`: For example, specify `-Dspark=3.5` to build for Spark 3.5.
 * `-Dscala`: `2.12` or `2.13`
 
-=== "Spark 3.4+ Scala 2.12"
+=== "Spark 3.5 Scala 2.12"
 	```
-	mvn clean install -DskipTests -Dspark=3.4 -Dscala=2.12
+	mvn clean install -DskipTests -Dspark=3.5 -Dscala=2.12
 	```
-    Please replace `3.4` with Spark major.minor version when building for higher Spark versions.
-=== "Spark 3.4+ Scala 2.13"
+    Scala 2.12 is only available for Spark 3.5. Spark 4.0 and above are built with Scala 2.13 only.
+=== "Spark 3.5+ Scala 2.13"
 	```
-	mvn clean install -DskipTests -Dspark=3.4 -Dscala=2.13
+	mvn clean install -DskipTests -Dspark=3.5 -Dscala=2.13
 	```
-    Please replace `3.4` with Spark major.minor version when building for higher Spark versions.
+    Please replace `3.5` with Spark major.minor version when building for higher Spark versions.
 
 !!!tip
-	To get the Sedona Spark Shaded jar with all GeoTools jars included, simply append `-Dgeotools` option. The command is like this:`mvn clean install -DskipTests -Dscala=2.12 -Dspark=3.4 -Dgeotools`
+	To get the Sedona Spark Shaded jar with all GeoTools jars included, simply append `-Dgeotools` option. The command is like this:`mvn clean install -DskipTests -Dscala=2.12 -Dspark=3.5 -Dgeotools`
 
 ### Download staged jars
 
@@ -78,7 +78,7 @@ Sedona uses GitHub Actions to automatically generate jars per commit. You can go
 1) Set up Spark (download if needed) and environment variables
 
 ```bash
-export SPARK_VERSION=3.4.0   # or another supported version
+export SPARK_VERSION=3.5.0   # or another supported version
 wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz
 tar -xvzf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 rm spark-${SPARK_VERSION}-bin-hadoop3.tgz

docs/setup/compile.zh.md

@@ -46,28 +46,28 @@ Sedona Scala/Java 代码是一个多模块项目，每个模块都是 Scala/Java
 	Geotools 相关 jar 会被打入生成的 fat jar 中。
 
 !!!note
-	默认情况下，该命令会针对 Spark 3.4 和 Scala 2.12 编译 Sedona。
+	默认情况下，该命令会针对 Spark 3.5 和 Scala 2.12 编译 Sedona。
 
 ### 针对不同目标编译
 
 可使用 `-Dspark` 与 `-Dscala` 命令行参数指定不同的目标：
 
-* `-Dspark`：`{major}.{minor}`，例如 `-Dspark=3.4` 表示针对 Spark 3.4 编译。
+* `-Dspark`：`{major}.{minor}`，例如 `-Dspark=3.5` 表示针对 Spark 3.5 编译。
 * `-Dscala`：`2.12` 或 `2.13`
 
-=== "Spark 3.4+ Scala 2.12"
+=== "Spark 3.5 Scala 2.12"
 	```
-	mvn clean install -DskipTests -Dspark=3.4 -Dscala=2.12
+	mvn clean install -DskipTests -Dspark=3.5 -Dscala=2.12
 	```
-    若要针对更高 Spark 版本编译，请将 `3.4` 替换为对应的 Spark major.minor 版本。
-=== "Spark 3.4+ Scala 2.13"
+    Scala 2.12 仅适用于 Spark 3.5。Spark 4.0 及以上仅提供 Scala 2.13 构建。
+=== "Spark 3.5+ Scala 2.13"
 	```
-	mvn clean install -DskipTests -Dspark=3.4 -Dscala=2.13
+	mvn clean install -DskipTests -Dspark=3.5 -Dscala=2.13
 	```
-    若要针对更高 Spark 版本编译，请将 `3.4` 替换为对应的 Spark major.minor 版本。
+    若要针对更高 Spark 版本编译，请将 `3.5` 替换为对应的 Spark major.minor 版本。
 
 !!!tip
-	如需获取打包好所有 GeoTools jar 的 Sedona Spark Shaded jar，只需追加 `-Dgeotools` 选项，例如：`mvn clean install -DskipTests -Dscala=2.12 -Dspark=3.4 -Dgeotools`
+	如需获取打包好所有 GeoTools jar 的 Sedona Spark Shaded jar，只需追加 `-Dgeotools` 选项，例如：`mvn clean install -DskipTests -Dscala=2.12 -Dspark=3.5 -Dgeotools`
 
 ### 下载 staged jar
 
@@ -78,7 +78,7 @@ Sedona 通过 GitHub Actions 在每次提交时自动生成 jar。您可以在 [
 1) 准备 Spark（如未安装则下载）并设置环境变量
 
 ```bash
-export SPARK_VERSION=3.4.0   # 或其他受支持的版本
+export SPARK_VERSION=3.5.0   # 或其他受支持的版本
 wget https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz
 tar -xvzf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 rm spark-${SPARK_VERSION}-bin-hadoop3.tgz

docs/setup/docker.md

@@ -115,7 +115,7 @@ Format:
 Example:
 
 ```bash
-./docker/build.sh 3.4.1 {{ sedona.current_version }}
+./docker/build.sh 4.0.1 {{ sedona.current_version }}
 ```
 
 `build_mode` is optional. If its value is not given or is `local`, the script will build the image locally. Otherwise, it will start a cross-platform compilation and push images directly to DockerHub.
@@ -133,7 +133,7 @@ Format:
 Example:
 
 ```bash
-./docker/build.sh 3.4.1 latest
+./docker/build.sh 4.0.1 latest
 ```
 
 `build_mode` is optional. If its value is not given or is `local`, the script will build the image locally. Otherwise, it will start a cross-platform compilation and push images directly to DockerHub.
@@ -175,5 +175,5 @@ Example:
 
 ```bash
 docker login
-./docker/build.sh 3.4.1 {{ sedona.current_version }} release
+./docker/build.sh 4.0.1 {{ sedona.current_version }} release
 ```

docs/setup/docker.zh.md

@@ -115,7 +115,7 @@ docker run -d -e DRIVER_MEM=6g -e EXECUTOR_MEM=8g \
 示例：
 
 ```bash
-./docker/build.sh 3.4.1 {{ sedona.current_version }}
+./docker/build.sh 4.0.1 {{ sedona.current_version }}
 ```
 
 `build_mode` 为可选项。如果未提供或值为 `local`，脚本会在本地构建镜像；否则会启动跨平台编译并直接将镜像推送到 DockerHub。
@@ -133,7 +133,7 @@ docker run -d -e DRIVER_MEM=6g -e EXECUTOR_MEM=8g \
 示例：
 
 ```bash
-./docker/build.sh 3.4.1 latest
+./docker/build.sh 4.0.1 latest
 ```
 
 `build_mode` 为可选项，含义与上文相同。
@@ -175,5 +175,5 @@ docker login
 
 ```bash
 docker login
-./docker/build.sh 3.4.1 {{ sedona.current_version }} release
+./docker/build.sh 4.0.1 {{ sedona.current_version }} release
 ```

docs/setup/emr.md

@@ -17,12 +17,12 @@
  under the License.
  -->
 
-We recommend Sedona-1.3.1-incubating and above for EMR. In the tutorial, we use AWS Elastic MapReduce (EMR) 6.9.0. It has the following applications installed: Hadoop 3.3.3, JupyterEnterpriseGateway 2.6.0, Livy 0.7.1, Spark 3.3.0.
+In the tutorial, we use AWS Elastic MapReduce (EMR) 7.9.0. It has the following applications installed: Hadoop 3.4.1, JupyterEnterpriseGateway 2.6.0, Livy 0.8.0-incubating, Spark 3.5.5. Any EMR 7.x release works, since they all ship Spark 3.5.
 
 This tutorial is tested on EMR on EC2 with EMR Studio (notebooks). EMR on EC2 uses YARN to manage resources.
 
 !!!note
-	If you are using Spark 3.4+ and Scala 2.12, please use `sedona-spark-shaded-3.4_2.12`. Please pay attention to the Spark version postfix and Scala version postfix.
+	Use the artifact whose Spark major.minor version matches your cluster: `sedona-spark-shaded-3.5_2.12` for Spark 3.5, `sedona-spark-shaded-4.0_2.13` for Spark 4.0, `sedona-spark-shaded-4.1_2.13` for Spark 4.1. Spark 4.0 and above are built with Scala 2.13 only.
 
 ## Prepare initialization script
 
@@ -35,7 +35,7 @@ In your S3 bucket, add a script that has the following content:
 sudo mkdir /jars
 
 # Download Sedona jar
-sudo curl -o /jars/sedona-spark-shaded-3.3_2.12-{{ sedona.current_version }}.jar "https://repo1.maven.org/maven2/org/apache/sedona/sedona-spark-shaded-3.3_2.12/{{ sedona.current_version }}/sedona-spark-shaded-3.3_2.12-{{ sedona.current_version }}.jar"
+sudo curl -o /jars/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar "https://repo1.maven.org/maven2/org/apache/sedona/sedona-spark-shaded-3.5_2.12/{{ sedona.current_version }}/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar"
 
 # Download GeoTools jar
 sudo curl -o /jars/geotools-wrapper-{{ sedona.current_geotools }}.jar "https://repo1.maven.org/maven2/org/datasyslab/geotools-wrapper/{{ sedona.current_geotools }}/geotools-wrapper-{{ sedona.current_geotools }}.jar"
@@ -60,7 +60,7 @@ When you create an EMR cluster, in the software configuration, add the following
   {
     "Classification":"spark-defaults",
     "Properties":{
-      "spark.yarn.dist.jars": "/jars/sedona-spark-shaded-3.3_2.12-{{ sedona.current_version }}.jar,/jars/geotools-wrapper-{{ sedona.current_geotools }}.jar",
+      "spark.yarn.dist.jars": "/jars/sedona-spark-shaded-3.5_2.12-{{ sedona.current_version }}.jar,/jars/geotools-wrapper-{{ sedona.current_geotools }}.jar",
       "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
       "spark.kryo.registrator": "org.apache.sedona.core.serde.SedonaKryoRegistrator",
       "spark.sql.extensions": "org.apache.sedona.viz.sql.SedonaVizExtensions,org.apache.sedona.sql.SedonaSqlExtensions"