diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala index 70950d44988..95e50228697 100644 --- a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala +++ b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala @@ -53,9 +53,9 @@ object ValuesMapper { case (GeoPackageType.BOOLEAN, _) => rs.getBoolean(column.name) case (GeoPackageType.DATE, _) => - DataTypesTransformations.getDays(rs.getString(column.name)) + parseNullable(rs.getString(column.name))(DataTypesTransformations.getDays) case (GeoPackageType.DATETIME, _) => - DataTypesTransformations.epoch(rs.getString(column.name)) * 1000 + parseNullable(rs.getString(column.name))(DataTypesTransformations.epoch(_) * 1000) case (GeoPackageType.POINT, _) => GeometryReader.extractWKB(rs.getBytes(column.name)) case (GeoPackageType.LINESTRING, _) => @@ -77,4 +77,12 @@ object ValuesMapper { } }) } + + /** + * A SQL NULL in a DATE or DATETIME column comes back from the driver as a null string. The + * parsers in [[DataTypesTransformations]] throw NullPointerException on null input, so map NULL + * straight through instead of parsing it. + */ + private def parseNullable[T](value: String)(parse: String => T): Any = + if (value == null) null else parse(value) } diff --git a/spark/common/src/test/resources/geopackage/test_null_datetime.gpkg b/spark/common/src/test/resources/geopackage/test_null_datetime.gpkg new file mode 100644 index 00000000000..9c4e134a39a Binary files /dev/null and b/spark/common/src/test/resources/geopackage/test_null_datetime.gpkg differ diff --git a/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala b/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index f37298661fe..059c4f34fdb 100644 --- a/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala +++ b/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -212,6 +212,37 @@ class GeoPackageReaderTest extends TestBaseScala with Matchers { metadataDf.select("last_change").collect() } } + + it("should read NULL date and datetime values as null") { + // A NULL in a DATE or DATETIME column used to surface as a NullPointerException + // from DataTypesTransformations and abort the whole scan. + val df = sparkSession.read + .format("geopackage") + .option("tableName", "test_features") + .load(resourceFolder + "geopackage/test_null_datetime.gpkg") + + df.schema.fields.find(_.name == "event_date").get.dataType shouldEqual DateType + df.schema.fields.find(_.name == "event_time").get.dataType shouldEqual TimestampType + + val rows = df + .select("fid", "event_date", "event_time") + .collect() + .map { row => + val date = Option(row.getAs[Date]("event_date")).map(_.toLocalDate.toString).orNull + val time = Option(row.getAs[Timestamp]("event_time")).map(_.toInstant.toString).orNull + (row.getInt(0), date, time) + } + .sortBy(_._1) + + rows shouldEqual Array( + (1, "2024-01-15", "2024-01-15T10:30:00Z"), + (2, null, null), + (3, "2024-03-01", null), + (4, null, "2024-03-01T00:00:00Z")) + + df.filter("event_date IS NULL").count() shouldEqual 2 + df.filter("event_time IS NULL").count() shouldEqual 2 + } } describe("GeoPackage Raster Data Test") { diff --git a/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala b/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index f37298661fe..059c4f34fdb 100644 --- a/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala +++ b/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -212,6 +212,37 @@ class GeoPackageReaderTest extends TestBaseScala with Matchers { metadataDf.select("last_change").collect() } } + + it("should read NULL date and datetime values as null") { + // A NULL in a DATE or DATETIME column used to surface as a NullPointerException + // from DataTypesTransformations and abort the whole scan. + val df = sparkSession.read + .format("geopackage") + .option("tableName", "test_features") + .load(resourceFolder + "geopackage/test_null_datetime.gpkg") + + df.schema.fields.find(_.name == "event_date").get.dataType shouldEqual DateType + df.schema.fields.find(_.name == "event_time").get.dataType shouldEqual TimestampType + + val rows = df + .select("fid", "event_date", "event_time") + .collect() + .map { row => + val date = Option(row.getAs[Date]("event_date")).map(_.toLocalDate.toString).orNull + val time = Option(row.getAs[Timestamp]("event_time")).map(_.toInstant.toString).orNull + (row.getInt(0), date, time) + } + .sortBy(_._1) + + rows shouldEqual Array( + (1, "2024-01-15", "2024-01-15T10:30:00Z"), + (2, null, null), + (3, "2024-03-01", null), + (4, null, "2024-03-01T00:00:00Z")) + + df.filter("event_date IS NULL").count() shouldEqual 2 + df.filter("event_time IS NULL").count() shouldEqual 2 + } } describe("GeoPackage Raster Data Test") { diff --git a/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala b/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index f37298661fe..059c4f34fdb 100644 --- a/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala +++ b/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -212,6 +212,37 @@ class GeoPackageReaderTest extends TestBaseScala with Matchers { metadataDf.select("last_change").collect() } } + + it("should read NULL date and datetime values as null") { + // A NULL in a DATE or DATETIME column used to surface as a NullPointerException + // from DataTypesTransformations and abort the whole scan. + val df = sparkSession.read + .format("geopackage") + .option("tableName", "test_features") + .load(resourceFolder + "geopackage/test_null_datetime.gpkg") + + df.schema.fields.find(_.name == "event_date").get.dataType shouldEqual DateType + df.schema.fields.find(_.name == "event_time").get.dataType shouldEqual TimestampType + + val rows = df + .select("fid", "event_date", "event_time") + .collect() + .map { row => + val date = Option(row.getAs[Date]("event_date")).map(_.toLocalDate.toString).orNull + val time = Option(row.getAs[Timestamp]("event_time")).map(_.toInstant.toString).orNull + (row.getInt(0), date, time) + } + .sortBy(_._1) + + rows shouldEqual Array( + (1, "2024-01-15", "2024-01-15T10:30:00Z"), + (2, null, null), + (3, "2024-03-01", null), + (4, null, "2024-03-01T00:00:00Z")) + + df.filter("event_date IS NULL").count() shouldEqual 2 + df.filter("event_time IS NULL").count() shouldEqual 2 + } } describe("GeoPackage Raster Data Test") {