From e680c999aed5677a76c8af8b609213917e0fbed5 Mon Sep 17 00:00:00 2001 From: Jia Yu Date: Fri, 11 Sep 2026 12:07:54 -0700 Subject: [PATCH] [GH-3352] Read NULL GeoPackage date and datetime values as null ValuesMapper handed the raw string from the result set straight to DataTypesTransformations, whose parsers throw NullPointerException on null input. A single SQL NULL in a DATE or DATETIME column therefore aborted the whole scan. Map NULL through as null instead of parsing it, matching what the other column branches already do. The new fixture test_null_datetime.gpkg is a four-row GDAL-written layer with NULLs in both a DATE and a DATETIME column. --- .../geopackage/transform/ValuesMapper.scala | 12 +++++-- .../geopackage/test_null_datetime.gpkg | Bin 0 -> 73728 bytes .../sedona/sql/GeoPackageReaderTest.scala | 31 ++++++++++++++++++ .../sedona/sql/GeoPackageReaderTest.scala | 31 ++++++++++++++++++ .../sedona/sql/GeoPackageReaderTest.scala | 31 ++++++++++++++++++ 5 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 spark/common/src/test/resources/geopackage/test_null_datetime.gpkg diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala index 70950d44988..95e50228697 100644 --- a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala +++ b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala @@ -53,9 +53,9 @@ object ValuesMapper { case (GeoPackageType.BOOLEAN, _) => rs.getBoolean(column.name) case (GeoPackageType.DATE, _) => - DataTypesTransformations.getDays(rs.getString(column.name)) + parseNullable(rs.getString(column.name))(DataTypesTransformations.getDays) case (GeoPackageType.DATETIME, _) => - DataTypesTransformations.epoch(rs.getString(column.name)) * 1000 + parseNullable(rs.getString(column.name))(DataTypesTransformations.epoch(_) * 1000) case (GeoPackageType.POINT, _) => GeometryReader.extractWKB(rs.getBytes(column.name)) case (GeoPackageType.LINESTRING, _) => @@ -77,4 +77,12 @@ object ValuesMapper { } }) } + + /** + * A SQL NULL in a DATE or DATETIME column comes back from the driver as a null string. The + * parsers in [[DataTypesTransformations]] throw NullPointerException on null input, so map NULL + * straight through instead of parsing it. + */ + private def parseNullable[T](value: String)(parse: String => T): Any = + if (value == null) null else parse(value) } diff --git a/spark/common/src/test/resources/geopackage/test_null_datetime.gpkg b/spark/common/src/test/resources/geopackage/test_null_datetime.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..9c4e134a39ac807dad2b97c80501f17f75ac2f46 GIT binary patch literal 73728 zcmeI5&u<&Y6~}k=i~3=Sa$<#T)z}jP#)>dylhjWtc56mjQ*0zsCYNwzJ6{q+3;ebm`kB+e`gb?YV7>n^vR|-1+DIY^1V8`;KmY_l00ck)1V8`; zKmY_l;29D)I&80OkY02S{*Lj!?+w!r93TJ!AOHd&00JNY0w4eaAOHgYZvwZE{ju10 ziW%v6SHAV~rFe$SPLh&TEl9eY=T|cE;YV>k6`#*s@mrbx;6hBuE}NHPI66D&4`x;t z`E)9=&=i@Tn3)Yv%mgFR+0&7!nMh<}YI-6V%LrRmo(_d)CZ{GM(a2Od z8l9YIr$aOxiI_UPk)VxS%IUJcQJ{sBskE?2RkM<-N~xC4$1*}YjVCRQ%XP^!(bptWW3R2RNorLtTxC4xn33KSm8?>bin1aV zjBY8Zx%D+UPcrFDGL_^v>UoV5yHNAKM3st?Dk*u1Xq%cYRjzzfKRmB8BkYOm&AZI! zRH~?)Q|dSQrcrt0gfxYAxryDBjrgDS{*s|D93TJ!AOHd&00JNY0w4eaAOHd&00RGe z0ws2Udzl+N=b=Xj4fS=gDCP7GRnqF8j>JePGCdxOj)%iSXnJlkIv1WkJv}>n`QEh; z`_HX^^x%B_i(^|4;dy?`g)ZZ>NEC5C8!X009sH0T2KI5C8!X009s< zv;=mI|E=%;KV-ZQ4{iIPI0%3M2!H?xfB*=900@8p2!H?xfWT3!3m00ck)1V8`;KmY_l00ck)1okFiJpV`hzc(jHfB*=900@8p z2!H?xfB*=900@A-$>woakU#(hPH3U11a*VUw zzs9b<|M>^k?!Eu{*JD@z5wU=37Y+~r0T2KI5C8!X009sH0T2KI5CDP0 zO5iBl$+5bm>0(jJ=^Lt~b=jLTIP{1a?i^)?{(j{C(4U6Z2fjY?>%OZ49}kEgweQ95 zPr9#nebRNgbCmspdCy)B=OFMMC$N2X*fV-@y!YL1U6o5ENiD5kFNwNbmc&YqCfyXb zs?~~Emfn%dqO53=s=FoT(|k*AxJn&E4uAS^xRuziEu3%a2$C@u9oQ* zr$+KsMbp)stmt#3*))>RDN0o*Z%L#qX&TYjataA?0di@Pr+SVh`Aes5(q~C1@G9Yx z3oG6p->FmVs(EMa?Xs*(q9(n)At`yubvaz;6nkxnC$7vfC0CJxv}EAc*&&aQhnAD{A7&yi*mtF2@AZ@D=X>5axA?{UguXyEGwiENm_iFPYNWNqVMd| zQjqwK|4sFi6`D!8`VE!*MBv{rE@=NNx@hbY`^f_B?naWaX>274iDf?U)4?8JW}IbY zr6Ap`H8QuMSIuj&#Wce1l%d=6M?AjqarPHcv;FRT_w}HIf;*RtnU#n}Q_Pk$TWjmh zI->@On&sb+3v^q0r`$D3F0JXU*{1L8P}I!Zso=VNlls_AQIoeMvXIIeX3|i}%G_+9 zIiE^q1nLyDH`lL=x?XW~w~ay2X7Yictyqy3t5ifAEyF((@uX=mwi*H?&0nA{k(}o< z#KcVNoF9-AFxwYRl%wDp`W=gOig6-8}R zBV#qeKzyLbcV%kdeoD!UExu~>$B}m#kH958(BI)X;n6H*MDvD-yzdNzF zwXfYGJ9l=_s=1wGH!9IL*QIs>JK4q-5faH&!-*PRS7}YM^)&T0<7P%HHM{lM)eh1L z$6#P8;&8bQS~_pHl58?@F-zx~rV~2ksf!x&inHTw!u#?}w>9O(r?;g4EqCe65ApkXt(UqzzB6aoTUVNfSuLrwx2*BXRKT9l(`u0RY+JuL zrxJVD2W#;My~~EuoE#ST3$g5yHO~fOJw3kcnO$+&?_2Xbdy?n&k2}mo!krt7muQde6=<=*+tSZL1ldIz=E8`~%!pX(Clgo1_i1P@kh12@gG4cM-Eq172w!ySy7;~=AJV{naj_4t;#U3<9R3pGcmUv@`3 z_g~a9{CZI%;_Wq}mF-T}oO^Evn#SRp2Ta>{rFFELic7+pJ=(0Lb{J6|j!{)hIYr*0 z(^1_r8l$W|i#kqKEzqf|?etxcin1b`Rk+O2ZFd2C+%`DyduD+3G5srjeV#w}l)48x zKkLY|zM8O_qzZ*-^F&%WwBX!O^nj&Do+XIG6>HZ(K-E2(q~ ziH>Tn5YVtzGYoVT`bj+y++OurEMMK7rM0{lH`a|%%5+0ptX=oO9V2P$-i|sNEVC=d zm`YL?h+XO_V{1##gT&SjC7z6JEd*+^O}w-(JFRg!;4Xz-;hL>C*gofd3cG4gXs0bK zS4TXfm#EK2ce2x3@M}e>2Q}1ol{9EJE!83R+)UM}MZ2A#uyZHKS>P-QTbn`Ki^nXs zzWuR1prJbM{w=dqhi+{*)%J-}c+qnC@E$15jh3ghQhOac8>zLW^K@I42OmyrU@ax~ zZ=#lb(?S;es5O%i3sPC4PXinq0-|G=-f`ht z`}y}@ji1CW2y}~ofj%5D7Jy0Oei&dqJgM1D%f1JFsv;1Fn&h18dYpu7PeRJo8MiJv!=e dk0p(uq4t2`dChWmRiAL7ThqJ|G}wof{{qle&o%%6 literal 0 HcmV?d00001 diff --git a/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala b/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index f37298661fe..059c4f34fdb 100644 --- a/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala +++ b/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -212,6 +212,37 @@ class GeoPackageReaderTest extends TestBaseScala with Matchers { metadataDf.select("last_change").collect() } } + + it("should read NULL date and datetime values as null") { + // A NULL in a DATE or DATETIME column used to surface as a NullPointerException + // from DataTypesTransformations and abort the whole scan. + val df = sparkSession.read + .format("geopackage") + .option("tableName", "test_features") + .load(resourceFolder + "geopackage/test_null_datetime.gpkg") + + df.schema.fields.find(_.name == "event_date").get.dataType shouldEqual DateType + df.schema.fields.find(_.name == "event_time").get.dataType shouldEqual TimestampType + + val rows = df + .select("fid", "event_date", "event_time") + .collect() + .map { row => + val date = Option(row.getAs[Date]("event_date")).map(_.toLocalDate.toString).orNull + val time = Option(row.getAs[Timestamp]("event_time")).map(_.toInstant.toString).orNull + (row.getInt(0), date, time) + } + .sortBy(_._1) + + rows shouldEqual Array( + (1, "2024-01-15", "2024-01-15T10:30:00Z"), + (2, null, null), + (3, "2024-03-01", null), + (4, null, "2024-03-01T00:00:00Z")) + + df.filter("event_date IS NULL").count() shouldEqual 2 + df.filter("event_time IS NULL").count() shouldEqual 2 + } } describe("GeoPackage Raster Data Test") { diff --git a/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala b/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index f37298661fe..059c4f34fdb 100644 --- a/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala +++ b/spark/spark-4.0/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -212,6 +212,37 @@ class GeoPackageReaderTest extends TestBaseScala with Matchers { metadataDf.select("last_change").collect() } } + + it("should read NULL date and datetime values as null") { + // A NULL in a DATE or DATETIME column used to surface as a NullPointerException + // from DataTypesTransformations and abort the whole scan. + val df = sparkSession.read + .format("geopackage") + .option("tableName", "test_features") + .load(resourceFolder + "geopackage/test_null_datetime.gpkg") + + df.schema.fields.find(_.name == "event_date").get.dataType shouldEqual DateType + df.schema.fields.find(_.name == "event_time").get.dataType shouldEqual TimestampType + + val rows = df + .select("fid", "event_date", "event_time") + .collect() + .map { row => + val date = Option(row.getAs[Date]("event_date")).map(_.toLocalDate.toString).orNull + val time = Option(row.getAs[Timestamp]("event_time")).map(_.toInstant.toString).orNull + (row.getInt(0), date, time) + } + .sortBy(_._1) + + rows shouldEqual Array( + (1, "2024-01-15", "2024-01-15T10:30:00Z"), + (2, null, null), + (3, "2024-03-01", null), + (4, null, "2024-03-01T00:00:00Z")) + + df.filter("event_date IS NULL").count() shouldEqual 2 + df.filter("event_time IS NULL").count() shouldEqual 2 + } } describe("GeoPackage Raster Data Test") { diff --git a/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala b/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index f37298661fe..059c4f34fdb 100644 --- a/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala +++ b/spark/spark-4.1/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -212,6 +212,37 @@ class GeoPackageReaderTest extends TestBaseScala with Matchers { metadataDf.select("last_change").collect() } } + + it("should read NULL date and datetime values as null") { + // A NULL in a DATE or DATETIME column used to surface as a NullPointerException + // from DataTypesTransformations and abort the whole scan. + val df = sparkSession.read + .format("geopackage") + .option("tableName", "test_features") + .load(resourceFolder + "geopackage/test_null_datetime.gpkg") + + df.schema.fields.find(_.name == "event_date").get.dataType shouldEqual DateType + df.schema.fields.find(_.name == "event_time").get.dataType shouldEqual TimestampType + + val rows = df + .select("fid", "event_date", "event_time") + .collect() + .map { row => + val date = Option(row.getAs[Date]("event_date")).map(_.toLocalDate.toString).orNull + val time = Option(row.getAs[Timestamp]("event_time")).map(_.toInstant.toString).orNull + (row.getInt(0), date, time) + } + .sortBy(_._1) + + rows shouldEqual Array( + (1, "2024-01-15", "2024-01-15T10:30:00Z"), + (2, null, null), + (3, "2024-03-01", null), + (4, null, "2024-03-01T00:00:00Z")) + + df.filter("event_date IS NULL").count() shouldEqual 2 + df.filter("event_time IS NULL").count() shouldEqual 2 + } } describe("GeoPackage Raster Data Test") {