From 79a49108b34046c39b5851dda51d7f52960dcb30 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Mon, 6 Jul 2026 18:56:02 -0700 Subject: [PATCH 1/8] feat(rust/sedona-raster-gdal): add RS_FromGDALRaster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode GDAL-readable raster bytes (e.g. GeoTIFF) into an in-db raster — the inverse of RS_AsGeoTiff and the binary counterpart of RS_FromPath. Ported from the GDAL raster draft #704 reusing that implementation, but adapted to main: decode via the public open + append_as_indb_raster path (dropping #704's test-only load_as_indb_raster and the arrow umbrella concat), accumulating into a single RasterBuilder. Tests reworked onto the RasterSpec/ScalarUdfTester harness with GDAL-generated bytes (no .tiff fixtures); adds a benchmark and docs. --- docs/reference/sql/rs_fromgdalraster.qmd | 47 +++ rust/sedona-raster-gdal/Cargo.toml | 5 + .../benches/rs_from_gdal_raster.rs | 55 ++++ rust/sedona-raster-gdal/src/lib.rs | 2 + rust/sedona-raster-gdal/src/register.rs | 1 + .../src/rs_from_gdal_raster.rs | 276 ++++++++++++++++++ 6 files changed, 386 insertions(+) create mode 100644 docs/reference/sql/rs_fromgdalraster.qmd create mode 100644 rust/sedona-raster-gdal/benches/rs_from_gdal_raster.rs create mode 100644 rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs diff --git a/docs/reference/sql/rs_fromgdalraster.qmd b/docs/reference/sql/rs_fromgdalraster.qmd new file mode 100644 index 0000000000..49bcfc2340 --- /dev/null +++ b/docs/reference/sql/rs_fromgdalraster.qmd @@ -0,0 +1,47 @@ +--- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +title: RS_FromGDALRaster +description: > + Decodes a GDAL-readable raster (e.g. a GeoTIFF) from binary content into an + in-database raster. +kernels: + - returns: raster + args: + - name: content + type: binary + description: > + Raster file bytes in any format GDAL can read (GeoTIFF, PNG, …). A NULL + input yields a NULL raster. +--- + +## Description + +`RS_FromGDALRaster` parses raster file bytes with GDAL and returns an in-database +raster with all band data materialised inline — the inverse of +[`RS_AsGeoTiff`](rs_asgeotiff.qmd), and the binary counterpart of +[`RS_FromPath`](rs_frompath.qmd) (which references a file on disk as an out-db +raster instead). A `NULL` input yields a `NULL` raster. + +It is typically applied to a binary column holding encoded rasters — for example +rasters produced by `RS_AsGeoTiff`, which round-trips back through +`RS_FromGDALRaster`: + +```text +SELECT RS_Width(RS_FromGDALRaster(RS_AsGeoTiff(RS_Example()))); +``` diff --git a/rust/sedona-raster-gdal/Cargo.toml b/rust/sedona-raster-gdal/Cargo.toml index 03a82513d8..5f0b961af6 100644 --- a/rust/sedona-raster-gdal/Cargo.toml +++ b/rust/sedona-raster-gdal/Cargo.toml @@ -58,6 +58,11 @@ sedona-testing = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +[[bench]] +harness = false +name = "rs_from_gdal_raster" +path = "benches/rs_from_gdal_raster.rs" + [[bench]] harness = false name = "rs_frompath" diff --git a/rust/sedona-raster-gdal/benches/rs_from_gdal_raster.rs b/rust/sedona-raster-gdal/benches/rs_from_gdal_raster.rs new file mode 100644 index 0000000000..10ce6b01c8 --- /dev/null +++ b/rust/sedona-raster-gdal/benches/rs_from_gdal_raster.rs @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmarks for the RS_FromGDALRaster UDF (GeoTIFF bytes → in-db raster). + +use std::hint::black_box; +use std::sync::Arc; + +use arrow_array::{ArrayRef, BinaryArray}; +use arrow_schema::DataType; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use datafusion_expr::ScalarUDF; +use sedona_schema::datatypes::SedonaType; +use sedona_testing::{data::test_raster, testers::ScalarUdfTester}; + +/// Read a fixture GeoTIFF's bytes and replicate them across `rows` (mirrors the +/// fixture-driven inputs used by the rs_frompath / rs_metadata benchmarks). +fn geotiff_bytes_array(name: &str, rows: usize) -> ArrayRef { + assert!(rows > 0, "benchmark rows must be positive"); + let path = test_raster(name).unwrap(); + let bytes = std::fs::read(path).unwrap(); + Arc::new(BinaryArray::from(vec![bytes.as_slice(); rows])) +} + +fn bench_rs_from_gdal_raster(c: &mut Criterion) { + let udf: ScalarUDF = sedona_raster_gdal::rs_from_gdal_raster_udf().into(); + let tester = ScalarUdfTester::new(udf, vec![SedonaType::Arrow(DataType::Binary)]); + + let mut group = c.benchmark_group("rs_from_gdal_raster"); + for rows in [1usize, 32] { + let input = geotiff_bytes_array("test4.tiff", rows); + group.throughput(Throughput::Elements(rows as u64)); + group.bench_with_input(BenchmarkId::new("decode", rows), &input, |b, input| { + b.iter(|| black_box(tester.invoke_arrays(vec![input.clone()]).unwrap())) + }); + } + group.finish(); +} + +criterion_group!(benches, bench_rs_from_gdal_raster); +criterion_main!(benches); diff --git a/rust/sedona-raster-gdal/src/lib.rs b/rust/sedona-raster-gdal/src/lib.rs index f878f34a12..6171a538c0 100644 --- a/rust/sedona-raster-gdal/src/lib.rs +++ b/rust/sedona-raster-gdal/src/lib.rs @@ -33,6 +33,7 @@ mod gdal_common; mod gdal_dataset_provider; mod raster_loader; +mod rs_from_gdal_raster; mod rs_frompath; mod rs_metadata; mod rs_polygonize; @@ -45,6 +46,7 @@ pub use gdal_common::{ nodata_bytes_to_f64, nodata_f64_to_bytes, GdalBandLayout, GdalBandPlan, }; pub use raster_loader::{GdalLoader, GDAL_FORMAT}; +pub use rs_from_gdal_raster::rs_from_gdal_raster_udf; pub use rs_frompath::rs_frompath_udf; pub use rs_metadata::rs_metadata_udf; pub use rs_polygonize::rs_polygonize_udf; diff --git a/rust/sedona-raster-gdal/src/register.rs b/rust/sedona-raster-gdal/src/register.rs index 52f7dfd488..6d70daf65a 100644 --- a/rust/sedona-raster-gdal/src/register.rs +++ b/rust/sedona-raster-gdal/src/register.rs @@ -20,6 +20,7 @@ use sedona_expr::function_set::FunctionSet; /// Export the set of GDAL-backed functions defined in this crate. pub fn default_function_set() -> FunctionSet { let mut function_set = FunctionSet::new(); + function_set.insert_scalar_udf(crate::rs_from_gdal_raster::rs_from_gdal_raster_udf()); function_set.insert_scalar_udf(crate::rs_frompath::rs_frompath_udf()); function_set.insert_scalar_udf(crate::rs_metadata::rs_metadata_udf()); function_set.insert_scalar_udf(crate::rs_polygonize::rs_polygonize_udf()); diff --git a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs new file mode 100644 index 0000000000..75249c83e0 --- /dev/null +++ b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs @@ -0,0 +1,276 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! RS_FromGDALRaster UDF - Parse binary content using GDAL driver as in-db raster +//! +//! Similar to PostGIS's ST_FromGDALRaster. Parses binary content using GDAL driver +//! and loads it as an in-db raster with all band data stored inline. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use arrow_array::Array; +use arrow_schema::DataType; +use datafusion_common::cast::as_binary_array; +use datafusion_common::config::ConfigOptions; +use datafusion_common::error::Result; +use datafusion_common::{exec_datafusion_err, ScalarValue}; +use datafusion_expr::{ColumnarValue, Volatility}; +use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF}; +use sedona_gdal::gdal::Gdal; +use sedona_gdal::gdal_dyn_bindgen::{GDAL_OF_RASTER, GDAL_OF_READONLY}; +use sedona_gdal::raster::types::DatasetOptions; +use sedona_raster::builder::RasterBuilder; +use sedona_schema::datatypes::{SedonaType, RASTER}; +use sedona_schema::matchers::ArgMatcher; + +use crate::gdal_common::{convert_gdal_err, with_gdal}; +use crate::gdal_dataset_provider::configure_thread_local_options; +use crate::utils::append_as_indb_raster; + +/// Counter for generating unique VSI memory file names +static VSI_FILE_COUNTER: AtomicUsize = AtomicUsize::new(0); + +/// RS_FromGDALRaster() scalar UDF implementation +/// +/// Parse binary content using GDAL driver and load it as in-db raster +pub fn rs_from_gdal_raster_udf() -> SedonaScalarUDF { + SedonaScalarUDF::new( + "rs_fromgdalraster", + vec![Arc::new(RsFromGDALRaster)], + Volatility::Immutable, + ) +} + +/// Kernel implementation for RS_FromGDALRaster +#[derive(Debug)] +pub(crate) struct RsFromGDALRaster; + +impl RsFromGDALRaster { + /// Generate a unique VSI memory file path + fn generate_vsi_path() -> String { + let counter = VSI_FILE_COUNTER.fetch_add(1, Ordering::SeqCst); + let thread_id = std::thread::current().id(); + format!( + "/vsimem/rs_from_gdal_raster_{:?}_{}.bin", + thread_id, counter + ) + } + + /// Write `content` to a temporary `/vsimem` file, open it with GDAL, and + /// append the decoded raster to `builder` as an in-db raster (all band data + /// materialised inline). The VSI file is always cleaned up. + fn append_gdal_raster(gdal: &Gdal, content: &[u8], builder: &mut RasterBuilder) -> Result<()> { + let vsi_path = Self::generate_vsi_path(); + gdal.create_mem_file(&vsi_path, content) + .map_err(|e| exec_datafusion_err!("Failed to create VSI memory file: {e}"))?; + + // Open + decode, then always unlink the VSI file (the dataset is dropped + // at the end of the closure, before the unlink). + let result = (|| { + let dataset = gdal + .open_ex_with_options( + &vsi_path, + DatasetOptions { + open_flags: GDAL_OF_RASTER | GDAL_OF_READONLY, + ..Default::default() + }, + ) + .map_err(convert_gdal_err)?; + append_as_indb_raster(&dataset, builder) + })(); + let _ = gdal.unlink_mem_file(&vsi_path); + result + } + + /// Parse binary content into a single in-db raster. Test-only convenience + /// around [`append_gdal_raster`](Self::append_gdal_raster); the kernel + /// appends directly into a shared builder. + #[cfg(test)] + pub(crate) fn parse_gdal_raster( + gdal: &Gdal, + content: &[u8], + ) -> Result { + let mut builder = RasterBuilder::new(1); + Self::append_gdal_raster(gdal, content, &mut builder)?; + builder + .finish() + .map_err(|e| exec_datafusion_err!("Failed to build raster: {e}")) + } +} + +impl SedonaScalarKernel for RsFromGDALRaster { + fn return_type(&self, args: &[SedonaType]) -> Result> { + let matcher = ArgMatcher::new(vec![ArgMatcher::is_binary()], RASTER); + matcher.match_args(args) + } + + fn invoke_batch( + &self, + arg_types: &[SedonaType], + args: &[ColumnarValue], + ) -> Result { + self.invoke_batch_from_args(arg_types, args, &SedonaType::Arrow(DataType::Null), 0, None) + } + + fn invoke_batch_from_args( + &self, + _arg_types: &[SedonaType], + args: &[ColumnarValue], + _return_type: &SedonaType, + _num_rows: usize, + config_options: Option<&ConfigOptions>, + ) -> Result { + with_gdal(|gdal| { + configure_thread_local_options(gdal, config_options)?; + + let content_array = match &args[0] { + ColumnarValue::Scalar(scalar) => scalar + .to_array() + .map_err(|e| exec_datafusion_err!("Failed to convert scalar to array: {e}"))?, + ColumnarValue::Array(array) => array.clone(), + }; + let binary_array = as_binary_array(&content_array)?; + let len = binary_array.len(); + + // Decode every row into one raster array. A NULL input row yields a + // NULL raster row; a non-null row is decoded to an in-db raster. + let mut builder = RasterBuilder::new(len); + for i in 0..len { + if binary_array.is_null(i) { + builder + .append_null() + .map_err(|e| exec_datafusion_err!("Failed to append null: {e}"))?; + } else { + Self::append_gdal_raster(gdal, binary_array.value(i), &mut builder)?; + } + } + let result = builder + .finish() + .map_err(|e| exec_datafusion_err!("Failed to build raster: {e}"))?; + + match &args[0] { + ColumnarValue::Scalar(_) => Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)), + ColumnarValue::Array(_) => Ok(ColumnarValue::Array(Arc::new(result))), + } + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::gdal_common::with_gdal; + use arrow_array::{ArrayRef, BinaryArray}; + use datafusion_common::cast::as_struct_array; + use sedona_gdal::raster::types::Buffer; + use sedona_raster::array::RasterStructArray; + use sedona_raster::traits::RasterRef; + use sedona_testing::testers::ScalarUdfTester; + + /// Build a small 4x4 single-band GeoTIFF (EPSG:4326) with GDAL and return its + /// bytes — the fixture-free stand-in for a `.tiff` on disk, so tests exercise + /// the real decode path without shipping a binary fixture. + fn make_geotiff_bytes(gdal: &Gdal) -> Vec { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("src.tif"); + let path_str = path.to_string_lossy().to_string(); + { + let driver = gdal.get_driver_by_name("GTiff").unwrap(); + let dataset = driver + .create_with_band_type::(&path_str, 4, 4, 1) + .unwrap(); + dataset + .set_geo_transform(&[0.0, 1.0, 0.0, 4.0, 0.0, -1.0]) + .unwrap(); + dataset.set_projection("EPSG:4326").unwrap(); + let band = dataset.rasterband(1).unwrap(); + let mut buffer = Buffer::new((4, 4), (0..16u8).collect::>()); + band.write((0, 0), (4, 4), &mut buffer).unwrap(); + } // drop flushes the dataset to disk + std::fs::read(&path).unwrap() + } + + fn from_gdal_tester() -> ScalarUdfTester { + ScalarUdfTester::new( + rs_from_gdal_raster_udf().into(), + vec![SedonaType::Arrow(DataType::Binary)], + ) + } + + #[test] + fn test_generate_vsi_path() { + let path1 = RsFromGDALRaster::generate_vsi_path(); + let path2 = RsFromGDALRaster::generate_vsi_path(); + + assert!(path1.starts_with("/vsimem/rs_from_gdal_raster_")); + assert!(path2.starts_with("/vsimem/rs_from_gdal_raster_")); + assert_ne!(path1, path2); + } + + #[test] + fn udf_from_gdal_raster() { + let udf: datafusion_expr::ScalarUDF = rs_from_gdal_raster_udf().into(); + assert_eq!(udf.name(), "rs_fromgdalraster"); + } + + #[test] + fn parse_gdal_raster_builds_indb_raster() { + // GeoTIFF bytes decode to an in-db raster with the source dimensions/CRS. + with_gdal(|gdal| { + let bytes = make_geotiff_bytes(gdal); + let arr = RsFromGDALRaster::parse_gdal_raster(gdal, &bytes)?; + let rasters = RasterStructArray::try_new(&arr).unwrap(); + assert_eq!(rasters.len(), 1); + let raster = rasters.get(0).unwrap(); + assert_eq!(raster.metadata().width(), 4); + assert_eq!(raster.metadata().height(), 4); + assert_eq!(raster.num_bands(), 1); + assert!(raster.crs().is_some(), "EPSG:4326 should survive decode"); + // In-db: band data is materialised inline, not an out-db reference. + assert!(raster.band_outdb_uri(0).is_none()); + Ok::<_, datafusion_common::DataFusionError>(()) + }) + .unwrap(); + } + + #[test] + fn from_gdal_raster_decodes_via_udf() { + // End-to-end through the UDF: GeoTIFF binary in, raster out. + let bytes = with_gdal(|gdal| Ok(make_geotiff_bytes(gdal))).unwrap(); + let input: ArrayRef = Arc::new(BinaryArray::from(vec![bytes.as_slice()])); + let result = from_gdal_tester().invoke_arrays(vec![input]).unwrap(); + let rasters = RasterStructArray::try_new(as_struct_array(&result).unwrap()).unwrap(); + let raster = rasters.get(0).unwrap(); + assert_eq!(raster.metadata().width(), 4); + assert_eq!(raster.metadata().height(), 4); + assert_eq!(raster.num_bands(), 1); + } + + #[test] + fn null_binary_yields_null_raster() { + let input: ArrayRef = Arc::new(BinaryArray::from(vec![None::<&[u8]>])); + let result = from_gdal_tester().invoke_arrays(vec![input]).unwrap(); + let struct_arr = as_struct_array(&result).unwrap(); + assert!( + struct_arr.is_null(0), + "NULL bytes should yield a NULL raster" + ); + } +} From eff9e074c9220769a60e2eb7d681cd0d112185ca Mon Sep 17 00:00:00 2001 From: jameswillis Date: Mon, 6 Jul 2026 20:03:00 -0700 Subject: [PATCH 2/8] docs(rs_fromgdalraster): drop link to RS_AsGeoTiff page (not present on this branch) RS_AsGeoTiff's doc page ships in a separate PR, so linking rs_asgeotiff.qmd breaks the mkdocs --strict build here. Reference it as plain text; keep the valid RS_FromPath link. --- docs/reference/sql/rs_fromgdalraster.qmd | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference/sql/rs_fromgdalraster.qmd b/docs/reference/sql/rs_fromgdalraster.qmd index 49bcfc2340..382f6e66b0 100644 --- a/docs/reference/sql/rs_fromgdalraster.qmd +++ b/docs/reference/sql/rs_fromgdalraster.qmd @@ -33,10 +33,10 @@ kernels: ## Description `RS_FromGDALRaster` parses raster file bytes with GDAL and returns an in-database -raster with all band data materialised inline — the inverse of -[`RS_AsGeoTiff`](rs_asgeotiff.qmd), and the binary counterpart of -[`RS_FromPath`](rs_frompath.qmd) (which references a file on disk as an out-db -raster instead). A `NULL` input yields a `NULL` raster. +raster with all band data materialised inline — the inverse of `RS_AsGeoTiff`, +and the binary counterpart of [`RS_FromPath`](rs_frompath.qmd) (which references +a file on disk as an out-db raster instead). A `NULL` input yields a `NULL` +raster. It is typically applied to a binary column holding encoded rasters — for example rasters produced by `RS_AsGeoTiff`, which round-trips back through From 8d7913cfb36865e9ef3b7318c18a4e577eeba9d0 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 16 Jul 2026 23:59:13 -0700 Subject: [PATCH 3/8] perf(rust/sedona-raster-gdal): move GDAL band bytes via append_band_data_buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The in-db decode path (append_as_indb_raster, used by RS_FromGDALRaster, RS_Metadata and RS_Polygonize) already handed each band's freshly-read Vec to Arrow without a copy, but did so by open-coding append_block + try_append_view. Route it through the dedicated RasterBuilder helper instead: it still attaches the allocation as a shared data block (a refcount bump, never a copy), and additionally stores sub-inline-threshold bands inline so the view stays canonical (a block-referencing view of <= 12 bytes fails array validation on roundtrip — a latent bug for small bands). Perf-neutral, as expected for an already-zero-copy path. rs_from_gdal_raster criterion bench (GeoTIFF bytes -> in-db raster), decode of test4.tiff: decode/1: 62.8 us -> 63.1 us (within noise threshold) decode/32: 1.741 ms -> 1.746 ms (no change detected) --- rust/sedona-raster-gdal/src/utils.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/rust/sedona-raster-gdal/src/utils.rs b/rust/sedona-raster-gdal/src/utils.rs index 7c8cc80c52..5ff9ea4619 100644 --- a/rust/sedona-raster-gdal/src/utils.rs +++ b/rust/sedona-raster-gdal/src/utils.rs @@ -84,12 +84,11 @@ pub fn append_as_indb_raster(dataset: &Dataset, builder: &mut RasterBuilder) -> .map_err(|e| exec_datafusion_err!("Failed to read band {} data: {}", band_idx, e))?; let band_data_len = u32::try_from(band_data.len()) .map_err(|_| exec_datafusion_err!("Band {} data too large for Arrow view", band_idx))?; - let block = builder - .band_data_writer() - .append_block(Buffer::from_vec(band_data)); + // Hand the freshly-read allocation to Arrow as a shared data block (a + // refcount bump, never a copy). `append_band_data_buffer` also stores + // sub-inline-threshold bands inline, keeping the view canonical. builder - .band_data_writer() - .try_append_view(block, 0, band_data_len) + .append_band_data_buffer(&Buffer::from_vec(band_data), 0, band_data_len) .map_err(|e| exec_datafusion_err!("Failed to append band {} data: {}", band_idx, e))?; builder From 1ea1242560d604068169278cd60032c66d659f63 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 16 Jul 2026 23:59:21 -0700 Subject: [PATCH 4/8] test(rust/sedona-raster-gdal): assert RS_FromGDALRaster results with raster_spec helpers The decode tests navigated the result by hand (RasterStructArray -> metadata -> band chains) and only checked width/height/band count. Replace that with assert_rasters_equal / assert_raster_scalar_equals against declarative RasterSpec expectations derived from each fixture's own GeoTIFF construction, which additionally pin the geotransform, per-band data type, nodata, in-db storage and exact pixel values. The CRS is read back from the fixture bytes (not copied from a decode result) so the spec pins CRS preservation without hard-coding a PROJJSON blob that drifts across PROJ versions. Also add coverage that was missing: - decoded_two_band_raster_is_zero_copy pins the no-copy property (one band-data buffer per band; a copying path would consolidate them). - error pathways: empty, unparseable and truncated bytes all error cleanly rather than panic. --- .../src/rs_from_gdal_raster.rs | 184 ++++++++++++++---- 1 file changed, 149 insertions(+), 35 deletions(-) diff --git a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs index 75249c83e0..41fb34151b 100644 --- a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs +++ b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs @@ -176,18 +176,23 @@ impl SedonaScalarKernel for RsFromGDALRaster { #[cfg(test)] mod tests { use super::*; - use crate::gdal_common::with_gdal; - use arrow_array::{ArrayRef, BinaryArray}; - use datafusion_common::cast::as_struct_array; + use arrow_array::{ArrayRef, BinaryArray, BinaryViewArray, ListArray, StructArray}; use sedona_gdal::raster::types::Buffer; - use sedona_raster::array::RasterStructArray; - use sedona_raster::traits::RasterRef; + use sedona_schema::raster::{band_indices, raster_indices}; + use sedona_testing::raster_spec::{ + assert_raster_scalar_equals, assert_rasters_equal, RasterSpec, + }; use sedona_testing::testers::ScalarUdfTester; /// Build a small 4x4 single-band GeoTIFF (EPSG:4326) with GDAL and return its - /// bytes — the fixture-free stand-in for a `.tiff` on disk, so tests exercise - /// the real decode path without shipping a binary fixture. - fn make_geotiff_bytes(gdal: &Gdal) -> Vec { + /// bytes together with the CRS GDAL reads back from them (PROJJSON) — the + /// fixture-free stand-in for a `.tiff` on disk, so tests exercise the real + /// decode path without shipping a binary fixture. + /// + /// The CRS is read from the fixture itself, not copied from a decode result, + /// so a [`RasterSpec`] can pin CRS preservation without hard-coding a + /// PROJJSON blob that drifts across PROJ versions. + fn make_geotiff_fixture(gdal: &Gdal) -> (Vec, Option) { let tmp = tempfile::tempdir().unwrap(); let path = tmp.path().join("src.tif"); let path_str = path.to_string_lossy().to_string(); @@ -204,9 +209,65 @@ mod tests { let mut buffer = Buffer::new((4, 4), (0..16u8).collect::>()); band.write((0, 0), (4, 4), &mut buffer).unwrap(); } // drop flushes the dataset to disk + let bytes = std::fs::read(&path).unwrap(); + let crs = gdal + .open_ex_with_options( + &path_str, + DatasetOptions { + open_flags: GDAL_OF_RASTER | GDAL_OF_READONLY, + ..Default::default() + }, + ) + .unwrap() + .spatial_ref() + .ok() + .and_then(|sr| sr.to_projjson().ok()); + (bytes, crs) + } + + /// Build a 4x4 two-band UInt8 GeoTIFF (no CRS). Each band's 16 bytes exceed + /// the inline view threshold, so an in-db decode attaches one shared data + /// block per band — the property [`decoded_two_band_raster_is_zero_copy`] + /// pins. + fn make_two_band_geotiff_bytes(gdal: &Gdal) -> Vec { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("two_band.tif"); + let path_str = path.to_string_lossy().to_string(); + { + let driver = gdal.get_driver_by_name("GTiff").unwrap(); + let dataset = driver + .create_with_band_type::(&path_str, 4, 4, 2) + .unwrap(); + dataset + .set_geo_transform(&[0.0, 1.0, 0.0, 4.0, 0.0, -1.0]) + .unwrap(); + let band1 = dataset.rasterband(1).unwrap(); + let mut b1 = Buffer::new((4, 4), (0..16u8).collect::>()); + band1.write((0, 0), (4, 4), &mut b1).unwrap(); + let band2 = dataset.rasterband(2).unwrap(); + let mut b2 = Buffer::new((4, 4), (100..116u8).collect::>()); + band2.write((0, 0), (4, 4), &mut b2).unwrap(); + } // drop flushes the dataset to disk std::fs::read(&path).unwrap() } + /// The band-data `BinaryViewArray` inside a raster `StructArray` + /// (bands list -> band struct -> data column). + fn band_data_view(arr: &StructArray) -> &BinaryViewArray { + arr.column(raster_indices::BANDS) + .as_any() + .downcast_ref::() + .unwrap() + .values() + .as_any() + .downcast_ref::() + .unwrap() + .column(band_indices::DATA) + .as_any() + .downcast_ref::() + .unwrap() + } + fn from_gdal_tester() -> ScalarUdfTester { ScalarUdfTester::new( rs_from_gdal_raster_udf().into(), @@ -214,6 +275,16 @@ mod tests { ) } + /// The single-band fixture's declarative expectation, derived from the + /// fixture's construction: 4x4 UInt8, north-up unit pixels with origin + /// (0, 4), sequential values 0..16, no nodata, in-db. + fn single_band_spec(crs: Option<&str>) -> RasterSpec { + RasterSpec::d2(4, 4) + .transform([0.0, 1.0, 0.0, 4.0, 0.0, -1.0]) + .crs(crs) + .band_values(&(0..16u8).collect::>()) + } + #[test] fn test_generate_vsi_path() { let path1 = RsFromGDALRaster::generate_vsi_path(); @@ -232,45 +303,88 @@ mod tests { #[test] fn parse_gdal_raster_builds_indb_raster() { - // GeoTIFF bytes decode to an in-db raster with the source dimensions/CRS. + // The direct builder path: GeoTIFF bytes decode to an in-db raster + // matching the fixture's dimensions, transform, CRS and pixel values. + // The spec's in-db bands assert (via storage_type) that band data was + // materialised inline rather than left as an out-db reference. with_gdal(|gdal| { - let bytes = make_geotiff_bytes(gdal); - let arr = RsFromGDALRaster::parse_gdal_raster(gdal, &bytes)?; - let rasters = RasterStructArray::try_new(&arr).unwrap(); - assert_eq!(rasters.len(), 1); - let raster = rasters.get(0).unwrap(); - assert_eq!(raster.metadata().width(), 4); - assert_eq!(raster.metadata().height(), 4); - assert_eq!(raster.num_bands(), 1); - assert!(raster.crs().is_some(), "EPSG:4326 should survive decode"); - // In-db: band data is materialised inline, not an out-db reference. - assert!(raster.band_outdb_uri(0).is_none()); - Ok::<_, datafusion_common::DataFusionError>(()) + let (bytes, crs) = make_geotiff_fixture(gdal); + let arr: ArrayRef = Arc::new(RsFromGDALRaster::parse_gdal_raster(gdal, &bytes)?); + assert_rasters_equal(&arr, &[Some(single_band_spec(crs.as_deref()))]); + Ok(()) }) .unwrap(); } #[test] fn from_gdal_raster_decodes_via_udf() { - // End-to-end through the UDF: GeoTIFF binary in, raster out. - let bytes = with_gdal(|gdal| Ok(make_geotiff_bytes(gdal))).unwrap(); - let input: ArrayRef = Arc::new(BinaryArray::from(vec![bytes.as_slice()])); - let result = from_gdal_tester().invoke_arrays(vec![input]).unwrap(); - let rasters = RasterStructArray::try_new(as_struct_array(&result).unwrap()).unwrap(); - let raster = rasters.get(0).unwrap(); - assert_eq!(raster.metadata().width(), 4); - assert_eq!(raster.metadata().height(), 4); - assert_eq!(raster.num_bands(), 1); + // End-to-end through the UDF's scalar path: GeoTIFF binary in, raster out. + let (bytes, crs) = with_gdal(|gdal| Ok(make_geotiff_fixture(gdal))).unwrap(); + let result = from_gdal_tester() + .invoke_scalar(ScalarValue::Binary(Some(bytes))) + .unwrap(); + assert_raster_scalar_equals(&result, &single_band_spec(crs.as_deref())); + } + + #[test] + fn decoded_two_band_raster_is_zero_copy() { + with_gdal(|gdal| { + let bytes = make_two_band_geotiff_bytes(gdal); + let arr = RsFromGDALRaster::parse_gdal_raster(gdal, &bytes)?; + + // Zero-copy: each band's freshly-read allocation is attached as its + // own shared data block (a refcount bump), so the band-data view + // has one buffer per band. A copying `append_value` path would + // consolidate both bands into a single builder-owned buffer. + assert_eq!(band_data_view(&arr).data_buffers().len(), 2); + + // ...and the decoded values/structure match the fixture. + let arr: ArrayRef = Arc::new(arr); + assert_rasters_equal( + &arr, + &[Some( + RasterSpec::d2(4, 4) + .crs(None) + .transform([0.0, 1.0, 0.0, 4.0, 0.0, -1.0]) + .band_values(&(0..16u8).collect::>()) + .band_values(&(100..116u8).collect::>()), + )], + ); + Ok(()) + }) + .unwrap(); } #[test] fn null_binary_yields_null_raster() { let input: ArrayRef = Arc::new(BinaryArray::from(vec![None::<&[u8]>])); let result = from_gdal_tester().invoke_arrays(vec![input]).unwrap(); - let struct_arr = as_struct_array(&result).unwrap(); - assert!( - struct_arr.is_null(0), - "NULL bytes should yield a NULL raster" - ); + assert_rasters_equal(&result, &[None]); + } + + #[test] + fn empty_binary_errors() { + // Non-null but empty bytes: GDAL has nothing to open. A clean error, + // not a panic. + let input: ArrayRef = Arc::new(BinaryArray::from(vec![Some(&b""[..])])); + assert!(from_gdal_tester().invoke_arrays(vec![input]).is_err()); + } + + #[test] + fn unparseable_bytes_error() { + // Bytes GDAL cannot identify as any raster format. + let input: ArrayRef = Arc::new(BinaryArray::from(vec![Some(&b"not a raster at all"[..])])); + assert!(from_gdal_tester().invoke_arrays(vec![input]).is_err()); + } + + #[test] + fn truncated_geotiff_errors() { + // A real GeoTIFF cut down to its 8-byte header: the IFD offset now + // points past EOF, so GDAL cannot open it. Exercises the malformed + // header path without a panic. + let (bytes, _) = with_gdal(|gdal| Ok(make_geotiff_fixture(gdal))).unwrap(); + let truncated = bytes[..8.min(bytes.len())].to_vec(); + let input: ArrayRef = Arc::new(BinaryArray::from(vec![Some(truncated.as_slice())])); + assert!(from_gdal_tester().invoke_arrays(vec![input]).is_err()); } } From 701b80600d948f7ca6c809c9b14ec5e9412c3c8e Mon Sep 17 00:00:00 2001 From: jameswillis Date: Fri, 17 Jul 2026 15:01:17 -0700 Subject: [PATCH 5/8] docs(rs_fromgdalraster): run the round-trip example as sql and link RS_AsGeoTiff --- docs/reference/sql/rs_fromgdalraster.qmd | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/sql/rs_fromgdalraster.qmd b/docs/reference/sql/rs_fromgdalraster.qmd index 382f6e66b0..36bf2f5e10 100644 --- a/docs/reference/sql/rs_fromgdalraster.qmd +++ b/docs/reference/sql/rs_fromgdalraster.qmd @@ -33,15 +33,15 @@ kernels: ## Description `RS_FromGDALRaster` parses raster file bytes with GDAL and returns an in-database -raster with all band data materialised inline — the inverse of `RS_AsGeoTiff`, -and the binary counterpart of [`RS_FromPath`](rs_frompath.qmd) (which references -a file on disk as an out-db raster instead). A `NULL` input yields a `NULL` -raster. +raster with all band data materialised inline — the inverse of +[`RS_AsGeoTiff`](rs_asgeotiff.qmd), and the binary counterpart of +[`RS_FromPath`](rs_frompath.qmd) (which references a file on disk as an out-db +raster instead). A `NULL` input yields a `NULL` raster. It is typically applied to a binary column holding encoded rasters — for example rasters produced by `RS_AsGeoTiff`, which round-trips back through `RS_FromGDALRaster`: -```text +```sql SELECT RS_Width(RS_FromGDALRaster(RS_AsGeoTiff(RS_Example()))); ``` From 6bade3c8050755a78d53de76ade6c9eec88b4508 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Fri, 17 Jul 2026 15:01:21 -0700 Subject: [PATCH 6/8] test(rust/sedona-raster-gdal): adopt RasterSpec::bbox and band_data_array helpers --- .../src/rs_from_gdal_raster.rs | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs index 41fb34151b..6f9b5523c1 100644 --- a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs +++ b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs @@ -176,9 +176,9 @@ impl SedonaScalarKernel for RsFromGDALRaster { #[cfg(test)] mod tests { use super::*; - use arrow_array::{ArrayRef, BinaryArray, BinaryViewArray, ListArray, StructArray}; + use arrow_array::{ArrayRef, BinaryArray}; use sedona_gdal::raster::types::Buffer; - use sedona_schema::raster::{band_indices, raster_indices}; + use sedona_raster::array::RasterStructArray; use sedona_testing::raster_spec::{ assert_raster_scalar_equals, assert_rasters_equal, RasterSpec, }; @@ -251,23 +251,6 @@ mod tests { std::fs::read(&path).unwrap() } - /// The band-data `BinaryViewArray` inside a raster `StructArray` - /// (bands list -> band struct -> data column). - fn band_data_view(arr: &StructArray) -> &BinaryViewArray { - arr.column(raster_indices::BANDS) - .as_any() - .downcast_ref::() - .unwrap() - .values() - .as_any() - .downcast_ref::() - .unwrap() - .column(band_indices::DATA) - .as_any() - .downcast_ref::() - .unwrap() - } - fn from_gdal_tester() -> ScalarUdfTester { ScalarUdfTester::new( rs_from_gdal_raster_udf().into(), @@ -276,11 +259,11 @@ mod tests { } /// The single-band fixture's declarative expectation, derived from the - /// fixture's construction: 4x4 UInt8, north-up unit pixels with origin - /// (0, 4), sequential values 0..16, no nodata, in-db. + /// fixture's construction: 4x4 UInt8 spanning the bbox (0, 0)-(4, 4) + /// (north-up unit pixels), sequential values 0..16, no nodata, in-db. fn single_band_spec(crs: Option<&str>) -> RasterSpec { RasterSpec::d2(4, 4) - .transform([0.0, 1.0, 0.0, 4.0, 0.0, -1.0]) + .bbox(0.0, 0.0, 4.0, 4.0) .crs(crs) .band_values(&(0..16u8).collect::>()) } @@ -336,7 +319,12 @@ mod tests { // own shared data block (a refcount bump), so the band-data view // has one buffer per band. A copying `append_value` path would // consolidate both bands into a single builder-owned buffer. - assert_eq!(band_data_view(&arr).data_buffers().len(), 2); + let num_data_buffers = RasterStructArray::try_new(&arr) + .unwrap() + .band_data_array() + .data_buffers() + .len(); + assert_eq!(num_data_buffers, 2); // ...and the decoded values/structure match the fixture. let arr: ArrayRef = Arc::new(arr); @@ -345,7 +333,7 @@ mod tests { &[Some( RasterSpec::d2(4, 4) .crs(None) - .transform([0.0, 1.0, 0.0, 4.0, 0.0, -1.0]) + .bbox(0.0, 0.0, 4.0, 4.0) .band_values(&(0..16u8).collect::>()) .band_values(&(100..116u8).collect::>()), )], From 174d9d99c72437b8e079a92bbd2c948843f702a0 Mon Sep 17 00:00:00 2001 From: jameswillis Date: Mon, 20 Jul 2026 13:26:37 -0700 Subject: [PATCH 7/8] fix(rust/sedona-raster-gdal): RS_FromGDALRaster accepts BinaryView input RS_AsGeoTiff returns BinaryView, and the is_binary matcher accepts both Binary and BinaryView, but the kernel narrowed its input through as_binary_array (i32-offset Binary only). That made RS_FromGDALRaster(RS_AsGeoTiff(...)) fail at execution with a BinaryView -> Binary cast error, breaking the round-trip documented in rs_fromgdalraster.qmd. Read Binary and BinaryView arrays directly instead of narrowing view offsets into Binary's i32 range. Add regression coverage for BinaryView input and for the RS_AsGeoTiff -> RS_FromGDALRaster round trip. --- .../src/rs_from_gdal_raster.rs | 103 +++++++++++++++--- 1 file changed, 87 insertions(+), 16 deletions(-) diff --git a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs index 6f9b5523c1..586f21466c 100644 --- a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs +++ b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs @@ -23,13 +23,13 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use arrow_array::Array; +use arrow_array::{cast::AsArray, Array}; use arrow_schema::DataType; -use datafusion_common::cast::as_binary_array; use datafusion_common::config::ConfigOptions; use datafusion_common::error::Result; use datafusion_common::{exec_datafusion_err, ScalarValue}; use datafusion_expr::{ColumnarValue, Volatility}; +use sedona_common::sedona_internal_err; use sedona_expr::scalar_udf::{SedonaScalarKernel, SedonaScalarUDF}; use sedona_gdal::gdal::Gdal; use sedona_gdal::gdal_dyn_bindgen::{GDAL_OF_RASTER, GDAL_OF_READONLY}; @@ -97,6 +97,25 @@ impl RsFromGDALRaster { result } + /// Decode each input row into `builder`: a NULL row appends a NULL raster, a + /// non-null row is decoded to an in-db raster. Generic over the binary array + /// flavour so `Binary` and `BinaryView` iterate through the same path. + fn append_rows<'a>( + gdal: &Gdal, + rows: impl Iterator>, + builder: &mut RasterBuilder, + ) -> Result<()> { + for row in rows { + match row { + None => builder + .append_null() + .map_err(|e| exec_datafusion_err!("Failed to append null: {e}"))?, + Some(content) => Self::append_gdal_raster(gdal, content, builder)?, + } + } + Ok(()) + } + /// Parse binary content into a single in-db raster. Test-only convenience /// around [`append_gdal_raster`](Self::append_gdal_raster); the kernel /// appends directly into a shared builder. @@ -144,19 +163,23 @@ impl SedonaScalarKernel for RsFromGDALRaster { .map_err(|e| exec_datafusion_err!("Failed to convert scalar to array: {e}"))?, ColumnarValue::Array(array) => array.clone(), }; - let binary_array = as_binary_array(&content_array)?; - let len = binary_array.len(); - - // Decode every row into one raster array. A NULL input row yields a - // NULL raster row; a non-null row is decoded to an in-db raster. - let mut builder = RasterBuilder::new(len); - for i in 0..len { - if binary_array.is_null(i) { - builder - .append_null() - .map_err(|e| exec_datafusion_err!("Failed to append null: {e}"))?; - } else { - Self::append_gdal_raster(gdal, binary_array.value(i), &mut builder)?; + + // Decode every row into one raster array. The binary matcher accepts + // both `Binary` and `BinaryView` (`RS_AsGeoTiff` produces the latter), + // so read each flavour directly rather than narrowing `BinaryView` + // offsets into `Binary`'s i32 range. + let mut builder = RasterBuilder::new(content_array.len()); + match content_array.data_type() { + DataType::Binary => { + Self::append_rows(gdal, content_array.as_binary::().iter(), &mut builder)? + } + DataType::BinaryView => { + Self::append_rows(gdal, content_array.as_binary_view().iter(), &mut builder)? + } + other => { + return sedona_internal_err!( + "RS_FromGDALRaster expected Binary or BinaryView content, got {other:?}" + ) } } let result = builder @@ -176,14 +199,17 @@ impl SedonaScalarKernel for RsFromGDALRaster { #[cfg(test)] mod tests { use super::*; - use arrow_array::{ArrayRef, BinaryArray}; + use arrow_array::{ArrayRef, BinaryArray, BinaryViewArray}; use sedona_gdal::raster::types::Buffer; use sedona_raster::array::RasterStructArray; + use sedona_schema::datatypes::RASTER; use sedona_testing::raster_spec::{ assert_raster_scalar_equals, assert_rasters_equal, RasterSpec, }; use sedona_testing::testers::ScalarUdfTester; + use crate::rs_as_geotiff::rs_as_geotiff_udf; + /// Build a small 4x4 single-band GeoTIFF (EPSG:4326) with GDAL and return its /// bytes together with the CRS GDAL reads back from them (PROJJSON) — the /// fixture-free stand-in for a `.tiff` on disk, so tests exercise the real @@ -258,6 +284,13 @@ mod tests { ) } + fn from_gdal_tester_binary_view() -> ScalarUdfTester { + ScalarUdfTester::new( + rs_from_gdal_raster_udf().into(), + vec![SedonaType::Arrow(DataType::BinaryView)], + ) + } + /// The single-band fixture's declarative expectation, derived from the /// fixture's construction: 4x4 UInt8 spanning the bbox (0, 0)-(4, 4) /// (north-up unit pixels), sequential values 0..16, no nodata, in-db. @@ -309,6 +342,44 @@ mod tests { assert_raster_scalar_equals(&result, &single_band_spec(crs.as_deref())); } + #[test] + fn from_gdal_raster_decodes_binary_view_input() { + // The `is_binary` matcher also accepts `BinaryView`, which is what + // `RS_AsGeoTiff` produces. The same GeoTIFF bytes wrapped in a + // `BinaryView` array must decode to the same raster as the `Binary` + // path — not fail casting `BinaryView` to i32-offset `Binary`. + let (bytes, crs) = with_gdal(|gdal| Ok(make_geotiff_fixture(gdal))).unwrap(); + let input: ArrayRef = Arc::new(BinaryViewArray::from(vec![Some(bytes.as_slice())])); + let result = from_gdal_tester_binary_view() + .invoke_arrays(vec![input]) + .unwrap(); + assert_rasters_equal(&result, &[Some(single_band_spec(crs.as_deref()))]); + } + + #[test] + fn as_geotiff_from_gdal_raster_round_trips() { + // The `RS_AsGeoTiff` -> `RS_FromGDALRaster` round trip used in the docs: + // encoding yields `BinaryView` bytes that must decode back to the source + // raster. A north-up single-band UInt8 raster (no CRS, no nodata) is + // preserved exactly through GeoTIFF. + let source = RasterSpec::d2(3, 3) + .crs(None) + .bbox(0.0, 0.0, 3.0, 3.0) + .band_values(&[1u8, 2, 3, 4, 5, 6, 7, 8, 9]); + + let encoded = ScalarUdfTester::new(rs_as_geotiff_udf().into(), vec![RASTER]) + .invoke_scalar(&source) + .unwrap(); + let ScalarValue::BinaryView(Some(bytes)) = encoded else { + panic!("expected a BinaryView result, got {encoded:?}"); + }; + + let decoded = from_gdal_tester_binary_view() + .invoke_scalar(ScalarValue::BinaryView(Some(bytes))) + .unwrap(); + assert_raster_scalar_equals(&decoded, &source); + } + #[test] fn decoded_two_band_raster_is_zero_copy() { with_gdal(|gdal| { From 8d4021866ce06e87dce2bcf9773534b1bbff38ef Mon Sep 17 00:00:00 2001 From: jameswillis Date: Thu, 23 Jul 2026 09:20:11 -0700 Subject: [PATCH 8/8] feat(rust/sedona-raster-gdal): RS_FromGDALRaster sets returns_bytes so RS_EnsureLoaded skips re-wrapping --- .../src/rs_from_gdal_raster.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs index 586f21466c..281692dcea 100644 --- a/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs +++ b/rust/sedona-raster-gdal/src/rs_from_gdal_raster.rs @@ -35,6 +35,7 @@ use sedona_gdal::gdal::Gdal; use sedona_gdal::gdal_dyn_bindgen::{GDAL_OF_RASTER, GDAL_OF_READONLY}; use sedona_gdal::raster::types::DatasetOptions; use sedona_raster::builder::RasterBuilder; +use sedona_raster_functions::rs_ensure_loaded::RETURNS_BYTES_METADATA_KEY; use sedona_schema::datatypes::{SedonaType, RASTER}; use sedona_schema::matchers::ArgMatcher; @@ -54,6 +55,11 @@ pub fn rs_from_gdal_raster_udf() -> SedonaScalarUDF { vec![Arc::new(RsFromGDALRaster)], Volatility::Immutable, ) + // Emits a fully-materialized in-db raster, so its output is already loaded + // and RS_EnsureLoaded must not wrap it again. It takes no raster argument + // (its input is binary), so there is nothing to materialize on the way in + // and NEEDS_PIXELS_METADATA_KEY does not apply. + .with_metadata(RETURNS_BYTES_METADATA_KEY, "true") } /// Kernel implementation for RS_FromGDALRaster @@ -317,6 +323,24 @@ mod tests { assert_eq!(udf.name(), "rs_fromgdalraster"); } + #[test] + fn udf_carries_returns_bytes_metadata() { + // RS_FromGDALRaster unconditionally returns a fully-materialized in-db + // raster, so it sets `returns_bytes`; the RS_EnsureLoaded rule reads + // this to skip redundantly wrapping the already-loaded output. + use sedona_raster_functions::rs_ensure_loaded::NEEDS_PIXELS_METADATA_KEY; + let udf = rs_from_gdal_raster_udf(); + assert_eq!( + udf.metadata() + .get(RETURNS_BYTES_METADATA_KEY) + .map(String::as_str), + Some("true"), + ); + // Its input is binary, not a raster, so there is no raster argument to + // materialize and it must NOT claim to need pixels. + assert!(udf.metadata().get(NEEDS_PIXELS_METADATA_KEY).is_none()); + } + #[test] fn parse_gdal_raster_builds_indb_raster() { // The direct builder path: GeoTIFF bytes decode to an in-db raster