From ba0c1548c58b7ede5ad1b8677ff62442c93920ee Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Tue, 3 Nov 2026 10:43:06 +0100 Subject: [PATCH 1/2] feat(data): the memory limits follow the computer's memory on the desktop (#1291 step 2, #1318) Three fixed memory caps become functions of the computer's memory, read once through sysinfo (rustgis_data::memory, a container's limit counting when it is lower). Each plans for a quarter of it, never below today's fixed value (still the browser's limit) and never above a ceiling: - raster::max_samples (was MAX_SAMPLES): 8 bytes a sample, 120 million to 1 billion. - rasterops::max_pixels (was MAX_PIXELS): 16 bytes a pixel, 64 million to 512 million. - ogc::max_paged_features (was MAX_PAGED_FEATURES): 20 KB a feature, 100,000 to 1 million. gp.env returns them ("limits"), and the errors and notes that name one give this computer's value. memory::scaled takes the memory as an argument, so tests pass fixed amounts. The work caps (viewshed, IDW, kriging) are left for a follow-up. --- Cargo.lock | 34 ++++++++++++ crates/analysis/src/kriging.rs | 6 +-- crates/analysis/src/rasterops.rs | 17 ++++-- crates/analysis/src/warp.rs | 21 +++++--- crates/data/Cargo.toml | 2 + crates/data/src/lib.rs | 1 + crates/data/src/memory.rs | 68 ++++++++++++++++++++++++ crates/data/src/ogc.rs | 20 +++++-- crates/data/src/raster/ascii.rs | 4 +- crates/data/src/raster/geotiff.rs | 28 +++++----- crates/data/src/raster/mod.rs | 25 ++++++--- crates/data/src/raster/zarr.rs | 7 +-- crates/engine/src/cmd/cube.rs | 15 ++++++ crates/engine/src/cmd/imagery.rs | 4 +- crates/engine/src/cmd/service.rs | 2 +- crates/engine/src/cmd/workflows.rs | 14 +++-- crates/engine/src/tests.rs | 12 +++-- crates/ui-egui/src/serviceui.rs | 2 +- docs/user-guide/analysis.md | 21 ++++++-- docs/user-guide/data-formats.md | 6 +-- docs/user-guide/data.md | 5 +- fuzz/Cargo.lock | 85 ++++++++++++++++++++++++++++++ 22 files changed, 333 insertions(+), 66 deletions(-) create mode 100644 crates/data/src/memory.rs diff --git a/Cargo.lock b/Cargo.lock index fb1d7a11a..40c889060 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3353,6 +3353,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.4.3" @@ -3634,6 +3643,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-io-surface" version = "0.3.2" @@ -4752,6 +4771,7 @@ dependencies = [ "serde", "serde_json", "shapefile", + "sysinfo", "tempfile", "thiserror 2.0.21", "tiberius", @@ -5603,6 +5623,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "sysinfo" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8a3edbe0394ce115f3917a00349dd242be09eb5a091ed509eaf67b55cfb8a8" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + [[package]] name = "tar" version = "0.4.46" diff --git a/crates/analysis/src/kriging.rs b/crates/analysis/src/kriging.rs index 24a3310a5..c175ba1a4 100644 --- a/crates/analysis/src/kriging.rs +++ b/crates/analysis/src/kriging.rs @@ -12,7 +12,7 @@ use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; use rustgis_geom::{Coord, Extent}; -use crate::rasterops::{MAX_PIXELS, NODATA}; +use crate::rasterops::{NODATA, max_pixels}; /// Maximum cell-point pairs Kriging computes (about a second's work). pub const MAX_KRIGING_WORK: usize = 40_000_000; @@ -337,8 +337,8 @@ fn kriging_on( return Err("the area to interpolate over is empty".into()); } let (w, h) = ((extent.width() / cell_deg).ceil().max(1.0) as usize, (extent.height() / cell_deg).ceil().max(1.0) as usize); - if w.saturating_mul(h) > MAX_PIXELS / 4 { - return Err(format!("{w} × {h} cells is too many; use a larger cell size")); + if w.saturating_mul(h) > max_pixels() / 4 { + return Err(format!("{w} × {h} cells is too many, more than this computer's {}; use a larger cell size", max_pixels() / 4)); } let k = if neighbours == 0 { 12.min(points.len()) } else { neighbours.clamp(2, points.len().min(32)) }; let k3 = (k * k * k) / 3; diff --git a/crates/analysis/src/rasterops.rs b/crates/analysis/src/rasterops.rs index 19feba4a2..6764dae91 100644 --- a/crates/analysis/src/rasterops.rs +++ b/crates/analysis/src/rasterops.rs @@ -9,8 +9,17 @@ use rustgis_geom::{Coord, Extent}; /// The no-data value written by these operations. pub const NODATA: f32 = -3.4e38; -/// Largest output raster built (pixels). -pub const MAX_PIXELS: usize = 64_000_000; +/// The fewest pixels [`max_pixels`] allows, and its limit in the browser. +pub const PIXELS_FLOOR: usize = 64_000_000; +/// The most [`max_pixels`] allows, however much memory the computer has (2 GB a band). +pub const PIXELS_CEILING: usize = 512_000_000; + +/// Largest output raster built (pixels): on the desktop, as many as a quarter of the computer's +/// memory holds at 16 bytes each (a tool holds its input, its output and a working grid or two), +/// from [`PIXELS_FLOOR`] up to [`PIXELS_CEILING`] (#1291 step 2). +pub fn max_pixels() -> usize { + rustgis_data::memory::scaled(rustgis_data::memory::physical(), PIXELS_FLOOR, PIXELS_CEILING, 16) +} /// Room for a grid of `w × h` cells of `T` (an output band, or a working grid of one), empty: a /// grid the machine can't hold is an error saying so, instead of ending the program @@ -449,8 +458,8 @@ fn idw_on( return Err("the area to interpolate over is empty".into()); } let (w, h) = ((extent.width() / cell_deg).ceil().max(1.0) as usize, (extent.height() / cell_deg).ceil().max(1.0) as usize); - if w.saturating_mul(h) > MAX_PIXELS / 4 { - return Err(format!("{w} × {h} cells is too many; use a larger cell size")); + if w.saturating_mul(h) > max_pixels() / 4 { + return Err(format!("{w} × {h} cells is too many, more than this computer's {}; use a larger cell size", max_pixels() / 4)); } // Every cell weighs every point. if w.saturating_mul(h).saturating_mul(points.len()) > MAX_IDW_WORK { diff --git a/crates/analysis/src/warp.rs b/crates/analysis/src/warp.rs index 20b13c572..b53ee808f 100644 --- a/crates/analysis/src/warp.rs +++ b/crates/analysis/src/warp.rs @@ -13,7 +13,7 @@ use std::collections::HashMap; -use rustgis_data::raster::{ColorHint, Crs, GeoTransform, MAX_SAMPLES, Raster, Transformer}; +use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster, Transformer, max_samples}; use rustgis_geom::Coord; use crate::georef::Fit; @@ -237,12 +237,15 @@ pub fn warp(r: &Raster, fit: &Fit, o: &Options) -> Result { let cells = |cell: f64| -> (f64, f64) { (((max_x - min_x) / cell).ceil().max(1.0), ((max_y - min_y) / cell).ceil().max(1.0)) }; // The samples held, and the buffers a row needs (they add up on a long thin grid). let load = |(ow, oh): (f64, f64)| ow * (oh * out_bands as f64 + COLUMN_COST); - let too_many = |n: (f64, f64)| load(n) > MAX_SAMPLES as f64; + let max = max_samples(); + let too_many = |n: (f64, f64)| load(n) > max as f64; let cell = match o.cell { Some(c) if !(c.is_finite() && c > 0.0) => return Err("cellSize must be a positive number".into()), Some(c) => { if too_many(cells(c)) { - return Err(format!("a cell size of {c} makes a grid of more than {MAX_SAMPLES} samples: give a larger cellSize")); + return Err(format!( + "a cell size of {c} makes a grid of more than {max} samples, this computer's limit: give a larger cellSize" + )); } c } @@ -254,7 +257,7 @@ pub fn warp(r: &Raster, fit: &Fit, o: &Options) -> Result { if !too_many(n) { break; } - cell *= (load(n) / MAX_SAMPLES as f64).sqrt().max(1.01); + cell *= (load(n) / max as f64).sqrt().max(1.01); } if too_many(cells(cell)) { return Err("the warped image is too large to hold: give a cellSize".into()); @@ -361,9 +364,10 @@ impl Grid { return Err("the origin and cellSize can't make a grid over this area".into()); } let (w, h) = (((max_x - min_x) / cx).round().max(1.0), ((max_y - min_y) / cy).round().max(1.0)); - if w * h > MAX_SAMPLES as f64 { + let max = max_samples(); + if w * h > max as f64 { return Err(format!( - "a cell size of {cx} makes a grid of {} × {} cells, more than {MAX_SAMPLES}: give a larger cellSize or a smaller extent", + "a cell size of {cx} makes a grid of {} × {} cells, more than this computer's {max}: give a larger cellSize or a smaller extent", w as u64, h as u64 )); } @@ -452,9 +456,10 @@ pub fn project(r: &Raster, g: &Grid, method: Method, name: String) -> Result MAX_SAMPLES as f64 { + let max = max_samples(); + if g.width as f64 * (g.height as f64 * out_bands as f64 + COLUMN_COST) > max as f64 { return Err(format!( - "{} × {} cells of {} band(s) are more than {MAX_SAMPLES} samples: give a larger cellSize or a smaller extent", + "{} × {} cells of {} band(s) are more than this computer's {max} samples: give a larger cellSize or a smaller extent", g.width, g.height, out_bands )); } diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index 668920ebf..e9097049f 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -57,6 +57,8 @@ rustls = { workspace = true } webpki-roots = { workspace = true } # Private temporary workspaces for downloaded JPEG 2000 conversion through GDAL. tempfile = "3" +# The computer's physical memory, for the limits that scale with it (`memory`, #1291 step 2). +sysinfo = { version = "0.40", default-features = false, features = ["system"] } [target.'cfg(all(unix, not(target_arch = "wasm32")))'.dependencies] # Safe effective-UID and no-follow operations for private Shapefile index caches. diff --git a/crates/data/src/lib.rs b/crates/data/src/lib.rs index 0abaa2ee0..c3301e9ff 100644 --- a/crates/data/src/lib.rs +++ b/crates/data/src/lib.rs @@ -23,6 +23,7 @@ pub mod landxml; pub mod lidar; pub mod mapinfo; pub mod mbtiles; +pub mod memory; pub mod ogc; pub mod osm; pub mod photo; diff --git a/crates/data/src/memory.rs b/crates/data/src/memory.rs new file mode 100644 index 000000000..191a75953 --- /dev/null +++ b/crates/data/src/memory.rs @@ -0,0 +1,68 @@ +//! The computer's memory, and the limits that follow it (#1291 step 2, #1318). +//! +//! On the desktop a memory limit plans for a share of the physical memory ([`SHARE`]): never below +//! the fixed value it had before (its floor, still the browser's limit), never above a ceiling. A +//! container's memory limit counts, when it is lower. The memory is read once, at the first limit +//! asked for, so a session keeps the same limits; [`scaled`] takes it as an argument, so tests pass +//! fixed amounts. + +/// A limit plans for a quarter of the physical memory: room for the layers already held, the +/// drawing, and a second job running at the same time. +pub const SHARE: u64 = 4; + +/// The computer's physical memory in bytes (a container's limit, when it is lower), read once; +/// `None` in the browser, or when it can't be read. +pub fn physical() -> Option { + #[cfg(not(target_arch = "wasm32"))] + { + static MEMORY: std::sync::OnceLock> = std::sync::OnceLock::new(); + *MEMORY.get_or_init(read) + } + #[cfg(target_arch = "wasm32")] + { + None + } +} + +#[cfg(not(target_arch = "wasm32"))] +fn read() -> Option { + let mut s = sysinfo::System::new().ok()?; + s.refresh_memory_specifics(sysinfo::MemoryRefreshKind::nothing().with_ram()); + let total = Some(s.total_memory()).filter(|m| *m > 0)?; + // A container's limit, on Linux; none is reported elsewhere. + Some(s.cgroup_limits().map(|c| c.total_memory).filter(|m| *m > 0).map_or(total, |c| c.min(total))) +} + +/// A limit of units costing `bytes` each: as many as [`SHARE`] of `memory` holds, within `floor` +/// and `ceiling`; `floor` when the memory isn't known. +pub fn scaled(memory: Option, floor: usize, ceiling: usize, bytes: u64) -> usize { + let Some(memory) = memory else { return floor }; + let n = memory / SHARE / bytes.max(1); + usize::try_from(n).unwrap_or(usize::MAX).clamp(floor, ceiling.max(floor)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + + #[test] + fn a_limit_follows_the_memory_between_its_floor_and_ceiling() { + // 8 bytes a unit: 16 GiB plans for 4 GiB, 512 Mi units. + assert_eq!(scaled(Some(16 * GIB), 100, 1 << 40, 8), 512 << 20); + assert_eq!(scaled(Some(GIB), 200_000_000, 1 << 40, 8), 200_000_000, "never below the floor"); + assert_eq!(scaled(Some(1024 * GIB), 100, 1_000, 8), 1_000, "never above the ceiling"); + assert_eq!(scaled(None, 100, 1_000, 8), 100, "unknown memory: the floor"); + assert_eq!(scaled(Some(u64::MAX), 100, usize::MAX, 0) as u64, u64::MAX / SHARE, "no overflow or division by zero"); + assert_eq!(scaled(Some(16 * GIB), 100, 50, 8), 100, "a ceiling below the floor is the floor"); + } + + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn the_memory_is_read_once_and_stays() { + let first = physical(); + assert!(first.is_none_or(|m| m >= 64 << 20), "{first:?}"); + assert_eq!(physical(), first); + } +} diff --git a/crates/data/src/ogc.rs b/crates/data/src/ogc.rs index 161cc0cff..f425b5022 100644 --- a/crates/data/src/ogc.rs +++ b/crates/data/src/ogc.rs @@ -2201,7 +2201,8 @@ fn arcgis(url: &str, v: &serde_json::Value) -> Result { && service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })) { service.notes.push(format!( - "a feature layer is read up to {MAX_PAGED_FEATURES} features, {max} per request, where the server pages its layers" + "a feature layer is read up to {} features on this computer, {max} per request, where the server pages its layers", + max_paged_features() )); } // A secured service's token goes with every request. @@ -2484,7 +2485,7 @@ fn ogc_api(url: &str, v: &serde_json::Value) -> Result { } service.incomplete = unreadable > 0 || collections.len() > MAX_LAYERS; if service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })) { - service.notes.push(format!("a collection is read up to {MAX_PAGED_FEATURES} features, page by page")); + service.notes.push(format!("a collection is read up to {} features on this computer, page by page", max_paged_features())); } if tiled > 0 { service.kind = if service.layers.iter().any(|l| matches!(l.offer, Offer::Features { .. })) { @@ -2681,8 +2682,17 @@ pub fn partial_note(bytes: &[u8]) -> Option { (next(&head) || next(&tail)).then(|| "the server sent one page of its features; the rest aren't read".into()) } -/// Most features read from a collection that comes in pages. -pub const MAX_PAGED_FEATURES: usize = 100_000; +/// The fewest features [`max_paged_features`] allows, and its limit in the browser. +pub const PAGED_FEATURES_FLOOR: usize = 100_000; +/// The most [`max_paged_features`] allows, however much memory the computer has. +pub const PAGED_FEATURES_CEILING: usize = 1_000_000; + +/// Most features read from a collection that comes in pages: on the desktop, as many as a quarter +/// of the computer's memory holds at 20 KB each (about 100,000 for every 8 GB, the pages read +/// included), from [`PAGED_FEATURES_FLOOR`] up to [`PAGED_FEATURES_CEILING`] (#1318). +pub fn max_paged_features() -> usize { + crate::memory::scaled(crate::memory::physical(), PAGED_FEATURES_FLOOR, PAGED_FEATURES_CEILING, 20 << 10) +} /// A feature collection page's JSON `next` link, resolved against the page's address. fn next_page(url: &str, page: &serde_json::Value) -> Option { @@ -2726,7 +2736,7 @@ fn arcgis_paging(layer: &serde_json::Value) -> Option { .get("maxRecordCount") .and_then(|m| m.as_u64()) .and_then(|m| usize::try_from(m).ok()) - .filter(|m| (1..=MAX_PAGED_FEATURES).contains(m)); + .filter(|m| (1..=max_paged_features()).contains(m)); let oid_field = || { layer .get("fields")? diff --git a/crates/data/src/raster/ascii.rs b/crates/data/src/raster/ascii.rs index b9d6d3124..c4b3af0c4 100644 --- a/crates/data/src/raster/ascii.rs +++ b/crates/data/src/raster/ascii.rs @@ -1,6 +1,6 @@ //! ASCII grids (`.asc`): a short header, then rows of numbers. -use super::{ColorHint, Crs, Exact, GeoTransform, MAX_SAMPLES, Raster, Uncounted}; +use super::{ColorHint, Crs, Exact, GeoTransform, Raster, Uncounted, max_samples}; use crate::{DataError, Result}; /// Reads an ASCII grid. Without a known coordinate system, coordinates that fit in degrees are @@ -59,7 +59,7 @@ pub(super) fn read_with_nodata(text: &str, name: &str, crs: Option, sidecar if v >= 1.0 && v <= u32::MAX as f64 && v.fract() == 0.0 { Ok(v as u32) } else { Err(bad(&format!("bad {what}"))) } }; let (w, h) = (size(ncols, "ncols")?, size(nrows, "nrows")?); - let n = (w as usize).checked_mul(h as usize).filter(|n| *n <= MAX_SAMPLES).ok_or_else(|| bad("too large"))?; + let n = (w as usize).checked_mul(h as usize).filter(|n| *n <= max_samples()).ok_or_else(|| bad("too large"))?; // CELLSIZE takes precedence over DX/DY, as in GDAL's ASCII grid reader. let (dx, dy) = if let Some(cell) = cell { (Some(cell), Some(cell)) } else { (dx, dy) }; let dx = dx.filter(|c| c.is_finite() && *c > 0.0).ok_or_else(|| bad("missing or invalid cell width"))?; diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 0e98d0f79..0da1052ab 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -14,7 +14,7 @@ use tiff::tags::Tag; use super::cog::Sparse; use super::crs::CustomCrs; -use super::{ColorHint, Crs, Exact, GeoTransform, MAX_SAMPLES, Raster, Uncounted}; +use super::{ColorHint, Crs, Exact, GeoTransform, Raster, Uncounted, max_samples}; use crate::{DataError, Result}; fn err(e: impl std::fmt::Display) -> DataError { @@ -948,7 +948,7 @@ fn decimate_parallel( out.into_inner().map_err(|_| err("a chunk couldn't be decoded")) } -/// Reads a GeoTIFF. Images too large for [`MAX_SAMPLES`] are read from their largest overview that +/// Reads a GeoTIFF. Images too large for [`max_samples`] are read from their largest overview that /// fits, when the file has overviews; otherwise an overview is made as the image is read, a strip /// or tile at a time (a regular, not cloud-optimised, file). pub fn read(r: R, name: &str) -> Result { @@ -957,7 +957,7 @@ pub fn read(r: R, name: &str) -> Result { /// [`read`], with the no-data value a sidecar gives (see [`read_limited_with_nodata`]). pub(super) fn read_with_nodata(r: R, name: &str, sidecar_nodata: Option) -> Result { - read_limited_with_nodata(reader(r).map_err(err)?, name, MAX_SAMPLES, None, sidecar_nodata) + read_limited_with_nodata(reader(r).map_err(err)?, name, max_samples(), None, sidecar_nodata) } /// What reading a GeoTIFF from the parts of it read so far came to. @@ -972,7 +972,7 @@ pub enum Partial { /// to read next. The web app reads a large file so, a part at a time, keeping only its header and /// the overview it opens from (or the strips or tiles an overview made as it is read needs). pub fn read_sparse(parts: &Sparse, name: &str) -> Result { - read_sparse_limited(parts, name, MAX_SAMPLES) + read_sparse_limited(parts, name, max_samples()) } fn read_sparse_limited(parts: &Sparse, name: &str, max: usize) -> Result { @@ -997,7 +997,7 @@ pub fn read_file(path: &std::path::Path, name: &str) -> Result { pub(super) fn read_file_with_nodata(path: &std::path::Path, name: &str, sidecar_nodata: Option) -> Result { let path = path.to_path_buf(); let open = move || -> std::io::Result> { reader(std::io::BufReader::new(std::fs::File::open(&path)?)) }; - read_limited_with_nodata(open().map_err(err)?, name, MAX_SAMPLES, Some(&open), sidecar_nodata) + read_limited_with_nodata(open().map_err(err)?, name, max_samples(), Some(&open), sidecar_nodata) } /// [`read`], holding at most `max` samples (the tests' small limit exercises the overviews), with @@ -1021,7 +1021,7 @@ pub(super) fn read_limited_with_nodata( let palette = r.first_is_palette(); let limits = { let mut l = Limits::default(); - l.decoding_buffer_size = MAX_SAMPLES.saturating_mul(4); + l.decoding_buffer_size = max_samples().saturating_mul(4); l.intermediate_buffer_size = 256 << 20; l.ifd_value_size = 64 << 20; l @@ -1821,7 +1821,7 @@ mod tests { #[test] fn a_part_container_opens_as_a_tif() { let file = super::super::cog::example(512, 256); - let (r, parts) = read_in_parts(&file, 4096, MAX_SAMPLES); + let (r, parts) = read_in_parts(&file, 4096, max_samples()); let back = super::super::read_bytes("parts.tif", &parts.container()).unwrap(); assert_eq!((back.width, back.height, back.value(0, 300, 200)), (r.width, r.height, r.value(0, 300, 200))); } @@ -1829,7 +1829,7 @@ mod tests { #[test] fn damaged_part_containers_never_panic() { let file = super::super::cog::example(512, 256); - let (_, parts) = read_in_parts(&file, 4096, MAX_SAMPLES); + let (_, parts) = read_in_parts(&file, 4096, max_samples()); let good = parts.container(); for cut in (0..good.len()).step_by(97) { let _ = super::super::read_bytes("x.tif", &good[..cut]); @@ -1930,7 +1930,7 @@ mod tests { let data: Vec = (0..w * h).map(|i| ((i % w * 7 + i / w * 3) % 253) as u8).collect(); let bytes = grey8(w, h, &data, None); let one = read(std::io::Cursor::new(&bytes), "one").unwrap(); - let many = read_threaded(&bytes, MAX_SAMPLES).unwrap(); + let many = read_threaded(&bytes, max_samples()).unwrap(); assert_eq!((many.width, many.height), (w, h)); assert!(many.bands.iter().zip(&one.bands).all(|(a, b)| a.data == b.data) && many.bands.len() == one.bands.len()); assert_eq!(many.data_type, one.data_type); @@ -1949,7 +1949,7 @@ mod tests { // Cut off partway: the strips past the end can't be read. let bytes = grey8(w, h, &data, None); let cut = &bytes[..bytes.len() / 2]; - assert!(read_threaded(cut, MAX_SAMPLES).is_err()); + assert!(read_threaded(cut, max_samples()).is_err()); assert!(read_threaded(cut, 100_000).is_err()); } @@ -2129,7 +2129,7 @@ mod tests { for (w, h, tile) in [(40, 24, (16, 16)), (32, 32, (16, 16)), (17, 9, (16, 16)), (24, 40, (16, 16)), (40, 20, (16, 8))] { let bytes = planar_tiled(w, h, tile, 8, pixel); // The whole image, and every 2nd pixel of it (a limit it doesn't fit). - for (k, max) in [(1, MAX_SAMPLES), (2, (w * h) as usize)] { + for (k, max) in [(1, max_samples()), (2, (w * h) as usize)] { let r = read_limited(reader(std::io::Cursor::new(&bytes)).unwrap(), "planar", max, None).unwrap(); assert_eq!((r.width, r.height, r.band_count()), (w.div_ceil(k), h.div_ceil(k), 3)); for y in 0..r.height { @@ -2424,8 +2424,8 @@ mod tests { }; check(&read(std::io::Cursor::new(&bytes), "dtm").unwrap()); // From a file, a chunk at a time; and in the browser, in parts. - check(&read_threaded(&bytes, MAX_SAMPLES).unwrap()); - check(&read_in_parts(&bytes, 512, MAX_SAMPLES).0); + check(&read_threaded(&bytes, max_samples()).unwrap()); + check(&read_in_parts(&bytes, 512, max_samples()).0); // The same tile of a larger image, as one of the tiles a COG is shown with. let wide = lzw_tiled(232, 200, 128, 1, |x, y| if x < 128 && y < 128 { pixel(x, y) } else { (x + y) as f32 }); let r = read(std::io::Cursor::new(&wide), "wide").unwrap(); @@ -2570,7 +2570,7 @@ mod tests { check(&scan, 32, 32); // A million pixels in 16 strips, read a chunk at a time on several threads. let big = ycbcr_jpeg_tiff(&[(1024, 1024, 64)], colors, true); - check(&read_threaded(&big, MAX_SAMPLES).unwrap(), 1024, 1024); + check(&read_threaded(&big, max_samples()).unwrap(), 1024, 1024); // An image too large for the limit is shown by its overview, a JPEG of its own. let cog = ycbcr_jpeg_tiff(&[(64, 64, 64), (32, 32, 32)], colors, true); let r = read_limited(reader(std::io::Cursor::new(&cog)).unwrap(), "cog", 4_000, None).unwrap(); diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index f54d9a1ae..1e24eab90 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -3,7 +3,7 @@ //! Pixels are held as `f32` per band: whole numbers are exact up to ±16,777,216 and decimals to //! about 7 significant digits. Readers count the values that change on the way in //! ([`Raster::rounded`]). Readers never panic on malformed input and refuse rasters -//! larger than [`MAX_SAMPLES`]; cloud-optimised GeoTIFFs fall back to their largest overview that +//! larger than [`max_samples`]; cloud-optimised GeoTIFFs fall back to their largest overview that //! fits. pub mod ascii; @@ -66,12 +66,21 @@ fn no_memory(w: usize, h: usize) -> DataError { DataError::Format(format!("not enough memory for a {w} × {h} raster")) } -/// Most samples (pixels × bands) held in memory: 120 million (480 MB), or 32 million in the -/// browser, whose memory is much tighter. Reading briefly needs about twice this. +/// The fewest samples (pixels × bands) [`max_samples`] allows: 120 million (480 MB), or 32 +/// million in the browser, whose memory is much tighter (its limit there). #[cfg(not(target_arch = "wasm32"))] -pub const MAX_SAMPLES: usize = 120_000_000; +pub const SAMPLES_FLOOR: usize = 120_000_000; #[cfg(target_arch = "wasm32")] -pub const MAX_SAMPLES: usize = 32_000_000; +pub const SAMPLES_FLOOR: usize = 32_000_000; +/// The most [`max_samples`] allows, however much memory the computer has: a billion (4 GB). +pub const SAMPLES_CEILING: usize = 1_000_000_000; + +/// Most samples (pixels × bands) held in memory: on the desktop, as many as a quarter of the +/// computer's memory holds at 8 bytes each (a sample takes 4, and reading briefly needs about +/// twice that), from [`SAMPLES_FLOOR`] up to [`SAMPLES_CEILING`] (#1291 step 2, #1318). +pub fn max_samples() -> usize { + crate::memory::scaled(crate::memory::physical(), SAMPLES_FLOOR, SAMPLES_CEILING, 8) +} /// Largest ASCII grid file read (1 GiB). pub const MAX_ASCII_BYTES: u64 = 1 << 30; /// File extensions read as rasters. @@ -884,10 +893,10 @@ fn deinterleave>(raw: &[T], w: u32, h: u32, ch: usize, k: u3 /// Reads a PNG or JPEG as a raster with no coordinates (`georeferenced` is false): its bands as /// they are (grey, grey and alpha, RGB, RGBA; 8 or 16 bits). A picture is decoded whole, so one -/// whose decoded size passes four times [`MAX_SAMPLES`] bytes is refused (save it as a TIFF); one -/// with more than [`MAX_SAMPLES`] samples is read at every second pixel, with its `overview` set. +/// whose decoded size passes four times [`max_samples`] bytes is refused (save it as a TIFF); one +/// with more than [`max_samples`] samples is read at every second pixel, with its `overview` set. pub fn read_scan(name: &str, source: R) -> Result { - read_scan_limited(name, source, MAX_SAMPLES) + read_scan_limited(name, source, max_samples()) } /// [`read_scan`], holding at most `max` samples (the tests' small limit exercises the reduction). diff --git a/crates/data/src/raster/zarr.rs b/crates/data/src/raster/zarr.rs index 73809b387..68ada6a52 100644 --- a/crates/data/src/raster/zarr.rs +++ b/crates/data/src/raster/zarr.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use serde_json::{Map, Value}; -use super::{ColorHint, Crs, Exact, GeoTransform, MAX_SAMPLES, Raster}; +use super::{ColorHint, Crs, Exact, GeoTransform, Raster, max_samples}; use crate::{DataError, Result}; /// The metadata files a store may have at its root: Zarr 3's `zarr.json`, Zarr 2's consolidated @@ -996,8 +996,9 @@ pub fn plan(store: &Store, sel: &Selection) -> Result { let (y, x) = map_dims(a); let (h, w) = (a.shape.get(y).copied().unwrap_or(0), a.shape.get(x).copied().unwrap_or(0)); let cells = h.checked_mul(w).ok_or_else(|| bad("too large"))?; - if cells == 0 || cells > MAX_SAMPLES as u64 { - return Err(bad(format!("{} is {w} × {h}, more than a raster holds", a.path))); + let max = max_samples(); + if cells == 0 || cells > max as u64 { + return Err(bad(format!("{} is {w} × {h}, more than the {max} cells a raster holds on this computer", a.path))); } a.chunk_values()?; let mut fixed = vec![0u64; a.shape.len()]; diff --git a/crates/engine/src/cmd/cube.rs b/crates/engine/src/cmd/cube.rs index 2821e5740..856867411 100644 --- a/crates/engine/src/cmd/cube.rs +++ b/crates/engine/src/cmd/cube.rs @@ -1968,6 +1968,21 @@ mod tests { assert_eq!(one, vec![(None, vec![1.0, 2.0, 3.0])]); } + /// Totals and columns the machine can't hold are errors, not crashes (#1291); small ones are + /// made as before. + #[test] + fn totals_and_columns_too_large_to_hold_are_errors() { + for stat in [super::Stat::Min, super::Stat::Std, super::Stat::Count] { + let e = super::Totals::new(stat, usize::MAX / 2).err().unwrap_or_default(); + assert!(e.contains("not enough memory"), "{e}"); + assert!(super::Totals::new(stat, 4).is_ok()); + } + let mut outs = Vec::new(); + let e = super::stack_columns(&mut outs, &[], vec![(Some(1.0), vec![2.0])], &[], Some(usize::MAX / 2)).unwrap_err(); + assert!(e.contains("not enough memory"), "{e}"); + assert!(super::stack_columns(&mut outs, &[], vec![(Some(1.0), vec![2.0])], &[], Some(3)).is_ok()); + } + /// The class counts of the groups being read share one cap: two groups each within it can be /// over it together. #[test] diff --git a/crates/engine/src/cmd/imagery.rs b/crates/engine/src/cmd/imagery.rs index 125162d41..66d57731d 100644 --- a/crates/engine/src/cmd/imagery.rs +++ b/crates/engine/src/cmd/imagery.rs @@ -898,7 +898,7 @@ fn raster_mosaic(s: &mut Session, p: &Value) -> CmdResult { if inputs.len() > MAX_MOSAIC_INPUTS { return Err(format!("a mosaic takes at most {MAX_MOSAIC_INPUTS} inputs")); } - let max = rustgis_data::raster::MAX_SAMPLES as i64; + let max = rustgis_data::raster::max_samples() as i64; let mut rs: Vec> = Vec::with_capacity(inputs.len()); // Each input's place on the first one's grid, and the bounds of them all so far. let mut places: Vec<(i64, i64)> = Vec::with_capacity(inputs.len()); @@ -1357,7 +1357,7 @@ fn mosaic(s: &mut Session, p: &Value) -> CmdResult { .fold(f64::INFINITY, f64::min); let cell = w.cell_size.unwrap_or(finest).max(1e-7); let (cols, rows) = ((e.width() / cell).ceil().max(1.0) as usize, (e.height() / cell).ceil().max(1.0) as usize); - if cols.saturating_mul(rows) > rasterops::MAX_PIXELS / 2 { + if cols.saturating_mul(rows) > rasterops::max_pixels() / 2 { return Err(format!("the mosaic would be {cols} × {rows} cells; set a larger cell size in the workspace options")); } let nb = imgs.iter().map(|r| r.band_count()).min().unwrap_or(1).min(if w.kind == "reality" { 1 } else { 4 }); diff --git a/crates/engine/src/cmd/service.rs b/crates/engine/src/cmd/service.rs index 7d06972ef..833a8e834 100644 --- a/crates/engine/src/cmd/service.rs +++ b/crates/engine/src/cmd/service.rs @@ -249,7 +249,7 @@ pub(crate) fn download_features(s: &mut Session, name: &str, url: &str, max: usi let until = std::time::Instant::now() + cache::BLOCKING_TIMEOUT; let first = cache::download_blocking_limited(url, max).map_err(|e| format!("{url}: {e}"))?; let mut left = max.saturating_sub(first.len()); - let mut step = rustgis_data::ogc::read_pages(url, first, rustgis_data::ogc::MAX_PAGED_FEATURES); + let mut step = rustgis_data::ogc::read_pages(url, first, rustgis_data::ogc::max_paged_features()); let (bytes, note) = loop { let pages = match step { rustgis_data::ogc::Step::Done(bytes, note) => break (bytes, note), diff --git a/crates/engine/src/cmd/workflows.rs b/crates/engine/src/cmd/workflows.rs index bd72ac96b..f5f49a5c4 100644 --- a/crates/engine/src/cmd/workflows.rs +++ b/crates/engine/src/cmd/workflows.rs @@ -173,7 +173,15 @@ fn environments(s: &mut Session, p: &Value) -> CmdResult { }; } let env = &s.project.gp_env; - Ok(json!({"extent": env.extent.map(|e| [e.min_x, e.min_y, e.max_x, e.max_y]), "cellSize": env.cell_size})) + // The memory limits on this computer (#1291 step 2), so a script or model can see why a run + // that passes elsewhere is refused here. + let limits = json!({ + "memory": rustgis_data::memory::physical(), + "rasterSamples": rustgis_data::raster::max_samples(), + "outputPixels": rasterops::max_pixels(), + "pagedFeatures": rustgis_data::ogc::max_paged_features(), + }); + Ok(json!({"extent": env.extent.map(|e| [e.min_x, e.min_y, e.max_x, e.max_y]), "cellSize": env.cell_size, "limits": limits})) } /// Curated tool lists: everyday tools, feature analysis and raster analysis. @@ -524,8 +532,8 @@ fn apply_function(s: &Session, r: &Raster, p: &Value) -> Result, String } "resample" => { let t = raster_of(s, ps(p, "target").ok_or("target (a raster to match) is required")?)?; - if t.width as usize * t.height as usize > rasterops::MAX_PIXELS { - return Err("the target grid is too large".into()); + if t.width as usize * t.height as usize > rasterops::max_pixels() { + return Err(format!("the target grid is larger than this computer's {} cells", rasterops::max_pixels())); } rasterops::resample_to(r, band, &t) } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f03725d58..c34063f39 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6439,7 +6439,12 @@ fn analysis_workflows_models_scripts_and_raster_functions() { let europe = s.map().layer_by_name("Europe buffers").unwrap().dataset.features.len(); let all = s.map().layer_by_name("Cities").unwrap().dataset.features.len(); assert!(europe > 10 && europe < all / 2, "{europe} of {all}"); - s.run("gp.env", &json!({"extent": null, "cellSize": null})).unwrap(); + let env = s.run("gp.env", &json!({"extent": null, "cellSize": null})).unwrap(); + // This computer's memory limits are said, never below the browser's fixed ones (#1291 step 2). + let limit = |k: &str| env["limits"][k].as_u64().unwrap_or(0) as usize; + assert!(limit("rasterSamples") >= rustgis_data::raster::SAMPLES_FLOOR, "{env}"); + assert!(limit("outputPixels") >= rustgis_analysis::rasterops::PIXELS_FLOOR, "{env}"); + assert!(limit("pagedFeatures") >= rustgis_data::ogc::PAGED_FEATURES_FLOOR, "{env}"); // Network route along the rivers: two points on the same river. let rivers = s.map().layer_by_name("Rivers").unwrap(); let (a, b2) = match rivers.dataset.features.iter().filter_map(|f| f.geometry.as_ref()).find_map(|g| match g { @@ -6772,8 +6777,9 @@ fn a_warp_refuses_a_globe_in_degrees_and_bounds_its_memory() { assert!(e.contains("projected crs"), "{e}"); // In Web Mercator it is placed (to its latitude limit), with the pole inside it. assert!(warp::warp(&r, &fit, &o(Crs::WebMercator, polar, Some(100_000.0))).is_ok()); - // A strip a hundred million metres long and a metre tall, from hostile points. - let strip = georef::fit(Kind::Affine, &gcps(&|c, r| (500_000.0 + c * 2.5e6, 4_000_000.0 - r * 0.025))).unwrap(); + // A strip a billion metres long and a metre tall, from hostile points: past the most samples + // any computer is allowed (rustgis_data::raster::SAMPLES_CEILING), its row buffers counted. + let strip = georef::fit(Kind::Affine, &gcps(&|c, r| (500_000.0 + c * 2.5e7, 4_000_000.0 - r * 0.025))).unwrap(); let utm13 = Crs::from_epsg(32613).unwrap(); let e = warp::warp(&r, &strip, &o(utm13, utm13, Some(1.0))).unwrap_err(); assert!(e.contains("cellSize"), "{e}"); diff --git a/crates/ui-egui/src/serviceui.rs b/crates/ui-egui/src/serviceui.rs index aad69113f..e5a3e7d31 100644 --- a/crates/ui-egui/src/serviceui.rs +++ b/crates/ui-egui/src/serviceui.rs @@ -297,7 +297,7 @@ pub struct PageRead { /// leads nowhere, else read on in the background ([`poll_pages`]). pub(crate) fn read_pages(app: &mut RustGisApp, landed: Landed, first: Vec) { let left = rustgis_engine::render::rustgis_tiles::cache::MAX_DOWNLOAD.saturating_sub(first.len()); - let step = ogc::read_pages(&landed.url, first, ogc::MAX_PAGED_FEATURES); + let step = ogc::read_pages(&landed.url, first, ogc::max_paged_features()); if matches!(step, Step::Fetch(_)) { app.session.notify(i18n::fmt("Reading {url} page by page…", &[("url", &landed.url)]), false); } diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 0583a9346..dd130de68 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -853,7 +853,20 @@ honour it, this restricts an unselected main feature input by bounding-box inter clipping its geometry; an existing selection takes precedence and overlay inputs stay whole. The Geostatistical Wizard uses the environment's extent and raster cell size (degrees). IDW Interpolation and Ordinary Kriging instead use their own `extent` and `pixelSize` parameters; -the environment does not set their output grids. Project Raster (`raster.project`) puts a raster +the environment does not set their output grids. + +`gp.env` also returns the memory limits on this computer (`limits`). On the desktop they follow the +computer's memory, or a container's limit when it is lower: each plans for a quarter of it, never +below the web app's fixed limits and never above a ceiling. + +| Limit | Fixed (web app, smallest) | Desktop: a quarter of memory at | Ceiling | +|---|---|---|---| +| `rasterSamples`: samples a raster read holds (cells times bands) | 120 million (32 million in the web app) | 8 bytes a sample | 1 billion | +| `outputPixels`: pixels of a raster a tool makes | 64 million | 16 bytes a pixel | 512 million | +| `pagedFeatures`: features read from a service page by page | 100,000 | 20 KB a feature (about 100,000 for every 8 GB) | 1 million | + +So a computer with 16 GB reads rasters of up to about 500 million samples. A run that passes on one +computer can be refused on a smaller one; the error names the limit there. Project Raster (`raster.project`) puts a raster on a grid of any coordinate system, cell size and extent; Raster Functions › Resample matches a target raster's grid. @@ -990,7 +1003,7 @@ The Analyze tab's imagery groups open their tools in the Workflows pane: are layers or file paths; where they overlap, the first one with data wins. Tiles in different coordinate systems or cell sizes, or opened from an overview, are refused rather than quietly resampled, as are tiles whose cells don't line up; the mosaic has to fit in memory (120 million - values: cells times bands). + values, cells times bands, or more on a computer with more memory: see Environments). - **Project Raster** (`raster.project`) puts a raster on a grid of your choosing, so rasters from different sources line up cell for cell (for map algebra, the Suitability Modeler, a series for Trend Analysis, or the inputs of a model): `{"layer": "ndvi", "crs": 31983, "cellSize": 100, @@ -1025,8 +1038,8 @@ The Analyze tab's imagery groups open their tools in the Workflows pane: colour image gains an alpha band instead, so they are see-through. - Datums are shifted as everywhere in RustGIS, through each system's transformation to WGS 84 (a Helmert step for SAD69 or Córrego Alegre, a few metres from the IBGE's grid shifts). - - The result has to fit in memory (120 million values, cells times bands; 32 million in the - web app): a grid past that is refused, naming the limit. A raster read from its overview is + - The result has to fit in memory (120 million values, cells times bands, or more on a computer + with more memory; 32 million in the web app): a grid past that is refused, naming the limit. A raster read from its overview is projected at that resolution, and the result says so. - **Classification Wizard** (unsupervised k-means) and **Classification Tools** (minimum distance from training polygons). diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index c2a2d76d2..cf51d9fb1 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -213,11 +213,11 @@ Scripts use `service.import {"url":"…","name":"My OGC Layer"}` and | --- | --- | --- | | WMS 1.1.1 / 1.3.0 | Named layers offered in Web Mercator or in a coordinate system RustGIS can transform (an EPSG code, CRS:84) | A layer of GetMap images (PNG when the service offers it, else JPEG): one picture of the view, or tiles; see [One picture of the view, or tiles](#one-picture-of-the-view-or-tiles) and [Services in other coordinate systems](#services-in-other-coordinate-systems) | | WMTS | Layers whose tile matrix set is Web Mercator's pyramid, or a tile matrix set in a coordinate system RustGIS can transform | A tile layer from the tile template (or KVP GetTile) | -| WFS 1.1 / 2.0 | Feature types | Their features downloaded as GeoJSON, following next links. Without a next link, WFS 1.x reads only the first response (requested limit: 50,000); reaching that limit produces an incomplete-read notice. A server may cap this response below the requested limit. WFS 2.0 verifies the hits total and uses offset requests when needed, up to 100,000 features; unknown totals, repeated offset pages and interrupted reads produce a notice | -| OGC API Features | Collections | Their items downloaded as GeoJSON, following the service's `next` pages up to 100,000 features (a command such as `service.add` waits a minute at most) | +| WFS 1.1 / 2.0 | Feature types | Their features downloaded as GeoJSON, following next links. Without a next link, WFS 1.x reads only the first response (requested limit: 50,000); reaching that limit produces an incomplete-read notice. A server may cap this response below the requested limit. WFS 2.0 verifies the hits total and uses offset requests when needed, up to 100,000 features or more on a computer with more memory; unknown totals, repeated offset pages and interrupted reads produce a notice | +| OGC API Features | Collections | Their items downloaded as GeoJSON, following the service's `next` pages up to 100,000 features, or more on a computer with more memory (see Environments in the analysis guide; a command such as `service.add` waits a minute at most) | | OGC API Tiles | Collections' map and vector tiles; a landing page's dataset map tiles | A tile layer in WebMercatorQuad, from the tile template the tileset advertises (its `item` link; `…/tiles/WebMercatorQuad/{z}/{y}/{x}`, the usual path, when that can't be read), keeping the tiles link's query (a token), map tiles in the server's image format, vector tiles drawn layer by layer in colours of their own | | WCS 1.0 / 2.0 (a 1.1 service is asked in 1.0) | Coverages, with their bounds | A raster: GetCoverage of the map's view within the coverage's bounds (or its whole extent: `"bbox": "full"`), as a GeoTIFF in EPSG:4326, at most 2048 pixels on a side | -| ArcGIS REST | FeatureServer and MapServer layers; a MapServer's map; an ImageServer; a VectorTileServer | Layers queried as GeoJSON, page by page past the server's record limit (`resultOffset`) up to 100,000 features (a command such as `service.add` waits a minute at most), where the layer supports paging (ArcGIS Server 10.3 and later; otherwise up to the record limit); a map as its tile cache when it is Web Mercator's pyramid, else exported images; an ImageServer as exported images; a VectorTileServer as its MapLibre style | +| ArcGIS REST | FeatureServer and MapServer layers; a MapServer's map; an ImageServer; a VectorTileServer | Layers queried as GeoJSON, page by page past the server's record limit (`resultOffset`) up to 100,000 features or more on a computer with more memory (a command such as `service.add` waits a minute at most), where the layer supports paging (ArcGIS Server 10.3 and later; otherwise up to the record limit); a map as its tile cache when it is Web Mercator's pyramid, else exported images; an ImageServer as exported images; a VectorTileServer as its MapLibre style | A WMS or WMTS layer's tiles are asked for down to the detail its service gives, and stretched only past it: a WMTS layer's over the zooms of its tile matrix set (or those its diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 6162dda33..f43f69e01 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -913,8 +913,9 @@ as RGB; other YCbCr and CMYK images aren't read. A rotated or skewed image is re north-up grid as it is read. The coordinate system is an EPSG code, or the file's own projection described in its GeoKeys (Albers, Lambert conformal conic, transverse Mercator, Mercator, Lambert azimuthal, stereographic, equirectangular, sinusoidal and others), which an -exported GeoTIFF keeps. A raster too large to hold in memory (more than 120 million samples, 32 -million in the web app) opens from its largest overview that fits; one without overviews opens as +exported GeoTIFF keeps. A raster too large to hold in memory (more than 120 million samples, or +more on a computer with more memory, see Environments in the analysis guide; 32 million in the +web app) opens from its largest overview that fits; one without overviews opens as an overview made as it is read, a strip or tile at a time (every second, fourth… pixel each way), so reading it takes as long as decoding the whole file once. A cloud-optimised GeoTIFF added by its web address is read in place instead, a tile at a time (see [Files on the diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 8e06e7e78..7db5c859f 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -1762,6 +1762,15 @@ dependencies = [ "libc", ] +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -1816,6 +1825,16 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-system-configuration" version = "0.3.2" @@ -2341,6 +2360,7 @@ dependencies = [ "serde", "serde_json", "shapefile", + "sysinfo", "tempfile", "thiserror 2.0.21", "tiberius", @@ -2896,6 +2916,20 @@ dependencies = [ "web-sys", ] +[[package]] +name = "sysinfo" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e8a3edbe0394ce115f3917a00349dd242be09eb5a091ed509eaf67b55cfb8a8" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + [[package]] name = "tempfile" version = "3.27.0" @@ -3584,6 +3618,27 @@ dependencies = [ "winapi", ] +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -3597,6 +3652,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -3625,6 +3691,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -3677,6 +3753,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" From 7a760152db62d143b6991708b192ea438279202b Mon Sep 17 00:00:00 2001 From: Dominik Strebel Date: Tue, 3 Nov 2026 11:23:09 +0100 Subject: [PATCH 2/2] fix(data): saved GeoTIFF parts are read with the browser's limit; notices for sysinfo From the review of #1372: - The parts of a GeoTIFF the web app saves hold the overview it chose under its 32 million sample limit. Read back on a desktop with a larger limit, a finer overview the parts don't hold could be chosen, and the file failed to open. read_bytes now reads them with raster::BROWSER_SAMPLES (geotiff::read_saved_parts). - THIRD-PARTY-LICENSES.txt and the Python package's, refreshed (cargo xtask notices) for sysinfo, ntapi and objc2-io-kit. --- THIRD-PARTY-LICENSES.txt | 31 ++++++++++++++- apps/rustgis-python/THIRD-PARTY-LICENSES.txt | 40 +++++++++++++++++--- crates/data/src/raster/geotiff.rs | 10 ++++- crates/data/src/raster/mod.rs | 12 ++++-- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 1b5f5b864..7bd654831 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -29,8 +29,8 @@ indexmap, inout, io-enum, itertools, itoa, jobserver, js-sys, keyboard-types, kh khronos_api, kurbo, laz, lazy_static, leb128fmt, libc, linebender_resource_handle, linux-raw-sys, litrs, lock_api, log, lyon, lyon_algorithms, lyon_geom, lyon_path, lyon_tessellation, md-5, md5, miniz_oxide, muda, multiversion_no_op, mysql, mysql_common, naga, naga-types, named_pipe, -nohash-hasher, num, num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational, -num-traits, objc2-app-kit, objc2-core-foundation, objc2-core-graphics, objc2-metal, +nohash-hasher, ntapi, num, num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational, +num-traits, objc2-app-kit, objc2-core-foundation, objc2-core-graphics, objc2-io-kit, objc2-metal, objc2-quartz-core, objc2-system-configuration, once_cell, openssl-probe, ordered-stream, owned_ttf_parser, parking, parking_lot, parking_lot_core, parquet, peniko, percent-encoding, petgraph, pin-project, pin-project-internal, pin-project-lite, piper, pkg-config, png, polling, @@ -10219,6 +10219,33 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +==================================================================================================== +MIT License +Used by: sysinfo +==================================================================================================== + +The MIT License (MIT) + +Copyright (c) 2015 Guillaume Gomez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + ==================================================================================================== MIT License Used by: tiff diff --git a/apps/rustgis-python/THIRD-PARTY-LICENSES.txt b/apps/rustgis-python/THIRD-PARTY-LICENSES.txt index 2cb649601..830797b02 100644 --- a/apps/rustgis-python/THIRD-PARTY-LICENSES.txt +++ b/apps/rustgis-python/THIRD-PARTY-LICENSES.txt @@ -21,10 +21,10 @@ flatbuffers, flate2, form_urlencoded, futures-lite, futures-timer, geo, geo-trai geozero, gif, half, hash32, hashbrown, hashlink, heapless, heck, hkdf, hmac, httparse, hybrid-array, i_overlay, idna, idna_adapter, image, image-webp, indexmap, inout, io-enum, itoa, jobserver, js-sys, laz, lazy_static, libc, linux-raw-sys, lock_api, log, lyon, lyon_algorithms, lyon_geom, lyon_path, -lyon_tessellation, md-5, md5, miniz_oxide, multiversion_no_op, mysql, mysql_common, named_pipe, num, -num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational, num-traits, -objc2-core-foundation, objc2-system-configuration, once_cell, openssl-probe, ordered-stream, -owned_ttf_parser, parking, parking_lot, parking_lot_core, parquet, percent-encoding, +lyon_tessellation, md-5, md5, miniz_oxide, multiversion_no_op, mysql, mysql_common, named_pipe, +ntapi, num, num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational, num-traits, +objc2-core-foundation, objc2-io-kit, objc2-system-configuration, once_cell, openssl-probe, +ordered-stream, owned_ttf_parser, parking, parking_lot, parking_lot_core, parquet, percent-encoding, pin-project-lite, piper, pkg-config, png, polling, postgres, postgres-protocol, postgres-types, proc-macro2, proj4rs, proj4rs-geodesic, pyo3, pyo3-build-config, pyo3-ffi, pyo3-macros, pyo3-macros-backend, quote, rand, rayon, rayon-core, regex, regex-automata, regex-syntax, ring, @@ -5669,8 +5669,9 @@ Used by: web-time ==================================================================================================== Apache License 2.0 -Used by: windows-core, windows-implement, windows-interface, windows-link, windows-result, windows-strings, -windows-sys, windows-targets, windows_aarch64_msvc, windows_x86_64_gnu, windows_x86_64_msvc +Used by: windows, windows-collections, windows-core, windows-future, windows-implement, windows-interface, +windows-link, windows-numerics, windows-result, windows-strings, windows-sys, windows-targets, +windows-threading, windows_aarch64_msvc, windows_x86_64_gnu, windows_x86_64_msvc ==================================================================================================== Apache License @@ -8487,6 +8488,33 @@ The above copyright notice and this permission notice shall be included in all c THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +==================================================================================================== +MIT License +Used by: sysinfo +==================================================================================================== + +The MIT License (MIT) + +Copyright (c) 2015 Guillaume Gomez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + ==================================================================================================== MIT License Used by: tiff diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 0da1052ab..90c68299d 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -975,6 +975,13 @@ pub fn read_sparse(parts: &Sparse, name: &str) -> Result { read_sparse_limited(parts, name, max_samples()) } +/// [`read_sparse`] of the parts the web app saved, with its limit ([`super::BROWSER_SAMPLES`]): it +/// kept the overview it chose with that, which a larger limit could pass over for a finer one the +/// parts don't hold. +pub fn read_saved_parts(parts: &Sparse, name: &str) -> Result { + read_sparse_limited(parts, name, super::BROWSER_SAMPLES) +} + fn read_sparse_limited(parts: &Sparse, name: &str, max: usize) -> Result { let r = parts.reader(); let watch = r.clone(); @@ -1821,7 +1828,8 @@ mod tests { #[test] fn a_part_container_opens_as_a_tif() { let file = super::super::cog::example(512, 256); - let (r, parts) = read_in_parts(&file, 4096, max_samples()); + // Read as the web app reads, with its limit, as the saved parts are read back anywhere. + let (r, parts) = read_in_parts(&file, 4096, super::super::BROWSER_SAMPLES); let back = super::super::read_bytes("parts.tif", &parts.container()).unwrap(); assert_eq!((back.width, back.height, back.value(0, 300, 200)), (r.width, r.height, r.value(0, 300, 200))); } diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index 1e24eab90..31dc5efdb 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -66,12 +66,16 @@ fn no_memory(w: usize, h: usize) -> DataError { DataError::Format(format!("not enough memory for a {w} × {h} raster")) } -/// The fewest samples (pixels × bands) [`max_samples`] allows: 120 million (480 MB), or 32 -/// million in the browser, whose memory is much tighter (its limit there). +/// Most samples (pixels × bands) held in memory in the browser, whose memory is much tighter: 32 +/// million. The parts of a GeoTIFF the web app saves are read with it anywhere, so that the same +/// overview is chosen. +pub const BROWSER_SAMPLES: usize = 32_000_000; +/// The fewest samples [`max_samples`] allows: 120 million (480 MB) on the desktop, and +/// [`BROWSER_SAMPLES`] (its limit) in the browser. #[cfg(not(target_arch = "wasm32"))] pub const SAMPLES_FLOOR: usize = 120_000_000; #[cfg(target_arch = "wasm32")] -pub const SAMPLES_FLOOR: usize = 32_000_000; +pub const SAMPLES_FLOOR: usize = BROWSER_SAMPLES; /// The most [`max_samples`] allows, however much memory the computer has: a billion (4 GB). pub const SAMPLES_CEILING: usize = 1_000_000_000; @@ -1110,7 +1114,7 @@ pub fn read_bytes(file_name: &str, bytes: &[u8]) -> Result { // The parts of a large GeoTIFF the web app read (see `geotiff::read_sparse`). "tif" | "tiff" if cog::Sparse::is_container(bytes) => { let parts = cog::Sparse::from_container(bytes).ok_or_else(|| DataError::Format("a damaged GeoTIFF part file".into()))?; - match geotiff::read_sparse(&parts, &name)? { + match geotiff::read_saved_parts(&parts, &name)? { geotiff::Partial::Done(r) => Ok(*r), geotiff::Partial::Need(..) => Err(DataError::Format("the parts of the GeoTIFF kept don't hold the whole overview".into())), }