feat(data): the memory limits follow the computer's memory on the desktop (#1291 step 2)

THIRD-PARTY-LICENSES.txt

@@ -29,8 +29,8 @@ indexmap, inout, io-enum, itertools, itoa, jobserver, js-sys, keyboard-types, kh
 khronos_api, kurbo, laz, lazy_static, leb128fmt, libc, linebender_resource_handle, linux-raw-sys,
 litrs, lock_api, log, lyon, lyon_algorithms, lyon_geom, lyon_path, lyon_tessellation, md-5, md5,
 miniz_oxide, muda, multiversion_no_op, mysql, mysql_common, naga, naga-types, named_pipe,
-nohash-hasher, num, num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational,
-num-traits, objc2-app-kit, objc2-core-foundation, objc2-core-graphics, objc2-metal,
+nohash-hasher, ntapi, num, num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational,
+num-traits, objc2-app-kit, objc2-core-foundation, objc2-core-graphics, objc2-io-kit, objc2-metal,
 objc2-quartz-core, objc2-system-configuration, once_cell, openssl-probe, ordered-stream,
 owned_ttf_parser, parking, parking_lot, parking_lot_core, parquet, peniko, percent-encoding,
 petgraph, pin-project, pin-project-internal, pin-project-lite, piper, pkg-config, png, polling,
@@ -10219,6 +10219,33 @@ The above copyright notice and this permission notice shall be included in all c
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+====================================================================================================
+MIT License
+Used by: sysinfo
+====================================================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Guillaume Gomez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ====================================================================================================
 MIT License
 Used by: tiff

apps/rustgis-python/THIRD-PARTY-LICENSES.txt

@@ -21,10 +21,10 @@ flatbuffers, flate2, form_urlencoded, futures-lite, futures-timer, geo, geo-trai
 geozero, gif, half, hash32, hashbrown, hashlink, heapless, heck, hkdf, hmac, httparse, hybrid-array,
 i_overlay, idna, idna_adapter, image, image-webp, indexmap, inout, io-enum, itoa, jobserver, js-sys,
 laz, lazy_static, libc, linux-raw-sys, lock_api, log, lyon, lyon_algorithms, lyon_geom, lyon_path,
-lyon_tessellation, md-5, md5, miniz_oxide, multiversion_no_op, mysql, mysql_common, named_pipe, num,
-num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational, num-traits,
-objc2-core-foundation, objc2-system-configuration, once_cell, openssl-probe, ordered-stream,
-owned_ttf_parser, parking, parking_lot, parking_lot_core, parquet, percent-encoding,
+lyon_tessellation, md-5, md5, miniz_oxide, multiversion_no_op, mysql, mysql_common, named_pipe,
+ntapi, num, num-bigint, num-complex, num-conv, num-integer, num-iter, num-rational, num-traits,
+objc2-core-foundation, objc2-io-kit, objc2-system-configuration, once_cell, openssl-probe,
+ordered-stream, owned_ttf_parser, parking, parking_lot, parking_lot_core, parquet, percent-encoding,
 pin-project-lite, piper, pkg-config, png, polling, postgres, postgres-protocol, postgres-types,
 proc-macro2, proj4rs, proj4rs-geodesic, pyo3, pyo3-build-config, pyo3-ffi, pyo3-macros,
 pyo3-macros-backend, quote, rand, rayon, rayon-core, regex, regex-automata, regex-syntax, ring,
@@ -5669,8 +5669,9 @@ Used by: web-time
 
 ====================================================================================================
 Apache License 2.0
-Used by: windows-core, windows-implement, windows-interface, windows-link, windows-result, windows-strings,
-windows-sys, windows-targets, windows_aarch64_msvc, windows_x86_64_gnu, windows_x86_64_msvc
+Used by: windows, windows-collections, windows-core, windows-future, windows-implement, windows-interface,
+windows-link, windows-numerics, windows-result, windows-strings, windows-sys, windows-targets,
+windows-threading, windows_aarch64_msvc, windows_x86_64_gnu, windows_x86_64_msvc
 ====================================================================================================
 
                                  Apache License
@@ -8487,6 +8488,33 @@ The above copyright notice and this permission notice shall be included in all c
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
+====================================================================================================
+MIT License
+Used by: sysinfo
+====================================================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2015 Guillaume Gomez
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ====================================================================================================
 MIT License
 Used by: tiff

crates/analysis/src/kriging.rs

@@ -12,7 +12,7 @@
 use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster};
 use rustgis_geom::{Coord, Extent};
 
-use crate::rasterops::{MAX_PIXELS, NODATA};
+use crate::rasterops::{NODATA, max_pixels};
 
 /// Maximum cell-point pairs Kriging computes (about a second's work).
 pub const MAX_KRIGING_WORK: usize = 40_000_000;
@@ -337,8 +337,8 @@ fn kriging_on(
         return Err("the area to interpolate over is empty".into());
     }
     let (w, h) = ((extent.width() / cell_deg).ceil().max(1.0) as usize, (extent.height() / cell_deg).ceil().max(1.0) as usize);
-    if w.saturating_mul(h) > MAX_PIXELS / 4 {
-        return Err(format!("{w} × {h} cells is too many; use a larger cell size"));
+    if w.saturating_mul(h) > max_pixels() / 4 {
+        return Err(format!("{w} × {h} cells is too many, more than this computer's {}; use a larger cell size", max_pixels() / 4));
     }
     let k = if neighbours == 0 { 12.min(points.len()) } else { neighbours.clamp(2, points.len().min(32)) };
     let k3 = (k * k * k) / 3;

crates/analysis/src/rasterops.rs

@@ -9,8 +9,17 @@ use rustgis_geom::{Coord, Extent};
 
 /// The no-data value written by these operations.
 pub const NODATA: f32 = -3.4e38;
-/// Largest output raster built (pixels).
-pub const MAX_PIXELS: usize = 64_000_000;
+/// The fewest pixels [`max_pixels`] allows, and its limit in the browser.
+pub const PIXELS_FLOOR: usize = 64_000_000;
+/// The most [`max_pixels`] allows, however much memory the computer has (2 GB a band).
+pub const PIXELS_CEILING: usize = 512_000_000;
+
+/// Largest output raster built (pixels): on the desktop, as many as a quarter of the computer's
+/// memory holds at 16 bytes each (a tool holds its input, its output and a working grid or two),
+/// from [`PIXELS_FLOOR`] up to [`PIXELS_CEILING`] (#1291 step 2).
+pub fn max_pixels() -> usize {
+    rustgis_data::memory::scaled(rustgis_data::memory::physical(), PIXELS_FLOOR, PIXELS_CEILING, 16)
+}
 
 /// Room for a grid of `w × h` cells of `T` (an output band, or a working grid of one), empty: a
 /// grid the machine can't hold is an error saying so, instead of ending the program
@@ -449,8 +458,8 @@ fn idw_on(
         return Err("the area to interpolate over is empty".into());
     }
     let (w, h) = ((extent.width() / cell_deg).ceil().max(1.0) as usize, (extent.height() / cell_deg).ceil().max(1.0) as usize);
-    if w.saturating_mul(h) > MAX_PIXELS / 4 {
-        return Err(format!("{w} × {h} cells is too many; use a larger cell size"));
+    if w.saturating_mul(h) > max_pixels() / 4 {
+        return Err(format!("{w} × {h} cells is too many, more than this computer's {}; use a larger cell size", max_pixels() / 4));
     }
     // Every cell weighs every point.
     if w.saturating_mul(h).saturating_mul(points.len()) > MAX_IDW_WORK {

crates/analysis/src/warp.rs

@@ -13,7 +13,7 @@
 
 use std::collections::HashMap;
 
-use rustgis_data::raster::{ColorHint, Crs, GeoTransform, MAX_SAMPLES, Raster, Transformer};
+use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster, Transformer, max_samples};
 use rustgis_geom::Coord;
 
 use crate::georef::Fit;
@@ -237,12 +237,15 @@ pub fn warp(r: &Raster, fit: &Fit, o: &Options) -> Result<Raster, String> {
     let cells = |cell: f64| -> (f64, f64) { (((max_x - min_x) / cell).ceil().max(1.0), ((max_y - min_y) / cell).ceil().max(1.0)) };
     // The samples held, and the buffers a row needs (they add up on a long thin grid).
     let load = |(ow, oh): (f64, f64)| ow * (oh * out_bands as f64 + COLUMN_COST);
-    let too_many = |n: (f64, f64)| load(n) > MAX_SAMPLES as f64;
+    let max = max_samples();
+    let too_many = |n: (f64, f64)| load(n) > max as f64;
     let cell = match o.cell {
         Some(c) if !(c.is_finite() && c > 0.0) => return Err("cellSize must be a positive number".into()),
         Some(c) => {
             if too_many(cells(c)) {
-                return Err(format!("a cell size of {c} makes a grid of more than {MAX_SAMPLES} samples: give a larger cellSize"));
+                return Err(format!(
+                    "a cell size of {c} makes a grid of more than {max} samples, this computer's limit: give a larger cellSize"
+                ));
             }
             c
         }
@@ -254,7 +257,7 @@ pub fn warp(r: &Raster, fit: &Fit, o: &Options) -> Result<Raster, String> {
                 if !too_many(n) {
                     break;
                 }
-                cell *= (load(n) / MAX_SAMPLES as f64).sqrt().max(1.01);
+                cell *= (load(n) / max as f64).sqrt().max(1.01);
             }
             if too_many(cells(cell)) {
                 return Err("the warped image is too large to hold: give a cellSize".into());
@@ -361,9 +364,10 @@ impl Grid {
             return Err("the origin and cellSize can't make a grid over this area".into());
         }
         let (w, h) = (((max_x - min_x) / cx).round().max(1.0), ((max_y - min_y) / cy).round().max(1.0));
-        if w * h > MAX_SAMPLES as f64 {
+        let max = max_samples();
+        if w * h > max as f64 {
             return Err(format!(
-                "a cell size of {cx} makes a grid of {} × {} cells, more than {MAX_SAMPLES}: give a larger cellSize or a smaller extent",
+                "a cell size of {cx} makes a grid of {} × {} cells, more than this computer's {max}: give a larger cellSize or a smaller extent",
                 w as u64, h as u64
             ));
         }
@@ -452,9 +456,10 @@ pub fn project(r: &Raster, g: &Grid, method: Method, name: String) -> Result<Ras
     let canvas = Canvas::of(r, false);
     let out_bands = canvas.out_bands;
     // The samples held, and the buffers a row needs.
-    if g.width as f64 * (g.height as f64 * out_bands as f64 + COLUMN_COST) > MAX_SAMPLES as f64 {
+    let max = max_samples();
+    if g.width as f64 * (g.height as f64 * out_bands as f64 + COLUMN_COST) > max as f64 {
         return Err(format!(
-            "{} × {} cells of {} band(s) are more than {MAX_SAMPLES} samples: give a larger cellSize or a smaller extent",
+            "{} × {} cells of {} band(s) are more than this computer's {max} samples: give a larger cellSize or a smaller extent",
             g.width, g.height, out_bands
         ));
     }

crates/data/Cargo.toml

@@ -57,6 +57,8 @@ rustls = { workspace = true }
 webpki-roots = { workspace = true }
 # Private temporary workspaces for downloaded JPEG 2000 conversion through GDAL.
 tempfile = "3"
+# The computer's physical memory, for the limits that scale with it (`memory`, #1291 step 2).
+sysinfo = { version = "0.40", default-features = false, features = ["system"] }
 
 [target.'cfg(all(unix, not(target_arch = "wasm32")))'.dependencies]
 # Safe effective-UID and no-follow operations for private Shapefile index caches.

crates/data/src/lib.rs

@@ -23,6 +23,7 @@ pub mod landxml;
 pub mod lidar;
 pub mod mapinfo;
 pub mod mbtiles;
+pub mod memory;
 pub mod ogc;
 pub mod osm;
 pub mod photo;

crates/data/src/memory.rs

@@ -0,0 +1,68 @@
+//! The computer's memory, and the limits that follow it (#1291 step 2, #1318).
+//!
+//! On the desktop a memory limit plans for a share of the physical memory ([`SHARE`]): never below
+//! the fixed value it had before (its floor, still the browser's limit), never above a ceiling. A
+//! container's memory limit counts, when it is lower. The memory is read once, at the first limit
+//! asked for, so a session keeps the same limits; [`scaled`] takes it as an argument, so tests pass
+//! fixed amounts.
+
+/// A limit plans for a quarter of the physical memory: room for the layers already held, the
+/// drawing, and a second job running at the same time.
+pub const SHARE: u64 = 4;
+
+/// The computer's physical memory in bytes (a container's limit, when it is lower), read once;
+/// `None` in the browser, or when it can't be read.
+pub fn physical() -> Option<u64> {
+    #[cfg(not(target_arch = "wasm32"))]
+    {
+        static MEMORY: std::sync::OnceLock<Option<u64>> = std::sync::OnceLock::new();
+        *MEMORY.get_or_init(read)
+    }
+    #[cfg(target_arch = "wasm32")]
+    {
+        None
+    }
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn read() -> Option<u64> {
+    let mut s = sysinfo::System::new().ok()?;
+    s.refresh_memory_specifics(sysinfo::MemoryRefreshKind::nothing().with_ram());
+    let total = Some(s.total_memory()).filter(|m| *m > 0)?;
+    // A container's limit, on Linux; none is reported elsewhere.
+    Some(s.cgroup_limits().map(|c| c.total_memory).filter(|m| *m > 0).map_or(total, |c| c.min(total)))
+}
+
+/// A limit of units costing `bytes` each: as many as [`SHARE`] of `memory` holds, within `floor`
+/// and `ceiling`; `floor` when the memory isn't known.
+pub fn scaled(memory: Option<u64>, floor: usize, ceiling: usize, bytes: u64) -> usize {
+    let Some(memory) = memory else { return floor };
+    let n = memory / SHARE / bytes.max(1);
+    usize::try_from(n).unwrap_or(usize::MAX).clamp(floor, ceiling.max(floor))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const GIB: u64 = 1 << 30;
+
+    #[test]
+    fn a_limit_follows_the_memory_between_its_floor_and_ceiling() {
+        // 8 bytes a unit: 16 GiB plans for 4 GiB, 512 Mi units.
+        assert_eq!(scaled(Some(16 * GIB), 100, 1 << 40, 8), 512 << 20);
+        assert_eq!(scaled(Some(GIB), 200_000_000, 1 << 40, 8), 200_000_000, "never below the floor");
+        assert_eq!(scaled(Some(1024 * GIB), 100, 1_000, 8), 1_000, "never above the ceiling");
+        assert_eq!(scaled(None, 100, 1_000, 8), 100, "unknown memory: the floor");
+        assert_eq!(scaled(Some(u64::MAX), 100, usize::MAX, 0) as u64, u64::MAX / SHARE, "no overflow or division by zero");
+        assert_eq!(scaled(Some(16 * GIB), 100, 50, 8), 100, "a ceiling below the floor is the floor");
+    }
+
+    #[cfg(not(target_arch = "wasm32"))]
+    #[test]
+    fn the_memory_is_read_once_and_stays() {
+        let first = physical();
+        assert!(first.is_none_or(|m| m >= 64 << 20), "{first:?}");
+        assert_eq!(physical(), first);
+    }
+}