feat(ui): Add Data and Load All read their file off the frame (#1291 step 3)

apps/rustgis/src/main.rs

@@ -515,6 +515,8 @@ fn main() -> eframe::Result {
         app
     };
     let mut app = app.with_services(services);
+    // Add Data and Load All read on a thread of their own, so a large file doesn't hold the window.
+    app.reads_off_frame = true;
     // Settings › Start-up; `--sample` still opens the sample.
     let blank = if args.iter().any(|a| a == "--sample") { Some(false) } else { (!files.is_empty()).then_some(true) };
     if let Err(e) = app.start(&rustgis_ui_egui::Start { blank, after_crash: unclean, ..Default::default() }) {

crates/engine/src/cmd/layer.rs

@@ -1286,6 +1286,8 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult {
     #[cfg(not(target_arch = "wasm32"))]
     if is_geojson(path)
         && !rustgis_data::cityjson::is_cityjson_name(path)
+        // One read whole ahead keeps that route, though the file grew since.
+        && !super::reads::placed_for(s, &whole_key(path))
         && b(p, "stream").unwrap_or_else(|| {
             std::fs::metadata(path).is_ok_and(|m| m.is_file() && m.len() > rustgis_data::source::geojson::IN_PLACE_ABOVE)
         })
@@ -1371,14 +1373,10 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult {
     }
     // A spreadsheet: one sheet ("layer"), read like delimited text.
     if rustgis_data::sheets::is_sheet_name(path) {
-        let meta = std::fs::metadata(path).map_err(|e| format!("{path}: {e}"))?;
-        if meta.len() > super::raster::MAX_UPLOAD_BYTES as u64 {
-            return Err(format!("{path} is too large to open"));
-        }
-        let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
-        // Read from disk: the table keeps the workbook's path, so an export can't replace it.
-        let mut read = read_sheet(p, path, &bytes)?;
-        read.0.source = Some(std::path::absolute(path).map_or_else(|_| path.to_string(), |a| a.display().to_string()));
+        let read = match super::reads::take(s, &whole_key(path)) {
+            Some(read) => read,
+            None => read_sheet_file(p, path),
+        }?;
         return add_read(s, p, read, None);
     }
     #[cfg(not(target_arch = "wasm32"))]
@@ -1394,6 +1392,36 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult {
         let full = std::path::absolute(path).map(|p| p.display().to_string()).unwrap_or_else(|_| path.to_string());
         return add_layers(s, p, &full, &bytes, None);
     }
+    let read = match super::reads::take(s, &whole_key(path)) {
+        Some(read) => read,
+        None => read_whole(p, path),
+    }?;
+    add_read(s, p, read, None)
+}
+
+/// A file Add Data reads whole: the features, and what reading them said.
+type WholeRead = Result<(rustgis_data::Dataset, rustgis_data::ReadReport), String>;
+
+/// The key Add Data's whole read of `path` is taken by (see [`super::reads`]).
+fn whole_key(path: &str) -> String {
+    format!("whole:{path}")
+}
+
+/// A spreadsheet's sheet ("layer"), read like delimited text.
+fn read_sheet_file(p: &Value, path: &str) -> WholeRead {
+    let meta = std::fs::metadata(path).map_err(|e| format!("{path}: {e}"))?;
+    if meta.len() > super::raster::MAX_UPLOAD_BYTES as u64 {
+        return Err(format!("{path} is too large to open"));
+    }
+    let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
+    // Read from disk: the table keeps the workbook's path, so an export can't replace it.
+    let mut read = read_sheet(p, path, &bytes)?;
+    read.0.source = Some(std::path::absolute(path).map_or_else(|_| path.to_string(), |a| a.display().to_string()));
+    Ok(read)
+}
+
+/// Delimited text with the options `p` gives, or another file its format's reader reads whole.
+fn read_whole(p: &Value, path: &str) -> WholeRead {
     let delimited = read_delimited(p, path, || {
         let meta = std::fs::metadata(path).map_err(|e| format!("{path}: {e}"))?;
         // The limit the text readers keep without options too.
@@ -1402,11 +1430,58 @@ fn add_data(s: &mut Session, p: &Value) -> CmdResult {
         }
         std::fs::read(path).map_err(|e| format!("{path}: {e}"))
     })?;
-    let read = match delimited {
-        Some(read) => read,
-        None => rustgis_data::io::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}"))?,
-    };
-    add_read(s, p, read, None)
+    match delimited {
+        Some(read) => Ok(read),
+        None => rustgis_data::io::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}")),
+    }
+}
+
+/// Add Data's read of the file `p` names, made before it runs ([`super::reads`]): a raster read
+/// whole or converted by GDAL, a GeoJSON read whole, delimited text or a spreadsheet. `None` for
+/// any other file, which [`add_data`] reads as it adds it. Each is picked only where `add_data`
+/// takes that read, so a read made is never wasted; a few it makes whole are left to it (a GeoJSON
+/// with `"stream"` given, a `.json` with a coordinate system or layer given, a large GeoJSON that
+/// isn't a FeatureCollection).
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn add_data_ahead(p: &Value) -> Option<super::reads::ReadAhead> {
+    use super::reads::ahead;
+    const NAME: &str = "Add Data";
+    let path = gdb_path(ps(p, "path")?).into_owned();
+    let scan = super::raster::wants_scan(p, &path).ok()?;
+    if !scan && rustgis_data::photo::is_photo_name(&path) {
+        return None;
+    }
+    if rustgis_data::raster::is_raster_name(&path) {
+        let key = super::raster::read_key(&path);
+        return Some(ahead(NAME, key, move || super::raster::read_raster(&path)));
+    }
+    if rustgis_data::raster::zarr::is_store(Path::new(&path)) {
+        return None;
+    }
+    if rustgis_data::raster::gdal::is_gdal_raster_name(&path) {
+        let text = |k: &str| ps(p, k).map(str::trim).filter(|v| !v.is_empty()).map(str::to_string);
+        let (key, layer, crs) = (super::raster::gdal_key(&path), text("layer"), text("crs"));
+        return Some(ahead(NAME, key, move || super::raster::read_gdal(&path, layer.as_deref(), crs.as_deref())));
+    }
+    let given = |k: &str| ps(p, k).is_some_and(|v| !v.trim().is_empty());
+    let lower = path.to_ascii_lowercase();
+    // A GeoJSON read whole: small, with no "stream" given (which add_data decides by), and not a
+    // CityJSON model (named so, or a .json with a coordinate system or a layer given).
+    let geojson = is_geojson(&path)
+        && !rustgis_data::cityjson::is_cityjson_name(&path)
+        && b(p, "stream").is_none()
+        && !(lower.ends_with(".json") && (given("crs") || given("layer")))
+        && std::fs::metadata(&path).is_ok_and(|m| m.is_file() && m.len() <= rustgis_data::source::geojson::IN_PLACE_ABOVE);
+    let (key, q) = (whole_key(&path), p.clone());
+    if rustgis_data::sheets::is_sheet_name(&path) {
+        return Some(ahead(NAME, key, move || read_sheet_file(&q, &path)));
+    }
+    (geojson || is_delimited_name(&path)).then(|| ahead(NAME, key, move || read_whole(&q, &path)))
+}
+
+#[cfg(target_arch = "wasm32")]
+pub(crate) fn add_data_ahead(_: &Value) -> Option<super::reads::ReadAhead> {
+    None
 }
 
 /// Adds every layer of a file that holds several: the Shapefiles (and other files) of a zip, a KML
@@ -2622,9 +2697,23 @@ fn define_projection(s: &mut Session, p: &Value) -> CmdResult {
 /// Loads a streamed layer's features into memory, so it can be edited like any other layer.
 fn load_all(s: &mut Session, p: &Value) -> CmdResult {
     let id = s.layer_id(p)?;
+    let key = load_all_key(s.map().layer(id).ok_or("no such layer")?);
+    let ahead = super::reads::take::<LoadedAll>(s, &key);
     let l = s.map().layer(id).ok_or("no such layer")?;
-    let ds = l.load_all()?;
-    let n = ds.features.len();
+    let (mut ds, unread) = match ahead {
+        // Read (or failed) from the source the layer still reads, listing the fields it lists
+        // now: one opened afresh since, or with other fields, is read again.
+        Some((copied, fields, read)) if l.source.as_ref().is_some_and(|src| src.same_opening(&copied)) && fields == l.dataset.fields => {
+            read?
+        }
+        _ => l.load_all_counted()?,
+    };
+    // Named as the layer is now: it may have been renamed while the read ran.
+    ds.name = l.dataset.name.clone();
+    let (n, name) = (ds.features.len(), l.name.clone());
+    if unread > 0 {
+        s.warn(format!("{name}: {unread} rows couldn't be read and were left out"));
+    }
     let l = s.map_mut().layer_mut(id).ok_or("no such layer")?;
     l.dataset = ds;
     l.dataset.windowed = false;
@@ -2634,6 +2723,27 @@ fn load_all(s: &mut Session, p: &Value) -> CmdResult {
     Ok(json!({"layer": id, "features": n}))
 }
 
+/// Load All's read made before it runs: the source it read, the fields the layer listed then, and
+/// what it read with the number of rows that couldn't be read.
+type LoadedAll = (rustgis_map::source::SourceLayer, Vec<rustgis_data::Field>, Result<(rustgis_data::Dataset, usize), String>);
+
+/// The key Load All's read of `l` is taken by (see [`super::reads`]).
+fn load_all_key(l: &rustgis_map::Layer) -> String {
+    format!("loadAll:{}", l.instance_id())
+}
+
+/// Load All's read of the streamed layer `p` names, made before it runs ([`super::reads`]), on a
+/// copy of the layer.
+pub(crate) fn load_all_ahead(s: &Session, p: &Value) -> Option<super::reads::ReadAhead> {
+    let l = s.map().layer(s.layer_id(p).ok()?)?;
+    let mut copy = l.clone();
+    let source = copy.source.as_mut()?;
+    // Its own read: the live layer keeps the one under way.
+    source.detach_read();
+    let (copied, fields) = (source.clone(), l.dataset.fields.clone());
+    Some(super::reads::ahead("Load All Features", load_all_key(l), move || (copied, fields, copy.load_all_counted())))
+}
+
 /// Adds a layer file: a layer with its data, symbology and labels.
 fn add_layer_file(s: &mut Session, path: &str) -> CmdResult {
     let bytes = crate::project::read_file(Path::new(path), 512 << 20)?;

crates/engine/src/cmd/mod.rs

@@ -38,6 +38,7 @@ pub mod presentation;
 pub mod project;
 pub mod qgis;
 pub mod raster;
+pub mod reads;
 pub mod repair;
 pub mod report;
 pub mod s3;

crates/engine/src/cmd/raster.rs

@@ -536,7 +536,10 @@ pub(crate) fn say_losses(s: &mut Session, losses: Vec<(String, String)>) {
 
 /// Opens a raster file by path.
 pub fn add_path(s: &mut Session, path: &str, name: Option<&str>, scan: bool) -> CmdResult {
-    let mut r = raster::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}"))?;
+    let mut r = match super::reads::take(s, &read_key(path)) {
+        Some(r) => r,
+        None => read_raster(path),
+    }?;
     if scan {
         r = r.into_scan().map_err(|e| format!("{path}: {e}"))?;
     }
@@ -547,14 +550,45 @@ pub fn add_path(s: &mut Session, path: &str, name: Option<&str>, scan: bool) ->
     add_raster(s, &name, RasterLayer::from_path(r, full))
 }
 
+/// The raster at `path`, read whole.
+pub(crate) fn read_raster(path: &str) -> Result<Raster, String> {
+    raster::read_path(Path::new(path)).map_err(|e| format!("{path}: {e}"))
+}
+
+/// The key [`add_path`]'s read of `path` is taken by (see [`super::reads`]).
+pub(crate) fn read_key(path: &str) -> String {
+    format!("raster:{path}")
+}
+
+/// A raster GDAL converts and the conversion it read.
+#[cfg(not(target_arch = "wasm32"))]
+type GdalRead = Result<(rustgis_data::raster::gdal::Converted, Raster), String>;
+
+/// The raster GDAL converts from `path` (`layer` its subdataset, `crs` a coordinate system the
+/// file doesn't say), with the conversion.
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn read_gdal(path: &str, layer: Option<&str>, crs: Option<&str>) -> GdalRead {
+    use rustgis_data::raster::gdal;
+    let c = gdal::convert(Path::new(path), layer, crs).map_err(|e| format!("{path}: {e}"))?;
+    let r = raster::read_converted(Path::new(path), &c).map_err(|e| format!("{path}: {e}"))?;
+    Ok((c, r))
+}
+
+/// The key [`add_gdal_path`]'s read of `path` is taken by (see [`super::reads`]).
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn gdal_key(path: &str) -> String {
+    format!("gdal:{path}")
+}
+
 /// Adds a raster GDAL reads (converted once to a GeoTIFF in the cache): `"layer"` picks a
 /// subdataset (a NetCDF variable) and `"crs"` gives a coordinate system the file doesn't say.
 #[cfg(not(target_arch = "wasm32"))]
 pub(crate) fn add_gdal_path(s: &mut Session, p: &Value, path: &str) -> CmdResult {
-    use rustgis_data::raster::gdal;
     let text = |k: &str| ps(p, k).map(str::trim).filter(|v| !v.is_empty());
-    let c = gdal::convert(Path::new(path), text("layer"), text("crs")).map_err(|e| format!("{path}: {e}"))?;
-    let r = raster::read_converted(Path::new(path), &c).map_err(|e| format!("{path}: {e}"))?;
+    let (c, r) = match super::reads::take::<GdalRead>(s, &gdal_key(path)) {
+        Some(read) => read,
+        None => read_gdal(path, text("layer"), text("crs")),
+    }?;
     if c.assumed_lonlat {
         s.notify(format!("{path} doesn't say its coordinate system; its coordinates look like longitude/latitude, so it's read as WGS 84 (give \"crs\" if not)"), false);
     }

crates/engine/src/cmd/reads.rs

@@ -0,0 +1,87 @@
+//! Reads the desktop app makes on a thread of its own before a command adds what they read
+//! (#1291 step 3): Add Data's whole-file reads (a raster, a raster GDAL converts, a GeoJSON,
+//! delimited text or spreadsheet read whole) and Load All Features. [`read_ahead`] takes what
+//! the read needs of the session, [`ReadAhead::run`] reads on the thread, and the app then runs
+//! the command as it would have with the read [`Read::placed`] in the session: the command takes
+//! it instead of reading again, so what it adds, says, logs and puts in Undo is the same as when
+//! it reads here. A script or the command line reads where the command runs.
+
+use std::any::Any;
+
+use serde_json::Value;
+
+use crate::Session;
+
+/// What a read gave, for the command it was made for: kept in the session ([`Read::placed`])
+/// while that command runs, and taken by the site that would have read it.
+pub struct Read {
+    key: String,
+    value: Box<dyn Any + Send>,
+}
+
+impl Read {
+    /// Puts the read in the session for the next command to take, until [`Session::drop_read`].
+    pub fn placed(self, s: &mut Session) {
+        s.read_ahead = Some(self);
+    }
+}
+
+/// A read started for a command, to run on a thread of its own.
+pub struct ReadAhead {
+    /// What the read is called, for its progress and a cancelled run's error.
+    pub name: &'static str,
+    key: String,
+    work: Box<dyn FnOnce() -> Box<dyn Any + Send> + Send>,
+}
+
+impl ReadAhead {
+    /// Reads, blocking until it's done; `control` stops it and counts its progress (see
+    /// [`rustgis_analysis::control`]). A read that fails gives its failure to the command, which
+    /// says it as it would have; only a cancelled run (or a bug in a reader) is an error here.
+    pub fn run(self, control: &std::sync::Arc<rustgis_analysis::control::Control>) -> Result<Read, String> {
+        let ReadAhead { name, key, work } = self;
+        let mut value = None;
+        // A bug in a reader must not take the app down.
+        let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            rustgis_analysis::run_under(name, control, || {
+                value = Some(work());
+                Ok(())
+            })
+        }))
+        .unwrap_or_else(|_| Err(format!("{name} failed unexpectedly")));
+        res?;
+        let value = value.ok_or_else(|| format!("{name} read nothing"))?;
+        Ok(Read { key, value })
+    }
+}
+
+/// The read `command` makes with `p`, when the app makes it on a thread of its own before the
+/// command runs: `None` for one it runs as it is (a layer read in place, a web service, a file
+/// the command reads differently).
+pub fn read_ahead(s: &Session, command: &str, p: &Value) -> Option<ReadAhead> {
+    match command {
+        "layer.addData" => super::layer::add_data_ahead(p),
+        "layer.loadAll" => super::layer::load_all_ahead(s, p),
+        _ => None,
+    }
+}
+
+/// `work` as a read for the command to take under `key`.
+pub(crate) fn ahead<T: Send + 'static>(name: &'static str, key: String, work: impl FnOnce() -> T + Send + 'static) -> ReadAhead {
+    ReadAhead { name, key, work: Box::new(move || Box::new(work()) as Box<dyn Any + Send>) }
+}
+
+/// The read placed for `key`, taken: `None` when none was, so the caller reads here.
+pub(crate) fn take<T: 'static>(s: &mut Session, key: &str) -> Option<T> {
+    if s.read_ahead.as_ref().is_none_or(|r| r.key != key) {
+        return None;
+    }
+    let read = s.read_ahead.take()?;
+    read.value.downcast::<T>().ok().map(|v| *v)
+}
+
+/// Whether a read is placed for `key`, without taking it.
+#[cfg(not(target_arch = "wasm32"))]
+pub(crate) fn placed_for(s: &Session, key: &str) -> bool {
+    s.read_ahead.as_ref().is_some_and(|r| r.key == key)
+}

crates/engine/src/lib.rs

@@ -750,6 +750,9 @@ pub struct Session {
     pub zarr_off_frame: bool,
     /// The Zarr steps landed that [`Session::follow_time_landed`] has seen.
     zarr_seen: u64,
+    /// A read the app made on a thread of its own for the command it runs next, which takes it
+    /// instead of reading again ([`cmd::reads`]).
+    pub(crate) read_ahead: Option<cmd::reads::Read>,
     /// Set where a person at a terminal waits on a command (`rustgis-cli run` with stdin and
     /// stderr a terminal), who can stop it with Ctrl-C: tools and rasters take no work cap
     /// ([`rustgis_analysis::control::Control::stoppable`]). A program that calls it (an agent, a
@@ -1003,6 +1006,7 @@ impl Session {
             long_runs: None,
             zarr_off_frame: false,
             zarr_seen: 0,
+            read_ahead: None,
             stoppable: false,
         };
         for text in label_notices {
@@ -1143,6 +1147,12 @@ impl Session {
         self.zarr_off_frame && cmd::zarr::waiting(self)
     }
 
+    /// Drops a read placed for a command ([`cmd::reads::Read::placed`]) that the command didn't
+    /// take: one refused before it read, or that read another way.
+    pub fn drop_read(&mut self) {
+        self.read_ahead = None;
+    }
+
     /// Shows the Zarr steps read off the frame ([`Session::zarr_off_frame`]) once they land: the
     /// app asks each frame. True when one landed, so the map draws again.
     pub fn follow_time_landed(&mut self) -> bool {