From c52b4387f72e500cf0bedf12506fe98553561123 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 16:08:26 -0330 Subject: [PATCH 01/27] feat(raster): cloud-optimised GeoTIFFs are raster layers read in place A COG holds what a GeoTIFF file holds, so it is now a raster layer like one: the same Symbology pane, raster commands, legend, identify and tools, with no COG-only dialogs or menus. The layer keeps an overview of every band (about two million values) for statistics and tools; zoomed in past it, the map reads the part in view from the level the view needs as it draws, styled with the overview's statistics so the image looks the same at every zoom. - Every band is read whatever the PhotometricInterpretation tag says (palette, RGB with extra bands, alpha), and that tag sets the first look: RGB in colour, a palette as a colormap, MinIsBlack in grey on band 1 or as RGB with three bands. GDAL metadata isn't consulted. File rasters follow the same rule. - The stretch statistics come from a level chosen by tile budget and pixel count, so images in tall strips no longer stretch over the type's full range. - Decoded tiles are evicted least recently used, view windows fit the cache, and missing tiles are filled from levels already decoded until they arrive, so blocks no longer flicker between overview levels while zooming. - Dated COG series, data cube files and AlphaEarth embeddings stay tile layers. Projects that saved a plain COG as a tile layer open it as a raster layer, keeping its visibility, transparency, blend, order and look. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/raster/cog.rs | 87 ++++++- crates/data/src/raster/geotiff.rs | 98 ++++++-- crates/data/src/raster/mod.rs | 40 ++- crates/engine/src/cmd/geolibre.rs | 2 +- crates/engine/src/cmd/layer.rs | 50 +++- crates/engine/src/cmd/raster.rs | 2 + crates/engine/src/cmd/stac.rs | 2 +- crates/engine/src/cmd/tiles.rs | 54 ++++ crates/engine/src/lib.rs | 4 +- crates/engine/src/project.rs | 2 + crates/engine/src/s3_saved_tests.rs | 12 +- crates/engine/src/tests.rs | 193 +++++++++++++-- crates/map/src/raster.rs | 142 ++++++++++- crates/render/src/lib.rs | 12 +- crates/render/src/raster.rs | 173 +++++++++++++ crates/render/src/raster_export.rs | 28 ++- crates/tiles/src/cog.rs | 291 +++++++++++++++++----- crates/tiles/src/cog/raster.rs | 371 ++++++++++++++++++++++++++++ crates/ui-egui/src/mapview.rs | 14 +- crates/ui-egui/src/panes.rs | 27 +- crates/ui-egui/tests/smoke.rs | 65 +++-- docs/user-guide/data-formats.md | 38 ++- docs/user-guide/web-data.md | 9 +- 23 files changed, 1549 insertions(+), 167 deletions(-) create mode 100644 crates/tiles/src/cog/raster.rs diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 28c432406..5621df3d7 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -309,12 +309,16 @@ pub fn layout(bytes: &Sparse) -> Result { } fn read_layout(r: Sparse) -> Result { - let mut dec = Decoder::new(super::geotiff::reader(r).map_err(err)?).map_err(err)?.with_limits(limits()); + let reader = super::geotiff::reader_all_bands(r).map_err(err)?; + let (palette, wide_rgb) = (reader.palette, reader.wide_rgb); + let mut dec = Decoder::new(reader).map_err(err)?.with_limits(limits()); let Some(geo) = super::geotiff::cog_geo(&mut dec)? else { return Ok(Parse::NotTiled("a rotated image, or one without coordinates, is read whole".into())); }; let ct = dec.colortype().map_err(err)?; let (bands, color) = match ct { + // A palette image (shown to the decoder as grey) is coloured by its table. + ColorType::Gray(bits) if palette => (1, super::geotiff::palette_table(&mut dec, bits).map_or(ColorHint::Gray, ColorHint::Palette)), ColorType::Gray(_) => (1, ColorHint::Gray), ColorType::GrayA(_) => (2, ColorHint::GrayAlpha), ColorType::RGB(_) => (3, ColorHint::Rgb { alpha: false }), @@ -322,6 +326,11 @@ fn read_layout(r: Sparse) -> Result { (3, ColorHint::Rgb { alpha: false }) } ColorType::RGBA(_) => (4, ColorHint::Rgb { alpha: true }), + // An RGB image with more bands (shown to the decoder as grey): every band, the first three + // red, green and blue. + ColorType::Multiband { num_samples, .. } if wide_rgb && num_samples >= 3 => { + (num_samples as usize, ColorHint::Rgb { alpha: super::geotiff::extra_alpha(&mut dec) }) + } ColorType::Multiband { num_samples, .. } => { let color = super::geotiff::multiband_color(&mut dec, num_samples); (num_samples.max(1) as usize, color) @@ -397,7 +406,7 @@ pub fn decode_tile(header: &Sparse, layout: &Layout, level: usize, index: u32, b let l = layout.levels.get(level).ok_or_else(|| err("no such level"))?; let at = *l.offsets.get(index as usize).ok_or_else(|| err("no such tile"))?; let r = header.clone().with(at, bytes).reader(); - let mut dec = Decoder::new(super::geotiff::reader(r).map_err(err)?).map_err(err)?.with_limits(limits()); + let mut dec = Decoder::new(super::geotiff::reader_all_bands(r).map_err(err)?).map_err(err)?.with_limits(limits()); dec.seek_to_image(l.ifd).map_err(err)?; let values = match super::geotiff::read_chunk(&mut dec, index)? { DecodingResult::U8(v) => v.into_iter().map(f32::from).collect::>(), @@ -469,11 +478,16 @@ pub struct Example { pub int8: bool, /// Stored bottom up: a positive Y pixel size, the first row the southernmost (at 45° N). pub bottom_up: bool, + /// Its PhotometricInterpretation: 1 grey, 2 RGB (the bands past the third extra samples), 3 + /// palette (one band; colour `i` is `(i, 255 − i, 0)`). + pub photometric: u16, + /// The first extra sample (the band past the colour ones) is unassociated alpha. + pub alpha: bool, } impl Default for Example { fn default() -> Self { - Example { size: 1024, tile: 256, bands: 1, planar: false, int8: false, bottom_up: false } + Example { size: 1024, tile: 256, bands: 1, planar: false, int8: false, bottom_up: false, photometric: 1, alpha: false } } } @@ -537,7 +551,7 @@ pub fn example_with(e: &Example) -> Vec { (257, V::L(vec![s])), (258, V::S(vec![8; nb as usize])), (259, V::S(vec![1])), - (262, V::S(vec![1])), + (262, V::S(vec![e.photometric])), (277, V::S(vec![nb as u16])), (284, V::S(vec![if e.planar { 2 } else { 1 }])), (322, V::L(vec![tile])), @@ -545,8 +559,19 @@ pub fn example_with(e: &Example) -> Vec { (324, V::L(vec![0; n])), (325, V::L(vec![chunk_len; n])), ]; - if nb > 1 { - t.push((338, V::S(vec![0; nb as usize - 1]))); + let colours = if e.photometric == 2 { 3 } else { 1 }; + if nb > colours { + let mut extra = vec![0; (nb - colours) as usize]; + if e.alpha + && let Some(first) = extra.first_mut() + { + *first = 2; + } + t.push((338, V::S(extra))); + } + if e.photometric == 3 { + let ramp: Vec = (0..256u16).map(|i| i * 257).collect(); + t.push((320, V::S([ramp.clone(), ramp.iter().rev().copied().collect(), vec![0; 256]].concat()))); } if e.int8 { t.push((339, V::S(vec![2; nb as usize]))); @@ -706,7 +731,7 @@ pub(crate) mod tests { fn bands_stored_apart_signed_bytes_bottom_up() { // As AlphaEarth's embeddings are: planar, Int8 with no-data −128, a positive Y pixel size; // 600 pixels in tiles of 256, so the right and bottom tiles are cut by the image's edge. - let e = Example { size: 600, tile: 256, bands: 4, planar: true, int8: true, bottom_up: true }; + let e = Example { size: 600, tile: 256, bands: 4, planar: true, int8: true, bottom_up: true, ..Example::default() }; let file = example_with(&e); let (s, l) = layout_of(&file); assert!(l.planar && l.bands == 4 && l.data_type == "Int8" && l.chunk_bands() == 1); @@ -763,4 +788,52 @@ pub(crate) mod tests { } let _ = layout(&Sparse::default().with(0, b"II*\0\xff\xff\xff\x7f".to_vec())); } + + /// Pixel (x, y)'s value in every band of an example's full image, read a tile at a time. + fn pixel(file: &[u8], x: u32, y: u32) -> (Box, Vec) { + let (s, l) = layout_of(file); + let lv = &l.levels[0]; + let tile = (y / lv.tile_height) * lv.across() + x / lv.tile_width; + let at = ((y % lv.tile_height) * lv.tile_width + x % lv.tile_width) as usize; + let values = (0..l.bands) + .map(|b| { + let (chunk, i) = l.chunk(lv, tile, b).unwrap(); + let (o, n) = (lv.offsets[chunk as usize] as usize, lv.counts[chunk as usize] as usize); + decode_tile(&s, &l, 0, chunk, file[o..o + n].to_vec()).unwrap()[i][at] + }) + .collect(); + (l, values) + } + + #[test] + fn every_band_whatever_the_photometric_interpretation_says() { + let want = |b: u32| ((300 + 10 * b) % 256) as f32; + let e = Example { size: 300, tile: 128, ..Example::default() }; + // Grey, as remote sensing images mostly are: eight bands (a WorldView multispectral image), + // interleaved or stored apart; a panchromatic one; a pan band with alpha. + for planar in [false, true] { + let (l, v) = pixel(&example_with(&Example { bands: 8, planar, ..e }), 150, 150); + assert_eq!((l.bands, &l.color), (8, &ColorHint::Gray)); + assert_eq!(v, (0..8).map(want).collect::>()); + } + let (l, v) = pixel(&example_with(&Example { bands: 1, ..e }), 150, 150); + assert_eq!((l.bands, &l.color, v), (1, &ColorHint::Gray, vec![want(0)])); + let (l, v) = pixel(&example_with(&Example { bands: 2, alpha: true, ..e }), 150, 150); + assert_eq!((l.bands, &l.color, v), (2, &ColorHint::GrayAlpha, vec![want(0), want(1)])); + // RGB: three bands, with alpha, and with bands past the colour ones (RGB and near-infrared, + // seven bands), which the decoder alone leaves out or refuses: every band, the first three + // red, green and blue. + for (bands, alpha) in [(3, false), (4, true), (4, false), (7, false), (5, true)] { + for planar in [false, true] { + let file = example_with(&Example { bands, alpha, planar, photometric: 2, ..e }); + let (l, v) = pixel(&file, 150, 150); + assert_eq!((l.bands, &l.color), (usize::from(bands), &ColorHint::Rgb { alpha }), "{bands} bands, alpha {alpha}"); + assert_eq!(v, (0..u32::from(bands)).map(want).collect::>(), "{bands} bands, alpha {alpha}, planar {planar}"); + } + } + // A palette image: its values, and its colour table. + let (l, v) = pixel(&example_with(&Example { photometric: 3, ..e }), 150, 150); + let ColorHint::Palette(table) = &l.color else { panic!("{:?}", l.color) }; + assert_eq!((table.len(), table[44], v), (256, [44, 211, 0, 255], vec![want(0)])); + } } diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index e281824b0..bb35e9cd7 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -50,6 +50,21 @@ pub(super) fn multiband_color(dec: &mut Decoder, n: u16) -> C if alpha { ColorHint::GrayAlpha } else { ColorHint::Gray } } +/// Whether the first sample past the colour ones is declared alpha (extra sample 1, associated, or +/// 2, unassociated). +pub(super) fn extra_alpha(dec: &mut Decoder) -> bool { + dec.find_tag_unsigned_vec::(Tag::ExtraSamples).ok().flatten().is_some_and(|v| matches!(v.first(), Some(1 | 2))) +} + +/// A palette image's colours, from its ColorMap tag (16-bit red, then green, then blue values); +/// `None` when the table is missing or short. +pub(super) fn palette_table(dec: &mut Decoder, bits: u8) -> Option> { + let map: Vec = dec.find_tag(Tag::ColorMap).ok().flatten().and_then(|v| v.into_u16_vec().ok()).unwrap_or_default(); + let n = 1usize << bits.min(16); + let c = |i: usize| (map.get(i).copied().unwrap_or(0) / 257) as u8; + (map.len() >= 3 * n).then(|| (0..n).map(|i| [c(i), c(n + i), c(2 * n + i), 255]).collect()) +} + fn geo_tags(dec: &mut Decoder) -> Result { let doubles = |dec: &mut Decoder, tag: u16| -> Option> { dec.find_tag(Tag::Unknown(tag)).ok().flatten()?.into_f64_vec().ok() }; @@ -562,10 +577,19 @@ fn samples(dec: &mut Decoder) -> usize { /// A reader that shows palette images as grey ones: the `tiff` crate decodes no palette images, but /// decodes their colour indices as grey samples, and the colour table is read from the ColorMap tag /// as it is. The only change is the PhotometricInterpretation value (3 → 1) in each image's IFD. +/// +/// Made by [`reader_all_bands`], it also shows RGB images with samples past the colour ones (RGB +/// and near-infrared, or more bands, not declared alpha) as grey ones (2 → 1): the crate decodes +/// only an RGB image's colour samples, and a fifth or later one not at all, but decodes every +/// sample of a grey image. pub(crate) struct Unpaletted { inner: R, - /// Where each palette image's PhotometricInterpretation value is, and what to show there. + /// Where each patched image's PhotometricInterpretation value is, and what to show there. patches: Vec<(u64, [u8; 2])>, + /// Palette images were patched. + pub(crate) palette: bool, + /// RGB images with more samples were patched: their first three samples are red, green, blue. + pub(crate) wide_rgb: bool, pos: u64, } @@ -573,11 +597,20 @@ pub(crate) struct Unpaletted { const MAX_IFDS: usize = 64; const MAX_ENTRIES: u64 = 4096; -/// Where the palette images' PhotometricInterpretation values are, walking the IFD chain -/// (classic and BigTIFF). A file it can't walk has none: the decoder reports what's wrong. -fn palette_patches(r: &mut R) -> Vec<(u64, [u8; 2])> { +/// What [`photometric_patches`] found to patch. +#[derive(Default)] +struct Patches { + at: Vec<(u64, [u8; 2])>, + palette: bool, + wide_rgb: bool, +} + +/// Where the palette images' PhotometricInterpretation values are (and, with `wide_rgb`, those of +/// RGB images with samples past the colour ones), walking the IFD chain (classic and BigTIFF). A +/// file it can't walk has none: the decoder reports what's wrong. +fn photometric_patches(r: &mut R, wide_rgb: bool) -> Patches { use std::io::SeekFrom; - let mut patches = Vec::new(); + let mut patches = Patches::default(); let mut walk = || -> Option<()> { let mut head = [0u8; 16]; r.seek(SeekFrom::Start(0)).ok()?; @@ -624,16 +657,35 @@ fn palette_patches(r: &mut R) -> Vec<(u64, [u8; 2])> { } let mut entries = vec![0u8; usize::try_from(n * entry_len).ok()?]; r.read_exact(&mut entries).ok()?; + let value_at = if big { 12 } else { 8 }; + let grey = if little { [1, 0] } else { [0, 1] }; + // PhotometricInterpretation (262): where its value is, and the value; SamplesPerPixel + // (277); the first ExtraSamples (338) value, when it is inline. Each a SHORT. + let (mut photometric, mut samples, mut extra) = (None, 1u16, None); for (i, e) in entries.chunks(entry_len as usize).enumerate() { - // PhotometricInterpretation (262), a SHORT, its value inline. - if u16_(e)? == 262 && u16_(e.get(2..)?)? == 3 { - let value_at = if big { 12 } else { 8 }; - if u16_(e.get(value_at..)?)? == 3 { - let at = ifd + count_len + i as u64 * entry_len + value_at as u64; - patches.push((at, if little { [1, 0] } else { [0, 1] })); + let short = u16_(e.get(2..)?)? == 3; + match u16_(e)? { + 262 if short => { + photometric = Some((ifd + count_len + i as u64 * entry_len + value_at as u64, u16_(e.get(value_at..)?)?)) } + 277 if short => samples = u16_(e.get(value_at..)?)?, + 338 if short => extra = u16_(e.get(value_at..)?), + _ => {} } } + match photometric { + Some((at, 3)) => { + patches.at.push((at, grey)); + patches.palette = true; + } + // An RGB image whose fourth sample isn't alpha, or with five or more: RGBI + // orthophotos, multispectral images. + Some((at, 2)) if wide_rgb && (samples > 4 || (samples == 4 && !matches!(extra, Some(1 | 2)))) => { + patches.at.push((at, grey)); + patches.wide_rgb = true; + } + _ => {} + } let mut next = [0u8; 8]; r.read_exact(&mut next[..next_len]).ok()?; ifd = if big { u64_(&next)? } else { u64::from(u32_(&next)?) }; @@ -654,14 +706,20 @@ pub(crate) type Reader<'a> = Unpaletted>; /// A reader for the decoder. pub(crate) fn reader<'a>(r: impl Read + Seek + 'a) -> std::io::Result> { - Unpaletted::new(Box::new(r)) + Unpaletted::new(Box::new(r), false) +} + +/// A reader for the decoder that decodes every band of an RGB image with more (see +/// [`Unpaletted`]). +pub(crate) fn reader_all_bands<'a>(r: impl Read + Seek + 'a) -> std::io::Result> { + Unpaletted::new(Box::new(r), true) } impl Unpaletted { - fn new(mut inner: R) -> std::io::Result { - let patches = palette_patches(&mut inner); + fn new(mut inner: R, wide_rgb: bool) -> std::io::Result { + let p = photometric_patches(&mut inner, wide_rgb); inner.seek(std::io::SeekFrom::Start(0))?; - Ok(Unpaletted { inner, patches, pos: 0 }) + Ok(Unpaletted { inner, patches: p.at, palette: p.palette, wide_rgb: p.wide_rgb, pos: 0 }) } } @@ -991,7 +1049,7 @@ pub(super) fn read_limited_with_nodata( open: Option>, sidecar_nodata: Option, ) -> Result { - let palette = !r.patches.is_empty(); + let palette = r.palette; let limits = { let mut l = Limits::default(); l.decoding_buffer_size = MAX_SAMPLES.saturating_mul(4); @@ -1092,13 +1150,7 @@ pub(super) fn read_limited_with_nodata( // conversion), so it is turned into RGB below (issue #692); other YCbCr data isn't read. ColorType::YCbCr(_) if ycbcr => (3, ColorHint::Rgb { alpha: false }), ColorType::RGBA(_) => (4, ColorHint::Rgb { alpha: true }), - ColorType::Palette(bits) => { - let map: Vec = dec.find_tag(Tag::ColorMap).ok().flatten().and_then(|v| v.into_u16_vec().ok()).unwrap_or_default(); - let n = 1usize << bits.min(16); - let c = |i: usize| (map.get(i).copied().unwrap_or(0) / 257) as u8; - let table = if map.len() >= 3 * n { (0..n).map(|i| [c(i), c(n + i), c(2 * n + i), 255]).collect() } else { Vec::new() }; - (1, if table.is_empty() { ColorHint::Gray } else { ColorHint::Palette(table) }) - } + ColorType::Palette(bits) => (1, palette_table(&mut dec, bits).map_or(ColorHint::Gray, ColorHint::Palette)), ColorType::Multiband { num_samples, .. } => (num_samples.max(1) as usize, multiband_color(&mut dec, num_samples)), other => return Err(err(format!("{other:?} images aren't supported"))), }; diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index 442c09290..c2e5d3887 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -417,6 +417,39 @@ impl Raster { nodata: Option, color: ColorHint, data_type: impl Into, + ) -> Result { + Raster::build(name, width, height, bands, transform, crs, nodata, color, data_type, true) + } + + /// [`Raster::new`] without statistics (each band's are empty): a part of a larger raster, + /// drawn with the whole's (a cloud-optimised GeoTIFF's view, read in place). + #[allow(clippy::too_many_arguments)] + pub fn unmeasured( + name: impl Into, + width: u32, + height: u32, + bands: Vec>, + transform: GeoTransform, + crs: Crs, + nodata: Option, + color: ColorHint, + data_type: impl Into, + ) -> Result { + Raster::build(name, width, height, bands, transform, crs, nodata, color, data_type, false) + } + + #[allow(clippy::too_many_arguments)] + fn build( + name: impl Into, + width: u32, + height: u32, + bands: Vec>, + transform: GeoTransform, + crs: Crs, + nodata: Option, + color: ColorHint, + data_type: impl Into, + measure: bool, ) -> Result { let n = (width as usize).checked_mul(height as usize).ok_or_else(|| DataError::Format("raster too large".into()))?; if n == 0 || bands.is_empty() { @@ -434,7 +467,12 @@ impl Raster { let bands = bands .into_iter() .enumerate() - .map(|(i, data)| Band { name: format!("Band {}", i + 1), stats: BandStats::compute(&data, valid), data, classes: Vec::new() }) + .map(|(i, data)| Band { + name: format!("Band {}", i + 1), + stats: if measure { BandStats::compute(&data, valid) } else { BandStats::default() }, + data, + classes: Vec::new(), + }) .collect(); let mut r = Raster { name: name.into(), diff --git a/crates/engine/src/cmd/geolibre.rs b/crates/engine/src/cmd/geolibre.rs index 1b4f3a990..721e1a00e 100644 --- a/crates/engine/src/cmd/geolibre.rs +++ b/crates/engine/src/cmd/geolibre.rs @@ -367,7 +367,7 @@ fn add_layer(s: &mut Session, r: &Value, dir: Option<&str>, report: &mut Report, return Ok(Added::Feature(id)); } if kind == "cog" && remote { - super::layer::add_cog(s, &q, &address)? + super::layer::add_cog_with(s, &q, &address, false)? } else if kind == "zarr" { if let Some(v) = src.get("variable") { q["variable"] = v.clone(); diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index b53f6d245..604f94053 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -40,7 +40,7 @@ pub fn commands() -> Vec { .params(r#"{"layer"?: name|id, "crs": "EPSG:… or WKT; reinterpret native coordinates, without rewriting the source file"}"#), C::new("layer.loadAll", "Load All Features", "Layers", load_all).when(need_streamed).params(r#"{"layer"?: name|id}"#), C::new("layer.addUrl", "Add Data From URL", "Home › Add › Add Data", add_url).params( - r#"{"url": "https://…/data.geojson|.csv|.parquet|.fgb|.gpkg|.zip|.kml|.kmz|.gpx|.tif|.jp2 (JPEG 2000: desktop, through GDAL) (the format from the address, else the contents) | a .pmtiles archive", "name"?: text, "layer"?: "one layer of a zip, KML or GPX", "zoom"?: "bool: true zooms to what was added; false keeps the view of a GeoParquet or FlatGeobuf read in place (zoomed to only when the map was empty)", "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "whole"?: "bool: download a cloud-optimised GeoTIFF whole (as a raster layer) instead of reading it in place (an address without a file name is read in place when its first bytes are a cloud-optimised GeoTIFF)", "cog"?: "bool: the address is a cloud-optimised GeoTIFF whatever its name says (the app's probe found it so)", "start"?, "end"?, "unit"?, "interval"?, "dates"?: "a GeoTIFF address with date placeholders ({date:YYYY}, {YYYY}{MM}{DD}), one file per date: the dates it spans and steps through (as time.set), turning the time slider on"}"#, + r#"{"url": "https://…/data.geojson|.csv|.parquet|.fgb|.gpkg|.zip|.kml|.kmz|.gpx|.tif|.jp2 (JPEG 2000: desktop, through GDAL) (the format from the address, else the contents) | a .pmtiles archive", "name"?: text, "layer"?: "one layer of a zip, KML or GPX", "zoom"?: "bool: true zooms to what was added; false keeps the view of a GeoParquet or FlatGeobuf read in place (zoomed to only when the map was empty)", "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "whole"?: "bool: download a cloud-optimised GeoTIFF whole instead of reading it in place (either way a raster layer; read in place it holds an overview, and the part in view is read at the resolution the map needs as it draws; an address without a file name is read in place when its first bytes are a cloud-optimised GeoTIFF)", "cog"?: "bool: the address is a cloud-optimised GeoTIFF whatever its name says (the app's probe found it so)", "cube"?: "bool: a data cube's file: a tile layer that follows the time slider (stac.addCube adds them)", "start"?, "end"?, "unit"?, "interval"?, "dates"?: "a GeoTIFF address with date placeholders ({date:YYYY}, {YYYY}{MM}{DD}), one file per date: the dates it spans and steps through (as time.set), turning the time slider on"}"#, ), C::new("layer.addDownload", "Add Downloaded Data", "Home › Add", add_download) .params(r#"{"id": a download the app finished (Home › Add › Add Data › From URL), "url": its address, "name"?: text, "layer"?: text, "zoom"?: bool, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "service"?: bool (a service's features: their axes and paging are checked), "bounds"?: [west, south, east, north] (with service: the layer's longitude/latitude bounds from its capabilities, which its axes are checked against; without bounds they are left as they come; ignored without service), "coverage"?: bool (a WCS coverage: a GeoTIFF, else the service's error), "geolibreProject"?: bool (internal: downloaded project JSON), "geolibreLayer"?: {"token", "record"} (internal: reserved imported layer and its saved style), "geolibreZarr"?: bool (internal: an imported Zarr slice as GeoTIFF)}"#), @@ -2956,8 +2956,24 @@ pub fn is_unnamed_url(url: &str) -> bool { /// Adds a cloud-optimised GeoTIFF by its address as a layer read in place (its tiles by range, /// as the view needs them), framed when its header is in. pub(crate) fn add_cog(s: &mut Session, p: &Value, url: &str) -> CmdResult { + add_cog_with(s, p, url, true) +} + +/// [`add_cog`], a GeoTIFF that can't be read now added all the same (unless `strict`), showing +/// why: a project being imported keeps its layers. +pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) -> CmdResult { let stem = cog_name(url); let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).unwrap_or(&stem); + // A GeoTIFF is a raster layer like a file's, read in place; a dated series, a data cube's file + // (which follow the time slider) and AlphaEarth's quantised embeddings (which raster styles + // don't dequantise) are tile layers. + let (file, look) = crate::render::rustgis_tiles::cog::split(url); + let dated = rustgis_data::time::is_template(file) + || ["start", "end", "unit", "interval", "dates"].iter().any(|k| p.get(k).is_some()) + || b(p, "cube") == Some(true); + if !dated && !look.aef { + return add_in_place(s, name, url, strict); + } let mut q = json!({"url": url, "kind": "cog", "name": name, "top": true}); // A dated address's span and steps (see map.addTiles). for k in ["start", "end", "unit", "interval", "dates"] { @@ -2988,6 +3004,38 @@ pub(crate) fn add_cog(s: &mut Session, p: &Value, url: &str) -> CmdResult { Ok(r) } +/// Adds a cloud-optimised GeoTIFF as a raster layer read in place: its overview held (read now, +/// natively; in the browser as it lands), the part in view read as the map draws. A look at the +/// address's end (`#bands=…&range=…`) is its style. +fn add_in_place(s: &mut Session, name: &str, url: &str, strict: bool) -> CmdResult { + use crate::render::rustgis_tiles::cog; + let (file, look) = cog::split(url); + #[cfg(not(target_arch = "wasm32"))] + let mut layer = if strict { + rustgis_map::raster::RasterLayer::in_place(cog::raster::overview_blocking(file)?, file) + } else { + rustgis_map::raster::RasterLayer::opening_in_place(file) + }; + #[cfg(target_arch = "wasm32")] + let mut layer = { + let _ = strict; + rustgis_map::raster::RasterLayer::opening_in_place(file) + }; + if let Some(d) = layer.data.clone() { + layer.style = super::tiles::look_style(&look, &d).unwrap_or_else(|| layer.style.clone()); + } + let mut r = super::raster::add_raster(s, name, layer)?; + if let Ok(Some(info)) = cog::info(file) { + let [w, so, e, n] = info.bounds; + s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)); + r["bounds"] = json!(info.bounds); + } + r["inPlace"] = json!(true); + r["kind"] = json!("raster"); + r["name"] = json!(s.map().layer(r["layer"].as_u64().unwrap_or(0) as u32).map(|l| l.name.clone())); + Ok(r) +} + /// A GeoTIFF's name from its address: the file name without its extension, or for a dated /// address (`…/landsat/{date:YYYY}.tif`) what is left of it once the dates are taken out, else its /// folder's name. diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index a6fb4abfc..3a50a58e1 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -348,6 +348,8 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes } let (w, h, n, overview) = layer.data.as_ref().map(|d| (d.width, d.height, d.band_count(), d.overview)).unwrap_or_default(); let rounded = layer.data.as_deref().and_then(rounded_count); + // Read in place, the map shows the full image as it draws: the overview is only what's held. + let overview = overview.filter(|_| !layer.in_place); let empty_before = s.map().layers.is_empty(); let id = s.map_mut().add_raster(unique.clone(), layer); s.active_layer = Some(id); diff --git a/crates/engine/src/cmd/stac.rs b/crates/engine/src/cmd/stac.rs index 74b8cddda..ae3a39a92 100644 --- a/crates/engine/src/cmd/stac.rs +++ b/crates/engine/src/cmd/stac.rs @@ -958,7 +958,7 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { dates.extend(urls.iter().map(|(d, _)| *d)); let Some((_, first)) = urls.first() else { continue }; let name = if many { format!("{base} {tile}") } else { base.clone() }; - let r = match super::layer::add_url(s, &json!({"url": first, "name": name})) { + let r = match super::layer::add_url(s, &json!({"url": first, "name": name, "cube": true})) { Ok(r) => r, Err(e) => { undo(s); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index f4c48b013..4ab2b27c0 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -532,6 +532,60 @@ fn blend(p: &Value) -> Result, String> { ps(p, "blend").map(|m| Blend::from_code(m).ok_or_else(|| format!("unknown blend mode '{m}'"))).transpose() } +/// Turns a project's COG tile layers (saved before GeoTIFFs read in place were raster layers) into +/// raster layers read in place, each where it was drawn, with its name, visibility, opacity, blend +/// and look. A dated series, a data cube's files and AlphaEarth's embeddings stay tile layers. +pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { + use rustgis_map::raster::RasterLayer; + let mut i = 0; + while let Some(t) = m.tiles.get(i) { + let (file, look) = cog::split(&t.url); + if t.kind != TileKind::Cog || t.time_url.is_some() || !t.time_urls.is_empty() || look.aef || rustgis_data::time::is_template(file) { + i += 1; + continue; + } + let t = m.tiles.remove(i); + let (file, look) = cog::split(&t.url); + // Read now natively; in the browser its own look once it lands. + let mut rl = RasterLayer::opening_in_place(file); + if let Some(style) = rl.data.clone().and_then(|d| look_style(&look, &d)) { + rl.style = style; + } + let id = m.add_raster(t.name.clone(), rl); + if let Some(l) = m.layer_mut(id) { + l.visible = t.visible; + l.transparency = ((1.0 - t.opacity.clamp(0.0, 1.0)) * 100.0).round(); + l.blend = t.blend; + } + // Just over the layer it was drawn over, in its group; under them all without one. + let group = t.over.and_then(|o| m.group_of(o).map(|g| g.id)); + if m.place_layer(id, group, t.over).is_err() { + let _ = m.place_layer(id, None, None); + } + } +} + +/// The raster style a GeoTIFF's look says (see [`Look`]): its bands, one grey or three as red, +/// green and blue, and its fixed range; `None` for the file's own look. A COG tile layer becomes +/// a raster layer read in place with it. +pub(crate) fn look_style(look: &Look, r: &rustgis_data::raster::Raster) -> Option { + use rustgis_map::raster::{RasterRenderer, RasterStyle, Stretch}; + if look.bands.is_empty() && look.range.is_none() { + return None; + } + let mut style = RasterStyle::default_for(r); + let n = r.band_count(); + match look.bands[..] { + [b] if b < n => style.renderer = RasterRenderer::Stretch { band: b, ramp: rustgis_map::Ramp::gray(), invert: false }, + [red, green, blue] if red < n && green < n && blue < n => style.renderer = RasterRenderer::Rgb { red, green, blue }, + _ => {} + } + if let Some((lo, hi)) = look.range { + style.stretch = Stretch::custom(f64::from(lo), f64::from(hi)); + } + Some(style) +} + /// A change of a COG layer's look a command asks for (`bands`, `range`, `dequantize`), applied to /// the look it has; `None` when it asks for none. #[allow(clippy::type_complexity)] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d0dab5d78..f83189cae 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -826,8 +826,10 @@ impl Session { #[cfg(feature = "geolibre")] rustgis_geolibre::register(); rustgis_plugins::tools::register(); - // FlatGeobuf and GeoParquet by their web address are read through the tile fetchers. + // FlatGeobuf and GeoParquet by their web address are read through the tile fetchers, and + // cloud-optimised GeoTIFFs' overviews for raster layers read in place. render::rustgis_tiles::remote::install(); + map::raster::set_in_place_reader(render::rustgis_tiles::cog::raster::overview); rustgis_data::diagnostics::set_clock(|| { web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) }); diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index aa574f29f..162fb3bb0 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -667,6 +667,8 @@ impl Project { // Tile layers with an address that can't be fetched (a hand edit) are dropped. m.tiles.retain_mut(|t| t.repair().map_err(|e| log::warn!("{}: {e}", m.name)).is_ok()); m.tiles.truncate(rustgis_map::tiles::MAX_TILE_LAYERS); + // GeoTIFFs read in place are raster layers (a project from before saved tile layers). + crate::cmd::tiles::cog_tiles_to_rasters(m); #[cfg(not(target_arch = "wasm32"))] for t in &mut m.tiles { if t.error.is_none() diff --git a/crates/engine/src/s3_saved_tests.rs b/crates/engine/src/s3_saved_tests.rs index 6111c9ee6..aff2c145e 100644 --- a/crates/engine/src/s3_saved_tests.rs +++ b/crates/engine/src/s3_saved_tests.rs @@ -688,15 +688,17 @@ fn s3_saved_projects_hold_no_keys_and_reload_through_their_one_connection() { assert!(read_view(&mut s, id) > 0, "{:?}", s.map().layer(id).unwrap().source.as_ref().unwrap().error); let layer = s.map().layers.iter().find(|l| l.name == "Grid").unwrap(); assert!(layer.source.as_ref().unwrap().path.as_deref().is_some_and(|x| x.contains(s.s3_binding("c1").unwrap()))); - // The COG's tile is drawn from its pixels. - let tiles = s.map().tiles.clone(); - let scene = tiles.iter().find(|t| t.name == "Scene").unwrap(); - assert!(scene.url.contains(s.s3_binding("c1").unwrap())); + // The COG, a raster layer read in place, holds its overview, read through the connection; + // its tiles are drawn from its pixels. + let scene = s.map().layers.iter().find(|l| l.name == "Scene").and_then(|l| l.raster.clone()).unwrap(); + let address = scene.path.clone().unwrap(); + assert!(scene.in_place && address.contains(s.s3_binding("c1").unwrap()), "{address}"); + assert!(scene.data.is_some(), "{:?}", scene.error); let (z, lon, lat) = (9u32, 10.5f64, 45.5f64); let n = f64::from(1u32 << z); let x = ((lon + 180.0) / 360.0 * n) as u32; let y = ((1.0 - lat.to_radians().tan().asinh() / std::f64::consts::PI) / 2.0 * n) as u32; - let tile = format!("cog:{z}/{x}/{y}@{}", scene.url); + let tile = format!("cog:{z}/{x}/{y}@{address}"); let mut got = None; for _ in 0..300 { match cache::get(&tile, cache::Want::Image) { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 7e434930e..b8576c575 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -12910,17 +12910,43 @@ fn cloud_optimised_geotiffs_are_read_in_place() { let mut s = Session::new(Project::default()); let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif")})).unwrap(); assert_eq!(r["inPlace"], true, "{r}"); - let t = s.map().tiles.last().unwrap().clone(); - assert_eq!((t.kind, t.name.as_str()), (rustgis_map::TileKind::Cog, "scene")); + // A raster layer like a file's: it holds an overview (1024 pixels a side of the 2048), whose + // statistics its symbology reads. + let l = s.map().layers.iter().find(|l| l.name == "scene").unwrap().clone(); + let rl = l.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.path.as_deref() == Some(&*format!("{base}/scene.tif"))); + let d = rl.data.as_ref().unwrap(); + assert_eq!((d.width, d.overview), (1024, Some((2048, 2048)))); + assert!(d.bands[0].stats.count > 0 && d.bands[0].stats.max > 200.0); // Framed on the file: 10–11° E, 45–46° N. let b = r["bounds"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect::>(); assert!((b[0] - 10.0).abs() < 1e-6 && (b[3] - 46.0).abs() < 1e-6, "{b:?}"); - // A map tile inside it (zoom 9) is drawn from a few of its tiles, not the whole file. + // Drawn zoomed in past the overview, the part in view is read at full resolution: a few of + // its tiles, not the whole file. Pixel (x, y) of the full image is (x + y) % 256: neighbouring + // screen pixels differ by one, finer than the overview's steps of two. + let before = sent.load(std::sync::atomic::Ordering::Relaxed); + let c = rustgis_geom::lonlat_to_merc(rustgis_geom::Coord::new(10.5, 45.5)); + s.map_mut().camera.center = c; + // A metre on the ground is about 1.43 Web Mercator units here: a third of a full pixel's. + s.map_mut().camera.res = 25.0; + s.view_size = (64.0, 64.0); + let png = std::env::temp_dir().join(format!("rustgis-cog-in-place-{}.png", std::process::id())); + s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap(); + let img = image::open(&png).unwrap().to_rgba8(); + let _ = std::fs::remove_file(&png); + let row: Vec = (0..64).map(|x| img.get_pixel(x, 32)[0]).collect(); + assert!(row.iter().any(|v| *v > 0), "drawn: {row:?}"); + assert!(row.windows(2).any(|w| w[1].abs_diff(w[0]) == 1), "full resolution (steps of one), not the overview's (of two): {row:?}"); + let window = sent.load(std::sync::atomic::Ordering::Relaxed) - before; + assert!(window > 0 && window < total / 8, "{window} of {total} bytes read for the view"); + // A map tile inside it (zoom 9) is drawn from a few of its tiles, not the whole file (a dated + // series of GeoTIFFs is drawn so, as a tile layer). let (z, lon, lat) = (9u32, 10.5f64, 45.5f64); let n = f64::from(1u32 << z); let x = ((lon + 180.0) / 360.0 * n) as u32; let y = ((1.0 - lat.to_radians().tan().asinh() / std::f64::consts::PI) / 2.0 * n) as u32; let tile = format!("cog:{z}/{x}/{y}@{base}/scene.tif"); + let before = sent.load(std::sync::atomic::Ordering::Relaxed); let mut got = None; for _ in 0..200 { match cache::get(&tile, Want::Image) { @@ -12936,13 +12962,13 @@ fn cloud_optimised_geotiffs_are_read_in_place() { let Payload::Image(pm) = &*p else { panic!("not a picture") }; assert_eq!((pm.width(), pm.height()), (256, 256)); assert!(pm.pixels().iter().any(|px| px.alpha() == 255)); - let read = sent.load(std::sync::atomic::Ordering::Relaxed); + let read = sent.load(std::sync::atomic::Ordering::Relaxed) - before; assert!(read < total / 4, "{read} of {total} bytes read"); // A tile outside it is empty. assert!(matches!(cache::get(&format!("cog:9/0/0@{base}/scene.tif"), Want::Image), State::Pending | State::Ready(_))); // The project keeps the address; asked to, the file is downloaded whole instead. let text = serde_json::to_string(&s.project).unwrap(); - assert!(text.contains("\"kind\":\"cog\"") && text.contains("scene.tif")); + assert!(text.contains("\"in_place\":true") && text.contains("scene.tif")); let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif"), "whole": true, "name": "Whole"})).unwrap(); assert!(r.get("inPlace").is_none(), "{r}"); let id = r["layer"].as_u64().unwrap() as u32; @@ -13007,8 +13033,9 @@ fn unnamed_cloud_optimised_geotiffs_are_read_in_place() { let mut s = Session::new(Project::default()); let r = s.run("layer.addUrl", &json!({"url": url, "name": "Ortho"})).unwrap(); assert_eq!(r["inPlace"], true, "{r}"); - let t = s.map().tiles.last().unwrap().clone(); - assert_eq!((t.kind, t.name.as_str(), t.url.as_str()), (rustgis_map::TileKind::Cog, "Ortho", url.as_str())); + let l = s.map().layers.iter().find(|l| l.name == "Ortho").unwrap(); + let rl = l.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.path.as_deref() == Some(url.as_str()), "{:?}", rl.path); let read = sent.load(std::sync::atomic::Ordering::Relaxed); assert!(read < total / 4, "{read} of {total} bytes read: only the header, not the whole file"); let r = s.run("layer.addUrl", &json!({"url": url, "name": "Probed", "cog": true})).unwrap(); @@ -13028,6 +13055,140 @@ fn unnamed_cloud_optimised_geotiffs_are_read_in_place() { assert!(e.contains("not a data format"), "{e}"); } +/// A cloud-optimised GeoTIFF read in place is a raster layer like a file's: its first look is what +/// its PhotometricInterpretation says (RGB in colour, a palette by its table, grey (MinIsBlack, as +/// most remote sensing images are) as its first band, but three grey bands as red, green and +/// blue), every band there for the raster commands and the Symbology pane to combine and stretch. +#[test] +fn cog_layers_are_rasters_styled_as_their_photometric_interpretation_says() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{RasterRenderer, Stretch}; + let e = Example { size: 512, tile: 256, ..Example::default() }; + let rgb = RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }; + let cases: [(&str, Example, usize, Option); 7] = [ + ("Pan", Example { bands: 1, ..e }, 1, None), + ("PanAlpha", Example { bands: 2, alpha: true, ..e }, 2, None), + ("Grey3", Example { bands: 3, ..e }, 3, Some(rgb.clone())), + ("Multispectral", Example { bands: 8, ..e }, 8, None), + ("Rgb", Example { bands: 3, photometric: 2, ..e }, 3, Some(rgb.clone())), + ("Rgbn", Example { bands: 4, photometric: 2, ..e }, 4, Some(rgb.clone())), + ("RgbaNir", Example { bands: 5, photometric: 2, alpha: true, planar: true, ..e }, 5, Some(rgb.clone())), + ]; + let mut s = Session::new(Project::default()); + for (name, ex, bands, renderer) in cases { + let (base, _) = serve_ranges(example_with(&ex)); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/{name}.tif"), "name": name})).unwrap(); + assert_eq!(r["inPlace"], true, "{name}: {r}"); + let id = r["layer"].as_u64().unwrap() as u32; + let rl = s.map().layer(id).and_then(|l| l.raster.clone()).unwrap(); + assert!(rl.in_place, "{name}"); + assert_eq!(rl.data.as_ref().map(|d| d.band_count()), Some(bands), "{name}"); + match renderer { + Some(want) => assert_eq!(rl.style.renderer, want, "{name}"), + None => assert!(matches!(rl.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{name}: {:?}", rl.style.renderer), + } + // The raster commands combine and stretch its bands, as a file's. + if bands >= 3 { + let last = bands as u64; + s.run("raster.bands", &json!({"layer": name, "red": last, "green": 2, "blue": 1})).unwrap(); + let st = s.map().layer(id).and_then(|l| l.raster.as_ref()).map(|r| r.style.clone()).unwrap(); + assert_eq!(st.renderer, RasterRenderer::Rgb { red: bands - 1, green: 1, blue: 0 }, "{name}"); + } + s.run("raster.stretch", &json!({"layer": name, "type": "standardDeviation", "n": 2})).unwrap(); + let st = s.map().layer(id).and_then(|l| l.raster.as_ref()).map(|r| r.style.clone()).unwrap(); + assert_eq!(st.stretch, Stretch::StandardDeviation { n: 2.0 }, "{name}"); + } + assert!(s.map().tiles.is_empty(), "no tile layers"); + // A palette image, by its table. + let (base, _) = serve_ranges(example_with(&Example { photometric: 3, ..e })); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/classes.tif")})).unwrap(); + let rl = s.map().layer(r["layer"].as_u64().unwrap() as u32).and_then(|l| l.raster.clone()).unwrap(); + assert_eq!(rl.style.renderer, RasterRenderer::Colormap); + // Saved and opened again, it is read in place again, its style kept. + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + let ms = t.map().layers.iter().find(|l| l.name == "Multispectral").and_then(|l| l.raster.clone()).unwrap(); + assert!(ms.in_place && ms.data.is_some() && ms.style.stretch == Stretch::StandardDeviation { n: 2.0 }); +} + +/// A view of a GeoTIFF read in place zoomed in past its overview shows the level zoomed from while +/// its own tiles are on their way: no holes, no blocks of the overview, then its own pixels. +#[test] +fn cog_windows_stand_in_the_level_kept_until_their_tiles_arrive() { + use crate::render::rustgis_tiles::cog::raster::window; + use std::sync::atomic::{AtomicBool, Ordering}; + let gate = std::sync::Arc::new(AtomicBool::new(true)); + let (base, _) = serve_gated(rustgis_data::raster::cog::example(2048, 256), true, Some(gate.clone())); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + // Its overview: level 1 (1024 pixels a side), whose tiles are kept. + s.run("layer.addUrl", &json!({"url": url})).unwrap(); + // Level 0's tiles held back: a window of it is whole, from level 1, and still to come. + gate.store(false, Ordering::Relaxed); + let (cols, rows) = ((1000, 1100), (900, 960)); + let gray = rustgis_data::raster::ColorHint::Gray; + let w = window(&url, 0, cols, rows, &[0], gray.clone()).unwrap(); + let part = w.raster.unwrap(); + assert!(w.pending); + let v = |r: &rustgis_data::raster::Raster, x: u32, y: u32| r.bands[0].data[((y - rows.0) * (cols.1 - cols.0) + (x - cols.0)) as usize]; + assert!(part.bands[0].data.iter().all(|v| v.is_finite()), "no holes"); + // Level 1's pixel under (1001, 901) is (500, 450): full pixel (1000, 900), (1000 + 900) % 256. + assert_eq!(v(&part, 1001, 901), ((1000 + 900) % 256) as f32); + // Its own tiles arrive: its own pixels, (x + y) % 256. + gate.store(true, Ordering::Relaxed); + let mut done = None; + for _ in 0..200 { + let w = window(&url, 0, cols, rows, &[0], gray.clone()).unwrap(); + if !w.pending { + done = w.raster; + break; + } + crate::render::rustgis_tiles::cache::wait(std::time::Duration::from_millis(50)); + } + let part = done.expect("its tiles never came"); + assert_eq!(v(&part, 1001, 901), ((1001 + 901) % 256) as f32); +} + +/// A project saved when cloud-optimised GeoTIFFs read in place were tile layers opens them as +/// raster layers read in place, each where it was drawn, with its name, visibility, opacity and +/// look; a dated series stays a tile layer. +#[test] +fn cog_tile_layers_of_older_projects_open_as_rasters() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{RasterRenderer, Stretch}; + let (base, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, bands: 8, ..Example::default() })); + let mut s = Session::new(Project::default()); + s.run("layer.addData", &json!({"path": "points.geojson", "text": "{\"type\":\"FeatureCollection\",\"features\":[]}"})).ok(); + let mut text: Value = serde_json::to_value(&s.project).unwrap(); + let tile = |name: &str, url: String, opacity: f64, visible: bool| { + let mut t = serde_json::to_value(rustgis_map::TileLayer::new(name, rustgis_map::TileKind::Cog, url)).unwrap(); + t["opacity"] = json!(opacity); + t["visible"] = json!(visible); + t + }; + let tiles = json!([ + tile("Scene", format!("{base}/scene.tif#bands=5,3,2&range=10,200"), 0.5, false), + tile("Series", format!("{base}/{{date:YYYY}}.tif"), 1.0, true), + ]); + let maps = text.pointer_mut("/maps").and_then(Value::as_array_mut); + match maps { + Some(maps) => maps[0]["tiles"] = tiles, + None => text["map"]["tiles"] = tiles, + } + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text.to_string()})).unwrap(); + let scene = t.map().layers.iter().find(|l| l.name == "Scene").unwrap(); + let rl = scene.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.data.is_some() && rl.path.as_deref() == Some(&*format!("{base}/scene.tif")), "{:?}", rl.path); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + assert!(!scene.visible && (scene.opacity() - 0.5).abs() < 1e-6); + // Drawn under the map's layers, as it was. + assert_eq!(t.map().layers.last().map(|l| l.name.as_str()), Some("Scene")); + assert_eq!(t.map().tiles.iter().map(|x| x.name.as_str()).collect::>(), ["Series"]); +} + /// A server answering every request with `body`, logging each as `HEAD`, `GET` or `GET `. /// Its HEAD answers carry `announce` as `X-TIFF-Header-Length`, `kind` as `Content-Type` and, /// with `ranges`, `Accept-Ranges: bytes`. @@ -23675,18 +23836,20 @@ fn geolibre_group_without_id_does_not_claim_ungrouped_layers() { assert!(s.map().group_of(id).is_none()); } -/// Declared COGs stream even without a TIFF suffix; a tile's folder look is baked only once. +/// Declared COGs are raster layers read in place even without a TIFF suffix, kept when their server +/// can't be reached at import. #[test] -fn geolibre_extensionless_cog_uses_tile_storage_and_inherited_opacity() { +fn geolibre_extensionless_cog_is_a_raster_read_in_place() { let mut s = world(); let text = json!({"version":"0.2.0","name":"COG","mapView":{},"basemapStyleUrl":"","layerGroups":[{"id":"g","name":"Folder","opacity":0.5}],"layers":[{"id":"c","name":"Coverage","type":"cog","groupId":"g","opacity":0.6,"source":{"url":"http://127.0.0.1:9/extensionless-coverage"}}]}).to_string(); let out = s.run("project.importGeolibre", &json!({"text":text})).unwrap(); - assert_eq!(out["layers"], json!(["Coverage"])); - assert!(s.map().layers.is_empty()); - assert_eq!(s.map().tiles.len(), 1); - assert_eq!(s.map().tiles[0].kind, rustgis_map::tiles::TileKind::Cog); - assert!((s.map().tiles[0].opacity - 0.3).abs() < 1e-6); - assert!(s.map().groups.iter().all(|g| g.children.is_empty())); + assert_eq!(out["layers"], json!(["Coverage"]), "{out}"); + // A raster layer read in place, kept although its server can't be reached now (it says why). + assert!(s.map().tiles.is_empty()); + let l = s.map().layers.iter().find(|l| l.name == "Coverage").unwrap(); + let rl = l.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.data.is_none() && rl.error.is_some(), "{:?}", rl.error); + assert!((l.opacity() - 0.6).abs() < 1e-6 || (l.opacity() - 0.3).abs() < 1e-6, "{}", l.opacity()); assert!(s.take_url_requests().is_empty()); } diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index ae357e7b1..7c6649f08 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -20,6 +20,11 @@ pub struct RasterLayer { /// The file it was read from; reopened when the project loads. #[serde(default)] pub path: Option, + /// `path` is a cloud-optimised GeoTIFF's web address, read in place: the layer holds an + /// overview of it (its statistics, histograms and tools read that), and the map reads the part + /// in view as it draws (see [`set_in_place_reader`]). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub in_place: bool, /// The file's own name (for its format) when it is embedded. #[serde(default)] pub file_name: Option, @@ -55,6 +60,24 @@ pub struct RasterLayer { /// A remote reload shared with render snapshots, never serialized. #[serde(skip)] opening: Option, + /// The style is the file's own look once its pixels land (a layer added before they have). + #[serde(skip)] + restyle: bool, +} + +/// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. +pub type InPlaceRead = Box) + Send>; +type InPlaceReader = Arc; + +fn in_place_reader() -> &'static Mutex> { + static R: std::sync::OnceLock>> = std::sync::OnceLock::new(); + R.get_or_init(Mutex::default) +} + +/// Sets how a cloud-optimised GeoTIFF's overview is read by its address (the tile cache's reader, +/// which checks addresses and signs requests; this crate does no networking). +pub fn set_in_place_reader(f: impl Fn(&str, InPlaceRead) + Send + Sync + 'static) { + *in_place_reader().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); } /// The renderer bands' statistics without the masked pixels, and what they were worked out for: @@ -80,6 +103,9 @@ impl RasterLayer { if self.embedded.is_some() { return Err("the raster's data is kept in the project: there's no file to repair".into()); } + if self.in_place { + return Err("the raster is read in place by its address: add the address again to change it".into()); + } let sub = subdataset.or(self.subdataset.as_deref()); let r = rustgis_data::raster::read_path_with(std::path::Path::new(path), sub, self.crs.as_deref()) .and_then(|r| placed(r, self.placement)) @@ -100,6 +126,7 @@ impl RasterLayer { style.repair(&r); Ok(RasterLayer { path: Some(path.into()), + in_place: false, file_name: None, embedded: None, subdataset: sub.map(String::from), @@ -111,6 +138,7 @@ impl RasterLayer { error: None, masked: MaskedStats::default(), opening: None, + restyle: false, }) } @@ -119,6 +147,7 @@ impl RasterLayer { let style = RasterStyle::default_for(&raster); RasterLayer { path: Some(path.into()), + in_place: false, file_name: None, embedded: None, subdataset: None, @@ -130,6 +159,7 @@ impl RasterLayer { error: None, masked: MaskedStats::default(), opening: None, + restyle: false, } } @@ -138,6 +168,7 @@ impl RasterLayer { let style = RasterStyle::default_for(&raster); RasterLayer { path: None, + in_place: false, file_name: Some(file_name.into()), embedded: Some(Arc::new(bytes)), subdataset: None, @@ -149,9 +180,38 @@ impl RasterLayer { error: None, masked: MaskedStats::default(), opening: None, + restyle: false, } } + /// A cloud-optimised GeoTIFF read in place by its address, its overview `raster` read. + pub fn in_place(raster: Raster, address: impl Into) -> RasterLayer { + RasterLayer { in_place: true, ..RasterLayer::from_path(raster, address) } + } + + /// A cloud-optimised GeoTIFF read in place by its address, its overview still to be read + /// (where nothing may wait: the browser): drawn, in its own look, once it lands. + pub fn opening_in_place(address: impl Into) -> RasterLayer { + let mut layer = RasterLayer { + path: Some(address.into()), + in_place: true, + file_name: None, + embedded: None, + subdataset: None, + crs: None, + time_zarr: None, + placement: None, + style: RasterStyle::placeholder(), + data: None, + error: None, + masked: MaskedStats::default(), + opening: None, + restyle: true, + }; + let _ = layer.reload(); + layer + } + /// Reads the pixels again from the path or the embedded file (after a project opens). A /// layer that follows the time slider is left to the engine, which reads the step shown. pub fn reload(&mut self) -> Result<(), String> { @@ -160,6 +220,50 @@ impl RasterLayer { } self.opening = None; self.error = None; + // Read in place: its overview, by the reader the app installed; waited for natively (as a + // file is read), landing later in the browser. + if self.in_place { + let Some(address) = self.path.clone() else { return Err("the raster has no source".into()) }; + let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { + let e = format!("{address}: can't be read in place here"); + self.error = Some(e.clone()); + return Err(e); + }; + #[cfg(not(target_arch = "wasm32"))] + { + let (tx, rx) = std::sync::mpsc::channel(); + reader(&address, Box::new(move |r| tx.send(r).unwrap_or_default())); + let r = rx + .recv_timeout(std::time::Duration::from_secs(120)) + .unwrap_or_else(|_| Err(format!("{address}: no answer in two minutes"))); + return match r { + Ok(raster) => { + self.land(Ok(Arc::new(raster))); + Ok(()) + } + Err(e) => { + self.data = None; + self.land(Err(e.clone())); + Err(e) + } + }; + } + #[cfg(target_arch = "wasm32")] + { + let slot = Arc::new(Mutex::new(None)); + let land = slot.clone(); + self.data = None; + self.opening = Some(slot); + reader( + &address, + Box::new(move |result| { + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); + crate::source::landed(); + }), + ); + return Ok(()); + } + } if self.embedded.is_none() && let Some(path) = self .path @@ -220,14 +324,24 @@ impl RasterLayer { let Some(slot) = self.opening.as_ref() else { return false }; let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() else { return true }; self.opening = None; + self.land(result); + false + } + + /// Installs pixels read (or why they couldn't be): in the file's own look for a layer added + /// before they arrived, else in its style, repaired for them. + fn land(&mut self, result: Result, String>) { match result { Ok(raster) => { - self.style.repair(&raster); + if std::mem::take(&mut self.restyle) { + self.style = RasterStyle::default_for(&raster); + } else { + self.style.repair(&raster); + } self.data = Some(raster); } Err(error) => self.error = Some(error), } - false } /// Statistics of the renderer's bands (one per band, in its order) without the pixels the @@ -700,8 +814,26 @@ fn one() -> f64 { } impl RasterStyle { - /// The default look: the file's colours, RGB for colour images, unique values for a few - /// categories, otherwise a grey stretch. + /// A style for a layer whose pixels haven't arrived (its own look replaces it when they do). + pub fn placeholder() -> RasterStyle { + RasterStyle { + renderer: RasterRenderer::Stretch { band: 0, ramp: Ramp::gray(), invert: false }, + stretch: Stretch::default(), + dra: false, + resampling: Resampling::Nearest, + brightness: 0.0, + contrast: 0.0, + gamma: 1.0, + rotation: 0.0, + mask: RasterMask::default(), + } + } + + /// The default look, as the file's PhotometricInterpretation says: the file's colours, RGB for + /// colour images and for grey ones of exactly three bands (most remote sensing images are + /// grey, whatever their bands: three are taken as red, green and blue), unique values for a + /// few categories, otherwise a grey stretch of the first band (an image of more bands is + /// combined as the user picks). pub fn default_for(r: &Raster) -> RasterStyle { RasterStyle::default_with(r, &RasterDefaults::default()) } @@ -715,7 +847,7 @@ impl RasterStyle { ColorHint::Rgb { .. } if r.band_count() >= 3 => { (RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, if byte_rgb { Stretch::None } else { Stretch::default() }) } - _ if r.band_count() >= 3 => (RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, Stretch::default()), + ColorHint::Gray if r.band_count() == 3 => (RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, Stretch::default()), _ => match r.bands.first().and_then(|b| b.stats.categories.as_ref()).filter(|c| c.len() <= 16 && r.data_type.contains("Int")) { Some(_) => (unique_values(r, 0, Ramp::Harbour), Stretch::None), None => (RasterRenderer::Stretch { band: 0, ramp: d.ramp.clone(), invert: d.invert }, d.stretch), diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 39b688543..87485f22d 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -3159,12 +3159,20 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac }; let stats = rl.masked_stats(); let stats = stats.as_ref().map(|s| s.as_slice()); + // Read in place: the part in view, at the level the view needs. + let address = rl.path.as_deref().filter(|_| rl.in_place); + let mut draw_it = |pm: &mut Pixmap, opacity: f32| match address { + Some(a) => { + tile_out.pending |= raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) + } + None => raster::draw(pm, data, &rl.style, stats, view, opacity, full_res_rasters, place), + }; if offscreen { let mut target = Pixmap::new(w, h)?; - raster::draw(&mut target, data, &rl.style, stats, view, 1.0, full_res_rasters, place); + draw_it(&mut target, 1.0); composite(&mut pm, &target); } else { - raster::draw(&mut pm, data, &rl.style, stats, view, opacity, full_res_rasters, place); + draw_it(&mut pm, opacity); } cpu_dirty = true; } diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 393cc92d8..58eee9f17 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -102,6 +102,179 @@ pub(crate) fn draw( draw_with(pm, r, style, stats, view, opacity, full_res, place, threads); } +/// Screen points a side sampled to find the part of a raster read in place that is in view, and +/// how finely. +const WINDOW_GRID: usize = 17; + +/// Draws a raster layer read in place (a cloud-optimised GeoTIFF by its address, see +/// [`rustgis_map::raster::RasterLayer::in_place`]) as [`draw`] draws one held whole: its overview +/// `r` (the layer's pixels), or, zoomed in past it, the part in view read from the level the view +/// needs, drawn over the overview until its tiles arrive. `stats` are the renderer's bands' +/// statistics (the overview's when not given), so the part in view is stretched as the whole is. +/// Returns whether tiles are still to come (the map is drawn again as they land). +#[allow(clippy::too_many_arguments)] +pub(crate) fn draw_in_place( + pm: &mut Pixmap, + address: &str, + r: &Raster, + style: &RasterStyle, + stats: Option<&[BandStats]>, + view: &View, + opacity: f32, + full_res: bool, + place: Place, +) -> bool { + let bands = style.renderer.bands(); + let own: Vec = bands.iter().map(|b| r.bands.get(*b).map(|b| b.stats.clone()).unwrap_or_default()).collect(); + let stats = stats.unwrap_or(&own); + let whole = |pm: &mut Pixmap| draw(pm, r, style, Some(stats), view, opacity, full_res, place); + // A rotated raster turns about the whole image's centre, not the part in view's. + let full = r.overview.unwrap_or((r.width, r.height)); + if style.rotation.rem_euclid(360.0).abs() > 1e-6 || full.0 <= r.width { + whole(pm); + return false; + } + let info = match rustgis_tiles::cog::info(address) { + Ok(Some(info)) => info, + Ok(None) => { + whole(pm); + return true; + } + Err(_) => { + whole(pm); + return false; + } + }; + let Some(want) = in_view(r, view, place, pm.width(), pm.height(), full_res) else { + whole(pm); + return false; + }; + // The coarsest level at least as fine as the view (within a quarter); zoomed in past the full + // image, the full image. + let levels = &info.layout.levels; + let full_px = want.per_px * f64::from(full.0) / f64::from(r.width.max(1)); + let fine_enough = |l: &rustgis_data::raster::cog::Level| f64::from(full.0) / f64::from(l.width.max(1)) <= full_px * 1.25; + let mut level = levels.iter().rposition(fine_enough).unwrap_or(0); + // The renderer's bands, then the image's alpha band, read as the window's own bands. + let mut read = bands.clone(); + let alpha = match (&style.renderer, &r.color) { + (RasterRenderer::Rgb { .. }, ColorHint::Rgb { alpha: true }) if r.band_count() >= 4 && !read.contains(&3) => Some(3), + (RasterRenderer::Stretch { band: 0, .. } | RasterRenderer::ShadedRelief { band: 0, .. }, ColorHint::GrayAlpha) => Some(1), + _ => None, + }; + read.extend(alpha); + let mut window = None; + while let Some(l) = levels.get(level) { + // The part in view, in the level's pixels, a pixel past each side (for bilinear sampling). + let (sx, sy) = (f64::from(l.width) / f64::from(r.width.max(1)), f64::from(l.height) / f64::from(r.height.max(1))); + let edge = |v: f64, k: f64, max: u32| ((v * k).floor() as i64).clamp(0, i64::from(max)) as u32; + let cols = (edge(want.cols.0, sx, l.width).saturating_sub(1), (edge(want.cols.1, sx, l.width) + 2).min(l.width)); + let rows = (edge(want.rows.0, sy, l.height).saturating_sub(1), (edge(want.rows.1, sy, l.height) + 2).min(l.height)); + if rustgis_tiles::cog::raster::fits(address, level, cols, rows, &read) { + window = Some((level, cols, rows)); + break; + } + // Too large a part for the tiles kept (a view far wider than the screen, a file of many + // bands stored together): a coarser level. + level += 1; + } + let Some((level, cols, rows)) = window else { + whole(pm); + return false; + }; + // The image at the overview's level is the overview. + if levels.get(level).is_some_and(|l| l.width <= r.width) { + whole(pm); + return false; + } + let color = match (&style.renderer, &r.color) { + (_, ColorHint::Palette(p)) => ColorHint::Palette(p.clone()), + (RasterRenderer::Rgb { .. }, _) => ColorHint::Rgb { alpha: alpha.is_some() }, + _ if alpha.is_some() => ColorHint::GrayAlpha, + _ => ColorHint::Gray, + }; + let w = match rustgis_tiles::cog::raster::window(address, level, cols, rows, &read, color) { + Ok(w) => w, + Err(_) => { + whole(pm); + return false; + } + }; + let Some(part) = w.raster else { + whole(pm); + return w.pending; + }; + // Until its tiles arrive, the part in view goes over the overview. + if w.pending { + whole(pm); + } + let mut own_style = style.clone(); + own_style.renderer = renumbered(&style.renderer); + draw(pm, &part, &own_style, Some(stats), view, opacity, full_res, place); + w.pending +} + +/// A renderer drawing the same bands of a raster holding only those, in its order (see +/// [`RasterRenderer::bands`]). +fn renumbered(renderer: &RasterRenderer) -> RasterRenderer { + let mut out = renderer.clone(); + match &mut out { + RasterRenderer::Stretch { band, .. } + | RasterRenderer::Classify { band, .. } + | RasterRenderer::UniqueValues { band, .. } + | RasterRenderer::ShadedRelief { band, .. } => *band = 0, + RasterRenderer::Rgb { red, green, blue } => (*red, *green, *blue) = (0, 1, 2), + RasterRenderer::Colormap => {} + } + out +} + +/// The part of a raster in view: its columns and rows (half-open, in its own pixels, possibly +/// fractional), and how many of its pixels a pixel drawn spans. +#[derive(Debug, PartialEq)] +struct InView { + cols: (f64, f64), + rows: (f64, f64), + per_px: f64, +} + +/// [`InView`] of a raster drawn into a `width × height` pixmap, from a grid of screen points; +/// `None` when it isn't in view. +fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_res: bool) -> Option { + let k = if view.pixels_per_point >= 1.5 && !full_res { view.pixels_per_point } else { 1.0 }; + let (w, h) = (f64::from(width) / k, f64::from(height) / k); + let n = WINDOW_GRID; + let at = |i: usize, j: usize| source(r, view, place, k, w * i as f64 / (n - 1) as f64, h * j as f64 / (n - 1) as f64); + let grid: Vec> = (0..n * n).map(|p| at(p % n, p / n)).collect(); + let (mut c0, mut c1, mut r0, mut r1) = (f64::INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::NEG_INFINITY); + let (rw, rh) = (f64::from(r.width), f64::from(r.height)); + for (c, row) in grid.iter().flatten() { + c0 = c0.min(c.clamp(0.0, rw)); + c1 = c1.max(c.clamp(0.0, rw)); + r0 = r0.min(row.clamp(0.0, rh)); + r1 = r1.max(row.clamp(0.0, rh)); + } + if !(c0 < c1 && r0 < r1) { + return None; + } + // The raster's pixels a screen pixel spans: the smallest step between neighbouring points, + // where the view is finest (a tilted or curved view is finer near its centre). + let step = (w / (n - 1) as f64).max(1.0); + let mut per_px = f64::INFINITY; + for j in 0..n { + for i in 0..n - 1 { + if let (Some(Some(a)), Some(Some(b))) = (grid.get(j * n + i), grid.get(j * n + i + 1)) { + let d = ((b.0 - a.0).powi(2) + (b.1 - a.1).powi(2)).sqrt() / step; + if d > 0.0 { + per_px = per_px.min(d); + } + } + } + } + // In the pixels drawn (a work pixel is `k` device pixels). + per_px.is_finite().then_some(InView { cols: (c0, c1), rows: (r0, r1), per_px }) +} + /// [`draw`], its rows spread over `threads` threads. #[allow(clippy::too_many_arguments)] fn draw_with( diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index 627b1dc43..6eea3e2da 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -66,17 +66,25 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> let center = Coord::new(-half + (f64::from(x) + 0.5) * size, half - (f64::from(y) + 0.5) * size); let view = View { width: 256.0, height: 256.0, pixels_per_point: 1.0, camera: Camera::new(center, size / 256.0), sky: false }; + let place = raster::Place::Flat(&Flat::Mercator, "EPSG:3857"); + let stats = stats.as_deref().map(Vec::as_slice); let mut image = Pixmap::new(TILE_SIZE, TILE_SIZE).ok_or("couldn't allocate a raster tile")?; - raster::draw( - &mut image, - r, - &layer.style, - stats.as_deref().map(Vec::as_slice), - &view, - opacity, - true, - raster::Place::Flat(&Flat::Mercator, "EPSG:3857"), - ); + match layer.path.as_deref().filter(|_| layer.in_place) { + // Read in place: at the tile's level, once its parts have arrived (natively; + // the browser draws what has). + Some(address) => { + let mut passes = 0; + while raster::draw_in_place(&mut image, address, r, &layer.style, stats, &view, opacity, true, place) + && passes < 6 + && cfg!(not(target_arch = "wasm32")) + { + crate::rustgis_tiles::cache::wait(web_time::Duration::from_secs(30)); + image.fill(tiny_skia::Color::TRANSPARENT); + passes += 1; + } + } + None => raster::draw(&mut image, r, &layer.style, stats, &view, opacity, true, place), + } if image.data().as_chunks::<4>().0.iter().all(|pixel| pixel[3] == 0) { continue; } diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 3b8cd6ed4..f644a5eb7 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -3,24 +3,29 @@ //! tile (`cog:{z}/{x}/{y}@
`) is drawn from them in Web Mercator, as a picture, so a COG is //! a tile layer like a PMTiles archive: drawn as its tiles land, on the desktop and in the browser. //! -//! Grey images are stretched between their 2nd and 98th percentiles (from the coarsest -//! overview); 8-bit colour images are shown as they are. An address may end with a [`Look`] -//! (`#bands=2,17,10&range=-0.3,0.3`): the bands shown as red, green and blue (or one, grey) and -//! their stretch. Bands stored apart (planar files, such as AlphaEarth's 64 embedding bands) are -//! read a band at a time, only those shown. Files are hostile input: header reads, tile counts and -//! sizes are bounded (see [`rustgis_data::raster::cog`]). +//! A file's own look is what its PhotometricInterpretation says: RGB images in colour, palette +//! images by their colour table, grey (MinIsBlack) ones grey, their first band, but for three bands +//! (taken as red, green and blue), each with its alpha band. Other bands of an image with more are +//! for the user to pick. Grey and colour images are stretched between their 2nd and 98th +//! percentiles (from the coarsest overview); 8-bit colour images are shown as they are. An address +//! may end with a [`Look`] (`#bands=2,17,10&range=-0.3,0.3`): the bands shown as red, green and +//! blue (or one, grey) and their stretch. Bands stored apart (planar files, such as AlphaEarth's 64 +//! embedding bands) are read a band at a time, only those shown. Files are hostile input: header +//! reads, tile counts and sizes are bounded (see [`rustgis_data::raster::cog`]). use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; use rustgis_data::raster::ColorHint; -use rustgis_data::raster::cog::{self, Layout, Parse, Sparse}; +use rustgis_data::raster::cog::{self, Layout, Level, Parse, Sparse}; use rustgis_data::raster::crs::Transformer; use rustgis_geom::Coord; use web_time::{Duration, Instant}; use crate::cache::{Done, MAX_BYTES, RANGE}; +pub mod raster; + /// The scheme of a map tile drawn from a COG. pub const SCHEME: &str = "cog:"; @@ -168,33 +173,51 @@ pub struct Info { /// Stretches whose read failed, and when: drawn with the type's full range meanwhile (not /// kept), read again after [`RETRY`]. stretch_failed: Mutex, bool), Instant>>, - /// The file's own bands shown as red, green, blue (or one, grey), and alpha. - rgb: bool, - alpha: bool, + /// The file's alpha band, shown with any look. + alpha: Option, } impl Info { - /// The bands a look shows (0-based): three as red, green and blue (and the file's alpha, as a - /// fourth, for its own), or one, grey. - pub fn shown(&self, look: &Look) -> Vec { + /// The bands a look colours the map with (0-based): three as red, green and blue, or one, + /// grey. Its own: an RGB image's colour bands, or a grey image's three bands (most remote + /// sensing images are grey, whatever their bands: three are taken as red, green and blue); + /// else the first band, the others for the user to pick. + pub fn colour(&self, look: &Look) -> Vec { let n = self.layout.bands.max(1); // Bands the file has, as many as asked (one or three); otherwise the file's own look. let picked: Vec = look.bands.iter().copied().filter(|b| *b < n).collect(); match picked.len() { 1 | 3 if picked.len() == look.bands.len() => picked, - _ if self.rgb && self.alpha => vec![0, 1, 2, 3], - _ if self.rgb => vec![0, 1, 2], + _ if matches!(self.layout.color, ColorHint::Rgb { .. }) && n >= 3 => vec![0, 1, 2], + _ if matches!(self.layout.color, ColorHint::Gray) && n == 3 => vec![0, 1, 2], _ => vec![0], } } + /// The bands a look reads: its colour bands, then the file's alpha band, if it has one. + pub fn shown(&self, look: &Look) -> Vec { + let mut shown = self.colour(look); + if let Some(a) = self.alpha.filter(|a| !shown.contains(a)) { + shown.push(a); + } + shown + } + + /// The colours of a palette image's values, when its own look shows them. + fn palette(&self, look: &Look) -> Option<&[[u8; 4]]> { + match &self.layout.color { + ColorHint::Palette(table) if look.bands.is_empty() && look.range.is_none() && !look.aef => Some(table), + _ => None, + } + } + /// The bands' (low, high) values for a look, when known. - fn stretch_for(&self, look: &Look, shown: &[usize]) -> Option> { - let colour = &shown[..shown.len().min(3)]; + fn stretch_for(&self, look: &Look) -> Option> { + let colour = self.colour(look); if let Some(r) = look.range { return Some(vec![r; colour.len()]); } - self.stretch.lock().unwrap_or_else(|e| e.into_inner()).get(&(colour.to_vec(), look.aef)).cloned() + self.stretch.lock().unwrap_or_else(|e| e.into_inner()).get(&(colour, look.aef)).cloned() } } @@ -212,7 +235,9 @@ fn infos() -> &'static Mutex> { type Bands = Arc>>; enum TileState { - Ready(Bands, usize), + /// Decoded: its bands, how many samples they hold, and when it was last used (see + /// [`Tiles::clock`]). + Ready(Bands, usize, u64), Loading(Vec) + Send>>), } @@ -220,6 +245,48 @@ enum TileState { struct Tiles { map: HashMap<(String, usize, u32), TileState>, samples: usize, + /// Goes up with each use: past [`BUDGET`], the tile used longest ago makes room, never one + /// the map in view has just used (which would be read again, and make room for itself by + /// throwing out another in view: the view flickering between levels). + clock: u64, +} + +impl Tiles { + /// A tile ready, marked as just used. + fn used(&mut self, key: &(String, usize, u32)) -> Option { + self.clock += 1; + let now = self.clock; + match self.map.get_mut(key) { + Some(TileState::Ready(b, _, used)) => { + *used = now; + Some(b.clone()) + } + _ => None, + } + } + + /// Keeps a tile ready, making room for it past [`BUDGET`] (the tiles used longest ago go); + /// returns what was kept under its key before. + fn keep(&mut self, key: (String, usize, u32), bands: Bands, samples: usize) -> Option { + while self.samples + samples > BUDGET { + let oldest = self + .map + .iter() + .filter_map(|(k, s)| match s { + TileState::Ready(_, _, used) => Some((*used, k)), + TileState::Loading(_) => None, + }) + .min_by_key(|(used, _)| *used) + .map(|(_, k)| k.clone()); + let Some(k) = oldest else { break }; + if let Some(TileState::Ready(_, n, _)) = self.map.remove(&k) { + self.samples = self.samples.saturating_sub(n); + } + } + self.clock += 1; + self.samples += samples; + self.map.insert(key, TileState::Ready(bands, samples, self.clock)) + } } fn tiles() -> &'static Mutex { @@ -244,7 +311,15 @@ pub(crate) fn forget_file(url: &str) { } drop(all); let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); - t.map.retain(|(u, _, _), s| u != url || matches!(s, TileState::Loading(_))); + let mut freed = 0; + t.map.retain(|(u, _, _), s| match s { + TileState::Ready(_, n, _) if u == url => { + freed += *n; + false + } + _ => true, + }); + t.samples = t.samples.saturating_sub(freed); } /// A file's info: `Ok(None)` while it is being read (asking for it starts the read). @@ -481,8 +556,17 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box 0.0 { (2.0 * HALF / f64::from(SIZE) / res).log2().ceil().clamp(0.0, 24.0) as u8 } else { 18 }; - let rgb = layout.bands >= 3; - let alpha = matches!(layout.color, ColorHint::Rgb { alpha: true }); + let alpha = match layout.color { + ColorHint::Rgb { alpha: true } if layout.bands >= 4 => Some(3), + ColorHint::GrayAlpha if layout.bands >= 2 => Some(1), + _ => None, + }; + let rgb = match layout.color { + ColorHint::Rgb { .. } => layout.bands >= 3, + ColorHint::Gray => layout.bands == 3, + _ => false, + }; + let palette = matches!(layout.color, ColorHint::Palette(_)); let bytes = layout.data_type == "UInt8"; let info = Arc::new(Info { layout, @@ -495,14 +579,15 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box usize { - layout.levels.iter().rposition(|l| l.width >= 256 && l.height >= 256).unwrap_or(0) +/// The image the stretch is read from: of those few enough tiles to read for it, the coarsest of +/// at least 256 × 256 pixels (GDAL writes overviews down to a pixel, too few to take percentiles +/// of), else the finest. A long strip's overviews are narrow: one 256 pixels wide is too many tiles +/// long, a narrower one plenty of pixels. None small enough: the coarsest (the type's range is +/// shown). +fn stats_level(levels: &[Level]) -> usize { + let few = |l: &Level| (l.tiles() as usize) <= MAX_STATS_TILES; + let pixels = |l: &Level| u64::from(l.width) * u64::from(l.height); + levels + .iter() + .rposition(|l| few(l) && pixels(l) >= 256 * 256) + .or_else(|| levels.iter().position(few)) + .unwrap_or(levels.len().saturating_sub(1)) } /// The full range of a file's type of values, as a look shows them (floats have none: 0–255 is a @@ -541,7 +635,7 @@ fn type_range(layout: &Layout, aef: bool) -> (f32, f32) { /// Keeps a look's stretch: a few looks a file (a script trying many is hostile input); past them, /// the others are read again when asked for. fn keep_stretch(info: &Info, key: (Vec, bool), s: Vec<(f32, f32)>) { - let own = (info.shown(&Look::default()).into_iter().take(3).collect::>(), false); + let own = (info.colour(&Look::default()), false); let mut kept = info.stretch.lock().unwrap_or_else(|e| e.into_inner()); // Past the cap one other look goes, never the file's own (an 8-bit colour image's is "as it // is", not read). @@ -570,13 +664,12 @@ fn percentiles(mut v: Vec) -> (f32, f32) { type StretchThen = Box, String>) + Send>; fn with_stretch(info: Arc, file: String, look: Look, then: StretchThen) { - let shown = info.shown(&look); - if let Some(s) = info.stretch_for(&look, &shown) { + if let Some(s) = info.stretch_for(&look) { then(Ok(s)); return; } - let colour: Vec = shown.iter().copied().take(3).collect(); - let coarsest = stats_level(&info.layout); + let colour = info.colour(&look); + let coarsest = stats_level(&info.layout.levels); let n = info.layout.levels.get(coarsest).map(|l| l.tiles() as usize).unwrap_or(0); // Too many tiles to read for it: the values' full range for the type. if n == 0 || n > MAX_STATS_TILES { @@ -785,13 +878,13 @@ fn get_tile(info: Arc, url: String, level: usize, idx: u32, then: Box { - let b = b.clone(); - drop(t); - then(Ok(b)); - return; - } + Some(TileState::Ready(..)) => {} Some(TileState::Loading(w)) => { w.push(then); return; @@ -810,19 +903,8 @@ fn get_tile(info: Arc, url: String, level: usize, idx: u32, then: Box { - // Over the budget: tiles ready make room (the cache is a working set). - while t.samples + samples > BUDGET { - let Some(k) = t.map.iter().find(|(_, s)| matches!(s, TileState::Ready(..))).map(|(k, _)| k.clone()) else { - break; - }; - if let Some(TileState::Ready(_, n)) = t.map.remove(&k) { - t.samples = t.samples.saturating_sub(n); - } - } - t.samples += samples; - t.map.insert(key, TileState::Ready(b.clone(), samples)) - } + // Over the budget: the tiles used longest ago make room. + Ok(b) => t.keep(key, b.clone(), samples), // A failed tile is read again by the next map tile that needs it. Err(_) => t.map.remove(&key), }; @@ -974,7 +1056,15 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 return; } let shown = info.shown(look); - let pen = Pen { rgb: shown.len() >= 3, alpha: shown.len() >= 4, aef: look.aef, stretch: stretch.to_vec() }; + let colours = info.colour(look).len(); + let pen = Pen { + rgb: colours >= 3, + alpha: (shown.len() > colours).then_some(colours), + alpha_max: type_range(&info.layout, false).1.max(1.0), + palette: info.palette(look).map(<[[u8; 4]]>::to_vec), + aef: look.aef, + stretch: stretch.to_vec(), + }; let w2 = wanted.clone(); fetch_bands( info, @@ -991,9 +1081,13 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 /// How a map tile's pixels are coloured from the bands shown. struct Pen { - /// Three bands as red, green and blue (else one, grey), and a fourth as alpha. + /// Three bands as red, green and blue (else one, grey). rgb: bool, - alpha: bool, + /// Where the alpha band is among those shown, after the colour ones, and its opaque value. + alpha: Option, + alpha_max: f32, + /// A palette image's colours, by value (no stretch). + palette: Option>, aef: bool, stretch: Vec<(f32, f32)>, } @@ -1011,23 +1105,38 @@ fn paint(pen: &Pen, spots: &[Option<(u32, usize)>], wanted: &[u32], tiles: &[Pic let Some((t, at)) = spot else { continue }; let Some(bands) = wanted.binary_search(t).ok().and_then(|i| tiles.get(i)) else { continue }; let v = |b: usize| bands.get(b).and_then(|(chunk, i)| chunk.get(*i)).and_then(|band| band.get(*at)).copied().unwrap_or(f32::NAN); + // The alpha band's value, 0 to 255 (opaque without one); none where it hides the pixel. + let a = match pen.alpha { + Some(i) => { + let a = v(i); + if a.is_nan() || a <= 0.0 { + continue; + } + ((a / pen.alpha_max).clamp(0.0, 1.0) * 255.0).round() as u8 + } + None => 255, + }; let px = if pen.rgb { let (r, g, b) = (v(0), v(1), v(2)); if !(r.is_finite() && g.is_finite() && b.is_finite()) { continue; } - let a = if pen.alpha { v(3) } else { 255.0 }; - if pen.alpha && (a.is_nan() || a <= 0.0) { + [scale(0, r), scale(1, g), scale(2, b), a] + } else if let Some(table) = &pen.palette { + let g = v(0); + // A value past the table: nothing. + let Some(c) = (g.is_finite() && g >= 0.0).then(|| table.get(g as usize)).flatten() else { continue }; + if c[3] == 0 { continue; } - [scale(0, r), scale(1, g), scale(2, b), a.clamp(0.0, 255.0) as u8] + [c[0], c[1], c[2], ((u16::from(c[3]) * u16::from(a)) / 255) as u8] } else { let g = v(0); if !g.is_finite() { continue; } let g = scale(0, g); - [g, g, g, 255] + [g, g, g, a] }; if let Some(out) = rgba.get_mut(k * 4..k * 4 + 4) { out.copy_from_slice(&px); @@ -1239,7 +1348,28 @@ mod tests { let rustgis_data::raster::cog::Parse::Done(l) = cog::layout(&Sparse::default().with(0, file)).unwrap() else { panic!() }; // Overviews down to 64 pixels: the 256-pixel one is read. assert_eq!(l.levels.last().map(|l| l.width), Some(64)); - assert_eq!(l.levels[stats_level(&l)].width, 256); + assert_eq!(l.levels[stats_level(&l.levels)].width, 256); + // A long strip (a WorldView image 44 626 × 353 888 pixels, in tiles of 256): its overview + // 349 pixels wide is 22 tiles, too many; the one 175 wide, 6 tiles, still a quarter of a + // million pixels, is read. + let level = |width: u32, height: u32| Level { + ifd: 0, + width, + height, + tile_width: 256, + tile_height: 256, + offsets: Vec::new(), + counts: Vec::new(), + }; + let strip: Vec = [(44_626, 353_888), (5579, 44_236), (698, 5530), (349, 2765), (175, 1383), (88, 692)] + .into_iter() + .map(|(w, h)| level(w, h)) + .collect(); + assert_eq!(strip[stats_level(&strip)].width, 175); + // Only small ones: the finest of them; none: the coarsest. + assert_eq!(stats_level(&[level(4096, 4096), level(200, 200), level(100, 100)]), 1); + assert_eq!(stats_level(&[level(8192, 8192), level(4096, 4096)]), 1); + assert_eq!(stats_level(&[]), 0); } /// Reads `file`'s layout as [`load`] and [`step`] do, a range at a time: the layout, how many @@ -1356,7 +1486,7 @@ mod tests { red[1] = f32::NAN; let tile: Picked = vec![(Arc::new(vec![red]), 0), (chunk(20.0), 0), (chunk(30.0), 0)]; let spots: Vec> = (0..(SIZE * SIZE) as usize).map(|k| if k < 2 { Some((7, k)) } else { None }).collect(); - let pen = Pen { rgb: true, alpha: false, aef: false, stretch: vec![(0.0, 40.0); 3] }; + let pen = Pen { rgb: true, alpha: None, alpha_max: 255.0, palette: None, aef: false, stretch: vec![(0.0, 40.0); 3] }; let out = paint(&pen, &spots, &[7], &[tile]).unwrap(); let px = &out[crate::cache::RAW.len() + 8..]; assert_eq!(&px[..4], &[64, 128, 191, 255]); @@ -1383,4 +1513,37 @@ mod tests { // Nothing there: no picture. assert!(paint(&pen, &vec![None; spots.len()], &[], &[]).unwrap().is_empty()); } + + #[test] + fn paints_grey_with_alpha_and_palettes() { + let chunk = |v: Vec| -> Bands { Arc::new(vec![v]) }; + let spots: Vec> = (0..(SIZE * SIZE) as usize).map(|k| (k < 3).then_some((7, k))).collect(); + let px = |out: Vec| out[crate::cache::RAW.len() + 8..crate::cache::RAW.len() + 8 + 12].to_vec(); + // A 16-bit pan band and its 16-bit alpha band: opaque, half, hidden. + let tile: Picked = vec![(chunk(vec![500.0, 1000.0, 1000.0, 0.0]), 0), (chunk(vec![65535.0, 32768.0, 0.0, 0.0]), 0)]; + let pen = Pen { rgb: false, alpha: Some(1), alpha_max: 65535.0, palette: None, aef: false, stretch: vec![(0.0, 1000.0)] }; + assert_eq!(px(paint(&pen, &spots, &[7], &[tile]).unwrap()), [128, 128, 128, 255, 255, 255, 255, 128, 0, 0, 0, 0]); + // A palette image: each value its colour, a transparent one or one past the table nothing. + let tile: Picked = vec![(chunk(vec![1.0, 2.0, 9.0, 0.0]), 0)]; + let table = vec![[0, 0, 0, 255], [200, 10, 20, 255], [0, 0, 0, 0]]; + let pen = Pen { rgb: false, alpha: None, alpha_max: 255.0, palette: Some(table), aef: false, stretch: vec![(0.0, 255.0)] }; + let out = px(paint(&pen, &spots, &[7], &[tile]).unwrap()); + assert_eq!(&out[..4], &[200, 10, 20, 255]); + } + + #[test] + fn the_tiles_used_longest_ago_make_room() { + let mut t = Tiles::default(); + let key = |i: u32| ("https://x.org/a.tif".to_string(), 0usize, i); + let bands: Bands = Arc::new(Vec::new()); + let half = BUDGET / 2; + t.keep(key(1), bands.clone(), half); + t.keep(key(2), bands.clone(), half); + // Tile 1 is used again (the view still shows it): tile 2 makes room for tile 3. + assert!(t.used(&key(1)).is_some()); + t.keep(key(3), bands.clone(), half); + assert!(t.used(&key(1)).is_some() && t.used(&key(3)).is_some()); + assert!(t.used(&key(2)).is_none()); + assert_eq!(t.samples, 2 * half); + } } diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs new file mode 100644 index 000000000..1967ebeda --- /dev/null +++ b/crates/tiles/src/cog/raster.rs @@ -0,0 +1,371 @@ +//! A cloud-optimised GeoTIFF as a raster layer's pixels, read in place: the same raster a file +//! gives, its symbology, statistics and tools alike, without downloading it. +//! +//! A layer holds an [`overview`]: every band of the finest overview that fits a budget, which its +//! statistics, histograms and tools read, as a file too large to hold whole is opened from an +//! overview. Drawn zoomed in past it, the part in view is read at the level the view needs as a +//! [`window`], a level's tiles at a time, from the tiles kept or fetched by range; the map draws +//! again as they land. + +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, OnceLock}; + +use rustgis_data::raster::{ColorHint, Raster}; +use web_time::Instant; + +use super::{BUDGET, Bands, Info, RETRY, TileState, Tiles, fetch_all, get_tile, split, tiles, with_info}; + +/// The most samples (every band) of the overview a layer holds. +const OVERVIEW_SAMPLES: usize = if cfg!(target_arch = "wasm32") { 1 << 20 } else { 2 << 20 }; +/// The most chunks read for it. +const OVERVIEW_CHUNKS: usize = 512; +/// The most samples of one band of a view's window (a large screen's worth, and more). +pub const WINDOW_SAMPLES: usize = 16 << 20; + +/// The level a layer holds as its overview: the finest whose every band fits [`OVERVIEW_SAMPLES`] +/// in at most [`OVERVIEW_CHUNKS`] chunks, else the coarsest. +pub fn overview_level(info: &Info) -> usize { + let layout = &info.layout; + let bands = layout.bands.max(1); + let planes = if layout.planar { bands } else { 1 }; + layout + .levels + .iter() + .position(|l| { + let samples = (l.width as usize).saturating_mul(l.height as usize).saturating_mul(bands); + let chunks = (l.tiles() as usize).saturating_mul(planes); + samples <= OVERVIEW_SAMPLES && chunks <= OVERVIEW_CHUNKS + }) + .unwrap_or(layout.levels.len().saturating_sub(1)) +} + +/// Reads a file's overview (see [`overview_level`]) as a raster, every band, then calls `done`. +/// The raster says it is an overview of the full image when it is one. +pub fn overview(url: &str, done: Box) + Send>) { + let file = split(url).0.to_string(); + let f = file.clone(); + with_info( + &file, + Box::new(move |r| match r { + Ok(info) => read_overview(info, f, done), + Err(e) => done(Err(e)), + }), + ); +} + +/// [`overview`], waited for (natively; at most two minutes). +#[cfg(not(target_arch = "wasm32"))] +pub fn overview_blocking(url: &str) -> Result { + let (tx, rx) = std::sync::mpsc::channel(); + overview(url, Box::new(move |r| tx.send(r).unwrap_or_default())); + rx.recv_timeout(std::time::Duration::from_secs(120)).map_err(|_| format!("{}: no answer in two minutes", split(url).0))? +} + +fn read_overview(info: Arc, url: String, done: Box) + Send>) { + let level = overview_level(&info); + let Some(l) = info.layout.levels.get(level) else { + done(Err(format!("{url}: the GeoTIFF has no image"))); + return; + }; + let bands: Vec = (0..info.layout.bands.max(1)).collect(); + let planes = if info.layout.planar { bands.len() } else { 1 }; + let Some(n) = l.tiles().checked_mul(u32::try_from(planes).unwrap_or(u32::MAX)) else { + done(Err(format!("{url}: too many tiles"))); + return; + }; + let (w, h) = (l.width, l.height); + let i = info.clone(); + fetch_all( + info, + url.clone(), + level, + (0..n).collect(), + Box::new(move |r| { + done(r.and_then(|got| { + let (data, _) = assemble(&i, level, (0, w), (0, h), &bands, |c| got.get(c as usize).cloned()); + let l = i.layout.levels.get(level).ok_or("no such overview")?; + let mut r = Raster::new( + file_name(&url), + w, + h, + data, + i.layout.transform_of(l), + i.layout.crs, + i.layout.nodata, + i.layout.color.clone(), + i.layout.data_type, + ) + .map_err(|e| format!("{url}: {e}"))?; + if let Some(full) = i.layout.levels.first().filter(|_| level > 0) { + r.overview = Some((full.width, full.height)); + } + r.source = Some(url.clone()); + Ok(r) + })) + }), + ); +} + +/// A file's name, from its address (without its query). +fn file_name(url: &str) -> String { + let path = url.split(['?', '#']).next().unwrap_or(url).trim_end_matches('/'); + path.rsplit('/').next().filter(|n| !n.is_empty()).unwrap_or("raster").to_string() +} + +/// The chunks a window of a level needs, of the bands given (each band of a tile its own chunk +/// when they are stored apart, else one chunk a tile). +fn chunks_of(info: &Info, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize]) -> HashSet { + let mut chunks = HashSet::new(); + let Some(l) = info.layout.levels.get(level) else { return chunks }; + let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); + for ty in rows.0 / th..rows.1.min(l.height).div_ceil(th) { + for tx in cols.0 / tw..cols.1.min(l.width).div_ceil(tw) { + let tile = ty * l.across() + tx; + for &b in bands { + if let Some((c, _)) = info.layout.chunk(l, tile, b) { + chunks.insert(c); + } + } + } + } + chunks +} + +/// Whether a window of a level (see [`window`]) fits the tiles kept: at most half of them, so the +/// tiles of the level before stay while its own arrive. One that doesn't is read from a coarser +/// level. +pub fn fits(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize]) -> bool { + let Ok(Some(info)) = super::info(split(url).0) else { return true }; + let Some(l) = info.layout.levels.get(level) else { return false }; + let per_chunk = (l.tile_width as usize) * (l.tile_height as usize) * info.layout.chunk_bands(); + let n = chunks_of(&info, level, cols, rows, bands).len(); + ((cols.1.saturating_sub(cols.0)) as usize).saturating_mul(rows.1.saturating_sub(rows.0) as usize) <= WINDOW_SAMPLES + && n.saturating_mul(per_chunk) <= BUDGET / 2 +} + +/// The part of a level in view, as [`window`] has it now. +#[derive(Debug, Default)] +pub struct Window { + /// Its pixels (NaN where a tile hasn't arrived); `None` before the file's header has. + pub raster: Option, + /// Tiles (or the header) are still on their way: the map draws again as they land. + pub pending: bool, +} + +/// Tiles whose read failed lately, and when: not asked for again until [`RETRY`] has passed, so a +/// map drawn again and again doesn't ask for them each time. +fn failed() -> &'static Mutex> { + static F: OnceLock>> = OnceLock::new(); + F.get_or_init(Mutex::default) +} + +/// The most failed tiles remembered. +const MAX_FAILED: usize = 4096; + +/// The columns `cols` and rows `rows` (half-open, in the level's pixels) of level `level` of a +/// file, bands `bands` (0-based, in that order, as the raster's bands), coloured as `color` says: +/// from the tiles kept, without waiting. Those missing are fetched (each once at a time); the +/// tile cache's generation goes up as each lands, so the map is drawn again. +pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize], color: ColorHint) -> Result { + let file = split(url).0; + let Some(info) = super::info(file)? else { return Ok(Window { raster: None, pending: true }) }; + let l = info.layout.levels.get(level).ok_or("no such overview")?; + let (c0, c1) = (cols.0.min(l.width), cols.1.min(l.width)); + let (r0, r1) = (rows.0.min(l.height), rows.1.min(l.height)); + if c0 >= c1 || r0 >= r1 || bands.is_empty() { + return Ok(Window::default()); + } + if ((c1 - c0) as usize).saturating_mul((r1 - r0) as usize) > WINDOW_SAMPLES { + return Err("the view is too large a part of the GeoTIFF to read at once".into()); + } + if let Some(b) = bands.iter().find(|b| **b >= info.layout.bands.max(1)) { + return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b + 1)); + } + // The chunks the window needs: those kept (marked as used, so they stay), and the others + // asked for. + let chunks = chunks_of(&info, level, (c0, c1), (r0, r1), bands); + let mut kept: HashMap = HashMap::new(); + let mut ask = Vec::new(); + { + let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + for &c in &chunks { + let key = (file.to_string(), level, c); + match t.used(&key) { + Some(b) => { + kept.insert(c, b); + } + None if t.map.contains_key(&key) => {} + None => ask.push(c), + } + } + } + let mut pending = kept.len() < chunks.len(); + { + let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); + f.retain(|_, at| at.elapsed() < RETRY); + ask.retain(|c| !f.contains_key(&(file.to_string(), level, *c))); + } + // Tiles that failed lately aren't waited for (the map would wait for them for nothing). + if ask.is_empty() && pending { + let t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + pending = chunks.iter().any(|c| matches!(t.map.get(&(file.to_string(), level, *c)), Some(TileState::Loading(_)))); + } + for c in ask { + let key = (file.to_string(), level, c); + get_tile( + info.clone(), + file.to_string(), + level, + c, + Box::new(move |r| { + if r.is_err() { + let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); + if f.len() < MAX_FAILED { + f.insert(key, Instant::now()); + } + } + crate::cache::notify(); + }), + ); + pending = true; + } + let (mut data, missing) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); + // Until they arrive, its missing tiles show the levels kept (the one zoomed from), not holes. + if missing { + let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + stand_in(&mut t, &info, file, level, (c0, c1), (r0, r1), bands, &mut data); + } + let t = info.layout.transform_of(l); + let transform = rustgis_data::raster::GeoTransform { + origin_x: t.origin_x + f64::from(c0) * t.pixel_width, + origin_y: t.origin_y + f64::from(r0) * t.pixel_height, + ..t + }; + let raster = Raster::unmeasured( + file_name(file), + c1 - c0, + r1 - r0, + data, + transform, + info.layout.crs, + info.layout.nodata, + color, + info.layout.data_type, + ) + .map_err(|e| e.to_string())?; + Ok(Window { raster: Some(raster), pending }) +} + +/// Fills the pixels of a window of `level` that have no value (NaN) from the other levels' tiles +/// kept, the nearest first (the finer, then the coarser, then further): the level zoomed from +/// stands in for the one zoomed to until its tiles arrive. Nothing is fetched. +#[allow(clippy::too_many_arguments)] +fn stand_in( + t: &mut Tiles, + info: &Info, + file: &str, + level: usize, + cols: (u32, u32), + rows: (u32, u32), + bands: &[usize], + data: &mut [Vec], +) { + let levels = &info.layout.levels; + let Some(l) = levels.get(level) else { return }; + let ww = (cols.1 - cols.0) as usize; + // The other levels, nearest first. + let others: Vec = + (1..levels.len()).flat_map(|d| [level.checked_sub(d), Some(level + d)]).flatten().filter(|o| *o < levels.len()).collect(); + for o in others { + let Some(ol) = levels.get(o) else { continue }; + let (sx, sy) = (f64::from(ol.width) / f64::from(l.width.max(1)), f64::from(ol.height) / f64::from(l.height.max(1))); + let (tw, th) = (ol.tile_width.max(1), ol.tile_height.max(1)); + // The chunks of that level found so far (none: not kept). + let mut found: HashMap> = HashMap::new(); + let mut left = false; + for (k, &b) in bands.iter().enumerate() { + let Some(dst) = data.get_mut(k) else { continue }; + // The chunk the last pixel came from: its neighbours mostly share it. + let mut last: Option<(u32, Option)> = None; + for (i, v) in dst.iter_mut().enumerate().filter(|(_, v)| v.is_nan()) { + let (x, y) = (cols.0 as usize + i % ww, rows.0 as usize + i / ww); + // The pixel of that level under this one's centre. + let ox = (((x as f64 + 0.5) * sx) as u32).min(ol.width.saturating_sub(1)); + let oy = (((y as f64 + 0.5) * sy) as u32).min(ol.height.saturating_sub(1)); + let tile = (oy / th) * ol.across() + ox / tw; + let Some((c, at)) = info.layout.chunk(ol, tile, b) else { continue }; + if last.as_ref().is_none_or(|(lc, _)| *lc != c) { + let chunk = found.entry(c).or_insert_with(|| t.used(&(file.to_string(), o, c))).clone(); + last = Some((c, chunk)); + } + let value = last + .as_ref() + .and_then(|(_, ch)| ch.as_ref()) + .and_then(|ch| ch.get(at)) + .and_then(|p| p.get(((oy % th) * tw + ox % tw) as usize)); + match value { + Some(s) => *v = *s, + None => left = true, + } + } + } + if !left { + return; + } + } +} + +/// The pixels of columns `cols` and rows `rows` (half-open) of a level, bands `bands` in that +/// order, each row by row, from the chunks `chunk` gives (NaN where it gives none), and whether +/// any was missing. +fn assemble( + info: &Info, + level: usize, + cols: (u32, u32), + rows: (u32, u32), + bands: &[usize], + chunk: impl Fn(u32) -> Option, +) -> (Vec>, bool) { + let (ww, wh) = ((cols.1 - cols.0) as usize, (rows.1 - rows.0) as usize); + let mut out = vec![vec![f32::NAN; ww * wh]; bands.len()]; + let Some(l) = info.layout.levels.get(level) else { return (out, true) }; + let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); + let mut missing = false; + for ty in rows.0 / th..rows.1.div_ceil(th) { + for tx in cols.0 / tw..cols.1.div_ceil(tw) { + let tile = ty * l.across() + tx; + // The tile's part of the window, in the level's pixels. + let (x0, x1) = ((tx * tw).max(cols.0), ((tx + 1) * tw).min(cols.1)); + let (y0, y1) = ((ty * th).max(rows.0), ((ty + 1) * th).min(rows.1)); + for (k, &b) in bands.iter().enumerate() { + let Some((c, at)) = info.layout.chunk(l, tile, b) else { continue }; + let Some(data) = chunk(c) else { + missing = true; + continue; + }; + let (Some(plane), Some(dst)) = (data.get(at), out.get_mut(k)) else { continue }; + for y in y0..y1 { + let src = ((y - ty * th) * tw + (x0 - tx * tw)) as usize; + let to = (y - rows.0) as usize * ww + (x0 - cols.0) as usize; + let n = (x1 - x0) as usize; + if let (Some(s), Some(d)) = (plane.get(src..src + n), dst.get_mut(to..to + n)) { + d.copy_from_slice(s); + } + } + } + } + } + (out, missing) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_from_addresses() { + assert_eq!(file_name("https://x.org/a/scene.tif?sig=1"), "scene.tif"); + assert_eq!(file_name("https://api.maxar.com/analytics/v1/raster/ortho/geotiff?function=f"), "geotiff"); + assert_eq!(file_name("https://x.org/"), "x.org"); + } +} diff --git a/crates/ui-egui/src/mapview.rs b/crates/ui-egui/src/mapview.rs index 6de26997c..be1686fd6 100644 --- a/crates/ui-egui/src/mapview.rs +++ b/crates/ui-egui/src/mapview.rs @@ -768,7 +768,12 @@ fn main_view(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { let mut key = key; // A scene whose tile layers are all live places them for every frame: tiles arriving need no // new scene (#635). - if gpu_ok && cam.bearing == 0.0 && cam.pitch == 0.0 && app.map.live_rev == Some(app.session.revision) { + if gpu_ok + && cam.bearing == 0.0 + && cam.pitch == 0.0 + && app.map.live_rev == Some(app.session.revision) + && !reads_in_place(app.session.map()) + { key.tiles = 0; } else if fetches_tiles(app.session.map()) { key.tiles = rustgis_engine::render::rustgis_tiles::cache::generation(); @@ -1237,6 +1242,13 @@ fn drape_state(app: &RustGisApp) -> (crate::globe_drape::DrapeKey, u64) { fn fetches_tiles(m: &rustgis_engine::map::Map) -> bool { !m.tiles.is_empty() || m.view.terrain.as_ref().is_some_and(|t| matches!(t.source, rustgis_engine::map::view::TerrainSource::Tiles { .. })) + || reads_in_place(m) +} + +/// Whether the map has raster layers read in place: the part in view arrives a tile at a time, +/// through the tile cache, and the map is drawn again as it does. +fn reads_in_place(m: &rustgis_engine::map::Map) -> bool { + m.layers.iter().any(|l| l.raster.as_ref().is_some_and(|r| r.in_place)) } /// Uploads a composed GPU scene, whether built in place or by the frame worker. diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index c03209f01..acbd65e2e 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -5930,8 +5930,31 @@ fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens ("South", format!("{:.4}°", e.min_y)), ("North", format!("{:.4}°", e.max_y)), ]; - if let Some((w, h)) = d.overview { - rows.insert(1, ("Full size", format!("{w} × {h}"))); + match d.overview { + // Read in place, the map draws the full image: the overview is what the layer holds. + Some((w, h)) if rl.in_place => { + let (sx, sy) = (f64::from(d.width) / f64::from(w.max(1)), f64::from(d.height) / f64::from(h.max(1))); + let f = rustgis_engine::map::symbology::fmt_num; + if let Some(r) = rows.get_mut(0) { + *r = ("Size", format!("{w} × {h}")); + } + if let Some(r) = rows.get_mut(3) { + *r = ("Cell size", format!("{} × {}{unit}", f(cx * sx), f(cy * sy))); + } + rows.insert( + 1, + ( + "Read", + crate::i18n::fmt( + "In place, holding a {w} × {h} overview", + &[("w", &d.width.to_string()), ("h", &d.height.to_string())], + ), + ), + ); + } + Some((w, h)) => rows.insert(1, ("Full size", format!("{w} × {h}"))), + None if rl.in_place => rows.insert(1, ("Read", tl!("In place").to_string())), + None => {} } let value_w = (ui.available_width() - 140.0).max(80.0); egui::Grid::new("raster_details").num_columns(2).striped(true).max_col_width(value_w).spacing(vec2(12.0, 4.0)).show(ui, |ui| { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index 5ed03ad83..187cdd7d8 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -1928,13 +1928,16 @@ fn data_from_a_web_address_lands_later() { assert!(h.state().ui.downloads.is_empty()); } -/// Add Data From URL reads a cloud-optimised GeoTIFF in place from an address that names no file -/// (an API such as Vantor's `…/geotiff?…`): its first bytes are probed off the frame, then the -/// layer reads its tiles by range, never downloading it whole. -#[test] -fn an_unnamed_cog_address_is_read_in_place() { +/// A local server answering with a small cloud-optimised GeoTIFF by range, its HEAD answers +/// announcing the TIFF header's length as Vantor's Transform API does: its address, the bytes it +/// has sent and the file's length. +fn serve_cog() -> (std::net::SocketAddr, std::sync::Arc, usize) { + serve_cog_bytes(rustgis_engine::data::raster::cog::example(2048, 256)) +} + +/// [`serve_cog`] serving `body`. +fn serve_cog_bytes(body: Vec) -> (std::net::SocketAddr, std::sync::Arc, usize) { use std::io::{Read, Write}; - let body = rustgis_engine::data::raster::cog::example(2048, 256); let total = body.len(); let sent = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let count = sent.clone(); @@ -1946,7 +1949,6 @@ fn an_unnamed_cog_address_is_read_in_place() { let mut buf = [0u8; 4096]; let read = s.read(&mut buf).unwrap_or(0); let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); - // Its headers announce the TIFF header's length, as Vantor's Transform API does. if req.starts_with("head ") { let _ = write!( s, @@ -1968,6 +1970,41 @@ fn an_unnamed_cog_address_is_read_in_place() { let _ = s.write_all(part); } }); + (addr, sent, total) +} + +/// A cloud-optimised GeoTIFF read in place is a raster layer like a file's: the Symbology pane +/// shows it with the same raster controls (its renderer, its bands), not COG ones of its own. +#[test] +fn a_cog_read_in_place_uses_the_raster_symbology_pane() { + use rustgis_engine::data::raster::cog::{Example, example_with}; + let (addr, _, _) = serve_cog_bytes(example_with(&Example { size: 512, tile: 256, bands: 8, ..Example::default() })); + let mut h = harness(); + h.state_mut().run("ui.language", json!({"code": "en"})).unwrap(); + h.state_mut().run("project.new", json!(null)).unwrap(); + let r = h.state_mut().run("layer.addUrl", json!({"url": format!("http://{addr}/multispectral.tif"), "name": "Multispectral"})).unwrap(); + assert_eq!(r["inPlace"], true, "{r}"); + assert!(h.state().session.map().tiles.is_empty(), "a raster layer, not a tile layer"); + h.state_mut().run("raster.bands", json!({"layer": "Multispectral", "red": 5, "green": 3, "blue": 2})).unwrap(); + h.state_mut().run("ui.pane.symbology", json!(null)).unwrap(); + for _ in 0..3 { + h.step(); + } + assert_eq!(h.state().ui.right, Some(rustgis_ui_egui::RightPane::Symbology)); + h.get_by_label("Primary symbology"); + // The band pickers a file raster's RGB symbology has, on the bands chosen. + for colour in ["Red", "Green", "Blue"] { + assert!(h.query_by_label(colour).is_some(), "{colour}"); + } + assert!(h.query_by_label("The raster's data couldn't be read.").is_none()); +} + +/// Add Data From URL reads a cloud-optimised GeoTIFF in place from an address that names no file +/// (an API such as Vantor's `…/geotiff?…`): its first bytes are probed off the frame, then the +/// layer reads its tiles by range, never downloading it whole. +#[test] +fn an_unnamed_cog_address_is_read_in_place() { + let (addr, sent, total) = serve_cog(); let mut h = harness(); h.step(); let url = format!("http://{addr}/analytics/v1/raster/ortho-with-overviews/geotiff?function=pansharp_ortho&key=k"); @@ -1975,13 +2012,13 @@ fn an_unnamed_cog_address_is_read_in_place() { assert!(h.state().ui.downloads.is_empty(), "probed first, not downloaded"); for _ in 0..200 { h.step(); - if h.state().session.map().tiles.iter().any(|t| t.name == "Ortho") { + if h.state().session.map().layers.iter().any(|l| l.name == "Ortho") { break; } std::thread::sleep(std::time::Duration::from_millis(20)); } - let t = h.state().session.map().tiles.iter().find(|t| t.name == "Ortho").cloned(); - assert!(t.is_some_and(|t| t.url == url), "added as a tile layer read in place"); + let t = h.state().session.map().layers.iter().find(|l| l.name == "Ortho").and_then(|l| l.raster.clone()); + assert!(t.is_some_and(|r| r.in_place && r.path.as_deref() == Some(url.as_str())), "added as a raster layer read in place"); assert!(h.state().ui.downloads.is_empty() && h.state().ui.cog_probes.is_empty()); let read = sent.load(std::sync::atomic::Ordering::Relaxed); assert!(read < total / 4, "{read} of {total} bytes read"); @@ -1991,13 +2028,13 @@ fn an_unnamed_cog_address_is_read_in_place() { assert!(h.state().ui.downloads.is_empty() && h.state().ui.cog_probes.len() == 1); for _ in 0..200 { h.step(); - if h.state().session.map().tiles.iter().any(|t| t.name == "Null") { + if h.state().session.map().layers.iter().any(|l| l.name == "Null") { break; } std::thread::sleep(std::time::Duration::from_millis(20)); } - let t = h.state().session.map().tiles.iter().find(|t| t.name == "Null").cloned(); - assert!(t.is_some_and(|t| t.url == null_url), "added as a tile layer read in place"); + let t = h.state().session.map().layers.iter().find(|l| l.name == "Null").and_then(|l| l.raster.clone()); + assert!(t.is_some_and(|r| r.in_place && r.path.as_deref() == Some(null_url.as_str())), "added as a raster layer read in place"); // A probe under way when another project opens is dropped: its layer never lands there. h.state_mut().run("ui.addUrl", json!({"url": format!("{url}&n=3"), "name": "Late"})).unwrap(); h.state_mut().run("project.new", json!({})).unwrap(); @@ -2006,7 +2043,7 @@ fn an_unnamed_cog_address_is_read_in_place() { h.step(); std::thread::sleep(std::time::Duration::from_millis(20)); } - assert!(!h.state().session.map().tiles.iter().any(|t| t.name == "Late")); + assert!(!h.state().session.map().layers.iter().any(|l| l.name == "Late")); } /// A service layer added from the app (Add From Service, the Connections tree) is read past its diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 0f49cf4f8..6e28b42e1 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -100,15 +100,33 @@ unresponsive. The original JPEG 2000 byte buffer is released once it has been st before GDAL runs or the GeoTIFF is read back. The converted GeoTIFF and decoded raster still occupy memory together; the 512 MiB file limit is not a limit on total process memory. -A tiled GeoTIFF is read in place rather than downloaded: its header first, then, by HTTP range -requests, the tiles of the overview each part of the view needs. Overviews make zoomed-out reads -smaller. It is a tile layer (`map.addTiles` kind `cog`, saved by its address), drawn under the map's -own layers: RGB(A) 8-bit images in colour, other images stretched between the 2nd and 98th -percentiles of their coarsest overview. Planar files (bands stored apart), signed-byte data and -bottom-up images work too. `map.setTiles` selects one or three `"bands"` (from 1) and a fixed -`"range"`; see [Satellite embeddings](web-data.md#satellite-embeddings). -A GeoTIFF in strips, a rotated image or an unsupported tiled image downloads whole as a raster layer, as does a -GeoTIFF added with `"whole": true` (for raster tools and symbology). The web app needs the server +A tiled GeoTIFF (a cloud-optimised one) is read in place rather than downloaded: its header +first, then, by HTTP range requests, only the parts the map needs. It is a raster layer like a +GeoTIFF file, saved by its address: the Symbology pane, the raster commands (`raster.symbology`, +`raster.bands`, `raster.stretch`, `raster.dra`, `raster.enhance`, `raster.mask`…), its legend, +identify and the raster tools work on it as on a file's. The layer holds an overview of the image +(every band, about two million values), from which its statistics, histograms and tools read, as a +file too large to hold whole is opened from an overview; zoomed in past it, the map reads the part +in view at the resolution it needs as it draws, and draws it with the layer's style and the +overview's statistics, so the image looks the same at every zoom. The Properties pane says the +image's full size and that it is read in place. + +Its first look is what the file's PhotometricInterpretation tag says: + +- **RGB:** red, green and blue, with the alpha band if there is one. Bands past the colour ones + (an RGB and near-infrared orthophoto, say) are read too. +- **Palette:** its colour table (Colormap). +- **Grey (MinIsBlack, as most remote sensing images are):** the first band stretched in grey, or + three bands as red, green and blue. An alpha band is shown with it. + +In an image with more bands (an 8-band multispectral image, say) you pick the combination in the +Symbology pane, as for any raster. GeoTIFF files follow the same rule. Planar files (bands stored +apart), 8- to 64-bit integer and floating-point data, signed-byte data and bottom-up images work +too. A dated series of GeoTIFFs (an address with date placeholders), a data cube's files and +AlphaEarth's embeddings are tile layers that follow the time slider or dequantise their values; +see [Satellite embeddings](web-data.md#satellite-embeddings). +A GeoTIFF in strips, a rotated image or an unsupported tiled image downloads whole, as does a +GeoTIFF added with `"whole": true` (for raster tools that need every pixel at full resolution). The web app needs the server to allow range requests (CORS with the `Range` header). A GeoParquet or FlatGeobuf file by its address is read in place too, a view at a time, however @@ -128,7 +146,7 @@ desktop, while in the web app they need the file downloaded first. On desktop, `layer.addUrl` waits for downloads, up to a minute; the From URL… dialog (`ui.addUrl`) and a plugin's `layer.addUrl` work in the background. In the web app the same dialog handles asynchronous downloads. -`layer.addUrl` can add PMTiles and COG tile layers directly, but an ordinary download returns +`layer.addUrl` can add PMTiles archives and GeoTIFFs read in place directly, but an ordinary download returns `{"pending": true, "url": "…"}` and is added later by the app. To open a remote GeoParquet or FlatGeobuf whose header hasn't been fetched yet, use the From URL… dialog or a `?data=` link. diff --git a/docs/user-guide/web-data.md b/docs/user-guide/web-data.md index d09d2b8d4..3ddf0e001 100644 --- a/docs/user-guide/web-data.md +++ b/docs/user-guide/web-data.md @@ -610,10 +610,11 @@ their footprints drawn as a layer. | **Tessera** (Cambridge; 128 values a 10 m pixel, 2017–2025) | **Add view** reads three bands (1, 2 and 3 by default) of the view from Tessera's [Zarr store](https://data.source.coop/tessera/tessera/zarr/v1.1-dclimate) in place (the shard's index, then the chunks the view needs: at most about 6 km a side, its centre when the view is larger), as a 32-bit raster in UTM. | | **Clay**, **Major TOM**, **Copernicus-Embed** | Listed with links to their data and papers; RustGIS doesn't read them. | -AlphaEarth's tiles keep their look in their layer's address (`#bands=2,17,10&range=-0.3,0.3&dequantize=aef`), -which `map.setTiles` changes for any cloud-optimised GeoTIFF read in place (`"bands"`, `"range"`, -`"dequantize"`); GeoTIFFs with their bands stored apart (planar), signed bytes and bottom-up images -are read in place too. The commands: `embeddings.datasets`, `embeddings.search` (`"dataset"`, +AlphaEarth's tiles are tile layers that keep their look in their address +(`#bands=2,17,10&range=-0.3,0.3&dequantize=aef`), which `map.setTiles` changes (`"bands"`, +`"range"`, `"dequantize"`); other cloud-optimised GeoTIFFs are raster layers read in place, styled +in the Symbology pane. GeoTIFFs with their bands stored apart (planar), signed bytes and bottom-up +images are read in place too. The commands: `embeddings.datasets`, `embeddings.search` (`"dataset"`, `"year"`, `"bbox"` or the view, `"footprints"`) and `embeddings.add` (`"dataset"`, `"url"`, `"bands"`, `"stretch"`, `"dimension"`, `"bbox"`, `"year"`): From d44edc180751ed3c1848028c586f518c5690282b Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 17:24:49 -0330 Subject: [PATCH 02/27] fix(raster): address review of COGs read in place; wait for Retry-After on 429 and 503 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Hold the coarsest level as the overview only when it fits half the tile cache in 4,096 tiles; refuse a larger image without overviews instead of reading it whole. - Stand-ins fill only the tiles still missing; a tile's own no-data pixels stay empty. - Keep an address's look (#bands=…&range=…) until the pixels land (browser, migrated projects); a palette image stretched by a range is drawn in grey. - Read a project's and a GeoLibre import's in-place overviews together, once each; migrate COG tile layers before the tile list is cut. - GeoTIFF files read every band, as COGs do; a palette image without its colour table says so. - Scale alpha by its pixel type in both drawing paths; blend a see-through layer once while its tiles load; read a wrapped 0–360° view's full width; keep the alpha band when a colour uses it. - Raster to PMTiles waits for in-place tiles (two minutes at most) or refuses, never writing the overview in their place. - A failed read waits a minute, or what its server's Retry-After says on a 429 or 503 (seconds or an HTTP date, within 1 s to 1 h); the map draws again when the wait is over. Co-Authored-By: Claude Opus 5.5 --- apps/rustgis-web/src/web.rs | 17 ++- crates/data/src/raster/cog.rs | 72 ++++++++++- crates/data/src/raster/geotiff.rs | 43 +++++-- crates/data/src/raster/mod.rs | 17 +++ crates/engine/src/cmd/geolibre.rs | 2 + crates/engine/src/cmd/layer.rs | 12 +- crates/engine/src/cmd/raster.rs | 4 + crates/engine/src/cmd/tiles.rs | 32 ++--- crates/engine/src/project.rs | 31 ++++- crates/engine/src/tests.rs | 29 ++++- crates/map/src/raster.rs | 200 +++++++++++++++++++++++------ crates/render/src/raster.rs | 41 ++++-- crates/render/src/raster_export.rs | 31 ++++- crates/tiles/src/cache.rs | 100 ++++++++++++++- crates/tiles/src/cog.rs | 8 +- crates/tiles/src/cog/raster.rs | 191 +++++++++++++++++++++------ crates/tiles/src/failure.rs | 80 +++++++++++- crates/tiles/src/pmtiles.rs | 4 +- crates/ui-egui/src/mapview.rs | 3 + crates/ui-egui/src/panes.rs | 23 ++-- docs/user-guide/analysis.md | 5 +- docs/user-guide/data-formats.md | 16 ++- 22 files changed, 787 insertions(+), 174 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index fe18cec36..4ff197400 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -574,6 +574,17 @@ async fn unless_stalled( /// `rustgis_tiles::cache::split_range`). /// A status in `omit` is an expected answer (a tile past a map's edge): its failure makes no /// record of its own, as on the desktop. +/// A refusal's text (`HTTP 503`), with how long the server asked to be left when it says so in a +/// `Retry-After` header the browser lets the page read (its CORS rules must expose it). +fn refused(resp: &web_sys::Response) -> String { + let wait = resp.headers().get("retry-after").ok().flatten(); + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::failure::with_retry_after( + format!("HTTP {}", resp.status()), + resp.status(), + wait.as_deref(), + ) +} + async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Result, String> { use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "GET", capture); @@ -656,7 +667,7 @@ async fn fetch_bytes_inner( .map_err(err)?; diagnostics.answered(resp.status()); if !resp.ok() { - return Err(format!("HTTP {}", resp.status())); + return Err(refused(&resp)); } // The last bytes asked for and the whole file answered: fine when it is no longer. let whole_tail = tail.filter(|_| resp.status() != 206); @@ -756,7 +767,7 @@ async fn post_json_inner( .map_err(err)?; diagnostics.answered(resp.status()); if !resp.ok() { - return Err(format!("HTTP {}", resp.status())); + return Err(refused(&resp)); } let buf = wasm_bindgen_futures::JsFuture::from(resp.array_buffer().map_err(err)?).await.map_err(err)?; let bytes = js_sys::Uint8Array::new(&buf); @@ -803,7 +814,7 @@ async fn head_inner( .map_err(err)?; diagnostics.answered(resp.status()); if !resp.ok() { - return Err(format!("HTTP {}", resp.status())); + return Err(refused(&resp)); } let header = |k: &str| resp.headers().get(k).ok().flatten(); Ok(cache::Head { diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 5621df3d7..23938665b 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -223,6 +223,8 @@ pub struct Layout { /// The bands are stored apart (planar configuration 2): each tile's bands are chunks of their /// own, band `b` of tile `i` at chunk `b × tiles + i`, so a band is read without the others. pub planar: bool, + /// A palette image whose colour table is missing or short: drawn in grey instead. + pub palette_missing: bool, } impl Layout { @@ -309,16 +311,26 @@ pub fn layout(bytes: &Sparse) -> Result { } fn read_layout(r: Sparse) -> Result { - let reader = super::geotiff::reader_all_bands(r).map_err(err)?; + let reader = super::geotiff::reader(r).map_err(err)?; let (palette, wide_rgb) = (reader.palette, reader.wide_rgb); let mut dec = Decoder::new(reader).map_err(err)?.with_limits(limits()); let Some(geo) = super::geotiff::cog_geo(&mut dec)? else { return Ok(Parse::NotTiled("a rotated image, or one without coordinates, is read whole".into())); }; let ct = dec.colortype().map_err(err)?; + let mut palette_missing = false; let (bands, color) = match ct { - // A palette image (shown to the decoder as grey) is coloured by its table. - ColorType::Gray(bits) if palette => (1, super::geotiff::palette_table(&mut dec, bits).map_or(ColorHint::Gray, ColorHint::Palette)), + // A palette image (shown to the decoder as grey) is coloured by its table, else grey. + ColorType::Gray(bits) if palette => ( + 1, + super::geotiff::palette_table(&mut dec, bits).map_or_else( + || { + palette_missing = true; + ColorHint::Gray + }, + ColorHint::Palette, + ), + ), ColorType::Gray(_) => (1, ColorHint::Gray), ColorType::GrayA(_) => (2, ColorHint::GrayAlpha), ColorType::RGB(_) => (3, ColorHint::Rgb { alpha: false }), @@ -396,7 +408,17 @@ fn read_layout(r: Sparse) -> Result { if levels.is_empty() { return Ok(Parse::NotTiled("no image".into())); } - Ok(Parse::Done(Box::new(Layout { levels, crs: geo.0, transform: geo.1, nodata: geo.2, bands, color, data_type, planar }))) + Ok(Parse::Done(Box::new(Layout { + levels, + crs: geo.0, + transform: geo.1, + nodata: geo.2, + bands, + color, + data_type, + planar, + palette_missing, + }))) } /// Decodes one chunk (a tile, or one band of a tile when the bands are stored apart, see @@ -406,7 +428,7 @@ pub fn decode_tile(header: &Sparse, layout: &Layout, level: usize, index: u32, b let l = layout.levels.get(level).ok_or_else(|| err("no such level"))?; let at = *l.offsets.get(index as usize).ok_or_else(|| err("no such tile"))?; let r = header.clone().with(at, bytes).reader(); - let mut dec = Decoder::new(super::geotiff::reader_all_bands(r).map_err(err)?).map_err(err)?.with_limits(limits()); + let mut dec = Decoder::new(super::geotiff::reader(r).map_err(err)?).map_err(err)?.with_limits(limits()); dec.seek_to_image(l.ifd).map_err(err)?; let values = match super::geotiff::read_chunk(&mut dec, index)? { DecodingResult::U8(v) => v.into_iter().map(f32::from).collect::>(), @@ -835,5 +857,45 @@ pub(crate) mod tests { let (l, v) = pixel(&example_with(&Example { photometric: 3, ..e }), 150, 150); let ColorHint::Palette(table) = &l.color else { panic!("{:?}", l.color) }; assert_eq!((table.len(), table[44], v), (256, [44, 211, 0, 255], vec![want(0)])); + assert!(!l.palette_missing); + } + + /// A GeoTIFF file is read as the same image read in place is: every band, whatever its + /// PhotometricInterpretation says, coloured alike. + #[test] + fn files_have_every_band_a_cog_has() { + let e = Example { size: 300, tile: 128, ..Example::default() }; + for (bands, alpha, photometric) in [(8, false, 1), (2, true, 1), (4, true, 2), (4, false, 2), (7, false, 2), (5, true, 2)] { + for planar in [false, true] { + let file = example_with(&Example { bands, alpha, planar, photometric, ..e }); + let (l, v) = pixel(&file, 150, 150); + let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "f").unwrap(); + let read: Vec = (0..r.band_count()).map(|b| r.value(b, 150, 150).unwrap()).collect(); + assert_eq!((r.band_count(), &r.color, read), (l.bands, &l.color, v), "{bands} bands, photometric {photometric}"); + } + } + } + + /// A palette image whose colour table is missing is drawn in grey, and says so, read in place + /// or as a file. + #[test] + fn a_palette_without_its_table_says_so() { + let mut file = example_with(&Example { size: 300, tile: 128, photometric: 3, ..Example::default() }); + // The ColorMap entries (tag 320, of shorts) renamed to a tag nothing reads. + let mut renamed = 0; + for i in 0..file.len().saturating_sub(4) { + if file[i..i + 4] == [0x40, 0x01, 0x03, 0x00] { + file[i..i + 2].copy_from_slice(&[0xFF, 0xFE]); + renamed += 1; + } + } + assert!(renamed > 0); + let (l, _) = pixel(&file, 150, 150); + assert_eq!((&l.color, l.palette_missing), (&ColorHint::Gray, true)); + let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "p").unwrap(); + assert_eq!((&r.color, r.palette_missing), (&ColorHint::Gray, true)); + let clean = + crate::raster::geotiff::read(std::io::Cursor::new(example_with(&Example { photometric: 3, ..Example::default() })), "p"); + assert!(!clean.unwrap().palette_missing); } } diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index bb35e9cd7..fdb8536d6 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -578,7 +578,7 @@ fn samples(dec: &mut Decoder) -> usize { /// decodes their colour indices as grey samples, and the colour table is read from the ColorMap tag /// as it is. The only change is the PhotometricInterpretation value (3 → 1) in each image's IFD. /// -/// Made by [`reader_all_bands`], it also shows RGB images with samples past the colour ones (RGB +/// It also shows RGB images with samples past the colour ones (RGB /// and near-infrared, or more bands, not declared alpha) as grey ones (2 → 1): the crate decodes /// only an RGB image's colour samples, and a fifth or later one not at all, but decodes every /// sample of a grey image. @@ -704,14 +704,9 @@ impl ReadSeek for T {} /// What the decoder reads from. pub(crate) type Reader<'a> = Unpaletted>; -/// A reader for the decoder. -pub(crate) fn reader<'a>(r: impl Read + Seek + 'a) -> std::io::Result> { - Unpaletted::new(Box::new(r), false) -} - -/// A reader for the decoder that decodes every band of an RGB image with more (see +/// A reader for the decoder: every band of the image, an RGB image's with more too (see /// [`Unpaletted`]). -pub(crate) fn reader_all_bands<'a>(r: impl Read + Seek + 'a) -> std::io::Result> { +pub(crate) fn reader<'a>(r: impl Read + Seek + 'a) -> std::io::Result> { Unpaletted::new(Box::new(r), true) } @@ -1049,7 +1044,7 @@ pub(super) fn read_limited_with_nodata( open: Option>, sidecar_nodata: Option, ) -> Result { - let palette = r.palette; + let (palette, wide_rgb) = (r.palette, r.wide_rgb); let limits = { let mut l = Limits::default(); l.decoding_buffer_size = MAX_SAMPLES.saturating_mul(4); @@ -1137,11 +1132,14 @@ pub(super) fn read_limited_with_nodata( let ct = dec.colortype().map_err(err)?; // JPEG-compressed YCbCr, the only YCbCr read (the colour match below rejects the rest). let ycbcr = matches!(ct, ColorType::YCbCr(_)) && dec.find_tag_unsigned::(Tag::Compression).ok().flatten() == Some(7); - // A palette image (shown to the decoder as grey) is coloured by its table. + // A palette image (shown to the decoder as grey) is coloured by its table (in grey without one, + // as said below). let ct = match ct { - ColorType::Gray(bits) if palette && dec.find_tag(Tag::ColorMap).ok().flatten().is_some() => ColorType::Palette(bits), + ColorType::Gray(bits) if palette => ColorType::Palette(bits), other => other, }; + // A palette image without a whole colour table is drawn in grey, and says so. + let mut no_table = false; let (nbands, color) = match ct { ColorType::Gray(_) => (1, ColorHint::Gray), ColorType::GrayA(_) => (2, ColorHint::GrayAlpha), @@ -1150,7 +1148,21 @@ pub(super) fn read_limited_with_nodata( // conversion), so it is turned into RGB below (issue #692); other YCbCr data isn't read. ColorType::YCbCr(_) if ycbcr => (3, ColorHint::Rgb { alpha: false }), ColorType::RGBA(_) => (4, ColorHint::Rgb { alpha: true }), - ColorType::Palette(bits) => (1, palette_table(&mut dec, bits).map_or(ColorHint::Gray, ColorHint::Palette)), + ColorType::Palette(bits) => ( + 1, + palette_table(&mut dec, bits).map_or_else( + || { + no_table = true; + ColorHint::Gray + }, + ColorHint::Palette, + ), + ), + // An RGB image with more bands (shown to the decoder as grey, see `Unpaletted`): every + // band, the first three red, green and blue. + ColorType::Multiband { num_samples, .. } if wide_rgb && num_samples >= 3 => { + (num_samples as usize, ColorHint::Rgb { alpha: extra_alpha(&mut dec) }) + } ColorType::Multiband { num_samples, .. } => (num_samples.max(1) as usize, multiband_color(&mut dec, num_samples)), other => return Err(err(format!("{other:?} images aren't supported"))), }; @@ -1240,6 +1252,7 @@ pub(super) fn read_limited_with_nodata( r.georeferenced = false; r.overview = overview; r.rounded = rounded; + r.palette_missing = no_table; if let Some(p) = &described { r.apply_pam(p); } @@ -1290,6 +1303,7 @@ pub(super) fn read_limited_with_nodata( let mut r = Raster::new(name, w, h, bands, transform, crs, geo.nodata, color, data_type)?; r.overview = overview; r.rounded = rounded; + r.palette_missing = no_table; if let Some(p) = &described { r.apply_pam(p); } @@ -1999,13 +2013,14 @@ mod tests { fn padded_tiles_with_an_extra_sample_stay_in_their_rows() { // Issue #162: an RGBI orthophoto (5000 pixels in 128-pixel tiles) opened with black // columns down its west edge, the padding of each row's last tile spilt into the next row. + // Its near-infrared band is read too, the fourth, after red, green and blue. let pixel = |b: u32, x: u32, y: u32| ((x + 3 * y + 60 * b) % 250 + 1) as u8; let bytes = rgbi_tiled(&[(40, 16), (20, 16)], pixel); let check = |r: &Raster, k: u32| { - assert_eq!((r.band_count(), r.color.clone()), (3, ColorHint::Rgb { alpha: false })); + assert_eq!((r.band_count(), r.color.clone()), (4, ColorHint::Rgb { alpha: false })); for y in 0..r.height { for x in 0..r.width { - for b in 0..3 { + for b in 0..4 { let want = f32::from(pixel(b, x * k, y * k)); assert_eq!(r.value(b as usize, i64::from(x), i64::from(y)), Some(want), "band {b} at {x},{y} (k {k})"); } diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index c2e5d3887..25dda64ff 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -351,6 +351,20 @@ impl Band { } } +/// An alpha band's opaque value for a pixel type (`UInt16`: 65535): the type's largest, 1 for +/// floating point (as TIFF keeps it), 255 for bytes and anything else. +pub fn alpha_max(data_type: &str) -> f32 { + match data_type { + "Int8" => 127.0, + "Int16" => 32767.0, + "UInt16" => 65535.0, + "Int32" => 2_147_483_647.0, + "UInt32" => 4_294_967_295.0, + "Float32" | "Float64" => 1.0, + _ => 255.0, + } +} + /// How a raster's bands were meant to be shown, from the file. #[derive(Clone, Debug, Default, PartialEq)] pub enum ColorHint { @@ -389,6 +403,8 @@ pub struct Raster { /// values with more digits than `f32` holds), by band; empty when none did. NaN and no-data /// values aren't counted. pub rounded: Vec, + /// A palette image whose colour table is missing or short: drawn in grey instead. + pub palette_missing: bool, transformer: Transformer, extent: Extent, } @@ -488,6 +504,7 @@ impl Raster { overview: None, georeferenced: true, rounded: Vec::new(), + palette_missing: false, transformer, extent: Extent::EMPTY, }; diff --git a/crates/engine/src/cmd/geolibre.rs b/crates/engine/src/cmd/geolibre.rs index 721e1a00e..f46dcae72 100644 --- a/crates/engine/src/cmd/geolibre.rs +++ b/crates/engine/src/cmd/geolibre.rs @@ -157,6 +157,8 @@ fn import_project(s: &mut Session, p: &Value) -> CmdResult { report.approximated(string(g, "name"), "nested folder flattened; inherited visibility and opacity retained"); } } + // Its GeoTIFFs read in place, read together rather than one after another. + crate::project::read_in_place(s.map_mut()); order.reverse(); s.map_mut().layers.sort_by_key(|l| order.iter().position(|id| *id == l.id).unwrap_or(usize::MAX)); s.request_only = false; diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 604f94053..5cfcb5e29 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3010,20 +3010,20 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) fn add_in_place(s: &mut Session, name: &str, url: &str, strict: bool) -> CmdResult { use crate::render::rustgis_tiles::cog; let (file, look) = cog::split(url); + // Strict, natively: read now, failing when it can't be. Otherwise read with the other layers + // added with it (see `project::read_in_place`); in the browser, landing later. #[cfg(not(target_arch = "wasm32"))] - let mut layer = if strict { + let layer = if strict { rustgis_map::raster::RasterLayer::in_place(cog::raster::overview_blocking(file)?, file) } else { - rustgis_map::raster::RasterLayer::opening_in_place(file) + rustgis_map::raster::RasterLayer::unread_in_place(file) }; #[cfg(target_arch = "wasm32")] - let mut layer = { + let layer = { let _ = strict; rustgis_map::raster::RasterLayer::opening_in_place(file) }; - if let Some(d) = layer.data.clone() { - layer.style = super::tiles::look_style(&look, &d).unwrap_or_else(|| layer.style.clone()); - } + let layer = layer.with_look(super::tiles::address_look(&look)); let mut r = super::raster::add_raster(s, name, layer)?; if let Ok(Some(info)) = cog::info(file) { let [w, so, e, n] = info.bounds; diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index 3a50a58e1..8d3fc3fbd 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -348,6 +348,7 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes } let (w, h, n, overview) = layer.data.as_ref().map(|d| (d.width, d.height, d.band_count(), d.overview)).unwrap_or_default(); let rounded = layer.data.as_deref().and_then(rounded_count); + let palette_missing = layer.data.as_deref().is_some_and(|d| d.palette_missing); // Read in place, the map shows the full image as it draws: the overview is only what's held. let overview = overview.filter(|_| !layer.in_place); let empty_before = s.map().layers.is_empty(); @@ -356,6 +357,9 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes if let Some((fw, fh)) = overview { s.notify(format!("{unique}: showing a {w} × {h} overview of the {fw} × {fh} raster"), false); } + if palette_missing { + s.notify(format!("{unique}: its colour table is missing or short, so it is drawn in grey"), false); + } if let Some(count) = rounded { let held = "pixels are held as 32-bit floats, exact for whole numbers up to 16,777,216 and to about 7 significant digits"; s.notify(format!("{unique}: {count} rounded as the file was read: {held}, so an export writes the rounded values"), false); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 4ab2b27c0..36bb00643 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -546,11 +546,9 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { } let t = m.tiles.remove(i); let (file, look) = cog::split(&t.url); - // Read now natively; in the browser its own look once it lands. - let mut rl = RasterLayer::opening_in_place(file); - if let Some(style) = rl.data.clone().and_then(|d| look_style(&look, &d)) { - rl.style = style; - } + // Read with the project's other rasters read in place (see `project::read_in_place`), in + // its look once it lands. + let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)); let id = m.add_raster(t.name.clone(), rl); if let Some(l) = m.layer_mut(id) { l.visible = t.visible; @@ -565,25 +563,11 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { } } -/// The raster style a GeoTIFF's look says (see [`Look`]): its bands, one grey or three as red, -/// green and blue, and its fixed range; `None` for the file's own look. A COG tile layer becomes -/// a raster layer read in place with it. -pub(crate) fn look_style(look: &Look, r: &rustgis_data::raster::Raster) -> Option { - use rustgis_map::raster::{RasterRenderer, RasterStyle, Stretch}; - if look.bands.is_empty() && look.range.is_none() { - return None; - } - let mut style = RasterStyle::default_for(r); - let n = r.band_count(); - match look.bands[..] { - [b] if b < n => style.renderer = RasterRenderer::Stretch { band: b, ramp: rustgis_map::Ramp::gray(), invert: false }, - [red, green, blue] if red < n && green < n && blue < n => style.renderer = RasterRenderer::Rgb { red, green, blue }, - _ => {} - } - if let Some((lo, hi)) = look.range { - style.stretch = Stretch::custom(f64::from(lo), f64::from(hi)); - } - Some(style) +/// The look a COG tile layer's address keeps (see [`Look`]) as a raster layer's: its bands, one +/// grey or three as red, green and blue, and its fixed range. A COG tile layer becomes a raster +/// layer read in place with it. +pub(crate) fn address_look(look: &Look) -> rustgis_map::raster::AddressLook { + rustgis_map::raster::AddressLook { bands: look.bands.clone(), range: look.range.map(|(lo, hi)| (f64::from(lo), f64::from(hi))) } } /// A change of a COG layer's look a command asks for (`bands`, `range`, `dequantize`), applied to diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 162fb3bb0..af17a4fb3 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -666,9 +666,10 @@ impl Project { } // Tile layers with an address that can't be fetched (a hand edit) are dropped. m.tiles.retain_mut(|t| t.repair().map_err(|e| log::warn!("{}: {e}", m.name)).is_ok()); - m.tiles.truncate(rustgis_map::tiles::MAX_TILE_LAYERS); - // GeoTIFFs read in place are raster layers (a project from before saved tile layers). + // GeoTIFFs read in place are raster layers (a project from before saved tile layers), + // so they don't count toward the tile layers kept. crate::cmd::tiles::cog_tiles_to_rasters(m); + m.tiles.truncate(rustgis_map::tiles::MAX_TILE_LAYERS); #[cfg(not(target_arch = "wasm32"))] for t in &mut m.tiles { if t.error.is_none() @@ -697,8 +698,10 @@ impl Project { src.error = Some(error); } } + // Read in place, read together below. if let Some(rl) = &mut l.raster && rl.data.is_none() + && !rl.in_place && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed) && unresolved.is_none() && let Err(e) = rl.reload() @@ -726,6 +729,7 @@ impl Project { } l.touch_geometry(); } + read_in_place(m); } } @@ -923,6 +927,29 @@ impl Project { } } +/// Reads the overviews of a map's raster layers read in place and not read yet, all at once (see +/// [`rustgis_map::raster::read_together`]): a project or an import of several waits for the +/// slowest, not for each in turn. One read from an S3 bucket waits for its connection (see +/// [`crate::s3_connections::resolve_project_s3`]). +pub(crate) fn read_in_place(m: &mut Map) { + let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m + .layers + .iter_mut() + .filter_map(|l| { + let name = l.name.clone(); + let rl = l.raster.as_mut()?; + (rl.in_place && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed)).then_some((name, rl)) + }) + .collect(); + let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); + let failed = rustgis_map::raster::read_together(&mut layers); + for (i, e) in failed { + if let Some((name, _)) = named.get(i) { + log::warn!("{name}: {e}"); + } + } +} + fn empty_map() -> &'static Map { static M: std::sync::OnceLock = std::sync::OnceLock::new(); M.get_or_init(|| Map::new("Map")) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index b8576c575..c6378cf7e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13150,6 +13150,32 @@ fn cog_windows_stand_in_the_level_kept_until_their_tiles_arrive() { assert_eq!(v(&part, 1001, 901), ((1001 + 901) % 256) as f32); } +/// A GeoTIFF's address look (`#bands=…&range=…`) is its layer's once its pixels land, read later +/// (as the browser and an import read them) as when read at once; a palette image stretched by a +/// range is drawn in grey, as its tile layer was, and without a look in its colours. +#[test] +fn an_address_look_holds_when_the_pixels_land_later() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{AddressLook, RasterLayer, RasterRenderer, Stretch, read_together}; + // A session installs the reader. + let _s = Session::new(Project::default()); + let (base, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, bands: 8, ..Example::default() })); + let look = AddressLook { bands: vec![4, 2, 1], range: Some((10.0, 200.0)) }; + let mut rl = RasterLayer::unread_in_place(format!("{base}/scene.tif")).with_look(look); + assert!(rl.data.is_none()); + assert!(read_together(&mut [&mut rl]).is_empty()); + assert!(rl.data.is_some()); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + let (classes, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, photometric: 3, ..Example::default() })); + let mut stretched = + RasterLayer::unread_in_place(format!("{classes}/classes.tif")).with_look(AddressLook { bands: vec![], range: Some((0.0, 100.0)) }); + let mut own = RasterLayer::unread_in_place(format!("{classes}/classes.tif")); + assert!(read_together(&mut [&mut stretched, &mut own]).is_empty()); + assert!(matches!(stretched.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", stretched.style.renderer); + assert_eq!(own.style.renderer, RasterRenderer::Colormap); +} + /// A project saved when cloud-optimised GeoTIFFs read in place were tile layers opens them as /// raster layers read in place, each where it was drawn, with its name, visibility, opacity and /// look; a dated series stays a tile layer. @@ -23849,7 +23875,8 @@ fn geolibre_extensionless_cog_is_a_raster_read_in_place() { let l = s.map().layers.iter().find(|l| l.name == "Coverage").unwrap(); let rl = l.raster.as_ref().unwrap(); assert!(rl.in_place && rl.data.is_none() && rl.error.is_some(), "{:?}", rl.error); - assert!((l.opacity() - 0.6).abs() < 1e-6 || (l.opacity() - 0.3).abs() < 1e-6, "{}", l.opacity()); + // Drawn at its own opacity in its folder's: 0.6 × 0.5. + assert!((s.map().layer_opacity(l) - 0.3).abs() < 1e-6, "{}", s.map().layer_opacity(l)); assert!(s.take_url_requests().is_empty()); } diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 7c6649f08..64a386ead 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -63,10 +63,74 @@ pub struct RasterLayer { /// The style is the file's own look once its pixels land (a layer added before they have). #[serde(skip)] restyle: bool, + /// The look its address gave a layer read in place, applied when its pixels land. + #[serde(skip)] + look: Option, } /// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. pub type InPlaceRead = Box) + Send>; +/// The longest a layer read in place waits for its overview natively. +#[cfg(not(target_arch = "wasm32"))] +const IN_PLACE_WAIT: std::time::Duration = std::time::Duration::from_secs(120); + +/// Reads the overviews of `layers` read in place and not read yet, all at once: a project of +/// several opens in the time of the slowest, not of them all (waited for natively, at most two +/// minutes in all; landing later in the browser). Other layers are left as they are. Returns each +/// failure, by its place in `layers`. +pub fn read_together(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { + let mut failed = Vec::new(); + let unread = |l: &RasterLayer| l.in_place && l.data.is_none() && l.opening.is_none() && l.time_zarr.is_none(); + #[cfg(not(target_arch = "wasm32"))] + let deadline = std::time::Instant::now() + IN_PLACE_WAIT; + #[cfg(not(target_arch = "wasm32"))] + let mut started = Vec::new(); + for (i, l) in layers.iter_mut().enumerate() { + if !unread(l) { + continue; + } + // Started without waiting: natively, each is waited for below, together. + #[cfg(not(target_arch = "wasm32"))] + { + let address = l.path.clone(); + let reader = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone(); + match (address, reader) { + (Some(address), Some(reader)) => { + l.error = None; + let slot = Arc::new(Mutex::new(None)); + let land = slot.clone(); + l.opening = Some(slot); + reader( + &address, + Box::new(move |result| { + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); + crate::source::landed(); + }), + ); + started.push(i); + } + _ => { + if let Err(e) = l.reload() { + failed.push((i, e)); + } + } + } + } + #[cfg(target_arch = "wasm32")] + if let Err(e) = l.reload() { + failed.push((i, e)); + } + } + #[cfg(not(target_arch = "wasm32"))] + for i in started { + if let Some(l) = layers.get_mut(i) + && let Err(e) = l.wait_for_read(deadline) + { + failed.push((i, e)); + } + } + failed +} type InPlaceReader = Arc; fn in_place_reader() -> &'static Mutex> { @@ -139,6 +203,7 @@ impl RasterLayer { masked: MaskedStats::default(), opening: None, restyle: false, + look: None, }) } @@ -160,6 +225,7 @@ impl RasterLayer { masked: MaskedStats::default(), opening: None, restyle: false, + look: None, } } @@ -181,6 +247,7 @@ impl RasterLayer { masked: MaskedStats::default(), opening: None, restyle: false, + look: None, } } @@ -189,10 +256,10 @@ impl RasterLayer { RasterLayer { in_place: true, ..RasterLayer::from_path(raster, address) } } - /// A cloud-optimised GeoTIFF read in place by its address, its overview still to be read - /// (where nothing may wait: the browser): drawn, in its own look, once it lands. - pub fn opening_in_place(address: impl Into) -> RasterLayer { - let mut layer = RasterLayer { + /// A cloud-optimised GeoTIFF read in place by its address, its overview not read yet: drawn, + /// in its own look, once [`RasterLayer::reload`] (or [`read_together`]) has read it. + pub fn unread_in_place(address: impl Into) -> RasterLayer { + RasterLayer { path: Some(address.into()), in_place: true, file_name: None, @@ -207,11 +274,32 @@ impl RasterLayer { masked: MaskedStats::default(), opening: None, restyle: true, - }; + look: None, + } + } + + /// [`RasterLayer::unread_in_place`], its overview read now (waited for natively; landing + /// later in the browser, where nothing may wait). + pub fn opening_in_place(address: impl Into) -> RasterLayer { + let mut layer = RasterLayer::unread_in_place(address); let _ = layer.reload(); layer } + /// The layer in the look its address gives (see [`AddressLook`]): now, when its pixels are + /// here, else once they land. + pub fn with_look(mut self, look: AddressLook) -> RasterLayer { + match self.data.clone() { + Some(d) => { + if let Some(style) = RasterStyle::for_look(&d, &look) { + self.style = style; + } + } + None => self.look = Some(look), + } + self + } + /// Reads the pixels again from the path or the embedded file (after a project opens). A /// layer that follows the time slider is left to the engine, which reads the step shown. pub fn reload(&mut self) -> Result<(), String> { @@ -229,40 +317,21 @@ impl RasterLayer { self.error = Some(e.clone()); return Err(e); }; + self.data = None; + let slot = Arc::new(Mutex::new(None)); + let land = slot.clone(); + self.opening = Some(slot); + reader( + &address, + Box::new(move |result| { + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); + crate::source::landed(); + }), + ); #[cfg(not(target_arch = "wasm32"))] - { - let (tx, rx) = std::sync::mpsc::channel(); - reader(&address, Box::new(move |r| tx.send(r).unwrap_or_default())); - let r = rx - .recv_timeout(std::time::Duration::from_secs(120)) - .unwrap_or_else(|_| Err(format!("{address}: no answer in two minutes"))); - return match r { - Ok(raster) => { - self.land(Ok(Arc::new(raster))); - Ok(()) - } - Err(e) => { - self.data = None; - self.land(Err(e.clone())); - Err(e) - } - }; - } + return self.wait_for_read(std::time::Instant::now() + IN_PLACE_WAIT); #[cfg(target_arch = "wasm32")] - { - let slot = Arc::new(Mutex::new(None)); - let land = slot.clone(); - self.data = None; - self.opening = Some(slot); - reader( - &address, - Box::new(move |result| { - *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); - crate::source::landed(); - }), - ); - return Ok(()); - } + return Ok(()); } if self.embedded.is_none() && let Some(path) = self @@ -319,6 +388,28 @@ impl RasterLayer { } } + /// Waits (natively, until `deadline`) for a read under way, then installs it. + #[cfg(not(target_arch = "wasm32"))] + fn wait_for_read(&mut self, deadline: std::time::Instant) -> Result<(), String> { + let Some(slot) = self.opening.clone() else { return Ok(()) }; + loop { + if let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() { + self.opening = None; + let error = result.as_ref().err().cloned(); + self.land(result); + return error.map_or(Ok(()), Err); + } + if std::time::Instant::now() >= deadline { + let address = self.path.clone().unwrap_or_default(); + let e = format!("{address}: no answer in two minutes"); + self.opening = None; + self.land(Err(e.clone())); + return Err(e); + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + /// Installs a completed remote read; each render copy can observe the same shared result. pub(crate) fn poll_reload(&mut self) -> bool { let Some(slot) = self.opening.as_ref() else { return false }; @@ -334,7 +425,8 @@ impl RasterLayer { match result { Ok(raster) => { if std::mem::take(&mut self.restyle) { - self.style = RasterStyle::default_for(&raster); + let look = self.look.take().and_then(|look| RasterStyle::for_look(&raster, &look)); + self.style = look.unwrap_or_else(|| RasterStyle::default_for(&raster)); } else { self.style.repair(&raster); } @@ -813,7 +905,39 @@ fn one() -> f64 { 1.0 } +/// The look an address gives a GeoTIFF read in place (`…/scene.tif#bands=5,3,2&range=0,3000`, as +/// tile layers kept it): the bands drawn (from 0; one in grey, or red, green and blue) and a fixed +/// stretch. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct AddressLook { + pub bands: Vec, + pub range: Option<(f64, f64)>, +} + impl RasterStyle { + /// The style `look` gives `r`: its own look (see [`RasterStyle::default_for`]) with the bands + /// and stretch the look names (bands it hasn't are left as they are); `None` for a look that + /// names neither. A palette image stretched by a range is drawn in grey, as its tile layer was. + pub fn for_look(r: &Raster, look: &AddressLook) -> Option { + if look.bands.is_empty() && look.range.is_none() { + return None; + } + let mut style = RasterStyle::default_for(r); + let n = r.band_count(); + match look.bands[..] { + [b] if b < n => style.renderer = RasterRenderer::Stretch { band: b, ramp: Ramp::gray(), invert: false }, + [red, green, blue] if red < n && green < n && blue < n => style.renderer = RasterRenderer::Rgb { red, green, blue }, + [] if matches!(style.renderer, RasterRenderer::Colormap) => { + style.renderer = RasterRenderer::Stretch { band: 0, ramp: Ramp::gray(), invert: false }; + } + _ => {} + } + if let Some((lo, hi)) = look.range { + style.stretch = Stretch::custom(lo, hi); + } + Some(style) + } + /// A style for a layer whose pixels haven't arrived (its own look replaces it when they do). pub fn placeholder() -> RasterStyle { RasterStyle { diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 58eee9f17..4ee9ce783 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -155,10 +155,11 @@ pub(crate) fn draw_in_place( let full_px = want.per_px * f64::from(full.0) / f64::from(r.width.max(1)); let fine_enough = |l: &rustgis_data::raster::cog::Level| f64::from(full.0) / f64::from(l.width.max(1)) <= full_px * 1.25; let mut level = levels.iter().rposition(fine_enough).unwrap_or(0); - // The renderer's bands, then the image's alpha band, read as the window's own bands. + // The renderer's bands, then the image's alpha band (again, when a colour draws it too), read + // as the window's own bands, as `draw` reads a raster held whole. let mut read = bands.clone(); let alpha = match (&style.renderer, &r.color) { - (RasterRenderer::Rgb { .. }, ColorHint::Rgb { alpha: true }) if r.band_count() >= 4 && !read.contains(&3) => Some(3), + (RasterRenderer::Rgb { .. }, ColorHint::Rgb { alpha: true }) if r.band_count() >= 4 => Some(3), (RasterRenderer::Stretch { band: 0, .. } | RasterRenderer::ShadedRelief { band: 0, .. }, ColorHint::GrayAlpha) => Some(1), _ => None, }; @@ -204,14 +205,23 @@ pub(crate) fn draw_in_place( whole(pm); return w.pending; }; - // Until its tiles arrive, the part in view goes over the overview. - if w.pending { - whole(pm); - } let mut own_style = style.clone(); own_style.renderer = renumbered(&style.renderer); + // Until its tiles arrive, the part in view goes over the overview: both drawn opaque, then + // blended once, so a see-through layer isn't darker where they overlap. + if w.pending { + if opacity >= 1.0 { + whole(pm); + draw(pm, &part, &own_style, Some(stats), view, 1.0, full_res, place); + } else if let Some(mut both) = Pixmap::new(pm.width(), pm.height()) { + draw(&mut both, r, style, Some(stats), view, 1.0, full_res, place); + draw(&mut both, &part, &own_style, Some(stats), view, 1.0, full_res, place); + pm.draw_pixmap(0, 0, both.as_ref(), &PixmapPaint { opacity, ..PixmapPaint::default() }, Transform::identity(), None); + } + return true; + } draw(pm, &part, &own_style, Some(stats), view, opacity, full_res, place); - w.pending + false } /// A renderer drawing the same bands of a raster holding only those, in its order (see @@ -248,12 +258,20 @@ fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_ let grid: Vec> = (0..n * n).map(|p| at(p % n, p / n)).collect(); let (mut c0, mut c1, mut r0, mut r1) = (f64::INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::NEG_INFINITY); let (rw, rh) = (f64::from(r.width), f64::from(r.height)); + // A geographic grid stored from 0 to 360° has the places west of 0° at its far columns (see + // `Raster::wrap_col`): seen across that seam, the part in view is the whole width. + let mut wrapped = false; for (c, row) in grid.iter().flatten() { - c0 = c0.min(c.clamp(0.0, rw)); - c1 = c1.max(c.clamp(0.0, rw)); + let w = r.wrap_col(*c); + wrapped |= w != *c; + c0 = c0.min(w.clamp(0.0, rw)); + c1 = c1.max(w.clamp(0.0, rw)); r0 = r0.min(row.clamp(0.0, rh)); r1 = r1.max(row.clamp(0.0, rh)); } + if wrapped { + (c0, c1) = (0.0, rw); + } if !(c0 < c1 && r0 < r1) { return None; } @@ -399,6 +417,7 @@ fn draw_with( relief, resampling, alpha_band, + alpha_max: rustgis_data::raster::alpha_max(&r.data_type), cell: r.cell_size_m(), mask: &style.mask, }; @@ -486,6 +505,8 @@ struct Pass1<'a> { relief: Option<(f64, f64, f64)>, resampling: Resampling, alpha_band: Option, + /// The alpha band's opaque value (its pixel type's: 65535 for 16 bits). + alpha_max: f32, cell: (f64, f64), /// Values drawn as no data. mask: &'a RasterMask, @@ -558,7 +579,7 @@ impl Pass1<'_> { last = Some((cell, dx)); } if let (Some(ab), Some(a)) = (self.alpha_band, alpha.get_mut(i)) { - *a = r.value(ab, cell.0, cell.1).map(|v| v.clamp(0.0, 255.0) as u8).unwrap_or(0); + *a = r.value(ab, cell.0, cell.1).map(|v| ((v / self.alpha_max).clamp(0.0, 1.0) * 255.0).round() as u8).unwrap_or(0); if *a == 0 { continue; } diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index 6eea3e2da..476b740e9 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -12,6 +12,10 @@ const MAX_TILES: u64 = 512; const MAX_BYTES: usize = 64 * 1024 * 1024; const TILE_SIZE: u32 = 256; const MAX_LAT: f64 = 85.051_128_779_806_6; +/// The longest an export waits for the tiles of a raster read in place, in all. +const IN_PLACE_WAIT: web_time::Duration = web_time::Duration::from_secs(120); +/// Waits in a row that bring nothing new before an export stops waiting for them. +const IDLE_WAITS: usize = 6; /// Half-open tile interval, retaining a tile for a zero-width extent. fn tile_range(start: f64, end: f64, n: u32) -> (u32, u32) { @@ -57,6 +61,8 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> ranges.push((z, x0, x1, y0, y1)); } let stats = layer.masked_stats(); + // A raster read in place waits for its tiles natively, two minutes at most in all. + let deadline = web_time::Instant::now() + IN_PLACE_WAIT; let mut tiles = Vec::new(); let mut bytes = 0usize; for (z, x0, x1, y0, y1) in ranges { @@ -73,14 +79,25 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> // Read in place: at the tile's level, once its parts have arrived (natively; // the browser draws what has). Some(address) => { - let mut passes = 0; - while raster::draw_in_place(&mut image, address, r, &layer.style, stats, &view, opacity, true, place) - && passes < 6 - && cfg!(not(target_arch = "wasm32")) - { - crate::rustgis_tiles::cache::wait(web_time::Duration::from_secs(30)); + let mut idle = 0; + loop { + let drawn_at = crate::rustgis_tiles::cache::generation(); + if !raster::draw_in_place(&mut image, address, r, &layer.style, stats, &view, opacity, true, place) { + break; + } + // An archive of the overview where the image is wanted isn't written. + let left = deadline.saturating_duration_since(web_time::Instant::now()); + if cfg!(target_arch = "wasm32") { + return Err(format!("{}: its tiles are still being read: export again once the map has drawn it", r.name)); + } + if left.is_zero() || idle >= IDLE_WAITS { + return Err(format!("{}: its tiles didn't all arrive, so the archive isn't written", r.name)); + } + crate::rustgis_tiles::cache::wait_for_news(drawn_at, left.min(web_time::Duration::from_secs(30))); + if crate::rustgis_tiles::cache::generation() == drawn_at { + idle += 1; + } image.fill(tiny_skia::Color::TRANSPARENT); - passes += 1; } } None => raster::draw(&mut image, r, &layer.style, stats, &view, opacity, true, place), diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index cd884b1d5..c5e881973 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -166,11 +166,73 @@ pub fn set_waker(f: impl Fn() + Send + Sync + 'static) { *waker().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); } +/// When the map is next to draw again by itself (a failed read may be tried again then), in +/// milliseconds after [`alarm_base`] plus one; 0 for none. +static ALARM: AtomicU64 = AtomicU64::new(0); + +fn alarm_base() -> Instant { + static B: OnceLock = OnceLock::new(); + *B.get_or_init(Instant::now) +} + +fn alarm_hook() -> &'static Mutex>> { + static A: OnceLock>>> = OnceLock::new(); + A.get_or_init(Mutex::default) +} + +/// Sets how the app is asked to draw again after a while (a repaint after it): a read that failed +/// is tried again once its wait is over, though nothing else draws the map. +pub fn set_alarm(f: impl Fn(Duration) + Send + Sync + 'static) { + *alarm_hook().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); +} + +/// Draws the map again at `when` (the earliest asked for is kept): [`generation`] goes up then, so +/// a map waiting on a failed read asks for it again. +pub(crate) fn notify_at(when: Instant) { + let at = (when.saturating_duration_since(alarm_base()).as_millis() as u64).saturating_add(1); + let mut current = ALARM.load(Ordering::Relaxed); + loop { + if current != 0 && current <= at { + return; + } + match ALARM.compare_exchange_weak(current, at, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => break, + Err(c) => current = c, + } + } + let hook = alarm_hook().lock().unwrap_or_else(|e| e.into_inner()).clone(); + if let Some(hook) = hook { + hook(when.saturating_duration_since(Instant::now())); + } +} + /// Goes up each time a fetch finishes: a map drawn at an older generation may now draw more. pub fn generation() -> u64 { + let at = ALARM.load(Ordering::Relaxed); + if at != 0 + && (alarm_base().elapsed().as_millis() as u64).saturating_add(1) >= at + && ALARM.compare_exchange(at, 0, Ordering::Relaxed, Ordering::Relaxed).is_ok() + { + GENERATION.fetch_add(1, Ordering::Relaxed); + } GENERATION.load(Ordering::Relaxed) } +/// How long a server that refused with 429 or 503 asked to be left before it is asked again, as +/// the error's text says it (`http status: 503 (retry after 30 s)`; see +/// [`crate::failure::with_retry_after`]). +pub fn retry_after(error: &str) -> Option { + let (_, rest) = error.rsplit_once("(retry after ")?; + let secs: u64 = rest.strip_suffix(" s)")?.parse().ok()?; + Some(Duration::from_secs(secs)) +} + +/// How long a failed read waits before it is tried again: what its server asked for (see +/// [`retry_after`]), else `default`. +pub(crate) fn retry_wait(error: &str, default: Duration) -> Duration { + retry_after(error).unwrap_or(default) +} + /// How many fetches are under way. Once a fetch no longer counts, its [`generation`] shows. pub fn in_flight() -> usize { IN_FLIGHT.load(Ordering::Acquire) @@ -209,7 +271,7 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { match &slot.entry { Entry::Ready(p, _) => return State::Ready(p.clone()), Entry::Pending => return State::Pending, - Entry::Failed(e, at) if at.elapsed() < RETRY => return State::Failed(e.clone()), + Entry::Failed(e, at) if at.elapsed() < retry_wait(e, RETRY) => return State::Failed(e.clone()), Entry::Failed(..) => {} } } @@ -719,8 +781,11 @@ impl<'a> RequestDiagnostics<'a> { /// another message (an address can hold "404"). pub fn http_status(error: &str) -> Option { let e = error.trim(); - let code = e.strip_prefix("http status: ").or_else(|| e.strip_prefix("HTTP "))?; - (code.len() == 3 && code.bytes().all(|b| b.is_ascii_digit())).then(|| code.parse().ok()).flatten() + let rest = e.strip_prefix("http status: ").or_else(|| e.strip_prefix("HTTP "))?; + // The status, and nothing after it but how long to wait (see [`retry_after`]). + let (code, after) = rest.split_at_checked(3)?; + let known = after.is_empty() || (after.starts_with(" (retry after ") && retry_after(after).is_some()); + (known && code.bytes().all(|b| b.is_ascii_digit())).then(|| code.parse().ok()).flatten() } /// Downloads a file the user named (data added by its web address): up to [`MAX_DOWNLOAD`] @@ -1385,7 +1450,8 @@ mod native { let status = resp.status().as_u16(); diagnostics.answered(status); if !(200..300).contains(&status) { - return Err(Failure::without_answer("", format!("http status: {status}"), Reason::Status(status))); + let message = crate::failure::with_retry_after(format!("http status: {status}"), status, retry_after_header(&resp)); + return Err(Failure::without_answer("", message, Reason::Status(status))); } let mut buf = Vec::new(); resp.body_mut() @@ -1470,6 +1536,11 @@ mod native { } /// An answer with an error status (#509): its kept headers and the start of its body. + /// An answer's `Retry-After` header, as sent. + fn retry_after_header(resp: &ureq::http::Response) -> Option<&str> { + resp.headers().get("retry-after").and_then(|v| v.to_str().ok()) + } + fn refused(resp: &mut ureq::http::Response) -> Failure { use std::io::Read; let headers: Vec<(String, String)> = resp @@ -1604,7 +1675,8 @@ mod native { return Err(if detail && !prepared.signed { refused(&mut resp) } else { - Failure::without_answer("", format!("http status: {status}"), Reason::Status(status)) + let message = crate::failure::with_retry_after(format!("http status: {status}"), status, retry_after_header(&resp)); + Failure::without_answer("", message, Reason::Status(status)) }); } if !resp.status().is_redirection() { @@ -2092,6 +2164,24 @@ mod tests { assert_eq!(http_status("https://x.org/404/a.zarr: timed out"), None); assert_eq!(http_status("HTTP 4040"), None); assert_eq!(http_status("io: connection reset (read 404 bytes)"), None); + // With how long the server asked to be left: the status, and the wait. + assert_eq!(http_status("http status: 503 (retry after 30 s)"), Some(503)); + assert_eq!(http_status("HTTP 429 (retry after 2 s)"), Some(429)); + assert_eq!(http_status("HTTP 429 (something else)"), None); + assert_eq!(retry_after("http status: 503 (retry after 30 s)"), Some(Duration::from_secs(30))); + assert_eq!(retry_after("http status: 503"), None); + assert_eq!(retry_wait("HTTP 429 (retry after 2 s)", RETRY), Duration::from_secs(2)); + assert_eq!(retry_wait("http status: 500", RETRY), RETRY); + } + + /// A read that failed asks the map to draw again when it may be tried: the generation goes up + /// once that time has come. + #[test] + fn the_map_draws_again_when_a_wait_is_over() { + let before = generation(); + notify_at(Instant::now() + Duration::from_millis(20)); + std::thread::sleep(std::time::Duration::from_millis(40)); + assert!(generation() > before); } #[test] diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index f644a5eb7..a79d84333 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -330,7 +330,7 @@ pub fn info(url: &str) -> Result>, String> { match i.get(url) { Some(State::Ready(info)) => return Ok(Some(info.clone())), Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => return Err(e.clone()), + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => return Err(e.clone()), _ => {} } } @@ -352,7 +352,7 @@ fn with_info(url: &str, then: Box, String>) + Send>) waiting.push(then); return; } - Some(State::Failed(e, at)) if at.elapsed() < RETRY => { + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { let e = e.clone(); drop(i); then(Err(e)); @@ -1060,7 +1060,7 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 let pen = Pen { rgb: colours >= 3, alpha: (shown.len() > colours).then_some(colours), - alpha_max: type_range(&info.layout, false).1.max(1.0), + alpha_max: rustgis_data::raster::alpha_max(info.layout.data_type), palette: info.palette(look).map(<[[u8; 4]]>::to_vec), aef: look.aef, stretch: stretch.to_vec(), @@ -1529,6 +1529,8 @@ mod tests { let pen = Pen { rgb: false, alpha: None, alpha_max: 255.0, palette: Some(table), aef: false, stretch: vec![(0.0, 255.0)] }; let out = px(paint(&pen, &spots, &[7], &[tile]).unwrap()); assert_eq!(&out[..4], &[200, 10, 20, 255]); + assert_eq!(&out[4..8], &[0, 0, 0, 0], "a transparent colour"); + assert_eq!(&out[8..12], &[0, 0, 0, 0], "a value past the table"); } #[test] diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 1967ebeda..b309b8408 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -19,24 +19,35 @@ use super::{BUDGET, Bands, Info, RETRY, TileState, Tiles, fetch_all, get_tile, s const OVERVIEW_SAMPLES: usize = if cfg!(target_arch = "wasm32") { 1 << 20 } else { 2 << 20 }; /// The most chunks read for it. const OVERVIEW_CHUNKS: usize = 512; +/// The most chunks of the coarsest level, held when no level fits [`OVERVIEW_SAMPLES`] (a file +/// with few or no overviews); its samples are bounded by half the tiles kept. +const FALLBACK_CHUNKS: usize = 4096; /// The most samples of one band of a view's window (a large screen's worth, and more). pub const WINDOW_SAMPLES: usize = 16 << 20; /// The level a layer holds as its overview: the finest whose every band fits [`OVERVIEW_SAMPLES`] -/// in at most [`OVERVIEW_CHUNKS`] chunks, else the coarsest. -pub fn overview_level(info: &Info) -> usize { +/// in at most [`OVERVIEW_CHUNKS`] chunks, else the coarsest when it fits half the tiles kept in at +/// most [`FALLBACK_CHUNKS`]; `None` when not even that does (a large image without overviews). +pub fn overview_level(info: &Info) -> Option { let layout = &info.layout; let bands = layout.bands.max(1); let planes = if layout.planar { bands } else { 1 }; + let size = |l: &rustgis_data::raster::cog::Level| { + let samples = (l.width as usize).saturating_mul(l.height as usize).saturating_mul(bands); + (samples, (l.tiles() as usize).saturating_mul(planes)) + }; layout .levels .iter() .position(|l| { - let samples = (l.width as usize).saturating_mul(l.height as usize).saturating_mul(bands); - let chunks = (l.tiles() as usize).saturating_mul(planes); + let (samples, chunks) = size(l); samples <= OVERVIEW_SAMPLES && chunks <= OVERVIEW_CHUNKS }) - .unwrap_or(layout.levels.len().saturating_sub(1)) + .or_else(|| { + let last = layout.levels.len().checked_sub(1)?; + let (samples, chunks) = size(layout.levels.get(last)?); + (samples <= BUDGET / 2 && chunks <= FALLBACK_CHUNKS).then_some(last) + }) } /// Reads a file's overview (see [`overview_level`]) as a raster, every band, then calls `done`. @@ -62,11 +73,18 @@ pub fn overview_blocking(url: &str) -> Result { } fn read_overview(info: Arc, url: String, done: Box) + Send>) { - let level = overview_level(&info); - let Some(l) = info.layout.levels.get(level) else { + let Some(coarsest) = info.layout.levels.last() else { done(Err(format!("{url}: the GeoTIFF has no image"))); return; }; + let Some((level, l)) = overview_level(&info).and_then(|i| Some((i, info.layout.levels.get(i)?))) else { + done(Err(format!( + "{url}: its smallest image, {} × {} with {} band(s), is too large to hold as the layer's overview: \ + a GeoTIFF with overviews (as GDAL's COG driver or gdaladdo make) is read in place", + coarsest.width, coarsest.height, info.layout.bands + ))); + return; + }; let bands: Vec = (0..info.layout.bands.max(1)).collect(); let planes = if info.layout.planar { bands.len() } else { 1 }; let Some(n) = l.tiles().checked_mul(u32::try_from(planes).unwrap_or(u32::MAX)) else { @@ -99,6 +117,7 @@ fn read_overview(info: Arc, url: String, done: Box 0) { r.overview = Some((full.width, full.height)); } + r.palette_missing = i.layout.palette_missing; r.source = Some(url.clone()); Ok(r) })) @@ -152,8 +171,9 @@ pub struct Window { pub pending: bool, } -/// Tiles whose read failed lately, and when: not asked for again until [`RETRY`] has passed, so a -/// map drawn again and again doesn't ask for them each time. +/// Tiles whose read failed lately, and until when they aren't asked for again ([`RETRY`] after, or +/// as long as their server asked to be left), so a map drawn again and again doesn't ask for them +/// each time. The map is drawn again when that time comes. fn failed() -> &'static Mutex> { static F: OnceLock>> = OnceLock::new(); F.get_or_init(Mutex::default) @@ -202,8 +222,20 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands let mut pending = kept.len() < chunks.len(); { let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); - f.retain(|_, at| at.elapsed() < RETRY); - ask.retain(|c| !f.contains_key(&(file.to_string(), level, *c))); + let now = Instant::now(); + f.retain(|_, until| now < *until); + // Those failed lately wait; the map draws again when the first may be asked for. + let mut next: Option = None; + ask.retain(|c| match f.get(&(file.to_string(), level, *c)) { + Some(until) => { + next = Some(next.map_or(*until, |n| n.min(*until))); + false + } + None => true, + }); + if let Some(at) = next { + crate::cache::notify_at(at); + } } // Tiles that failed lately aren't waited for (the map would wait for them for nothing). if ask.is_empty() && pending { @@ -218,22 +250,25 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands level, c, Box::new(move |r| { - if r.is_err() { + if let Err(e) = r { + let until = Instant::now() + crate::cache::retry_wait(&e, RETRY); let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); if f.len() < MAX_FAILED { - f.insert(key, Instant::now()); + f.insert(key, until); } + drop(f); + crate::cache::notify_at(until); } crate::cache::notify(); }), ); pending = true; } - let (mut data, missing) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); + let (mut data, gaps) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); // Until they arrive, its missing tiles show the levels kept (the one zoomed from), not holes. - if missing { + if !gaps.is_empty() { let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); - stand_in(&mut t, &info, file, level, (c0, c1), (r0, r1), bands, &mut data); + stand_in(&mut t, &info, file, level, (c0, c1), (r0, r1), bands, &mut data, &gaps); } let t = info.layout.transform_of(l); let transform = rustgis_data::raster::GeoTransform { @@ -256,9 +291,18 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands Ok(Window { raster: Some(raster), pending }) } -/// Fills the pixels of a window of `level` that have no value (NaN) from the other levels' tiles -/// kept, the nearest first (the finer, then the coarser, then further): the level zoomed from -/// stands in for the one zoomed to until its tiles arrive. Nothing is fetched. +/// A part of a window a chunk not kept leaves empty: the band (its place among those read), and +/// its columns and rows (half-open, in the level's pixels). +struct Gap { + k: usize, + cols: (u32, u32), + rows: (u32, u32), +} + +/// Fills the `gaps` of a window of `level` from the other levels' tiles kept, the nearest first +/// (the finer, then the coarser, then further): the level zoomed from stands in for the one zoomed +/// to until its tiles arrive. Only the gaps are filled: a tile's own no-data pixels stay empty. +/// Nothing is fetched. #[allow(clippy::too_many_arguments)] fn stand_in( t: &mut Tiles, @@ -269,10 +313,19 @@ fn stand_in( rows: (u32, u32), bands: &[usize], data: &mut [Vec], + gaps: &[Gap], ) { let levels = &info.layout.levels; let Some(l) = levels.get(level) else { return }; let ww = (cols.1 - cols.0) as usize; + // The pixels still to fill, by band (its place among those read), as places in the window. + let mut left: Vec<(usize, Vec)> = gaps + .iter() + .map(|g| { + let at = |y: u32, x: u32| (y - rows.0) as usize * ww + (x - cols.0) as usize; + (g.k, (g.rows.0..g.rows.1).flat_map(|y| (g.cols.0..g.cols.1).map(move |x| at(y, x))).collect()) + }) + .collect(); // The other levels, nearest first. let others: Vec = (1..levels.len()).flat_map(|d| [level.checked_sub(d), Some(level + d)]).flatten().filter(|o| *o < levels.len()).collect(); @@ -282,18 +335,20 @@ fn stand_in( let (tw, th) = (ol.tile_width.max(1), ol.tile_height.max(1)); // The chunks of that level found so far (none: not kept). let mut found: HashMap> = HashMap::new(); - let mut left = false; - for (k, &b) in bands.iter().enumerate() { - let Some(dst) = data.get_mut(k) else { continue }; + for (k, todo) in &mut left { + let (Some(&b), Some(dst)) = (bands.get(*k), data.get_mut(*k)) else { + todo.clear(); + continue; + }; // The chunk the last pixel came from: its neighbours mostly share it. let mut last: Option<(u32, Option)> = None; - for (i, v) in dst.iter_mut().enumerate().filter(|(_, v)| v.is_nan()) { + todo.retain(|&i| { let (x, y) = (cols.0 as usize + i % ww, rows.0 as usize + i / ww); // The pixel of that level under this one's centre. let ox = (((x as f64 + 0.5) * sx) as u32).min(ol.width.saturating_sub(1)); let oy = (((y as f64 + 0.5) * sy) as u32).min(ol.height.saturating_sub(1)); let tile = (oy / th) * ol.across() + ox / tw; - let Some((c, at)) = info.layout.chunk(ol, tile, b) else { continue }; + let Some((c, at)) = info.layout.chunk(ol, tile, b) else { return true }; if last.as_ref().is_none_or(|(lc, _)| *lc != c) { let chunk = found.entry(c).or_insert_with(|| t.used(&(file.to_string(), o, c))).clone(); last = Some((c, chunk)); @@ -303,21 +358,26 @@ fn stand_in( .and_then(|(_, ch)| ch.as_ref()) .and_then(|ch| ch.get(at)) .and_then(|p| p.get(((oy % th) * tw + ox % tw) as usize)); - match value { - Some(s) => *v = *s, - None => left = true, + match (value, dst.get_mut(i)) { + // That level's value, no data included: it is what the image has there. + (Some(s), Some(v)) => { + *v = *s; + false + } + _ => true, } - } + }); } - if !left { + left.retain(|(_, todo)| !todo.is_empty()); + if left.is_empty() { return; } } } /// The pixels of columns `cols` and rows `rows` (half-open) of a level, bands `bands` in that -/// order, each row by row, from the chunks `chunk` gives (NaN where it gives none), and whether -/// any was missing. +/// order, each row by row, from the chunks `chunk` gives (NaN where it gives none), and the gaps +/// the chunks it doesn't give leave. fn assemble( info: &Info, level: usize, @@ -325,12 +385,12 @@ fn assemble( rows: (u32, u32), bands: &[usize], chunk: impl Fn(u32) -> Option, -) -> (Vec>, bool) { +) -> (Vec>, Vec) { let (ww, wh) = ((cols.1 - cols.0) as usize, (rows.1 - rows.0) as usize); let mut out = vec![vec![f32::NAN; ww * wh]; bands.len()]; - let Some(l) = info.layout.levels.get(level) else { return (out, true) }; + let mut gaps = Vec::new(); + let Some(l) = info.layout.levels.get(level) else { return (out, gaps) }; let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); - let mut missing = false; for ty in rows.0 / th..rows.1.div_ceil(th) { for tx in cols.0 / tw..cols.1.div_ceil(tw) { let tile = ty * l.across() + tx; @@ -340,7 +400,7 @@ fn assemble( for (k, &b) in bands.iter().enumerate() { let Some((c, at)) = info.layout.chunk(l, tile, b) else { continue }; let Some(data) = chunk(c) else { - missing = true; + gaps.push(Gap { k, cols: (x0, x1), rows: (y0, y1) }); continue; }; let (Some(plane), Some(dst)) = (data.get(at), out.get_mut(k)) else { continue }; @@ -355,13 +415,72 @@ fn assemble( } } } - (out, missing) + (out, gaps) } #[cfg(test)] mod tests { use super::*; + use rustgis_data::raster::cog::{Example, Level, Parse, Sparse, example_with, layout}; + + /// A file's info, as reading its header gives it. + fn info_of(file: Vec) -> Info { + let Ok(Parse::Done(layout)) = layout(&Sparse::default().with(0, file)) else { panic!("not a COG") }; + let transformer = layout.crs.transformer().unwrap(); + Info { + layout: *layout, + header: Sparse::default(), + transformer, + bounds: [0.0; 4], + merc: [0.0; 4], + min_zoom: 0, + max_zoom: 0, + stretch: Mutex::default(), + stretching: Mutex::default(), + stretch_failed: Mutex::default(), + alpha: None, + } + } + + /// The overview held is the finest small one; without one, the coarsest if it fits half the + /// tiles kept; else none, rather than reading a large image whole. + #[test] + fn the_overview_held_is_bounded() { + let mut info = info_of(example_with(&Example { size: 4096, tile: 256, ..Example::default() })); + // 4096, 2048, 1024 … a side: 1024 a side is a million values. + assert_eq!(overview_level(&info).and_then(|l| info.layout.levels.get(l)).map(|l| l.width), Some(1024)); + let first = info.layout.levels[0].clone(); + let level = |side: u32| Level { width: side, height: side, ..first.clone() }; + // Only the full image, 4096 a side (16 million values in 256 tiles): held. + info.layout.levels = vec![level(4096)]; + assert_eq!(overview_level(&info), Some(0)); + // One 20 000 a side (400 million values in 6241 tiles): none. + info.layout.levels = vec![level(20_000)]; + assert_eq!(overview_level(&info), None); + } + + /// A window's missing tiles are filled from the level kept, but not its own no-data pixels. + #[test] + fn stand_ins_fill_only_the_tiles_missing() { + let info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); + let file = "test://stand-in.tif"; + // Level 1 (32 a side) kept: its tile 0 is level 0's pixels 0 to 32, all 7. + let mut t = Tiles::default(); + let _ = t.keep((file.to_string(), 1, 0), Arc::new(vec![vec![7.0; 16 * 16]]), 256); + // Level 0's tile 0 here, one pixel of it no data; tile 1, right of it, not yet. + let mut own = vec![3.0; 16 * 16]; + own[16 + 1] = f32::NAN; + let own: Bands = Arc::new(vec![own]); + let (cols, rows) = ((0, 32), (0, 16)); + let (mut data, gaps) = assemble(&info, 0, cols, rows, &[0], |c| (c == 0).then(|| own.clone())); + assert_eq!(gaps.len(), 1); + stand_in(&mut t, &info, file, 0, cols, rows, &[0], &mut data, &gaps); + let at = |x: usize, y: usize| data[0][y * 32 + x]; + assert_eq!((at(0, 0), at(20, 5), at(31, 15)), (3.0, 7.0, 7.0), "tile 1 from level 1"); + assert!(at(1, 1).is_nan(), "tile 0's no-data pixel stays empty"); + } + #[test] fn names_from_addresses() { assert_eq!(file_name("https://x.org/a/scene.tif?sig=1"), "scene.tif"); diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index 4a0ccf76e..a0ea4802c 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -86,10 +86,11 @@ impl Failure { } } let html = kept.iter().any(|(k, v)| k == "content-type" && v.to_ascii_lowercase().contains("html")); + let wait = kept.iter().find(|(k, _)| k == "retry-after").map(|(_, v)| v.as_str()); Failure { url: url.to_string(), // As ureq has always said it, so text that reads the status back still finds it. - message: format!("http status: {status}"), + message: with_retry_after(format!("http status: {status}"), status, wait), reason: Reason::Status(status), answer: Some(Box::new(Answer { headers: kept, excerpt: excerpt(body, html), user_agent: None })), } @@ -192,6 +193,59 @@ impl std::fmt::Display for Failure { } } +/// The longest a server's `Retry-After` is waited for: a longer wait (or a hostile one) is cut +/// to it, so the read is tried again within the hour. +const MAX_RETRY_AFTER: u64 = 3600; + +/// A refusal's text (`http status: 503`, or the browser's `HTTP 503`), with how long the server +/// asked to be left (`… (retry after 30 s)`) when it refused with 429 or 503 and its `Retry-After` +/// header (`retry_after`) says so, in seconds or as a date. Read back by +/// [`crate::cache::retry_after`]. +pub fn with_retry_after(text: String, status: u16, retry_after: Option<&str>) -> String { + let now = web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + match retry_after.filter(|_| matches!(status, 429 | 503)).and_then(|v| retry_after_secs(v, now)) { + Some(secs) => format!("{text} (retry after {secs} s)"), + None => text, + } +} + +/// A `Retry-After` header's wait in seconds (at least one, at most [`MAX_RETRY_AFTER`]): a number +/// of seconds, or an HTTP date (`Wed, 21 Oct 2015 07:28:00 GMT`) after `now` (seconds since 1970). +fn retry_after_secs(value: &str, now: u64) -> Option { + let value = value.trim(); + let secs = match value.parse::() { + Ok(secs) => secs, + Err(_) => http_date(value)?.saturating_sub(now), + }; + Some(secs.clamp(1, MAX_RETRY_AFTER)) +} + +/// An HTTP date's seconds since 1970 (the IMF-fixdate form servers send: `Sun, 06 Nov 1994 +/// 08:49:37 GMT`). +fn http_date(s: &str) -> Option { + let mut parts = s.split_whitespace(); + let (_, day, month, year, time, zone) = (parts.next()?, parts.next()?, parts.next()?, parts.next()?, parts.next()?, parts.next()?); + if zone != "GMT" || parts.next().is_some() { + return None; + } + const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + let month = MONTHS.iter().position(|m| *m == month)? as i64 + 1; + let (day, year): (i64, i64) = (day.parse().ok()?, year.parse().ok()?); + let mut hms = time.split(':').map(|p| p.parse::().ok()); + let (h, m, sec) = (hms.next()??, hms.next()??, hms.next()??); + if !(1..=31).contains(&day) || !(1970..=9999).contains(&year) || h > 23 || m > 59 || sec > 60 || hms.next().is_some() { + return None; + } + // Days since 1970 of a date in the proleptic Gregorian calendar (Howard Hinnant's algorithm). + let y = if month <= 2 { year - 1 } else { year }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146_097 + doe - 719_468; + u64::try_from(days * 86_400 + h * 3600 + m * 60 + sec).ok() +} + /// A status's usual name (`403` → `Forbidden`), for the common ones. pub fn status_name(status: u16) -> Option<&'static str> { Some(match status { @@ -384,4 +438,28 @@ mod tests { assert_eq!(Failure::without_answer("u", "timeout: global", Reason::Timeout).reason, Reason::Timeout); assert_eq!((status_name(403), status_name(299)), (Some("Forbidden"), None)); } + + /// A 429 or 503 refusal says how long its server asked to be left, from its `Retry-After` + /// header in seconds or as a date, within an hour; other refusals, and a header that can't be + /// read, say nothing more. + #[test] + fn retry_after_on_429_and_503() { + let base = || "http status: 503".to_string(); + assert_eq!(with_retry_after(base(), 503, Some("30")), "http status: 503 (retry after 30 s)"); + assert_eq!(with_retry_after("HTTP 429".into(), 429, Some(" 2 ")), "HTTP 429 (retry after 2 s)"); + assert_eq!(with_retry_after(base(), 503, Some("0")), "http status: 503 (retry after 1 s)"); + assert_eq!(with_retry_after(base(), 503, Some("86400")), "http status: 503 (retry after 3600 s)"); + assert_eq!(with_retry_after(base(), 503, Some("soon")), "http status: 503"); + assert_eq!(with_retry_after(base(), 503, None), "http status: 503"); + assert_eq!(with_retry_after("http status: 500".into(), 500, Some("30")), "http status: 500"); + // A date: the seconds until it (Sun, 06 Nov 1994 08:49:37 GMT is 784 111 777). + assert_eq!(http_date("Sun, 06 Nov 1994 08:49:37 GMT"), Some(784_111_777)); + assert_eq!(retry_after_secs("Sun, 06 Nov 1994 08:50:37 GMT", 784_111_777), Some(60)); + assert_eq!(retry_after_secs("Sun, 06 Nov 1994 08:49:37 GMT", 784_200_000), Some(1), "past: at once, after a second"); + assert_eq!(http_date("Sun, 06 Nov 1994 08:49:37 PST"), None); + assert_eq!(http_date("Sun, 32 Nov 1994 08:49:37 GMT"), None); + // A refused answer's own header. + let headers = vec![("Retry-After".to_string(), "12".to_string())]; + assert_eq!(Failure::answered("u", 429, &headers, b"").message, "http status: 429 (retry after 12 s)"); + } } diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index cb6608633..c1e7be3e6 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -414,7 +414,7 @@ pub fn info(archive: &str) -> Result>, String> { match all.get(archive) { Some(State::Ready(i)) => return Ok(Some(i.clone())), Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => return Err(e.clone()), + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => return Err(e.clone()), _ => {} } all.insert(archive.to_string(), State::Loading(Vec::new())); @@ -442,7 +442,7 @@ fn with_info(archive: &str, f: Box, String>) + Send> f(Ok(i)); } Some(State::Loading(waiting)) => waiting.push(f), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => { + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { let e = e.clone(); drop(all); f(Err(e)); diff --git a/crates/ui-egui/src/mapview.rs b/crates/ui-egui/src/mapview.rs index be1686fd6..dd2642c5a 100644 --- a/crates/ui-egui/src/mapview.rs +++ b/crates/ui-egui/src/mapview.rs @@ -762,6 +762,9 @@ fn main_view(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { WAKER.call_once(|| { let ctx = ui.ctx().clone(); rustgis_engine::render::rustgis_tiles::cache::set_waker(move || ctx.request_repaint()); + // A read that failed is tried again when its wait is over, though nothing else moves. + let ctx = ui.ctx().clone(); + rustgis_engine::render::rustgis_tiles::cache::set_alarm(move |after| ctx.request_repaint_after(after)); let ctx = ui.ctx().clone(); rustgis_engine::map::source::set_waker(move || ctx.request_repaint()); }); diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index acbd65e2e..21c5bb02e 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -5913,13 +5913,18 @@ fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens return; }; let e = d.extent(); - let (cx, cy) = d.cell_size(); let unit = if d.crs.is_geographic() { "°" } else { " m" }; + // Read in place, the map draws the full image: its size and cell size, not the overview's. + let full = d.overview.filter(|_| rl.in_place); + let (w, h) = full.unwrap_or((d.width, d.height)); + let (cx, cy) = d.cell_size(); + let (cx, cy) = (cx * f64::from(d.width) / f64::from(w.max(1)), cy * f64::from(d.height) / f64::from(h.max(1))); + let f = rustgis_engine::map::symbology::fmt_num; let mut rows = vec![ - ("Size", format!("{} × {}", d.width, d.height)), + ("Size", format!("{w} × {h}")), ("Bands", d.band_count().to_string()), ("Pixel type", d.data_type.clone()), - ("Cell size", format!("{} × {}{unit}", rustgis_engine::map::symbology::fmt_num(cx), rustgis_engine::map::symbology::fmt_num(cy))), + ("Cell size", format!("{} × {}{unit}", f(cx), f(cy))), ("Coordinate system", d.crs.label()), ("NoData", d.nodata.map(|v| v.to_string()).unwrap_or_else(|| "—".into())), ("Symbology", tl!(rl.style.renderer.label()).to_string()), @@ -5931,16 +5936,8 @@ fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens ("North", format!("{:.4}°", e.max_y)), ]; match d.overview { - // Read in place, the map draws the full image: the overview is what the layer holds. - Some((w, h)) if rl.in_place => { - let (sx, sy) = (f64::from(d.width) / f64::from(w.max(1)), f64::from(d.height) / f64::from(h.max(1))); - let f = rustgis_engine::map::symbology::fmt_num; - if let Some(r) = rows.get_mut(0) { - *r = ("Size", format!("{w} × {h}")); - } - if let Some(r) = rows.get_mut(3) { - *r = ("Cell size", format!("{} × {}{unit}", f(cx * sx), f(cy * sy))); - } + // Read in place: the overview is what the layer holds. + Some(_) if rl.in_place => { rows.insert( 1, ( diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 3501110ba..7f00e2c36 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -146,7 +146,10 @@ bands, colors, mask and resampling. Set the minimum and maximum zoom from 0 to 1 transparency are included. The raster must be georeferenced and fit within one world. Turn off dynamic range adjustment and display rotation first. The in-memory export is limited to 512 tiles and 64 MiB of compressed tile data; reduce the extent or zoom range -if it exceeds a limit. Areas beyond the Web Mercator latitude limit are clipped. +if it exceeds a limit. Areas beyond the Web Mercator latitude limit are clipped. A cloud-optimised +GeoTIFF read in place is exported from its full-resolution tiles: the desktop waits for them (two +minutes at most), and in the web app, which can't wait, export again once the map has drawn it. +Without them nothing is written. **Vector to PMTiles** (`vectorToPmtiles`) writes a single-layer MVT archive in standard Web Mercator. Choose `minZoom` and `maxZoom` from 0 to 16; `sourceLayer` defaults to the input diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 6e28b42e1..f4849f98e 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -108,8 +108,14 @@ identify and the raster tools work on it as on a file's. The layer holds an over (every band, about two million values), from which its statistics, histograms and tools read, as a file too large to hold whole is opened from an overview; zoomed in past it, the map reads the part in view at the resolution it needs as it draws, and draws it with the layer's style and the -overview's statistics, so the image looks the same at every zoom. The Properties pane says the -image's full size and that it is read in place. +overview's statistics, so the image looks the same at every zoom. Until a part's tiles arrive, the +level zoomed from stands in for them. The Properties pane says the image's full size and that it +is read in place. A file whose smallest image is too large to hold as the overview (a large image +saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` makes the overviews. A tile +that fails to arrive is asked for again a minute later, or when its server says to in a +`Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again then. +Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, +and isn't written without them; in the web app, export again once the map has drawn the image. Its first look is what the file's PhotometricInterpretation tag says: @@ -120,7 +126,11 @@ Its first look is what the file's PhotometricInterpretation tag says: three bands as red, green and blue. An alpha band is shown with it. In an image with more bands (an 8-band multispectral image, say) you pick the combination in the -Symbology pane, as for any raster. GeoTIFF files follow the same rule. Planar files (bands stored +Symbology pane, as for any raster. GeoTIFF files follow the same rule and have every band too (an +RGB and near-infrared file has four), and so do other rasters without colours of their own (Zarr +stores, files read through GDAL): four or more bands open as the first in grey, where any three +or more used to open as red, green and blue. A palette image without its colour table is drawn in +grey, and says so. Planar files (bands stored apart), 8- to 64-bit integer and floating-point data, signed-byte data and bottom-up images work too. A dated series of GeoTIFFs (an address with date placeholders), a data cube's files and AlphaEarth's embeddings are tile layers that follow the time slider or dequantise their values; From 6cdb0e8bf9f8c41ab2682e62e83f008137aecdf9 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 17:51:10 -0330 Subject: [PATCH 03/27] fix(raster): say a missing colour table when pixels land later; make room for failed tiles - A palette image without its colour table is said once for its pixels whenever they land (a project opened, an import, the browser), not only when added with them: the layer remembers it was said, and the session says it after each command and each frame. - A full memory of failed tiles drops the one whose wait ends soonest, so a new failure still waits (Retry-After included) instead of being asked for at every draw. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/raster.rs | 3 ++- crates/engine/src/lib.rs | 27 ++++++++++++++++++++++++ crates/engine/src/tests.rs | 32 ++++++++++++++++++++++++++++ crates/map/src/raster.rs | 18 ++++++++++++++++ crates/tiles/src/cog/raster.rs | 37 ++++++++++++++++++++++++++++----- crates/ui-egui/src/lib.rs | 2 ++ 6 files changed, 113 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index 8d3fc3fbd..ecc98c0db 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -348,7 +348,8 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes } let (w, h, n, overview) = layer.data.as_ref().map(|d| (d.width, d.height, d.band_count(), d.overview)).unwrap_or_default(); let rounded = layer.data.as_deref().and_then(rounded_count); - let palette_missing = layer.data.as_deref().is_some_and(|d| d.palette_missing); + // Said here for pixels added with the layer; for those landing later, by `notice_landed_rasters`. + let palette_missing = layer.palette_unsaid(); // Read in place, the map shows the full image as it draws: the overview is only what's held. let overview = overview.filter(|_| !layer.in_place); let empty_before = s.map().layers.is_empty(); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f83189cae..b383346d0 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1064,6 +1064,8 @@ impl Session { let joins = self.merge.is_some() && self.merge == self.merged && !self.undo.is_empty(); let (at, start) = (project_log::now_ms(), web_time::Instant::now()); let res = self.run_command(id, params); + // Pixels the command read in place (a project, an import) say what they lost. + self.notice_landed_rasters(); // A plugin's command waiting for its fetches answers `pending` and runs again when they // land: logged then, once. (Another command's `pending`, a download the web app makes, is // logged as it is asked for: nothing runs it again.) @@ -1489,6 +1491,31 @@ impl Session { } } + /// Says what reading a raster layer's pixels lost when it wasn't said as the layer was added + /// (pixels read in place land later: in the browser, a project, an import): a palette image + /// without its colour table, drawn in grey. Once for each layer's pixels; the app calls it each + /// frame. + pub fn notice_landed_rasters(&mut self) { + let mut names = Vec::new(); + for m in &mut self.project.maps { + for l in &mut m.layers { + if let Some(rl) = &mut l.raster + && rl.palette_unsaid() + { + names.push(l.name.clone()); + } + } + } + match &names[..] { + [] => {} + [one] => self.notify(format!("{one}: its colour table is missing or short, so it is drawn in grey"), false), + many => self.notify( + format!("{} palette rasters have no whole colour table, so they are drawn in grey: {}", many.len(), many.join(", ")), + false, + ), + } + } + pub fn notify(&mut self, text: impl Into, error: bool) { use rustgis_data::diagnostics::Severity; self.notice(text.into(), if error { Severity::Error } else { Severity::Info }); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 97d579961..b38a92523 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13272,6 +13272,38 @@ fn an_address_look_holds_when_the_pixels_land_later() { assert_eq!(own.style.renderer, RasterRenderer::Colormap); } +/// A palette COG without its colour table is drawn in grey and says so, also when its pixels land +/// after the layer is added (a project opened, as in the browser and an import): once, and not for +/// one with its table. +#[test] +fn a_palette_without_its_table_says_so_when_its_pixels_land() { + use rustgis_data::raster::cog::{Example, example_with}; + let clean = example_with(&Example { size: 300, tile: 128, photometric: 3, ..Example::default() }); + let mut lossy = clean.clone(); + // The ColorMap entries (tag 320, of shorts) renamed to a tag nothing reads. + for i in 0..lossy.len().saturating_sub(4) { + if lossy[i..i + 4] == [0x40, 0x01, 0x03, 0x00] { + lossy[i..i + 2].copy_from_slice(&[0xFF, 0xFE]); + } + } + let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("colour table is missing")).count(); + for (file, want) in [(lossy, 1), (clean, 0)] { + let (base, _) = serve_ranges(file); + let mut s = Session::new(Project::default()); + let layer = rustgis_map::raster::RasterLayer::unread_in_place(format!("{base}/classes.tif")); + s.map_mut().add_raster("Classes".to_string(), layer); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + assert!(s.map().layers.iter().any(|l| l.raster.as_ref().is_some_and(|r| r.data.is_some())), "read on opening"); + assert_eq!(said(&s), want); + // Said once for these pixels, however often asked. + s.notice_landed_rasters(); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), want); + } +} + /// A project saved when cloud-optimised GeoTIFFs read in place were tile layers opens them as /// raster layers read in place, each where it was drawn, with its name, visibility, opacity and /// look; a dated series stays a tile layer. diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 64a386ead..3aace790d 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -66,6 +66,9 @@ pub struct RasterLayer { /// The look its address gave a layer read in place, applied when its pixels land. #[serde(skip)] look: Option, + /// Its pixels' missing colour table has been said (see [`RasterLayer::palette_unsaid`]). + #[serde(skip)] + palette_said: bool, } /// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. @@ -204,6 +207,7 @@ impl RasterLayer { opening: None, restyle: false, look: None, + palette_said: false, }) } @@ -226,6 +230,7 @@ impl RasterLayer { opening: None, restyle: false, look: None, + palette_said: false, } } @@ -248,6 +253,7 @@ impl RasterLayer { opening: None, restyle: false, look: None, + palette_said: false, } } @@ -275,6 +281,7 @@ impl RasterLayer { opening: None, restyle: true, look: None, + palette_said: false, } } @@ -378,6 +385,7 @@ impl RasterLayer { Ok(r) => { self.style.repair(&r); self.data = Some(Arc::new(r)); + self.palette_said = false; Ok(()) } Err(e) => { @@ -410,6 +418,15 @@ impl RasterLayer { } } + /// Whether its pixels are a palette image without a whole colour table (drawn in grey) and + /// that is still to be said: true once for the pixels it has, whenever they landed (added with + /// them, or read in place later, in the browser, a project or an import). + pub fn palette_unsaid(&mut self) -> bool { + let due = !self.palette_said && self.data.as_ref().is_some_and(|d| d.palette_missing); + self.palette_said |= due; + due + } + /// Installs a completed remote read; each render copy can observe the same shared result. pub(crate) fn poll_reload(&mut self) -> bool { let Some(slot) = self.opening.as_ref() else { return false }; @@ -424,6 +441,7 @@ impl RasterLayer { fn land(&mut self, result: Result, String>) { match result { Ok(raster) => { + self.palette_said = false; if std::mem::take(&mut self.restyle) { let look = self.look.take().and_then(|look| RasterStyle::for_look(&raster, &look)); self.style = look.unwrap_or_else(|| RasterStyle::default_for(&raster)); diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index b309b8408..2fc8655eb 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -182,6 +182,19 @@ fn failed() -> &'static Mutex> { /// The most failed tiles remembered. const MAX_FAILED: usize = 4096; +/// Remembers a failed tile until `until`. With [`MAX_FAILED`] already, the one whose wait ends +/// soonest makes room: one forgotten would be asked for again at each draw, whatever its server +/// asked for. +fn remember_failed(f: &mut HashMap<(String, usize, u32), Instant>, key: (String, usize, u32), until: Instant) { + if f.len() >= MAX_FAILED + && !f.contains_key(&key) + && let Some(soonest) = f.iter().min_by_key(|(_, at)| **at).map(|(k, _)| k.clone()) + { + f.remove(&soonest); + } + f.insert(key, until); +} + /// The columns `cols` and rows `rows` (half-open, in the level's pixels) of level `level` of a /// file, bands `bands` (0-based, in that order, as the raster's bands), coloured as `color` says: /// from the tiles kept, without waiting. Those missing are fetched (each once at a time); the @@ -252,11 +265,7 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands Box::new(move |r| { if let Err(e) = r { let until = Instant::now() + crate::cache::retry_wait(&e, RETRY); - let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); - if f.len() < MAX_FAILED { - f.insert(key, until); - } - drop(f); + remember_failed(&mut failed().lock().unwrap_or_else(|e| e.into_inner()), key, until); crate::cache::notify_at(until); } crate::cache::notify(); @@ -481,6 +490,24 @@ mod tests { assert!(at(1, 1).is_nan(), "tile 0's no-data pixel stays empty"); } + /// A full memory of failed tiles makes room for a new one: the one whose wait ends soonest + /// goes, so every tile failing waits its turn. + #[test] + fn failed_tiles_make_room_for_the_newest() { + let mut f = HashMap::new(); + let now = Instant::now(); + let key = |i: u32| ("https://x.org/a.tif".to_string(), 0usize, i); + for i in 0..MAX_FAILED as u32 { + remember_failed(&mut f, key(i), now + std::time::Duration::from_secs(60 + u64::from(i))); + } + remember_failed(&mut f, key(u32::MAX), now + std::time::Duration::from_secs(30)); + assert_eq!(f.len(), MAX_FAILED); + assert!(f.contains_key(&key(u32::MAX)) && !f.contains_key(&key(0)), "the soonest gone, the newest kept"); + // One already remembered is waited for again, nothing else goes. + remember_failed(&mut f, key(5), now + std::time::Duration::from_secs(90)); + assert!(f.contains_key(&key(1)) && f.len() == MAX_FAILED); + } + #[test] fn names_from_addresses() { assert_eq!(file_name("https://x.org/a/scene.tif?sig=1"), "scene.tif"); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index b070db732..b92197cc9 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -3818,6 +3818,8 @@ impl RustGisApp { // Plugin commands that waited for `host.fetch` answers (the browser) run again once // they are in; their errors are notices already. let _ = self.session.poll_plugin_retries(); + // Rasters read in place whose pixels landed since (the browser) say what they lost. + self.session.notice_landed_rasters(); // A frame may not come until the user moves: look again while commands wait. if self.session.plugin_retries_waiting() { ctx.request_repaint_after(std::time::Duration::from_millis(100)); From 991575cf5bc0724c7334f27afe691fba36fa0d6b Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 18:41:07 -0330 Subject: [PATCH 04/27] fix(raster): second review round of COGs read in place - Say an in-place layer's statistics and tools read an overview, when added and when its pixels land later; landing notices list at most five names, and the app gets a frame once pixels land. - Exports refuse an in-place layer drawn from less than the image (the overview after a read error, a view too large, stand-ins for tiles that couldn't be read). - The overview budget counts whole decoded tiles; a full failed-tile memory clears expired waits before making room. - The tile path keeps an alpha band drawn as a colour; alpha_max covers 64-bit and half floats. - A grey image is a palette image only when the whole file is one or it has its ColorMap. - A path-less in-place layer keeps its error; fetch_bytes has its doc comment back. - Tests: a GET answering 503 with Retry-After, two converted COG layers keep their order, the overview notice, and an unreadable in-place layer's export. Co-Authored-By: Claude Opus 5.5 --- apps/rustgis-web/src/web.rs | 8 ++-- crates/data/src/raster/cog.rs | 7 ++-- crates/data/src/raster/geotiff.rs | 19 ++++++--- crates/data/src/raster/mod.rs | 4 +- crates/engine/src/cmd/raster.rs | 10 ++++- crates/engine/src/lib.rs | 66 ++++++++++++++++++++++++------ crates/engine/src/tests.rs | 24 ++++++++++- crates/map/src/raster.rs | 41 +++++++++++-------- crates/map/src/source.rs | 5 +++ crates/render/src/lib.rs | 3 +- crates/render/src/raster.rs | 33 ++++++++------- crates/render/src/raster_export.rs | 29 ++++++++++++- crates/tiles/src/cache.rs | 6 +++ crates/tiles/src/cog.rs | 3 +- crates/tiles/src/cog/raster.rs | 33 ++++++++++----- 15 files changed, 217 insertions(+), 74 deletions(-) diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index 4ff197400..9d95be53d 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -570,10 +570,6 @@ async fn unless_stalled( Some(raced) } -/// Downloads `url`, or the byte range it asks for (a PMTiles archive's part: see -/// `rustgis_tiles::cache::split_range`). -/// A status in `omit` is an expected answer (a tile past a map's edge): its failure makes no -/// record of its own, as on the desktop. /// A refusal's text (`HTTP 503`), with how long the server asked to be left when it says so in a /// `Retry-After` header the browser lets the page read (its CORS rules must expose it). fn refused(resp: &web_sys::Response) -> String { @@ -585,6 +581,10 @@ fn refused(resp: &web_sys::Response) -> String { ) } +/// Downloads `url`, or the byte range it asks for (a PMTiles archive's part: see +/// `rustgis_tiles::cache::split_range`). +/// A status in `omit` is an expected answer (a tile past a map's edge): its failure makes no +/// record of its own, as on the desktop. async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Result, String> { use rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::{cache::RequestDiagnostics, failure::Failure}; let mut diagnostics = RequestDiagnostics::with_capture(url, "GET", capture); diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 23938665b..cb5a2036a 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -312,7 +312,7 @@ pub fn layout(bytes: &Sparse) -> Result { fn read_layout(r: Sparse) -> Result { let reader = super::geotiff::reader(r).map_err(err)?; - let (palette, wide_rgb) = (reader.palette, reader.wide_rgb); + let (palette, all_palette, wide_rgb) = (reader.palette, reader.all_palette, reader.wide_rgb); let mut dec = Decoder::new(reader).map_err(err)?.with_limits(limits()); let Some(geo) = super::geotiff::cog_geo(&mut dec)? else { return Ok(Parse::NotTiled("a rotated image, or one without coordinates, is read whole".into())); @@ -320,8 +320,9 @@ fn read_layout(r: Sparse) -> Result { let ct = dec.colortype().map_err(err)?; let mut palette_missing = false; let (bands, color) = match ct { - // A palette image (shown to the decoder as grey) is coloured by its table, else grey. - ColorType::Gray(bits) if palette => ( + // A palette image (shown to the decoder as grey) is coloured by its table, else grey (see + // `geotiff::read_limited_with_nodata`). + ColorType::Gray(bits) if all_palette || (palette && dec.find_tag(Tag::ColorMap).ok().flatten().is_some()) => ( 1, super::geotiff::palette_table(&mut dec, bits).map_or_else( || { diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index fdb8536d6..919e377f4 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -588,6 +588,9 @@ pub(crate) struct Unpaletted { patches: Vec<(u64, [u8; 2])>, /// Palette images were patched. pub(crate) palette: bool, + /// Every image is a palette image (transparency masks aside): one without its ColorMap is + /// one still, its table missing. + pub(crate) all_palette: bool, /// RGB images with more samples were patched: their first three samples are red, green, blue. pub(crate) wide_rgb: bool, pos: u64, @@ -602,6 +605,8 @@ const MAX_ENTRIES: u64 = 4096; struct Patches { at: Vec<(u64, [u8; 2])>, palette: bool, + /// An image that isn't a palette image (nor a transparency mask) was seen. + other: bool, wide_rgb: bool, } @@ -673,6 +678,7 @@ fn photometric_patches(r: &mut R, wide_rgb: bool) -> Patches { _ => {} } } + patches.other |= photometric.is_some_and(|(_, p)| p != 3 && p != 4); match photometric { Some((at, 3)) => { patches.at.push((at, grey)); @@ -714,7 +720,7 @@ impl Unpaletted { fn new(mut inner: R, wide_rgb: bool) -> std::io::Result { let p = photometric_patches(&mut inner, wide_rgb); inner.seek(std::io::SeekFrom::Start(0))?; - Ok(Unpaletted { inner, patches: p.at, palette: p.palette, wide_rgb: p.wide_rgb, pos: 0 }) + Ok(Unpaletted { inner, patches: p.at, palette: p.palette, all_palette: p.palette && !p.other, wide_rgb: p.wide_rgb, pos: 0 }) } } @@ -1044,7 +1050,7 @@ pub(super) fn read_limited_with_nodata( open: Option>, sidecar_nodata: Option, ) -> Result { - let (palette, wide_rgb) = (r.palette, r.wide_rgb); + let (palette, all_palette, wide_rgb) = (r.palette, r.all_palette, r.wide_rgb); let limits = { let mut l = Limits::default(); l.decoding_buffer_size = MAX_SAMPLES.saturating_mul(4); @@ -1132,10 +1138,13 @@ pub(super) fn read_limited_with_nodata( let ct = dec.colortype().map_err(err)?; // JPEG-compressed YCbCr, the only YCbCr read (the colour match below rejects the rest). let ycbcr = matches!(ct, ColorType::YCbCr(_)) && dec.find_tag_unsigned::(Tag::Compression).ok().flatten() == Some(7); - // A palette image (shown to the decoder as grey) is coloured by its table (in grey without one, - // as said below). + // A palette image (shown to the decoder as grey) is coloured by its table: in a file of palette + // images, one without it is drawn in grey, as said below; in a file of others too, only one with + // its table is taken for one. let ct = match ct { - ColorType::Gray(bits) if palette => ColorType::Palette(bits), + ColorType::Gray(bits) if all_palette || (palette && dec.find_tag(Tag::ColorMap).ok().flatten().is_some()) => { + ColorType::Palette(bits) + } other => other, }; // A palette image without a whole colour table is drawn in grey, and says so. diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index 42cf2f3d3..67ec88ab9 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -385,7 +385,9 @@ pub fn alpha_max(data_type: &str) -> f32 { "UInt16" => 65535.0, "Int32" => 2_147_483_647.0, "UInt32" => 4_294_967_295.0, - "Float32" | "Float64" => 1.0, + "Int64" => 9.223_372e18, + "UInt64" => 1.844_674_4e19, + "Float16" | "Float32" | "Float64" => 1.0, _ => 255.0, } } diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index ecc98c0db..e6f88bd34 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -349,8 +349,11 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes let (w, h, n, overview) = layer.data.as_ref().map(|d| (d.width, d.height, d.band_count(), d.overview)).unwrap_or_default(); let rounded = layer.data.as_deref().and_then(rounded_count); // Said here for pixels added with the layer; for those landing later, by `notice_landed_rasters`. - let palette_missing = layer.palette_unsaid(); - // Read in place, the map shows the full image as it draws: the overview is only what's held. + let unsaid = layer.pixels_unsaid(); + let palette_missing = unsaid.as_ref().is_some_and(|d| d.palette_missing); + // Read in place, the map shows the full image as it draws, but statistics and tools read the + // overview held: said so. + let held = unsaid.filter(|_| layer.in_place).and_then(|d| crate::in_place_overview(&d)); let overview = overview.filter(|_| !layer.in_place); let empty_before = s.map().layers.is_empty(); let id = s.map_mut().add_raster(unique.clone(), layer); @@ -361,6 +364,9 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes if palette_missing { s.notify(format!("{unique}: its colour table is missing or short, so it is drawn in grey"), false); } + if let Some(held) = held { + s.notify(format!("{unique}: {held}"), false); + } if let Some(count) = rounded { let held = "pixels are held as 32-bit floats, exact for whole numbers up to 16,777,216 and to about 7 significant digits"; s.notify(format!("{unique}: {count} rounded as the file was read: {held}, so an export writes the rounded values"), false); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b383346d0..3c10bb7d1 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -264,6 +264,29 @@ fn clip(s: &str, n: usize) -> String { if s.chars().count() > n { s.chars().take(n).chain(['…']).collect() } else { s.to_string() } } +/// The most layer names a notice lists (the rest counted), each cut to 80 characters: the notice +/// stays well within what a notice keeps. +const NAMES_SAID: usize = 5; + +/// Layer names for a notice: the first [`NAMES_SAID`], each clipped, and how many more. +fn names_said(names: &[String]) -> String { + let shown: Vec = names.iter().take(NAMES_SAID).map(|n| clip(n, 80)).collect(); + match names.len().saturating_sub(NAMES_SAID) { + 0 => shown.join(", "), + more => format!("{} and {more} more", shown.join(", ")), + } +} + +/// What a raster layer read in place holds for its statistics and tools, when it holds an +/// overview of the image the map draws. +pub(crate) fn in_place_overview(d: &rustgis_data::raster::Raster) -> Option { + let (fw, fh) = d.overview?; + Some(format!( + "read in place: the map draws its {fw} × {fh} pixels as it needs them, while statistics, histograms and raster tools read a {} × {} overview", + d.width, d.height + )) +} + pub struct Registry { pub commands: Vec, } @@ -1491,29 +1514,48 @@ impl Session { } } - /// Says what reading a raster layer's pixels lost when it wasn't said as the layer was added - /// (pixels read in place land later: in the browser, a project, an import): a palette image - /// without its colour table, drawn in grey. Once for each layer's pixels; the app calls it each - /// frame. + /// Says what reading raster layers' pixels lost when it wasn't said as each was added (pixels + /// read in place land later: in the browser, a project, an import): a palette image without + /// its colour table, drawn in grey; an overview statistics and tools read. Once for each + /// layer's pixels, one notice of each kind; the app calls it each frame. pub fn notice_landed_rasters(&mut self) { - let mut names = Vec::new(); + let (mut grey, mut held) = (Vec::new(), Vec::new()); for m in &mut self.project.maps { for l in &mut m.layers { - if let Some(rl) = &mut l.raster - && rl.palette_unsaid() - { - names.push(l.name.clone()); + let Some(rl) = &mut l.raster else { continue }; + let in_place = rl.in_place; + let Some(d) = rl.pixels_unsaid() else { continue }; + if d.palette_missing { + grey.push(l.name.clone()); + } + if let Some(what) = in_place.then(|| in_place_overview(&d)).flatten() { + held.push((l.name.clone(), what)); } } } - match &names[..] { + match &grey[..] { [] => {} - [one] => self.notify(format!("{one}: its colour table is missing or short, so it is drawn in grey"), false), + [one] => self.notify(format!("{}: its colour table is missing or short, so it is drawn in grey", clip(one, 80)), false), many => self.notify( - format!("{} palette rasters have no whole colour table, so they are drawn in grey: {}", many.len(), many.join(", ")), + format!("{} palette rasters have no whole colour table, so they are drawn in grey: {}", many.len(), names_said(many)), false, ), } + match &held[..] { + [] => {} + [(one, what)] => self.notify(format!("{}: {what}", clip(one, 80)), false), + many => { + let names: Vec = many.iter().map(|(n, _)| n.clone()).collect(); + self.notify( + format!( + "{} rasters are read in place: the map draws them at full resolution as it needs it, while statistics, histograms and raster tools read an overview of each: {}", + many.len(), + names_said(&names) + ), + false, + ) + } + } } pub fn notify(&mut self, text: impl Into, error: bool) { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index b38a92523..806aa7ffe 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13304,6 +13304,23 @@ fn a_palette_without_its_table_says_so_when_its_pixels_land() { } } +/// A layer read in place says its statistics and tools read an overview: once when added, once +/// when a project with it opens (its pixels landing then), and not again for the same pixels. +#[test] +fn in_place_layers_say_their_overview_once() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(2048, 256)); + let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("raster tools read a 1024 × 1024 overview")).count(); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif")})).unwrap(); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), 1, "{:?}", s.notices.last()); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + s.notice_landed_rasters(); + assert_eq!(said(&s), 1, "{:?}", s.notices.last()); +} + /// A project saved when cloud-optimised GeoTIFFs read in place were tile layers opens them as /// raster layers read in place, each where it was drawn, with its name, visibility, opacity and /// look; a dated series stays a tile layer. @@ -13321,9 +13338,11 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { t["visible"] = json!(visible); t }; + // Tile layers list the top one first: Scene over Aerial, both under the map's layers. let tiles = json!([ tile("Scene", format!("{base}/scene.tif#bands=5,3,2&range=10,200"), 0.5, false), tile("Series", format!("{base}/{{date:YYYY}}.tif"), 1.0, true), + tile("Aerial", format!("{base}/aerial.tif"), 1.0, true), ]); let maps = text.pointer_mut("/maps").and_then(Value::as_array_mut); match maps { @@ -13338,8 +13357,9 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); assert!(!scene.visible && (scene.opacity() - 0.5).abs() < 1e-6); - // Drawn under the map's layers, as it was. - assert_eq!(t.map().layers.last().map(|l| l.name.as_str()), Some("Scene")); + // Drawn under the map's layers, as they were, Scene over Aerial. + let names: Vec<&str> = t.map().layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names[names.len() - 2..], ["Scene", "Aerial"], "{names:?}"); assert_eq!(t.map().tiles.iter().map(|x| x.name.as_str()).collect::>(), ["Series"]); } diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 3aace790d..9f7be44c4 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -66,9 +66,9 @@ pub struct RasterLayer { /// The look its address gave a layer read in place, applied when its pixels land. #[serde(skip)] look: Option, - /// Its pixels' missing colour table has been said (see [`RasterLayer::palette_unsaid`]). + /// What reading its pixels lost has been said (see [`RasterLayer::pixels_unsaid`]). #[serde(skip)] - palette_said: bool, + pixels_said: bool, } /// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. @@ -207,7 +207,7 @@ impl RasterLayer { opening: None, restyle: false, look: None, - palette_said: false, + pixels_said: false, }) } @@ -230,7 +230,7 @@ impl RasterLayer { opening: None, restyle: false, look: None, - palette_said: false, + pixels_said: false, } } @@ -253,7 +253,7 @@ impl RasterLayer { opening: None, restyle: false, look: None, - palette_said: false, + pixels_said: false, } } @@ -281,7 +281,7 @@ impl RasterLayer { opening: None, restyle: true, look: None, - palette_said: false, + pixels_said: false, } } @@ -318,7 +318,11 @@ impl RasterLayer { // Read in place: its overview, by the reader the app installed; waited for natively (as a // file is read), landing later in the browser. if self.in_place { - let Some(address) = self.path.clone() else { return Err("the raster has no source".into()) }; + let Some(address) = self.path.clone() else { + let e = "the raster has no source".to_string(); + self.error = Some(e.clone()); + return Err(e); + }; let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { let e = format!("{address}: can't be read in place here"); self.error = Some(e.clone()); @@ -385,7 +389,7 @@ impl RasterLayer { Ok(r) => { self.style.repair(&r); self.data = Some(Arc::new(r)); - self.palette_said = false; + self.pixels_said = false; Ok(()) } Err(e) => { @@ -418,13 +422,16 @@ impl RasterLayer { } } - /// Whether its pixels are a palette image without a whole colour table (drawn in grey) and - /// that is still to be said: true once for the pixels it has, whenever they landed (added with - /// them, or read in place later, in the browser, a project or an import). - pub fn palette_unsaid(&mut self) -> bool { - let due = !self.palette_said && self.data.as_ref().is_some_and(|d| d.palette_missing); - self.palette_said |= due; - due + /// Its pixels, the first time they are asked for after they landed (added with them, or read + /// in place later: in the browser, a project, an import), so what reading them lost (a colour + /// table missing, an overview held) is said once for them. + pub fn pixels_unsaid(&mut self) -> Option> { + if self.pixels_said { + return None; + } + let d = self.data.clone()?; + self.pixels_said = true; + Some(d) } /// Installs a completed remote read; each render copy can observe the same shared result. @@ -433,6 +440,8 @@ impl RasterLayer { let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() else { return true }; self.opening = None; self.land(result); + // Installed while the map draws: a frame after it says what reading them lost. + crate::source::wake(); false } @@ -441,7 +450,7 @@ impl RasterLayer { fn land(&mut self, result: Result, String>) { match result { Ok(raster) => { - self.palette_said = false; + self.pixels_said = false; if std::mem::take(&mut self.restyle) { let look = self.look.take().and_then(|look| RasterStyle::for_look(&raster, &look)); self.style = look.unwrap_or_else(|| RasterStyle::default_for(&raster)); diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 83223919a..4dadafe66 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -528,6 +528,11 @@ type Opening = Arc, String>>>>; /// Tells the app something landed (a read, or a source opened by its address). pub(crate) fn landed() { GENERATION.fetch_add(1, Ordering::Relaxed); + wake(); +} + +/// Asks the app for a frame (the waker), with nothing new to draw. +pub(crate) fn wake() { if let Some(w) = waker().lock().ok().and_then(|w| w.clone()) { w(); } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 87485f22d..fd278d639 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -3163,7 +3163,8 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac let address = rl.path.as_deref().filter(|_| rl.in_place); let mut draw_it = |pm: &mut Pixmap, opacity: f32| match address { Some(a) => { - tile_out.pending |= raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) + tile_out.pending |= + raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) == Ok(true) } None => raster::draw(pm, data, &rl.style, stats, view, opacity, full_res_rasters, place), }; diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 4ee9ce783..266e5e900 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -111,7 +111,9 @@ const WINDOW_GRID: usize = 17; /// `r` (the layer's pixels), or, zoomed in past it, the part in view read from the level the view /// needs, drawn over the overview until its tiles arrive. `stats` are the renderer's bands' /// statistics (the overview's when not given), so the part in view is stretched as the whole is. -/// Returns whether tiles are still to come (the map is drawn again as they land). +/// Returns whether tiles are still to come (the map is drawn again as they land), or why it drew +/// less than the view needs (the overview, or other levels for tiles that couldn't be read): the +/// map shows what it has, an export refuses it. #[allow(clippy::too_many_arguments)] pub(crate) fn draw_in_place( pm: &mut Pixmap, @@ -123,7 +125,7 @@ pub(crate) fn draw_in_place( opacity: f32, full_res: bool, place: Place, -) -> bool { +) -> Result { let bands = style.renderer.bands(); let own: Vec = bands.iter().map(|b| r.bands.get(*b).map(|b| b.stats.clone()).unwrap_or_default()).collect(); let stats = stats.unwrap_or(&own); @@ -132,22 +134,22 @@ pub(crate) fn draw_in_place( let full = r.overview.unwrap_or((r.width, r.height)); if style.rotation.rem_euclid(360.0).abs() > 1e-6 || full.0 <= r.width { whole(pm); - return false; + return Ok(false); } let info = match rustgis_tiles::cog::info(address) { Ok(Some(info)) => info, Ok(None) => { whole(pm); - return true; + return Ok(true); } - Err(_) => { + Err(e) => { whole(pm); - return false; + return Err(e); } }; let Some(want) = in_view(r, view, place, pm.width(), pm.height(), full_res) else { whole(pm); - return false; + return Ok(false); }; // The coarsest level at least as fine as the view (within a quarter); zoomed in past the full // image, the full image. @@ -181,12 +183,12 @@ pub(crate) fn draw_in_place( } let Some((level, cols, rows)) = window else { whole(pm); - return false; + return Err("the view is too large a part of the image to read at the resolution it needs".into()); }; // The image at the overview's level is the overview. if levels.get(level).is_some_and(|l| l.width <= r.width) { whole(pm); - return false; + return Ok(false); } let color = match (&style.renderer, &r.color) { (_, ColorHint::Palette(p)) => ColorHint::Palette(p.clone()), @@ -196,14 +198,14 @@ pub(crate) fn draw_in_place( }; let w = match rustgis_tiles::cog::raster::window(address, level, cols, rows, &read, color) { Ok(w) => w, - Err(_) => { + Err(e) => { whole(pm); - return false; + return Err(e); } }; let Some(part) = w.raster else { whole(pm); - return w.pending; + return Ok(w.pending); }; let mut own_style = style.clone(); own_style.renderer = renumbered(&style.renderer); @@ -218,10 +220,13 @@ pub(crate) fn draw_in_place( draw(&mut both, &part, &own_style, Some(stats), view, 1.0, full_res, place); pm.draw_pixmap(0, 0, both.as_ref(), &PixmapPaint { opacity, ..PixmapPaint::default() }, Transform::identity(), None); } - return true; + return Ok(true); } draw(pm, &part, &own_style, Some(stats), view, opacity, full_res, place); - false + if w.failed { + return Err("some of its tiles couldn't be read".into()); + } + Ok(false) } /// A renderer drawing the same bands of a raster holding only those, in its order (see diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index 476b740e9..0cb3115d3 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -82,8 +82,11 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> let mut idle = 0; loop { let drawn_at = crate::rustgis_tiles::cache::generation(); - if !raster::draw_in_place(&mut image, address, r, &layer.style, stats, &view, opacity, true, place) { - break; + match raster::draw_in_place(&mut image, address, r, &layer.style, stats, &view, opacity, true, place) { + Ok(false) => break, + Ok(true) => {} + // Drawn from less than the tile needs: not written in its place. + Err(e) => return Err(format!("{}: {e}, so the archive isn't written", r.name)), } // An archive of the overview where the image is wanted isn't written. let left = deadline.saturating_duration_since(web_time::Instant::now()); @@ -148,6 +151,28 @@ mod tests { assert_eq!((last - first + 1).pow(2), 484); } + /// A raster read in place whose image can't be read isn't exported from its overview. + #[test] + #[allow(clippy::panic)] + fn an_in_place_raster_unread_is_not_exported_from_its_overview() { + let mut overview = Raster::new( + "gone", + 4, + 4, + vec![(0..16).map(|v| v as f32).collect()], + GeoTransform { origin_x: -180.0, origin_y: 80.0, pixel_width: 90.0, pixel_height: -40.0 }, + Crs::LonLat, + None, + ColorHint::Gray, + "Float32", + ) + .unwrap_or_else(|e| panic!("{e}")); + overview.overview = Some((4096, 4096)); + let layer = RasterLayer::in_place(overview, "http://127.0.0.1:9/gone.tif"); + let e = export(&layer, 2, 2, 1.0).err().unwrap_or_else(|| panic!("an archive of the overview")); + assert!(e.contains("the archive isn't written"), "{e}"); + } + #[test] #[allow(clippy::panic)] fn exports_png_tiles_and_rejects_unbounded_or_unsupported_requests() { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index c5e881973..828a8b9d5 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -1999,6 +1999,12 @@ mod tests { let records = network_records(&history, &url); assert_eq!(records.len(), 1); assert_eq!(records[0].network.as_ref().unwrap().status, Some(503)); + // One that says when to ask again: its wait goes with the status, read back as the wait. + let wait = "HTTP/1.1 503 Unavailable\r\nRetry-After: 30\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let base = serve(vec![wait.into()]); + let e = native::get(&native::agent(), &format!("{base}/tiles/0/0/0.png"), &[404]).unwrap_err(); + assert_eq!(e, "http status: 503 (retry after 30 s)"); + assert_eq!((http_status(&e), retry_wait(&e, RETRY)), (Some(503), Duration::from_secs(30))); // A download's absence names the address asked for, not the module's markers. let base = serve(vec![not_found.into()]); let url = format!("{base}/data.zarr/zarr.json"); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index a79d84333..359c632b5 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -1059,7 +1059,8 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 let colours = info.colour(look).len(); let pen = Pen { rgb: colours >= 3, - alpha: (shown.len() > colours).then_some(colours), + // The alpha band where it is among those shown: after the colours, or one of them. + alpha: info.alpha.and_then(|a| shown.iter().rposition(|b| *b == a)), alpha_max: rustgis_data::raster::alpha_max(info.layout.data_type), palette: info.palette(look).map(<[[u8; 4]]>::to_vec), aef: look.aef, diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 2fc8655eb..19940ce42 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -32,8 +32,10 @@ pub fn overview_level(info: &Info) -> Option { let layout = &info.layout; let bands = layout.bands.max(1); let planes = if layout.planar { bands } else { 1 }; + // Samples decoded, whole tiles (an edge tile's padding too), and chunks read. let size = |l: &rustgis_data::raster::cog::Level| { - let samples = (l.width as usize).saturating_mul(l.height as usize).saturating_mul(bands); + let tile = (l.tile_width as usize).saturating_mul(l.tile_height as usize); + let samples = (l.tiles() as usize).saturating_mul(tile).saturating_mul(bands); (samples, (l.tiles() as usize).saturating_mul(planes)) }; layout @@ -169,6 +171,9 @@ pub struct Window { pub raster: Option, /// Tiles (or the header) are still on their way: the map draws again as they land. pub pending: bool, + /// Tiles in it couldn't be read lately (other levels stand in for them, and they are asked + /// for again when their wait is over). + pub failed: bool, } /// Tiles whose read failed lately, and until when they aren't asked for again ([`RETRY`] after, or @@ -182,15 +187,19 @@ fn failed() -> &'static Mutex> { /// The most failed tiles remembered. const MAX_FAILED: usize = 4096; -/// Remembers a failed tile until `until`. With [`MAX_FAILED`] already, the one whose wait ends -/// soonest makes room: one forgotten would be asked for again at each draw, whatever its server -/// asked for. +/// Remembers a failed tile until `until`. With [`MAX_FAILED`] already, those whose wait is over go +/// first; failing that, the one whose wait ends soonest (asked for again a little early, once) +/// makes room, rather than the new one, which would be asked for again at each draw, whatever its +/// server asked for. fn remember_failed(f: &mut HashMap<(String, usize, u32), Instant>, key: (String, usize, u32), until: Instant) { - if f.len() >= MAX_FAILED - && !f.contains_key(&key) - && let Some(soonest) = f.iter().min_by_key(|(_, at)| **at).map(|(k, _)| k.clone()) - { - f.remove(&soonest); + if f.len() >= MAX_FAILED && !f.contains_key(&key) { + let now = Instant::now(); + f.retain(|_, at| now < *at); + if f.len() >= MAX_FAILED + && let Some(soonest) = f.iter().min_by_key(|(_, at)| **at).map(|(k, _)| k.clone()) + { + f.remove(&soonest); + } } f.insert(key, until); } @@ -201,7 +210,7 @@ fn remember_failed(f: &mut HashMap<(String, usize, u32), Instant>, key: (String, /// tile cache's generation goes up as each lands, so the map is drawn again. pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize], color: ColorHint) -> Result { let file = split(url).0; - let Some(info) = super::info(file)? else { return Ok(Window { raster: None, pending: true }) }; + let Some(info) = super::info(file)? else { return Ok(Window { raster: None, pending: true, failed: false }) }; let l = info.layout.levels.get(level).ok_or("no such overview")?; let (c0, c1) = (cols.0.min(l.width), cols.1.min(l.width)); let (r0, r1) = (rows.0.min(l.height), rows.1.min(l.height)); @@ -233,6 +242,7 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands } } let mut pending = kept.len() < chunks.len(); + let waiting: bool; { let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); let now = Instant::now(); @@ -249,6 +259,7 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands if let Some(at) = next { crate::cache::notify_at(at); } + waiting = next.is_some(); } // Tiles that failed lately aren't waited for (the map would wait for them for nothing). if ask.is_empty() && pending { @@ -297,7 +308,7 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands info.layout.data_type, ) .map_err(|e| e.to_string())?; - Ok(Window { raster: Some(raster), pending }) + Ok(Window { raster: Some(raster), pending, failed: waiting }) } /// A part of a window a chunk not kept leaves empty: the band (its place among those read), and From ba55251e82a83323d33da12b0bda4a5a2d15d793 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 19:09:06 -0330 Subject: [PATCH 05/27] fix(raster): third review round of COGs read in place - An in-place layer that can't be read says so once (an error notice), wherever it was read. - Extra-band images are taken for RGB only when the file holds no grey multiband image. - A layer saved before its pixels landed keeps its pending look and its own-look restyle. - The overview is drawn under failed tiles too, so they show it rather than holes. - layer.addUrl on an in-place COG returns overviewOf again; only the file's notice is left out. - stand_in walks the gap rectangles, allocating nothing per pixel. - Doc comments back above fetch_bytes and refused; no always-true wide_rgb flag; no layer-0 fallback; no shadowed view width; impossible HTTP dates refused; docs lines rewrapped. Co-Authored-By: Claude Opus 5.5 --- crates/data/src/raster/cog.rs | 18 ++++++++ crates/data/src/raster/geotiff.rs | 32 ++++++++----- crates/engine/src/cmd/layer.rs | 3 +- crates/engine/src/cmd/raster.rs | 6 ++- crates/engine/src/lib.rs | 18 ++++++-- crates/engine/src/tests.rs | 32 ++++++++++++- crates/map/src/raster.rs | 39 +++++++++++++--- crates/render/src/raster.rs | 32 ++++++------- crates/tiles/src/cache.rs | 2 +- crates/tiles/src/cog/raster.rs | 74 +++++++++++++++---------------- crates/tiles/src/failure.rs | 11 ++++- docs/user-guide/data-formats.md | 7 +-- 12 files changed, 192 insertions(+), 82 deletions(-) diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index cb5a2036a..4588c69ef 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -877,6 +877,24 @@ pub(crate) mod tests { } } + /// A file holding a grey image of several bands as well as an RGB one with more bands can't + /// tell which is which once the RGB one is shown to the decoder as grey: neither is taken for + /// colour. + #[test] + fn a_file_mixing_grey_and_rgb_images_of_several_bands_stays_grey() { + let e = Example { size: 300, tile: 128, bands: 4, photometric: 2, ..Example::default() }; + let mut file = example_with(&e); + // The overview's PhotometricInterpretation (tag 262, one short) from RGB to grey. + let entry = [0x06, 0x01, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00]; + let at: Vec = (0..file.len().saturating_sub(entry.len())).filter(|&i| file[i..i + entry.len()] == entry).collect(); + assert_eq!(at.len(), 3, "the full image and its overviews"); + file[at[2] + 8] = 1; + let (l, _) = pixel(&file, 150, 150); + assert_eq!((l.bands, &l.color), (4, &ColorHint::Gray)); + let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "mixed").unwrap(); + assert_eq!((r.band_count(), &r.color), (4, &ColorHint::Gray)); + } + /// A palette image whose colour table is missing is drawn in grey, and says so, read in place /// or as a file. #[test] diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 919e377f4..e9c85348f 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -591,7 +591,9 @@ pub(crate) struct Unpaletted { /// Every image is a palette image (transparency masks aside): one without its ColorMap is /// one still, its table missing. pub(crate) all_palette: bool, - /// RGB images with more samples were patched: their first three samples are red, green, blue. + /// RGB images with more samples were patched and the file has no grey image of three or more + /// bands: an image of several bands is one of them, its first three samples red, green, blue. + /// (In a file with both, they can't be told apart, so neither is taken for colour.) pub(crate) wide_rgb: bool, pos: u64, } @@ -608,12 +610,14 @@ struct Patches { /// An image that isn't a palette image (nor a transparency mask) was seen. other: bool, wide_rgb: bool, + /// A grey image of three or more bands was seen. + grey_bands: bool, } -/// Where the palette images' PhotometricInterpretation values are (and, with `wide_rgb`, those of -/// RGB images with samples past the colour ones), walking the IFD chain (classic and BigTIFF). A -/// file it can't walk has none: the decoder reports what's wrong. -fn photometric_patches(r: &mut R, wide_rgb: bool) -> Patches { +/// Where the palette images' PhotometricInterpretation values are, and those of RGB images with +/// samples past the colour ones, walking the IFD chain (classic and BigTIFF). A file it can't walk +/// has none: the decoder reports what's wrong. +fn photometric_patches(r: &mut R) -> Patches { use std::io::SeekFrom; let mut patches = Patches::default(); let mut walk = || -> Option<()> { @@ -679,6 +683,7 @@ fn photometric_patches(r: &mut R, wide_rgb: bool) -> Patches { } } patches.other |= photometric.is_some_and(|(_, p)| p != 3 && p != 4); + patches.grey_bands |= photometric.is_some_and(|(_, p)| p <= 1) && samples >= 3; match photometric { Some((at, 3)) => { patches.at.push((at, grey)); @@ -686,7 +691,7 @@ fn photometric_patches(r: &mut R, wide_rgb: bool) -> Patches { } // An RGB image whose fourth sample isn't alpha, or with five or more: RGBI // orthophotos, multispectral images. - Some((at, 2)) if wide_rgb && (samples > 4 || (samples == 4 && !matches!(extra, Some(1 | 2)))) => { + Some((at, 2)) if samples > 4 || (samples == 4 && !matches!(extra, Some(1 | 2))) => { patches.at.push((at, grey)); patches.wide_rgb = true; } @@ -713,14 +718,21 @@ pub(crate) type Reader<'a> = Unpaletted>; /// A reader for the decoder: every band of the image, an RGB image's with more too (see /// [`Unpaletted`]). pub(crate) fn reader<'a>(r: impl Read + Seek + 'a) -> std::io::Result> { - Unpaletted::new(Box::new(r), true) + Unpaletted::new(Box::new(r)) } impl Unpaletted { - fn new(mut inner: R, wide_rgb: bool) -> std::io::Result { - let p = photometric_patches(&mut inner, wide_rgb); + fn new(mut inner: R) -> std::io::Result { + let p = photometric_patches(&mut inner); inner.seek(std::io::SeekFrom::Start(0))?; - Ok(Unpaletted { inner, patches: p.at, palette: p.palette, all_palette: p.palette && !p.other, wide_rgb: p.wide_rgb, pos: 0 }) + Ok(Unpaletted { + inner, + patches: p.at, + palette: p.palette, + all_palette: p.palette && !p.other, + wide_rgb: p.wide_rgb && !p.grey_bands, + pos: 0, + }) } } diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index a972a6e2e..0a1242a17 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3037,7 +3037,8 @@ fn add_in_place(s: &mut Session, name: &str, url: &str, strict: bool) -> CmdResu } r["inPlace"] = json!(true); r["kind"] = json!("raster"); - r["name"] = json!(s.map().layer(r["layer"].as_u64().unwrap_or(0) as u32).map(|l| l.name.clone())); + let added = r["layer"].as_u64().and_then(|id| u32::try_from(id).ok()).and_then(|id| s.map().layer(id)); + r["name"] = json!(added.map(|l| l.name.clone())); Ok(r) } diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index e6f88bd34..78c4ba1e3 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -354,11 +354,13 @@ pub fn add_raster(s: &mut Session, name: &str, mut layer: RasterLayer) -> CmdRes // Read in place, the map shows the full image as it draws, but statistics and tools read the // overview held: said so. let held = unsaid.filter(|_| layer.in_place).and_then(|d| crate::in_place_overview(&d)); - let overview = overview.filter(|_| !layer.in_place); + let in_place = layer.in_place; let empty_before = s.map().layers.is_empty(); let id = s.map_mut().add_raster(unique.clone(), layer); s.active_layer = Some(id); - if let Some((fw, fh)) = overview { + // Read in place, its own notice says the overview it holds (`held`); the result names the full + // size all the same. + if let Some((fw, fh)) = overview.filter(|_| !in_place) { s.notify(format!("{unique}: showing a {w} × {h} overview of the {fw} × {fh} raster"), false); } if palette_missing { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3c10bb7d1..f7544857a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1516,13 +1516,17 @@ impl Session { /// Says what reading raster layers' pixels lost when it wasn't said as each was added (pixels /// read in place land later: in the browser, a project, an import): a palette image without - /// its colour table, drawn in grey; an overview statistics and tools read. Once for each - /// layer's pixels, one notice of each kind; the app calls it each frame. + /// its colour table, drawn in grey; an overview statistics and tools read; a layer read in place + /// that couldn't be read, and why. Once for each layer's pixels or failure, one notice of each + /// kind; the app calls it each frame. pub fn notice_landed_rasters(&mut self) { - let (mut grey, mut held) = (Vec::new(), Vec::new()); + let (mut grey, mut held, mut failed) = (Vec::new(), Vec::new(), Vec::new()); for m in &mut self.project.maps { for l in &mut m.layers { let Some(rl) = &mut l.raster else { continue }; + if let Some(e) = rl.error_unsaid() { + failed.push((l.name.clone(), e)); + } let in_place = rl.in_place; let Some(d) = rl.pixels_unsaid() else { continue }; if d.palette_missing { @@ -1541,6 +1545,14 @@ impl Session { false, ), } + match &failed[..] { + [] => {} + [(one, e)] => self.notify(format!("{}: couldn't be read in place: {e}", clip(one, 80)), true), + many => { + let names: Vec = many.iter().map(|(n, _)| n.clone()).collect(); + self.notify(format!("{} raster layers couldn't be read in place: {}", many.len(), names_said(&names)), true); + } + } match &held[..] { [] => {} [(one, what)] => self.notify(format!("{}: {what}", clip(one, 80)), false), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 3cb065f97..a46041e7f 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13325,6 +13325,34 @@ fn an_address_look_holds_when_the_pixels_land_later() { assert!(read_together(&mut [&mut stretched, &mut own]).is_empty()); assert!(matches!(stretched.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", stretched.style.renderer); assert_eq!(own.style.renderer, RasterRenderer::Colormap); + // Saved before its pixels landed (the browser, a read that failed), a layer opens in its look, + // or its own one, still. + let mut s = Session::new(Project::default()); + let look = AddressLook { bands: vec![4, 2, 1], range: Some((10.0, 200.0)) }; + s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place(format!("{base}/scene.tif")).with_look(look)); + s.map_mut().add_raster("Classes".to_string(), RasterLayer::unread_in_place(format!("{classes}/classes.tif"))); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let style = + |name: &str| s.map().layers.iter().find(|l| l.name == name).and_then(|l| l.raster.as_ref()).map(|r| r.style.clone()).unwrap(); + assert_eq!(style("Scene").renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); + assert_eq!(style("Scene").stretch, Stretch::custom(10.0, 200.0)); + assert_eq!(style("Classes").renderer, RasterRenderer::Colormap); +} + +/// A layer read in place that can't be read says so, once, as its project opens. +#[test] +fn an_in_place_layer_that_cant_be_read_says_so() { + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Gone".to_string(), rustgis_map::raster::RasterLayer::unread_in_place("http://127.0.0.1:9/gone.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Gone: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices.last()); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), 1); } /// A palette COG without its colour table is drawn in grey and says so, also when its pixels land @@ -13366,7 +13394,9 @@ fn in_place_layers_say_their_overview_once() { let (base, _) = serve_ranges(rustgis_data::raster::cog::example(2048, 256)); let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("raster tools read a 1024 × 1024 overview")).count(); let mut s = Session::new(Project::default()); - s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif")})).unwrap(); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif")})).unwrap(); + // The result names the full size, as a file opened from an overview's does. + assert_eq!((r["width"].clone(), r["overviewOf"].clone()), (json!(1024), json!([2048, 2048])), "{r}"); s.run("map.zoomIn", &Value::Null).unwrap(); assert_eq!(said(&s), 1, "{:?}", s.notices.last()); let text = serde_json::to_string(&s.project).unwrap(); diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 9f7be44c4..f5c9c1243 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -60,15 +60,20 @@ pub struct RasterLayer { /// A remote reload shared with render snapshots, never serialized. #[serde(skip)] opening: Option, - /// The style is the file's own look once its pixels land (a layer added before they have). - #[serde(skip)] + /// The style is the file's own look once its pixels land (a layer added before they have; + /// saved with a project saved before they did, so it opens in that look still). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] restyle: bool, - /// The look its address gave a layer read in place, applied when its pixels land. - #[serde(skip)] + /// The look its address gave a layer read in place, applied when its pixels land (saved, as + /// `restyle` is, until they have). + #[serde(default, skip_serializing_if = "Option::is_none")] look: Option, /// What reading its pixels lost has been said (see [`RasterLayer::pixels_unsaid`]). #[serde(skip)] pixels_said: bool, + /// Why it couldn't be read in place has been said (see [`RasterLayer::error_unsaid`]). + #[serde(skip)] + error_said: bool, } /// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. @@ -208,6 +213,7 @@ impl RasterLayer { restyle: false, look: None, pixels_said: false, + error_said: false, }) } @@ -231,6 +237,7 @@ impl RasterLayer { restyle: false, look: None, pixels_said: false, + error_said: false, } } @@ -254,6 +261,7 @@ impl RasterLayer { restyle: false, look: None, pixels_said: false, + error_said: false, } } @@ -282,6 +290,7 @@ impl RasterLayer { restyle: true, look: None, pixels_said: false, + error_said: false, } } @@ -321,11 +330,13 @@ impl RasterLayer { let Some(address) = self.path.clone() else { let e = "the raster has no source".to_string(); self.error = Some(e.clone()); + self.error_said = false; return Err(e); }; let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { let e = format!("{address}: can't be read in place here"); self.error = Some(e.clone()); + self.error_said = false; return Err(e); }; self.data = None; @@ -434,6 +445,17 @@ impl RasterLayer { Some(d) } + /// Why a layer read in place couldn't be read, the first time it is asked for after the read + /// failed, so it is said once (a project, an import or the browser read it after adding it). + pub fn error_unsaid(&mut self) -> Option { + if !self.in_place || self.error_said { + return None; + } + let e = self.error.clone()?; + self.error_said = true; + Some(e) + } + /// Installs a completed remote read; each render copy can observe the same shared result. pub(crate) fn poll_reload(&mut self) -> bool { let Some(slot) = self.opening.as_ref() else { return false }; @@ -459,7 +481,10 @@ impl RasterLayer { } self.data = Some(raster); } - Err(error) => self.error = Some(error), + Err(error) => { + self.error = Some(error); + self.error_said = false; + } } } @@ -935,9 +960,11 @@ fn one() -> f64 { /// The look an address gives a GeoTIFF read in place (`…/scene.tif#bands=5,3,2&range=0,3000`, as /// tile layers kept it): the bands drawn (from 0; one in grey, or red, green and blue) and a fixed /// stretch. -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct AddressLook { + #[serde(default, skip_serializing_if = "Vec::is_empty")] pub bands: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] pub range: Option<(f64, f64)>, } diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 266e5e900..290ab6cc9 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -209,9 +209,10 @@ pub(crate) fn draw_in_place( }; let mut own_style = style.clone(); own_style.renderer = renumbered(&style.renderer); - // Until its tiles arrive, the part in view goes over the overview: both drawn opaque, then - // blended once, so a see-through layer isn't darker where they overlap. - if w.pending { + // While it has gaps (tiles to come, or that couldn't be read and no level kept stands in for), + // the part in view goes over the overview: both drawn opaque, then blended once, so a + // see-through layer isn't darker where they overlap. + if w.pending || w.failed { if opacity >= 1.0 { whole(pm); draw(pm, &part, &own_style, Some(stats), view, 1.0, full_res, place); @@ -220,13 +221,14 @@ pub(crate) fn draw_in_place( draw(&mut both, &part, &own_style, Some(stats), view, 1.0, full_res, place); pm.draw_pixmap(0, 0, both.as_ref(), &PixmapPaint { opacity, ..PixmapPaint::default() }, Transform::identity(), None); } - return Ok(true); + } else { + draw(pm, &part, &own_style, Some(stats), view, opacity, full_res, place); } - draw(pm, &part, &own_style, Some(stats), view, opacity, full_res, place); - if w.failed { - return Err("some of its tiles couldn't be read".into()); + match (w.pending, w.failed) { + (true, _) => Ok(true), + (false, true) => Err("some of its tiles couldn't be read".into()), + (false, false) => Ok(false), } - Ok(false) } /// A renderer drawing the same bands of a raster holding only those, in its order (see @@ -257,9 +259,9 @@ struct InView { /// `None` when it isn't in view. fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_res: bool) -> Option { let k = if view.pixels_per_point >= 1.5 && !full_res { view.pixels_per_point } else { 1.0 }; - let (w, h) = (f64::from(width) / k, f64::from(height) / k); + let (vw, vh) = (f64::from(width) / k, f64::from(height) / k); let n = WINDOW_GRID; - let at = |i: usize, j: usize| source(r, view, place, k, w * i as f64 / (n - 1) as f64, h * j as f64 / (n - 1) as f64); + let at = |i: usize, j: usize| source(r, view, place, k, vw * i as f64 / (n - 1) as f64, vh * j as f64 / (n - 1) as f64); let grid: Vec> = (0..n * n).map(|p| at(p % n, p / n)).collect(); let (mut c0, mut c1, mut r0, mut r1) = (f64::INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::NEG_INFINITY); let (rw, rh) = (f64::from(r.width), f64::from(r.height)); @@ -267,10 +269,10 @@ fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_ // `Raster::wrap_col`): seen across that seam, the part in view is the whole width. let mut wrapped = false; for (c, row) in grid.iter().flatten() { - let w = r.wrap_col(*c); - wrapped |= w != *c; - c0 = c0.min(w.clamp(0.0, rw)); - c1 = c1.max(w.clamp(0.0, rw)); + let col = r.wrap_col(*c); + wrapped |= col != *c; + c0 = c0.min(col.clamp(0.0, rw)); + c1 = c1.max(col.clamp(0.0, rw)); r0 = r0.min(row.clamp(0.0, rh)); r1 = r1.max(row.clamp(0.0, rh)); } @@ -282,7 +284,7 @@ fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_ } // The raster's pixels a screen pixel spans: the smallest step between neighbouring points, // where the view is finest (a tilted or curved view is finer near its centre). - let step = (w / (n - 1) as f64).max(1.0); + let step = (vw / (n - 1) as f64).max(1.0); let mut per_px = f64::INFINITY; for j in 0..n { for i in 0..n - 1 { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 828a8b9d5..b332ec3bf 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -1535,12 +1535,12 @@ mod native { Failure::without_answer("", e.to_string(), reason) } - /// An answer with an error status (#509): its kept headers and the start of its body. /// An answer's `Retry-After` header, as sent. fn retry_after_header(resp: &ureq::http::Response) -> Option<&str> { resp.headers().get("retry-after").and_then(|v| v.to_str().ok()) } + /// An answer with an error status (#509): its kept headers and the start of its body. fn refused(resp: &mut ureq::http::Response) -> Failure { use std::io::Read; let headers: Vec<(String, String)> = resp diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 19940ce42..d9a09b3c4 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -338,14 +338,8 @@ fn stand_in( let levels = &info.layout.levels; let Some(l) = levels.get(level) else { return }; let ww = (cols.1 - cols.0) as usize; - // The pixels still to fill, by band (its place among those read), as places in the window. - let mut left: Vec<(usize, Vec)> = gaps - .iter() - .map(|g| { - let at = |y: u32, x: u32| (y - rows.0) as usize * ww + (x - cols.0) as usize; - (g.k, (g.rows.0..g.rows.1).flat_map(|y| (g.cols.0..g.cols.1).map(move |x| at(y, x))).collect()) - }) - .collect(); + // The gaps a level kept hasn't filled yet: walked a row at a time, nothing kept per pixel. + let mut open: Vec<&Gap> = gaps.iter().collect(); // The other levels, nearest first. let others: Vec = (1..levels.len()).flat_map(|d| [level.checked_sub(d), Some(level + d)]).flatten().filter(|o| *o < levels.len()).collect(); @@ -355,41 +349,43 @@ fn stand_in( let (tw, th) = (ol.tile_width.max(1), ol.tile_height.max(1)); // The chunks of that level found so far (none: not kept). let mut found: HashMap> = HashMap::new(); - for (k, todo) in &mut left { - let (Some(&b), Some(dst)) = (bands.get(*k), data.get_mut(*k)) else { - todo.clear(); - continue; - }; + open.retain(|g| { + let (Some(&b), Some(dst)) = (bands.get(g.k), data.get_mut(g.k)) else { return false }; + // Whether a pixel still empty found that level's tile not kept. + let mut short = false; // The chunk the last pixel came from: its neighbours mostly share it. let mut last: Option<(u32, Option)> = None; - todo.retain(|&i| { - let (x, y) = (cols.0 as usize + i % ww, rows.0 as usize + i / ww); - // The pixel of that level under this one's centre. - let ox = (((x as f64 + 0.5) * sx) as u32).min(ol.width.saturating_sub(1)); - let oy = (((y as f64 + 0.5) * sy) as u32).min(ol.height.saturating_sub(1)); - let tile = (oy / th) * ol.across() + ox / tw; - let Some((c, at)) = info.layout.chunk(ol, tile, b) else { return true }; - if last.as_ref().is_none_or(|(lc, _)| *lc != c) { - let chunk = found.entry(c).or_insert_with(|| t.used(&(file.to_string(), o, c))).clone(); - last = Some((c, chunk)); - } - let value = last - .as_ref() - .and_then(|(_, ch)| ch.as_ref()) - .and_then(|ch| ch.get(at)) - .and_then(|p| p.get(((oy % th) * tw + ox % tw) as usize)); - match (value, dst.get_mut(i)) { - // That level's value, no data included: it is what the image has there. - (Some(s), Some(v)) => { - *v = *s; - false + for y in g.rows.0..g.rows.1 { + let oy = (((f64::from(y) + 0.5) * sy) as u32).min(ol.height.saturating_sub(1)); + let row = (y - rows.0) as usize * ww; + for x in g.cols.0..g.cols.1 { + let Some(v) = dst.get_mut(row + (x - cols.0) as usize).filter(|v| v.is_nan()) else { continue }; + // The pixel of that level under this one's centre. + let ox = (((f64::from(x) + 0.5) * sx) as u32).min(ol.width.saturating_sub(1)); + let tile = (oy / th) * ol.across() + ox / tw; + let Some((c, at)) = info.layout.chunk(ol, tile, b) else { + short = true; + continue; + }; + if last.as_ref().is_none_or(|(lc, _)| *lc != c) { + let chunk = found.entry(c).or_insert_with(|| t.used(&(file.to_string(), o, c))).clone(); + last = Some((c, chunk)); + } + let value = last + .as_ref() + .and_then(|(_, ch)| ch.as_ref()) + .and_then(|ch| ch.get(at)) + .and_then(|p| p.get(((oy % th) * tw + ox % tw) as usize)); + match value { + // That level's value, no data included: it is what the image has there. + Some(s) => *v = *s, + None => short = true, } - _ => true, } - }); - } - left.retain(|(_, todo)| !todo.is_empty()); - if left.is_empty() { + } + short + }); + if open.is_empty() { return; } } diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index a0ea4802c..e7cc4c53e 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -233,7 +233,14 @@ fn http_date(s: &str) -> Option { let (day, year): (i64, i64) = (day.parse().ok()?, year.parse().ok()?); let mut hms = time.split(':').map(|p| p.parse::().ok()); let (h, m, sec) = (hms.next()??, hms.next()??, hms.next()??); - if !(1..=31).contains(&day) || !(1970..=9999).contains(&year) || h > 23 || m > 59 || sec > 60 || hms.next().is_some() { + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let days_in_month = match month { + 2 if leap => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + if !(1..=days_in_month).contains(&day) || !(1970..=9999).contains(&year) || h > 23 || m > 59 || sec > 60 || hms.next().is_some() { return None; } // Days since 1970 of a date in the proleptic Gregorian calendar (Howard Hinnant's algorithm). @@ -458,6 +465,8 @@ mod tests { assert_eq!(retry_after_secs("Sun, 06 Nov 1994 08:49:37 GMT", 784_200_000), Some(1), "past: at once, after a second"); assert_eq!(http_date("Sun, 06 Nov 1994 08:49:37 PST"), None); assert_eq!(http_date("Sun, 32 Nov 1994 08:49:37 GMT"), None); + assert_eq!(http_date("Wed, 31 Feb 2024 00:00:00 GMT"), None, "no 31 February"); + assert!(http_date("Thu, 29 Feb 2024 00:00:00 GMT").is_some() && http_date("Sat, 29 Feb 2025 00:00:00 GMT").is_none()); // A refused answer's own header. let headers = vec![("Retry-After".to_string(), "12".to_string())]; assert_eq!(Failure::answered("u", 429, &headers, b"").message, "http status: 429 (retry after 12 s)"); diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index bc001c5bd..c4ea310d2 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -136,8 +136,8 @@ too. A dated series of GeoTIFFs (an address with date placeholders), a data cube AlphaEarth's embeddings are tile layers that follow the time slider or dequantise their values; see [Satellite embeddings](web-data.md#satellite-embeddings). A GeoTIFF in strips, a rotated image or an unsupported tiled image downloads whole, as does a -GeoTIFF added with `"whole": true` (for raster tools that need every pixel at full resolution). The web app needs the server -to allow range requests (CORS with the `Range` header). +GeoTIFF added with `"whole": true` (for raster tools that need every pixel at full resolution). +The web app needs the server to allow range requests (CORS with the `Range` header). A GeoParquet or FlatGeobuf file by its address is read in place too, a view at a time, however large: the layer is streamed like a large local file, and the view's features are fetched in the @@ -156,7 +156,8 @@ desktop, while in the web app they need the file downloaded first. On desktop, `layer.addUrl` waits for downloads, up to a minute; the From URL… dialog (`ui.addUrl`) and a plugin's `layer.addUrl` work in the background. In the web app the same dialog handles asynchronous downloads. -`layer.addUrl` can add PMTiles archives and GeoTIFFs read in place directly, but an ordinary download returns +`layer.addUrl` can add PMTiles archives and GeoTIFFs read in place directly, but an ordinary +download returns `{"pending": true, "url": "…"}` and is added later by the app. To open a remote GeoParquet or FlatGeobuf whose header hasn't been fetched yet, use the From URL… dialog or a `?data=` link. From b5b668409983436f442a3d5c8de5c7f15285753f Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 19:41:40 -0330 Subject: [PATCH 06/27] fix(raster): fourth review round of COGs read in place - A view read from a coarser level than it needs (too large a part for the tiles kept) is reported, so exports refuse it; the map still draws it. - Dated-series keys given as null ask for nothing: the GeoTIFF stays a raster read in place. - stand_in looks kept tiles up one at a time, never holding the tile cache across the fill. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/layer.rs | 6 ++++-- crates/engine/src/tests.rs | 12 ++++++++++++ crates/render/src/raster.rs | 4 ++++ crates/tiles/src/cog/raster.rs | 17 +++++++++-------- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 0a1242a17..088073405 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2973,8 +2973,10 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) // (which follow the time slider) and AlphaEarth's quantised embeddings (which raster styles // don't dequantise) are tile layers. let (file, look) = crate::render::rustgis_tiles::cog::split(url); + // A key given as `null` (a script's unused one) asks for nothing. + let given = |k: &str| p.get(k).filter(|v| !v.is_null()); let dated = rustgis_data::time::is_template(file) - || ["start", "end", "unit", "interval", "dates"].iter().any(|k| p.get(k).is_some()) + || ["start", "end", "unit", "interval", "dates"].iter().any(|k| given(k).is_some()) || b(p, "cube") == Some(true); if !dated && !look.aef { return add_in_place(s, name, url, strict); @@ -2982,7 +2984,7 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) let mut q = json!({"url": url, "kind": "cog", "name": name, "top": true}); // A dated address's span and steps (see map.addTiles). for k in ["start", "end", "unit", "interval", "dates"] { - if let Some(v) = p.get(k) { + if let Some(v) = given(k) { q[k] = v.clone(); } } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index a46041e7f..42bf0ecc5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13387,6 +13387,18 @@ fn a_palette_without_its_table_says_so_when_its_pixels_land() { } } +/// Dated-series keys given as `null` (a script's unused ones) ask for nothing: the GeoTIFF is a +/// raster layer read in place, not a tile layer. +#[test] +fn null_series_keys_leave_a_geotiff_read_in_place() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + let p = json!({"url": format!("{base}/scene.tif"), "start": null, "end": null, "dates": null}); + let r = s.run("layer.addUrl", &p).unwrap(); + assert_eq!((r["kind"].clone(), r["inPlace"].clone()), (json!("raster"), json!(true)), "{r}"); + assert!(s.map().tiles.is_empty()); +} + /// A layer read in place says its statistics and tools read an overview: once when added, once /// when a project with it opens (its pixels landing then), and not again for the same pixels. #[test] diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 290ab6cc9..c72c4eff7 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -157,6 +157,7 @@ pub(crate) fn draw_in_place( let full_px = want.per_px * f64::from(full.0) / f64::from(r.width.max(1)); let fine_enough = |l: &rustgis_data::raster::cog::Level| f64::from(full.0) / f64::from(l.width.max(1)) <= full_px * 1.25; let mut level = levels.iter().rposition(fine_enough).unwrap_or(0); + let needed = level; // The renderer's bands, then the image's alpha band (again, when a colour draws it too), read // as the window's own bands, as `draw` reads a raster held whole. let mut read = bands.clone(); @@ -227,6 +228,9 @@ pub(crate) fn draw_in_place( match (w.pending, w.failed) { (true, _) => Ok(true), (false, true) => Err("some of its tiles couldn't be read".into()), + // Read coarser than the view needs (too large a part for the tiles kept): the map shows + // it, an export refuses it. + (false, false) if level > needed => Err("the view is too large a part of the image to read at the resolution it needs".into()), (false, false) => Ok(false), } } diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index d9a09b3c4..7aad36d2a 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use rustgis_data::raster::{ColorHint, Raster}; use web_time::Instant; -use super::{BUDGET, Bands, Info, RETRY, TileState, Tiles, fetch_all, get_tile, split, tiles, with_info}; +use super::{BUDGET, Bands, Info, RETRY, TileState, fetch_all, get_tile, split, tiles, with_info}; /// The most samples (every band) of the overview a layer holds. const OVERVIEW_SAMPLES: usize = if cfg!(target_arch = "wasm32") { 1 << 20 } else { 2 << 20 }; @@ -287,8 +287,9 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands let (mut data, gaps) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); // Until they arrive, its missing tiles show the levels kept (the one zoomed from), not holes. if !gaps.is_empty() { - let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); - stand_in(&mut t, &info, file, level, (c0, c1), (r0, r1), bands, &mut data, &gaps); + // The tiles kept, looked up one at a time: the cache isn't held while the gaps fill. + let kept = |o: usize, c: u32| tiles().lock().unwrap_or_else(|e| e.into_inner()).used(&(file.to_string(), o, c)); + stand_in(&info, level, (c0, c1), (r0, r1), bands, &mut data, &gaps, kept); } let t = info.layout.transform_of(l); let transform = rustgis_data::raster::GeoTransform { @@ -322,18 +323,17 @@ struct Gap { /// Fills the `gaps` of a window of `level` from the other levels' tiles kept, the nearest first /// (the finer, then the coarser, then further): the level zoomed from stands in for the one zoomed /// to until its tiles arrive. Only the gaps are filled: a tile's own no-data pixels stay empty. -/// Nothing is fetched. +/// `kept` gives a level's chunk when it is kept (marking it used); nothing is fetched. #[allow(clippy::too_many_arguments)] fn stand_in( - t: &mut Tiles, info: &Info, - file: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize], data: &mut [Vec], gaps: &[Gap], + mut kept: impl FnMut(usize, u32) -> Option, ) { let levels = &info.layout.levels; let Some(l) = levels.get(level) else { return }; @@ -368,7 +368,7 @@ fn stand_in( continue; }; if last.as_ref().is_none_or(|(lc, _)| *lc != c) { - let chunk = found.entry(c).or_insert_with(|| t.used(&(file.to_string(), o, c))).clone(); + let chunk = found.entry(c).or_insert_with(|| kept(o, c)).clone(); last = Some((c, chunk)); } let value = last @@ -438,6 +438,7 @@ fn assemble( mod tests { use super::*; + use super::super::Tiles; use rustgis_data::raster::cog::{Example, Level, Parse, Sparse, example_with, layout}; /// A file's info, as reading its header gives it. @@ -491,7 +492,7 @@ mod tests { let (cols, rows) = ((0, 32), (0, 16)); let (mut data, gaps) = assemble(&info, 0, cols, rows, &[0], |c| (c == 0).then(|| own.clone())); assert_eq!(gaps.len(), 1); - stand_in(&mut t, &info, file, 0, cols, rows, &[0], &mut data, &gaps); + stand_in(&info, 0, cols, rows, &[0], &mut data, &gaps, |o, c| t.used(&(file.to_string(), o, c))); let at = |x: usize, y: usize| data[0][y * 32 + x]; assert_eq!((at(0, 0), at(20, 5), at(31, 15)), (3.0, 7.0, 7.0), "tile 1 from level 1"); assert!(at(1, 1).is_nan(), "tile 0's no-data pixel stays empty"); From 2585cc4aae0077bd84825a6caa01b0d8165a36cf Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 20:18:26 -0330 Subject: [PATCH 07/27] fix(raster): fifth review round of COGs read in place - A view that falls back to the overview when it needs a finer level is refused by exports too. - The wait read back from an error's text is bounded as when written: no overflow. - The tile cache evicts through an ordered index, not a scan; a re-kept tile counts once. - Opening a COG tile layer as a raster layer says which settings (attribution, zoom range, bounds) a raster layer hasn't, once. - One in-place wait constant; the address split once; the Details pane named in the docs. Co-Authored-By: Claude Fable 5.1 --- crates/data/src/raster/mod.rs | 4 +++ crates/engine/src/cmd/layer.rs | 7 +++-- crates/engine/src/cmd/tiles.rs | 15 ++++++++++- crates/engine/src/lib.rs | 21 ++++++++++++++- crates/engine/src/tests.rs | 14 +++++++++- crates/map/src/raster.rs | 29 +++++++++++++++++---- crates/render/src/raster.rs | 12 ++++++--- crates/render/src/raster_export.rs | 4 +-- crates/tiles/src/cache.rs | 5 +++- crates/tiles/src/cog.rs | 42 +++++++++++++++++++++--------- crates/tiles/src/cog/raster.rs | 2 +- crates/tiles/src/failure.rs | 2 +- docs/user-guide/data-formats.md | 2 +- 13 files changed, 123 insertions(+), 36 deletions(-) diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index 67ec88ab9..f1f7dcc5b 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -376,6 +376,10 @@ impl Band { } } +/// The longest a raster read in place is waited for natively: its overview as a layer opens, its +/// tiles as it is exported. +pub const IN_PLACE_WAIT: std::time::Duration = std::time::Duration::from_secs(120); + /// An alpha band's opaque value for a pixel type (`UInt16`: 65535): the type's largest, 1 for /// floating point (as TIFF keeps it), 255 for bytes and anything else. pub fn alpha_max(data_type: &str) -> f32 { diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 088073405..b3e5fb209 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2979,7 +2979,7 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) || ["start", "end", "unit", "interval", "dates"].iter().any(|k| given(k).is_some()) || b(p, "cube") == Some(true); if !dated && !look.aef { - return add_in_place(s, name, url, strict); + return add_in_place(s, name, file, &look, strict); } let mut q = json!({"url": url, "kind": "cog", "name": name, "top": true}); // A dated address's span and steps (see map.addTiles). @@ -3014,9 +3014,8 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) /// Adds a cloud-optimised GeoTIFF as a raster layer read in place: its overview held (read now, /// natively; in the browser as it lands), the part in view read as the map draws. A look at the /// address's end (`#bands=…&range=…`) is its style. -fn add_in_place(s: &mut Session, name: &str, url: &str, strict: bool) -> CmdResult { +fn add_in_place(s: &mut Session, name: &str, file: &str, look: &crate::render::rustgis_tiles::cog::Look, strict: bool) -> CmdResult { use crate::render::rustgis_tiles::cog; - let (file, look) = cog::split(url); // Strict, natively: read now, failing when it can't be. Otherwise read with the other layers // added with it (see `project::read_in_place`); in the browser, landing later. #[cfg(not(target_arch = "wasm32"))] @@ -3030,7 +3029,7 @@ fn add_in_place(s: &mut Session, name: &str, url: &str, strict: bool) -> CmdResu let _ = strict; rustgis_map::raster::RasterLayer::opening_in_place(file) }; - let layer = layer.with_look(super::tiles::address_look(&look)); + let layer = layer.with_look(super::tiles::address_look(look)); let mut r = super::raster::add_raster(s, name, layer)?; if let Ok(Some(info)) = cog::info(file) { let [w, so, e, n] = info.bounds; diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 36bb00643..874c30978 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -546,9 +546,22 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { } let t = m.tiles.remove(i); let (file, look) = cog::split(&t.url); + // What a raster layer hasn't, said once the project is open (see + // `Session::notice_landed_rasters`). + let plain = rustgis_map::TileLayer::new(&t.name, TileKind::Cog, file); + let mut dropped = Vec::new(); + if !t.attribution.trim().is_empty() { + dropped.push("attribution"); + } + if (t.min_zoom, t.max_zoom) != (plain.min_zoom, plain.max_zoom) { + dropped.push("zoom range"); + } + if t.bounds.is_some() { + dropped.push("bounds"); + } // Read with the project's other rasters read in place (see `project::read_in_place`), in // its look once it lands. - let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)); + let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)).from_tiles_dropping(dropped); let id = m.add_raster(t.name.clone(), rl); if let Some(l) = m.layer_mut(id) { l.visible = t.visible; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f7544857a..df8903f1d 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1520,13 +1520,17 @@ impl Session { /// that couldn't be read, and why. Once for each layer's pixels or failure, one notice of each /// kind; the app calls it each frame. pub fn notice_landed_rasters(&mut self) { - let (mut grey, mut held, mut failed) = (Vec::new(), Vec::new(), Vec::new()); + let (mut grey, mut held, mut failed, mut dropped) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); for m in &mut self.project.maps { for l in &mut m.layers { let Some(rl) = &mut l.raster else { continue }; if let Some(e) = rl.error_unsaid() { failed.push((l.name.clone(), e)); } + let lost = rl.dropped_unsaid(); + if !lost.is_empty() { + dropped.push((l.name.clone(), lost)); + } let in_place = rl.in_place; let Some(d) = rl.pixels_unsaid() else { continue }; if d.palette_missing { @@ -1545,6 +1549,21 @@ impl Session { false, ), } + if !dropped.is_empty() { + let mut kinds: Vec<&str> = dropped.iter().flat_map(|(_, k)| k.iter().copied()).collect(); + kinds.sort_unstable(); + kinds.dedup(); + let names: Vec = dropped.iter().map(|(n, _)| n.clone()).collect(); + self.notify( + format!( + "{} COG tile layer(s) opened as raster layers read in place, which have no {}: not kept for {}", + dropped.len(), + kinds.join(", "), + names_said(&names) + ), + false, + ); + } match &failed[..] { [] => {} [(one, e)] => self.notify(format!("{}: couldn't be read in place: {e}", clip(one, 80)), true), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 42bf0ecc5..30724049e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13430,7 +13430,13 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { s.run("layer.addData", &json!({"path": "points.geojson", "text": "{\"type\":\"FeatureCollection\",\"features\":[]}"})).ok(); let mut text: Value = serde_json::to_value(&s.project).unwrap(); let tile = |name: &str, url: String, opacity: f64, visible: bool| { - let mut t = serde_json::to_value(rustgis_map::TileLayer::new(name, rustgis_map::TileKind::Cog, url)).unwrap(); + let mut l = rustgis_map::TileLayer::new(name, rustgis_map::TileKind::Cog, url); + // Scene's credit and zoom range: what a raster layer hasn't. + if name == "Scene" { + l.attribution = "© Example".into(); + l.min_zoom = 5; + } + let mut t = serde_json::to_value(l).unwrap(); t["opacity"] = json!(opacity); t["visible"] = json!(visible); t @@ -13458,6 +13464,12 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { let names: Vec<&str> = t.map().layers.iter().map(|l| l.name.as_str()).collect(); assert_eq!(names[names.len() - 2..], ["Scene", "Aerial"], "{names:?}"); assert_eq!(t.map().tiles.iter().map(|x| x.name.as_str()).collect::>(), ["Series"]); + // Scene's settings a raster layer hasn't are said to be dropped, once; Aerial had none. + let said = |t: &Session| t.notices.iter().filter(|n| n.text.contains("which have no attribution, zoom range")).count(); + assert_eq!(said(&t), 1, "{:?}", t.notices.last()); + assert!(t.notices.iter().any(|n| n.text.contains("not kept for Scene") && !n.text.contains("Aerial")), "{:?}", t.notices.last()); + t.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&t), 1); } /// A server answering every request with `body`, logging each as `HEAD`, `GET` or `GET `. diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index f5c9c1243..a95edec2b 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -74,13 +74,14 @@ pub struct RasterLayer { /// Why it couldn't be read in place has been said (see [`RasterLayer::error_unsaid`]). #[serde(skip)] error_said: bool, + /// Settings of the tile layer this one was opened from that a raster layer hasn't (see + /// [`RasterLayer::dropped_unsaid`]). + #[serde(skip)] + dropped: Vec<&'static str>, } /// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. pub type InPlaceRead = Box) + Send>; -/// The longest a layer read in place waits for its overview natively. -#[cfg(not(target_arch = "wasm32"))] -const IN_PLACE_WAIT: std::time::Duration = std::time::Duration::from_secs(120); /// Reads the overviews of `layers` read in place and not read yet, all at once: a project of /// several opens in the time of the slowest, not of them all (waited for natively, at most two @@ -90,7 +91,7 @@ pub fn read_together(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { let mut failed = Vec::new(); let unread = |l: &RasterLayer| l.in_place && l.data.is_none() && l.opening.is_none() && l.time_zarr.is_none(); #[cfg(not(target_arch = "wasm32"))] - let deadline = std::time::Instant::now() + IN_PLACE_WAIT; + let deadline = std::time::Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; #[cfg(not(target_arch = "wasm32"))] let mut started = Vec::new(); for (i, l) in layers.iter_mut().enumerate() { @@ -214,6 +215,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + dropped: Vec::new(), }) } @@ -238,6 +240,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + dropped: Vec::new(), } } @@ -262,6 +265,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + dropped: Vec::new(), } } @@ -291,6 +295,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + dropped: Vec::new(), } } @@ -351,7 +356,7 @@ impl RasterLayer { }), ); #[cfg(not(target_arch = "wasm32"))] - return self.wait_for_read(std::time::Instant::now() + IN_PLACE_WAIT); + return self.wait_for_read(std::time::Instant::now() + rustgis_data::raster::IN_PLACE_WAIT); #[cfg(target_arch = "wasm32")] return Ok(()); } @@ -445,6 +450,20 @@ impl RasterLayer { Some(d) } + /// The layer, opened from a tile layer whose `dropped` settings (`"attribution"`, `"zoom + /// range"`, `"bounds"`) a raster layer hasn't, so they aren't kept: said once, by + /// [`RasterLayer::dropped_unsaid`]. + pub fn from_tiles_dropping(mut self, dropped: Vec<&'static str>) -> RasterLayer { + self.dropped = dropped; + self + } + + /// The tile layer settings dropped when this layer was opened from one, the first time they + /// are asked for. + pub fn dropped_unsaid(&mut self) -> Vec<&'static str> { + std::mem::take(&mut self.dropped) + } + /// Why a layer read in place couldn't be read, the first time it is asked for after the read /// failed, so it is said once (a project, an import or the browser read it after adding it). pub fn error_unsaid(&mut self) -> Option { diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index c72c4eff7..c2bcdedee 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -105,6 +105,8 @@ pub(crate) fn draw( /// Screen points a side sampled to find the part of a raster read in place that is in view, and /// how finely. const WINDOW_GRID: usize = 17; +/// Why a part in view was drawn from a coarser level than it needs. +const TOO_LARGE: &str = "the view is too large a part of the image to read at the resolution it needs"; /// Draws a raster layer read in place (a cloud-optimised GeoTIFF by its address, see /// [`rustgis_map::raster::RasterLayer::in_place`]) as [`draw`] draws one held whole: its overview @@ -184,12 +186,14 @@ pub(crate) fn draw_in_place( } let Some((level, cols, rows)) = window else { whole(pm); - return Err("the view is too large a part of the image to read at the resolution it needs".into()); + return Err(TOO_LARGE.into()); }; - // The image at the overview's level is the overview. + // The image at the overview's level is the overview: reached from a finer level the view + // needs (too large a part for the tiles kept), it is drawn but refused, as a coarser level is + // below. if levels.get(level).is_some_and(|l| l.width <= r.width) { whole(pm); - return Ok(false); + return if level > needed { Err(TOO_LARGE.into()) } else { Ok(false) }; } let color = match (&style.renderer, &r.color) { (_, ColorHint::Palette(p)) => ColorHint::Palette(p.clone()), @@ -230,7 +234,7 @@ pub(crate) fn draw_in_place( (false, true) => Err("some of its tiles couldn't be read".into()), // Read coarser than the view needs (too large a part for the tiles kept): the map shows // it, an export refuses it. - (false, false) if level > needed => Err("the view is too large a part of the image to read at the resolution it needs".into()), + (false, false) if level > needed => Err(TOO_LARGE.into()), (false, false) => Ok(false), } } diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index 0cb3115d3..d4c078473 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -12,8 +12,6 @@ const MAX_TILES: u64 = 512; const MAX_BYTES: usize = 64 * 1024 * 1024; const TILE_SIZE: u32 = 256; const MAX_LAT: f64 = 85.051_128_779_806_6; -/// The longest an export waits for the tiles of a raster read in place, in all. -const IN_PLACE_WAIT: web_time::Duration = web_time::Duration::from_secs(120); /// Waits in a row that bring nothing new before an export stops waiting for them. const IDLE_WAITS: usize = 6; @@ -62,7 +60,7 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> } let stats = layer.masked_stats(); // A raster read in place waits for its tiles natively, two minutes at most in all. - let deadline = web_time::Instant::now() + IN_PLACE_WAIT; + let deadline = web_time::Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; let mut tiles = Vec::new(); let mut bytes = 0usize; for (z, x0, x1, y0, y1) in ranges { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index b332ec3bf..88d7dc8ce 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -224,7 +224,8 @@ pub fn generation() -> u64 { pub fn retry_after(error: &str) -> Option { let (_, rest) = error.rsplit_once("(retry after ")?; let secs: u64 = rest.strip_suffix(" s)")?.parse().ok()?; - Some(Duration::from_secs(secs)) + // Bounded as when written: a text from elsewhere (an address echoed) can't ask for years. + Some(Duration::from_secs(secs.clamp(1, crate::failure::MAX_RETRY_AFTER))) } /// How long a failed read waits before it is tried again: what its server asked for (see @@ -2176,6 +2177,8 @@ mod tests { assert_eq!(http_status("HTTP 429 (something else)"), None); assert_eq!(retry_after("http status: 503 (retry after 30 s)"), Some(Duration::from_secs(30))); assert_eq!(retry_after("http status: 503"), None); + assert_eq!(retry_after("x (retry after 18446744073709551615 s)"), Some(Duration::from_secs(3600)), "bounded, never overflowing"); + assert_eq!(retry_after("x (retry after 0 s)"), Some(Duration::from_secs(1))); assert_eq!(retry_wait("HTTP 429 (retry after 2 s)", RETRY), Duration::from_secs(2)); assert_eq!(retry_wait("http status: 500", RETRY), RETRY); } diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 359c632b5..fc9e0542a 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -13,7 +13,7 @@ //! embedding bands) are read a band at a time, only those shown. Files are hostile input: header //! reads, tile counts and sizes are bounded (see [`rustgis_data::raster::cog`]). -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::{Arc, Mutex, OnceLock}; use rustgis_data::raster::ColorHint; @@ -249,6 +249,9 @@ struct Tiles { /// the map in view has just used (which would be read again, and make room for itself by /// throwing out another in view: the view flickering between levels). clock: u64, + /// The tiles ready, by when each was last used (the clock then, unique): the first is the + /// one to make room, found without a scan of them all. + order: BTreeMap, } impl Tiles { @@ -258,7 +261,9 @@ impl Tiles { let now = self.clock; match self.map.get_mut(key) { Some(TileState::Ready(b, _, used)) => { + self.order.remove(used); *used = now; + self.order.insert(now, key.clone()); Some(b.clone()) } _ => None, @@ -268,24 +273,23 @@ impl Tiles { /// Keeps a tile ready, making room for it past [`BUDGET`] (the tiles used longest ago go); /// returns what was kept under its key before. fn keep(&mut self, key: (String, usize, u32), bands: Bands, samples: usize) -> Option { + // Kept under this key already: counted once. + let before = self.map.remove(&key); + if let Some(TileState::Ready(_, n, used)) = &before { + self.samples = self.samples.saturating_sub(*n); + self.order.remove(used); + } while self.samples + samples > BUDGET { - let oldest = self - .map - .iter() - .filter_map(|(k, s)| match s { - TileState::Ready(_, _, used) => Some((*used, k)), - TileState::Loading(_) => None, - }) - .min_by_key(|(used, _)| *used) - .map(|(_, k)| k.clone()); - let Some(k) = oldest else { break }; + let Some((_, k)) = self.order.pop_first() else { break }; if let Some(TileState::Ready(_, n, _)) = self.map.remove(&k) { self.samples = self.samples.saturating_sub(n); } } self.clock += 1; self.samples += samples; - self.map.insert(key, TileState::Ready(bands, samples, self.clock)) + self.order.insert(self.clock, key.clone()); + self.map.insert(key, TileState::Ready(bands, samples, self.clock)); + before } } @@ -299,6 +303,7 @@ pub(crate) fn forget() { infos().lock().unwrap_or_else(|e| e.into_inner()).retain(|_, s| matches!(s, State::Loading(_))); let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); t.map.retain(|_, s| matches!(s, TileState::Loading(_))); + t.order.clear(); t.samples = 0; } @@ -310,7 +315,11 @@ pub(crate) fn forget_file(url: &str) { all.remove(url); } drop(all); - let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + forget_file_in(&mut tiles().lock().unwrap_or_else(|e| e.into_inner()), url); +} + +/// Forgets a file's tiles kept (not those loading) from `t`. +fn forget_file_in(t: &mut Tiles, url: &str) { let mut freed = 0; t.map.retain(|(u, _, _), s| match s { TileState::Ready(_, n, _) if u == url => { @@ -320,6 +329,7 @@ pub(crate) fn forget_file(url: &str) { _ => true, }); t.samples = t.samples.saturating_sub(freed); + t.order.retain(|_, (u, _, _)| u != url); } /// A file's info: `Ok(None)` while it is being read (asking for it starts the read). @@ -1548,5 +1558,11 @@ mod tests { assert!(t.used(&key(1)).is_some() && t.used(&key(3)).is_some()); assert!(t.used(&key(2)).is_none()); assert_eq!(t.samples, 2 * half); + // Kept again under its key: counted once, nothing else making room for it. + t.keep(key(3), bands.clone(), half); + assert!(t.used(&key(1)).is_some() && t.used(&key(3)).is_some()); + assert_eq!((t.samples, t.order.len()), (2 * half, 2)); + forget_file_in(&mut t, "https://x.org/a.tif"); + assert_eq!((t.samples, t.order.len(), t.map.len()), (0, 0, 0)); } } diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 7aad36d2a..e96c218cb 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -71,7 +71,7 @@ pub fn overview(url: &str, done: Box) + Send>) pub fn overview_blocking(url: &str) -> Result { let (tx, rx) = std::sync::mpsc::channel(); overview(url, Box::new(move |r| tx.send(r).unwrap_or_default())); - rx.recv_timeout(std::time::Duration::from_secs(120)).map_err(|_| format!("{}: no answer in two minutes", split(url).0))? + rx.recv_timeout(rustgis_data::raster::IN_PLACE_WAIT).map_err(|_| format!("{}: no answer in two minutes", split(url).0))? } fn read_overview(info: Arc, url: String, done: Box) + Send>) { diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index e7cc4c53e..d543156f0 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -195,7 +195,7 @@ impl std::fmt::Display for Failure { /// The longest a server's `Retry-After` is waited for: a longer wait (or a hostile one) is cut /// to it, so the read is tried again within the hour. -const MAX_RETRY_AFTER: u64 = 3600; +pub(crate) const MAX_RETRY_AFTER: u64 = 3600; /// A refusal's text (`http status: 503`, or the browser's `HTTP 503`), with how long the server /// asked to be left (`… (retry after 30 s)`) when it refused with 429 or 503 and its `Retry-After` diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index c4ea310d2..d0975509c 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -109,7 +109,7 @@ identify and the raster tools work on it as on a file's. The layer holds an over file too large to hold whole is opened from an overview; zoomed in past it, the map reads the part in view at the resolution it needs as it draws, and draws it with the layer's style and the overview's statistics, so the image looks the same at every zoom. Until a part's tiles arrive, the -level zoomed from stands in for them. The Properties pane says the image's full size and that it +level zoomed from stands in for them. The Details pane says the image's full size and that it is read in place. A file whose smallest image is too large to hold as the overview (a large image saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` makes the overviews. A tile that fails to arrive is asked for again a minute later, or when its server says to in a From 22b22a8c44251266590df7983ec00ae21f64cc92 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 20:44:14 -0330 Subject: [PATCH 08/27] fix(raster): check an in-place address as a tile layer's before reading it A project edited by hand asks for http(s) or s3 addresses only: `read_together` and `reload` refuse any other, keeping why on the layer, which is said once the project is open. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/tests.rs | 19 +++++++++++++++++++ crates/map/src/raster.rs | 10 +++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index e07609bd3..3a5de096c 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13339,6 +13339,25 @@ fn an_address_look_holds_when_the_pixels_land_later() { assert_eq!(style("Classes").renderer, RasterRenderer::Colormap); } +/// An in-place address a project keeps is checked as a tile layer's is before anything is asked +/// for: a hand-edited one that isn't http(s) or s3 is refused, and said, on opening and on adding. +#[test] +fn an_unchecked_in_place_address_is_refused() { + use rustgis_map::raster::RasterLayer; + let bad = "ftp://127.0.0.1/scene.tif"; + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Odd".to_string(), RasterLayer::unread_in_place(bad)); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let rl = s.map().layers.iter().find(|l| l.name == "Odd").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(rl.data.is_none() && rl.error.as_deref().is_some_and(|e| !e.contains("no answer")), "{:?}", rl.error); + assert_eq!(s.notices.iter().filter(|n| n.error && n.text.contains("Odd: couldn't be read in place")).count(), 1); + // Added directly (the browser's path), refused before the reader is asked. + let added = RasterLayer::opening_in_place(bad); + assert!(added.error.is_some() && added.data.is_none()); +} + /// A layer read in place that can't be read says so, once, as its project opens. #[test] fn an_in_place_layer_that_cant_be_read_says_so() { diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index a95edec2b..bcf78ac6d 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -104,7 +104,9 @@ pub fn read_together(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { let address = l.path.clone(); let reader = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone(); match (address, reader) { - (Some(address), Some(reader)) => { + // An address checked as a tile layer's is (http(s) or s3, a host, no control + // characters): a project edited by hand asks for nothing else. + (Some(address), Some(reader)) if crate::tiles::check_url(&address).is_ok() => { l.error = None; let slot = Arc::new(Mutex::new(None)); let land = slot.clone(); @@ -338,6 +340,12 @@ impl RasterLayer { self.error_said = false; return Err(e); }; + // Checked as a tile layer's address is: a project edited by hand asks for nothing else. + if let Err(e) = crate::tiles::check_url(&address) { + self.error = Some(e.clone()); + self.error_said = false; + return Err(e); + } let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { let e = format!("{address}: can't be read in place here"); self.error = Some(e.clone()); From 2ca64af572d0ea607fbd4926080d9966b403a0ad Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 21:05:32 -0330 Subject: [PATCH 09/27] fix(raster): the map says when an in-place layer draws less than the view needs; PNG exports refuse it - A coarser level than the view needs, or tiles that couldn't be read, reach the app as a tile error named by the layer, shown once; the map keeps drawing what it has. - A PNG export (share.exportMap) refuses a raster read in place whose tiles are still on their way or couldn't be read, as Raster to PMTiles does, instead of writing the overview. - The window's edges use saturating arithmetic on hostile widths. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/tests.rs | 30 ++++++++++++++++++++++++++++++ crates/render/src/lib.rs | 28 ++++++++++++++++++++++++---- crates/render/src/raster.rs | 4 ++-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 3a5de096c..1469b86a5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13358,6 +13358,36 @@ fn an_unchecked_in_place_address_is_refused() { assert!(added.error.is_some() && added.data.is_none()); } +/// A PNG export of a raster read in place whose image can't be read (its tiles, past the overview +/// it holds) is refused, not written from the overview. +#[test] +fn a_png_export_of_an_unreadable_in_place_layer_is_refused() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let mut overview = Raster::new( + "gone", + 4, + 4, + vec![(0..16).map(|v| v as f32).collect()], + GeoTransform { origin_x: -180.0, origin_y: 80.0, pixel_width: 90.0, pixel_height: -40.0 }, + Crs::LonLat, + None, + ColorHint::Gray, + "Float32", + ) + .unwrap(); + overview.overview = Some((4096, 4096)); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Gone".to_string(), rustgis_map::raster::RasterLayer::in_place(overview, "http://127.0.0.1:9/gone.tif")); + // Zoomed in far past the overview: the image's tiles are needed, and can't be read. + s.map_mut().camera.center = rustgis_geom::Coord::new(0.0, 0.0); + s.map_mut().camera.res = 25.0; + s.view_size = (64.0, 64.0); + let png = std::env::temp_dir().join(format!("rustgis-in-place-refused-{}.png", std::process::id())); + let e = s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap_err(); + assert!(e.contains("isn't written") || e.contains("still on their way"), "{e}"); + assert!(!png.exists()); +} + /// A layer read in place that can't be read says so, once, as its project opens. #[test] fn an_in_place_layer_that_cant_be_read_says_so() { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index ad1c5c5e2..c0776127e 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -3172,10 +3172,17 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac // Read in place: the part in view, at the level the view needs. let address = rl.path.as_deref().filter(|_| rl.in_place); let mut draw_it = |pm: &mut Pixmap, opacity: f32| match address { - Some(a) => { - tile_out.pending |= - raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) == Ok(true) - } + // Drawn from less than the view needs (a coarser level, tiles that couldn't be + // read): shown, and said with the tile errors. + Some(a) => match raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) { + Ok(pending) => tile_out.pending |= pending, + Err(e) => { + let e = format!("{}: {e}", layer.name); + if !tile_out.errors.contains(&e) { + tile_out.errors.push(e); + } + } + }, None => raster::draw(pm, data, &rl.style, stats, view, opacity, full_res_rasters, place), }; if offscreen { @@ -4866,6 +4873,19 @@ pub(crate) fn render_complete(map: &mut Map, view: &View) -> Option { /// Renders a map with its labels, a scale bar and a north arrow to PNG bytes. pub fn render_png(map: &mut Map, view: &View) -> Result, String> { let r = render_complete(map, view).ok_or("could not allocate the image")?; + // A raster read in place drawn from less than the view needs isn't written: its tiles still + // on their way (the browser can't wait; natively the wait ran out), or ones it couldn't read + // (said by the layer's name among the tile errors). + let in_place: Vec<&str> = + map.layers.iter().filter(|l| l.visible && l.raster.as_ref().is_some_and(|r| r.in_place)).map(|l| l.name.as_str()).collect(); + if !in_place.is_empty() { + if r.tiles_pending { + return Err("a raster read in place has tiles still on their way: export again once the map has drawn it".into()); + } + if let Some(e) = r.tile_errors.iter().find(|e| in_place.iter().any(|n| e.starts_with(&format!("{n}: ")))) { + return Err(format!("{e}, so the image isn't written")); + } + } let mut pm = Canvas::from_pixmap(r.pixmap); for l in r.unplaced.iter().chain(&r.labels) { text::draw(&mut pm, l, view.pixels_per_point as f32); diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index c2bcdedee..e446284b6 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -174,8 +174,8 @@ pub(crate) fn draw_in_place( // The part in view, in the level's pixels, a pixel past each side (for bilinear sampling). let (sx, sy) = (f64::from(l.width) / f64::from(r.width.max(1)), f64::from(l.height) / f64::from(r.height.max(1))); let edge = |v: f64, k: f64, max: u32| ((v * k).floor() as i64).clamp(0, i64::from(max)) as u32; - let cols = (edge(want.cols.0, sx, l.width).saturating_sub(1), (edge(want.cols.1, sx, l.width) + 2).min(l.width)); - let rows = (edge(want.rows.0, sy, l.height).saturating_sub(1), (edge(want.rows.1, sy, l.height) + 2).min(l.height)); + let cols = (edge(want.cols.0, sx, l.width).saturating_sub(1), edge(want.cols.1, sx, l.width).saturating_add(2).min(l.width)); + let rows = (edge(want.rows.0, sy, l.height).saturating_sub(1), edge(want.rows.1, sy, l.height).saturating_add(2).min(l.height)); if rustgis_tiles::cog::raster::fits(address, level, cols, rows, &read) { window = Some((level, cols, rows)); break; From 055128b5c03fa7941fc9a9605ddfa48d93dda575 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 21:28:59 -0330 Subject: [PATCH 10/27] fix(raster): say a dropped overlay flag; refuse partial in-place exports by layer id - Opening a COG tile layer as a raster layer reports its overlay flag with the other settings a raster layer hasn't. - The render output carries the raster layers read in place drawn from less than the view needs (`partial`, by id), so a PNG export refuses them by id rather than by matching names. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/tiles.rs | 3 ++ crates/engine/src/tests.rs | 5 ++- crates/render/src/gpu_scene.rs | 3 ++ crates/render/src/lib.rs | 67 +++++++++++++++++++++++++++------- crates/render/src/tiles.rs | 3 ++ crates/ui-egui/src/mapgpu.rs | 2 + 6 files changed, 68 insertions(+), 15 deletions(-) diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 874c30978..c6b9dc557 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -559,6 +559,9 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { if t.bounds.is_some() { dropped.push("bounds"); } + if t.overlay { + dropped.push("overlay"); + } // Read with the project's other rasters read in place (see `project::read_in_place`), in // its look once it lands. let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)).from_tiles_dropping(dropped); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 1469b86a5..711a6e5b7 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13478,10 +13478,11 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { let mut text: Value = serde_json::to_value(&s.project).unwrap(); let tile = |name: &str, url: String, opacity: f64, visible: bool| { let mut l = rustgis_map::TileLayer::new(name, rustgis_map::TileKind::Cog, url); - // Scene's credit and zoom range: what a raster layer hasn't. + // Scene's credit, zoom range and overlay flag: what a raster layer hasn't. if name == "Scene" { l.attribution = "© Example".into(); l.min_zoom = 5; + l.overlay = true; } let mut t = serde_json::to_value(l).unwrap(); t["opacity"] = json!(opacity); @@ -13512,7 +13513,7 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { assert_eq!(names[names.len() - 2..], ["Scene", "Aerial"], "{names:?}"); assert_eq!(t.map().tiles.iter().map(|x| x.name.as_str()).collect::>(), ["Series"]); // Scene's settings a raster layer hasn't are said to be dropped, once; Aerial had none. - let said = |t: &Session| t.notices.iter().filter(|n| n.text.contains("which have no attribution, zoom range")).count(); + let said = |t: &Session| t.notices.iter().filter(|n| n.text.contains("which have no attribution, overlay, zoom range")).count(); assert_eq!(said(&t), 1, "{:?}", t.notices.last()); assert!(t.notices.iter().any(|n| n.text.contains("not kept for Scene") && !n.text.contains("Aerial")), "{:?}", t.notices.last()); t.run("map.zoomIn", &Value::Null).unwrap(); diff --git a/crates/render/src/gpu_scene.rs b/crates/render/src/gpu_scene.rs index 15a44c60f..ece9d57b3 100644 --- a/crates/render/src/gpu_scene.rs +++ b/crates/render/src/gpu_scene.rs @@ -57,6 +57,9 @@ pub struct GpuScene { pub data_pending: bool, /// Tile layers that couldn't be drawn, and why. pub tile_errors: Vec, + /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles + /// they couldn't read), by id, their errors among `tile_errors`: an export refuses them. + pub partial: Vec, /// A turned or tilted scene: the camera it was drawn for, which the vertex shader's full /// perspective takes, and a tilted one's sky and fog. `None` for a flat, north-up scene. pub tilt: Option, diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index c0776127e..519dbfd7a 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -270,6 +270,9 @@ pub struct Rendered { pub data_pending: bool, /// Tile layers that couldn't be drawn, and why. pub tile_errors: Vec, + /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles + /// they couldn't read), by id, their errors among `tile_errors`: an export refuses them. + pub partial: Vec, } /// Which rasteriser [`render_backend`] targets: the tiny-skia CPU pixmap (the default, and the @@ -1914,6 +1917,7 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea let mut kept: Vec<(View, (f64, f64), Pixmap)> = Vec::new(); let last = bands_of.len().saturating_sub(1); let (mut attribution, mut tiles_pending, mut tile_errors) = (Vec::::new(), false, Vec::::new()); + let mut partial = Vec::::new(); for (i, b) in bands_of.into_iter().enumerate() { let v = band_view(view, &b); let Some(RenderOutput::Cpu(r)) = render_frame(map, &v, full_res_rasters, Backend::Cpu, Labels::Tilted(view, &mut candidates)) @@ -1948,6 +1952,11 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea tile_errors.push(e); } } + for id in r.partial { + if !partial.contains(&id) { + partial.push(id); + } + } tiles_pending |= r.tiles_pending; if ground.is_some() { // The nearest band reaches the bottom of the view, the farthest the sky. @@ -1976,7 +1985,16 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea } } let unplaced = if map.label_display.show_unplaced { placed.unplaced } else { Vec::new() }; - Some(Rendered { pixmap: pm.into_pixmap(), labels: placed.labels, unplaced, attribution, tiles_pending, data_pending, tile_errors }) + Some(Rendered { + pixmap: pm.into_pixmap(), + labels: placed.labels, + unplaced, + attribution, + tiles_pending, + data_pending, + tile_errors, + partial, + }) } /// Lays a tilted view's bands, drawn flat, over the ground's relief: each band as a grid of @@ -2150,6 +2168,7 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: let mut candidates = Vec::new(); let (mut cmds, mut clips) = (Vec::new(), Vec::new()); let (mut attribution, mut tiles_pending, mut tile_errors, mut clear) = (Vec::::new(), false, Vec::::new(), [0.0; 4]); + let mut partial = Vec::::new(); let mut plain = true; let last = bands.len().saturating_sub(1); for (i, b) in bands.iter().enumerate() { @@ -2191,6 +2210,11 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: tile_errors.push(e); } } + for id in scene.partial { + if !partial.contains(&id) { + partial.push(id); + } + } tiles_pending |= scene.tiles_pending; // The nearest band reaches the bottom of the view and the farthest the sky, so no ground // falls between them. @@ -2225,7 +2249,19 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: clips.push(CmdClip { upright: true, ..CmdClip::ALL }); } let unplaced = if map.label_display.show_unplaced { placed.unplaced } else { Vec::new() }; - Some(GpuScene { clear, cmds, labels: placed.labels, unplaced, attribution, tiles_pending, data_pending, tile_errors, tilt, clips }) + Some(GpuScene { + clear, + cmds, + labels: placed.labels, + unplaced, + attribution, + tiles_pending, + data_pending, + tile_errors, + partial, + tilt, + clips, + }) } /// Places the labels a tilted view's bands offered. A feature drawn in two bands offers its @@ -3181,6 +3217,9 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac if !tile_out.errors.contains(&e) { tile_out.errors.push(e); } + if !tile_out.partial.contains(&layer.id) { + tile_out.partial.push(layer.id); + } } }, None => raster::draw(pm, data, &rl.style, stats, view, opacity, full_res_rasters, place), @@ -3997,6 +4036,7 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac let attribution = tile_out.attribution; let tiles_pending = tile_out.pending; let tile_errors = tile_out.errors; + let partial = tile_out.partial; match backend { Backend::Cpu => Some(RenderOutput::Cpu(Rendered { pixmap: pm.into_pixmap(), @@ -4006,6 +4046,7 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac tiles_pending, data_pending, tile_errors, + partial, })), Backend::Gpu | Backend::GpuLive => { flush_acc(pm.pixels(), &mut cmds, &mut cpu_dirty)?; @@ -4024,6 +4065,7 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac tiles_pending, data_pending, tile_errors, + partial, tilt: None, clips: Vec::new(), })) @@ -4874,17 +4916,16 @@ pub(crate) fn render_complete(map: &mut Map, view: &View) -> Option { pub fn render_png(map: &mut Map, view: &View) -> Result, String> { let r = render_complete(map, view).ok_or("could not allocate the image")?; // A raster read in place drawn from less than the view needs isn't written: its tiles still - // on their way (the browser can't wait; natively the wait ran out), or ones it couldn't read - // (said by the layer's name among the tile errors). - let in_place: Vec<&str> = - map.layers.iter().filter(|l| l.visible && l.raster.as_ref().is_some_and(|r| r.in_place)).map(|l| l.name.as_str()).collect(); - if !in_place.is_empty() { - if r.tiles_pending { - return Err("a raster read in place has tiles still on their way: export again once the map has drawn it".into()); - } - if let Some(e) = r.tile_errors.iter().find(|e| in_place.iter().any(|n| e.starts_with(&format!("{n}: ")))) { - return Err(format!("{e}, so the image isn't written")); - } + // on their way (the browser can't wait; natively the wait ran out), or a coarser level or + // tiles it couldn't read (`partial`). + if r.tiles_pending && map.layers.iter().any(|l| l.visible && l.raster.as_ref().is_some_and(|r| r.in_place)) { + return Err("a raster read in place has tiles still on their way: export again once the map has drawn it".into()); + } + if let Some(l) = r.partial.first().and_then(|id| map.layers.iter().find(|l| l.id == *id)) { + return Err(format!( + "{}: drawn from less than the view needs (a coarser level, or tiles that couldn't be read), so the image isn't written", + l.name + )); } let mut pm = Canvas::from_pixmap(r.pixmap); for l in r.unplaced.iter().chain(&r.labels) { diff --git a/crates/render/src/tiles.rs b/crates/render/src/tiles.rs index 0f803e3de..badec0b89 100644 --- a/crates/render/src/tiles.rs +++ b/crates/render/src/tiles.rs @@ -27,6 +27,9 @@ pub(crate) struct TileOutput { pub attribution: Vec, pub pending: bool, pub errors: Vec, + /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles + /// they couldn't read), by id: shown, with the error among `errors`; an export refuses them. + pub partial: Vec, } impl TileOutput { diff --git a/crates/ui-egui/src/mapgpu.rs b/crates/ui-egui/src/mapgpu.rs index b23daaffe..65e3bf0b7 100644 --- a/crates/ui-egui/src/mapgpu.rs +++ b/crates/ui-egui/src/mapgpu.rs @@ -1732,6 +1732,7 @@ mod tests { tiles_pending: false, data_pending: false, tile_errors: vec![], + partial: vec![], tilt: None, clips: vec![], }; @@ -2346,6 +2347,7 @@ mod tests { tiles_pending: false, data_pending: false, tile_errors: vec![], + partial: vec![], tilt: None, clips: vec![], }; From 93b4652c52d2acd4dbffea3c2edaae21c5f9f400 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 21:54:22 -0330 Subject: [PATCH 11/27] fix(raster): refuse a PNG export only for an in-place layer's own pending tiles; palette and RGB by image - The render output carries the in-place layers whose tiles are still on their way, by id; render_png refuses those alone, so other layers' tiles pending export what has drawn. - A GeoTIFF's palette and RGB-with-more-bands interpretations are decided per image, from the image decoded: a tableless palette image beside grey overviews says so; a grey image beside an RGB one is grey. - The docs paragraph on layer.addUrl's pending downloads is rewrapped. Co-Authored-By: Claude Fable 5.1 --- crates/data/src/raster/cog.rs | 53 ++++++++++++++++++++++++----- crates/data/src/raster/geotiff.rs | 55 ++++++++++++------------------- crates/render/src/gpu_scene.rs | 3 ++ crates/render/src/lib.rs | 34 ++++++++++++++++--- crates/render/src/tiles.rs | 2 ++ crates/ui-egui/src/mapgpu.rs | 2 ++ docs/user-guide/data-formats.md | 6 ++-- 7 files changed, 105 insertions(+), 50 deletions(-) diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 4588c69ef..df68ee08a 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -312,7 +312,8 @@ pub fn layout(bytes: &Sparse) -> Result { fn read_layout(r: Sparse) -> Result { let reader = super::geotiff::reader(r).map_err(err)?; - let (palette, all_palette, wide_rgb) = (reader.palette, reader.all_palette, reader.wide_rgb); + // The full image's photometric interpretation (the first image). + let (palette, wide_rgb) = (reader.palette_ifds.first().copied().unwrap_or(false), reader.rgb_ifds.first().copied().unwrap_or(false)); let mut dec = Decoder::new(reader).map_err(err)?.with_limits(limits()); let Some(geo) = super::geotiff::cog_geo(&mut dec)? else { return Ok(Parse::NotTiled("a rotated image, or one without coordinates, is read whole".into())); @@ -322,7 +323,7 @@ fn read_layout(r: Sparse) -> Result { let (bands, color) = match ct { // A palette image (shown to the decoder as grey) is coloured by its table, else grey (see // `geotiff::read_limited_with_nodata`). - ColorType::Gray(bits) if all_palette || (palette && dec.find_tag(Tag::ColorMap).ok().flatten().is_some()) => ( + ColorType::Gray(bits) if palette => ( 1, super::geotiff::palette_table(&mut dec, bits).map_or_else( || { @@ -877,11 +878,10 @@ pub(crate) mod tests { } } - /// A file holding a grey image of several bands as well as an RGB one with more bands can't - /// tell which is which once the RGB one is shown to the decoder as grey: neither is taken for - /// colour. + /// A file holding an RGB image with more bands and a grey image of several bands: each is + /// read as what its own PhotometricInterpretation says. #[test] - fn a_file_mixing_grey_and_rgb_images_of_several_bands_stays_grey() { + fn a_file_mixing_grey_and_rgb_images_tells_them_apart_by_image() { let e = Example { size: 300, tile: 128, bands: 4, photometric: 2, ..Example::default() }; let mut file = example_with(&e); // The overview's PhotometricInterpretation (tag 262, one short) from RGB to grey. @@ -889,10 +889,47 @@ pub(crate) mod tests { let at: Vec = (0..file.len().saturating_sub(entry.len())).filter(|&i| file[i..i + entry.len()] == entry).collect(); assert_eq!(at.len(), 3, "the full image and its overviews"); file[at[2] + 8] = 1; + // The full image: RGB (and near-infrared); the coarsest overview, read for a small limit: + // grey. let (l, _) = pixel(&file, 150, 150); - assert_eq!((l.bands, &l.color), (4, &ColorHint::Gray)); + assert_eq!((l.bands, &l.color), (4, &ColorHint::Rgb { alpha: false })); let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "mixed").unwrap(); - assert_eq!((r.band_count(), &r.color), (4, &ColorHint::Gray)); + assert_eq!((r.band_count(), &r.color), (4, &ColorHint::Rgb { alpha: false })); + let small = crate::raster::geotiff::read_limited( + crate::raster::geotiff::reader(std::io::Cursor::new(&file)).unwrap(), + "mixed", + 30_000, + None, + ) + .unwrap(); + assert_eq!((small.width, small.band_count(), &small.color), (75, 4, &ColorHint::Gray)); + } + + /// A palette image without its colour table beside grey overviews: it is drawn in grey and + /// says so; the overviews, grey by their own interpretation, don't. + #[test] + fn a_tableless_palette_image_beside_grey_ones_says_so() { + let mut file = example_with(&Example { size: 300, tile: 128, photometric: 3, ..Example::default() }); + for i in 0..file.len().saturating_sub(4) { + if file[i..i + 4] == [0x40, 0x01, 0x03, 0x00] { + file[i..i + 2].copy_from_slice(&[0xFF, 0xFE]); + } + } + // The overviews' PhotometricInterpretation (tag 262) from palette to grey. + let entry = [0x06, 0x01, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0x00]; + let at: Vec = (0..file.len().saturating_sub(entry.len())).filter(|&i| file[i..i + entry.len()] == entry).collect(); + assert_eq!(at.len(), 3); + for &i in &at[1..] { + file[i + 8] = 1; + } + let (l, _) = pixel(&file, 150, 150); + assert_eq!((&l.color, l.palette_missing), (&ColorHint::Gray, true)); + let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "p").unwrap(); + assert_eq!((&r.color, r.palette_missing), (&ColorHint::Gray, true)); + let small = + crate::raster::geotiff::read_limited(crate::raster::geotiff::reader(std::io::Cursor::new(&file)).unwrap(), "p", 30_000, None) + .unwrap(); + assert_eq!((small.width, &small.color, small.palette_missing), (150, &ColorHint::Gray, false)); } /// A palette image whose colour table is missing is drawn in grey, and says so, read in place diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index e9c85348f..7eb07d9ed 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -586,15 +586,12 @@ pub(crate) struct Unpaletted { inner: R, /// Where each patched image's PhotometricInterpretation value is, and what to show there. patches: Vec<(u64, [u8; 2])>, - /// Palette images were patched. - pub(crate) palette: bool, - /// Every image is a palette image (transparency masks aside): one without its ColorMap is - /// one still, its table missing. - pub(crate) all_palette: bool, - /// RGB images with more samples were patched and the file has no grey image of three or more - /// bands: an image of several bands is one of them, its first three samples red, green, blue. - /// (In a file with both, they can't be told apart, so neither is taken for colour.) - pub(crate) wide_rgb: bool, + /// Which images (in the file's order) are palette images: shown to the decoder as grey, one + /// without its ColorMap is one still, its table missing. + pub(crate) palette_ifds: Vec, + /// Which images are RGB with more samples (shown to the decoder as grey): their first three + /// samples are red, green and blue. + pub(crate) rgb_ifds: Vec, pos: u64, } @@ -606,12 +603,9 @@ const MAX_ENTRIES: u64 = 4096; #[derive(Default)] struct Patches { at: Vec<(u64, [u8; 2])>, - palette: bool, - /// An image that isn't a palette image (nor a transparency mask) was seen. - other: bool, - wide_rgb: bool, - /// A grey image of three or more bands was seen. - grey_bands: bool, + /// By image: a palette image; an RGB image with more samples. + palette_ifds: Vec, + rgb_ifds: Vec, } /// Where the palette images' PhotometricInterpretation values are, and those of RGB images with @@ -682,21 +676,22 @@ fn photometric_patches(r: &mut R) -> Patches { _ => {} } } - patches.other |= photometric.is_some_and(|(_, p)| p != 3 && p != 4); - patches.grey_bands |= photometric.is_some_and(|(_, p)| p <= 1) && samples >= 3; + let (mut is_palette, mut is_rgb) = (false, false); match photometric { Some((at, 3)) => { patches.at.push((at, grey)); - patches.palette = true; + is_palette = true; } // An RGB image whose fourth sample isn't alpha, or with five or more: RGBI // orthophotos, multispectral images. Some((at, 2)) if samples > 4 || (samples == 4 && !matches!(extra, Some(1 | 2))) => { patches.at.push((at, grey)); - patches.wide_rgb = true; + is_rgb = true; } _ => {} } + patches.palette_ifds.push(is_palette); + patches.rgb_ifds.push(is_rgb); let mut next = [0u8; 8]; r.read_exact(&mut next[..next_len]).ok()?; ifd = if big { u64_(&next)? } else { u64::from(u32_(&next)?) }; @@ -725,14 +720,7 @@ impl Unpaletted { fn new(mut inner: R) -> std::io::Result { let p = photometric_patches(&mut inner); inner.seek(std::io::SeekFrom::Start(0))?; - Ok(Unpaletted { - inner, - patches: p.at, - palette: p.palette, - all_palette: p.palette && !p.other, - wide_rgb: p.wide_rgb && !p.grey_bands, - pos: 0, - }) + Ok(Unpaletted { inner, patches: p.at, palette_ifds: p.palette_ifds, rgb_ifds: p.rgb_ifds, pos: 0 }) } } @@ -1062,7 +1050,7 @@ pub(super) fn read_limited_with_nodata( open: Option>, sidecar_nodata: Option, ) -> Result { - let (palette, all_palette, wide_rgb) = (r.palette, r.all_palette, r.wide_rgb); + let (palette_ifds, rgb_ifds) = (r.palette_ifds.clone(), r.rgb_ifds.clone()); let limits = { let mut l = Limits::default(); l.decoding_buffer_size = MAX_SAMPLES.saturating_mul(4); @@ -1150,13 +1138,12 @@ pub(super) fn read_limited_with_nodata( let ct = dec.colortype().map_err(err)?; // JPEG-compressed YCbCr, the only YCbCr read (the colour match below rejects the rest). let ycbcr = matches!(ct, ColorType::YCbCr(_)) && dec.find_tag_unsigned::(Tag::Compression).ok().flatten() == Some(7); - // A palette image (shown to the decoder as grey) is coloured by its table: in a file of palette - // images, one without it is drawn in grey, as said below; in a file of others too, only one with - // its table is taken for one. + // The image read is a palette image (shown to the decoder as grey): coloured by its table, or + // in grey without one, as said below. + let palette = palette_ifds.get(image).copied().unwrap_or(false); + let wide_rgb = rgb_ifds.get(image).copied().unwrap_or(false); let ct = match ct { - ColorType::Gray(bits) if all_palette || (palette && dec.find_tag(Tag::ColorMap).ok().flatten().is_some()) => { - ColorType::Palette(bits) - } + ColorType::Gray(bits) if palette => ColorType::Palette(bits), other => other, }; // A palette image without a whole colour table is drawn in grey, and says so. diff --git a/crates/render/src/gpu_scene.rs b/crates/render/src/gpu_scene.rs index ece9d57b3..bae92d1b3 100644 --- a/crates/render/src/gpu_scene.rs +++ b/crates/render/src/gpu_scene.rs @@ -60,6 +60,9 @@ pub struct GpuScene { /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles /// they couldn't read), by id, their errors among `tile_errors`: an export refuses them. pub partial: Vec, + /// Raster layers read in place whose tiles are still on their way, by id: drawn again as + /// they land; an export that can't wait refuses them. + pub pending_in_place: Vec, /// A turned or tilted scene: the camera it was drawn for, which the vertex shader's full /// perspective takes, and a tilted one's sky and fog. `None` for a flat, north-up scene. pub tilt: Option, diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 519dbfd7a..9fa418505 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -273,6 +273,9 @@ pub struct Rendered { /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles /// they couldn't read), by id, their errors among `tile_errors`: an export refuses them. pub partial: Vec, + /// Raster layers read in place whose tiles are still on their way, by id: drawn again as + /// they land; an export that can't wait refuses them. + pub pending_in_place: Vec, } /// Which rasteriser [`render_backend`] targets: the tiny-skia CPU pixmap (the default, and the @@ -1917,7 +1920,7 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea let mut kept: Vec<(View, (f64, f64), Pixmap)> = Vec::new(); let last = bands_of.len().saturating_sub(1); let (mut attribution, mut tiles_pending, mut tile_errors) = (Vec::::new(), false, Vec::::new()); - let mut partial = Vec::::new(); + let (mut partial, mut pending_in_place) = (Vec::::new(), Vec::::new()); for (i, b) in bands_of.into_iter().enumerate() { let v = band_view(view, &b); let Some(RenderOutput::Cpu(r)) = render_frame(map, &v, full_res_rasters, Backend::Cpu, Labels::Tilted(view, &mut candidates)) @@ -1957,6 +1960,11 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea partial.push(id); } } + for id in r.pending_in_place { + if !pending_in_place.contains(&id) { + pending_in_place.push(id); + } + } tiles_pending |= r.tiles_pending; if ground.is_some() { // The nearest band reaches the bottom of the view, the farthest the sky. @@ -1994,6 +2002,7 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea data_pending, tile_errors, partial, + pending_in_place, }) } @@ -2168,7 +2177,7 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: let mut candidates = Vec::new(); let (mut cmds, mut clips) = (Vec::new(), Vec::new()); let (mut attribution, mut tiles_pending, mut tile_errors, mut clear) = (Vec::::new(), false, Vec::::new(), [0.0; 4]); - let mut partial = Vec::::new(); + let (mut partial, mut pending_in_place) = (Vec::::new(), Vec::::new()); let mut plain = true; let last = bands.len().saturating_sub(1); for (i, b) in bands.iter().enumerate() { @@ -2215,6 +2224,11 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: partial.push(id); } } + for id in scene.pending_in_place { + if !pending_in_place.contains(&id) { + pending_in_place.push(id); + } + } tiles_pending |= scene.tiles_pending; // The nearest band reaches the bottom of the view and the farthest the sky, so no ground // falls between them. @@ -2259,6 +2273,7 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: data_pending, tile_errors, partial, + pending_in_place, tilt, clips, }) @@ -3211,7 +3226,12 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac // Drawn from less than the view needs (a coarser level, tiles that couldn't be // read): shown, and said with the tile errors. Some(a) => match raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) { - Ok(pending) => tile_out.pending |= pending, + Ok(pending) => { + tile_out.pending |= pending; + if pending && !tile_out.pending_in_place.contains(&layer.id) { + tile_out.pending_in_place.push(layer.id); + } + } Err(e) => { let e = format!("{}: {e}", layer.name); if !tile_out.errors.contains(&e) { @@ -4037,6 +4057,7 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac let tiles_pending = tile_out.pending; let tile_errors = tile_out.errors; let partial = tile_out.partial; + let pending_in_place = tile_out.pending_in_place; match backend { Backend::Cpu => Some(RenderOutput::Cpu(Rendered { pixmap: pm.into_pixmap(), @@ -4047,6 +4068,7 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac data_pending, tile_errors, partial, + pending_in_place, })), Backend::Gpu | Backend::GpuLive => { flush_acc(pm.pixels(), &mut cmds, &mut cpu_dirty)?; @@ -4066,6 +4088,7 @@ fn render_frame(map: &mut Map, view: &View, full_res_rasters: bool, backend: Bac data_pending, tile_errors, partial, + pending_in_place, tilt: None, clips: Vec::new(), })) @@ -4918,8 +4941,9 @@ pub fn render_png(map: &mut Map, view: &View) -> Result, String> { // A raster read in place drawn from less than the view needs isn't written: its tiles still // on their way (the browser can't wait; natively the wait ran out), or a coarser level or // tiles it couldn't read (`partial`). - if r.tiles_pending && map.layers.iter().any(|l| l.visible && l.raster.as_ref().is_some_and(|r| r.in_place)) { - return Err("a raster read in place has tiles still on their way: export again once the map has drawn it".into()); + // Other layers' tiles pending export what has drawn, as before. + if let Some(l) = r.pending_in_place.first().and_then(|id| map.layers.iter().find(|l| l.id == *id)) { + return Err(format!("{}: its tiles are still on their way: export again once the map has drawn it", l.name)); } if let Some(l) = r.partial.first().and_then(|id| map.layers.iter().find(|l| l.id == *id)) { return Err(format!( diff --git a/crates/render/src/tiles.rs b/crates/render/src/tiles.rs index badec0b89..6e4693f8a 100644 --- a/crates/render/src/tiles.rs +++ b/crates/render/src/tiles.rs @@ -30,6 +30,8 @@ pub(crate) struct TileOutput { /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles /// they couldn't read), by id: shown, with the error among `errors`; an export refuses them. pub partial: Vec, + /// Raster layers read in place whose tiles are still on their way, by id. + pub pending_in_place: Vec, } impl TileOutput { diff --git a/crates/ui-egui/src/mapgpu.rs b/crates/ui-egui/src/mapgpu.rs index 65e3bf0b7..77cb9eb95 100644 --- a/crates/ui-egui/src/mapgpu.rs +++ b/crates/ui-egui/src/mapgpu.rs @@ -1733,6 +1733,7 @@ mod tests { data_pending: false, tile_errors: vec![], partial: vec![], + pending_in_place: vec![], tilt: None, clips: vec![], }; @@ -2348,6 +2349,7 @@ mod tests { data_pending: false, tile_errors: vec![], partial: vec![], + pending_in_place: vec![], tilt: None, clips: vec![], }; diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index d0975509c..96b9f160d 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -157,9 +157,9 @@ desktop, while in the web app they need the file downloaded first. On desktop, `layer.addUrl` waits for downloads, up to a minute; the From URL… dialog (`ui.addUrl`) and a plugin's `layer.addUrl` work in the background. In the web app the same dialog handles asynchronous downloads. `layer.addUrl` can add PMTiles archives and GeoTIFFs read in place directly, but an ordinary -download returns -`{"pending": true, "url": "…"}` and is added later by the app. To open a remote GeoParquet or -FlatGeobuf whose header hasn't been fetched yet, use the From URL… dialog or a `?data=` link. +download returns `{"pending": true, "url": "…"}` and is added later by the app. To open a remote +GeoParquet or FlatGeobuf whose header hasn't been fetched yet, use the From URL… dialog or a +`?data=` link. ## Map and feature services From 83d28460b762512f6d071320e81223fa7fa9d8c0 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 22:25:35 -0330 Subject: [PATCH 12/27] fix(raster): three minor points from review - The export's idle-wait count resets when a wait brings news, as its doc says. - A migrated tile layer's opacity is kept exactly, not rounded to a percent. - A tile's colours are looked up once for its pen. Co-Authored-By: Claude Fable 5.1 --- crates/data/src/raster/geotiff.rs | 1 + crates/engine/src/cmd/tiles.rs | 2 +- crates/render/src/raster_export.rs | 2 ++ crates/tiles/src/cog.rs | 11 ++++++++--- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 7eb07d9ed..fb8721d59 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -666,6 +666,7 @@ fn photometric_patches(r: &mut R) -> Patches { // (277); the first ExtraSamples (338) value, when it is inline. Each a SHORT. let (mut photometric, mut samples, mut extra) = (None, 1u16, None); for (i, e) in entries.chunks(entry_len as usize).enumerate() { + // Each a SHORT, as the decoder itself requires of them. let short = u16_(e.get(2..)?)? == 3; match u16_(e)? { 262 if short => { diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index c6b9dc557..30577afcf 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -568,7 +568,7 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { let id = m.add_raster(t.name.clone(), rl); if let Some(l) = m.layer_mut(id) { l.visible = t.visible; - l.transparency = ((1.0 - t.opacity.clamp(0.0, 1.0)) * 100.0).round(); + l.transparency = (1.0 - t.opacity.clamp(0.0, 1.0)) * 100.0; l.blend = t.blend; } // Just over the layer it was drawn over, in its group; under them all without one. diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index d4c078473..f3defbce2 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -97,6 +97,8 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> crate::rustgis_tiles::cache::wait_for_news(drawn_at, left.min(web_time::Duration::from_secs(30))); if crate::rustgis_tiles::cache::generation() == drawn_at { idle += 1; + } else { + idle = 0; } image.fill(tiny_skia::Color::TRANSPARENT); } diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index fc9e0542a..c4a293ec7 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -196,7 +196,11 @@ impl Info { /// The bands a look reads: its colour bands, then the file's alpha band, if it has one. pub fn shown(&self, look: &Look) -> Vec { - let mut shown = self.colour(look); + self.shown_from(self.colour(look)) + } + + /// [`Info::shown`], from the look's colours already found. + fn shown_from(&self, mut shown: Vec) -> Vec { if let Some(a) = self.alpha.filter(|a| !shown.contains(a)) { shown.push(a); } @@ -1065,8 +1069,9 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 done(Err("too many GeoTIFF tiles for one map tile".into())); return; } - let shown = info.shown(look); - let colours = info.colour(look).len(); + let colour = info.colour(look); + let colours = colour.len(); + let shown = info.shown_from(colour); let pen = Pen { rgb: colours >= 3, // The alpha band where it is among those shown: after the colours, or one of them. From 9fe6717337e3fe19d3a785afb2236523c6139094 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 23:01:17 -0330 Subject: [PATCH 13/27] fix(raster): a tile's end saturates in assemble; a data cube's file stays a tile layer on reopen - A level as wide as u32 allows is assembled rather than wrapped to an empty copy. - "cube": true marks the address (#cube=1), so a data cube's file added without its series stays a tile layer when its project opens again; look changes keep the marker. - The engine tests' range server answers each connection on its own thread, so a layer's tiles asked for together don't hold a fetch worker each while the others wait. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/layer.rs | 11 ++--- crates/engine/src/cmd/tiles.rs | 9 +++- crates/engine/src/tests.rs | 79 ++++++++++++++++++++++------------ crates/tiles/src/cog.rs | 9 +++- crates/tiles/src/cog/raster.rs | 17 +++++++- crates/tiles/src/embeddings.rs | 2 +- 6 files changed, 89 insertions(+), 38 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index b3e5fb209..ef28ba672 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2972,15 +2972,16 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) // A GeoTIFF is a raster layer like a file's, read in place; a dated series, a data cube's file // (which follow the time slider) and AlphaEarth's quantised embeddings (which raster styles // don't dequantise) are tile layers. - let (file, look) = crate::render::rustgis_tiles::cog::split(url); + let (file, mut look) = crate::render::rustgis_tiles::cog::split(url); // A key given as `null` (a script's unused one) asks for nothing. let given = |k: &str| p.get(k).filter(|v| !v.is_null()); - let dated = rustgis_data::time::is_template(file) - || ["start", "end", "unit", "interval", "dates"].iter().any(|k| given(k).is_some()) - || b(p, "cube") == Some(true); - if !dated && !look.aef { + // A data cube's file stays a tile layer, its project opened again too: its address says so. + look.cube |= b(p, "cube") == Some(true); + let dated = rustgis_data::time::is_template(file) || ["start", "end", "unit", "interval", "dates"].iter().any(|k| given(k).is_some()); + if !dated && !look.aef && !look.cube { return add_in_place(s, name, file, &look, strict); } + let url = crate::render::rustgis_tiles::cog::with_look(url, &look); let mut q = json!({"url": url, "kind": "cog", "name": name, "top": true}); // A dated address's span and steps (see map.addTiles). for k in ["start", "end", "unit", "interval", "dates"] { diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 30577afcf..b40717d37 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -540,7 +540,13 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { let mut i = 0; while let Some(t) = m.tiles.get(i) { let (file, look) = cog::split(&t.url); - if t.kind != TileKind::Cog || t.time_url.is_some() || !t.time_urls.is_empty() || look.aef || rustgis_data::time::is_template(file) { + if t.kind != TileKind::Cog + || t.time_url.is_some() + || !t.time_urls.is_empty() + || look.aef + || look.cube + || rustgis_data::time::is_template(file) + { i += 1; continue; } @@ -626,6 +632,7 @@ fn cog_look(p: &Value) -> Result Look>>, String> { bands: bands.clone().unwrap_or(old.bands), range: range.unwrap_or(old.range), aef: aef.unwrap_or(old.aef), + cube: old.cube, }))) } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 711a6e5b7..151a1e108 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -12444,40 +12444,47 @@ fn serve_gated( let addr = listener.local_addr().unwrap(); let sent = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let count = sent.clone(); + let body = std::sync::Arc::new(body); std::thread::spawn(move || { + // Each connection on its own thread: a layer read in place asks for its tiles together, + // and answering them one by one would hold a fetch worker for each while the others wait + // (the fetch workers are shared by every test in the process). for stream in listener.incoming() { let Ok(mut s) = stream else { continue }; - let mut buf = [0u8; 4096]; - let read = s.read(&mut buf).unwrap_or(0); - if let Some(g) = &gate { - let start = std::time::Instant::now(); - while !g.load(std::sync::atomic::Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(60) { - std::thread::sleep(std::time::Duration::from_millis(5)); + let (body, count, gate) = (body.clone(), count.clone(), gate.clone()); + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + let read = s.read(&mut buf).unwrap_or(0); + if let Some(g) = &gate { + let start = std::time::Instant::now(); + while !g.load(std::sync::atomic::Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(60) { + std::thread::sleep(std::time::Duration::from_millis(5)); + } } - } - let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); - // A HEAD request is answered with the length alone. - if req.starts_with("head ") { - let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - continue; - } - let range = req.lines().filter(|_| ranges).find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { - let (a, b) = r.trim().split_once('-')?; - // A suffix (`-n`): the last n bytes. - if a.is_empty() { - let n = b.parse::().ok()?.min(body.len()); - return Some((body.len() - n, body.len().saturating_sub(1))); + let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); + // A HEAD request is answered with the length alone. + if req.starts_with("head ") { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + return; } - Some((a.parse::().ok()?, b.parse::().ok()?)) + let range = req.lines().filter(|_| ranges).find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + // A suffix (`-n`): the last n bytes. + if a.is_empty() { + let n = b.parse::().ok()?.min(body.len()); + return Some((body.len() - n, body.len().saturating_sub(1))); + } + Some((a.parse::().ok()?, b.parse::().ok()?)) + }); + let (status, part) = match range { + Some((a, b)) if a < body.len() => ("206 Partial Content", &body[a..=b.min(body.len() - 1)]), + Some(_) => ("416 Range Not Satisfiable", &body[..0]), + None => ("200 OK", &body[..]), + }; + count.fetch_add(part.len(), std::sync::atomic::Ordering::Relaxed); + let _ = write!(s, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); + let _ = s.write_all(part); }); - let (status, part) = match range { - Some((a, b)) if a < body.len() => ("206 Partial Content", &body[a..=b.min(body.len() - 1)]), - Some(_) => ("416 Range Not Satisfiable", &body[..0]), - None => ("200 OK", &body[..]), - }; - count.fetch_add(part.len(), std::sync::atomic::Ordering::Relaxed); - let _ = write!(s, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); - let _ = s.write_all(part); } }); (format!("http://{addr}"), sent) @@ -13434,6 +13441,22 @@ fn a_palette_without_its_table_says_so_when_its_pixels_land() { } } +/// A data cube's file (`"cube": true`) is a tile layer, and stays one when its project opens +/// again: its address keeps the marker, which the migration respects. +#[test] +fn a_data_cubes_file_stays_a_tile_layer_when_its_project_reopens() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/cube-2020.tif"), "cube": true})).unwrap(); + assert_eq!(r["kind"], "cog", "{r}"); + assert_eq!(s.map().tiles.len(), 1); + assert!(s.map().tiles[0].url.ends_with("#cube=1"), "{}", s.map().tiles[0].url); + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + assert_eq!((t.map().tiles.len(), t.map().layers.iter().filter(|l| l.raster.is_some()).count()), (1, 0)); +} + /// Dated-series keys given as `null` (a script's unused ones) ask for nothing: the GeoTIFF is a /// raster layer read in place, not a tile layer. #[test] diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index c4a293ec7..9f80fbd92 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -73,6 +73,9 @@ pub struct Look { pub range: Option<(f32, f32)>, /// Values are AlphaEarth's quantised embeddings, shown as `sign(v)·(v/127.5)²`. pub aef: bool, + /// A data cube's file (`stac.addCube`): a tile layer that follows the time slider, kept one + /// when its project opens again. + pub cube: bool, } impl Look { @@ -88,6 +91,9 @@ impl Look { if self.aef { parts.push("dequantize=aef".into()); } + if self.cube { + parts.push("cube=1".into()); + } if parts.is_empty() { String::new() } else { format!("#{}", parts.join("&")) } } @@ -113,6 +119,7 @@ impl Look { look.range = Some((lo, hi)); } "dequantize" => look.aef = v == "aef", + "cube" => look.cube = v == "1", _ => return None, } } @@ -1476,7 +1483,7 @@ mod tests { #[test] fn looks_in_addresses() { let u = "https://x.org/a.tiff"; - let look = Look { bands: vec![1, 16, 9], range: Some((-0.3, 0.3)), aef: true }; + let look = Look { bands: vec![1, 16, 9], range: Some((-0.3, 0.3)), aef: true, cube: false }; let shown = with_look(u, &look); assert_eq!(shown, "https://x.org/a.tiff#bands=2,17,10&range=-0.3,0.3&dequantize=aef"); assert_eq!(split(&shown), (u, look)); diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index e96c218cb..24c894bd8 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -411,8 +411,10 @@ fn assemble( for tx in cols.0 / tw..cols.1.div_ceil(tw) { let tile = ty * l.across() + tx; // The tile's part of the window, in the level's pixels. - let (x0, x1) = ((tx * tw).max(cols.0), ((tx + 1) * tw).min(cols.1)); - let (y0, y1) = ((ty * th).max(rows.0), ((ty + 1) * th).min(rows.1)); + // The tile's end saturates: a width near `u32::MAX` (the header bounds the tiles, + // not the width) stays a tile, not a wrapped, empty one. + let (x0, x1) = ((tx * tw).max(cols.0), tx.saturating_add(1).saturating_mul(tw).min(cols.1)); + let (y0, y1) = ((ty * th).max(rows.0), ty.saturating_add(1).saturating_mul(th).min(rows.1)); for (k, &b) in bands.iter().enumerate() { let Some((c, at)) = info.layout.chunk(l, tile, b) else { continue }; let Some(data) = chunk(c) else { @@ -516,6 +518,17 @@ mod tests { assert!(f.contains_key(&key(1)) && f.len() == MAX_FAILED); } + /// A level as wide as `u32` allows (a hostile header): its last tile's end doesn't wrap, so + /// the window is assembled, its missing tiles gaps. + #[test] + fn a_window_at_the_far_edge_of_a_huge_level_doesnt_overflow() { + let mut info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); + info.layout.levels[0].width = u32::MAX; + let (data, gaps) = assemble(&info, 0, (u32::MAX - 15, u32::MAX), (0, 16), &[0], |_| None); + assert_eq!((data.len(), data[0].len()), (1, 16 * 15)); + assert!(gaps.iter().all(|g| g.cols.0 <= g.cols.1 && g.rows.0 <= g.rows.1)); + } + #[test] fn names_from_addresses() { assert_eq!(file_name("https://x.org/a/scene.tif?sig=1"), "scene.tif"); diff --git a/crates/tiles/src/embeddings.rs b/crates/tiles/src/embeddings.rs index dda9b6ee9..98611f969 100644 --- a/crates/tiles/src/embeddings.rs +++ b/crates/tiles/src/embeddings.rs @@ -238,7 +238,7 @@ pub const MAX_TILES: usize = 60; /// stretched over [−stretch, stretch]. pub fn aef_look(bands: [usize; 3], stretch: f32) -> Look { let s = if stretch.is_finite() && stretch > 0.0 { stretch.min(1.0) } else { AEF_DEFAULT_STRETCH }; - Look { bands: bands.iter().map(|b| (*b).min(63)).collect(), range: Some((-s, s)), aef: true } + Look { bands: bands.iter().map(|b| (*b).min(63)).collect(), range: Some((-s, s)), aef: true, cube: false } } /// A file's web address from the index's `s3://` one. From 9df3209e58b2f15d57e358f30dc4f58b04bd9b42 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 23:24:03 -0330 Subject: [PATCH 14/27] =?UTF-8?q?fix(raster):=20a=20view=20across=20a=200?= =?UTF-8?q?=E2=80=93360=C2=B0=20grid's=20seam=20reads=20the=20two=20strips?= =?UTF-8?q?=20in=20view,=20not=20the=20whole=20width?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sample grid's points that wrapped and those that didn't give two runs of columns, each run to the seam; the part in view is read as a window for each at the same level, so a wide full-turn grid viewed at 0° no longer reads its full width, nor reports it as too large. Co-Authored-By: Claude Fable 5.1 --- crates/render/src/raster.rs | 107 ++++++++++++++++++++++++++++-------- 1 file changed, 83 insertions(+), 24 deletions(-) diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index e446284b6..b58543a9d 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -169,14 +169,19 @@ pub(crate) fn draw_in_place( _ => None, }; read.extend(alpha); + // The strips in view: one, or two across a 0–360° grid's seam. + let strips: Vec<(f64, f64)> = std::iter::once(want.cols).chain(want.cols2).collect(); let mut window = None; while let Some(l) = levels.get(level) { // The part in view, in the level's pixels, a pixel past each side (for bilinear sampling). let (sx, sy) = (f64::from(l.width) / f64::from(r.width.max(1)), f64::from(l.height) / f64::from(r.height.max(1))); let edge = |v: f64, k: f64, max: u32| ((v * k).floor() as i64).clamp(0, i64::from(max)) as u32; - let cols = (edge(want.cols.0, sx, l.width).saturating_sub(1), edge(want.cols.1, sx, l.width).saturating_add(2).min(l.width)); + let cols: Vec<(u32, u32)> = strips + .iter() + .map(|c| (edge(c.0, sx, l.width).saturating_sub(1), edge(c.1, sx, l.width).saturating_add(2).min(l.width))) + .collect(); let rows = (edge(want.rows.0, sy, l.height).saturating_sub(1), edge(want.rows.1, sy, l.height).saturating_add(2).min(l.height)); - if rustgis_tiles::cog::raster::fits(address, level, cols, rows, &read) { + if cols.iter().all(|cols| rustgis_tiles::cog::raster::fits(address, level, *cols, rows, &read)) { window = Some((level, cols, rows)); break; } @@ -201,35 +206,49 @@ pub(crate) fn draw_in_place( _ if alpha.is_some() => ColorHint::GrayAlpha, _ => ColorHint::Gray, }; - let w = match rustgis_tiles::cog::raster::window(address, level, cols, rows, &read, color) { - Ok(w) => w, - Err(e) => { - whole(pm); - return Err(e); + // Each strip read from the tiles kept, the missing ones asked for. + let (mut parts, mut pending, mut failed) = (Vec::new(), false, false); + for cols in cols { + match rustgis_tiles::cog::raster::window(address, level, cols, rows, &read, color.clone()) { + Ok(w) => { + pending |= w.pending; + failed |= w.failed; + parts.extend(w.raster); + } + Err(e) => { + whole(pm); + return Err(e); + } } - }; - let Some(part) = w.raster else { + } + if parts.is_empty() { whole(pm); - return Ok(w.pending); - }; + return Ok(pending); + } let mut own_style = style.clone(); own_style.renderer = renumbered(&style.renderer); // While it has gaps (tiles to come, or that couldn't be read and no level kept stands in for), // the part in view goes over the overview: both drawn opaque, then blended once, so a // see-through layer isn't darker where they overlap. - if w.pending || w.failed { + if pending || failed { if opacity >= 1.0 { whole(pm); - draw(pm, &part, &own_style, Some(stats), view, 1.0, full_res, place); + for part in &parts { + draw(pm, part, &own_style, Some(stats), view, 1.0, full_res, place); + } } else if let Some(mut both) = Pixmap::new(pm.width(), pm.height()) { draw(&mut both, r, style, Some(stats), view, 1.0, full_res, place); - draw(&mut both, &part, &own_style, Some(stats), view, 1.0, full_res, place); + for part in &parts { + draw(&mut both, part, &own_style, Some(stats), view, 1.0, full_res, place); + } pm.draw_pixmap(0, 0, both.as_ref(), &PixmapPaint { opacity, ..PixmapPaint::default() }, Transform::identity(), None); } } else { - draw(pm, &part, &own_style, Some(stats), view, opacity, full_res, place); + for part in &parts { + draw(pm, part, &own_style, Some(stats), view, opacity, full_res, place); + } } - match (w.pending, w.failed) { + match (pending, failed) { (true, _) => Ok(true), (false, true) => Err("some of its tiles couldn't be read".into()), // Read coarser than the view needs (too large a part for the tiles kept): the map shows @@ -259,6 +278,9 @@ fn renumbered(renderer: &RasterRenderer) -> RasterRenderer { #[derive(Debug, PartialEq)] struct InView { cols: (f64, f64), + /// A second run of columns: a geographic grid stored from 0 to 360° seen across its seam has + /// the places west of 0° at its far columns, so the part in view is two strips, not the width. + cols2: Option<(f64, f64)>, rows: (f64, f64), per_px: f64, } @@ -274,18 +296,30 @@ fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_ let (mut c0, mut c1, mut r0, mut r1) = (f64::INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::NEG_INFINITY); let (rw, rh) = (f64::from(r.width), f64::from(r.height)); // A geographic grid stored from 0 to 360° has the places west of 0° at its far columns (see - // `Raster::wrap_col`): seen across that seam, the part in view is the whole width. - let mut wrapped = false; + // `Raster::wrap_col`): seen across that seam, the part in view is two strips, one at each end. + let (mut w0, mut w1) = (f64::INFINITY, f64::NEG_INFINITY); for (c, row) in grid.iter().flatten() { let col = r.wrap_col(*c); - wrapped |= col != *c; - c0 = c0.min(col.clamp(0.0, rw)); - c1 = c1.max(col.clamp(0.0, rw)); + if col != *c { + w0 = w0.min(col.clamp(0.0, rw)); + w1 = w1.max(col.clamp(0.0, rw)); + } else { + c0 = c0.min(col.clamp(0.0, rw)); + c1 = c1.max(col.clamp(0.0, rw)); + } r0 = r0.min(row.clamp(0.0, rh)); r1 = r1.max(row.clamp(0.0, rh)); } - if wrapped { - (c0, c1) = (0.0, rw); + let mut cols2 = None; + if w0 < w1 { + if c0 < c1 { + // Both sides: the seam is in view, so each strip runs to it (the sample points stop + // short of it by up to a grid step). + cols2 = Some((w0, rw)); + c0 = 0.0; + } else { + (c0, c1) = (w0, w1); + } } if !(c0 < c1 && r0 < r1) { return None; @@ -305,7 +339,7 @@ fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_ } } // In the pixels drawn (a work pixel is `k` device pixels). - per_px.is_finite().then_some(InView { cols: (c0, c1), rows: (r0, r1), per_px }) + per_px.is_finite().then_some(InView { cols: (c0, c1), cols2, rows: (r0, r1), per_px }) } /// [`draw`], its rows spread over `threads` threads. @@ -742,6 +776,31 @@ mod tests { assert_eq!([actual.red(), actual.green(), actual.blue(), actual.alpha()], expected); } + /// A 0–360° grid seen across its seam: the part in view is two strips, one at each end of + /// the grid, not the whole width. + #[test] + fn a_view_across_the_seam_of_a_full_turn_grid_is_two_strips() { + let gt = GeoTransform { origin_x: 0.0, origin_y: 90.0, pixel_width: 10.0, pixel_height: -10.0 }; + let r = Raster::new("turn", 36, 18, vec![vec![1.0; 36 * 18]], gt, Crs::LonLat, None, ColorHint::Gray, "Float32").unwrap(); + // About 90° to each side of 0°: the east strip from column 0, the west one to column 36. + let view = View { + width: 400.0, + height: 200.0, + pixels_per_point: 1.0, + camera: Camera { center: Coord::new(0.0, 0.0), res: 50_000.0, bearing: 0.0, pitch: 0.0 }, + sky: false, + }; + let w = in_view(&r, &view, Place::Flat(&Flat::Mercator, "mercator"), 400, 200, true).unwrap(); + assert!(w.cols.0 == 0.0 && w.cols.1 > 5.0 && w.cols.1 < 18.0, "{:?}", w.cols); + let (a, b) = w.cols2.unwrap(); + assert!(a > 18.0 && a < 31.0 && b == 36.0, "{:?}", (a, b)); + // East of 0° only: one strip. + let east = + View { camera: Camera { center: lonlat_to_merc(Coord::new(100.0, 0.0)), res: 50_000.0, bearing: 0.0, pitch: 0.0 }, ..view }; + let w = in_view(&r, &east, Place::Flat(&Flat::Mercator, "mercator"), 400, 200, true).unwrap(); + assert!(w.cols2.is_none() && w.cols.0 > 0.0, "{:?}", (w.cols, w.cols2)); + } + #[test] fn shaded_relief_is_unchanged_when_the_geographic_seam_moves() { let elevations: Vec = (0..36).map(|c| (((c as f64 + 0.5) * 10.0).to_radians().sin() * 1_000_000.0) as f32).collect(); From d7d66752b308e89ce885ff9b6adfd940798f4bd3 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Mon, 2 Nov 2026 23:54:37 -0330 Subject: [PATCH 15/27] fix(raster): a turned in-place layer zoomed in past its overview says so, and exports refuse it - A turned layer draws its overview; zoomed in past it, the map reports it and the PNG and PMTiles exports refuse, quoting the layer's own reason. The user guide says so. - An HTTP date's hour, minute and second are parsed unsigned. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/tests.rs | 22 ++++++++++++++++++++++ crates/render/src/lib.rs | 10 ++++++---- crates/render/src/raster.rs | 13 +++++++++++-- crates/tiles/src/failure.rs | 4 +++- docs/user-guide/data-formats.md | 5 +++-- 5 files changed, 45 insertions(+), 9 deletions(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 151a1e108..ea987dc8d 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13395,6 +13395,28 @@ fn a_png_export_of_an_unreadable_in_place_layer_is_refused() { assert!(!png.exists()); } +/// A turned layer read in place is drawn from its overview: zoomed in past it, an export is +/// refused rather than written from the overview; zoomed out, it is written. +#[test] +fn a_turned_in_place_layer_is_not_exported_past_its_overview() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(2048, 256)); + let mut s = Session::new(Project::default()); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif"), "name": "Scene"})).unwrap(); + assert_eq!(r["width"], 1024, "{r}"); + let id = s.map().layer_by_name("Scene").unwrap().id; + s.map_mut().layer_mut(id).and_then(|l| l.raster.as_mut()).unwrap().style.rotation = 15.0; + let png = std::env::temp_dir().join(format!("rustgis-turned-{}.png", std::process::id())); + // Zoomed in past the overview (its pixels are about 50 m): refused. + s.map_mut().camera.res = 12.0; + s.view_size = (64.0, 64.0); + let e = s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap_err(); + assert!(e.contains("turned") && e.contains("isn't written"), "{e}"); + // Zoomed out to the overview's resolution and beyond: written. + s.map_mut().camera.res = 200.0; + s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap(); + assert!(png.exists()); +} + /// A layer read in place that can't be read says so, once, as its project opens. #[test] fn an_in_place_layer_that_cant_be_read_says_so() { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 9fa418505..0aa81f358 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -4946,10 +4946,12 @@ pub fn render_png(map: &mut Map, view: &View) -> Result, String> { return Err(format!("{}: its tiles are still on their way: export again once the map has drawn it", l.name)); } if let Some(l) = r.partial.first().and_then(|id| map.layers.iter().find(|l| l.id == *id)) { - return Err(format!( - "{}: drawn from less than the view needs (a coarser level, or tiles that couldn't be read), so the image isn't written", - l.name - )); + // Its own error, named by the layer, says why; else the general one. + let why = r.tile_errors.iter().find(|e| e.starts_with(&format!("{}: ", l.name))).cloned(); + let why = why.unwrap_or_else(|| { + format!("{}: drawn from less than the view needs (a coarser level, or tiles that couldn't be read)", l.name) + }); + return Err(format!("{why}, so the image isn't written")); } let mut pm = Canvas::from_pixmap(r.pixmap); for l in r.unplaced.iter().chain(&r.labels) { diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index b58543a9d..b6da3cd8b 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -107,6 +107,8 @@ pub(crate) fn draw( const WINDOW_GRID: usize = 17; /// Why a part in view was drawn from a coarser level than it needs. const TOO_LARGE: &str = "the view is too large a part of the image to read at the resolution it needs"; +/// Why a turned layer is drawn from its overview. +const TURNED: &str = "turned, it is drawn from its overview, not at the resolution the view needs"; /// Draws a raster layer read in place (a cloud-optimised GeoTIFF by its address, see /// [`rustgis_map::raster::RasterLayer::in_place`]) as [`draw`] draws one held whole: its overview @@ -132,12 +134,19 @@ pub(crate) fn draw_in_place( let own: Vec = bands.iter().map(|b| r.bands.get(*b).map(|b| b.stats.clone()).unwrap_or_default()).collect(); let stats = stats.unwrap_or(&own); let whole = |pm: &mut Pixmap| draw(pm, r, style, Some(stats), view, opacity, full_res, place); - // A rotated raster turns about the whole image's centre, not the part in view's. let full = r.overview.unwrap_or((r.width, r.height)); - if style.rotation.rem_euclid(360.0).abs() > 1e-6 || full.0 <= r.width { + if full.0 <= r.width { whole(pm); return Ok(false); } + // A rotated raster turns about the whole image's centre, not the part in view's: the + // overview is drawn, and zoomed in past it (a screen pixel under 0.8 of its pixels) that is + // less than the view needs: said, and refused by exports. + if style.rotation.rem_euclid(360.0).abs() > 1e-6 { + whole(pm); + let zoomed_in = in_view(r, view, place, pm.width(), pm.height(), full_res).is_some_and(|w| w.per_px < 0.8); + return if zoomed_in { Err(TURNED.into()) } else { Ok(false) }; + } let info = match rustgis_tiles::cog::info(address) { Ok(Some(info)) => info, Ok(None) => { diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index d543156f0..88302bc8b 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -231,7 +231,8 @@ fn http_date(s: &str) -> Option { const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; let month = MONTHS.iter().position(|m| *m == month)? as i64 + 1; let (day, year): (i64, i64) = (day.parse().ok()?, year.parse().ok()?); - let mut hms = time.split(':').map(|p| p.parse::().ok()); + // Each field unsigned: a sign isn't a time. + let mut hms = time.split(':').map(|p| p.parse::().ok().map(i64::from)); let (h, m, sec) = (hms.next()??, hms.next()??, hms.next()??); let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); let days_in_month = match month { @@ -466,6 +467,7 @@ mod tests { assert_eq!(http_date("Sun, 06 Nov 1994 08:49:37 PST"), None); assert_eq!(http_date("Sun, 32 Nov 1994 08:49:37 GMT"), None); assert_eq!(http_date("Wed, 31 Feb 2024 00:00:00 GMT"), None, "no 31 February"); + assert_eq!(http_date("Sun, 06 Nov 1994 -8:49:37 GMT"), None, "no negative hour"); assert!(http_date("Thu, 29 Feb 2024 00:00:00 GMT").is_some() && http_date("Sat, 29 Feb 2025 00:00:00 GMT").is_none()); // A refused answer's own header. let headers = vec![("Retry-After".to_string(), "12".to_string())]; diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 96b9f160d..ce28642f8 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -109,8 +109,9 @@ identify and the raster tools work on it as on a file's. The layer holds an over file too large to hold whole is opened from an overview; zoomed in past it, the map reads the part in view at the resolution it needs as it draws, and draws it with the layer's style and the overview's statistics, so the image looks the same at every zoom. Until a part's tiles arrive, the -level zoomed from stands in for them. The Details pane says the image's full size and that it -is read in place. A file whose smallest image is too large to hold as the overview (a large image +level zoomed from stands in for them. A turned layer (Rotation in the Symbology pane) is drawn +from the overview it holds: zoomed in past it, the map says so and an export refuses it. The +Details pane says the image's full size and that it is read in place. A file whose smallest image is too large to hold as the overview (a large image saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` makes the overviews. A tile that fails to arrive is asked for again a minute later, or when its server says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again then. From 95983b4ca478a1ebd09526c2adbf1dfbba4ceb17 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Tue, 3 Nov 2026 13:58:07 -0330 Subject: [PATCH 16/27] fix(raster): review of the merge with main - A raster read in place from an S3 bucket a project opens without its connection says so, once, and is read when the connection binds. - An address look naming bands the file hasn't is said, with what the pixels lost; the layer draws in its own look and keeps the look's range. - A COG tile layer opened as a raster layer takes a numbered name when another layer has its. - A dead line in the stretch setup and an over-long docs line are gone. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/tiles.rs | 9 ++++++- crates/engine/src/s3_connections.rs | 6 +++++ crates/engine/src/s3_saved_tests.rs | 18 +++++++++++++ crates/engine/src/tests.rs | 41 ++++++++++++++++++++++++++++ crates/map/src/raster.rs | 42 +++++++++++++++++++++++++++-- crates/tiles/src/cog.rs | 1 - docs/user-guide/data-formats.md | 3 ++- 7 files changed, 115 insertions(+), 5 deletions(-) diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index b40717d37..5e2e9fd02 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -571,7 +571,14 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { // Read with the project's other rasters read in place (see `project::read_in_place`), in // its look once it lands. let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)).from_tiles_dropping(dropped); - let id = m.add_raster(t.name.clone(), rl); + // A name another layer has: numbered, as a layer added is. + let mut name = t.name.clone(); + let mut i = 1; + while m.layer_by_name(&name).is_some() { + i += 1; + name = format!("{} ({i})", t.name); + } + let id = m.add_raster(name, rl); if let Some(l) = m.layer_mut(id) { l.visible = t.visible; l.transparency = (1.0 - t.opacity.clamp(0.0, 1.0)) * 100.0; diff --git a/crates/engine/src/s3_connections.rs b/crates/engine/src/s3_connections.rs index 803f6d76a..ab1063ba9 100644 --- a/crates/engine/src/s3_connections.rs +++ b/crates/engine/src/s3_connections.rs @@ -274,6 +274,12 @@ fn block(s: &mut Session, g: &Group) -> usize { src.error = Some(s3::UNRESOLVED.into()); } } + // A raster read in place from the bucket waits for the connection too, and says so. + for rl in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.raster.as_mut()) { + if rl.in_place && rl.data.is_none() && rl.path.as_deref().is_some_and(|p| g.has(p)) { + rl.fail(s3::UNRESOLVED); + } + } s.project.addresses_mut().into_iter().filter(|a| g.has(a)).count() } diff --git a/crates/engine/src/s3_saved_tests.rs b/crates/engine/src/s3_saved_tests.rs index aff2c145e..ff3df4c52 100644 --- a/crates/engine/src/s3_saved_tests.rs +++ b/crates/engine/src/s3_saved_tests.rs @@ -815,6 +815,24 @@ fn s3_saved_sources_without_keys_zero_matches_and_older_addresses_stay_blocked_o assert!(none.notices.iter().any(|n| n.error && n.text.contains("no saved connection")), "{:?}", none.notices); assert_eq!(none.map().layers.iter().find_map(|l| l.source.as_ref()).unwrap().error.as_deref(), Some(s3::UNRESOLVED)); assert_eq!(p.srv.requests(), from); + // A GeoTIFF read in place from the bucket waits for the connection too, and says so, once. + let fgb = none.map().layers.iter().find_map(|l| l.source.as_ref().and_then(|src| src.path.clone())).unwrap(); + let scene = fgb.replace("grid.fgb", "scene.tif"); + assert_ne!(scene, fgb); + none.map_mut().add_raster("Scene".to_string(), rustgis_map::raster::RasterLayer::unread_in_place(scene)); + let text = serde_json::to_string(&none.project).unwrap(); + let mut none = session(Some(&empty)); + none.run("project.openText", &json!({"text": text})).unwrap(); + let scene = none.map().layers.iter().find(|l| l.name == "Scene").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(scene.data.is_none() && scene.error.as_deref() == Some(s3::UNRESOLVED), "{:?}", scene.error); + // One notice (the project's maps each hold the layer: one line naming them all). + let said = |s: &Session| { + s.notices.iter().filter(|n| n.error && n.text.contains("couldn't be read in place") && n.text.contains("Scene")).count() + }; + assert_eq!(said(&none), 1, "{:?}", none.notices); + none.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&none), 1); + assert_eq!(p.srv.requests(), from, "nothing fetched without the connection"); // An older project's bare address, with a session-only connection of the bucket: as before. let legacy = p.text.replace(&p.fgb, "s3://sv-proj-edge/grid.fgb"); assert!(legacy.contains("s3://sv-proj-edge/grid.fgb") && !legacy.contains(&p.fgb), "the Grid layer is an older project's"); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ccf4bec49..5d4360089 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13777,6 +13777,47 @@ fn a_turned_in_place_layer_is_not_exported_past_its_overview() { assert!(png.exists()); } +/// An address look naming bands the file hasn't (`#bands=9,8,7` on a 3-band file) is said, and +/// the layer draws in its own look; its range is kept. +#[test] +fn a_look_naming_bands_the_file_hasnt_is_said() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{RasterRenderer, Stretch}; + let (base, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, bands: 3, ..Example::default() })); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif#bands=9,8,7&range=10,200"), "name": "Scene"})).unwrap(); + let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("asks for band(s) 9, 8, 7")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + let rl = s.map().layer_by_name("Scene").and_then(|l| l.raster.as_ref()).unwrap(); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, "the file's own look: three grey bands as RGB"); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), 1); +} + +/// A COG tile layer opened as a raster layer takes a numbered name when another layer has its +/// name, as a layer added does. +#[test] +fn a_migrated_cog_layer_takes_a_free_name() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + let empty = + r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}"#; + s.run("layer.addDataText", &json!({"fileName": "scene.geojson", "text": empty, "name": "Scene"})).unwrap(); + assert!(s.map().layer_by_name("Scene").is_some()); + let mut text: Value = serde_json::to_value(&s.project).unwrap(); + let tile = serde_json::to_value(rustgis_map::TileLayer::new("Scene", rustgis_map::TileKind::Cog, format!("{base}/scene.tif"))).unwrap(); + match text.pointer_mut("/maps").and_then(Value::as_array_mut) { + Some(maps) => maps[0]["tiles"] = json!([tile]), + None => text["map"]["tiles"] = json!([tile]), + } + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text.to_string()})).unwrap(); + let names: Vec<&str> = t.map().layers.iter().map(|l| l.name.as_str()).collect(); + assert!(names.contains(&"Scene") && names.contains(&"Scene (2)"), "{names:?}"); + assert!(t.map().layer_by_name("Scene (2)").and_then(|l| l.raster.as_ref()).is_some_and(|r| r.in_place)); +} + /// A layer read in place that can't be read says so, once, as its project opens. #[test] fn an_in_place_layer_that_cant_be_read_says_so() { diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 85a3771b1..9ab1db985 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -78,6 +78,10 @@ pub struct RasterLayer { /// [`RasterLayer::lost_unsaid`]). #[serde(skip)] lost_said: bool, + /// What its address's look asked for that the file hasn't (bands past its own), said with + /// what the pixels lost. + #[serde(skip)] + look_lost: Option, /// Settings of the tile layer this one was opened from that a raster layer hasn't (see /// [`RasterLayer::dropped_unsaid`]). #[serde(skip)] @@ -222,6 +226,7 @@ impl RasterLayer { pixels_said: false, error_said: false, lost_said: false, + look_lost: None, dropped: Vec::new(), }) } @@ -248,6 +253,7 @@ impl RasterLayer { pixels_said: false, error_said: false, lost_said: false, + look_lost: None, dropped: Vec::new(), } } @@ -274,6 +280,7 @@ impl RasterLayer { pixels_said: false, error_said: false, lost_said: false, + look_lost: None, dropped: Vec::new(), } } @@ -305,6 +312,7 @@ impl RasterLayer { pixels_said: false, error_said: false, lost_said: false, + look_lost: None, dropped: Vec::new(), } } @@ -322,6 +330,8 @@ impl RasterLayer { pub fn with_look(mut self, look: AddressLook) -> RasterLayer { match self.data.clone() { Some(d) => { + self.look_lost = RasterStyle::look_lost(&d, &look); + self.lost_said &= self.look_lost.is_none(); if let Some(style) = RasterStyle::for_look(&d, &look) { self.style = style; } @@ -331,6 +341,13 @@ impl RasterLayer { self } + /// Marks the layer as one that couldn't be read, and why (a project opened without the + /// connection its address needs), for [`RasterLayer::error_unsaid`] to say. + pub fn fail(&mut self, why: impl Into) { + self.error = Some(why.into()); + self.error_said = false; + } + /// Reads the pixels again from the path or the embedded file (after a project opens). A /// layer that follows the time slider is left to the engine, which reads the step shown. pub fn reload(&mut self) -> Result<(), String> { @@ -488,7 +505,8 @@ impl RasterLayer { if self.lost_said { return Vec::new(); } - let lost = self.data.as_ref().map(|d| d.lost.clone()).unwrap_or_default(); + let mut lost = self.data.as_ref().map(|d| d.lost.clone()).unwrap_or_default(); + lost.extend(self.look_lost.take()); self.lost_said = self.data.is_some(); lost } @@ -523,7 +541,9 @@ impl RasterLayer { self.pixels_said = false; self.lost_said = false; if std::mem::take(&mut self.restyle) { - let look = self.look.take().and_then(|look| RasterStyle::for_look(&raster, &look)); + let look = self.look.take(); + self.look_lost = look.as_ref().and_then(|look| RasterStyle::look_lost(&raster, look)); + let look = look.and_then(|look| RasterStyle::for_look(&raster, &look)); self.style = look.unwrap_or_else(|| RasterStyle::default_for(&raster)); } else { self.style.repair(&raster); @@ -1025,6 +1045,14 @@ impl RasterStyle { if look.bands.is_empty() && look.range.is_none() { return None; } + // Bands the file hasn't: its own look (said by `look_lost`). + if Self::look_lost(r, look).is_some() { + let mut style = RasterStyle::default_for(r); + if let Some((lo, hi)) = look.range { + style.stretch = Stretch::custom(lo, hi); + } + return Some(style); + } let mut style = RasterStyle::default_for(r); let n = r.band_count(); match look.bands[..] { @@ -1041,6 +1069,16 @@ impl RasterStyle { Some(style) } + /// What a look asks for that `r` hasn't: bands past its own, so the file's own look is drawn + /// instead; `None` when the look fits. + pub fn look_lost(r: &Raster, look: &AddressLook) -> Option { + let n = r.band_count(); + let past: Vec = look.bands.iter().filter(|b| **b >= n).map(|b| (b + 1).to_string()).collect(); + (!past.is_empty()).then(|| { + format!("its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): drawn in its own look", past.join(", ")) + }) + } + /// A style for a layer whose pixels haven't arrived (its own look replaces it when they do). pub fn placeholder() -> RasterStyle { RasterStyle { diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 6c35354c9..cdad5efc1 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -618,7 +618,6 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box Date: Tue, 3 Nov 2026 15:54:15 -0330 Subject: [PATCH 17/27] fix(raster): review of the merge with main, second round - A two-band `#bands=` look is said; the layer draws the file's own look with the look's range. - A tile a sparse file leaves out is kept as empty: a window over it is whole at once, and an export of the view over it is written rather than refused after waiting for it. `Example` writes such a file. - A header refused with `Retry-After` (a GeoTIFF's or a PMTiles archive's) asks the map to draw again when it may be read, on the failure and on each draw while it waits. - `add_in_place` checks the address as the project path does. - The bands' statistics are built only when the caller passes none; doc comments and a name counter tidied; the unchecked-address test asserts the rule named. Co-Authored-By: Claude Fable 5.1 --- crates/data/src/raster/cog.rs | 22 ++++++-- crates/data/src/raster/geotiff.rs | 6 +-- crates/engine/src/cmd/layer.rs | 2 + crates/engine/src/cmd/tiles.rs | 6 +-- crates/engine/src/tests.rs | 90 +++++++++++++++++++++++++++++-- crates/map/src/raster.rs | 19 +++++-- crates/render/src/raster.rs | 5 +- crates/tiles/src/cog.rs | 33 ++++++++++-- crates/tiles/src/cog/raster.rs | 24 ++++++--- crates/tiles/src/pmtiles.rs | 17 +++++- docs/user-guide/data-formats.md | 9 ++-- 11 files changed, 197 insertions(+), 36 deletions(-) diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 3d5ef233c..a43315322 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -639,6 +639,9 @@ pub struct Example { pub stray: bool, /// With `classes`: a palette image with no colour table. pub no_table: bool, + /// The full image's first tile left out, as GDAL's sparse files leave out tiles of no-data + /// (no bytes, no offset). + pub sparse: bool, } impl Default for Example { @@ -658,6 +661,7 @@ impl Default for Example { huge: false, stray: false, no_table: false, + sparse: false, } } } @@ -708,6 +712,12 @@ pub fn example_with(e: &Example) -> Vec { chunks.push(c); } } + if e.sparse + && k == 1 + && let Some(first) = chunks.first_mut() + { + first.clear(); + } chunks }) .collect(); @@ -756,7 +766,7 @@ pub fn example_with(e: &Example) -> Vec { (322, V::L(vec![tile])), (323, V::L(vec![tile])), (324, V::L(vec![0; n])), - (325, V::L(vec![chunk_len; n])), + (325, V::L(images.get(i).map_or(vec![chunk_len; n], |c| c.iter().map(|c| c.len() as u32).collect()))), ]; // The samples past the colour ones (three of an RGB image, else one): the first alpha // when asked, the others undefined. @@ -822,14 +832,16 @@ pub fn example_with(e: &Example) -> Vec { let mut data_at = at; let mut out = vec![b'I', b'I', 42, 0]; out.extend_from_slice(&header_len.to_le_bytes()); - for t in ifds.iter_mut() { + for (lvl, t) in ifds.iter_mut().enumerate() { for (tag, v) in t.iter_mut() { if *tag == 324 && let V::L(offs) = v { - for o in offs.iter_mut() { - *o = data_at as u32; - data_at += chunk_len as usize; + for (j, o) in offs.iter_mut().enumerate() { + // A chunk left out has no offset, as GDAL writes a sparse file. + let len = images.get(lvl).and_then(|c| c.get(j)).map_or(0, Vec::len); + *o = if len == 0 { 0 } else { data_at as u32 }; + data_at += len; } } } diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index 75772100a..c51967222 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -41,15 +41,15 @@ struct Geo { referenced: bool, } -/// How `n` samples of a grey image the decoder calls multiband are shown: grey with alpha when -/// there are two and the second is declared alpha (extra sample 1, associated, or 2, unassociated), -/// else grey. GDAL marks the second band of an ordinary two-band file as 0, unspecified data. /// Whether the first sample past the colour ones is declared alpha (extra sample 1, associated, or /// 2, unassociated). pub(super) fn extra_alpha(dec: &mut Decoder) -> bool { dec.find_tag_unsigned_vec::(Tag::ExtraSamples).ok().flatten().is_some_and(|v| matches!(v.first(), Some(1 | 2))) } +/// How `n` samples of a grey image the decoder calls multiband are shown: grey with alpha when +/// there are two and the second is declared alpha (extra sample 1, associated, or 2, unassociated), +/// else grey. GDAL marks the second band of an ordinary two-band file as 0, unspecified data. pub(super) fn multiband_color(dec: &mut Decoder, n: u16) -> ColorHint { let alpha = n == 2 && dec.find_tag_unsigned_vec::(Tag::ExtraSamples).ok().flatten().is_some_and(|v| matches!(v.first(), Some(1 | 2))); diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 640e83d9d..3c192e7df 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3157,6 +3157,8 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) /// address's end (`#bands=…&range=…`) is its style. fn add_in_place(s: &mut Session, name: &str, file: &str, look: &crate::render::rustgis_tiles::cog::Look, strict: bool) -> CmdResult { use crate::render::rustgis_tiles::cog; + // Judged as an address a project keeps is (a host, no spaces, not too long) before it is read. + rustgis_map::tiles::check_url(file)?; // Strict, natively: read now, failing when it can't be. Otherwise read with the other layers // added with it (see `project::read_in_place`); in the browser, landing later. #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 5e2e9fd02..f06e0ef85 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -573,10 +573,10 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)).from_tiles_dropping(dropped); // A name another layer has: numbered, as a layer added is. let mut name = t.name.clone(); - let mut i = 1; + let mut nth = 1; while m.layer_by_name(&name).is_some() { - i += 1; - name = format!("{} ({i})", t.name); + nth += 1; + name = format!("{} ({nth})", t.name); } let id = m.add_raster(name, rl); if let Some(l) = m.layer_mut(id) { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index c864d478b..52b92cf03 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13721,6 +13721,83 @@ fn cog_windows_stand_in_the_level_kept_until_their_tiles_arrive() { assert_eq!(v(&part, 1001, 901), ((1001 + 901) % 256) as f32); } +/// A tile a sparse file leaves out (GDAL's: no bytes) is empty, not waited for: a window over it +/// is whole at once and asks for nothing more, and an export of the view over it is written rather +/// than refused after waiting for the tile. +#[test] +fn a_tile_a_sparse_file_leaves_out_is_empty_not_waited_for() { + use crate::render::rustgis_tiles::cog::raster::window; + use rustgis_data::raster::cog::{Example, example_with}; + use std::sync::atomic::Ordering; + let (base, sent) = serve_ranges(example_with(&Example { size: 2048, tile: 256, sparse: true, ..Example::default() })); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + // Its overview is level 1 (1024 pixels a side); level 0's first tile has no bytes. + let r = s.run("layer.addUrl", &json!({"url": url.clone(), "name": "Scene"})).unwrap(); + assert_eq!(r["width"], 1024, "{r}"); + let before = sent.load(Ordering::Relaxed); + let gray = rustgis_data::raster::ColorHint::Gray; + for _ in 0..2 { + let w = window(&url, 0, (0, 64), (0, 64), &[0], gray.clone()).unwrap(); + assert!(!w.pending, "an empty tile isn't waited for"); + assert!(w.raster.unwrap().bands[0].data.iter().all(|v| v.is_nan()), "empty"); + } + assert_eq!(sent.load(Ordering::Relaxed), before, "nothing is fetched for it"); + // The view over it (the image's north-west corner), zoomed in past the overview: written. + s.view_size = (64.0, 64.0); + s.zoom_to(&rustgis_geom::Extent::new(10.02, 45.92, 10.08, 45.98)); + s.map_mut().camera.res = 12.0; + let png = std::env::temp_dir().join(format!("rustgis-sparse-{}.png", std::process::id())); + s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap(); + assert!(png.exists()); + let _ = std::fs::remove_file(png); +} + +/// A GeoTIFF whose server refuses its header with a 429 or 503 and a `Retry-After` isn't asked +/// for again before then, and the map is asked to draw again by then (as for a tile), so the +/// layer doesn't stay failed until something else draws the map. +#[test] +fn a_header_refused_with_retry_after_is_read_again_when_it_says() { + use crate::render::rustgis_tiles::{cache, cog}; + use std::io::{Read, Write}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut s) = stream else { continue }; + let mut buf = [0u8; 4096]; + let _ = s.read(&mut buf); + let _ = s.write_all(b"HTTP/1.1 503 Service Unavailable\r\nRetry-After: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + } + }); + // The app's alarm: each wait asked for is noted. + let asked = Arc::new(Mutex::new(Vec::new())); + let noted = asked.clone(); + cache::set_alarm(move |d| noted.lock().unwrap().push(d)); + let url = format!("http://{addr}/scene.tif"); + let mut s = Session::new(Project::default()); + let e = s.run("layer.addUrl", &json!({"url": url.clone(), "name": "Scene"})).unwrap_err(); + assert!(e.contains("retry after 1 s"), "{e}"); + // Not asked for again before then (the error stands), and the map is asked to draw again by + // then: on the failure, and on each draw while it waits (another alarm may have fired since). + let mut seen = None; + for _ in 0..30 { + let _ = cache::generation(); + // The error stands until then; its wait over, it is read again. + if let Err(e) = cog::info(&url) { + assert!(e.contains("retry after 1 s"), "{e}"); + } + seen = asked.lock().unwrap().iter().find(|d| **d <= Duration::from_secs(1)).copied(); + if seen.is_some() { + break; + } + cache::wait(Duration::from_millis(50)); + } + assert!(seen.is_some(), "no alarm within a second: {:?}", asked.lock().unwrap()); +} + /// A GeoTIFF's address look (`#bands=…&range=…`) is its layer's once its pixels land, read later /// (as the browser and an import read them) as when read at once; a palette image stretched by a /// range is drawn in grey, as its tile layer was, and without a look in its colours. @@ -13773,7 +13850,7 @@ fn an_unchecked_in_place_address_is_refused() { let mut s = Session::new(Project::default()); s.run("project.openText", &json!({"text": text})).unwrap(); let rl = s.map().layers.iter().find(|l| l.name == "Odd").and_then(|l| l.raster.as_ref()).unwrap(); - assert!(rl.data.is_none() && rl.error.as_deref().is_some_and(|e| !e.contains("no answer")), "{:?}", rl.error); + assert!(rl.data.is_none() && rl.error.as_deref().is_some_and(|e| e.contains("tile addresses start with")), "{:?}", rl.error); assert_eq!(s.notices.iter().filter(|n| n.error && n.text.contains("Odd: couldn't be read in place")).count(), 1); // Added directly (the browser's path), refused before the reader is asked. let added = RasterLayer::opening_in_place(bad); @@ -13832,8 +13909,9 @@ fn a_turned_in_place_layer_is_not_exported_past_its_overview() { assert!(png.exists()); } -/// An address look naming bands the file hasn't (`#bands=9,8,7` on a 3-band file) is said, and -/// the layer draws in its own look; its range is kept. +/// An address look naming bands the file hasn't (`#bands=9,8,7` on a 3-band file), or two bands +/// (`#bands=1,2`, which no raster look draws), is said, and the layer draws in its own look; its +/// range is kept. #[test] fn a_look_naming_bands_the_file_hasnt_is_said() { use rustgis_data::raster::cog::{Example, example_with}; @@ -13848,6 +13926,12 @@ fn a_look_naming_bands_the_file_hasnt_is_said() { assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); s.run("map.zoomIn", &Value::Null).unwrap(); assert_eq!(said(&s), 1); + // Two bands: neither grey nor RGB, said as such. + s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif#bands=1,2&range=10,200"), "name": "Pair"})).unwrap(); + assert_eq!(s.notices.iter().filter(|n| n.text.contains("Pair") && n.text.contains("asks for 2 bands")).count(), 1, "{:?}", s.notices); + let rl = s.map().layer_by_name("Pair").and_then(|l| l.raster.as_ref()).unwrap(); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); } /// A COG tile layer opened as a raster layer takes a numbered name when another layer has its diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 9ab1db985..988331b11 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -1069,13 +1069,24 @@ impl RasterStyle { Some(style) } - /// What a look asks for that `r` hasn't: bands past its own, so the file's own look is drawn - /// instead; `None` when the look fits. + /// What a look asks for that `r` can't draw: bands past its own, or two bands (one is drawn + /// in grey, three as red, green and blue), so the file's own look is drawn instead; `None` + /// when the look fits. pub fn look_lost(r: &Raster, look: &AddressLook) -> Option { let n = r.band_count(); let past: Vec = look.bands.iter().filter(|b| **b >= n).map(|b| (b + 1).to_string()).collect(); - (!past.is_empty()).then(|| { - format!("its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): drawn in its own look", past.join(", ")) + if !past.is_empty() { + return Some(format!( + "its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): drawn in its own look", + past.join(", ") + )); + } + // An address written by hand: `map.setTiles` takes one band or three. + (!matches!(look.bands.len(), 0 | 1 | 3)).then(|| { + format!( + "its address asks for {} bands; one is drawn in grey, three as red, green and blue: drawn in its own look", + look.bands.len() + ) }) } diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index b6da3cd8b..a0119a792 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -131,7 +131,10 @@ pub(crate) fn draw_in_place( place: Place, ) -> Result { let bands = style.renderer.bands(); - let own: Vec = bands.iter().map(|b| r.bands.get(*b).map(|b| b.stats.clone()).unwrap_or_default()).collect(); + let own: Vec = match stats { + Some(_) => Vec::new(), + None => bands.iter().map(|b| r.bands.get(*b).map(|b| b.stats.clone()).unwrap_or_default()).collect(), + }; let stats = stats.unwrap_or(&own); let whole = |pm: &mut Pixmap| draw(pm, r, style, Some(stats), view, opacity, full_res, place); let full = r.overview.unwrap_or((r.width, r.height)); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index cdad5efc1..2fc4718f5 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -351,7 +351,14 @@ pub fn info(url: &str) -> Result>, String> { match i.get(url) { Some(State::Ready(info)) => return Ok(Some(info.clone())), Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => return Err(e.clone()), + Some(State::Failed(e, at)) => { + // Not read again until its wait is over: the map draws again then. + let wait = crate::cache::retry_wait(e, RETRY); + if at.elapsed() < wait { + crate::cache::notify_at(*at + wait); + return Err(e.clone()); + } + } _ => {} } } @@ -374,6 +381,7 @@ fn with_info(url: &str, then: Box, String>) + Send>) return; } Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { + crate::cache::notify_at(*at + crate::cache::retry_wait(e, RETRY)); let e = e.clone(); drop(i); then(Err(e)); @@ -392,7 +400,12 @@ fn with_info(url: &str, then: Box, String>) + Send>) let mut i = infos().lock().unwrap_or_else(|e| e.into_inner()); let next = match &r { Ok(info) => State::Ready(info.clone()), - Err(e) => State::Failed(e.clone(), Instant::now()), + Err(e) => { + // Read again once its wait is over: the map draws again then (a 429 + // or 503 says when). + crate::cache::notify_at(Instant::now() + crate::cache::retry_wait(e, RETRY)); + State::Failed(e.clone(), Instant::now()) + } }; match i.insert(key, next) { Some(State::Loading(w)) => w, @@ -883,9 +896,21 @@ fn get_tile(info: Arc, url: String, level: usize, idx: u32, then: Box b, + None => { + let b: Bands = Arc::new(vec![vec![f32::NAN; per_band]; nb].into()); + t.keep(key, b.clone(), samples); + b + } + }; + drop(t); + then(Ok(empty)); return; } // Bytes at offset 0 would be the file's header: a broken index, said rather than drawn blank. diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 29bb0a595..a89dce4c0 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -249,7 +249,6 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands } } } - let mut pending = kept.len() < chunks.len(); let waiting: bool; { let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); @@ -269,11 +268,6 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands } waiting = next.is_some(); } - // Tiles that failed lately aren't waited for (the map would wait for them for nothing). - if ask.is_empty() && pending { - let t = tiles().lock().unwrap_or_else(|e| e.into_inner()); - pending = chunks.iter().any(|c| matches!(t.map.get(&(file.to_string(), level, *c)), Some(TileState::Loading(_)))); - } for c in ask { let key = (file.to_string(), level, c); get_tile( @@ -290,8 +284,24 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands crate::cache::notify(); }), ); - pending = true; + // A tile the file leaves out (a sparse file's) is kept at once: not waited for. + if let Some(b) = tiles().lock().unwrap_or_else(|e| e.into_inner()).used(&(file.to_string(), level, c)) { + kept.insert(c, b); + } } + // The tiles still missing are on their way (the map draws again as they land) or failed lately + // (not waited for: the map would wait for them for nothing; asked for again when their wait is + // over). + let missing: Vec = chunks.iter().copied().filter(|c| !kept.contains_key(c)).collect(); + let pending = !missing.is_empty() && { + let t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + missing.iter().any(|c| matches!(t.map.get(&(file.to_string(), level, *c)), Some(TileState::Loading(_)))) + }; + let waiting = waiting + || !missing.is_empty() && { + let f = failed().lock().unwrap_or_else(|e| e.into_inner()); + missing.iter().any(|c| f.contains_key(&(file.to_string(), level, *c))) + }; let (mut data, gaps) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); // Until they arrive, its missing tiles show the levels kept (the one zoomed from), not holes. if !gaps.is_empty() { diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index c1e7be3e6..50e217712 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -414,7 +414,14 @@ pub fn info(archive: &str) -> Result>, String> { match all.get(archive) { Some(State::Ready(i)) => return Ok(Some(i.clone())), Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => return Err(e.clone()), + Some(State::Failed(e, at)) => { + // Not read again until its wait is over: the map draws again then. + let wait = crate::cache::retry_wait(e, RETRY); + if at.elapsed() < wait { + crate::cache::notify_at(*at + wait); + return Err(e.clone()); + } + } _ => {} } all.insert(archive.to_string(), State::Loading(Vec::new())); @@ -443,6 +450,7 @@ fn with_info(archive: &str, f: Box, String>) + Send> } Some(State::Loading(waiting)) => waiting.push(f), Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { + crate::cache::notify_at(*at + crate::cache::retry_wait(e, RETRY)); let e = e.clone(); drop(all); f(Err(e)); @@ -500,7 +508,12 @@ fn finish(archive: &str, r: Result) { // A read the busy queue dropped isn't the archive's fault: forgotten, so the next // draw reads it again, rather than kept as a failure. Err(e) if e == crate::cache::DROPPED => all.remove(archive), - Err(e) => all.insert(archive.to_string(), State::Failed(e.clone(), Instant::now())), + Err(e) => { + // Read again once its wait is over: the map draws again then (a 429 or 503 says + // when). + crate::cache::notify_at(Instant::now() + crate::cache::retry_wait(e, RETRY)); + all.insert(archive.to_string(), State::Failed(e.clone(), Instant::now())) + } }; match old { Some(State::Loading(w)) => w, diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 636e1007f..f5f938163 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -114,10 +114,11 @@ overview's statistics, so the image looks the same at every zoom. Until a part's level zoomed from stands in for them. A turned layer (Rotation in the Symbology pane) is drawn from the overview it holds: zoomed in past it, the map says so and an export refuses it. The Details pane says the image's full size and that it is read in place. A file whose smallest -image is too large to hold as the overview (a large image -saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` makes the overviews. A tile -that fails to arrive is asked for again a minute later, or when its server says to in a -`Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again then. +image is too large to hold as the overview (a large image saved without overviews) isn't added: +GDAL's COG driver or `gdaladdo` makes the overviews. A tile a sparse file leaves out is empty. A +tile, or the file's header, that fails to arrive is asked for again a minute later, or when its +server says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map +draws again then. Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, and isn't written without them; in the web app, export again once the map has drawn the image. From 2ce92bce3c76c42e497ea7eb5c27bcebee8b3460 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Tue, 3 Nov 2026 17:24:54 -0330 Subject: [PATCH 18/27] fix(raster): review of the merges with main, third round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A failed overview of a layer read in place is read again once its wait is over (a minute, or the server's Retry-After), as a tile or the file's header is: `RasterLayer::read_failure`/`read_again`, `Session::read_in_place_again` each frame; the read lands as the map draws. A palette image whose address names bands the file hasn't is drawn in grey with the range, as its tile layer drew it, and the notice says so. A tile the fetch queue dropped isn't remembered as failed. Failed tiles are forgotten with `cache::clear` and `cache::forget`. The Details pane says "Reading in place…" while an overview is on its way and shows the error when it failed. The migration's doc comment says it runs on every open. `std::time` → `web_time` in the map crate. Co-Authored-By: Claude Fable 5.1 --- crates/data/src/raster/cog.rs | 1 + crates/engine/src/cmd/tiles.rs | 9 +- crates/engine/src/lib.rs | 21 +++++ crates/engine/src/tests.rs | 83 ++++++++++++++++- crates/map/src/raster.rs | 159 +++++++++++++++++++------------- crates/tiles/src/cache.rs | 6 +- crates/tiles/src/cog.rs | 9 +- crates/tiles/src/cog/raster.rs | 77 +++++++++++++--- crates/ui-egui/src/lib.rs | 2 + crates/ui-egui/src/panes.rs | 15 ++- docs/user-guide/data-formats.md | 6 +- 11 files changed, 292 insertions(+), 96 deletions(-) diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index a43315322..67263e538 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -1066,6 +1066,7 @@ pub(crate) mod tests { } let _ = layout(&Sparse::default().with(0, b"II*\0\xff\xff\xff\x7f".to_vec())); } + #[test] fn every_band_whatever_the_photometric_interpretation_says() { let want = |b: u32| ((300 + 10 * b) % 256) as f32; diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index f06e0ef85..46de70fa0 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -532,9 +532,12 @@ fn blend(p: &Value) -> Result, String> { ps(p, "blend").map(|m| Blend::from_code(m).ok_or_else(|| format!("unknown blend mode '{m}'"))).transpose() } -/// Turns a project's COG tile layers (saved before GeoTIFFs read in place were raster layers) into -/// raster layers read in place, each where it was drawn, with its name, visibility, opacity, blend -/// and look. A dated series, a data cube's files and AlphaEarth's embeddings stay tile layers. +/// Turns a map's plain COG tile layers into raster layers read in place, each where it was drawn, +/// with its name, visibility, opacity, blend and look: those of projects saved before GeoTIFFs read +/// in place were raster layers, and those `map.addTiles` (kind `cog`) still adds, on every open (a +/// plain GeoTIFF is one kind of layer, however it was added; what a tile layer had that a raster +/// layer hasn't is said once, see `Session::notice_landed_rasters`). A dated series, a data cube's +/// files and AlphaEarth's embeddings stay tile layers. pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { use rustgis_map::raster::RasterLayer; let mut i = 0; diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 33a4b6cbc..e40ec7e66 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1597,6 +1597,27 @@ impl Session { } } + /// Reads again the overview of a raster layer read in place whose read failed, once its wait + /// is over: a minute, or what its server's `Retry-After` said, as a tile or a file's header is + /// read again (see `rustgis_tiles::cache::retry_wait`); until then the map is asked to draw + /// again when the wait is over. The app calls it each frame, with + /// [`Session::notice_landed_rasters`]; the pixels land as the map draws. + pub fn read_in_place_again(&mut self) { + use render::rustgis_tiles::cache; + for m in &mut self.project.maps { + for l in &mut m.layers { + let Some(rl) = &mut l.raster else { continue }; + let Some((e, at)) = rl.read_failure() else { continue }; + let wait = cache::retry_wait(e, cache::RETRY); + if at.elapsed() < wait { + cache::notify_at(at + wait); + } else { + rl.read_again(); + } + } + } + } + pub fn notify(&mut self, text: impl Into, error: bool) { use rustgis_data::diagnostics::Severity; self.notice(text.into(), if error { Severity::Error } else { Severity::Info }); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 52b92cf03..548a6f348 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -12847,14 +12847,31 @@ fn serve_file(body: Vec, ranges: bool) -> (String, std::sync::Arc, n: usize) -> (String, std::sync::Arc) { + serve_with(body, true, None, n) +} + /// [`serve_file`] holding every answer until `gate` is set (a request still under way, for as long /// as a test needs). fn serve_gated( body: Vec, ranges: bool, gate: Option>, +) -> (String, std::sync::Arc) { + serve_with(body, ranges, gate, 0) +} + +/// [`serve_gated`], refusing the first `refuse` requests (see [`serve_refusing_first`]). +fn serve_with( + body: Vec, + ranges: bool, + gate: Option>, + refuse: usize, ) -> (String, std::sync::Arc) { use std::io::{Read, Write}; + let refusals = std::sync::Arc::new(std::sync::Mutex::new(refuse)); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let sent = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -12866,10 +12883,22 @@ fn serve_gated( // (the fetch workers are shared by every test in the process). for stream in listener.incoming() { let Ok(mut s) = stream else { continue }; - let (body, count, gate) = (body.clone(), count.clone(), gate.clone()); + let (body, count, gate, refusals) = (body.clone(), count.clone(), gate.clone(), refusals.clone()); std::thread::spawn(move || { let mut buf = [0u8; 4096]; let read = s.read(&mut buf).unwrap_or(0); + // Refused while there are refusals left: busy, ask again in a second. + let refused = { + let mut left = refusals.lock().unwrap(); + let refused = *left > 0; + *left = left.saturating_sub(1); + refused + }; + if refused { + let _ = s + .write_all(b"HTTP/1.1 503 Service Unavailable\r\nRetry-After: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + } if let Some(g) = &gate { let start = std::time::Instant::now(); while !g.load(std::sync::atomic::Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(60) { @@ -13932,6 +13961,58 @@ fn a_look_naming_bands_the_file_hasnt_is_said() { let rl = s.map().layer_by_name("Pair").and_then(|l| l.raster.as_ref()).unwrap(); assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }); assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + // A palette image: in grey with the range, as its tile layer draws one whose address names + // bands (its colours are of one band alone), not in its colours without the range. + let (classes, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, classes: true, ..Example::default() })); + s.run("layer.addUrl", &json!({"url": format!("{classes}/classes.tif#bands=9,8,7&range=10,200"), "name": "Classes"})).unwrap(); + let said = s + .notices + .iter() + .filter(|n| n.text.contains("Classes") && n.text.contains("asks for band(s) 9, 8, 7") && n.text.contains("drawn in grey")); + assert_eq!(said.count(), 1, "{:?}", s.notices); + let rl = s.map().layer_by_name("Classes").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(matches!(rl.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", rl.style.renderer); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); +} + +/// A layer read in place whose overview couldn't be read (its server refused the file's header, +/// saying when to ask again) is read again once its wait is over, as the app draws, and lands then: +/// not before, and failing was said once. +#[test] +fn an_in_place_layers_failed_overview_is_read_again_when_its_wait_is_over() { + use rustgis_map::raster::RasterLayer; + use std::sync::atomic::Ordering; + use std::time::Duration; + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 1); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place(format!("{base}/scene.tif"))); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let layer = |s: &Session| s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + let rl = layer(&s); + assert!(rl.data.is_none()); + assert!(rl.error.as_deref().is_some_and(|e| e.contains("retry after 1 s")), "{:?}", rl.error); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // Not before its wait is over. + s.read_in_place_again(); + assert!(layer(&s).data.is_none()); + assert_eq!(sent.load(Ordering::Relaxed), 0, "read again before its wait was over"); + // Then read again, landing as the map draws. + let view = rustgis_geom::Extent::new(10.0, 45.9, 10.1, 46.0); + let start = std::time::Instant::now(); + while layer(&s).data.is_none() && start.elapsed() < Duration::from_secs(20) { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + s.read_in_place_again(); + s.map_mut().sync_sources(&view, 1000.0, false); + s.notice_landed_rasters(); + } + let rl = layer(&s); + assert!(rl.data.is_some(), "{:?}", rl.error); + assert!(rl.error.is_none()); + assert!(sent.load(Ordering::Relaxed) > 0); + assert_eq!(said(&s), 1, "{:?}", s.notices); } /// A COG tile layer opened as a raster layer takes a numbered name when another layer has its diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 988331b11..c4299a390 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex, Weak}; use rustgis_data::raster::{BandStats, ColorHint, Raster}; use serde::{Deserialize, Serialize}; +use web_time::Instant; use crate::color::{Ramp, Rgba}; use crate::symbology::{ClassMethod, class_breaks, fmt_num}; @@ -74,6 +75,10 @@ pub struct RasterLayer { /// Why it couldn't be read in place has been said (see [`RasterLayer::error_unsaid`]). #[serde(skip)] error_said: bool, + /// When reading its overview in place failed, for the read to be made again once its wait is + /// over (see [`RasterLayer::read_failure`]). + #[serde(skip)] + failed_at: Option, /// What reading its pixels left out (`Raster::lost`) has been said (see /// [`RasterLayer::lost_unsaid`]). #[serde(skip)] @@ -99,7 +104,7 @@ pub fn read_together(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { let mut failed = Vec::new(); let unread = |l: &RasterLayer| l.in_place && l.data.is_none() && l.opening.is_none() && l.time_zarr.is_none(); #[cfg(not(target_arch = "wasm32"))] - let deadline = std::time::Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; + let deadline = Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; #[cfg(not(target_arch = "wasm32"))] let mut started = Vec::new(); for (i, l) in layers.iter_mut().enumerate() { @@ -109,29 +114,14 @@ pub fn read_together(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { // Started without waiting: natively, each is waited for below, together. #[cfg(not(target_arch = "wasm32"))] { - let address = l.path.clone(); - let reader = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone(); - match (address, reader) { - // An address checked as a tile layer's is (http(s) or s3, a host, no control - // characters): a project edited by hand asks for nothing else. - (Some(address), Some(reader)) if crate::tiles::check_url(&address).is_ok() => { - l.error = None; - let slot = Arc::new(Mutex::new(None)); - let land = slot.clone(); - l.opening = Some(slot); - reader( - &address, - Box::new(move |result| { - *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); - crate::source::landed(); - }), - ); - started.push(i); - } - _ => { - if let Err(e) = l.reload() { - failed.push((i, e)); - } + l.error = None; + l.failed_at = None; + match l.begin_in_place_read() { + Ok(()) => started.push(i), + Err(e) => { + l.error = Some(e.clone()); + l.error_said = false; + failed.push((i, e)); } } } @@ -225,6 +215,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + failed_at: None, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -252,6 +243,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + failed_at: None, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -279,6 +271,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + failed_at: None, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -311,6 +304,7 @@ impl RasterLayer { look: None, pixels_said: false, error_said: false, + failed_at: None, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -346,6 +340,57 @@ impl RasterLayer { pub fn fail(&mut self, why: impl Into) { self.error = Some(why.into()); self.error_said = false; + self.failed_at = None; + } + + /// Starts reading its overview in place by the reader the app installed, which answers later + /// (installed by [`RasterLayer::poll_reload`] as the map draws, or waited for by + /// [`RasterLayer::reload`] natively). What it can't start (no address, an address that isn't a + /// tile layer's, no reader here) is the caller's to say. + fn begin_in_place_read(&mut self) -> Result<(), String> { + let Some(address) = self.path.clone() else { return Err("the raster has no source".to_string()) }; + // Checked as a tile layer's address is (http(s) or s3, a host, no control characters): a + // project edited by hand asks for nothing else. + crate::tiles::check_url(&address)?; + let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { + return Err(format!("{address}: can't be read in place here")); + }; + self.data = None; + let slot = Arc::new(Mutex::new(None)); + let land = slot.clone(); + self.opening = Some(slot); + reader( + &address, + Box::new(move |result| { + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); + crate::source::landed(); + }), + ); + Ok(()) + } + + /// Why reading its overview in place failed, and when: for the read to be made again once its + /// wait is over (see [`RasterLayer::read_again`]), as a tile or a file's header is. `None` for + /// a layer with its pixels, one being read, and one that waits for something other than time + /// (its bucket's connection: see [`RasterLayer::fail`]). + pub fn read_failure(&self) -> Option<(&str, Instant)> { + if !self.in_place || self.data.is_some() || self.opening.is_some() { + return None; + } + Some((self.error.as_deref()?, self.failed_at?)) + } + + /// Reads its overview again after a read that failed (see [`RasterLayer::read_failure`]), + /// without waiting: it lands as the map draws, natively as in the browser (nothing may wait + /// each frame). The layer keeps its error until then, and failing the same way again isn't + /// said again. + pub fn read_again(&mut self) { + if self.read_failure().is_none() { + return; + } + if let Err(e) = self.begin_in_place_read() { + self.land(Err(e)); + } } /// Reads the pixels again from the path or the embedded file (after a project opens). A @@ -356,40 +401,17 @@ impl RasterLayer { } self.opening = None; self.error = None; + self.failed_at = None; // Read in place: its overview, by the reader the app installed; waited for natively (as a // file is read), landing later in the browser. if self.in_place { - let Some(address) = self.path.clone() else { - let e = "the raster has no source".to_string(); - self.error = Some(e.clone()); - self.error_said = false; - return Err(e); - }; - // Checked as a tile layer's address is: a project edited by hand asks for nothing else. - if let Err(e) = crate::tiles::check_url(&address) { + if let Err(e) = self.begin_in_place_read() { self.error = Some(e.clone()); self.error_said = false; return Err(e); } - let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { - let e = format!("{address}: can't be read in place here"); - self.error = Some(e.clone()); - self.error_said = false; - return Err(e); - }; - self.data = None; - let slot = Arc::new(Mutex::new(None)); - let land = slot.clone(); - self.opening = Some(slot); - reader( - &address, - Box::new(move |result| { - *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); - crate::source::landed(); - }), - ); #[cfg(not(target_arch = "wasm32"))] - return self.wait_for_read(std::time::Instant::now() + rustgis_data::raster::IN_PLACE_WAIT); + return self.wait_for_read(Instant::now() + rustgis_data::raster::IN_PLACE_WAIT); #[cfg(target_arch = "wasm32")] return Ok(()); } @@ -452,7 +474,7 @@ impl RasterLayer { /// Waits (natively, until `deadline`) for a read under way, then installs it. #[cfg(not(target_arch = "wasm32"))] - fn wait_for_read(&mut self, deadline: std::time::Instant) -> Result<(), String> { + fn wait_for_read(&mut self, deadline: Instant) -> Result<(), String> { let Some(slot) = self.opening.clone() else { return Ok(()) }; loop { if let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() { @@ -461,14 +483,14 @@ impl RasterLayer { self.land(result); return error.map_or(Ok(()), Err); } - if std::time::Instant::now() >= deadline { + if Instant::now() >= deadline { let address = self.path.clone().unwrap_or_default(); let e = format!("{address}: no answer in two minutes"); self.opening = None; self.land(Err(e.clone())); return Err(e); } - std::thread::sleep(std::time::Duration::from_millis(10)); + std::thread::sleep(web_time::Duration::from_millis(10)); } } @@ -549,10 +571,14 @@ impl RasterLayer { self.style.repair(&raster); } self.data = Some(raster); + self.error = None; + self.failed_at = None; } Err(error) => { + // A read made again that fails as before (see `read_again`) isn't said again. + self.error_said = self.error_said && self.error.as_deref() == Some(error.as_str()); self.error = Some(error); - self.error_said = false; + self.failed_at = Some(Instant::now()); } } } @@ -1040,14 +1066,20 @@ pub struct AddressLook { impl RasterStyle { /// The style `look` gives `r`: its own look (see [`RasterStyle::default_for`]) with the bands /// and stretch the look names (bands it hasn't are left as they are); `None` for a look that - /// names neither. A palette image stretched by a range is drawn in grey, as its tile layer was. + /// names neither. A palette image stretched by a range, or whose address names bands, is + /// drawn in grey, as its tile layer was. pub fn for_look(r: &Raster, look: &AddressLook) -> Option { if look.bands.is_empty() && look.range.is_none() { return None; } - // Bands the file hasn't: its own look (said by `look_lost`). + // Bands the file hasn't: its own look (said by `look_lost`); a palette image in grey, as + // its tile layer draws one whose address names bands (its colours are of one band alone). if Self::look_lost(r, look).is_some() { let mut style = RasterStyle::default_for(r); + if matches!(style.renderer, RasterRenderer::Colormap) { + style.renderer = RasterRenderer::Stretch { band: 0, ramp: Ramp::gray(), invert: false }; + style.stretch = Stretch::default(); + } if let Some((lo, hi)) = look.range { style.stretch = Stretch::custom(lo, hi); } @@ -1074,20 +1106,15 @@ impl RasterStyle { /// when the look fits. pub fn look_lost(r: &Raster, look: &AddressLook) -> Option { let n = r.band_count(); + // A palette image's colours are of one band alone: in grey, as its tile layer drew it. + let own = if matches!(r.color, ColorHint::Palette(_)) { "drawn in grey" } else { "drawn in its own look" }; let past: Vec = look.bands.iter().filter(|b| **b >= n).map(|b| (b + 1).to_string()).collect(); if !past.is_empty() { - return Some(format!( - "its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): drawn in its own look", - past.join(", ") - )); + return Some(format!("its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): {own}", past.join(", "))); } // An address written by hand: `map.setTiles` takes one band or three. - (!matches!(look.bands.len(), 0 | 1 | 3)).then(|| { - format!( - "its address asks for {} bands; one is drawn in grey, three as red, green and blue: drawn in its own look", - look.bands.len() - ) - }) + (!matches!(look.bands.len(), 0 | 1 | 3)) + .then(|| format!("its address asks for {} bands; one is drawn in grey, three as red, green and blue: {own}", look.bands.len())) } /// A style for a layer whose pixels haven't arrived (its own look replaces it when they do). diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index a48ad8cb9..c1abb818c 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -63,7 +63,7 @@ type ImageDecoder = Box, Decoded) + Send + Sync>; /// Bytes the cache keeps before dropping the least recently used entries. const BUDGET: usize = if cfg!(target_arch = "wasm32") { 160 << 20 } else { 384 << 20 }; /// A failed address is tried again after this long. -pub(crate) const RETRY: Duration = Duration::from_secs(60); +pub const RETRY: Duration = Duration::from_secs(60); /// The largest tile or document fetched. pub const MAX_BYTES: usize = 16 << 20; /// The error a fetcher gives a request it dropped unfetched (the view moved on): the address is @@ -188,7 +188,7 @@ pub fn set_alarm(f: impl Fn(Duration) + Send + Sync + 'static) { /// Draws the map again at `when` (the earliest asked for is kept): [`generation`] goes up then, so /// a map waiting on a failed read asks for it again. -pub(crate) fn notify_at(when: Instant) { +pub fn notify_at(when: Instant) { let at = (when.saturating_duration_since(alarm_base()).as_millis() as u64).saturating_add(1); let mut current = ALARM.load(Ordering::Relaxed); loop { @@ -230,7 +230,7 @@ pub fn retry_after(error: &str) -> Option { /// How long a failed read waits before it is tried again: what its server asked for (see /// [`retry_after`]), else `default`. -pub(crate) fn retry_wait(error: &str, default: Duration) -> Duration { +pub fn retry_wait(error: &str, default: Duration) -> Duration { retry_after(error).unwrap_or(default) } diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 2fc4718f5..35671a08b 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -309,23 +309,26 @@ fn tiles() -> &'static Mutex { T.get_or_init(Mutex::default) } -/// Forgets every file and tile, so they are read again (with [`crate::cache::clear`]). +/// Forgets every file and tile (those that failed too), so they are read again (with +/// [`crate::cache::clear`]). pub(crate) fn forget() { infos().lock().unwrap_or_else(|e| e.into_inner()).retain(|_, s| matches!(s, State::Loading(_))); + raster::forget_failed(None); let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); t.map.retain(|_, s| matches!(s, TileState::Loading(_))); t.order.clear(); t.samples = 0; } -/// Forgets what was read of one file (its info and its tiles), but not a read under way: a -/// failed read is made again with whatever login now signs the file's address. +/// Forgets what was read of one file (its info and its tiles, those that failed too), but not a +/// read under way: a failed read is made again with whatever login now signs the file's address. pub(crate) fn forget_file(url: &str) { let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); if all.get(url).is_some_and(|s| !matches!(s, State::Loading(_))) { all.remove(url); } drop(all); + raster::forget_failed(Some(url)); forget_file_in(&mut tiles().lock().unwrap_or_else(|e| e.into_inner()), url); } diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index a89dce4c0..78417b1d7 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -187,7 +187,7 @@ pub struct Window { /// Tiles whose read failed lately, and until when they aren't asked for again ([`RETRY`] after, or /// as long as their server asked to be left), so a map drawn again and again doesn't ask for them /// each time. The map is drawn again when that time comes. -fn failed() -> &'static Mutex> { +pub(super) fn failed() -> &'static Mutex> { static F: OnceLock>> = OnceLock::new(); F.get_or_init(Mutex::default) } @@ -195,6 +195,37 @@ fn failed() -> &'static Mutex> { /// The most failed tiles remembered. const MAX_FAILED: usize = 4096; +/// What a tile's arrival leaves: the map draws again. One that failed is remembered until its +/// wait is over (a minute, or what its server's `Retry-After` said), the map asked to draw again +/// then; one the fetch queue dropped (the view moved on, see [`crate::cache::DROPPED`]) is no +/// failure: not remembered, it is asked for again when it is drawn next, and the tiles that +/// pushed it out draw the map as they land. +fn tile_landed(key: (String, usize, u32), r: &Result) { + match r { + Err(e) if e == crate::cache::DROPPED => return, + Err(e) => { + let until = Instant::now() + crate::cache::retry_wait(e, RETRY); + remember_failed(&mut failed().lock().unwrap_or_else(|e| e.into_inner()), key, until); + crate::cache::notify_at(until); + } + Ok(_) => {} + } + crate::cache::notify(); +} + +/// Forgets the failed tiles of `file` (`None`: of every file), so they are asked for at once: a +/// login or a connection changed (see [`crate::cache::forget`] and [`crate::cache::clear`]). +pub(super) fn forget_failed(file: Option<&str>) { + forget_failed_in(&mut failed().lock().unwrap_or_else(|e| e.into_inner()), file); +} + +fn forget_failed_in(f: &mut HashMap<(String, usize, u32), Instant>, file: Option<&str>) { + match file { + None => f.clear(), + Some(file) => f.retain(|(u, _, _), _| u != file), + } +} + /// Remembers a failed tile until `until`. With [`MAX_FAILED`] already, those whose wait is over go /// first; failing that, the one whose wait ends soonest (asked for again a little early, once) /// makes room, rather than the new one, which would be asked for again at each draw, whatever its @@ -270,20 +301,7 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands } for c in ask { let key = (file.to_string(), level, c); - get_tile( - info.clone(), - file.to_string(), - level, - c, - Box::new(move |r| { - if let Err(e) = r { - let until = Instant::now() + crate::cache::retry_wait(&e, RETRY); - remember_failed(&mut failed().lock().unwrap_or_else(|e| e.into_inner()), key, until); - crate::cache::notify_at(until); - } - crate::cache::notify(); - }), - ); + get_tile(info.clone(), file.to_string(), level, c, Box::new(move |r| tile_landed(key, &r))); // A tile the file leaves out (a sparse file's) is kept at once: not waited for. if let Some(b) = tiles().lock().unwrap_or_else(|e| e.into_inner()).used(&(file.to_string(), level, c)) { kept.insert(c, b); @@ -536,6 +554,35 @@ mod tests { assert!(f.contains_key(&key(1)) && f.len() == MAX_FAILED); } + /// A tile the fetch queue dropped (the view moved on) is no failure: not remembered, so it is + /// asked for again when it is drawn next; one its server refused is, until its wait is over. + #[test] + fn a_dropped_tile_isnt_remembered_as_failed() { + let key = |c: u32| ("test://dropped.tif".to_string(), 0usize, c); + tile_landed(key(1), &Err(crate::cache::DROPPED.to_string())); + assert!(!failed().lock().unwrap().contains_key(&key(1))); + tile_landed(key(2), &Err("HTTP 503".to_string())); + assert!(failed().lock().unwrap().contains_key(&key(2))); + tile_landed(key(3), &Ok(Arc::new(vec![vec![0.0f32]].into()))); + assert!(!failed().lock().unwrap().contains_key(&key(3))); + } + + /// Failed tiles are forgotten with their file (a login or a connection changed: asked for at + /// once), and all of them when the cache is cleared. + #[test] + fn failed_tiles_are_forgotten_with_their_file() { + let until = Instant::now() + std::time::Duration::from_secs(60); + let a = ("test://forgotten-a.tif".to_string(), 0usize, 1u32); + let b = ("test://forgotten-b.tif".to_string(), 0usize, 1u32); + let mut f = HashMap::new(); + remember_failed(&mut f, a.clone(), until); + remember_failed(&mut f, b.clone(), until); + forget_failed_in(&mut f, Some("test://forgotten-a.tif")); + assert!(!f.contains_key(&a) && f.contains_key(&b)); + forget_failed_in(&mut f, None); + assert!(f.is_empty()); + } + /// A level as wide as `u32` allows (a hostile header): its last tile's end doesn't wrap, so /// the window is assembled, its missing tiles gaps. #[test] diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index c2f9c395e..96fec6068 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -3860,6 +3860,8 @@ impl RustGisApp { // Plugin commands that waited for `host.fetch` answers (the browser) run again once // they are in; their errors are notices already. let _ = self.session.poll_plugin_retries(); + // A raster read in place whose overview couldn't be read is read again once its wait is over. + self.session.read_in_place_again(); // Rasters read in place whose pixels landed since (the browser) say what they lost. self.session.notice_landed_rasters(); // A frame may not come until the user moves: look again while commands wait. diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index a7a249633..b973a5420 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -5945,7 +5945,18 @@ fn raster_legend_rows(ui: &mut Ui, entries: &[RasterLegend], indent: f32, t: &To fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens) { let Some(d) = &rl.data else { - ui.label(egui::RichText::new(tl!("The raster's data couldn't be read.")).color(t.red)); + // Read in place, its overview is on its way until it lands, or fails (said why). + match &rl.error { + None if rl.in_place => { + ui.label(egui::RichText::new(tl!("Reading in place…")).color(t.text_dim)); + } + error => { + ui.label(egui::RichText::new(tl!("The raster's data couldn't be read.")).color(t.red)); + if let Some(e) = error { + ui.label(egui::RichText::new(e).size(11.0)); + } + } + } if let Some(p) = &rl.path { ui.label(egui::RichText::new(p).monospace().size(11.0)); } @@ -5988,7 +5999,7 @@ fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens ), ); } - Some((w, h)) => rows.insert(1, ("Full size", format!("{w} × {h}"))), + Some((fw, fh)) => rows.insert(1, ("Full size", format!("{fw} × {fh}"))), None if rl.in_place => rows.insert(1, ("Read", tl!("In place").to_string())), None => {} } diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index f5f938163..2bbaa4c15 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -116,9 +116,9 @@ from the overview it holds: zoomed in past it, the map says so and an export ref Details pane says the image's full size and that it is read in place. A file whose smallest image is too large to hold as the overview (a large image saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` makes the overviews. A tile a sparse file leaves out is empty. A -tile, or the file's header, that fails to arrive is asked for again a minute later, or when its -server says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map -draws again then. +tile, the file's header or the overview a layer holds, that fails to arrive is asked for again a +minute later, or when its server says to in a `Retry-After` header (with a 429 or 503 answer, +within the hour); the map draws again then. Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, and isn't written without them; in the web app, export again once the map has drawn the image. From 3996a53f01b79202a4332e909d6a40615995510c Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Tue, 3 Nov 2026 20:41:55 -0330 Subject: [PATCH 19/27] fix(tiles): review of the merges with main, fourth round A joint read of tiles (a layer's overview) asks for them a batch at a time and asks again for one the fetch queue dropped, so an overview of more tiles than the queue holds (a file without overviews, in small tiles) lands rather than failing. The plain tile cache honours Retry-After: a tile its server refused saying when to come back has the map drawn again then, on the failure and from the gate on each draw while it waits. A test pins main's palette rule for a tableless palette overview under a grey full image. The example writer can leave overviews out. Co-Authored-By: Claude Fable 5.1 --- crates/data/src/raster/cog.rs | 5 +- crates/engine/src/tests.rs | 39 ++++++++++++ crates/tiles/src/cache.rs | 37 ++++++++++- crates/tiles/src/cog.rs | 117 +++++++++++++++++++++++++--------- 4 files changed, 167 insertions(+), 31 deletions(-) diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index 67263e538..ae9769855 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -642,6 +642,8 @@ pub struct Example { /// The full image's first tile left out, as GDAL's sparse files leave out tiles of no-data /// (no bytes, no offset). pub sparse: bool, + /// Without overviews: the full image alone (a file saved without them). + pub no_overviews: bool, } impl Default for Example { @@ -662,6 +664,7 @@ impl Default for Example { stray: false, no_table: false, sparse: false, + no_overviews: false, } } } @@ -674,7 +677,7 @@ pub fn example_with(e: &Example) -> Vec { let planes = if e.planar { nb } else { 1 }; let per_chunk = nb / planes; let mut sizes = vec![size]; - while *sizes.last().unwrap_or(&tile) > tile { + while !e.no_overviews && *sizes.last().unwrap_or(&tile) > tile { sizes.push(sizes.last().unwrap_or(&tile).div_ceil(2)); } let value = |x: u32, y: u32, b: u32| -> u8 { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f56c04266..322bca883 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13975,6 +13975,45 @@ fn a_look_naming_bands_the_file_hasnt_is_said() { assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); } +/// A file without overviews whose only image has more tiles than the fetch queue holds (small +/// tiles) is read in place all the same: its overview is asked for a batch at a time, and a tile +/// the queue dropped is asked for again, so the layer lands rather than failing with "dropped". +#[test] +fn an_overview_of_more_tiles_than_the_fetch_queue_holds_lands() { + use rustgis_data::raster::cog::{Example, example_with}; + let (base, _) = serve_ranges(example_with(&Example { size: 1024, tile: 32, no_overviews: true, ..Example::default() })); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/flat.tif"), "name": "Flat"})).unwrap(); + let rl = s.map().layer_by_name("Flat").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(rl.in_place); + let d = rl.data.as_ref().unwrap(); + assert_eq!((d.width, d.height), (1024, 1024)); + // Every tile, not a stand-in: pixel (x, y) of band 0 is (x + y) % 256. + let at = |x: u32, y: u32| d.bands[0].data[(y * d.width + x) as usize]; + assert_eq!((at(0, 1), at(1023, 1023), at(500, 700)), (1.0, 254.0, 176.0)); +} + +/// A palette image's overview without a colour table under a grey full image is read as the full +/// image is, in grey, and nothing is said: a file is what its full image says (as `main` reads a +/// grey full image with a stray colour table as grey), a window over it is read the same way, and +/// the overview's own say is the one thing the reader doesn't take over the full image's. +#[test] +fn a_palette_overview_under_a_grey_full_image_is_read_grey_as_the_full_image_is() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::RasterRenderer; + // 4096 × 4096 in 256 px tiles: the overview held is the 1024 × 1024 one (a palette image + // without a table), not the full image (grey, 16 M samples). + let file = example_with(&Example { size: 4096, tile: 256, classes: true, stray: true, no_table: true, ..Example::default() }); + let (base, _) = serve_ranges(file); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/stray.tif"), "name": "Stray"})).unwrap(); + let rl = s.map().layer_by_name("Stray").and_then(|l| l.raster.as_ref()).unwrap(); + let d = rl.data.as_ref().unwrap(); + assert_eq!((d.width, d.overview), (1024, Some((4096, 4096))), "the overview held"); + assert!(matches!(rl.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", rl.style.renderer); + assert!(!s.notices.iter().any(|n| n.text.contains("colour table")), "{:?}", s.notices); +} + /// A layer read in place whose overview couldn't be read (its server refused the file's header, /// saying when to ask again) is read again once its wait is over, as the app draws, and lands then: /// not before, and failing was said once. diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index c1abb818c..91d76ff98 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -277,7 +277,15 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { match &slot.entry { Entry::Ready(p, _) => return State::Ready(p.clone()), Entry::Pending => return State::Pending, - Entry::Failed(e, at) if at.elapsed() < retry_wait(e, RETRY) => return State::Failed(e.clone()), + Entry::Failed(e, at) if at.elapsed() < retry_wait(e, RETRY) => { + // A server that said when to come back (429 or 503 with `Retry-After`) is asked + // again then: the map draws again at that time, though nothing else draws it. + // One that didn't is asked again by the next draw after a minute, as before. + if let Some(wait) = retry_after(e) { + notify_at(*at + wait); + } + return State::Failed(e.clone()); + } Entry::Failed(..) => {} } } @@ -566,6 +574,10 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16]) { } Err(e) => { log::debug!("{}: {e}", answer.key); + // Refused until a time its server gave: the map draws again then. + if let Some(wait) = retry_after(&e) { + notify_at(Instant::now() + wait); + } Entry::Failed(e, Instant::now()) } }); @@ -2238,6 +2250,29 @@ mod tests { assert_eq!(retry_wait("http status: 500", RETRY), RETRY); } + /// A tile its server refused saying when to come back (429 or 503 with `Retry-After`) has the + /// map asked to draw again then, from the gate, on each draw while it waits (the alarm is + /// one-shot, and another may have fired since; the failure arms it first). The alarm is the + /// earliest pending one, read here as the map reads it. The entry is stored as a failed fetch + /// stores it: the fetcher is one for the process, and another test's may be in. + #[test] + fn a_tile_refused_with_retry_after_has_the_map_drawn_again_then() { + let within = |secs: u64| { + let at = ALARM.load(Ordering::Relaxed); + at != 0 && at <= (Instant::now() + Duration::from_secs(secs)).saturating_duration_since(alarm_base()).as_millis() as u64 + 1 + }; + let url = "test://cache/busy.json"; + store(url, Entry::Failed("http status: 503 (retry after 2 s)".into(), Instant::now())); + ALARM.store(0, Ordering::Relaxed); + let State::Failed(e) = get(url, Want::Json) else { panic!("not failed") }; + assert!(e.contains("retry after 2 s"), "{e}"); + assert!(within(2), "no alarm on a draw while waiting: {}", ALARM.load(Ordering::Relaxed)); + // Again on the next draw: the alarm fired (or was consumed) before the wait was over. + ALARM.store(0, Ordering::Relaxed); + assert!(matches!(get(url, Want::Json), State::Failed(_))); + assert!(within(2), "not armed again: {}", ALARM.load(Ordering::Relaxed)); + } + /// A read that failed asks the map to draw again when it may be tried: the generation goes up /// once that time has come. #[test] diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 35671a08b..f6f9d3f60 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -839,50 +839,109 @@ fn fetch_bands( type Gathered = Box, String>) + Send>; +/// Tiles a joint read ([`fetch_all`]) has under way at once: half the fetch queue, so a read of +/// many (the overview of a file with few overviews) doesn't push its own requests out of it. +const AT_ONCE: usize = 256; + /// Reads tiles of a level (each once, shared by the map tiles that need it), then calls `then` -/// with their bands in the order asked. +/// with their bands in the order asked. [`AT_ONCE`] are under way at a time, and a tile the fetch +/// queue dropped (other reads filled it: see [`crate::cache::DROPPED`]) is asked for again, so a +/// drop is a delay, not a failure. fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: Gathered) { if wanted.is_empty() { then(Ok(Vec::new())); return; } struct Gather { + info: Arc, + url: String, + level: usize, + wanted: Vec, got: Vec>, - left: usize, + /// Places in `wanted` not asked for yet (or dropped, to ask for again), the next last. + to_ask: Vec, + in_flight: usize, + /// Someone is asking for the next tiles, and asks again after: a tile answered at once + /// leaves it to them (asking from the answer would nest as deep as the tiles kept). + asking: bool, failed: Option, then: Option, } - let g = Arc::new(Mutex::new(Gather { got: vec![None; wanted.len()], left: wanted.len(), failed: None, then: Some(then) })); - for (k, idx) in wanted.into_iter().enumerate() { - let g = g.clone(); - get_tile( - info.clone(), - url.clone(), - level, - idx, - Box::new(move |r| { + let n = wanted.len(); + let g = Arc::new(Mutex::new(Gather { + info, + url, + level, + wanted, + got: vec![None; n], + to_ask: (0..n).rev().collect(), + in_flight: 0, + asking: false, + failed: None, + then: Some(then), + })); + ask(&g); + + /// Asks for the next tiles, [`AT_ONCE`] under way at most, until all are here or one failed + /// (then once the rest have answered), and answers `then` once. + fn ask(g: &Arc>) { + loop { + let (info, url, level, batch) = { let mut s = g.lock().unwrap_or_else(|e| e.into_inner()); - match r { - Ok(b) => { - if let Some(slot) = s.got.get_mut(k) { - *slot = Some(b); - } + if s.asking { + return; + } + if s.in_flight == 0 && (s.to_ask.is_empty() || s.failed.is_some()) { + if let Some(then) = s.then.take() { + let r = match s.failed.take() { + Some(e) => Err(e), + None => Ok(s.got.iter_mut().filter_map(Option::take).collect()), + }; + drop(s); + then(r); } - Err(e) => s.failed = Some(e), + return; } - s.left = s.left.saturating_sub(1); - if s.left == 0 - && let Some(then) = s.then.take() - { - let r = match s.failed.take() { - Some(e) => Err(e), - None => Ok(s.got.iter_mut().filter_map(Option::take).collect()), - }; - drop(s); - then(r); + let mut batch = Vec::new(); + while s.in_flight < AT_ONCE && s.failed.is_none() { + let Some(k) = s.to_ask.pop() else { break }; + let Some(&idx) = s.wanted.get(k) else { continue }; + s.in_flight += 1; + batch.push((k, idx)); + } + if batch.is_empty() { + return; } - }), - ); + s.asking = true; + (s.info.clone(), s.url.clone(), s.level, batch) + }; + for (k, idx) in batch { + let g = g.clone(); + get_tile( + info.clone(), + url.clone(), + level, + idx, + Box::new(move |r| { + { + let mut s = g.lock().unwrap_or_else(|e| e.into_inner()); + s.in_flight = s.in_flight.saturating_sub(1); + match r { + Ok(b) => { + if let Some(slot) = s.got.get_mut(k) { + *slot = Some(b); + } + } + Err(e) if e == crate::cache::DROPPED => s.to_ask.push(k), + Err(e) => s.failed = Some(e), + } + } + ask(&g); + }), + ); + } + g.lock().unwrap_or_else(|e| e.into_inner()).asking = false; + } } } From 5793700df9c013178e78b84a7ca19d1917cf6618 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Tue, 3 Nov 2026 21:21:56 -0330 Subject: [PATCH 20/27] fix(tiles): a window asks for its tiles in batches A window over more tiles than the fetch queue holds (small tiles) asks for at most 256 missing tiles per draw, the rest at the next, rather than all at once, which pushed its own first requests out of the queue to be asked for all again. A tile not asked for yet is pending, as one on its way is. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/tests.rs | 36 +++++++++++++++++++++++++++++++ crates/tiles/src/cog.rs | 2 +- crates/tiles/src/cog/raster.rs | 39 +++++++++++++++++++++------------- 3 files changed, 61 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 322bca883..ba798a5aa 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -13993,6 +13993,42 @@ fn an_overview_of_more_tiles_than_the_fetch_queue_holds_lands() { assert_eq!((at(0, 1), at(1023, 1023), at(500, 700)), (1.0, 254.0, 176.0)); } +/// A window over more tiles than the fetch queue holds (small tiles) asks for them a batch at a +/// time, the next as the map draws again, rather than all at once (which would push its own first +/// requests out of the queue, to ask for them all again at the next draw), and is pending until +/// the last lands. (The tile cache is one for the process: other tests' reads may evict some of +/// these tiles meanwhile, so how many are fetched in all isn't checked.) +#[test] +fn a_window_of_more_tiles_than_the_fetch_queue_holds_asks_for_them_in_batches() { + use crate::render::rustgis_tiles::{cache, cog}; + use rustgis_data::raster::ColorHint; + use rustgis_data::raster::cog::{Example, example_with}; + use std::time::Duration; + let _s = Session::new(Project::default()); + let (base, _) = serve_ranges(example_with(&Example { size: 1024, tile: 32, no_overviews: true, ..Example::default() })); + let url = format!("{base}/flat.tif"); + // The header first; then the window asks for its first batch. + let start = std::time::Instant::now(); + let mut w = cog::raster::window(&url, 0, (0, 1024), (0, 1024), &[0], ColorHint::Gray).unwrap(); + while w.raster.is_none() && start.elapsed() < Duration::from_secs(20) { + cache::wait(Duration::from_millis(20)); + w = cog::raster::window(&url, 0, (0, 1024), (0, 1024), &[0], ColorHint::Gray).unwrap(); + } + assert!(w.raster.is_some() && w.pending && !w.failed); + let asked = cog::raster::asked(&url, 0); + assert!(asked <= 256, "{asked} asked for at once"); + // Drawn again as tiles land, the window asks for the rest, and is whole in the end. + while w.pending && start.elapsed() < Duration::from_secs(60) { + cache::wait(Duration::from_millis(20)); + w = cog::raster::window(&url, 0, (0, 1024), (0, 1024), &[0], ColorHint::Gray).unwrap(); + } + assert!(!w.pending && !w.failed, "still pending after {:?}", start.elapsed()); + let r = w.raster.unwrap(); + let at = |x: u32, y: u32| r.bands[0].data[(y * r.width + x) as usize]; + assert_eq!((at(0, 1), at(1023, 1023), at(500, 700)), (1.0, 254.0, 176.0)); + assert!(r.bands[0].data.iter().all(|v| v.is_finite()), "a gap"); +} + /// A palette image's overview without a colour table under a grey full image is read as the full /// image is, in grey, and nothing is said: a file is what its full image says (as `main` reads a /// grey full image with a stray colour table as grey), a window over it is read the same way, and diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index f6f9d3f60..a4aaab23d 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -841,7 +841,7 @@ type Gathered = Box, String>) + Send>; /// Tiles a joint read ([`fetch_all`]) has under way at once: half the fetch queue, so a read of /// many (the overview of a file with few overviews) doesn't push its own requests out of it. -const AT_ONCE: usize = 256; +pub(super) const AT_ONCE: usize = 256; /// Reads tiles of a level (each once, shared by the map tiles that need it), then calls `then` /// with their bands in the order asked. [`AT_ONCE`] are under way at a time, and a tile the fetch diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 78417b1d7..e14eed05c 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -13,7 +13,7 @@ use std::sync::{Arc, Mutex, OnceLock}; use rustgis_data::raster::{ColorHint, Raster}; use web_time::Instant; -use super::{BUDGET, Bands, Info, RETRY, TileState, fetch_all, get_tile, split, tiles, with_info}; +use super::{AT_ONCE, BUDGET, Bands, Info, RETRY, TileState, fetch_all, get_tile, split, tiles, with_info}; /// The most samples (every band) of the overview a layer holds. const OVERVIEW_SAMPLES: usize = if cfg!(target_arch = "wasm32") { 1 << 20 } else { 2 << 20 }; @@ -213,6 +213,14 @@ fn tile_landed(key: (String, usize, u32), r: &Result) { crate::cache::notify(); } +/// How many tiles of level `level` of a file have been asked for and not forgotten (kept, or on +/// their way): for tests of how a window asks for them. +#[doc(hidden)] +pub fn asked(file: &str, level: usize) -> usize { + let t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + t.map.iter().filter(|((u, l, _), s)| u == file && *l == level && matches!(s, TileState::Ready(..) | TileState::Loading(_))).count() +} + /// Forgets the failed tiles of `file` (`None`: of every file), so they are asked for at once: a /// login or a connection changed (see [`crate::cache::forget`] and [`crate::cache::clear`]). pub(super) fn forget_failed(file: Option<&str>) { @@ -245,8 +253,9 @@ fn remember_failed(f: &mut HashMap<(String, usize, u32), Instant>, key: (String, /// The columns `cols` and rows `rows` (half-open, in the level's pixels) of level `level` of a /// file, bands `bands` (0-based, in that order, as the raster's bands), coloured as `color` says: -/// from the tiles kept, without waiting. Those missing are fetched (each once at a time); the -/// tile cache's generation goes up as each lands, so the map is drawn again. +/// from the tiles kept, without waiting. Those missing are fetched (each once at a time, and +/// [`AT_ONCE`] of them at a time: the rest as the map draws again); the tile cache's generation +/// goes up as each lands, so the map is drawn again. pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize], color: ColorHint) -> Result { let file = split(url).0; let Some(info) = super::info(file)? else { return Ok(Window { raster: None, pending: true, failed: false }) }; @@ -299,7 +308,11 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands } waiting = next.is_some(); } - for c in ask { + // Asked for `AT_ONCE` at a time (half the fetch queue, as a joint read asks): a view of small + // tiles needing more would push its own first requests out of the queue, to ask for them all + // again at the next draw. The rest are asked for as the map draws again (a tile landing draws + // it), and are on their way till then. + for c in ask.into_iter().take(AT_ONCE) { let key = (file.to_string(), level, c); get_tile(info.clone(), file.to_string(), level, c, Box::new(move |r| tile_landed(key, &r))); // A tile the file leaves out (a sparse file's) is kept at once: not waited for. @@ -307,19 +320,15 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands kept.insert(c, b); } } - // The tiles still missing are on their way (the map draws again as they land) or failed lately - // (not waited for: the map would wait for them for nothing; asked for again when their wait is - // over). + // The tiles still missing are on their way (asked for, or to be asked for at the next draw; + // the map draws again as they land) or failed lately (not waited for: the map would wait for + // them for nothing; asked for again when their wait is over). let missing: Vec = chunks.iter().copied().filter(|c| !kept.contains_key(c)).collect(); - let pending = !missing.is_empty() && { - let t = tiles().lock().unwrap_or_else(|e| e.into_inner()); - missing.iter().any(|c| matches!(t.map.get(&(file.to_string(), level, *c)), Some(TileState::Loading(_)))) + let (pending, waiting) = { + let f = failed().lock().unwrap_or_else(|e| e.into_inner()); + let failed_now = |c: &u32| f.contains_key(&(file.to_string(), level, *c)); + (missing.iter().any(|c| !failed_now(c)), waiting || missing.iter().any(failed_now)) }; - let waiting = waiting - || !missing.is_empty() && { - let f = failed().lock().unwrap_or_else(|e| e.into_inner()); - missing.iter().any(|c| f.contains_key(&(file.to_string(), level, *c))) - }; let (mut data, gaps) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); // Until they arrive, its missing tiles show the levels kept (the one zoomed from), not holes. if !gaps.is_empty() { From 767f99a4083d6be171ff4ee93d8ee6fedcd95601 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Wed, 4 Nov 2026 13:59:54 -0330 Subject: [PATCH 21/27] fix(engine): the desktop doesn't wait for a read in place, and a failing one waits longer each time Review findings on the merge with main: - `Session::in_place_wait` (true by default: the command line, Python and tests wait, so a result is complete) is false in the desktop binary, whose commands run on the window's own thread: `layer.addUrl` adds a GeoTIFF read in place at once and its overview lands as the map draws (`RasterLayer::opening_in_place` begins the read without waiting, `poll_reload` installs it); a project, an import and a layer file begin their reads together as before and `Session::wait_in_place_reads` waits only where the session waits. - Binding an S3 connection reads its in-place rasters together (`reopen_where` calls `read_together` once), not one after another, each waiting. - An overview that fails again waits twice as long each time, within the hour (`cache::backoff`), so a file that never reads isn't asked for every minute all session. - With no memory for the image that blends the part in view with its overview, the overview is drawn alone and the draw says so, rather than nothing and Ok. Tests: a_cog_added_without_waiting_lands_as_the_map_draws, binding_a_connection_reads_its_in_place_rasters_together (a stand-in bucket that answers only once both files are asked for), a_read_that_fails_again_waits_longer_each_time, and the overview retry test now refuses twice. Co-Authored-By: Claude Fable 5.1 --- apps/rustgis/src/main.rs | 5 +- crates/engine/src/cmd/geolibre.rs | 1 + crates/engine/src/cmd/items.rs | 1 + crates/engine/src/cmd/layer.rs | 19 ++- crates/engine/src/cmd/project.rs | 3 + crates/engine/src/lib.rs | 46 ++++++- crates/engine/src/project.rs | 11 +- crates/engine/src/s3_connections.rs | 13 +- crates/engine/src/tests.rs | 201 ++++++++++++++++++++++++++-- crates/map/src/raster.rs | 120 ++++++++++------- crates/render/src/raster.rs | 9 ++ crates/tiles/src/cache.rs | 26 ++++ docs/user-guide/data-formats.md | 15 ++- 13 files changed, 390 insertions(+), 80 deletions(-) diff --git a/apps/rustgis/src/main.rs b/apps/rustgis/src/main.rs index 5460bf488..afab2c482 100644 --- a/apps/rustgis/src/main.rs +++ b/apps/rustgis/src/main.rs @@ -448,11 +448,14 @@ fn main() -> eframe::Result { .collect(); // Installed plugins, before the first session lists commands and tools. let plugin_errors = rustgis_engine::cmd::plugins::load_installed(); - let app = if args.iter().any(|a| a == "--sample") || files.is_empty() { + let mut app = if args.iter().any(|a| a == "--sample") || files.is_empty() { RustGisApp::with_sample() } else { RustGisApp::new(Session::new(Project::default())) }; + // Commands run on the window's own thread: a raster read in place lands as the map draws + // rather than holding the window for a slow host. + app.session.in_place_wait = false; let mut services = prefs::services(); // Autosave, in a recovery folder only the copy holding the run marker uses: a second copy // running alongside would offer, or overwrite, the first one's unsaved work. diff --git a/crates/engine/src/cmd/geolibre.rs b/crates/engine/src/cmd/geolibre.rs index d03a1557f..322e25269 100644 --- a/crates/engine/src/cmd/geolibre.rs +++ b/crates/engine/src/cmd/geolibre.rs @@ -159,6 +159,7 @@ fn import_project(s: &mut Session, p: &Value) -> CmdResult { } // Its GeoTIFFs read in place, read together rather than one after another. crate::project::read_in_place(s.map_mut()); + s.wait_in_place_reads(); order.reverse(); s.map_mut().layers.sort_by_key(|l| order.iter().position(|id| *id == l.id).unwrap_or(usize::MAX)); s.request_only = false; diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 4bbd79257..c4c6730c8 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -464,6 +464,7 @@ fn import_maps(s: &mut Session, p: &Value) -> CmdResult { super::connections::restore_active(s); super::connections::resolve_project(s); crate::s3_connections::resolve_project_s3(s); + s.wait_in_place_reads(); super::connections::note_waiting(s); if let Some(n) = names.last() && let Some(i) = s.project.maps.iter().position(|m| &m.name == n) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 99ea598cc..e81b7d7d7 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -2660,6 +2660,7 @@ pub(crate) fn add_layer_text(s: &mut Session, text: &str) -> CmdResult { super::connections::clear_active(); s.project.normalize(); super::connections::restore_active(s); + s.wait_in_place_reads(); // Only the database the added layer reads can change here. let keys: Vec<_> = s.map().layer(id).and_then(|l| l.source.as_ref()).map(super::connections::layer_keys).unwrap_or_default(); super::connections::resolve_keys(s, &keys); @@ -3434,9 +3435,11 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool) Ok(r) } -/// Adds a cloud-optimised GeoTIFF as a raster layer read in place: its overview held (read now, -/// natively; in the browser as it lands), the part in view read as the map draws. A look at the -/// address's end (`#bands=…&range=…`) is its style. `zoom` is the adding command's `"zoom"`: +/// Adds a cloud-optimised GeoTIFF as a raster layer read in place: its overview held (read now +/// where the session waits for reads in place, failing the command when it can't be; landing as +/// the map draws on the desktop and in the browser, see [`Session::in_place_wait`]), the part in +/// view read as the map draws. A look at the address's end (`#bands=…&range=…`) is its style. +/// `zoom` is the adding command's `"zoom"`: /// `Some(true)` frames it, `Some(false)` keeps the view, `None` frames it only as the first content /// on an empty map. fn add_in_place( @@ -3451,11 +3454,15 @@ fn add_in_place( // Judged as an address a project keeps is (a host, no spaces, not too long) before it is read. rustgis_map::tiles::check_url(file)?; let empty_before = super::map_is_empty(s); - // Strict, natively: read now, failing when it can't be. Otherwise read with the other layers - // added with it (see `project::read_in_place`); in the browser, landing later. + // Strict, where the session waits: read now, failing when it can't be. Strict on the desktop: + // added now, its overview landing as the map draws (and failing there, said), so a slow host + // doesn't hold the window. Otherwise read with the other layers added with it (see + // `project::read_in_place`); in the browser, landing later. #[cfg(not(target_arch = "wasm32"))] - let layer = if strict { + let layer = if strict && s.in_place_wait { rustgis_map::raster::RasterLayer::in_place(cog::raster::overview_blocking(file)?, file) + } else if strict { + rustgis_map::raster::RasterLayer::opening_in_place(file) } else { rustgis_map::raster::RasterLayer::unread_in_place(file) }; diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index 686936525..13ee6f08c 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -171,6 +171,9 @@ fn reset(s: &mut Session, p: Project) { // nothing of the new project is fetched before. s.s3_pending.clear(); crate::s3_connections::resolve_project_s3(s); + // Its rasters read in place, begun together as it was read: waited for where the session + // waits (the command line, Python), landing as the map draws on the desktop. + s.wait_in_place_reads(); // What reading a raster left out (a colour table, names, a sidecar's scale) is said again as // the project opens, as when it was added. One still read in the background isn't here yet. let mut lost: Vec<(String, String)> = Vec::new(); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 33596a34a..062607e13 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -593,6 +593,12 @@ pub struct Session { /// (`map.addTiles` with `"view": true`), or a PMTiles/MBTiles archive whose bounds it goes to, and the /// camera and owning map instance then: moved or switched since, the map stays. pub style_view: Option<(rustgis_map::TileLayer, Camera, u64)>, + /// Whether a command waits for the overview of a raster read in place (natively, at most two + /// minutes; the browser never waits): the command line, Python and tests do, so a result is + /// complete; the desktop app doesn't, as its commands run on the window's own thread, and the + /// layer lands as the map draws (failing is said then, and read again: see + /// [`Session::read_in_place_again`]). + pub in_place_wait: bool, pub notices: Vec, /// Troubleshooting history, independent of the active pane, project files and undo. pub diagnostics: rustgis_data::diagnostics::History, @@ -919,6 +925,7 @@ impl Session { layer_selection_of: None, view_size: (1200.0, 720.0), style_view: None, + in_place_wait: true, notices: Vec::new(), diagnostics: rustgis_data::diagnostics::History::default(), notices_given: 0, @@ -1654,16 +1661,45 @@ impl Session { /// Reads again the overview of a raster layer read in place whose read failed, once its wait /// is over: a minute, or what its server's `Retry-After` said, as a tile or a file's header is - /// read again (see `rustgis_tiles::cache::retry_wait`); until then the map is asked to draw - /// again when the wait is over. The app calls it each frame, with - /// [`Session::notice_landed_rasters`]; the pixels land as the map draws. + /// read again (see `rustgis_tiles::cache::retry_wait`), doubled each time it fails again, + /// within the hour (`cache::backoff`: a file that never reads isn't asked for every minute all + /// session); until then the map is asked to draw again when the wait is over. The app calls it + /// each frame, with [`Session::notice_landed_rasters`]; the pixels land as the map draws. + /// Waits for the overviews of the rasters read in place that a project, an import or a layer + /// file began reading together (see [`crate::project::read_in_place`]), where the session + /// waits for them ([`Session::in_place_wait`]: natively, two minutes at most for all); where + /// it doesn't, they land as the map draws. Failing is the layer's to say (see + /// [`Session::notice_landed_rasters`]). + pub(crate) fn wait_in_place_reads(&mut self) { + if !self.in_place_wait { + return; + } + for m in &mut self.project.maps { + let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m + .layers + .iter_mut() + .filter_map(|l| { + let name = l.name.clone(); + let rl = l.raster.as_mut()?; + rl.in_place.then_some((name, rl)) + }) + .collect(); + let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); + for (i, e) in rustgis_map::raster::wait_for_reads(&mut layers) { + if let Some((name, _)) = named.get(i) { + log::warn!("{name}: {e}"); + } + } + } + } + pub fn read_in_place_again(&mut self) { use render::rustgis_tiles::cache; for m in &mut self.project.maps { for l in &mut m.layers { let Some(rl) = &mut l.raster else { continue }; - let Some((e, at)) = rl.read_failure() else { continue }; - let wait = cache::retry_wait(e, cache::RETRY); + let Some((e, at, failures)) = rl.read_failure() else { continue }; + let wait = cache::backoff(cache::retry_wait(e, cache::RETRY), failures); if at.elapsed() < wait { cache::notify_at(at + wait); } else { diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 65bd730eb..669ece645 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -929,10 +929,11 @@ impl Project { } } -/// Reads the overviews of a map's raster layers read in place and not read yet, all at once (see -/// [`rustgis_map::raster::read_together`]): a project or an import of several waits for the -/// slowest, not for each in turn. One read from an S3 bucket waits for its connection (see -/// [`crate::s3_connections::resolve_project_s3`]). +/// Starts reading the overviews of a map's raster layers read in place and not read yet, all at +/// once (see [`rustgis_map::raster::read_together`]): a project or an import of several waits for +/// the slowest, not for each in turn, where the session waits for them +/// ([`crate::Session::wait_in_place_reads`]); elsewhere they land as the map draws. One read from +/// an S3 bucket waits for its connection (see [`crate::s3_connections::resolve_project_s3`]). pub(crate) fn read_in_place(m: &mut Map) { let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m .layers @@ -944,7 +945,7 @@ pub(crate) fn read_in_place(m: &mut Map) { }) .collect(); let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); - let failed = rustgis_map::raster::read_together(&mut layers); + let failed = rustgis_map::raster::read_together(&mut layers, false); for (i, e) in failed { if let Some((name, _)) = named.get(i) { log::warn!("{name}: {e}"); diff --git a/crates/engine/src/s3_connections.rs b/crates/engine/src/s3_connections.rs index ab1063ba9..2a7917db7 100644 --- a/crates/engine/src/s3_connections.rs +++ b/crates/engine/src/s3_connections.rs @@ -284,9 +284,12 @@ fn block(s: &mut Session, g: &Group) -> usize { } /// Opens (again) the streamed sources whose address `which` accepts; how many opened and how -/// many failed. +/// many failed. Rasters read in place read their overviews together (see +/// [`rustgis_map::raster::read_together`]): a bucket of several waits for the slowest, not for +/// each in turn, and on the desktop for none (they land as the map draws). fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) { let (mut opened, mut failed) = (0, 0); + let wait = s.in_place_wait; for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { let Some(path) = src.path.clone() else { continue }; if (!src.is_open() || src.error.is_some()) && which(&path) { @@ -296,14 +299,22 @@ fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) } } } + let mut in_place = Vec::new(); for raster in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.raster.as_mut()) { if raster.data.is_none() && raster.time_zarr.is_none() && raster.path.as_deref().is_some_and(&which) { + if raster.in_place { + in_place.push(raster); + continue; + } match raster.reload() { Ok(()) => opened += 1, Err(_) => failed += 1, } } } + let unread = rustgis_map::raster::read_together(&mut in_place, wait).len(); + failed += unread; + opened += in_place.len().saturating_sub(unread); (opened, failed) } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index c26d46d2d..24c0d8ab4 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -14183,7 +14183,13 @@ fn serve_file(body: Vec, ranges: bool) -> (String, std::sync::Arc, n: usize) -> (String, std::sync::Arc) { - serve_with(body, true, None, n) + serve_with(body, true, None, n, 0) +} + +/// [`serve_ranges`], answering the first `n - 1` requests (a file's header) and refusing the rest +/// with a 500: a file whose header reads but whose tiles can't be. +fn serve_refusing_from(body: Vec, n: usize) -> (String, std::sync::Arc) { + serve_with(body, true, None, 0, n) } /// [`serve_file`] holding every answer until `gate` is set (a request still under way, for as long @@ -14193,18 +14199,21 @@ fn serve_gated( ranges: bool, gate: Option>, ) -> (String, std::sync::Arc) { - serve_with(body, ranges, gate, 0) + serve_with(body, ranges, gate, 0, 0) } -/// [`serve_gated`], refusing the first `refuse` requests (see [`serve_refusing_first`]). +/// [`serve_gated`], refusing the first `refuse` requests (see [`serve_refusing_first`]), and every +/// request from the `refuse_from`th on (see [`serve_refusing_from`]; 0 for none). fn serve_with( body: Vec, ranges: bool, gate: Option>, refuse: usize, + refuse_from: usize, ) -> (String, std::sync::Arc) { use std::io::{Read, Write}; let refusals = std::sync::Arc::new(std::sync::Mutex::new(refuse)); + let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let sent = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); @@ -14216,10 +14225,16 @@ fn serve_with( // (the fetch workers are shared by every test in the process). for stream in listener.incoming() { let Ok(mut s) = stream else { continue }; - let (body, count, gate, refusals) = (body.clone(), count.clone(), gate.clone(), refusals.clone()); + let (body, count, gate, refusals, seen) = (body.clone(), count.clone(), gate.clone(), refusals.clone(), seen.clone()); std::thread::spawn(move || { let mut buf = [0u8; 4096]; let read = s.read(&mut buf).unwrap_or(0); + // Refused from the `refuse_from`th request on: a file whose tiles can't be read. + let nth = seen.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + if refuse_from > 0 && nth >= refuse_from { + let _ = s.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + } // Refused while there are refusals left: busy, ask again in a second. let refused = { let mut left = refusals.lock().unwrap(); @@ -15173,7 +15188,7 @@ fn an_address_look_holds_when_the_pixels_land_later() { let look = AddressLook { bands: vec![4, 2, 1], range: Some((10.0, 200.0)) }; let mut rl = RasterLayer::unread_in_place(format!("{base}/scene.tif")).with_look(look); assert!(rl.data.is_none()); - assert!(read_together(&mut [&mut rl]).is_empty()); + assert!(read_together(&mut [&mut rl], true).is_empty()); assert!(rl.data.is_some()); assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); @@ -15181,7 +15196,7 @@ fn an_address_look_holds_when_the_pixels_land_later() { let mut stretched = RasterLayer::unread_in_place(format!("{classes}/classes.tif")).with_look(AddressLook { bands: vec![], range: Some((0.0, 100.0)) }); let mut own = RasterLayer::unread_in_place(format!("{classes}/classes.tif")); - assert!(read_together(&mut [&mut stretched, &mut own]).is_empty()); + assert!(read_together(&mut [&mut stretched, &mut own], true).is_empty()); assert!(matches!(stretched.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", stretched.style.renderer); assert_eq!(own.style.renderer, RasterRenderer::Colormap); // Saved before its pixels landed (the browser, a read that failed), a layer opens in its look, @@ -15391,7 +15406,7 @@ fn an_in_place_layers_failed_overview_is_read_again_when_its_wait_is_over() { use rustgis_map::raster::RasterLayer; use std::sync::atomic::Ordering; use std::time::Duration; - let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 1); + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 2); let mut s = Session::new(Project::default()); s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place(format!("{base}/scene.tif"))); let text = serde_json::to_string(&s.project).unwrap(); @@ -15401,28 +15416,192 @@ fn an_in_place_layers_failed_overview_is_read_again_when_its_wait_is_over() { let rl = layer(&s); assert!(rl.data.is_none()); assert!(rl.error.as_deref().is_some_and(|e| e.contains("retry after 1 s")), "{:?}", rl.error); + assert_eq!(rl.read_failure().map(|(_, _, failures)| failures), Some(1)); let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); assert_eq!(said(&s), 1, "{:?}", s.notices); // Not before its wait is over. s.read_in_place_again(); assert!(layer(&s).data.is_none()); assert_eq!(sent.load(Ordering::Relaxed), 0, "read again before its wait was over"); - // Then read again, landing as the map draws. + // Then read again, as the map draws: refused again, it waits twice as long this time (see + // `cache::backoff`), and isn't said again. let view = rustgis_geom::Extent::new(10.0, 45.9, 10.1, 46.0); - let start = std::time::Instant::now(); - while layer(&s).data.is_none() && start.elapsed() < Duration::from_secs(20) { + let draw = |s: &mut Session| { crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); s.read_in_place_again(); s.map_mut().sync_sources(&view, 1000.0, false); s.notice_landed_rasters(); + }; + let start = std::time::Instant::now(); + while layer(&s).read_failure().is_none_or(|(_, _, failures)| failures < 2) && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + let rl = layer(&s); + let (_, at, failures) = rl.read_failure().expect("refused again"); + assert_eq!(failures, 2); + assert_eq!(sent.load(Ordering::Relaxed), 0); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // Not before twice its wait is over (looked at while that is still ahead). + if at.elapsed() < Duration::from_millis(1800) { + s.read_in_place_again(); + assert!(layer(&s).read_failure().is_some(), "read again before twice its wait was over"); + } + // Then read a third time, landing as the map draws. + let start = std::time::Instant::now(); + while layer(&s).data.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); } let rl = layer(&s); assert!(rl.data.is_some(), "{:?}", rl.error); - assert!(rl.error.is_none()); + assert!(rl.error.is_none() && rl.read_failure().is_none()); assert!(sent.load(Ordering::Relaxed) > 0); assert_eq!(said(&s), 1, "{:?}", s.notices); } +/// Where the session doesn't wait for reads in place (the desktop app, whose commands run on the +/// window's own thread), a cloud-optimised GeoTIFF is added at once, its header probed as the app +/// does, and its overview lands as the map draws; one that can't be read is added too, and says so +/// once its read fails. A project opens the same way. Waiting, the command fails instead. +#[test] +fn a_cog_added_without_waiting_lands_as_the_map_draws() { + use std::time::Duration; + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + s.in_place_wait = false; + let _ = crate::render::rustgis_tiles::cog::probe_blocking(&url); + let r = s.run("layer.addUrl", &json!({"url": url, "name": "Scene"})).unwrap(); + assert_eq!((r["inPlace"].as_bool(), r["kind"].as_str()), (Some(true), Some("raster")), "{r}"); + assert_eq!(r["bounds"], json!([10.0, 45.0, 11.0, 46.0]), "framed on the header read: {r}"); + let layer = |s: &Session, name: &str| s.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); + let rl = layer(&s, "Scene"); + assert!(rl.in_place && rl.data.is_none() && rl.error.is_none(), "added before its overview is in: {rl:?}"); + let view = rustgis_geom::Extent::new(10.0, 45.0, 11.0, 46.0); + let draw = |s: &mut Session| { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + s.read_in_place_again(); + s.map_mut().sync_sources(&view, 1000.0, false); + s.notice_landed_rasters(); + }; + let start = std::time::Instant::now(); + while layer(&s, "Scene").data.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + let rl = layer(&s, "Scene"); + assert_eq!(rl.data.as_ref().map(|d| (d.width, d.height)), Some((512, 512)), "{:?}", rl.error); + // One whose header reads (the first request) but whose tiles can't be is added, and says so + // once its read fails. + let (broken, _) = serve_refusing_from(rustgis_data::raster::cog::example(512, 256), 2); + let broken = format!("{broken}/broken.tif"); + s.run("layer.addUrl", &json!({"url": broken, "name": "Broken"})).unwrap(); + let start = std::time::Instant::now(); + while layer(&s, "Broken").error.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + assert!(layer(&s, "Broken").error.as_deref().is_some_and(|e| e.contains("500")), "{:?}", layer(&s, "Broken").error); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Broken: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // Waiting, the same address fails the command. + s.in_place_wait = true; + assert!(s.run("layer.addUrl", &json!({"url": broken, "name": "Again"})).unwrap_err().contains("500")); + // A project opens the same way: its layers read in place land as the map draws. + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.in_place_wait = false; + t.run("project.openText", &json!({"text": text})).unwrap(); + assert!(layer(&t, "Scene").data.is_none(), "opened before its overview is in"); + let start = std::time::Instant::now(); + while layer(&t, "Scene").data.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut t); + } + assert!(layer(&t, "Scene").data.is_some(), "{:?}", layer(&t, "Scene").error); +} + +/// Binding a saved S3 connection to a project's rasters read in place reads their overviews +/// together, in the time of the slowest (see `s3_connections::reopen_where`), not one after the +/// other: the stand-in bucket answers nothing until both files have been asked for, so reads made +/// one at a time would never get their first answer. +#[test] +fn binding_a_connection_reads_its_in_place_rasters_together() { + use rustgis_map::raster::RasterLayer; + use std::time::Duration; + let base = serve_s3_together(vec![ + ("a.tif", rustgis_data::raster::cog::example(512, 256)), + ("b.tif", rustgis_data::raster::cog::example(256, 512)), + ]); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("A".to_string(), RasterLayer::unread_in_place("s3://s3-together-bucket/a.tif")); + s.map_mut().add_raster("B".to_string(), RasterLayer::unread_in_place("s3://s3-together-bucket/b.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run( + "connection.new", + &json!({"kind": "s3", "name": "Together", "address": base, "options": {"credentials": "none", "buckets": "s3-together-bucket"}}), + ) + .unwrap(); + let start = std::time::Instant::now(); + t.run("project.openText", &json!({"text": text})).unwrap(); + for (name, width) in [("A", 512), ("B", 256)] { + let rl = t.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); + assert_eq!(rl.data.as_ref().map(|d| d.width), Some(width), "{name}: {:?}", rl.error); + } + assert!(start.elapsed() < Duration::from_secs(60), "read one after the other: {:?}", start.elapsed()); + t.run("connection.remove", &json!({"id": "Together"})).unwrap(); +} + +/// A stand-in S3 endpoint serving `files` (key, bytes) by range, answering nothing until each key +/// has been asked for once: reads made together get their answers, reads made one after the other +/// never get the first. +fn serve_s3_together(files: Vec<(&'static str, Vec)>) -> String { + use std::io::{Read, Write}; + use std::time::Duration; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let files = std::sync::Arc::new(files); + let asked: std::sync::Arc>> = Default::default(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut s) = stream else { continue }; + let (files, asked) = (files.clone(), asked.clone()); + std::thread::spawn(move || { + let mut buf = [0u8; 8192]; + let read = s.read(&mut buf).unwrap_or(0); + let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); + let path = req.split_whitespace().nth(1).unwrap_or("/").split(['?', '#']).next().unwrap_or("/").to_string(); + let Some((key, body)) = files.iter().find(|(k, _)| path.ends_with(&format!("/{k}"))) else { + let _ = s.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + }; + asked.lock().unwrap().insert(key); + let start = std::time::Instant::now(); + while asked.lock().unwrap().len() < files.len() && start.elapsed() < Duration::from_secs(150) { + std::thread::sleep(Duration::from_millis(5)); + } + if req.starts_with("head ") { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + return; + } + let range = req.lines().find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + if a.is_empty() { + let n = b.parse::().ok()?.min(body.len()); + return Some((body.len() - n, body.len().saturating_sub(1))); + } + Some((a.parse::().ok()?, b.parse::().ok()?)) + }); + let (status, part) = match range { + Some((a, b)) if a < body.len() => ("206 Partial Content", &body[a..=b.min(body.len() - 1)]), + Some(_) => ("416 Range Not Satisfiable", &body[..0]), + None => ("200 OK", &body[..]), + }; + let _ = write!(s, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); + let _ = s.write_all(part); + }); + } + }); + base +} + /// A COG tile layer opened as a raster layer takes a numbered name when another layer has its /// name, as a layer added does. #[test] diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index c4299a390..e4aea702f 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -79,6 +79,10 @@ pub struct RasterLayer { /// over (see [`RasterLayer::read_failure`]). #[serde(skip)] failed_at: Option, + /// How many reads of its overview in place have failed in a row: each waits longer than the + /// last (see [`RasterLayer::read_failure`]). + #[serde(skip)] + failures: u32, /// What reading its pixels left out (`Raster::lost`) has been said (see /// [`RasterLayer::lost_unsaid`]). #[serde(skip)] @@ -97,48 +101,50 @@ pub struct RasterLayer { pub type InPlaceRead = Box) + Send>; /// Reads the overviews of `layers` read in place and not read yet, all at once: a project of -/// several opens in the time of the slowest, not of them all (waited for natively, at most two -/// minutes in all; landing later in the browser). Other layers are left as they are. Returns each -/// failure, by its place in `layers`. -pub fn read_together(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { +/// several opens in the time of the slowest, not of them all. Waited for natively when `wait` +/// (at most two minutes in all); otherwise, as always in the browser, they land as the map draws +/// (see [`wait_for_reads`]). Other layers are left as they are. Returns each failure, by its +/// place in `layers`: without waiting, the reads that couldn't start. +pub fn read_together(layers: &mut [&mut RasterLayer], wait: bool) -> Vec<(usize, String)> { let mut failed = Vec::new(); let unread = |l: &RasterLayer| l.in_place && l.data.is_none() && l.opening.is_none() && l.time_zarr.is_none(); - #[cfg(not(target_arch = "wasm32"))] - let deadline = Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; - #[cfg(not(target_arch = "wasm32"))] - let mut started = Vec::new(); for (i, l) in layers.iter_mut().enumerate() { - if !unread(l) { - continue; - } - // Started without waiting: natively, each is waited for below, together. - #[cfg(not(target_arch = "wasm32"))] + // Started without waiting: waited for below, together. + if unread(l) + && let Err(e) = l.begin_reload() { - l.error = None; - l.failed_at = None; - match l.begin_in_place_read() { - Ok(()) => started.push(i), - Err(e) => { - l.error = Some(e.clone()); - l.error_said = false; - failed.push((i, e)); - } - } - } - #[cfg(target_arch = "wasm32")] - if let Err(e) = l.reload() { failed.push((i, e)); } } + if wait { + failed.extend(wait_for_reads(layers)); + } + failed +} + +/// Waits (natively; at most two minutes for all) for the overviews of `layers` being read in +/// place (see [`read_together`]), installing them. Returns each failure, by its place in +/// `layers`. In the browser, where nothing may wait, they land as the map draws. +pub fn wait_for_reads(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { + #[cfg(target_arch = "wasm32")] + { + let _ = layers; + Vec::new() + } #[cfg(not(target_arch = "wasm32"))] - for i in started { - if let Some(l) = layers.get_mut(i) - && let Err(e) = l.wait_for_read(deadline) - { - failed.push((i, e)); + { + let deadline = Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; + let mut failed = Vec::new(); + for (i, l) in layers.iter_mut().enumerate() { + if l.in_place + && l.opening.is_some() + && let Err(e) = l.wait_for_read(deadline) + { + failed.push((i, e)); + } } + failed } - failed } type InPlaceReader = Arc; @@ -216,6 +222,7 @@ impl RasterLayer { pixels_said: false, error_said: false, failed_at: None, + failures: 0, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -244,6 +251,7 @@ impl RasterLayer { pixels_said: false, error_said: false, failed_at: None, + failures: 0, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -272,6 +280,7 @@ impl RasterLayer { pixels_said: false, error_said: false, failed_at: None, + failures: 0, lost_said: false, look_lost: None, dropped: Vec::new(), @@ -305,17 +314,19 @@ impl RasterLayer { pixels_said: false, error_said: false, failed_at: None, + failures: 0, lost_said: false, look_lost: None, dropped: Vec::new(), } } - /// [`RasterLayer::unread_in_place`], its overview read now (waited for natively; landing - /// later in the browser, where nothing may wait). + /// [`RasterLayer::unread_in_place`], its overview being read: it lands as the map draws + /// ([`RasterLayer::poll_reload`]), in the browser and on the desktop alike, and failing is + /// said then (see [`RasterLayer::error_unsaid`]). pub fn opening_in_place(address: impl Into) -> RasterLayer { let mut layer = RasterLayer::unread_in_place(address); - let _ = layer.reload(); + let _ = layer.begin_reload(); layer } @@ -369,15 +380,16 @@ impl RasterLayer { Ok(()) } - /// Why reading its overview in place failed, and when: for the read to be made again once its - /// wait is over (see [`RasterLayer::read_again`]), as a tile or a file's header is. `None` for - /// a layer with its pixels, one being read, and one that waits for something other than time - /// (its bucket's connection: see [`RasterLayer::fail`]). - pub fn read_failure(&self) -> Option<(&str, Instant)> { + /// Why reading its overview in place failed, when, and how many reads in a row have failed: + /// for the read to be made again once its wait is over (see [`RasterLayer::read_again`]), as a + /// tile or a file's header is, waiting longer each time. `None` for a layer with its pixels, + /// one being read, and one that waits for something other than time (its bucket's connection: + /// see [`RasterLayer::fail`]). + pub fn read_failure(&self) -> Option<(&str, Instant, u32)> { if !self.in_place || self.data.is_some() || self.opening.is_some() { return None; } - Some((self.error.as_deref()?, self.failed_at?)) + Some((self.error.as_deref()?, self.failed_at?, self.failures)) } /// Reads its overview again after a read that failed (see [`RasterLayer::read_failure`]), @@ -394,16 +406,29 @@ impl RasterLayer { } /// Reads the pixels again from the path or the embedded file (after a project opens). A - /// layer that follows the time slider is left to the engine, which reads the step shown. + /// layer that follows the time slider is left to the engine, which reads the step shown. A + /// layer read in place is waited for natively (at most two minutes), as a file is read. pub fn reload(&mut self) -> Result<(), String> { + self.reload_in(true) + } + + /// [`RasterLayer::reload`] without waiting for a layer read in place: its overview lands as + /// the map draws ([`RasterLayer::poll_reload`]), natively as in the browser, so a command on + /// the desktop's own thread isn't held by a slow host. The error is a read that can't start. + pub fn begin_reload(&mut self) -> Result<(), String> { + self.reload_in(false) + } + + fn reload_in(&mut self, wait: bool) -> Result<(), String> { if self.time_zarr.is_some() { return Ok(()); } self.opening = None; self.error = None; self.failed_at = None; - // Read in place: its overview, by the reader the app installed; waited for natively (as a - // file is read), landing later in the browser. + self.failures = 0; + // Read in place: its overview, by the reader the app installed; waited for natively when + // asked to (as a file is read), else landing later, as always in the browser. if self.in_place { if let Err(e) = self.begin_in_place_read() { self.error = Some(e.clone()); @@ -411,8 +436,11 @@ impl RasterLayer { return Err(e); } #[cfg(not(target_arch = "wasm32"))] - return self.wait_for_read(Instant::now() + rustgis_data::raster::IN_PLACE_WAIT); + if wait { + return self.wait_for_read(Instant::now() + rustgis_data::raster::IN_PLACE_WAIT); + } #[cfg(target_arch = "wasm32")] + let _ = wait; return Ok(()); } if self.embedded.is_none() @@ -573,12 +601,14 @@ impl RasterLayer { self.data = Some(raster); self.error = None; self.failed_at = None; + self.failures = 0; } Err(error) => { // A read made again that fails as before (see `read_again`) isn't said again. self.error_said = self.error_said && self.error.as_deref() == Some(error.as_str()); self.error = Some(error); self.failed_at = Some(Instant::now()); + self.failures = self.failures.saturating_add(1); } } } diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index a0119a792..bd9012d33 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -110,6 +110,10 @@ const TOO_LARGE: &str = "the view is too large a part of the image to read at th /// Why a turned layer is drawn from its overview. const TURNED: &str = "turned, it is drawn from its overview, not at the resolution the view needs"; +/// Why a see-through layer with tiles to come is drawn from its overview alone: no memory for +/// the image that blends the part in view with it. +const UNBLENDED: &str = "no memory to blend the part in view with its overview, so it is drawn from its overview"; + /// Draws a raster layer read in place (a cloud-optimised GeoTIFF by its address, see /// [`rustgis_map::raster::RasterLayer::in_place`]) as [`draw`] draws one held whole: its overview /// `r` (the layer's pixels), or, zoomed in past it, the part in view read from the level the view @@ -254,6 +258,11 @@ pub(crate) fn draw_in_place( draw(&mut both, part, &own_style, Some(stats), view, 1.0, full_res, place); } pm.draw_pixmap(0, 0, both.as_ref(), &PixmapPaint { opacity, ..PixmapPaint::default() }, Transform::identity(), None); + } else { + // No image to blend in: the overview alone, and said (the map shows it, an export + // refuses it), rather than nothing. + whole(pm); + return Err(UNBLENDED.into()); } } else { for part in &parts { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 2097d2108..2e736f64b 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -234,6 +234,16 @@ pub fn retry_wait(error: &str, default: Duration) -> Duration { retry_after(error).unwrap_or(default) } +/// How long a read that has failed `failures` times in a row waits before the next try: `wait`, +/// doubled for each failure past the first, within the hour (see [`retry_after`]'s bound), so a +/// file that never reads isn't asked for every minute all session. +pub fn backoff(wait: Duration, failures: u32) -> Duration { + let most = Duration::from_secs(crate::failure::MAX_RETRY_AFTER); + // Past twelve doublings the wait is over the hour whatever it was (it is a second at least). + let doublings = failures.saturating_sub(1).min(12); + wait.saturating_mul(1u32 << doublings).min(most) +} + /// How many fetches are under way. Once a fetch no longer counts, its [`generation`] shows. pub fn in_flight() -> usize { IN_FLIGHT.load(Ordering::Acquire) @@ -2286,6 +2296,22 @@ mod tests { assert_eq!(retry_wait("http status: 500", RETRY), RETRY); } + /// A read that fails again waits twice as long each time, within the hour: a file that never + /// reads isn't asked for every minute all session. + #[test] + fn a_read_that_fails_again_waits_longer_each_time() { + let minute = Duration::from_secs(60); + assert_eq!(backoff(minute, 0), minute); + assert_eq!(backoff(minute, 1), minute); + assert_eq!(backoff(minute, 2), Duration::from_secs(120)); + assert_eq!(backoff(minute, 3), Duration::from_secs(240)); + assert_eq!(backoff(minute, 7), Duration::from_secs(3600), "within the hour"); + assert_eq!(backoff(minute, u32::MAX), Duration::from_secs(3600), "bounded, never overflowing"); + // A server's own wait is doubled the same way, within the hour. + assert_eq!(backoff(Duration::from_secs(1), 2), Duration::from_secs(2)); + assert_eq!(backoff(Duration::from_secs(3600), 2), Duration::from_secs(3600)); + } + /// A tile its server refused saying when to come back (429 or 503 with `Retry-After`) has the /// map asked to draw again then, from the gate, on each draw while it waits (the alarm is /// one-shot, and another may have fired since; the failure arms it first). The alarm is the diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 5b9a98869..a45cb6b2c 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -122,12 +122,15 @@ in view at the resolution it needs as it draws, and draws it with the layer's st overview's statistics, so the image looks the same at every zoom. Until a part's tiles arrive, the level zoomed from stands in for them. A turned layer (Rotation in the Symbology pane) is drawn from the overview it holds: zoomed in past it, the map says so and an export refuses it. The -Details pane says the image's full size and that it is read in place. A file whose smallest -image is too large to hold as the overview (a large image saved without overviews) isn't added: -GDAL's COG driver or `gdaladdo` makes the overviews. A tile a sparse file leaves out is empty. A -tile, the file's header or the overview a layer holds, that fails to arrive is asked for again a -minute later, or when its server says to in a `Retry-After` header (with a 429 or 503 answer, -within the hour); the map draws again then. +Details pane says the image's full size and that it is read in place. The desktop app adds the +layer at once and reads its overview as the map draws (the Details pane says so until it is in), +so a slow host doesn't hold the window; the command line and Python wait for it, up to two +minutes, so a command's result is complete. A file whose smallest image is too large to hold as +the overview (a large image saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` +makes the overviews. A tile a sparse file leaves out is empty. A tile, the file's header or the +overview a layer holds, that fails to arrive is asked for again a minute later, or when its server +says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again +then. An overview that fails again waits twice as long each time, up to an hour. Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, and isn't written without them; in the web app, export again once the map has drawn the image. From e7d4e9fd19fa74d912e14c1d5d380fe93002aa61 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Wed, 4 Nov 2026 14:31:17 -0330 Subject: [PATCH 22/27] fix(tiles): a joint read the queue keeps dropping ends, and one wait bounds every map's reads Review findings on 767f99a4: - `Session::wait_in_place_reads` waits for every map's in-place reads at once, under one two-minute deadline, not each map's in turn. - `fetch_all` counts the tiles the fetch queue dropped and fails past eight a tile (64 at least), saying so, so the layer reads again later with its backing-off wait rather than asking without end (the browser has no other end to it). It takes its tile getter now (`fetch_all_with`), so the test answers "dropped" without the process-wide fetcher. - The tile index of a hostile header's huge level is computed checked in `chunks_of` and `assemble`: a tile `u32` can't count is skipped, not wrapped (a debug build panicked). - A look applied after the pixels' losses were said says its own loss alone, not the header's again. Tests: a_read_the_queue_keeps_dropping_ends, a_window_at_the_far_edge_of_a_huge_level_doesnt_overflow (the far corner too), a_look_applied_after_the_losses_were_said_says_its_own_alone. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/lib.rs | 31 ++++++------ crates/map/src/raster.rs | 27 ++++++++++- crates/tiles/src/cog.rs | 87 ++++++++++++++++++++++++++++++++-- crates/tiles/src/cog/raster.rs | 16 +++++-- 4 files changed, 136 insertions(+), 25 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 062607e13..0f0e3d182 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1674,21 +1674,22 @@ impl Session { if !self.in_place_wait { return; } - for m in &mut self.project.maps { - let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m - .layers - .iter_mut() - .filter_map(|l| { - let name = l.name.clone(); - let rl = l.raster.as_mut()?; - rl.in_place.then_some((name, rl)) - }) - .collect(); - let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); - for (i, e) in rustgis_map::raster::wait_for_reads(&mut layers) { - if let Some((name, _)) = named.get(i) { - log::warn!("{name}: {e}"); - } + // Every map's at once: one wait bounds them all, not each map's in turn. + let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = self + .project + .maps + .iter_mut() + .flat_map(|m| m.layers.iter_mut()) + .filter_map(|l| { + let name = l.name.clone(); + let rl = l.raster.as_mut()?; + rl.in_place.then_some((name, rl)) + }) + .collect(); + let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); + for (i, e) in rustgis_map::raster::wait_for_reads(&mut layers) { + if let Some((name, _)) = named.get(i) { + log::warn!("{name}: {e}"); } } } diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index e4aea702f..d33acce21 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -336,7 +336,6 @@ impl RasterLayer { match self.data.clone() { Some(d) => { self.look_lost = RasterStyle::look_lost(&d, &look); - self.lost_said &= self.look_lost.is_none(); if let Some(style) = RasterStyle::for_look(&d, &look) { self.style = style; } @@ -553,7 +552,9 @@ impl RasterLayer { /// land later (the browser). pub fn lost_unsaid(&mut self) -> Vec { if self.lost_said { - return Vec::new(); + // A look applied after the pixels' losses were said: what it asked for that the file + // hasn't, on its own, not the pixels' losses again. + return self.look_lost.take().into_iter().collect(); } let mut lost = self.data.as_ref().map(|d| d.lost.clone()).unwrap_or_default(); lost.extend(self.look_lost.take()); @@ -1609,6 +1610,28 @@ mod tests { Raster::new("dem", 4, 4, vec![data], gt, Crs::LonLat, None, ColorHint::Gray, "Float32").unwrap() } + /// What reading the pixels lost is said once; a look applied after that (bands the file + /// hasn't) says what it lost on its own, not the pixels' losses again. + #[test] + fn a_look_applied_after_the_losses_were_said_says_its_own_alone() { + let mut r = dem(); + r.lost.push("its colour table is left out".into()); + let mut rl = RasterLayer::in_place(r, "test://look-later.tif"); + assert_eq!(rl.lost_unsaid(), vec!["its colour table is left out".to_string()]); + assert!(rl.lost_unsaid().is_empty()); + let mut rl = rl.with_look(AddressLook { bands: vec![9], range: None }); + let said = rl.lost_unsaid(); + assert_eq!(said.len(), 1, "{said:?}"); + assert!(!said[0].contains("colour table"), "{said:?}"); + assert!(rl.lost_unsaid().is_empty()); + // A look applied before they are said is said with them, once. + let mut r = dem(); + r.lost.push("its colour table is left out".into()); + let mut rl = RasterLayer::in_place(r, "test://look-first.tif").with_look(AddressLook { bands: vec![9], range: None }); + assert_eq!(rl.lost_unsaid().len(), 2); + assert!(rl.lost_unsaid().is_empty()); + } + #[test] fn defaults_and_colours() { let r = dem(); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index a4aaab23d..939be6c53 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -839,6 +839,13 @@ fn fetch_bands( type Gathered = Box, String>) + Send>; +/// How many times the fetch queue may drop the tiles of a joint read of `tiles` before it fails: +/// eight a tile, 64 at least, so a busy queue delays a read but one that never gets its turn ends +/// (the browser has no other end to it). +fn drops_allowed(tiles: usize) -> usize { + tiles.saturating_mul(8).max(64) +} + /// Tiles a joint read ([`fetch_all`]) has under way at once: half the fetch queue, so a read of /// many (the overview of a file with few overviews) doesn't push its own requests out of it. pub(super) const AT_ONCE: usize = 256; @@ -846,13 +853,23 @@ pub(super) const AT_ONCE: usize = 256; /// Reads tiles of a level (each once, shared by the map tiles that need it), then calls `then` /// with their bands in the order asked. [`AT_ONCE`] are under way at a time, and a tile the fetch /// queue dropped (other reads filled it: see [`crate::cache::DROPPED`]) is asked for again, so a -/// drop is a delay, not a failure. +/// drop is a delay, not a failure, until the drops pass [`drops_allowed`]: then the read fails, +/// saying so, rather than asking without end (the layer reads again later). fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: Gathered) { + fetch_all_with(Arc::new(get_tile), info, url, level, wanted, then) +} + +/// How [`fetch_all_with`] gets a tile: [`get_tile`], or a test's stand-in. +type GetTile = Arc, String, usize, u32, Box) + Send>) + Send + Sync>; + +/// [`fetch_all`] through `get`. +fn fetch_all_with(get: GetTile, info: Arc, url: String, level: usize, wanted: Vec, then: Gathered) { if wanted.is_empty() { then(Ok(Vec::new())); return; } struct Gather { + get: GetTile, info: Arc, url: String, level: usize, @@ -864,11 +881,14 @@ fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: /// Someone is asking for the next tiles, and asks again after: a tile answered at once /// leaves it to them (asking from the answer would nest as deep as the tiles kept). asking: bool, + /// Tiles the fetch queue dropped so far, bounded by [`drops_allowed`]. + dropped: usize, failed: Option, then: Option, } let n = wanted.len(); let g = Arc::new(Mutex::new(Gather { + get, info, url, level, @@ -877,6 +897,7 @@ fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: to_ask: (0..n).rev().collect(), in_flight: 0, asking: false, + dropped: 0, failed: None, then: Some(then), })); @@ -886,7 +907,7 @@ fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: /// (then once the rest have answered), and answers `then` once. fn ask(g: &Arc>) { loop { - let (info, url, level, batch) = { + let (get, info, url, level, batch) = { let mut s = g.lock().unwrap_or_else(|e| e.into_inner()); if s.asking { return; @@ -913,11 +934,11 @@ fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: return; } s.asking = true; - (s.info.clone(), s.url.clone(), s.level, batch) + (s.get.clone(), s.info.clone(), s.url.clone(), s.level, batch) }; for (k, idx) in batch { let g = g.clone(); - get_tile( + get( info.clone(), url.clone(), level, @@ -932,7 +953,21 @@ fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: *slot = Some(b); } } - Err(e) if e == crate::cache::DROPPED => s.to_ask.push(k), + // Dropped: asked for again, within the drops allowed (a batch + // already asked for answers after the read failed: not counted). + Err(e) if e == crate::cache::DROPPED => { + if s.failed.is_none() { + s.dropped += 1; + if s.dropped > drops_allowed(s.wanted.len()) { + s.failed = Some(format!( + "the fetch queue dropped its tiles {} times (other reads filled it): busy", + s.dropped + )); + } else { + s.to_ask.push(k); + } + } + } Err(e) => s.failed = Some(e), } } @@ -1449,6 +1484,48 @@ pub fn read_values_counted_blocking( #[cfg(test)] mod tests { + /// A joint read whose tiles the fetch queue keeps dropping ends, saying so, rather than + /// asking without end: within eight drops a tile (64 at least). + #[test] + fn a_read_the_queue_keeps_dropping_ends() { + use rustgis_data::raster::cog::{Parse, Sparse, example, layout}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + let Ok(Parse::Done(layout)) = layout(&Sparse::default().with(0, example(64, 16))) else { panic!("not a COG") }; + let transformer = layout.crs.transformer().unwrap(); + let info = Arc::new(super::Info { + layout: *layout, + header: Sparse::default(), + transformer, + bounds: [0.0; 4], + merc: [0.0; 4], + min_zoom: 0, + max_zoom: 0, + stretch: Mutex::default(), + stretching: Mutex::default(), + stretch_failed: Mutex::default(), + alpha: None, + }); + let asked = Arc::new(AtomicUsize::new(0)); + let a = asked.clone(); + let get: super::GetTile = Arc::new( + move |_: Arc, _: String, _: usize, _: u32, done: Box) + Send>| { + a.fetch_add(1, Ordering::Relaxed); + done(Err(crate::cache::DROPPED.into())); + }, + ); + let (tx, rx) = std::sync::mpsc::channel(); + let wanted: Vec = (0..16).collect(); + super::fetch_all_with(get, info, "test://dropping.tif".into(), 0, wanted, Box::new(move |r| tx.send(r).unwrap_or_default())); + let r = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("the read ended"); + let e = r.map(|_| ()).unwrap_err(); + // Eight drops a tile, then the read fails on the next; the rest of that batch isn't counted. + assert!(e.contains("dropped its tiles 129 times"), "{e}"); + let asked = asked.load(Ordering::Relaxed); + assert!((129..=129 + 15).contains(&asked), "{asked}"); + assert_eq!((super::drops_allowed(1), super::drops_allowed(16), super::drops_allowed(4096)), (64, 128, 32768)); + } + #[test] fn announced_header_lengths_and_types() { use super::{HEAD, header_length, named_tiff, not_tiff_type}; diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index e14eed05c..03dfcbc23 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -149,7 +149,8 @@ fn chunks_of(info: &Info, level: usize, cols: (u32, u32), rows: (u32, u32), band let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); for ty in rows.0 / th..rows.1.min(l.height).div_ceil(th) { for tx in cols.0 / tw..cols.1.min(l.width).div_ceil(tw) { - let tile = ty * l.across() + tx; + // A level too large for `u32` to count its tiles (a hostile header) has none past it. + let Some(tile) = ty.checked_mul(l.across()).and_then(|t| t.checked_add(tx)) else { continue }; for &b in bands { if let Some((c, _)) = info.layout.chunk(l, tile, b) { chunks.insert(c); @@ -454,7 +455,9 @@ fn assemble( let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); for ty in rows.0 / th..rows.1.div_ceil(th) { for tx in cols.0 / tw..cols.1.div_ceil(tw) { - let tile = ty * l.across() + tx; + // A level too large for `u32` to count its tiles (a hostile header) has none past it: + // left empty, as a tile the header has no chunk for is. + let Some(tile) = ty.checked_mul(l.across()).and_then(|t| t.checked_add(tx)) else { continue }; // The tile's part of the window, in the level's pixels. // The tile's end saturates: a width near `u32::MAX` (the header bounds the tiles, // not the width) stays a tile, not a wrapped, empty one. @@ -593,7 +596,8 @@ mod tests { } /// A level as wide as `u32` allows (a hostile header): its last tile's end doesn't wrap, so - /// the window is assembled, its missing tiles gaps. + /// the window is assembled, its missing tiles gaps. As tall too, its far tiles are more than + /// `u32` counts: a window there is assembled all the same (empty), and asks for no chunk. #[test] fn a_window_at_the_far_edge_of_a_huge_level_doesnt_overflow() { let mut info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); @@ -601,6 +605,12 @@ mod tests { let (data, gaps) = assemble(&info, 0, (u32::MAX - 15, u32::MAX), (0, 16), &[0], |_| None); assert_eq!((data.len(), data[0].len()), (1, 16 * 15)); assert!(gaps.iter().all(|g| g.cols.0 <= g.cols.1 && g.rows.0 <= g.rows.1)); + info.layout.levels[0].height = u32::MAX; + let far = ((u32::MAX - 15, u32::MAX), (u32::MAX - 15, u32::MAX)); + let (data, gaps) = assemble(&info, 0, far.0, far.1, &[0], |_| None); + assert_eq!((data.len(), data[0].len()), (1, 15 * 15)); + assert!(data[0].iter().all(|v| v.is_nan()) && gaps.is_empty()); + assert!(chunks_of(&info, 0, far.0, far.1, &[0]).is_empty()); } #[test] From 93a65533c37ea6b2678604ffbdd4832574e2ce1f Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Wed, 4 Nov 2026 16:19:05 -0330 Subject: [PATCH 23/27] fix(engine): a bind counts a raster read in place as reading until its overview is in Where the session doesn't wait for reads in place (the desktop, the browser), a saved S3 connection chosen for a project's rasters read in place only starts their overviews reading. `reopen_where` counted them as opened; now it counts what is in as opened and the rest as reading, `connection.resolveS3` answers `{id, reopened, failed, reading}`, the chooser's notice says how many are still being read, and the user guide says so. Test: a_connection_chosen_without_waiting_counts_its_in_place_rasters_as_reading. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/connections.rs | 2 +- crates/engine/src/s3_connections.rs | 62 +++++++++++++++++++--------- crates/engine/src/tests.rs | 54 ++++++++++++++++++++++++ crates/ui-egui/src/connectionui.rs | 20 +++++---- docs/user-guide/data-formats.md | 4 +- 5 files changed, 112 insertions(+), 30 deletions(-) diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index 1ebe80945..70c777eb5 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -42,7 +42,7 @@ pub fn commands() -> Vec { r#"{"id": "a saved S3 connection's id or name"} (deletes stored keys; a login in this session keeps working)"#, ), C::new("connection.resolveS3", "Choose S3 Connection", "Home › Add › Data Sources", resolve_s3).view().params( - r#"{"source": "a pending project S3 source address", "id": "a candidate saved connection id"} → {id, reopened, failed}: explicitly grants this bucket and opens only its waiting sources"#, + r#"{"source": "a pending project S3 source address", "id": "a candidate saved connection id"} → {id, reopened, failed, reading}: explicitly grants this bucket and opens only its waiting sources; reading counts the rasters read in place whose overview is still being read (the desktop doesn't wait for it: it lands as the map draws, and one that fails is said then)"#, ), C::new("connection.forgetPassword", "Forget Saved Password", "Home › Add › Data Sources", forget_password).view().params(r#"{"id": "a database connection's id or name"}"#), C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).view().params( diff --git a/crates/engine/src/s3_connections.rs b/crates/engine/src/s3_connections.rs index 2a7917db7..6dde8602a 100644 --- a/crates/engine/src/s3_connections.rs +++ b/crates/engine/src/s3_connections.rs @@ -283,19 +283,31 @@ fn block(s: &mut Session, g: &Group) -> usize { s.project.addresses_mut().into_iter().filter(|a| g.has(a)).count() } -/// Opens (again) the streamed sources whose address `which` accepts; how many opened and how -/// many failed. Rasters read in place read their overviews together (see +/// What opening a bind's sources did: how many opened, how many failed, and how many rasters read +/// in place are still reading their overview. Where nothing waits for a read in place (the +/// desktop, the browser), a raster whose overview was only asked for is reading, not opened: it +/// lands as the map draws, and one that fails is said then. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Counts { + pub(crate) opened: usize, + pub(crate) failed: usize, + pub(crate) reading: usize, +} + +/// Opens (again) the streamed sources whose address `which` accepts: how many opened, failed, and +/// are still reading. Rasters read in place read their overviews together (see /// [`rustgis_map::raster::read_together`]): a bucket of several waits for the slowest, not for -/// each in turn, and on the desktop for none (they land as the map draws). -fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) { - let (mut opened, mut failed) = (0, 0); +/// each in turn, and on the desktop for none (they land as the map draws, and count as reading +/// until then). +fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> Counts { + let mut n = Counts::default(); let wait = s.in_place_wait; for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { let Some(path) = src.path.clone() else { continue }; if (!src.is_open() || src.error.is_some()) && which(&path) { match src.reopen() { - Ok(()) => opened += 1, - Err(_) => failed += 1, + Ok(()) => n.opened += 1, + Err(_) => n.failed += 1, } } } @@ -307,21 +319,31 @@ fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) continue; } match raster.reload() { - Ok(()) => opened += 1, - Err(_) => failed += 1, + Ok(()) => n.opened += 1, + Err(_) => n.failed += 1, } } } - let unread = rustgis_map::raster::read_together(&mut in_place, wait).len(); - failed += unread; - opened += in_place.len().saturating_sub(unread); - (opened, failed) + let unread = rustgis_map::raster::read_together(&mut in_place, wait); + n.failed += unread.len(); + // Counted by what is in, not by what was asked for: a read nobody waited for is still reading. + for (i, rl) in in_place.iter().enumerate() { + if unread.iter().any(|(j, _)| *j == i) { + continue; + } + if rl.data.is_some() { + n.opened += 1; + } else { + n.reading += 1; + } + } + n } /// Binds `g`'s sources to the saved connection `c`: logs it in if it isn't, lets its login read /// the bucket (the user's choice, or its only candidate's), rewrites the addresses to name it and -/// reads them. How many sources opened and how many failed. -fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result<(usize, usize), String> { +/// reads them. How many sources opened, failed, and are still reading. +fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result { let endpoint = endpoint_of(c).ok_or_else(|| s3::MISMATCH.to_string())?; let binding = crate::cmd::connections::login_saved(s, c)?; s3::grant_saved(&binding, &g.bucket)?; @@ -331,11 +353,11 @@ fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result<(usize, usiz cache::forget(part(a)); } let affected = |address: &str| addresses.iter().any(|a| a == address); - let (mut opened, failed) = reopen_where(s, affected); + let mut n = reopen_where(s, affected); // Tile resources fetch as they draw, by their new address. - opened += + n.opened += s.project.maps.iter().flat_map(|m| &m.tiles).filter(|t| affected(&t.url) || t.time_url.as_deref().is_some_and(affected)).count(); - Ok((opened, failed)) + Ok(n) } /// The user's choice of the saved connection `id` to read the sources of the pending resource @@ -350,9 +372,9 @@ pub(crate) fn choose(s: &mut Session, source: &str, id: &str) -> crate::CmdResul s.s3_pending.remove(index); return Err(STALE.into()); } - let (reopened, failed) = bind_group(s, &g, &c)?; + let n = bind_group(s, &g, &c)?; s.s3_pending.remove(index); - Ok(serde_json::json!({"id": c.id, "reopened": reopened, "failed": failed})) + Ok(serde_json::json!({"id": c.id, "reopened": n.opened, "failed": n.failed, "reading": n.reading})) } #[cfg(test)] diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 2a7890625..6bd6a0d70 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -15549,6 +15549,60 @@ fn binding_a_connection_reads_its_in_place_rasters_together() { t.run("connection.remove", &json!({"id": "Together"})).unwrap(); } +/// Where the session doesn't wait for reads in place (the desktop), a saved S3 connection chosen +/// for a project's rasters read in place starts their overviews reading and says so: they count +/// as `reading` in `connection.resolveS3`'s answer, not as `reopened`, until they land as the map +/// draws. Waiting, the same choice reads them before it answers, and they are reopened. +#[test] +fn a_connection_chosen_without_waiting_counts_its_in_place_rasters_as_reading() { + use rustgis_map::raster::RasterLayer; + use std::time::Duration; + let base = serve_s3_together(vec![ + ("a.tif", rustgis_data::raster::cog::example(512, 256)), + ("b.tif", rustgis_data::raster::cog::example(256, 512)), + ]); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("A".to_string(), RasterLayer::unread_in_place("s3://s3-chosen-bucket/a.tif")); + s.map_mut().add_raster("B".to_string(), RasterLayer::unread_in_place("s3://s3-chosen-bucket/b.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let layer = |s: &Session, name: &str| s.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); + // The connection lists no bucket, so the project's sources wait for the user's choice of it. + let open = |wait: bool| { + let mut t = Session::new(Project::default()); + t.in_place_wait = wait; + let c = + t.run("connection.new", &json!({"kind": "s3", "name": "Chosen", "address": base, "options": {"credentials": "none"}})).unwrap(); + let id = c["id"].as_str().unwrap_or_default().to_string(); + assert!(!id.is_empty(), "{c}"); + t.run("project.openText", &json!({"text": text})).unwrap(); + let source = t.s3_pending.first().map(|p| p.source.clone()).unwrap_or_default(); + assert!(!source.is_empty(), "nothing waited for a choice: {:?}", t.notices); + let r = t.run("connection.resolveS3", &json!({"source": source, "id": id})).unwrap(); + (t, r, id) + }; + let counts = |r: &Value| (r["reopened"].as_u64(), r["reading"].as_u64(), r["failed"].as_u64()); + let (mut t, r, id) = open(false); + assert_eq!(counts(&r), (Some(0), Some(2), Some(0)), "{r}"); + assert!(layer(&t, "A").data.is_none() && layer(&t, "B").data.is_none(), "chosen before the overviews are in"); + let view = rustgis_geom::Extent::new(10.0, 45.0, 11.0, 46.0); + let start = std::time::Instant::now(); + while (layer(&t, "A").data.is_none() || layer(&t, "B").data.is_none()) && start.elapsed() < Duration::from_secs(60) { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + t.read_in_place_again(); + t.map_mut().sync_sources(&view, 1000.0, false); + t.notice_landed_rasters(); + } + for (name, width) in [("A", 512), ("B", 256)] { + assert_eq!(layer(&t, name).data.as_ref().map(|d| d.width), Some(width), "{name}: {:?}", layer(&t, name).error); + } + t.run("connection.remove", &json!({"id": id})).unwrap(); + // Waiting, the choice answers once the overviews are in. + let (mut t, r, id) = open(true); + assert_eq!(counts(&r), (Some(2), Some(0), Some(0)), "{r}"); + assert!(layer(&t, "A").data.is_some() && layer(&t, "B").data.is_some()); + t.run("connection.remove", &json!({"id": id})).unwrap(); +} + /// A stand-in S3 endpoint serving `files` (key, bytes) by range, answering nothing until each key /// has been asked for once: reads made together get their answers, reads made one after the other /// never get the first. diff --git a/crates/ui-egui/src/connectionui.rs b/crates/ui-egui/src/connectionui.rs index 9c59c02d4..616f8fdda 100644 --- a/crates/ui-egui/src/connectionui.rs +++ b/crates/ui-egui/src/connectionui.rs @@ -894,15 +894,19 @@ fn resolve_waiting(app: &mut RustGisApp, id: &str) { fn choose_s3(app: &mut RustGisApp, source: &str, id: &str) -> Result<(), String> { let v = app.run("connection.resolveS3", json!({"source": source, "id": id}))?; let name = app.session.connections.find(id).map(|c| c.name.clone()).unwrap_or_else(|| id.to_string()); - let opened = v["reopened"].as_u64().unwrap_or(0).to_string(); + let opened = v["reopened"].as_u64().unwrap_or(0); let failed = v["failed"].as_u64().unwrap_or(0); - let text = if failed == 0 { - i18n::fmt("{name} now reads {n} project sources", &[("name", &name), ("n", &opened)]) - } else { - i18n::fmt( - "{name} now reads {n} project sources; {failed} could not be read", - &[("name", &name), ("n", &opened), ("failed", &failed.to_string())], - ) + // Rasters read in place whose overview is on its way: the connection reads them, but they + // aren't in yet (the desktop doesn't wait; they land as the map draws). + let reading = v["reading"].as_u64().unwrap_or(0); + let n = opened.saturating_add(reading).to_string(); + let (failed_n, reading_n) = (failed.to_string(), reading.to_string()); + let args: &[(&str, &str)] = &[("name", &name), ("n", &n), ("failed", &failed_n), ("reading", &reading_n)]; + let text = match (failed, reading) { + (0, 0) => i18n::fmt("{name} now reads {n} project sources", args), + (0, _) => i18n::fmt("{name} now reads {n} project sources; {reading} are still being read", args), + (_, 0) => i18n::fmt("{name} now reads {n} project sources; {failed} could not be read", args), + _ => i18n::fmt("{name} now reads {n} project sources; {reading} are still being read and {failed} could not be read", args), }; app.session.notify(text, failed > 0); Ok(()) diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index a45cb6b2c..6b90f7c67 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -125,7 +125,9 @@ from the overview it holds: zoomed in past it, the map says so and an export ref Details pane says the image's full size and that it is read in place. The desktop app adds the layer at once and reads its overview as the map draws (the Details pane says so until it is in), so a slow host doesn't hold the window; the command line and Python wait for it, up to two -minutes, so a command's result is complete. A file whose smallest image is too large to hold as +minutes, so a command's result is complete. A saved S3 connection chosen for such layers (Choose +S3 connection…) counts them as still being read until their overview is in, apart from what it +opened. A file whose smallest image is too large to hold as the overview (a large image saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` makes the overviews. A tile a sparse file leaves out is empty. A tile, the file's header or the overview a layer holds, that fails to arrive is asked for again a minute later, or when its server From 4cd9d68e1e3c548d23b99ba93564b88eaffbaaa6 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Wed, 4 Nov 2026 17:32:05 -0330 Subject: [PATCH 24/27] fix(engine): COG tile layers open as rasters as the project opens only, and a failed read in place isn't started over by a repair `Project::normalize` ran the migration of plain COG tile layers to raster layers read in place on every repair, so a tile layer added this session (`map.addTiles`, kind `cog`) was converted, losing its attribution, zoom range and bounds, when a layer file was added or maps imported. It runs as a project opens now (`normalize_opened`), and as a map file is imported. The same repair started the reads of the layers not read yet, a failed one among them, which skipped its wait and reset its failure count: a layer with a read failure is left to `Session::read_in_place_again`. A raster turned by a hair either side of none isn't turned (render). The chooser's count reads with one as with many. Tests: a_cog_tile_layer_added_this_session_stays_one_until_the_project_opens_again, a_hair_of_rotation_either_way_isnt_a_turn (render); the overview retry test adds a layer file after the first failure. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/items.rs | 3 ++ crates/engine/src/cmd/tiles.rs | 3 +- crates/engine/src/project.rs | 30 +++++++++++++++--- crates/engine/src/tests.rs | 51 ++++++++++++++++++++++++++++++ crates/render/src/raster.rs | 21 +++++++++++- crates/ui-egui/src/connectionui.rs | 4 +-- 6 files changed, 103 insertions(+), 9 deletions(-) diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index c4c6730c8..d98f47d96 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -434,6 +434,9 @@ fn import_maps(s: &mut Session, p: &Value) -> CmdResult { if let Some(d) = dir.as_deref().and_then(Path::to_str) { crate::project::resolve_file_paths(d, m.file_paths_mut()); } + // An older map file's plain COG tile layers open as raster layers read in place, as a + // project's do (a project imported had them opened as it was read). + super::tiles::cog_tiles_to_rasters(&mut m); vec![m] } Err(_) => match Project::from_import_json(&text, dir.as_deref()) { diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index cf1850d39..254005c86 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -563,7 +563,8 @@ fn blend(p: &Value) -> Result, String> { /// Turns a map's plain COG tile layers into raster layers read in place, each where it was drawn, /// with its name, visibility, opacity, blend and look: those of projects saved before GeoTIFFs read -/// in place were raster layers, and those `map.addTiles` (kind `cog`) still adds, on every open (a +/// in place were raster layers, and those `map.addTiles` (kind `cog`) still adds, as a project or +/// a map file opens, and then only: a tile layer added this session stays one until then (a /// plain GeoTIFF is one kind of layer, however it was added; what a tile layer had that a raster /// layer hasn't is said once, see `Session::notice_landed_rasters`). A dated series, a data cube's /// files and AlphaEarth's embeddings stay tile layers. diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 669ece645..f39705017 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -557,6 +557,19 @@ impl Project { /// Repairs invariants after loading (at least one map; a valid active index). pub fn normalize(&mut self) { + self.repair(false); + } + + /// [`Project::normalize`] for a project read from its file, which also opens its plain COG + /// tile layers (a project saved before GeoTIFFs read in place were raster layers, or with + /// those `map.addTiles` adds) as raster layers read in place: as the project opens only. A + /// later repair (a layer file added, maps imported) leaves a tile layer added this session + /// as it is. + pub(crate) fn normalize_opened(&mut self) { + self.repair(true); + } + + fn repair(&mut self, opened: bool) { if self.maps.is_empty() { self.maps.push(Map::new("Map")); } @@ -667,8 +680,11 @@ impl Project { // Tile layers with an address that can't be fetched (a hand edit) are dropped. m.tiles.retain_mut(|t| t.repair().map_err(|e| log::warn!("{}: {e}", m.name)).is_ok()); // GeoTIFFs read in place are raster layers (a project from before saved tile layers), - // so they don't count toward the tile layers kept. - crate::cmd::tiles::cog_tiles_to_rasters(m); + // so they don't count toward the tile layers kept. As the project opens only: a tile + // layer added this session stays one until the project is opened again. + if opened { + crate::cmd::tiles::cog_tiles_to_rasters(m); + } m.tiles.truncate(rustgis_map::tiles::MAX_TILE_LAYERS); #[cfg(not(target_arch = "wasm32"))] for t in &mut m.tiles { @@ -915,7 +931,7 @@ impl Project { if let Some(d) = dir.and_then(Path::to_str) { p.resolve_paths(d); } - p.normalize(); + p.normalize_opened(); Ok(p) } @@ -933,7 +949,10 @@ impl Project { /// once (see [`rustgis_map::raster::read_together`]): a project or an import of several waits for /// the slowest, not for each in turn, where the session waits for them /// ([`crate::Session::wait_in_place_reads`]); elsewhere they land as the map draws. One read from -/// an S3 bucket waits for its connection (see [`crate::s3_connections::resolve_project_s3`]). +/// an S3 bucket waits for its connection (see [`crate::s3_connections::resolve_project_s3`]). One +/// that failed is left to [`crate::Session::read_in_place_again`], which reads it again once its +/// wait is over: a repair of the project (a layer file added, maps imported) doesn't start that +/// wait over. pub(crate) fn read_in_place(m: &mut Map) { let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m .layers @@ -941,7 +960,8 @@ pub(crate) fn read_in_place(m: &mut Map) { .filter_map(|l| { let name = l.name.clone(); let rl = l.raster.as_mut()?; - (rl.in_place && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed)).then_some((name, rl)) + (rl.in_place && rl.read_failure().is_none() && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed)) + .then_some((name, rl)) }) .collect(); let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index e99384805..df948d0f5 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -15513,6 +15513,19 @@ fn an_in_place_layers_failed_overview_is_read_again_when_its_wait_is_over() { s.read_in_place_again(); assert!(layer(&s).data.is_none()); assert_eq!(sent.load(Ordering::Relaxed), 0, "read again before its wait was over"); + // Nor by a repair of the project (a layer file added, maps imported), which reads the layers + // not read yet together: a failed read is paced, not started over. + let at = layer(&s).read_failure().map(|(_, at, _)| at).unwrap(); + let mut other = rustgis_map::Map::new("Other"); + let id = other.add_dataset(crate::data::Dataset::new("Empty", crate::data::GeometryType::Point, Vec::new())); + crate::cmd::layer::add_layer_text(&mut s, &serde_json::to_string(other.layer(id).unwrap()).unwrap()).unwrap(); + assert_eq!( + layer(&s).read_failure().map(|(_, at, failures)| (at, failures)), + Some((at, 1)), + "started over by the repair: {:?}", + layer(&s).error + ); + assert_eq!(sent.load(Ordering::Relaxed), 0); // Then read again, as the map draws: refused again, it waits twice as long this time (see // `cache::backoff`), and isn't said again. let view = rustgis_geom::Extent::new(10.0, 45.9, 10.1, 46.0); @@ -15911,6 +15924,44 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { assert_eq!(said(&t), 1); } +/// A plain COG tile layer added this session (`map.addTiles`, kind `cog`) stays a tile layer, with +/// what a raster layer hasn't, through a layer file added and maps imported (each repairs the +/// project): it opens as a raster layer read in place once the project is saved and opened again, +/// as an older project's does. A map file's plain COG tile layers open as raster layers as the map +/// is imported, as a project's do. +#[test] +fn a_cog_tile_layer_added_this_session_stays_one_until_the_project_opens_again() { + use rustgis_map::{TileKind, TileLayer}; + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let url = format!("{base}/scene.tif"); + let _ = crate::render::rustgis_tiles::cog::probe_blocking(&url); + let mut s = Session::new(Project::default()); + s.run("map.addTiles", &json!({"url": url, "kind": "cog", "name": "Scene", "attribution": "© Example"})).unwrap(); + let scene = |s: &Session| s.project.maps[0].tiles.iter().find(|t| t.name == "Scene").map(|t| (t.kind, t.attribution.clone())); + assert_eq!(scene(&s), Some((TileKind::Cog, "© Example".to_string())), "{:?}", s.project.maps[0].tiles); + let mut other = rustgis_map::Map::new("Other"); + let id = other.add_dataset(crate::data::Dataset::new("Empty", crate::data::GeometryType::Point, Vec::new())); + crate::cmd::layer::add_layer_text(&mut s, &serde_json::to_string(other.layer(id).unwrap()).unwrap()).unwrap(); + s.run("map.import", &json!({"text": serde_json::to_string(&other).unwrap()})).unwrap(); + assert_eq!(scene(&s), Some((TileKind::Cog, "© Example".to_string())), "converted by a repair"); + assert!(s.project.maps[0].layer_by_name("Scene").is_none()); + assert!(!s.notices.iter().any(|n| n.text.contains("which have no attribution")), "{:?}", s.notices); + // An older map file's plain COG tile layer opens as a raster layer read in place as it is + // imported. + let mut old = serde_json::to_value(rustgis_map::Map::new("Old")).unwrap(); + old["tiles"] = json!([serde_json::to_value(TileLayer::new("Aerial", TileKind::Cog, format!("{base}/aerial.tif"))).unwrap()]); + s.run("map.import", &json!({"text": old.to_string()})).unwrap(); + let imported = s.project.maps.iter().find(|m| m.name == "Old").unwrap(); + let rl = imported.layer_by_name("Aerial").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(imported.tiles.is_empty() && rl.in_place && rl.data.is_some(), "{:?}", rl.error); + // Saved and opened again, Scene is a raster layer read in place too. + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + let rl = t.project.maps[0].layer_by_name("Scene").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(t.project.maps[0].tiles.is_empty() && rl.in_place && rl.data.is_some(), "{:?}", rl.error); +} + /// A server answering every request with `body`, logging each as `HEAD`, `GET` or `GET `. /// Its HEAD answers carry `announce` as `X-TIFF-Header-Length`, `kind` as `Content-Type` and, /// with `ranges`, `Accept-Ranges: bytes`. diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index bd9012d33..9b357a231 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -110,6 +110,13 @@ const TOO_LARGE: &str = "the view is too large a part of the image to read at th /// Why a turned layer is drawn from its overview. const TURNED: &str = "turned, it is drawn from its overview, not at the resolution the view needs"; +/// Whether `rotation` (degrees) turns the raster: anything but a whole number of turns. A hair +/// either side of none (`-1e-7` as much as `1e-7`) isn't one. +fn turned(rotation: f64) -> bool { + let r = rotation.rem_euclid(360.0); + r.min(360.0 - r) > 1e-6 +} + /// Why a see-through layer with tiles to come is drawn from its overview alone: no memory for /// the image that blends the part in view with it. const UNBLENDED: &str = "no memory to blend the part in view with its overview, so it is drawn from its overview"; @@ -149,7 +156,7 @@ pub(crate) fn draw_in_place( // A rotated raster turns about the whole image's centre, not the part in view's: the // overview is drawn, and zoomed in past it (a screen pixel under 0.8 of its pixels) that is // less than the view needs: said, and refused by exports. - if style.rotation.rem_euclid(360.0).abs() > 1e-6 { + if turned(style.rotation) { whole(pm); let zoomed_in = in_view(r, view, place, pm.width(), pm.height(), full_res).is_some_and(|w| w.per_px < 0.8); return if zoomed_in { Err(TURNED.into()) } else { Ok(false) }; @@ -728,6 +735,18 @@ mod tests { use super::*; + /// A raster turned by a hair either side of none isn't turned: it is drawn at the view's + /// resolution, not from its overview. + #[test] + fn a_hair_of_rotation_either_way_isnt_a_turn() { + for r in [0.0, 1e-7, -1e-7, 360.0, -360.0, 720.0 + 1e-7, f64::NAN] { + assert!(!turned(r), "{r}"); + } + for r in [1.0, -1.0, 359.0, 180.0, 1e-5, -1e-5] { + assert!(turned(r), "{r}"); + } + } + fn sampling_grid(pixel_width: f64, crs: Crs, nodata: Option) -> Raster { let gt = GeoTransform { origin_x: 0.0, origin_y: 90.0, pixel_width, pixel_height: -60.0 }; let data = [0.0, 10.0, 20.0, 30.0].repeat(3); diff --git a/crates/ui-egui/src/connectionui.rs b/crates/ui-egui/src/connectionui.rs index 616f8fdda..9ba3465f3 100644 --- a/crates/ui-egui/src/connectionui.rs +++ b/crates/ui-egui/src/connectionui.rs @@ -904,9 +904,9 @@ fn choose_s3(app: &mut RustGisApp, source: &str, id: &str) -> Result<(), String> let args: &[(&str, &str)] = &[("name", &name), ("n", &n), ("failed", &failed_n), ("reading", &reading_n)]; let text = match (failed, reading) { (0, 0) => i18n::fmt("{name} now reads {n} project sources", args), - (0, _) => i18n::fmt("{name} now reads {n} project sources; {reading} are still being read", args), + (0, _) => i18n::fmt("{name} now reads {n} project sources; {reading} still being read", args), (_, 0) => i18n::fmt("{name} now reads {n} project sources; {failed} could not be read", args), - _ => i18n::fmt("{name} now reads {n} project sources; {reading} are still being read and {failed} could not be read", args), + _ => i18n::fmt("{name} now reads {n} project sources; {reading} still being read and {failed} could not be read", args), }; app.session.notify(text, failed > 0); Ok(()) From 9dd924180e9befca1618a52c4bf29350367e445e Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Wed, 4 Nov 2026 20:49:28 -0330 Subject: [PATCH 25/27] fix(engine): a session that doesn't draw reads a failed overview again, and a read that couldn't start isn't started over - `Session::run` reads a failed in-place overview again after each command once its wait is over (the app did so each frame only), waiting for it where the session waits for reads in place: a long-lived Python session no longer keeps a failed layer failed. `read_in_place_again` says whether it started one. - A repair of the project (a layer file added, maps imported) leaves a layer whose read couldn't start as it is, said once, as it left a timed failure: `read_in_place` skips any layer carrying an error. - A GeoTIFF added in place before its overview is in answers its size from the header (the level it will hold, its bands, the full size) and `"reading": true`; before the header is in (the browser), `null`, not 0. - A band number past the last `usize` (a project file's) is printed, not overflowed: `look_lost`, a window's band check, `Look::suffix` and `map.setTiles`'s check use `saturating_add`. Tests: `a_waiting_sessions_next_command_reads_a_failed_overview_again`, `a_read_in_place_that_couldnt_start_isnt_started_over_by_a_repair`, `a_cog_added_before_its_header_is_in_says_it_is_being_read`, `a_look_asking_for_the_band_past_the_last_usize_is_said`, `a_window_of_the_band_past_the_last_usize_is_refused`; `a_cog_added_without_waiting_lands_as_the_map_draws` checks the answer's size. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/layer.rs | 25 +++++++++ crates/engine/src/cmd/tiles.rs | 2 +- crates/engine/src/lib.rs | 26 ++++++--- crates/engine/src/project.rs | 7 ++- crates/engine/src/tests.rs | 98 +++++++++++++++++++++++++++++++++ crates/map/src/raster.rs | 10 +++- crates/tiles/src/cog.rs | 2 +- crates/tiles/src/cog/raster.rs | 13 ++++- docs/user-guide/data-formats.md | 4 +- docs/user-guide/data.md | 4 +- 10 files changed, 175 insertions(+), 16 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index 452cb1d41..6bc808cdf 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3478,6 +3478,7 @@ fn add_in_place( rustgis_map::raster::RasterLayer::opening_in_place(file) }; let layer = layer.with_look(super::tiles::address_look(look)); + let landed = layer.data.is_some(); let mut r = super::raster::add_raster_after(s, name, layer, zoom)?; if let Ok(Some(info)) = cog::info(file) { let [w, so, e, n] = info.bounds; @@ -3487,6 +3488,30 @@ fn add_in_place( s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)); } r["bounds"] = json!(info.bounds); + // Added before its overview is in: sized by the header as the answer would be once the + // overview had landed (the level the layer holds, every band, and the full size when the + // level isn't it). + if !landed && let Some(level) = cog::raster::overview_level(&info) { + let levels = &info.layout.levels; + if let Some(l) = levels.get(level) { + r["width"] = json!(l.width); + r["height"] = json!(l.height); + r["bands"] = json!(info.layout.bands); + } + if let Some(full) = levels.first().filter(|_| level > 0) { + r["overviewOf"] = json!([full.width, full.height]); + } + } + } + if !landed { + // Its overview is still being read (it lands as the map draws): a size the answer can't + // give yet (the header not in, or no level small enough to hold) is null, not 0. + r["reading"] = json!(true); + if r["width"].as_u64() == Some(0) { + for key in ["width", "height", "bands"] { + r[key] = Value::Null; + } + } } r["inPlace"] = json!(true); r["kind"] = json!("raster"); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 5d74b3b8b..8140f64d0 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -776,7 +776,7 @@ fn set(s: &mut Session, p: &Value) -> CmdResult { if let Ok(Some(info)) = cog::info(&t.url) && let Some(b) = new.bands.iter().find(|b| **b >= info.layout.bands) { - return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b + 1)); + return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b.saturating_add(1))); } Some(cog::with_look(&t.url, &new)) } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index f3ea1680d..aa97b16e2 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -1179,6 +1179,12 @@ impl Session { let joins = self.merge.is_some() && self.merge == self.merged && !self.undo.is_empty(); let (at, start) = (project_log::now_ms(), web_time::Instant::now()); let res = self.run_command(id, params); + // A failed overview whose wait is over is read again (the app does this each frame; + // a session that doesn't draw, the command line or Python, as its next command runs), + // waited for where the session waits for reads in place. + if self.read_in_place_again() { + self.wait_in_place_reads(); + } // Pixels the command read in place (a project, an import) say what they lost. self.notice_landed_rasters(); // A plugin's command waiting for its fetches answers `pending` and runs again when they @@ -1697,12 +1703,6 @@ impl Session { } } - /// Reads again the overview of a raster layer read in place whose read failed, once its wait - /// is over: a minute, or what its server's `Retry-After` said, as a tile or a file's header is - /// read again (see `rustgis_tiles::cache::retry_wait`), doubled each time it fails again, - /// within the hour (`cache::backoff`: a file that never reads isn't asked for every minute all - /// session); until then the map is asked to draw again when the wait is over. The app calls it - /// each frame, with [`Session::notice_landed_rasters`]; the pixels land as the map draws. /// Waits for the overviews of the rasters read in place that a project, an import or a layer /// file began reading together (see [`crate::project::read_in_place`]), where the session /// waits for them ([`Session::in_place_wait`]: natively, two minutes at most for all); where @@ -1732,8 +1732,18 @@ impl Session { } } - pub fn read_in_place_again(&mut self) { + /// Reads again the overview of a raster layer read in place whose read failed, once its wait + /// is over: a minute, or what its server's `Retry-After` said, as a tile or a file's header is + /// read again (see `rustgis_tiles::cache::retry_wait`), doubled each time it fails again, + /// within the hour (`cache::backoff`: a file that never reads isn't asked for every minute all + /// session); until then the map is asked to draw again when the wait is over. The app calls it + /// each frame, with [`Session::notice_landed_rasters`], and [`Session::run`] after each command + /// (a session that doesn't draw, the command line or Python, asks again that way); the pixels + /// land as the map draws, or are waited for where the session waits for reads in place + /// ([`Session::in_place_wait`]). Whether a read was started. + pub fn read_in_place_again(&mut self) -> bool { use render::rustgis_tiles::cache; + let mut started = false; for m in &mut self.project.maps { for l in &mut m.layers { let Some(rl) = &mut l.raster else { continue }; @@ -1743,9 +1753,11 @@ impl Session { cache::notify_at(at + wait); } else { rl.read_again(); + started = true; } } } + started } pub fn notify(&mut self, text: impl Into, error: bool) { diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 9e25d82af..02f5ef1c2 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -1143,8 +1143,9 @@ impl Project { /// ([`crate::Session::wait_in_place_reads`]); elsewhere they land as the map draws. One read from /// an S3 bucket waits for its connection (see [`crate::s3_connections::resolve_project_s3`]). One /// that failed is left to [`crate::Session::read_in_place_again`], which reads it again once its -/// wait is over: a repair of the project (a layer file added, maps imported) doesn't start that -/// wait over. +/// wait is over, and one that couldn't start (an address the project keeps that can't be read) +/// stays as it is, said once: a repair of the project (a layer file added, maps imported) doesn't +/// start either over. pub(crate) fn read_in_place(m: &mut Map) { let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m .layers @@ -1152,7 +1153,7 @@ pub(crate) fn read_in_place(m: &mut Map) { .filter_map(|l| { let name = l.name.clone(); let rl = l.raster.as_mut()?; - (rl.in_place && rl.read_failure().is_none() && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed)) + (rl.in_place && rl.error.is_none() && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed)) .then_some((name, rl)) }) .collect(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 649505c6b..a4cc16088 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -15569,6 +15569,97 @@ fn an_in_place_layers_failed_overview_is_read_again_when_its_wait_is_over() { assert_eq!(said(&s), 1, "{:?}", s.notices); } +/// A cloud-optimised GeoTIFF added without waiting before its header is in (the browser, where +/// nothing may wait, so an address isn't probed first as it is natively) answers that it is still +/// being read, with no size yet rather than none; the size comes with the header (see +/// `a_cog_added_without_waiting_lands_as_the_map_draws`). +#[test] +fn a_cog_added_before_its_header_is_in_says_it_is_being_read() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; + let gate = std::sync::Arc::new(AtomicBool::new(false)); + let (base, _) = serve_gated(rustgis_data::raster::cog::example(512, 256), true, Some(gate.clone())); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + s.in_place_wait = false; + // As the browser adds it: straight in place, its header read with its overview. + let r = crate::cmd::layer::add_cog(&mut s, &json!({"name": "Scene"}), &url).unwrap(); + assert_eq!((r["inPlace"].clone(), r["reading"].clone()), (json!(true), json!(true)), "{r}"); + assert!(r["width"].is_null() && r["height"].is_null() && r["bands"].is_null(), "sized before its header is in: {r}"); + assert!(r.get("bounds").is_none(), "{r}"); + // Its header and overview arrive as the map draws. + gate.store(true, Ordering::Relaxed); + let layer = |s: &Session| s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + let view = rustgis_geom::Extent::new(10.0, 45.0, 11.0, 46.0); + let start = std::time::Instant::now(); + while layer(&s).data.is_none() && start.elapsed() < Duration::from_secs(20) { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + s.map_mut().sync_sources(&view, 1000.0, false); + s.notice_landed_rasters(); + } + let rl = layer(&s); + assert_eq!(rl.data.as_ref().map(|d| (d.width, d.band_count())), Some((512, 1)), "{:?}", rl.error); +} + +/// A raster layer read in place whose read can't start (a project edited by hand keeps an address +/// that can't be read) says so once, and a repair of the project (a layer file added, maps +/// imported) leaves it as it is: not started over, not said again. +#[test] +fn a_read_in_place_that_couldnt_start_isnt_started_over_by_a_repair() { + use rustgis_map::raster::RasterLayer; + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place("ftp://files.example.org/scene.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let layer = |s: &Session| s.project.maps[0].layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + let rl = layer(&s); + assert!(rl.data.is_none() && rl.read_failure().is_none(), "{rl:?}"); + let error = rl.error.clone().expect("couldn't start"); + assert!(error.contains("start with https://"), "{error}"); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + let mut other = rustgis_map::Map::new("Other"); + let id = other.add_dataset(crate::data::Dataset::new("Empty", crate::data::GeometryType::Point, Vec::new())); + crate::cmd::layer::add_layer_text(&mut s, &serde_json::to_string(other.layer(id).unwrap()).unwrap()).unwrap(); + s.run("map.import", &json!({"text": serde_json::to_string(&rustgis_map::Map::new("Other")).unwrap()})).unwrap(); + s.notice_landed_rasters(); + let rl = layer(&s); + assert_eq!((rl.error.as_deref(), rl.read_failure().is_none()), (Some(error.as_str()), true), "started over: {rl:?}"); + assert_eq!(said(&s), 1, "said again: {:?}", s.notices); +} + +/// Where the session waits for reads in place (the command line, Python), a failed overview is +/// read again by the next command run once its wait is over, and waited for: a long-lived session +/// that doesn't draw has nothing else to ask again. +#[test] +fn a_waiting_sessions_next_command_reads_a_failed_overview_again() { + use std::sync::atomic::Ordering; + use std::time::Duration; + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 1); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), rustgis_map::raster::RasterLayer::unread_in_place(format!("{base}/scene.tif"))); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let layer = |s: &Session| s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + assert_eq!(layer(&s).read_failure().map(|(_, _, failures)| failures), Some(1), "{:?}", layer(&s).error); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // A command run before its wait is over doesn't read it. + s.run("info.project", &json!({})).unwrap(); + assert!(layer(&s).data.is_none() && layer(&s).read_failure().is_some()); + assert_eq!(sent.load(Ordering::Relaxed), 0, "read again before its wait was over"); + // One run after does, waiting for it: the layer holds its overview as the command answers. + std::thread::sleep(Duration::from_millis(1100)); + s.run("info.project", &json!({})).unwrap(); + let rl = layer(&s); + assert!(rl.data.is_some(), "{:?}", rl.error); + assert!(rl.error.is_none() && rl.read_failure().is_none()); + assert!(sent.load(Ordering::Relaxed) > 0); + assert_eq!(said(&s), 1, "{:?}", s.notices); +} + /// Where the session doesn't wait for reads in place (the desktop app, whose commands run on the /// window's own thread), a cloud-optimised GeoTIFF is added at once, its header probed as the app /// does, and its overview lands as the map draws; one that can't be read is added too, and says so @@ -15584,6 +15675,13 @@ fn a_cog_added_without_waiting_lands_as_the_map_draws() { let r = s.run("layer.addUrl", &json!({"url": url, "name": "Scene"})).unwrap(); assert_eq!((r["inPlace"].as_bool(), r["kind"].as_str()), (Some(true), Some("raster")), "{r}"); assert_eq!(r["bounds"], json!([10.0, 45.0, 11.0, 46.0]), "framed on the header read: {r}"); + // Sized by the header read, as the answer is once the overview has landed (512 a side: held + // whole, so not an overview of anything). + assert_eq!( + (r["width"].clone(), r["height"].clone(), r["bands"].clone(), r["reading"].clone(), r.get("overviewOf")), + (json!(512), json!(512), json!(1), json!(true), None), + "{r}" + ); let layer = |s: &Session, name: &str| s.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); let rl = layer(&s, "Scene"); assert!(rl.in_place && rl.data.is_none() && rl.error.is_none(), "added before its overview is in: {rl:?}"); diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 91269ed35..e0a2cce16 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -1145,7 +1145,7 @@ impl RasterStyle { let n = r.band_count(); // A palette image's colours are of one band alone: in grey, as its tile layer drew it. let own = if matches!(r.color, ColorHint::Palette(_)) { "drawn in grey" } else { "drawn in its own look" }; - let past: Vec = look.bands.iter().filter(|b| **b >= n).map(|b| (b + 1).to_string()).collect(); + let past: Vec = look.bands.iter().filter(|b| **b >= n).map(|b| b.saturating_add(1).to_string()).collect(); if !past.is_empty() { return Some(format!("its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): {own}", past.join(", "))); } @@ -1638,6 +1638,14 @@ mod tests { assert!(rl.lost_unsaid().is_empty()); } + /// A project file may ask for any band number: the one past the last `usize` is said by + /// number, not overflowed. + #[test] + fn a_look_asking_for_the_band_past_the_last_usize_is_said() { + let lost = RasterStyle::look_lost(&dem(), &AddressLook { bands: vec![usize::MAX], range: None }).unwrap(); + assert!(lost.contains(&format!("band(s) {} (from 1)", usize::MAX)), "{lost}"); + } + #[test] fn defaults_and_colours() { let r = dem(); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 24f680e63..d9f96b223 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -83,7 +83,7 @@ impl Look { pub fn suffix(&self) -> String { let mut parts = Vec::new(); if !self.bands.is_empty() { - parts.push(format!("bands={}", self.bands.iter().map(|b| (b + 1).to_string()).collect::>().join(","))); + parts.push(format!("bands={}", self.bands.iter().map(|b| b.saturating_add(1).to_string()).collect::>().join(","))); } if let Some((lo, hi)) = self.range { parts.push(format!("range={lo},{hi}")); diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs index 03dfcbc23..0555506c6 100644 --- a/crates/tiles/src/cog/raster.rs +++ b/crates/tiles/src/cog/raster.rs @@ -270,7 +270,7 @@ pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands return Err("the view is too large a part of the GeoTIFF to read at once".into()); } if let Some(b) = bands.iter().find(|b| **b >= info.layout.bands.max(1)) { - return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b + 1)); + return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b.saturating_add(1))); } // The chunks the window needs: those kept (marked as used, so they stay), and the others // asked for. @@ -527,6 +527,17 @@ mod tests { assert_eq!(overview_level(&info), None); } + /// A project file may ask a window for any band number (its layer's renderer): the one past + /// the last `usize` is refused by number, not overflowed. + #[test] + fn a_window_of_the_band_past_the_last_usize_is_refused() { + let file = "test://past-last-band.tif"; + let info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); + super::super::infos().lock().unwrap().insert(file.to_string(), super::super::State::Ready(Arc::new(info))); + let e = window(file, 0, (0, 16), (0, 16), &[usize::MAX], ColorHint::Gray).unwrap_err(); + assert_eq!(e, format!("the GeoTIFF has 1 bands, not {}", usize::MAX)); + } + /// A window's missing tiles are filled from the level kept, but not its own no-data pixels. #[test] fn stand_ins_fill_only_the_tiles_missing() { diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 2c0f7b424..b3477dee3 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -132,7 +132,9 @@ the overview (a large image saved without overviews) isn't added: GDAL's COG dri makes the overviews. A tile a sparse file leaves out is empty. A tile, the file's header or the overview a layer holds, that fails to arrive is asked for again a minute later, or when its server says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again -then. An overview that fails again waits twice as long each time, up to an hour. +then. An overview that fails again waits twice as long each time, up to an hour. On the command +line and in Python, which don't draw, the next command run once the wait is over reads a failed +overview again, waiting for it as the project did. Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, and isn't written without them; in the web app, export again once the map has drawn the image. diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 3e33ef619..d24efa062 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -988,7 +988,9 @@ descriptions GDAL and rasterio write inside a GeoTIFF (its `GDAL_METADATA` tag) too, a sidecar's taking precedence. Exporting a raster opened from an overview writes that overview; the results of adding and of exporting it (`layer.addData`, `share.exportRaster`) give the full size as `overviewOf`, so scripts -and the MCP server see it too. +and the MCP server see it too. A GeoTIFF read in place that is added before its overview is in +(the desktop app, the browser) answers `"reading": true`, sized by its header when that is in, and +with `null` for its size and bands until then. `share.exportRaster` writes an untiled GeoTIFF; with `"cog": true` it writes a cloud-optimised one: 512-pixel tiles compressed with Deflate, overviews (each half the size of the last, down to one tile), and every directory ahead of the tiles, so a reader fetches the header once and then only the From fb449b182370fce9a5efd0e80401cb702ae88f9b Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Fri, 6 Nov 2026 16:12:04 -0330 Subject: [PATCH 26/27] fix(engine): a migrated COG tile layer keeps its connection, a palette series draws its table, and the browser export says what to do CodeRabbit's review of the merge of main: - `cog_tiles_to_rasters` copies the tile layer's `connection` and `anonymous` onto the raster layer it becomes, so a project reopened reads it through the connection it recorded (#1223) rather than asking to choose among those matching its origin. - `Info::palette` reads `layout.palette`, the table the header keeps apart from its colour hint since the window read (#1204): the tiles of a palette file's own look are coloured by it, as `paint` always could. `with_stretch` answers that look without a read and without keeping a stretch (a look of its bands would take it for its own). The notice for a dated series or a cube no longer says its tiles are grey. - The browser exporter's error, and both guides, say the export asked for the tiles still missing and to export again once they are in: a map draw doesn't read what the export needs beyond the view. - The two tile-cache tests that arm or clear the process-wide alarm hold a test-only lock. Tests: `a_palette_images_own_look_has_its_table`; `cog_tile_layers_of_older_projects_open_as_rasters` checks the connection and anonymity carried. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/layer.rs | 15 +++------ crates/engine/src/cmd/tiles.rs | 6 +++- crates/engine/src/tests.rs | 12 +++++-- crates/render/src/raster_export.rs | 5 ++- crates/tiles/src/cache.rs | 9 ++++++ crates/tiles/src/cog.rs | 51 +++++++++++++++++++++++++----- docs/user-guide/analysis.md | 5 +-- docs/user-guide/data-formats.md | 4 ++- 8 files changed, 82 insertions(+), 25 deletions(-) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index a44a72c39..3c6dcd530 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -3646,20 +3646,15 @@ pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool, s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)); } r["bounds"] = json!(info.bounds); - // Drawn in place, the tiles are drawn as values: what the header holds beside them (a - // colour table, band and class names) comes only with a window, and is said. + // Drawn in place, the tiles are drawn as values (a palette image's by its colour table): + // what the header holds beside them (band and class names) comes only with a window, and + // is said. let l = &info.layout; let named = l.described.as_ref().is_some_and(|d| d.bands.iter().any(|b| b.description.is_some() || !b.categories.is_empty())); let mut said: Vec = l.lost.clone(); said.extend(l.described.iter().flat_map(|d| d.lost.iter().map(|x| format!("{}: {x}", rustgis_data::raster::pam::GDAL_METADATA)))); - // Grey only for a palette file: an RGB file with band names is drawn in its colours. - match (l.palette.is_some(), named) { - (true, true) => { - said.push("drawn in place in grey, without its colour table and band or class names: a window (\"bbox\") reads them".into()) - } - (true, false) => said.push("drawn in place in grey, without its colour table: a window (\"bbox\") reads it".into()), - (false, true) => said.push("drawn in place without its band or class names: a window (\"bbox\") reads them".into()), - (false, false) => {} + if named { + said.push("drawn in place without its band or class names: a window (\"bbox\") reads them".into()); } // A tile layer's own name, as added. let shown = r.get("name").and_then(Value::as_str).unwrap_or(name).to_string(); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 8140f64d0..985a0a270 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -564,7 +564,7 @@ fn blend(p: &Value) -> Result, String> { } /// Turns a map's plain COG tile layers into raster layers read in place, each where it was drawn, -/// with its name, visibility, opacity, blend and look: those of projects saved before GeoTIFFs read +/// with its name, visibility, opacity, blend, look, connection and anonymity: those of projects saved before GeoTIFFs read /// in place were raster layers, and those `map.addTiles` (kind `cog`) still adds, as a project or /// a map file opens, and then only: a tile layer added this session stays one until then (a /// plain GeoTIFF is one kind of layer, however it was added; what a tile layer had that a raster @@ -618,6 +618,10 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { l.visible = t.visible; l.transparency = (1.0 - t.opacity.clamp(0.0, 1.0)) * 100.0; l.blend = t.blend; + // Read again through the saved connection it came from, as the tile layer was; one an + // anonymous plugin command added takes no saved login, as before. + l.connection = t.connection.clone(); + l.anonymous = t.anonymous; } // Just over the layer it was drawn over, in its group; under them all without one. let group = t.over.and_then(|o| m.group_of(o).map(|g| g.id)); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 75f932fb0..75bd4c1c1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -17043,8 +17043,8 @@ fn in_place_layers_say_their_overview_once() { } /// A project saved when cloud-optimised GeoTIFFs read in place were tile layers opens them as -/// raster layers read in place, each where it was drawn, with its name, visibility, opacity and -/// look; a dated series stays a tile layer. +/// raster layers read in place, each where it was drawn, with its name, visibility, opacity, +/// look, connection and anonymity; a dated series stays a tile layer. #[test] fn cog_tile_layers_of_older_projects_open_as_rasters() { use rustgis_data::raster::cog::{Example, example_with}; @@ -17060,6 +17060,10 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { l.attribution = "© Example".into(); l.min_zoom = 5; l.overlay = true; + l.connection = "conn-scene".into(); + } + if name == "Aerial" { + l.anonymous = true; } let mut t = serde_json::to_value(l).unwrap(); t["opacity"] = json!(opacity); @@ -17085,6 +17089,10 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); assert!(!scene.visible && (scene.opacity() - 0.5).abs() < 1e-6); + // Read again through the connection it came from; an anonymous layer stays one. + assert_eq!((scene.connection.as_str(), scene.anonymous), ("conn-scene", false)); + let aerial = t.map().layers.iter().find(|l| l.name == "Aerial").unwrap(); + assert!(aerial.anonymous && aerial.connection.is_empty(), "{aerial:?}"); // Drawn under the map's layers, as they were, Scene over Aerial. let names: Vec<&str> = t.map().layers.iter().map(|l| l.name.as_str()).collect(); assert_eq!(names[names.len() - 2..], ["Scene", "Aerial"], "{names:?}"); diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index f3defbce2..b23c50d8b 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -89,7 +89,10 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> // An archive of the overview where the image is wanted isn't written. let left = deadline.saturating_duration_since(web_time::Instant::now()); if cfg!(target_arch = "wasm32") { - return Err(format!("{}: its tiles are still being read: export again once the map has drawn it", r.name)); + return Err(format!( + "{}: its tiles are still being read (this export asked for the rest): export again once they are in", + r.name + )); } if left.is_zero() || idle >= IDLE_WAITS { return Err(format!("{}: its tiles didn't all arrive, so the archive isn't written", r.name)); diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 39c77e155..2cfea8842 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -2431,6 +2431,7 @@ mod tests { /// stores it: the fetcher is one for the process, and another test's may be in. #[test] fn a_tile_refused_with_retry_after_has_the_map_drawn_again_then() { + let _alarm = alarm_test_lock().lock().unwrap_or_else(|e| e.into_inner()); let within = |secs: u64| { let at = ALARM.load(Ordering::Relaxed); at != 0 && at <= (Instant::now() + Duration::from_secs(secs)).saturating_duration_since(alarm_base()).as_millis() as u64 + 1 @@ -2451,12 +2452,20 @@ mod tests { /// once that time has come. #[test] fn the_map_draws_again_when_a_wait_is_over() { + let _alarm = alarm_test_lock().lock().unwrap_or_else(|e| e.into_inner()); let before = generation(); notify_at(Instant::now() + Duration::from_millis(20)); std::thread::sleep(std::time::Duration::from_millis(40)); assert!(generation() > before); } + /// Held by the tests that arm or read the process-wide alarm: they run in parallel threads of + /// one process, and one's clearing would erase another's. + fn alarm_test_lock() -> &'static std::sync::Mutex<()> { + static L: std::sync::OnceLock> = std::sync::OnceLock::new(); + L.get_or_init(std::sync::Mutex::default) + } + #[test] fn a_decoder_set_for_pictures_decodes_them() { let mut pm = Pixmap::new(3, 2).unwrap_or_else(|| panic!("pixmap")); diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index d9f96b223..a5b17481e 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -216,10 +216,7 @@ impl Info { /// The colours of a palette image's values, when its own look shows them. fn palette(&self, look: &Look) -> Option<&[[u8; 4]]> { - match &self.layout.color { - ColorHint::Palette(table) if look.bands.is_empty() && look.range.is_none() && !look.aef => Some(table), - _ => None, - } + self.layout.palette.as_deref().filter(|_| look.bands.is_empty() && look.range.is_none() && !look.aef) } /// The bands' (low, high) values for a look, when known. @@ -616,7 +613,6 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box layout.bands == 3, _ => false, }; - let palette = matches!(layout.color, ColorHint::Palette(_)); let bytes = layout.data_type == "UInt8"; let info = Arc::new(Info { layout, @@ -631,10 +627,11 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box, file: String, look: Look, then: StretchThen) { then(Ok(s)); return; } + // A palette image's own look colours its values by its table (see `paint`): no stretch to + // read, and none kept, which a look of its bands would take for its own. + if info.palette(&look).is_some() { + then(Ok(vec![(0.0, 255.0)])); + return; + } let colour = info.colour(&look); let coarsest = stats_level(&info.layout.levels); let n = info.layout.levels.get(coarsest).map(|l| l.tiles() as usize).unwrap_or(0); @@ -1498,6 +1501,38 @@ pub fn read_values_counted_blocking( #[cfg(test)] mod tests { + /// A palette image's own look colours its values by the table the header keeps apart from its + /// colour hint (`Layout::palette`), with no stretch read or kept; a look of its bands doesn't. + #[test] + fn a_palette_images_own_look_has_its_table() { + use rustgis_data::raster::cog::{Parse, Sparse, example, layout}; + use std::sync::{Arc, Mutex}; + let Ok(Parse::Done(mut layout)) = layout(&Sparse::default().with(0, example(64, 16))) else { panic!("not a COG") }; + layout.palette = Some(vec![[1, 2, 3, 255]; 256]); + let transformer = layout.crs.transformer().unwrap(); + let info = Arc::new(super::Info { + layout: *layout, + header: Sparse::default(), + transformer, + bounds: [0.0; 4], + merc: [0.0; 4], + min_zoom: 0, + max_zoom: 0, + stretch: Mutex::default(), + stretching: Mutex::default(), + stretch_failed: Mutex::default(), + alpha: None, + }); + let own = super::Look::default(); + assert_eq!(info.palette(&own).map(<[[u8; 4]]>::len), Some(256)); + assert!(info.palette(&super::Look { bands: vec![0], ..super::Look::default() }).is_none()); + let (tx, rx) = std::sync::mpsc::channel(); + super::with_stretch(info.clone(), "test://palette.tif".into(), own.clone(), Box::new(move |r| tx.send(r).unwrap_or_default())); + let stretch = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("answered at once").unwrap(); + assert_eq!(stretch, vec![(0.0, 255.0)]); + assert!(info.stretch_for(&own).is_none(), "kept, which a look of its bands would take for its own"); + } + /// A joint read whose tiles the fetch queue keeps dropping ends, saying so, rather than /// asking without end: within eight drops a tile (64 at least). #[test] diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index 6b7be533f..80b530e6f 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -181,8 +181,9 @@ within one world. Turn off dynamic range adjustment and display rotation first. export is limited to 512 tiles and 64 MiB of compressed tile data; reduce the extent or zoom range if it exceeds a limit. Areas beyond the Web Mercator latitude limit are clipped. A cloud-optimised GeoTIFF read in place is exported from its full-resolution tiles: the desktop waits for them (two -minutes at most), and in the web app, which can't wait, export again once the map has drawn it. -Without them nothing is written. +minutes at most); the web app can't wait, so an export whose tiles aren't all in says so and asks +for the rest: export again once they are in (a large export may take a few tries). Without them +nothing is written. **Vector to PMTiles** (`vectorToPmtiles`) writes a single-layer MVT archive in standard Web Mercator. Choose `minZoom` and `maxZoom` from 0 to 16; `sourceLayer` defaults to the input diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 3850a27b4..7d249c621 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -136,7 +136,9 @@ then. An overview that fails again waits twice as long each time, up to an hour. line and in Python, which don't draw, the next command run once the wait is over reads a failed overview again, waiting for it as the project did. Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, -and isn't written without them; in the web app, export again once the map has drawn the image. +and isn't written without them; the web app can't wait, so an export whose tiles aren't all in +says so and asks for the rest: export again once they are in (a large archive may take a few +tries). Its first look is what the file's PhotometricInterpretation tag says: From 54a24d702fe29b07abd8fc63d775d11da982d214 Mon Sep 17 00:00:00 2001 From: Jerry Rumbolt Date: Fri, 6 Nov 2026 18:25:35 -0330 Subject: [PATCH 27/27] fix(tiles, engine): every wait asked for has the map drawn again, a draw before it asks again, and a project an older RustGIS saved again reads its COG in place - The tile cache keeps every wait asked for (the earliest 1,024), not the earliest alone: the draw when an earlier wait is over may not revisit a later read's gate (a style kept whole while its elevation source waits). `generation()` takes out every wait that is over at once and asks the app again for the earliest left on each draw, since egui's `request_repaint_after` lasts one frame and a draw before the deadline used it up. Tests for both; the tile test takes the cache's turn too. - The engine's header retry test asserts what the session sees (the add fails saying when to ask again, nothing is sent for the refusal, and a draw once the wait is over reads the header again, answered), with no alarm hook installed; that no read starts while the wait lasts, and that the alarm is armed, is checked in the tiles crate, where the gate's state and the alarm are read under the cache's turn and the alarm tests' turn. - A project an older RustGIS saved again (it didn't know `in_place`: it showed the layer as missing and saved it without the key) reads its cloud-optimised GeoTIFF in place again: a raster layer whose path is a plain web address, from no saved connection, is read no other way. - The COG tile-layer migration says a legend is dropped, with the attribution, bounds, overlay and zoom range. - A login made again reads a raster layer read in place again without waiting for it where the session doesn't wait (the desktop), so it doesn't hold the window; a test pins that a COG added from a saved connection records it, its address bound to the login, and follows the login. - The guide says a file without a small enough image is refused where the session waits, and added, saying why, elsewhere. Co-Authored-By: Claude Fable 5.1 --- crates/engine/src/cmd/tiles.rs | 3 + crates/engine/src/project.rs | 23 +++ crates/engine/src/service_logins.rs | 7 +- crates/engine/src/tests.rs | 76 +++++---- crates/engine/src/tests/service_logins.rs | 72 ++++++++- crates/map/src/raster.rs | 6 +- crates/tiles/src/cache.rs | 186 +++++++++++++++++----- crates/tiles/src/cog.rs | 27 ++++ docs/user-guide/data-formats.md | 5 +- 9 files changed, 320 insertions(+), 85 deletions(-) diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index 985a0a270..8d7193152 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -594,6 +594,9 @@ pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { if !t.attribution.trim().is_empty() { dropped.push("attribution"); } + if !t.legends.is_empty() { + dropped.push("legend"); + } if (t.min_zoom, t.max_zoom) != (plain.min_zoom, plain.max_zoom) { dropped.push("zoom range"); } diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index 4742b905e..b4e55eedc 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -588,6 +588,13 @@ pub(crate) fn resolve_file_paths<'a>(dir: &str, paths: impl IntoIterator bool { + let scheme = path.get(..8).map(str::to_ascii_lowercase).unwrap_or_default(); + scheme.starts_with("https://") || scheme.starts_with("http://") +} + /// Read only compatibility metadata before interpreting any saved payload. #[derive(Deserialize)] struct FileFormat { @@ -899,6 +906,22 @@ impl Project { // layer added this session stays one until the project is opened again. if opened { crate::cmd::tiles::cog_tiles_to_rasters(m); + // A raster layer whose path is a plain web address, from no saved connection, is + // read in place, the one way such a path is read (a login or S3 wrapper is + // downloaded whole, as is a connection's address, plain here until its login is + // made again): one saved by an older RustGIS, which didn't know `in_place` (it + // showed the layer as missing and saved it again without the key), reads in + // place again here, losing a pending look at most. + for l in &mut m.layers { + if l.connection.is_empty() + && let Some(rl) = &mut l.raster + && !rl.in_place + && rl.embedded.is_none() + && rl.path.as_deref().is_some_and(plain_web_address) + { + rl.in_place = true; + } + } } m.tiles.truncate(rustgis_map::tiles::MAX_TILE_LAYERS); #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/service_logins.rs b/crates/engine/src/service_logins.rs index f539b570c..01dbca38e 100644 --- a/crates/engine/src/service_logins.rs +++ b/crates/engine/src/service_logins.rs @@ -416,6 +416,7 @@ fn layer_address(l: &rustgis_map::Layer) -> Option<&str> { fn rebind_connection(s: &mut Session, c: &Connection) -> usize { let binding = s.web_logins.active.get(&c.id).map(|a| a.binding.as_str()); let origin = login_origin(c); + let wait = s.in_place_wait; let mut failed = 0usize; let mut rewrite = |address: &mut String| { let (prefix, raw) = crate::sources::template_archive(address).unwrap_or(("", address.as_str())); @@ -477,8 +478,10 @@ fn rebind_connection(s: &mut Session, c: &Connection) -> usize { && let Some(path) = raster.path.as_mut() && rewrite(path) { - // Reload keeps the style and publishes any failure through raster.error. - let _ = raster.reload(); + // Reload keeps the style and publishes any failure through raster.error. A layer + // read in place lands as the map draws where the session doesn't wait for reads + // in place (the desktop), so a login made again doesn't hold the window. + let _ = if raster.in_place && !wait { raster.begin_reload() } else { raster.reload() }; } if let Some(path) = &mut l.dataset.source { rewrite(path); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 75bd4c1c1..7f3edea40 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -16295,49 +16295,37 @@ fn a_tile_a_sparse_file_leaves_out_is_empty_not_waited_for() { let _ = std::fs::remove_file(png); } -/// A GeoTIFF whose server refuses its header with a 429 or 503 and a `Retry-After` isn't asked -/// for again before then, and the map is asked to draw again by then (as for a tile), so the -/// layer doesn't stay failed until something else draws the map. +/// A GeoTIFF whose server refuses its header with a 429 or 503 and a `Retry-After` fails to be +/// added, saying when to ask again, and is read again on a draw once the wait is over. (That it +/// isn't asked for before then, and that the map is asked to draw again by then, so the layer +/// doesn't stay failed until something else draws it: `rustgis_tiles::cog`'s +/// `a_header_refused_with_retry_after_has_the_map_drawn_again_then`, where the gate's state and +/// the alarm can be read; here another test may empty the cache meanwhile.) #[test] fn a_header_refused_with_retry_after_is_read_again_when_it_says() { use crate::render::rustgis_tiles::{cache, cog}; - use std::io::{Read, Write}; - use std::sync::{Arc, Mutex}; + use std::sync::atomic::Ordering; use std::time::Duration; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = listener.local_addr().unwrap(); - std::thread::spawn(move || { - for stream in listener.incoming() { - let Ok(mut s) = stream else { continue }; - let mut buf = [0u8; 4096]; - let _ = s.read(&mut buf); - let _ = s.write_all(b"HTTP/1.1 503 Service Unavailable\r\nRetry-After: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); - } - }); - // The app's alarm: each wait asked for is noted. - let asked = Arc::new(Mutex::new(Vec::new())); - let noted = asked.clone(); - cache::set_alarm(move |d| noted.lock().unwrap().push(d)); - let url = format!("http://{addr}/scene.tif"); + // Refused once (busy: ask again in a second), answered after. + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 1); + let url = format!("{base}/scene.tif"); let mut s = Session::new(Project::default()); let e = s.run("layer.addUrl", &json!({"url": url.clone(), "name": "Scene"})).unwrap_err(); assert!(e.contains("retry after 1 s"), "{e}"); - // Not asked for again before then (the error stands), and the map is asked to draw again by - // then: on the failure, and on each draw while it waits (another alarm may have fired since). - let mut seen = None; - for _ in 0..30 { + assert_eq!(sent.load(Ordering::Relaxed), 0, "refused: nothing sent"); + // Its wait over, it is read again on a draw, and answered. + let start = std::time::Instant::now(); + let mut read = None; + while read.is_none() && start.elapsed() < Duration::from_secs(10) { let _ = cache::generation(); - // The error stands until then; its wait over, it is read again. - if let Err(e) = cog::info(&url) { - assert!(e.contains("retry after 1 s"), "{e}"); - } - seen = asked.lock().unwrap().iter().find(|d| **d <= Duration::from_secs(1)).copied(); - if seen.is_some() { - break; + if let Ok(Some(info)) = cog::info(&url) { + read = Some(info); } cache::wait(Duration::from_millis(50)); } - assert!(seen.is_some(), "no alarm within a second: {:?}", asked.lock().unwrap()); + let width = read.as_ref().and_then(|i| i.layout.levels.first()).map(|l| l.width); + assert_eq!(width, Some(512), "not read again once its wait was over"); + assert!(sent.load(Ordering::Relaxed) > 0); } /// A GeoTIFF's address look (`#bands=…&range=…`) is its layer's once its pixels land, read later @@ -16969,6 +16957,25 @@ fn an_in_place_layer_that_cant_be_read_says_so() { assert_eq!(said(&s), 1); } +/// A project an older RustGIS saved again (it didn't know `in_place`: the layer was missing there, +/// and saved without the key) reads its cloud-optimised GeoTIFF in place again: a raster layer +/// whose path is a plain web address is read no other way. +#[test] +fn a_project_an_older_rustgis_saved_again_reads_its_cog_in_place() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), rustgis_map::raster::RasterLayer::unread_in_place(format!("{base}/scene.tif"))); + let mut text: Value = serde_json::to_value(&s.project).unwrap(); + // As the older RustGIS saved it: the path, without `in_place`. + let raster = text.pointer_mut("/maps/0/layers/0/raster").and_then(Value::as_object_mut).unwrap(); + assert_eq!(raster.remove("in_place"), Some(json!(true))); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text.to_string()})).unwrap(); + let rl = s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + assert!(rl.in_place && rl.error.is_none(), "{:?}", rl.error); + assert_eq!(rl.data.as_ref().map(|d| d.width), Some(512)); +} + /// A palette COG without its colour table is drawn in grey and says so, also when its pixels land /// after the layer is added (a project opened, as in the browser and an import): once, and not for /// one with its table. @@ -17055,11 +17062,12 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { let mut text: Value = serde_json::to_value(&s.project).unwrap(); let tile = |name: &str, url: String, opacity: f64, visible: bool| { let mut l = rustgis_map::TileLayer::new(name, rustgis_map::TileKind::Cog, url); - // Scene's credit, zoom range and overlay flag: what a raster layer hasn't. + // Scene's credit, zoom range, overlay flag and legend: what a raster layer hasn't. if name == "Scene" { l.attribution = "© Example".into(); l.min_zoom = 5; l.overlay = true; + l.legends = vec![rustgis_data::ogc::Legend { url: format!("{base}/legend.png"), min_scale: None, max_scale: None }]; l.connection = "conn-scene".into(); } if name == "Aerial" { @@ -17098,7 +17106,7 @@ fn cog_tile_layers_of_older_projects_open_as_rasters() { assert_eq!(names[names.len() - 2..], ["Scene", "Aerial"], "{names:?}"); assert_eq!(t.map().tiles.iter().map(|x| x.name.as_str()).collect::>(), ["Series"]); // Scene's settings a raster layer hasn't are said to be dropped, once; Aerial had none. - let said = |t: &Session| t.notices.iter().filter(|n| n.text.contains("which have no attribution, overlay, zoom range")).count(); + let said = |t: &Session| t.notices.iter().filter(|n| n.text.contains("which have no attribution, legend, overlay, zoom range")).count(); assert_eq!(said(&t), 1, "{:?}", t.notices.last()); assert!(t.notices.iter().any(|n| n.text.contains("not kept for Scene") && !n.text.contains("Aerial")), "{:?}", t.notices.last()); t.run("map.zoomIn", &Value::Null).unwrap(); diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index b700d1f54..2fb107c60 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -44,7 +44,17 @@ struct LoginFile { #[cfg(not(target_arch = "wasm32"))] impl LoginFile { + /// A server that sends the whole file, whatever is asked for (as before ranges were answered). fn new(body: Vec) -> Self { + Self::with(body, false) + } + + /// A server that answers a range asked for with that part (a file read in place asks so). + fn ranged(body: Vec) -> Self { + Self::with(body, true) + } + + fn with(body: Vec, ranges: bool) -> Self { use std::io::{Read, Write}; use std::sync::atomic::{AtomicBool, Ordering}; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); @@ -76,10 +86,30 @@ impl LoginFile { let request = String::from_utf8_lossy(&request).into_owned(); let authorized = request.to_ascii_lowercase().contains("\r\nx-key: fixture-login-757\r\n") || request.split_whitespace().nth(1).is_some_and(|p| p.starts_with("/public")); + // A range asked for (a file read in place): that part. + let range = + request.lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("range: bytes=").map(str::to_string)).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + let last = body.len().checked_sub(1)?; + if a.is_empty() { + let n = b.parse::().ok()?.min(body.len()); + return Some((body.len() - n, last)); + } + Some((a.parse::().ok()?, b.parse::().ok()?.min(last))) + }); + let range = range.filter(|_| ranges); + let head = request.to_ascii_lowercase().starts_with("head "); log.lock().unwrap().push(request); - let (status, bytes) = if authorized { ("200 OK", body.as_slice()) } else { ("401 Unauthorized", &[][..]) }; + let (status, bytes) = match (authorized, range) { + (false, _) => ("401 Unauthorized", &[][..]), + (true, Some((a, b))) if a <= b => ("206 Partial Content", &body[a..=b]), + (true, _) => ("200 OK", body.as_slice()), + }; let _ = write!(stream, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", bytes.len()); - let _ = stream.write_all(bytes); + // A HEAD request is answered with the length alone. + if !head { + let _ = stream.write_all(bytes); + } } }); Self { base, requests, stop, worker: Some(worker) } @@ -1401,3 +1431,41 @@ fn delayed_head_refusal_keeps_secrets_out_of_history_across_login_lifecycle() { } } } + +/// A cloud-optimised GeoTIFF added by address from a saved web-service connection is a raster +/// layer read in place that records the connection, as the tile layer it was did, its address +/// bound to the connection's login (never carrying the key) and read with it; the login made +/// again, its address is bound to the new login and it is read again through it. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_cog_read_in_place_from_a_connection_records_it_and_follows_its_login() { + let server = LoginFile::ranged(rustgis_data::raster::cog::example(512, 256)); + let mut s = session(None); + let id = new(&mut s, json!({"kind":"ogcapi","address":format!("{}/collections",server.base),"auth":"header","keyName":"X-Key"})); + s.run("connection.login", &json!({"id":id,"apiKey":"fixture-login-757","allowInsecure":true})).unwrap(); + let r = s.run("layer.addUrl", &json!({"url":format!("{}/scene.tif",server.base),"name":"Scene","connection":id})).unwrap(); + assert_eq!((r["inPlace"].as_bool(), r["width"].as_u64()), (Some(true), Some(512)), "{r}"); + let path = { + let layer = &s.map().layers[0]; + let raster = layer.raster.as_ref().unwrap(); + assert!(raster.in_place && raster.data.is_some(), "{:?}", raster.error); + assert_eq!(layer.connection, id); + raster.path.clone().unwrap() + }; + assert!(rustgis_data::login_address::is_wrapper(&path) && !path.contains("fixture-login-757"), "{path}"); + let carried = |server: &LoginFile| { + let requests = server.requests.lock().unwrap(); + !requests.is_empty() && requests.iter().all(|r| r.to_ascii_lowercase().contains("\r\nx-key: fixture-login-757\r\n")) + }; + assert!(carried(&server), "{:?}", server.requests.lock().unwrap()); + // Logged in again (a new binding): the layer's address follows it, and it is read through it. + server.requests.lock().unwrap().clear(); + s.run("connection.login", &json!({"id":id,"apiKey":"fixture-login-757","allowInsecure":true})).unwrap(); + let layer = &s.map().layers[0]; + let raster = layer.raster.as_ref().unwrap(); + let again = raster.path.clone().unwrap(); + assert!(again != path && rustgis_data::login_address::is_wrapper(&again), "{again}"); + assert!(raster.data.is_some() && raster.error.is_none(), "{:?}", raster.error); + assert_eq!(layer.connection, id); + assert!(carried(&server), "{:?}", server.requests.lock().unwrap()); +} diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index dbbeb0fd1..8d9235125 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -539,9 +539,9 @@ impl RasterLayer { Some(d) } - /// The layer, opened from a tile layer whose `dropped` settings (`"attribution"`, `"zoom - /// range"`, `"bounds"`) a raster layer hasn't, so they aren't kept: said once, by - /// [`RasterLayer::dropped_unsaid`]. + /// The layer, opened from a tile layer whose `dropped` settings (`"attribution"`, `"legend"`, + /// `"zoom range"`, `"bounds"`, `"overlay"`) a raster layer hasn't, so they aren't kept: said + /// once, by [`RasterLayer::dropped_unsaid`]. pub fn from_tiles_dropping(mut self, dropped: Vec<&'static str>) -> RasterLayer { self.dropped = dropped; self diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 2cfea8842..481ef2503 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -10,7 +10,7 @@ //! map stopped asking for is dropped unfetched ([`crate::schedule`]). Natively they are fetched on //! worker threads; the browser app installs its own fetcher with [`set_fetcher`]. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; @@ -191,58 +191,124 @@ pub fn set_waker(f: impl Fn() + Send + Sync + 'static) { *waker().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); } -/// When the map is next to draw again by itself (a failed read may be tried again then), in -/// milliseconds after [`alarm_base`] plus one; 0 for none. -static ALARM: AtomicU64 = AtomicU64::new(0); +/// When the map is next to draw again by itself (a failed read may be tried again then): each +/// wait asked for ([`notify_at`]), in milliseconds after [`alarm_base`] plus one, the earliest +/// first. Every wait is kept (the earliest [`MAX_ALARMS`]), not the earliest alone: the draw when +/// it is over may not ask again for a later one (a style kept whole while its elevation source +/// waits). +fn alarms() -> &'static Mutex> { + static A: OnceLock>> = OnceLock::new(); + A.get_or_init(Mutex::default) +} + +/// The most waits kept at once: a later one past them all is left to its read, which asks again +/// on each draw while it waits. +const MAX_ALARMS: usize = 1024; fn alarm_base() -> Instant { static B: OnceLock = OnceLock::new(); *B.get_or_init(Instant::now) } +/// `when` as the alarms keep it. +fn alarm_at(when: Instant) -> u64 { + (when.saturating_duration_since(alarm_base()).as_millis() as u64).saturating_add(1) +} + fn alarm_hook() -> &'static Mutex>> { static A: OnceLock>>> = OnceLock::new(); A.get_or_init(Mutex::default) } /// Sets how the app is asked to draw again after a while (a repaint after it): a read that failed -/// is tried again once its wait is over, though nothing else draws the map. +/// is tried again once its wait is over, though nothing else draws the map. The request lasts +/// one frame (egui's `request_repaint_after`): it is made again on each draw while the wait +/// lasts (see [`generation`]). pub fn set_alarm(f: impl Fn(Duration) + Send + Sync + 'static) { *alarm_hook().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); } -/// Draws the map again at `when` (the earliest asked for is kept): [`generation`] goes up then, so -/// a map waiting on a failed read asks for it again. -pub fn notify_at(when: Instant) { - let at = (when.saturating_duration_since(alarm_base()).as_millis() as u64).saturating_add(1); - let mut current = ALARM.load(Ordering::Relaxed); - loop { - if current != 0 && current <= at { - return; - } - match ALARM.compare_exchange_weak(current, at, Ordering::Relaxed, Ordering::Relaxed) { - Ok(_) => break, - Err(c) => current = c, - } - } +/// Asks the app to draw again after `after` (see [`set_alarm`]). +fn ring(after: Duration) { let hook = alarm_hook().lock().unwrap_or_else(|e| e.into_inner()).clone(); if let Some(hook) = hook { - hook(when.saturating_duration_since(Instant::now())); + hook(after); + } +} + +/// Draws the map again at `when`: [`generation`] goes up then, so a map waiting on a failed read +/// asks for it again. The app is asked for the earliest wait pending; each draw asks it again +/// for the earliest left (see [`generation`]). +pub fn notify_at(when: Instant) { + let at = alarm_at(when); + let earliest = { + let mut a = alarms().lock().unwrap_or_else(|e| e.into_inner()); + if a.len() >= MAX_ALARMS { + // Full: a wait past every one kept is left to its read; an earlier one takes the + // latest's place. + if a.last().is_some_and(|last| at >= *last) { + return; + } + a.pop_last(); + } + a.insert(at); + a.first() == Some(&at) + }; + if earliest { + ring(when.saturating_duration_since(Instant::now())); } } -/// Goes up each time a fetch finishes: a map drawn at an older generation may now draw more. +/// Goes up each time a fetch finishes, and once a wait asked for ([`notify_at`]) is over: a map +/// drawn at an older generation may now draw more. Read as the map draws, it asks the app to draw +/// again at the earliest wait still pending: the app's request lasts one frame (egui's +/// `request_repaint_after`), and a draw before the wait was over (the pointer moved) used it up. pub fn generation() -> u64 { - let at = ALARM.load(Ordering::Relaxed); - if at != 0 - && (alarm_base().elapsed().as_millis() as u64).saturating_add(1) >= at - && ALARM.compare_exchange(at, 0, Ordering::Relaxed, Ordering::Relaxed).is_ok() - { - GENERATION.fetch_add(1, Ordering::Relaxed); + let now = alarm_at(Instant::now()); + let next = { + let mut a = alarms().lock().unwrap_or_else(|e| e.into_inner()); + if a.first().is_some_and(|at| *at <= now) { + // Every wait over is one draw: the generation goes up once for them all. + let later = a.split_off(&now.saturating_add(1)); + *a = later; + GENERATION.fetch_add(1, Ordering::Relaxed); + } + a.first().copied() + }; + if let Some(at) = next { + ring(Duration::from_millis(at.saturating_sub(now))); } GENERATION.load(Ordering::Relaxed) } +/// How long until the earliest wait pending; none when there is none. +#[cfg(test)] +pub(crate) fn next_alarm() -> Option { + let at = alarms().lock().unwrap_or_else(|e| e.into_inner()).first().copied()?; + Some(Duration::from_millis(at.saturating_sub(alarm_at(Instant::now())))) +} + +/// Forgets every wait pending. +#[cfg(test)] +pub(crate) fn clear_alarms() { + alarms().lock().unwrap_or_else(|e| e.into_inner()).clear(); +} + +/// Undoes [`set_alarm`]: a test done with its own. +#[cfg(test)] +pub(crate) fn clear_alarm() { + *alarm_hook().lock().unwrap_or_else(|e| e.into_inner()) = None; +} + +/// A test's turn at the process-wide alarms, held by the tests that arm, read or clear them, or +/// set what rings: they run in parallel threads of one process, and one's clearing would erase +/// another's. +#[cfg(test)] +pub(crate) fn alarm_turn() -> std::sync::MutexGuard<'static, ()> { + static L: Mutex<()> = Mutex::new(()); + L.lock().unwrap_or_else(|e| e.into_inner()) +} + /// How long a server that refused with 429 or 503 asked to be left before it is asked again, as /// the error's text says it (`http status: 503 (retry after 30 s)`; see /// [`crate::failure::with_retry_after`]). @@ -2431,39 +2497,75 @@ mod tests { /// stores it: the fetcher is one for the process, and another test's may be in. #[test] fn a_tile_refused_with_retry_after_has_the_map_drawn_again_then() { - let _alarm = alarm_test_lock().lock().unwrap_or_else(|e| e.into_inner()); - let within = |secs: u64| { - let at = ALARM.load(Ordering::Relaxed); - at != 0 && at <= (Instant::now() + Duration::from_secs(secs)).saturating_duration_since(alarm_base()).as_millis() as u64 + 1 - }; + // Its turn at the cache (the entry is planted in it, and another test's clearing would + // drop it) and at the alarms. + let _turn = super::turn(); + let _alarm = super::alarm_turn(); + let within = |secs: u64| super::next_alarm().is_some_and(|d| d <= Duration::from_secs(secs)); let url = "test://cache/busy.json"; store(url, Entry::Failed("http status: 503 (retry after 2 s)".into(), Instant::now())); - ALARM.store(0, Ordering::Relaxed); + super::clear_alarms(); let State::Failed(e) = get(url, Want::Json) else { panic!("not failed") }; assert!(e.contains("retry after 2 s"), "{e}"); - assert!(within(2), "no alarm on a draw while waiting: {}", ALARM.load(Ordering::Relaxed)); + assert!(within(2), "no alarm on a draw while waiting: {:?}", super::next_alarm()); // Again on the next draw: the alarm fired (or was consumed) before the wait was over. - ALARM.store(0, Ordering::Relaxed); + super::clear_alarms(); assert!(matches!(get(url, Want::Json), State::Failed(_))); - assert!(within(2), "not armed again: {}", ALARM.load(Ordering::Relaxed)); + assert!(within(2), "not armed again: {:?}", super::next_alarm()); } /// A read that failed asks the map to draw again when it may be tried: the generation goes up /// once that time has come. #[test] fn the_map_draws_again_when_a_wait_is_over() { - let _alarm = alarm_test_lock().lock().unwrap_or_else(|e| e.into_inner()); + let _alarm = super::alarm_turn(); let before = generation(); notify_at(Instant::now() + Duration::from_millis(20)); std::thread::sleep(std::time::Duration::from_millis(40)); assert!(generation() > before); } - /// Held by the tests that arm or read the process-wide alarm: they run in parallel threads of - /// one process, and one's clearing would erase another's. - fn alarm_test_lock() -> &'static std::sync::Mutex<()> { - static L: std::sync::OnceLock> = std::sync::OnceLock::new(); - L.get_or_init(std::sync::Mutex::default) + /// Every wait asked for has the map drawn again when it is over, not the earliest alone: the + /// draw when the earlier is over may not ask for the later again (a style kept whole while + /// its elevation source waits), so the later is kept, and the app asked for it then. + #[test] + fn a_later_wait_isnt_lost_behind_an_earlier_one() { + let _alarm = super::alarm_turn(); + let asked = std::sync::Arc::new(Mutex::new(Vec::new())); + let noted = asked.clone(); + super::set_alarm(move |after| noted.lock().unwrap_or_else(|e| e.into_inner()).push(after)); + super::clear_alarms(); + let before = generation(); + notify_at(Instant::now() + Duration::from_millis(20)); + notify_at(Instant::now() + Duration::from_millis(150)); + std::thread::sleep(std::time::Duration::from_millis(50)); + let earlier = generation(); + assert!(earlier > before, "the earlier wait over"); + // The later is still pending, and asked of the app as the earlier was over. + assert!(super::next_alarm().is_some_and(|d| d <= Duration::from_millis(150)), "{:?}", super::next_alarm()); + let later = |asked: &Vec| asked.iter().any(|d| (Duration::from_millis(30)..=Duration::from_millis(150)).contains(d)); + assert!(later(&asked.lock().unwrap_or_else(|e| e.into_inner())), "{:?}", asked.lock().unwrap_or_else(|e| e.into_inner())); + std::thread::sleep(std::time::Duration::from_millis(150)); + assert!(generation() > earlier, "the later wait over"); + super::clear_alarm(); + } + + /// A draw before a wait is over asks the app for it again: the app's request lasts one frame, + /// and a draw since (the pointer moved) used it up. + #[test] + fn a_draw_before_a_wait_is_over_asks_for_it_again() { + let _alarm = super::alarm_turn(); + let asked = std::sync::Arc::new(Mutex::new(Vec::new())); + let noted = asked.clone(); + super::set_alarm(move |after| noted.lock().unwrap_or_else(|e| e.into_inner()).push(after)); + super::clear_alarms(); + notify_at(Instant::now() + Duration::from_millis(300)); + asked.lock().unwrap_or_else(|e| e.into_inner()).clear(); + let _ = generation(); + let again = asked.lock().unwrap_or_else(|e| e.into_inner()).iter().any(|d| *d <= Duration::from_millis(300)); + assert!(again, "{:?}", asked.lock().unwrap_or_else(|e| e.into_inner())); + super::clear_alarms(); + super::clear_alarm(); } #[test] diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index a5b17481e..f1dc6fe9a 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -1831,4 +1831,31 @@ mod tests { forget_file_in(&mut t, "https://x.org/a.tif"); assert_eq!((t.samples, t.order.len(), t.map.len()), (0, 0, 0)); } + + /// A file's header refused with a `Retry-After` (429 or 503) isn't asked for again before + /// then (the failure stands, no read starts), and the map is asked to draw again by then, on + /// each draw while it waits (as a tile's is), so a layer whose header failed doesn't stay + /// failed until something else draws the map. (That the file is read again then: the + /// engine's `a_header_refused_with_retry_after_is_read_again_when_it_says`.) + #[test] + fn a_header_refused_with_retry_after_has_the_map_drawn_again_then() { + // Its turn at the cache (another test's clearing would forget the failure) and the alarms. + let _turn = crate::cache::turn(); + let _alarm = crate::cache::alarm_turn(); + let file = "test://cog/busy.tif"; + let refused = || super::State::Failed("http status: 503 (retry after 2 s)".into(), Instant::now()); + super::infos().lock().unwrap().insert(file.to_string(), refused()); + crate::cache::clear_alarms(); + let within = || crate::cache::next_alarm().is_some_and(|d| d <= Duration::from_secs(2)); + let waiting = || matches!(super::infos().lock().unwrap().get(file), Some(super::State::Failed(..))); + let Err(e) = super::info(file) else { panic!("read while waiting") }; + assert!(e.contains("retry after 2 s"), "{e}"); + assert!(waiting(), "asked for again before its wait was over"); + assert!(within(), "no alarm on a draw while waiting: {:?}", crate::cache::next_alarm()); + // Again on the next draw: the alarm fired (or was consumed) before the wait was over. + crate::cache::clear_alarms(); + assert!(super::info(file).is_err() && waiting()); + assert!(within(), "not armed again: {:?}", crate::cache::next_alarm()); + super::infos().lock().unwrap().remove(file); + } } diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 7d249c621..e78555ba9 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -127,8 +127,9 @@ layer at once and reads its overview as the map draws (the Details pane says so so a slow host doesn't hold the window; the command line and Python wait for it, up to two minutes, so a command's result is complete. A saved S3 connection chosen for such layers (Choose S3 connection…) counts them as still being read until their overview is in, apart from what it -opened. A file whose smallest image is too large to hold as -the overview (a large image saved without overviews) isn't added: GDAL's COG driver or `gdaladdo` +opened. A file whose smallest image is too large to hold as the overview (a large image saved +without overviews) is refused where the session waits for it (the command line, Python); the desktop +app and the browser add the layer, which says why it can't be read. GDAL's COG driver or `gdaladdo` makes the overviews. A tile a sparse file leaves out is empty. A tile, the file's header or the overview a layer holds, that fails to arrive is asked for again a minute later, or when its server says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again