diff --git a/apps/rustgis-web/src/web.rs b/apps/rustgis-web/src/web.rs index c02f452dc..e80740dae 100644 --- a/apps/rustgis-web/src/web.rs +++ b/apps/rustgis-web/src/web.rs @@ -570,6 +570,17 @@ async fn unless_stalled( Some(raced) } +/// A refusal's text (`HTTP 503`), with how long the server asked to be left when it says so in a +/// `Retry-After` header the browser lets the page read (its CORS rules must expose it). +fn refused(resp: &web_sys::Response) -> String { + let wait = resp.headers().get("retry-after").ok().flatten(); + rustgis_ui_egui::rustgis_engine::render::rustgis_tiles::failure::with_retry_after( + format!("HTTP {}", resp.status()), + resp.status(), + wait.as_deref(), + ) +} + /// Downloads `url`, or the byte range it asks for (a PMTiles archive's part: see /// `rustgis_tiles::cache::split_range`). A status in `omit` is an expected answer (a tile past a /// map's edge): its failure isn't recorded, only noted once per server, as on the desktop. A @@ -670,7 +681,7 @@ async fn fetch_bytes(url: &str, capture: Capture, omit: &'static [u16]) -> Resul .map_err(err)?; diagnostics.answered(resp.status()); if !resp.ok() { - return Err(format!("HTTP {}", resp.status())); + return Err(refused(&resp)); } // The last bytes asked for and the whole file answered: fine when it is no longer. let whole_tail = tail.filter(|_| resp.status() != 206); @@ -793,7 +804,7 @@ async fn post_json(url: &str, body: &str, capture: Capture) -> Result, S .map_err(err)?; diagnostics.answered(resp.status()); if !resp.ok() { - return Err(format!("HTTP {}", resp.status())); + return Err(refused(&resp)); } let buf = wasm_bindgen_futures::JsFuture::from(resp.array_buffer().map_err(err)?).await.map_err(err)?; let bytes = js_sys::Uint8Array::new(&buf); @@ -861,7 +872,7 @@ async fn head(url: &str, capture: Capture) -> Result eframe::Result { .collect(); // Installed plugins, before the first session lists commands and tools. let plugin_errors = rustgis_engine::cmd::plugins::load_installed(); - let app = if args.iter().any(|a| a == "--sample") || files.is_empty() { + let mut app = if args.iter().any(|a| a == "--sample") || files.is_empty() { RustGisApp::with_sample() } else { RustGisApp::new(Session::new(Project::default())) }; + // Commands run on the window's own thread: a raster read in place lands as the map draws + // rather than holding the window for a slow host. + app.session.in_place_wait = false; let mut services = prefs::services(); // Autosave, in a recovery folder only the copy holding the run marker uses: a second copy // running alongside would offer, or overwrite, the first one's unsaved work. diff --git a/crates/data/src/raster/cog.rs b/crates/data/src/raster/cog.rs index cf7db8b8d..ae9769855 100644 --- a/crates/data/src/raster/cog.rs +++ b/crates/data/src/raster/cog.rs @@ -333,6 +333,9 @@ fn read_layout(r: Sparse) -> Result { // then is a colour table read, as the whole-file read does; a grey image's stray one isn't. // The full image's word, which the window and the drawing read. let palette_image = rd.first_is_palette(); + // An RGB image with more bands (shown to the decoder as grey, see `Unpaletted`): every band, + // the first three red, green and blue. + let wide_rgb = rd.first_is_wide_rgb(); let mut dec = Decoder::new(rd).map_err(err)?.with_limits(limits()); let Some(geo) = super::geotiff::cog_geo(&mut dec)? else { return Ok(Parse::NotTiled("a rotated image, or one without coordinates, is read whole".into())); @@ -346,6 +349,9 @@ fn read_layout(r: Sparse) -> Result { (3, ColorHint::Rgb { alpha: false }) } ColorType::RGBA(_) => (4, ColorHint::Rgb { alpha: true }), + ColorType::Multiband { num_samples, .. } if wide_rgb && num_samples >= 3 => { + (num_samples as usize, ColorHint::Rgb { alpha: super::geotiff::extra_alpha(&mut dec) }) + } ColorType::Multiband { num_samples, .. } => { let color = super::geotiff::multiband_color(&mut dec, num_samples); (num_samples.max(1) as usize, color) @@ -613,6 +619,11 @@ pub struct Example { pub int8: bool, /// Stored bottom up: a positive Y pixel size, the first row the southernmost (at 45° N). pub bottom_up: bool, + /// Its PhotometricInterpretation: 1 grey, 2 RGB (the bands past the third extra samples); + /// `classes` makes a palette image (3) whatever this says. + pub photometric: u16, + /// The first extra sample (the band past the colour ones) is unassociated alpha. + pub alpha: bool, /// A palette image (one band of bytes) whose index `i` is the colour (i, 255 − i, 0), with /// classes named in GDAL's metadata as the MapBiomas files name theirs (`CLASS_3`, `CLASS_15`). pub classes: bool, @@ -628,6 +639,11 @@ pub struct Example { pub stray: bool, /// With `classes`: a palette image with no colour table. pub no_table: bool, + /// The full image's first tile left out, as GDAL's sparse files leave out tiles of no-data + /// (no bytes, no offset). + pub sparse: bool, + /// Without overviews: the full image alone (a file saved without them). + pub no_overviews: bool, } impl Default for Example { @@ -639,12 +655,16 @@ impl Default for Example { planar: false, int8: false, bottom_up: false, + photometric: 1, + alpha: false, classes: false, lossy: false, wide: false, huge: false, stray: false, no_table: false, + sparse: false, + no_overviews: false, } } } @@ -657,7 +677,7 @@ pub fn example_with(e: &Example) -> Vec { let planes = if e.planar { nb } else { 1 }; let per_chunk = nb / planes; let mut sizes = vec![size]; - while *sizes.last().unwrap_or(&tile) > tile { + while !e.no_overviews && *sizes.last().unwrap_or(&tile) > tile { sizes.push(sizes.last().unwrap_or(&tile).div_ceil(2)); } let value = |x: u32, y: u32, b: u32| -> u8 { @@ -695,6 +715,12 @@ pub fn example_with(e: &Example) -> Vec { chunks.push(c); } } + if e.sparse + && k == 1 + && let Some(first) = chunks.first_mut() + { + first.clear(); + } chunks }) .collect(); @@ -737,16 +763,23 @@ pub fn example_with(e: &Example) -> Vec { ]), ), (259, V::S(vec![1])), - (262, V::S(vec![if e.classes && !(e.stray && i == 0) { 3 } else { 1 }])), + (262, V::S(vec![if e.classes && !(e.stray && i == 0) { 3 } else { e.photometric }])), (277, V::S(vec![nb as u16])), (284, V::S(vec![if e.planar { 2 } else { 1 }])), (322, V::L(vec![tile])), (323, V::L(vec![tile])), (324, V::L(vec![0; n])), - (325, V::L(vec![chunk_len; n])), + (325, V::L(images.get(i).map_or(vec![chunk_len; n], |c| c.iter().map(|c| c.len() as u32).collect()))), ]; - if nb > 1 { - t.push((338, V::S(vec![0; nb as usize - 1]))); + // The samples past the colour ones (three of an RGB image, else one): the first alpha + // when asked, the others undefined. + let colours = if e.photometric == 2 { 3 } else { 1 }; + if nb as usize > colours { + let mut extra = vec![0u16; nb as usize - colours]; + if e.alpha { + extra[0] = 2; + } + t.push((338, V::S(extra))); } if e.int8 || e.wide { t.push((339, V::S(vec![2; nb as usize]))); @@ -802,14 +835,16 @@ pub fn example_with(e: &Example) -> Vec { let mut data_at = at; let mut out = vec![b'I', b'I', 42, 0]; out.extend_from_slice(&header_len.to_le_bytes()); - for t in ifds.iter_mut() { + for (lvl, t) in ifds.iter_mut().enumerate() { for (tag, v) in t.iter_mut() { if *tag == 324 && let V::L(offs) = v { - for o in offs.iter_mut() { - *o = data_at as u32; - data_at += chunk_len as usize; + for (j, o) in offs.iter_mut().enumerate() { + // A chunk left out has no offset, as GDAL writes a sparse file. + let len = images.get(lvl).and_then(|c| c.get(j)).map_or(0, Vec::len); + *o = if len == 0 { 0 } else { data_at as u32 }; + data_at += len; } } } @@ -1034,4 +1069,92 @@ pub(crate) mod tests { } let _ = layout(&Sparse::default().with(0, b"II*\0\xff\xff\xff\x7f".to_vec())); } + + #[test] + fn every_band_whatever_the_photometric_interpretation_says() { + let want = |b: u32| ((300 + 10 * b) % 256) as f32; + let e = Example { size: 300, tile: 128, ..Example::default() }; + // Grey, as remote sensing images mostly are: eight bands (a WorldView multispectral image), + // interleaved or stored apart; a panchromatic one; a pan band with alpha. + for planar in [false, true] { + let (l, v) = pixel(&example_with(&Example { bands: 8, planar, ..e }), 150, 150); + assert_eq!((l.bands, &l.color), (8, &ColorHint::Gray)); + assert_eq!(v, (0..8).map(want).collect::>()); + } + let (l, v) = pixel(&example_with(&Example { bands: 1, ..e }), 150, 150); + assert_eq!((l.bands, &l.color, v), (1, &ColorHint::Gray, vec![want(0)])); + let (l, v) = pixel(&example_with(&Example { bands: 2, alpha: true, ..e }), 150, 150); + assert_eq!((l.bands, &l.color, v), (2, &ColorHint::GrayAlpha, vec![want(0), want(1)])); + // RGB: three bands, with alpha, and with bands past the colour ones (RGB and near-infrared, + // seven bands), which the decoder alone leaves out or refuses: every band, the first three + // red, green and blue. + for (bands, alpha) in [(3, false), (4, true), (4, false), (7, false), (5, true)] { + for planar in [false, true] { + let file = example_with(&Example { bands, alpha, planar, photometric: 2, ..e }); + let (l, v) = pixel(&file, 150, 150); + assert_eq!((l.bands, &l.color), (usize::from(bands), &ColorHint::Rgb { alpha }), "{bands} bands, alpha {alpha}"); + assert_eq!(v, (0..u32::from(bands)).map(want).collect::>(), "{bands} bands, alpha {alpha}, planar {planar}"); + } + } + // A palette image: its values, drawn grey, with its colour table apart. + let (l, v) = pixel(&example_with(&Example { classes: true, ..e }), 150, 150); + let table = l.palette.as_ref().unwrap(); + assert_eq!((&l.color, table.len(), table[44], v), (&ColorHint::Gray, 256, [44, 211, 0, 255], vec![want(0)])); + } + + /// A GeoTIFF file is read as the same image read in place is: every band, whatever its + /// PhotometricInterpretation says, coloured alike. + #[test] + fn files_have_every_band_a_cog_has() { + let e = Example { size: 300, tile: 128, ..Example::default() }; + for (bands, alpha, photometric) in [(8, false, 1), (2, true, 1), (4, true, 2), (4, false, 2), (7, false, 2), (5, true, 2)] { + for planar in [false, true] { + let file = example_with(&Example { bands, alpha, planar, photometric, ..e }); + let (l, v) = pixel(&file, 150, 150); + let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "f").unwrap(); + let read: Vec = (0..r.band_count()).map(|b| r.value(b, 150, 150).unwrap()).collect(); + assert_eq!((r.band_count(), &r.color, read), (l.bands, &l.color, v), "{bands} bands, photometric {photometric}"); + } + } + } + + /// A file holding an RGB image with more bands and a grey overview: the full image's word + /// goes, whichever image is read, as a palette file's does. + #[test] + fn a_file_mixing_grey_and_rgb_images_goes_by_the_full_image() { + let e = Example { size: 300, tile: 128, bands: 4, photometric: 2, ..Example::default() }; + let mut file = example_with(&e); + // The coarsest overview's PhotometricInterpretation (tag 262, one short) from RGB to grey. + let entry = [0x06, 0x01, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00]; + let at: Vec = (0..file.len().saturating_sub(entry.len())).filter(|&i| file[i..i + entry.len()] == entry).collect(); + assert_eq!(at.len(), 3, "the full image and its overviews"); + file[at[2] + 8] = 1; + let (l, _) = pixel(&file, 150, 150); + assert_eq!((l.bands, &l.color), (4, &ColorHint::Rgb { alpha: false })); + let r = crate::raster::geotiff::read(std::io::Cursor::new(&file), "mixed").unwrap(); + assert_eq!((r.band_count(), &r.color), (4, &ColorHint::Rgb { alpha: false })); + let small = crate::raster::geotiff::read_limited( + crate::raster::geotiff::reader(std::io::Cursor::new(&file)).unwrap(), + "mixed", + 30_000, + None, + ) + .unwrap(); + assert_eq!((small.width, small.band_count(), &small.color), (75, 4, &ColorHint::Rgb { alpha: false })); + } + /// Pixel (x, y)'s value in every band of an example's full image, read a tile at a time. + fn pixel(file: &[u8], x: u32, y: u32) -> (Box, Vec) { + let (s, l) = layout_of(file); + let lv = &l.levels[0]; + let tile = (y / lv.tile_height) * lv.across() + x / lv.tile_width; + let at = ((y % lv.tile_height) * lv.tile_width + x % lv.tile_width) as usize; + let values = (0..l.bands) + .map(|b| { + let (chunk, i) = l.chunk(lv, tile, b).unwrap(); + let (o, n) = (lv.offsets[chunk as usize] as usize, lv.counts[chunk as usize] as usize); + decode_tile(&s, &l, 0, chunk, file[o..o + n].to_vec()).unwrap()[i][at] + }) + .collect(); + (l, values) + } } diff --git a/crates/data/src/raster/geotiff.rs b/crates/data/src/raster/geotiff.rs index e1908fe77..b177ebadb 100644 --- a/crates/data/src/raster/geotiff.rs +++ b/crates/data/src/raster/geotiff.rs @@ -41,6 +41,12 @@ struct Geo { referenced: bool, } +/// Whether the first sample past the colour ones is declared alpha (extra sample 1, associated, or +/// 2, unassociated). +pub(super) fn extra_alpha(dec: &mut Decoder) -> bool { + dec.find_tag_unsigned_vec::(Tag::ExtraSamples).ok().flatten().is_some_and(|v| matches!(v.first(), Some(1 | 2))) +} + /// How `n` samples of a grey image the decoder calls multiband are shown: grey with alpha when /// there are two and the second is declared alpha (extra sample 1, associated, or 2, unassociated), /// else grey. GDAL marks the second band of an ordinary two-band file as 0, unspecified data. @@ -576,6 +582,9 @@ pub(crate) struct Unpaletted { patches: Vec<(u64, [u8; 2])>, /// Whether the first image (the full one) is a palette image. first_palette: bool, + /// Whether the first image is an RGB image with samples past the colour ones (shown to the + /// decoder as grey): its first three samples are red, green and blue, every sample read. + first_wide_rgb: bool, pos: u64, } @@ -583,13 +592,16 @@ pub(crate) struct Unpaletted { const MAX_IFDS: usize = 64; const MAX_ENTRIES: u64 = 4096; -/// Where the palette images' PhotometricInterpretation values are, walking the IFD chain -/// (classic and BigTIFF). A file it can't walk has none: the decoder reports what's wrong. -fn palette_patches(r: &mut R) -> (Vec<(u64, [u8; 2])>, bool) { +/// Where the palette images' PhotometricInterpretation values are, and those of RGB images with +/// samples past the colour ones (which the decoder would leave out, or refuse), walking the IFD +/// chain (classic and BigTIFF). A file it can't walk has none: the decoder reports what's wrong. +/// Returns the patches, whether the first image is a palette image, and whether it is such an RGB +/// image. +fn palette_patches(r: &mut R) -> (Vec<(u64, [u8; 2])>, bool, bool) { use std::io::SeekFrom; let mut patches = Vec::new(); - // Whether the first image is one of them. - let mut first = false; + // Whether the first image is one of them, and whether it is an RGB image with more samples. + let (mut first, mut first_wide) = (false, false); let mut walk = || -> Option<()> { let mut head = [0u8; 16]; r.seek(SeekFrom::Start(0)).ok()?; @@ -636,16 +648,35 @@ fn palette_patches(r: &mut R) -> (Vec<(u64, [u8; 2])>, bool) { } let mut entries = vec![0u8; usize::try_from(n * entry_len).ok()?]; r.read_exact(&mut entries).ok()?; + let value_at = if big { 12 } else { 8 }; + let grey = if little { [1, 0] } else { [0, 1] }; + // PhotometricInterpretation (262): where its value is, and the value; SamplesPerPixel + // (277); the first ExtraSamples (338) value, when it is inline. Each a SHORT, as the + // decoder itself requires of them. + let (mut photometric, mut samples, mut extra) = (None, 1u16, None); for (i, e) in entries.chunks(entry_len as usize).enumerate() { - // PhotometricInterpretation (262), a SHORT, its value inline. - if u16_(e)? == 262 && u16_(e.get(2..)?)? == 3 { - let value_at = if big { 12 } else { 8 }; - if u16_(e.get(value_at..)?)? == 3 { - let at = ifd + count_len + i as u64 * entry_len + value_at as u64; - patches.push((at, if little { [1, 0] } else { [0, 1] })); - first |= k == 0; + let short = u16_(e.get(2..)?)? == 3; + match u16_(e)? { + 262 if short => { + photometric = Some((ifd + count_len + i as u64 * entry_len + value_at as u64, u16_(e.get(value_at..)?)?)) } + 277 if short => samples = u16_(e.get(value_at..)?)?, + 338 if short => extra = u16_(e.get(value_at..)?), + _ => {} + } + } + match photometric { + Some((at, 3)) => { + patches.push((at, grey)); + first |= k == 0; + } + // An RGB image whose fourth sample isn't alpha, or with five or more: RGBI + // orthophotos, multispectral images. + Some((at, 2)) if samples > 4 || (samples == 4 && !matches!(extra, Some(1 | 2))) => { + patches.push((at, grey)); + first_wide |= k == 0; } + _ => {} } let mut next = [0u8; 8]; r.read_exact(&mut next[..next_len]).ok()?; @@ -654,7 +685,7 @@ fn palette_patches(r: &mut R) -> (Vec<(u64, [u8; 2])>, bool) { Some(()) }; let _ = walk(); - (patches, first) + (patches, first, first_wide) } /// Any reader, boxed: the decoder is compiled once for every kind of source (a file, bytes in @@ -676,13 +707,18 @@ impl Unpaletted { pub(crate) fn first_is_palette(&self) -> bool { self.first_palette } + + /// Whether the first image is an RGB image with more samples than its colours. + pub(crate) fn first_is_wide_rgb(&self) -> bool { + self.first_wide_rgb + } } impl Unpaletted { fn new(mut inner: R) -> std::io::Result { - let (patches, first_palette) = palette_patches(&mut inner); + let (patches, first_palette, first_wide_rgb) = palette_patches(&mut inner); inner.seek(std::io::SeekFrom::Start(0))?; - Ok(Unpaletted { inner, patches, first_palette, pos: 0 }) + Ok(Unpaletted { inner, patches, first_palette, first_wide_rgb, pos: 0 }) } } @@ -1026,6 +1062,7 @@ pub(super) fn read_limited_with_nodata( // COG window has it: a grey full image's stray colour table colours nothing, nor do palette // overviews of it. let palette = r.first_is_palette(); + let wide_rgb = r.first_is_wide_rgb(); let limits = { let mut l = Limits::default(); l.decoding_buffer_size = max_samples().saturating_mul(4); @@ -1162,6 +1199,11 @@ pub(super) fn read_limited_with_nodata( }; (1, table.map_or(ColorHint::Gray, ColorHint::Palette)) } + // An RGB image with more bands (shown to the decoder as grey, see `Unpaletted`): every + // band, the first three red, green and blue. + ColorType::Multiband { num_samples, .. } if wide_rgb && num_samples >= 3 => { + (num_samples as usize, ColorHint::Rgb { alpha: extra_alpha(&mut dec) }) + } ColorType::Multiband { num_samples, .. } => (num_samples.max(1) as usize, multiband_color(&mut dec, num_samples)), other => return Err(err(format!("{other:?} images aren't supported"))), }; @@ -2071,11 +2113,12 @@ mod tests { // columns down its west edge, the padding of each row's last tile spilt into the next row. let pixel = |b: u32, x: u32, y: u32| ((x + 3 * y + 60 * b) % 250 + 1) as u8; let bytes = rgbi_tiled(&[(40, 16), (20, 16)], pixel); + // Its near-infrared band is read too, the fourth, after red, green and blue. let check = |r: &Raster, k: u32| { - assert_eq!((r.band_count(), r.color.clone()), (3, ColorHint::Rgb { alpha: false })); + assert_eq!((r.band_count(), r.color.clone()), (4, ColorHint::Rgb { alpha: false })); for y in 0..r.height { for x in 0..r.width { - for b in 0..3 { + for b in 0..4 { let want = f32::from(pixel(b, x * k, y * k)); assert_eq!(r.value(b as usize, i64::from(x), i64::from(y)), Some(want), "band {b} at {x},{y} (k {k})"); } diff --git a/crates/data/src/raster/mod.rs b/crates/data/src/raster/mod.rs index e352f9698..4c0e6754d 100644 --- a/crates/data/src/raster/mod.rs +++ b/crates/data/src/raster/mod.rs @@ -517,6 +517,26 @@ impl Band { } } +/// The longest a raster read in place is waited for natively: its overview as a layer opens, its +/// tiles as it is exported. +pub const IN_PLACE_WAIT: std::time::Duration = std::time::Duration::from_secs(120); + +/// An alpha band's opaque value for a pixel type (`UInt16`: 65535): the type's largest, 1 for +/// floating point (as TIFF keeps it), 255 for bytes and anything else. +pub fn alpha_max(data_type: &str) -> f32 { + match data_type { + "Int8" => 127.0, + "Int16" => 32767.0, + "UInt16" => 65535.0, + "Int32" => 2_147_483_647.0, + "UInt32" => 4_294_967_295.0, + "Int64" => 9.223_372e18, + "UInt64" => 1.844_674_4e19, + "Float16" | "Float32" | "Float64" => 1.0, + _ => 255.0, + } +} + /// How a raster's bands were meant to be shown, from the file. #[derive(Clone, Debug, Default, PartialEq)] pub enum ColorHint { @@ -608,6 +628,39 @@ impl Raster { nodata: Option, color: ColorHint, data_type: impl Into, + ) -> Result { + Raster::build(name, width, height, bands, transform, crs, nodata, color, data_type, true) + } + + /// [`Raster::new`] without statistics (each band's are empty): a part of a larger raster, + /// drawn with the whole's (a cloud-optimised GeoTIFF's view, read in place). + #[allow(clippy::too_many_arguments)] + pub fn unmeasured( + name: impl Into, + width: u32, + height: u32, + bands: Vec>, + transform: GeoTransform, + crs: Crs, + nodata: Option, + color: ColorHint, + data_type: impl Into, + ) -> Result { + Raster::build(name, width, height, bands, transform, crs, nodata, color, data_type, false) + } + + #[allow(clippy::too_many_arguments)] + fn build( + name: impl Into, + width: u32, + height: u32, + bands: Vec>, + transform: GeoTransform, + crs: Crs, + nodata: Option, + color: ColorHint, + data_type: impl Into, + measure: bool, ) -> Result { let n = (width as usize).checked_mul(height as usize).ok_or_else(|| DataError::Format("raster too large".into()))?; if n == 0 || bands.is_empty() { @@ -625,7 +678,12 @@ impl Raster { let bands = bands .into_iter() .enumerate() - .map(|(i, data)| Band { name: format!("Band {}", i + 1), stats: BandStats::compute(&data, valid), data, classes: Vec::new() }) + .map(|(i, data)| Band { + name: format!("Band {}", i + 1), + stats: if measure { BandStats::compute(&data, valid) } else { BandStats::default() }, + data, + classes: Vec::new(), + }) .collect(); let mut r = Raster { name: name.into(), diff --git a/crates/engine/src/cmd/connections.rs b/crates/engine/src/cmd/connections.rs index ddcaace57..7bb5fb543 100644 --- a/crates/engine/src/cmd/connections.rs +++ b/crates/engine/src/cmd/connections.rs @@ -46,7 +46,7 @@ pub fn commands() -> Vec { r#"{"id": "a saved S3 connection's id or name"} (deletes stored keys; a login in this session keeps working)"#, ), C::new("connection.resolveS3", "Choose S3 Connection", "Home › Add › Data Sources", resolve_s3).view().params( - r#"{"source": "a pending project S3 source address", "id": "a candidate saved connection id"} → {id, reopened, failed}: explicitly grants this bucket and opens only its waiting sources"#, + r#"{"source": "a pending project S3 source address", "id": "a candidate saved connection id"} → {id, reopened, failed, reading}: explicitly grants this bucket and opens only its waiting sources; reading counts the rasters read in place whose overview is still being read (the desktop doesn't wait for it: it lands as the map draws, and one that fails is said then)"#, ), C::new("connection.forgetPassword", "Forget Saved Password", "Home › Add › Data Sources", forget_password).view().params(r#"{"id": "a database connection's id or name"}"#), C::new("connection.test", "Test Connection", "Home › Add › Data Sources", test).view().params( diff --git a/crates/engine/src/cmd/geolibre.rs b/crates/engine/src/cmd/geolibre.rs index 542f8ea27..a8d78fd02 100644 --- a/crates/engine/src/cmd/geolibre.rs +++ b/crates/engine/src/cmd/geolibre.rs @@ -157,6 +157,9 @@ fn import_project(s: &mut Session, p: &Value) -> CmdResult { report.approximated(string(g, "name"), "nested folder flattened; inherited visibility and opacity retained"); } } + // Its GeoTIFFs read in place, read together rather than one after another. + crate::project::read_in_place(s.map_mut()); + s.wait_in_place_reads(); order.reverse(); s.map_mut().layers.sort_by_key(|l| order.iter().position(|id| *id == l.id).unwrap_or(usize::MAX)); s.request_only = false; @@ -369,7 +372,7 @@ fn add_layer(s: &mut Session, r: &Value, dir: Option<&str>, report: &mut Report, return Ok(Added::Feature(id)); } if kind == "cog" && remote { - super::layer::add_cog(s, &q, &address, false)? + super::layer::add_cog_with(s, &q, &address, false, false)? } else if kind == "zarr" { if let Some(v) = src.get("variable") { q["variable"] = v.clone(); diff --git a/crates/engine/src/cmd/items.rs b/crates/engine/src/cmd/items.rs index 079a2b97f..07df243d0 100644 --- a/crates/engine/src/cmd/items.rs +++ b/crates/engine/src/cmd/items.rs @@ -447,6 +447,9 @@ fn import_maps(s: &mut Session, p: &Value) -> CmdResult { if let Some(d) = dir.as_deref().and_then(Path::to_str) { crate::project::resolve_file_paths(d, m.file_paths_mut()); } + // An older map file's plain COG tile layers open as raster layers read in place, as a + // project's do (a project imported had them opened as it was read). + super::tiles::cog_tiles_to_rasters(&mut m); for notice in crate::project::label_losses::map_notices(&text, &m) { s.notify(notice, false); } @@ -486,6 +489,7 @@ fn import_maps(s: &mut Session, p: &Value) -> CmdResult { super::connections::restore_active(s); super::connections::resolve_project(s); crate::s3_connections::resolve_project_s3(s); + s.wait_in_place_reads(); super::connections::note_waiting(s); if let Some(n) = names.last() && let Some(i) = s.project.maps.iter().position(|m| &m.name == n) diff --git a/crates/engine/src/cmd/layer.rs b/crates/engine/src/cmd/layer.rs index f34bd22e9..3c6dcd530 100644 --- a/crates/engine/src/cmd/layer.rs +++ b/crates/engine/src/cmd/layer.rs @@ -40,7 +40,7 @@ pub fn commands() -> Vec { .params(r#"{"layer"?: name|id, "crs": "EPSG:… or WKT; reinterpret native coordinates, without rewriting the source file"}"#), C::new("layer.loadAll", "Load All Features", "Layers", load_all).when(need_streamed).params(r#"{"layer"?: name|id}"#), C::new("layer.addUrl", "Add Data From URL", "Home › Add › Add Data", add_url).params( - r#"{"url": "https://…/data.geojson|.csv|.parquet|.fgb|.gpkg|.zip|.shp (its .dbf fetched beside it, and its .shx, .prj and .cpg when there)|.kml|.kmz|.gpx|.tif|.jp2 (JPEG 2000: desktop, through GDAL) (the format from the address, else the contents) | a .pmtiles archive", "name"?: text, "layer"?: "one layer of a zip, KML or GPX", "zoom"?: "bool: true frames everything added; false keeps the view, even on an empty map; omitted, the first content on an empty map is framed (a GeoParquet, FlatGeobuf or cloud-optimised GeoTIFF read in place likewise; a PMTiles or MBTiles archive ignores zoom and is framed on its bounds unless it covers the whole world)", "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "whole"?: "bool: download a cloud-optimised GeoTIFF whole (as a raster layer) instead of reading it in place (an address without a file name is read in place when its first bytes are a cloud-optimised GeoTIFF)", "cog"?: "bool: the address is a cloud-optimised GeoTIFF whatever its name says (the app's probe found it so)", "bbox"?: "[west, south, east, north] in degrees | "view": a cloud-optimised GeoTIFF's window as a raster layer (desktop and command line), read by range at full resolution, in the file's coordinate system, with its colour table and class names; at most 64 million values (all bands)", "cellSize"?: "with bbox: the coarsest overview whose cells are at most this size is read (the file's units; metres for longitude/latitude, a degree taken as 111.32 km)", "start"?, "end"?, "unit"?, "interval"?, "dates"?: "a GeoTIFF address with date placeholders ({date:YYYY}, {YYYY}{MM}{DD}), one file per date: the dates it spans and steps through (as time.set), turning the time slider on", "connection"?: "the id of the saved web-service connection it comes from (as service.add)", "anonymous"?: bool (internal: a plugin-origin download must never adopt a saved login), "request"?: "number (internal: the download a plugin command left to the app, #1202)"}"#, + r#"{"url": "https://…/data.geojson|.csv|.parquet|.fgb|.gpkg|.zip|.shp (its .dbf fetched beside it, and its .shx, .prj and .cpg when there)|.kml|.kmz|.gpx|.tif|.jp2 (JPEG 2000: desktop, through GDAL) (the format from the address, else the contents) | a .pmtiles archive", "name"?: text, "layer"?: "one layer of a zip, KML or GPX", "zoom"?: "bool: true frames everything added; false keeps the view, even on an empty map; omitted, the first content on an empty map is framed (a GeoParquet, FlatGeobuf or cloud-optimised GeoTIFF read in place likewise; a PMTiles or MBTiles archive ignores zoom and is framed on its bounds unless it covers the whole world)", "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "whole"?: "bool: download a cloud-optimised GeoTIFF whole (as a raster layer) instead of reading it in place (an address without a file name is read in place when its first bytes are a cloud-optimised GeoTIFF)", "cog"?: "bool: the address is a cloud-optimised GeoTIFF whatever its name says (the app's probe found it so)", "bbox"?: "[west, south, east, north] in degrees | "view": a cloud-optimised GeoTIFF's window as a raster layer (desktop and command line), read by range at full resolution, in the file's coordinate system, with its colour table and class names; at most 64 million values (all bands)", "cellSize"?: "with bbox: the coarsest overview whose cells are at most this size is read (the file's units; metres for longitude/latitude, a degree taken as 111.32 km)", "cube"?: "bool: a data cube's file: a tile layer that follows the time slider (stac.addCube adds them), kept one when the project opens again", "start"?, "end"?, "unit"?, "interval"?, "dates"?: "a GeoTIFF address with date placeholders ({date:YYYY}, {YYYY}{MM}{DD}), one file per date: the dates it spans and steps through (as time.set), turning the time slider on", "connection"?: "the id of the saved web-service connection it comes from (as service.add)", "anonymous"?: bool (internal: a plugin-origin download must never adopt a saved login), "request"?: "number (internal: the download a plugin command left to the app, #1202)"}"#, ), C::new("layer.addDownload", "Add Downloaded Data", "Home › Add", add_download) .params(r#"{"id": a download the app finished (Home › Add › Add Data › From URL), "url": its address, "name"?: text, "layer"?: text, "zoom"?: bool, "x"?, "y"?, "wkt"?, "crs"?, "delimiter"?, "paperSpace"?, "stream"?, "as"? (as layer.addData), "service"?: bool (a service's features: their axes (when they came as GeoJSON) and paging are checked, and the address is kept as the layer's source), "bounds"?: [west, south, east, north] (with service: the layer's longitude/latitude bounds from its capabilities, which its axes are checked against; without bounds they are left as they come; ignored without service), "coverage"?: bool (a WCS coverage: a GeoTIFF, else the service's error, and its request kept as the layer's source), "geolibreProject"?: bool (internal: downloaded project JSON), "geolibreLayer"?: {"token", "record"} (internal: reserved imported layer and its saved style), "geolibreZarr"?: bool (internal: an imported Zarr slice as GeoTIFF), "connection"?: "as layer.addUrl", "anonymous"?: bool (internal: preserves a plugin-origin download's anonymous reads), "request"?: number (internal: the download a plugin command left to the app, #1202)}"#), @@ -2833,6 +2833,7 @@ pub(crate) fn add_layer_text(s: &mut Session, text: &str) -> CmdResult { super::connections::clear_active(); s.project.normalize(); super::connections::restore_active(s); + s.wait_in_place_reads(); // Only the database the added layer reads can change here. let keys: Vec<_> = s.map().layer(id).and_then(|l| l.source.as_ref()).map(super::connections::layer_keys).unwrap_or_default(); super::connections::resolve_keys(s, &keys); @@ -3572,17 +3573,38 @@ pub fn is_unnamed_url(url: &str) -> bool { } /// Adds a cloud-optimised GeoTIFF by its address as a layer read in place (its tiles by range, -/// as the view needs them), framed when its header is in. `probed`: its header was just read -/// (natively), so it is waited for if something emptied the tile cache since (a login ended): the -/// layer would go unframed. One not read yet isn't waited for (a project opening its files). +/// as the view needs them), framed when its header is in and it is the first content on the map +/// (`"zoom"` says otherwise: `true` frames it, `false` keeps the view). `probed`: its header was +/// just read (natively), so it is waited for if something emptied the tile cache since (a login +/// ended): the layer would go unframed. One not read yet isn't waited for (a project opening its +/// files). pub(crate) fn add_cog(s: &mut Session, p: &Value, url: &str, probed: bool) -> CmdResult { + add_cog_with(s, p, url, true, probed) +} + +/// [`add_cog`], a GeoTIFF that can't be read now added all the same (unless `strict`), showing +/// why: a project being imported keeps its layers. +pub(crate) fn add_cog_with(s: &mut Session, p: &Value, url: &str, strict: bool, probed: bool) -> CmdResult { let stem = cog_name(url); let name = ps(p, "name").map(str::trim).filter(|n| !n.is_empty()).unwrap_or(&stem); let empty_before = super::map_is_empty(s); + // A GeoTIFF is a raster layer like a file's, read in place; a dated series, a data cube's file + // (which follow the time slider) and AlphaEarth's quantised embeddings (which raster styles + // don't dequantise) are tile layers. + let (file, mut look) = crate::render::rustgis_tiles::cog::split(url); + // A key given as `null` (a script's unused one) asks for nothing. + let given = |k: &str| p.get(k).filter(|v| !v.is_null()); + // A data cube's file stays a tile layer, its project opened again too: its address says so. + look.cube |= b(p, "cube") == Some(true); + let dated = rustgis_data::time::is_template(file) || ["start", "end", "unit", "interval", "dates"].iter().any(|k| given(k).is_some()); + if !dated && !look.aef && !look.cube { + return add_in_place(s, name, file, &look, strict, b(p, "zoom"), probed); + } + let url = crate::render::rustgis_tiles::cog::with_look(url, &look); let mut q = json!({"url": url, "kind": "cog", "name": name, "top": true}); // A dated address's span and steps (see map.addTiles). for k in ["start", "end", "unit", "interval", "dates", "connection"] { - if let Some(v) = p.get(k) { + if let Some(v) = given(k) { q[k] = v.clone(); } } @@ -3624,20 +3646,15 @@ pub(crate) fn add_cog(s: &mut Session, p: &Value, url: &str, probed: bool) -> Cm s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)); } r["bounds"] = json!(info.bounds); - // Drawn in place, the tiles are drawn as values: what the header holds beside them (a - // colour table, band and class names) comes only with a window, and is said. + // Drawn in place, the tiles are drawn as values (a palette image's by its colour table): + // what the header holds beside them (band and class names) comes only with a window, and + // is said. let l = &info.layout; let named = l.described.as_ref().is_some_and(|d| d.bands.iter().any(|b| b.description.is_some() || !b.categories.is_empty())); let mut said: Vec = l.lost.clone(); said.extend(l.described.iter().flat_map(|d| d.lost.iter().map(|x| format!("{}: {x}", rustgis_data::raster::pam::GDAL_METADATA)))); - // Grey only for a palette file: an RGB file with band names is drawn in its colours. - match (l.palette.is_some(), named) { - (true, true) => { - said.push("drawn in place in grey, without its colour table and band or class names: a window (\"bbox\") reads them".into()) - } - (true, false) => said.push("drawn in place in grey, without its colour table: a window (\"bbox\") reads it".into()), - (false, true) => said.push("drawn in place without its band or class names: a window (\"bbox\") reads them".into()), - (false, false) => {} + if named { + said.push("drawn in place without its band or class names: a window (\"bbox\") reads them".into()); } // A tile layer's own name, as added. let shown = r.get("name").and_then(Value::as_str).unwrap_or(name).to_string(); @@ -3660,6 +3677,105 @@ pub(crate) fn add_cog(s: &mut Session, p: &Value, url: &str, probed: bool) -> Cm Ok(r) } +/// Adds a cloud-optimised GeoTIFF as a raster layer read in place: its overview held (read now +/// where the session waits for reads in place, failing the command when it can't be; landing as +/// the map draws on the desktop and in the browser, see [`Session::in_place_wait`]), the part in +/// view read as the map draws. A look at the address's end (`#bands=…&range=…`) is its style. +/// `zoom` is the adding command's `"zoom"`: +/// `Some(true)` frames it, `Some(false)` keeps the view, `None` frames it only as the first content +/// on an empty map. `probed` is [`add_cog`]'s. +fn add_in_place( + s: &mut Session, + name: &str, + file: &str, + look: &crate::render::rustgis_tiles::cog::Look, + strict: bool, + zoom: Option, + probed: bool, +) -> CmdResult { + use crate::render::rustgis_tiles::cog; + // Judged as an address a project keeps is (a host, no spaces, not too long) before it is read. + rustgis_map::tiles::check_url(file)?; + let empty_before = super::map_is_empty(s); + // Strict, where the session waits: read now, failing when it can't be. Strict on the desktop: + // added now, its overview landing as the map draws (and failing there, said), so a slow host + // doesn't hold the window. Otherwise read with the other layers added with it (see + // `project::read_in_place`); in the browser, landing later. + #[cfg(not(target_arch = "wasm32"))] + let layer = if strict && s.in_place_wait { + rustgis_map::raster::RasterLayer::in_place(cog::raster::overview_blocking(file)?, file) + } else if strict { + rustgis_map::raster::RasterLayer::opening_in_place(file) + } else { + rustgis_map::raster::RasterLayer::unread_in_place(file) + }; + #[cfg(target_arch = "wasm32")] + let layer = { + let _ = strict; + rustgis_map::raster::RasterLayer::opening_in_place(file) + }; + let layer = layer.with_look(super::tiles::address_look(look)); + let landed = layer.data.is_some(); + let mut r = super::raster::add_raster_after(s, name, layer, zoom)?; + // A header just read (natively) is waited for if something emptied the tile cache since (a + // login ended): the layer would go unframed. A plugin's command in the app doesn't wait (it + // runs on the frame), as in the browser. + #[cfg(not(target_arch = "wasm32"))] + let info = if probed && !s.leave_downloads { cog::info_blocking(file).map(Some) } else { cog::info(file) }; + #[cfg(target_arch = "wasm32")] + let info = { + let _ = probed; + cog::info(file) + }; + // Its header couldn't be read (a failure recorded and not tried again yet, or now): the view + // stays where it was, and the answer says why (the layer says so too, once its read fails). + if let Err(e) = &info { + r["warning"] = json!(format!( + "{} couldn't be read ({}), so the map wasn't moved to it", + crate::sources::redact(file), + crate::sources::redact_text(e) + )); + } + if let Ok(Some(info)) = info { + let [w, so, e, n] = info.bounds; + // Framed on the file's bounds when it is the first content on the map, as other data is, + // or when asked to. + if zoom.unwrap_or(empty_before) { + s.zoom_to(&rustgis_geom::Extent::new(w, so, e, n)); + } + r["bounds"] = json!(info.bounds); + // Added before its overview is in: sized by the header as the answer would be once the + // overview had landed (the level the layer holds, every band, and the full size when the + // level isn't it). + if !landed && let Some(level) = cog::raster::overview_level(&info) { + let levels = &info.layout.levels; + if let Some(l) = levels.get(level) { + r["width"] = json!(l.width); + r["height"] = json!(l.height); + r["bands"] = json!(info.layout.bands); + } + if let Some(full) = levels.first().filter(|_| level > 0) { + r["overviewOf"] = json!([full.width, full.height]); + } + } + } + if !landed { + // Its overview is still being read (it lands as the map draws): a size the answer can't + // give yet (the header not in, or no level small enough to hold) is null, not 0. + r["reading"] = json!(true); + if r["width"].as_u64() == Some(0) { + for key in ["width", "height", "bands"] { + r[key] = Value::Null; + } + } + } + r["inPlace"] = json!(true); + r["kind"] = json!("raster"); + let added = r["layer"].as_u64().and_then(|id| u32::try_from(id).ok()).and_then(|id| s.map().layer(id)); + r["name"] = json!(added.map(|l| l.name.clone())); + Ok(r) +} + /// The most values a window of a cloud-optimised GeoTIFF holds, all its bands together: one read's /// most (256 MB of 32-bit values). Under test, 300,000, so the refusal is tried on a small file. #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index eee65c515..246fc4c40 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -174,16 +174,17 @@ fn reset(s: &mut Session, p: Project) { // nothing of the new project is fetched before. s.s3_pending.clear(); crate::s3_connections::resolve_project_s3(s); + // Its rasters read in place, begun together as it was read: waited for where the session + // waits (the command line, Python), landing as the map draws on the desktop. + s.wait_in_place_reads(); // What reading a raster left out (a colour table, names, a sidecar's scale) is said again as // the project opens, as when it was added. One still read in the background isn't here yet. - let lost: Vec<(String, String)> = s - .project - .maps - .iter() - .flat_map(|m| &m.layers) - .filter_map(|l| Some((&l.name, l.raster.as_ref()?.data.as_ref()?))) - .flat_map(|(name, d)| d.lost.iter().map(move |l| (name.clone(), l.clone()))) - .collect(); + let mut lost: Vec<(String, String)> = Vec::new(); + for l in s.project.maps.iter_mut().flat_map(|m| &mut m.layers) { + if let Some(rl) = &mut l.raster { + lost.extend(rl.lost_unsaid().into_iter().map(|what| (l.name.clone(), what))); + } + } super::raster::say_losses(s, lost); // A terrain saved on a DEM name two layers share draws the first of them: said, as the // project opens (its layers keep their names). diff --git a/crates/engine/src/cmd/raster.rs b/crates/engine/src/cmd/raster.rs index 57c525583..9c85fed81 100644 --- a/crates/engine/src/cmd/raster.rs +++ b/crates/engine/src/cmd/raster.rs @@ -355,17 +355,28 @@ pub(crate) fn add_raster_after(s: &mut Session, name: &str, mut layer: RasterLay } let (w, h, n, overview) = layer.data.as_ref().map(|d| (d.width, d.height, d.band_count(), d.overview)).unwrap_or_default(); // What reading it left out (a colour table, names), said once the layer has its name. - let lost = layer.data.as_ref().map(|d| d.lost.clone()).unwrap_or_default(); let rounded = layer.data.as_deref().and_then(rounded_count); + // Said here for pixels added with the layer; for those landing later, by `notice_landed_rasters`. + let lost = layer.lost_unsaid(); + let unsaid = layer.pixels_unsaid(); + // Read in place, the map shows the full image as it draws, but statistics and tools read the + // overview held: said so. + let held = unsaid.filter(|_| layer.in_place).and_then(|d| crate::in_place_overview(&d)); + let in_place = layer.in_place; let empty_before = super::map_is_empty(s); let id = s.map_mut().add_raster(unique.clone(), layer); s.active_layer = Some(id); - if let Some((fw, fh)) = overview { + // Read in place, its own notice says the overview it holds (`held`); the result names the full + // size all the same. + if let Some((fw, fh)) = overview.filter(|_| !in_place) { s.notify(format!("{unique}: showing a {w} × {h} overview of the {fw} × {fh} raster"), false); } for l in lost { s.notify(format!("{unique}: {l}"), false); } + if let Some(held) = held { + s.notify(format!("{unique}: {held}"), false); + } if let Some(count) = rounded { let held = "pixels are held as 32-bit floats, exact for whole numbers up to 16,777,216 and to about 7 significant digits"; s.notify(format!("{unique}: {count} rounded as the file was read: {held}, so an export writes the rounded values"), false); diff --git a/crates/engine/src/cmd/stac.rs b/crates/engine/src/cmd/stac.rs index 5988e4dba..58aaec07f 100644 --- a/crates/engine/src/cmd/stac.rs +++ b/crates/engine/src/cmd/stac.rs @@ -974,7 +974,7 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult { dates.extend(urls.iter().map(|(d, _)| *d)); let Some((_, first)) = urls.first() else { continue }; let name = if many { format!("{base} {tile}") } else { base.clone() }; - let r = match super::layer::add_url(s, &json!({"url": first, "name": name, "connection":p.get("connection")})) { + let r = match super::layer::add_url(s, &json!({"url": first, "name": name, "cube": true, "connection": p.get("connection")})) { Ok(r) => r, Err(e) => { undo(s); diff --git a/crates/engine/src/cmd/tiles.rs b/crates/engine/src/cmd/tiles.rs index f23cbaed3..8d7193152 100644 --- a/crates/engine/src/cmd/tiles.rs +++ b/crates/engine/src/cmd/tiles.rs @@ -563,6 +563,84 @@ fn blend(p: &Value) -> Result, String> { ps(p, "blend").map(|m| Blend::from_code(m).ok_or_else(|| format!("unknown blend mode '{m}'"))).transpose() } +/// Turns a map's plain COG tile layers into raster layers read in place, each where it was drawn, +/// with its name, visibility, opacity, blend, look, connection and anonymity: those of projects saved before GeoTIFFs read +/// in place were raster layers, and those `map.addTiles` (kind `cog`) still adds, as a project or +/// a map file opens, and then only: a tile layer added this session stays one until then (a +/// plain GeoTIFF is one kind of layer, however it was added; what a tile layer had that a raster +/// layer hasn't is said once, see `Session::notice_landed_rasters`). A dated series, a data cube's +/// files and AlphaEarth's embeddings stay tile layers. +pub(crate) fn cog_tiles_to_rasters(m: &mut rustgis_map::Map) { + use rustgis_map::raster::RasterLayer; + let mut i = 0; + while let Some(t) = m.tiles.get(i) { + let (file, look) = cog::split(&t.url); + if t.kind != TileKind::Cog + || t.time_url.is_some() + || !t.time_urls.is_empty() + || look.aef + || look.cube + || rustgis_data::time::is_template(file) + { + i += 1; + continue; + } + let t = m.tiles.remove(i); + let (file, look) = cog::split(&t.url); + // What a raster layer hasn't, said once the project is open (see + // `Session::notice_landed_rasters`). + let plain = rustgis_map::TileLayer::new(&t.name, TileKind::Cog, file); + let mut dropped = Vec::new(); + if !t.attribution.trim().is_empty() { + dropped.push("attribution"); + } + if !t.legends.is_empty() { + dropped.push("legend"); + } + if (t.min_zoom, t.max_zoom) != (plain.min_zoom, plain.max_zoom) { + dropped.push("zoom range"); + } + if t.bounds.is_some() { + dropped.push("bounds"); + } + if t.overlay { + dropped.push("overlay"); + } + // Read with the project's other rasters read in place (see `project::read_in_place`), in + // its look once it lands. + let rl = RasterLayer::unread_in_place(file).with_look(address_look(&look)).from_tiles_dropping(dropped); + // A name another layer has: numbered, as a layer added is. + let mut name = t.name.clone(); + let mut nth = 1; + while m.layer_by_name(&name).is_some() { + nth += 1; + name = format!("{} ({nth})", t.name); + } + let id = m.add_raster(name, rl); + if let Some(l) = m.layer_mut(id) { + l.visible = t.visible; + l.transparency = (1.0 - t.opacity.clamp(0.0, 1.0)) * 100.0; + l.blend = t.blend; + // Read again through the saved connection it came from, as the tile layer was; one an + // anonymous plugin command added takes no saved login, as before. + l.connection = t.connection.clone(); + l.anonymous = t.anonymous; + } + // Just over the layer it was drawn over, in its group; under them all without one. + let group = t.over.and_then(|o| m.group_of(o).map(|g| g.id)); + if m.place_layer(id, group, t.over).is_err() { + let _ = m.place_layer(id, None, None); + } + } +} + +/// The look a COG tile layer's address keeps (see [`Look`]) as a raster layer's: its bands, one +/// grey or three as red, green and blue, and its fixed range. A COG tile layer becomes a raster +/// layer read in place with it. +pub(crate) fn address_look(look: &Look) -> rustgis_map::raster::AddressLook { + rustgis_map::raster::AddressLook { bands: look.bands.clone(), range: look.range.map(|(lo, hi)| (f64::from(lo), f64::from(hi))) } +} + /// A change of a COG layer's look a command asks for (`bands`, `range`, `dequantize`), applied to /// the look it has; `None` when it asks for none. #[allow(clippy::type_complexity)] @@ -603,6 +681,7 @@ fn cog_look(p: &Value) -> Result Look>>, String> { bands: bands.clone().unwrap_or(old.bands), range: range.unwrap_or(old.range), aef: aef.unwrap_or(old.aef), + cube: old.cube, }))) } @@ -704,7 +783,7 @@ fn set(s: &mut Session, p: &Value) -> CmdResult { if let Ok(Some(info)) = cog::info(&t.url) && let Some(b) = new.bands.iter().find(|b| **b >= info.layout.bands) { - return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b + 1)); + return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b.saturating_add(1))); } Some(cog::with_look(&t.url, &new)) } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 3e3da2279..b007d1ba1 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -268,6 +268,29 @@ fn clip(s: &str, n: usize) -> String { if s.chars().count() > n { s.chars().take(n).chain(['…']).collect() } else { s.to_string() } } +/// The most layer names a notice lists (the rest counted), each cut to 80 characters: the notice +/// stays well within what a notice keeps. +const NAMES_SAID: usize = 5; + +/// Layer names for a notice: the first [`NAMES_SAID`], each clipped, and how many more. +fn names_said(names: &[String]) -> String { + let shown: Vec = names.iter().take(NAMES_SAID).map(|n| clip(n, 80)).collect(); + match names.len().saturating_sub(NAMES_SAID) { + 0 => shown.join(", "), + more => format!("{} and {more} more", shown.join(", ")), + } +} + +/// What a raster layer read in place holds for its statistics and tools, when it holds an +/// overview of the image the map draws. +pub(crate) fn in_place_overview(d: &rustgis_data::raster::Raster) -> Option { + let (fw, fh) = d.overview?; + Some(format!( + "read in place: the map draws its {fw} × {fh} pixels as it needs them, while statistics, histograms and raster tools read a {} × {} overview", + d.width, d.height + )) +} + pub struct Registry { pub commands: Vec, } @@ -594,6 +617,12 @@ pub struct Session { /// (`map.addTiles` with `"view": true`), or a PMTiles/MBTiles archive whose bounds it goes to, and the /// camera and owning map instance then: moved or switched since, the map stays. pub style_view: Option<(rustgis_map::TileLayer, Camera, u64)>, + /// Whether a command waits for the overview of a raster read in place (natively, at most two + /// minutes; the browser never waits): the command line, Python and tests do, so a result is + /// complete; the desktop app doesn't, as its commands run on the window's own thread, and the + /// layer lands as the map draws (failing is said then, and read again: see + /// [`Session::read_in_place_again`]). + pub in_place_wait: bool, pub notices: Vec, /// Troubleshooting history, independent of the active pane, project files and undo. pub diagnostics: rustgis_data::diagnostics::History, @@ -946,8 +975,10 @@ impl Session { #[cfg(feature = "geolibre")] rustgis_geolibre::register(); rustgis_plugins::tools::register(); - // FlatGeobuf and GeoParquet by their web address are read through the tile fetchers. + // FlatGeobuf and GeoParquet by their web address are read through the tile fetchers, and + // cloud-optimised GeoTIFFs' overviews for raster layers read in place. render::rustgis_tiles::remote::install(); + map::raster::set_in_place_reader(render::rustgis_tiles::cog::raster::overview); rustgis_data::diagnostics::set_clock(|| { web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_millis() as u64).unwrap_or(0) }); @@ -978,6 +1009,7 @@ impl Session { layer_selection_of: None, view_size: (1200.0, 720.0), style_view: None, + in_place_wait: true, notices: Vec::new(), diagnostics: rustgis_data::diagnostics::History::default(), notices_given: 0, @@ -1357,6 +1389,14 @@ impl Session { let added = cmd::plugins::fetch::Added::of(self.map(), &res); self.url_waits.land(n, res.clone(), added); } + // A failed overview whose wait is over is read again (the app does this each frame; + // a session that doesn't draw, the command line or Python, as its next command runs), + // waited for where the session waits for reads in place. + if self.read_in_place_again() { + self.wait_in_place_reads(); + } + // Pixels the command read in place (a project, an import) say what they lost. + self.notice_landed_rasters(); // A plugin's command waiting for its fetches answers `pending` and runs again when they // land: logged then, once. (Another command's `pending`, a download the web app makes, is // logged as it is asked for: nothing runs it again.) @@ -1807,6 +1847,131 @@ impl Session { } } + /// Says what reading raster layers' pixels lost when it wasn't said as each was added (pixels + /// read in place land later: in the browser, a project, an import): what the file's header + /// left out (`Raster::lost`: a colour table missing or short, GDAL metadata that doesn't + /// decode), an overview statistics and tools read, a layer read in place that couldn't be + /// read, and why. Once for each layer's pixels or failure, one notice of each kind; the app + /// calls it each frame. + pub fn notice_landed_rasters(&mut self) { + let (mut lost, mut held, mut failed, mut dropped) = (Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for m in &mut self.project.maps { + for l in &mut m.layers { + let Some(rl) = &mut l.raster else { continue }; + if let Some(e) = rl.error_unsaid() { + failed.push((l.name.clone(), e)); + } + let dropped_here = rl.dropped_unsaid(); + if !dropped_here.is_empty() { + dropped.push((l.name.clone(), dropped_here)); + } + let in_place = rl.in_place; + lost.extend(rl.lost_unsaid().into_iter().map(|what| (l.name.clone(), what))); + let Some(d) = rl.pixels_unsaid() else { continue }; + if let Some(what) = in_place.then(|| in_place_overview(&d)).flatten() { + held.push((l.name.clone(), what)); + } + } + } + // What a header left out, as adding the layer says it (one notice a kind, with counts). + cmd::raster::say_losses(self, lost); + if !dropped.is_empty() { + let mut kinds: Vec<&str> = dropped.iter().flat_map(|(_, k)| k.iter().copied()).collect(); + kinds.sort_unstable(); + kinds.dedup(); + let names: Vec = dropped.iter().map(|(n, _)| n.clone()).collect(); + self.notify( + format!( + "{} COG tile layer(s) opened as raster layers read in place, which have no {}: not kept for {}", + dropped.len(), + kinds.join(", "), + names_said(&names) + ), + false, + ); + } + match &failed[..] { + [] => {} + [(one, e)] => self.notify(format!("{}: couldn't be read in place: {e}", clip(one, 80)), true), + many => { + let names: Vec = many.iter().map(|(n, _)| n.clone()).collect(); + self.notify(format!("{} raster layers couldn't be read in place: {}", many.len(), names_said(&names)), true); + } + } + match &held[..] { + [] => {} + [(one, what)] => self.notify(format!("{}: {what}", clip(one, 80)), false), + many => { + let names: Vec = many.iter().map(|(n, _)| n.clone()).collect(); + self.notify( + format!( + "{} rasters are read in place: the map draws them at full resolution as it needs it, while statistics, histograms and raster tools read an overview of each: {}", + many.len(), + names_said(&names) + ), + false, + ) + } + } + } + + /// Waits for the overviews of the rasters read in place that a project, an import or a layer + /// file began reading together (see [`crate::project::read_in_place`]), where the session + /// waits for them ([`Session::in_place_wait`]: natively, two minutes at most for all); where + /// it doesn't, they land as the map draws. Failing is the layer's to say (see + /// [`Session::notice_landed_rasters`]). + pub(crate) fn wait_in_place_reads(&mut self) { + if !self.in_place_wait { + return; + } + // Every map's at once: one wait bounds them all, not each map's in turn. + let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = self + .project + .maps + .iter_mut() + .flat_map(|m| m.layers.iter_mut()) + .filter_map(|l| { + let name = l.name.clone(); + let rl = l.raster.as_mut()?; + rl.in_place.then_some((name, rl)) + }) + .collect(); + let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); + for (i, e) in rustgis_map::raster::wait_for_reads(&mut layers) { + if let Some((name, _)) = named.get(i) { + log::warn!("{name}: {e}"); + } + } + } + + /// Reads again the overview of a raster layer read in place whose read failed, once its wait + /// is over: a minute, or what its server's `Retry-After` said, as a tile or a file's header is + /// read again (see `rustgis_tiles::cache::retry_wait`), doubled each time it fails again, + /// within the hour (`cache::backoff`: a file that never reads isn't asked for every minute all + /// session); until then the map is asked to draw again when the wait is over. The app calls it + /// each frame, with [`Session::notice_landed_rasters`], and [`Session::run`] after each command + /// (a session that doesn't draw, the command line or Python, asks again that way); the pixels + /// land as the map draws, or are waited for where the session waits for reads in place + /// ([`Session::in_place_wait`]). Whether a read was started. + pub fn read_in_place_again(&mut self) -> bool { + use render::rustgis_tiles::cache; + let mut started = false; + for m in &mut self.project.maps { + for l in &mut m.layers { + let Some(rl) = &mut l.raster else { continue }; + let Some((e, at, failures)) = rl.read_failure() else { continue }; + let wait = cache::backoff(cache::retry_wait(e, cache::RETRY), failures); + if at.elapsed() < wait { + cache::notify_at(at + wait); + } else { + rl.read_again(); + started = true; + } + } + } + started + } + pub fn notify(&mut self, text: impl Into, error: bool) { use rustgis_data::diagnostics::Severity; self.notice(text.into(), if error { Severity::Error } else { Severity::Info }); diff --git a/crates/engine/src/project.rs b/crates/engine/src/project.rs index dae43f5ee..b4e55eedc 100644 --- a/crates/engine/src/project.rs +++ b/crates/engine/src/project.rs @@ -588,6 +588,13 @@ pub(crate) fn resolve_file_paths<'a>(dir: &str, paths: impl IntoIterator bool { + let scheme = path.get(..8).map(str::to_ascii_lowercase).unwrap_or_default(); + scheme.starts_with("https://") || scheme.starts_with("http://") +} + /// Read only compatibility metadata before interpreting any saved payload. #[derive(Deserialize)] struct FileFormat { @@ -772,6 +779,19 @@ impl Project { /// Repairs invariants after loading (at least one map; a valid active index). pub fn normalize(&mut self) { + self.repair(false); + } + + /// [`Project::normalize`] for a project read from its file, which also opens its plain COG + /// tile layers (a project saved before GeoTIFFs read in place were raster layers, or with + /// those `map.addTiles` adds) as raster layers read in place: as the project opens only. A + /// later repair (a layer file added, maps imported) leaves a tile layer added this session + /// as it is. + pub(crate) fn normalize_opened(&mut self) { + self.repair(true); + } + + fn repair(&mut self, opened: bool) { if self.maps.is_empty() { self.maps.push(Map::new("Map")); } @@ -881,6 +901,28 @@ impl Project { } // Tile layers with an address that can't be fetched (a hand edit) are dropped. m.tiles.retain_mut(|t| t.repair().map_err(|e| log::warn!("{}: {e}", m.name)).is_ok()); + // GeoTIFFs read in place are raster layers (a project from before saved tile layers), + // so they don't count toward the tile layers kept. As the project opens only: a tile + // layer added this session stays one until the project is opened again. + if opened { + crate::cmd::tiles::cog_tiles_to_rasters(m); + // A raster layer whose path is a plain web address, from no saved connection, is + // read in place, the one way such a path is read (a login or S3 wrapper is + // downloaded whole, as is a connection's address, plain here until its login is + // made again): one saved by an older RustGIS, which didn't know `in_place` (it + // showed the layer as missing and saved it again without the key), reads in + // place again here, losing a pending look at most. + for l in &mut m.layers { + if l.connection.is_empty() + && let Some(rl) = &mut l.raster + && !rl.in_place + && rl.embedded.is_none() + && rl.path.as_deref().is_some_and(plain_web_address) + { + rl.in_place = true; + } + } + } m.tiles.truncate(rustgis_map::tiles::MAX_TILE_LAYERS); #[cfg(not(target_arch = "wasm32"))] for t in &mut m.tiles { @@ -910,8 +952,10 @@ impl Project { src.error = Some(error); } } + // Read in place, read together below. if let Some(rl) = &mut l.raster && rl.data.is_none() + && !rl.in_place && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed) && unresolved.is_none() && let Err(e) = rl.reload() @@ -939,6 +983,7 @@ impl Project { } l.touch_geometry(); } + read_in_place(m); } } @@ -1128,7 +1173,7 @@ impl Project { if let Some(d) = dir.and_then(Path::to_str) { p.resolve_paths(d); } - p.normalize(); + p.normalize_opened(); p.label_read_notices = label_losses::notices(text, &p)?; Ok(p) } @@ -1143,6 +1188,35 @@ impl Project { } } +/// Starts reading the overviews of a map's raster layers read in place and not read yet, all at +/// once (see [`rustgis_map::raster::read_together`]): a project or an import of several waits for +/// the slowest, not for each in turn, where the session waits for them +/// ([`crate::Session::wait_in_place_reads`]); elsewhere they land as the map draws. One read from +/// an S3 bucket waits for its connection (see [`crate::s3_connections::resolve_project_s3`]). One +/// that failed is left to [`crate::Session::read_in_place_again`], which reads it again once its +/// wait is over, and one that couldn't start (an address the project keeps that can't be read) +/// stays as it is, said once: a repair of the project (a layer file added, maps imported) doesn't +/// start either over. +pub(crate) fn read_in_place(m: &mut Map) { + let mut named: Vec<(String, &mut rustgis_map::raster::RasterLayer)> = m + .layers + .iter_mut() + .filter_map(|l| { + let name = l.name.clone(); + let rl = l.raster.as_mut()?; + (rl.in_place && rl.error.is_none() && !rl.path.as_deref().is_some_and(crate::s3_connections::is_s3_backed)) + .then_some((name, rl)) + }) + .collect(); + let mut layers: Vec<&mut rustgis_map::raster::RasterLayer> = named.iter_mut().map(|(_, rl)| &mut **rl).collect(); + let failed = rustgis_map::raster::read_together(&mut layers, false); + for (i, e) in failed { + if let Some((name, _)) = named.get(i) { + log::warn!("{name}: {e}"); + } + } +} + /// A local file's bytes, refused past `max`. Only a regular file is opened: a device such as /// `/dev/zero` reports no size and never ends, and opening a named pipe waits for a writer. /// Reading stops a byte past `max`, so a file growing as it is read can't exhaust memory either. diff --git a/crates/engine/src/s3_connections.rs b/crates/engine/src/s3_connections.rs index 803f6d76a..6dde8602a 100644 --- a/crates/engine/src/s3_connections.rs +++ b/crates/engine/src/s3_connections.rs @@ -274,37 +274,76 @@ fn block(s: &mut Session, g: &Group) -> usize { src.error = Some(s3::UNRESOLVED.into()); } } + // A raster read in place from the bucket waits for the connection too, and says so. + for rl in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.raster.as_mut()) { + if rl.in_place && rl.data.is_none() && rl.path.as_deref().is_some_and(|p| g.has(p)) { + rl.fail(s3::UNRESOLVED); + } + } s.project.addresses_mut().into_iter().filter(|a| g.has(a)).count() } -/// Opens (again) the streamed sources whose address `which` accepts; how many opened and how -/// many failed. -fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> (usize, usize) { - let (mut opened, mut failed) = (0, 0); +/// What opening a bind's sources did: how many opened, how many failed, and how many rasters read +/// in place are still reading their overview. Where nothing waits for a read in place (the +/// desktop, the browser), a raster whose overview was only asked for is reading, not opened: it +/// lands as the map draws, and one that fails is said then. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Counts { + pub(crate) opened: usize, + pub(crate) failed: usize, + pub(crate) reading: usize, +} + +/// Opens (again) the streamed sources whose address `which` accepts: how many opened, failed, and +/// are still reading. Rasters read in place read their overviews together (see +/// [`rustgis_map::raster::read_together`]): a bucket of several waits for the slowest, not for +/// each in turn, and on the desktop for none (they land as the map draws, and count as reading +/// until then). +fn reopen_where(s: &mut Session, which: impl Fn(&str) -> bool) -> Counts { + let mut n = Counts::default(); + let wait = s.in_place_wait; for src in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.source.as_mut()) { let Some(path) = src.path.clone() else { continue }; if (!src.is_open() || src.error.is_some()) && which(&path) { match src.reopen() { - Ok(()) => opened += 1, - Err(_) => failed += 1, + Ok(()) => n.opened += 1, + Err(_) => n.failed += 1, } } } + let mut in_place = Vec::new(); for raster in s.project.maps.iter_mut().flat_map(|m| m.layers.iter_mut()).filter_map(|l| l.raster.as_mut()) { if raster.data.is_none() && raster.time_zarr.is_none() && raster.path.as_deref().is_some_and(&which) { + if raster.in_place { + in_place.push(raster); + continue; + } match raster.reload() { - Ok(()) => opened += 1, - Err(_) => failed += 1, + Ok(()) => n.opened += 1, + Err(_) => n.failed += 1, } } } - (opened, failed) + let unread = rustgis_map::raster::read_together(&mut in_place, wait); + n.failed += unread.len(); + // Counted by what is in, not by what was asked for: a read nobody waited for is still reading. + for (i, rl) in in_place.iter().enumerate() { + if unread.iter().any(|(j, _)| *j == i) { + continue; + } + if rl.data.is_some() { + n.opened += 1; + } else { + n.reading += 1; + } + } + n } /// Binds `g`'s sources to the saved connection `c`: logs it in if it isn't, lets its login read /// the bucket (the user's choice, or its only candidate's), rewrites the addresses to name it and -/// reads them. How many sources opened and how many failed. -fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result<(usize, usize), String> { +/// reads them. How many sources opened, failed, and are still reading. +fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result { let endpoint = endpoint_of(c).ok_or_else(|| s3::MISMATCH.to_string())?; let binding = crate::cmd::connections::login_saved(s, c)?; s3::grant_saved(&binding, &g.bucket)?; @@ -314,11 +353,11 @@ fn bind_group(s: &mut Session, g: &Group, c: &Connection) -> Result<(usize, usiz cache::forget(part(a)); } let affected = |address: &str| addresses.iter().any(|a| a == address); - let (mut opened, failed) = reopen_where(s, affected); + let mut n = reopen_where(s, affected); // Tile resources fetch as they draw, by their new address. - opened += + n.opened += s.project.maps.iter().flat_map(|m| &m.tiles).filter(|t| affected(&t.url) || t.time_url.as_deref().is_some_and(affected)).count(); - Ok((opened, failed)) + Ok(n) } /// The user's choice of the saved connection `id` to read the sources of the pending resource @@ -333,9 +372,9 @@ pub(crate) fn choose(s: &mut Session, source: &str, id: &str) -> crate::CmdResul s.s3_pending.remove(index); return Err(STALE.into()); } - let (reopened, failed) = bind_group(s, &g, &c)?; + let n = bind_group(s, &g, &c)?; s.s3_pending.remove(index); - Ok(serde_json::json!({"id": c.id, "reopened": reopened, "failed": failed})) + Ok(serde_json::json!({"id": c.id, "reopened": n.opened, "failed": n.failed, "reading": n.reading})) } #[cfg(test)] diff --git a/crates/engine/src/s3_saved_tests.rs b/crates/engine/src/s3_saved_tests.rs index 6111c9ee6..ff3df4c52 100644 --- a/crates/engine/src/s3_saved_tests.rs +++ b/crates/engine/src/s3_saved_tests.rs @@ -688,15 +688,17 @@ fn s3_saved_projects_hold_no_keys_and_reload_through_their_one_connection() { assert!(read_view(&mut s, id) > 0, "{:?}", s.map().layer(id).unwrap().source.as_ref().unwrap().error); let layer = s.map().layers.iter().find(|l| l.name == "Grid").unwrap(); assert!(layer.source.as_ref().unwrap().path.as_deref().is_some_and(|x| x.contains(s.s3_binding("c1").unwrap()))); - // The COG's tile is drawn from its pixels. - let tiles = s.map().tiles.clone(); - let scene = tiles.iter().find(|t| t.name == "Scene").unwrap(); - assert!(scene.url.contains(s.s3_binding("c1").unwrap())); + // The COG, a raster layer read in place, holds its overview, read through the connection; + // its tiles are drawn from its pixels. + let scene = s.map().layers.iter().find(|l| l.name == "Scene").and_then(|l| l.raster.clone()).unwrap(); + let address = scene.path.clone().unwrap(); + assert!(scene.in_place && address.contains(s.s3_binding("c1").unwrap()), "{address}"); + assert!(scene.data.is_some(), "{:?}", scene.error); let (z, lon, lat) = (9u32, 10.5f64, 45.5f64); let n = f64::from(1u32 << z); let x = ((lon + 180.0) / 360.0 * n) as u32; let y = ((1.0 - lat.to_radians().tan().asinh() / std::f64::consts::PI) / 2.0 * n) as u32; - let tile = format!("cog:{z}/{x}/{y}@{}", scene.url); + let tile = format!("cog:{z}/{x}/{y}@{address}"); let mut got = None; for _ in 0..300 { match cache::get(&tile, cache::Want::Image) { @@ -813,6 +815,24 @@ fn s3_saved_sources_without_keys_zero_matches_and_older_addresses_stay_blocked_o assert!(none.notices.iter().any(|n| n.error && n.text.contains("no saved connection")), "{:?}", none.notices); assert_eq!(none.map().layers.iter().find_map(|l| l.source.as_ref()).unwrap().error.as_deref(), Some(s3::UNRESOLVED)); assert_eq!(p.srv.requests(), from); + // A GeoTIFF read in place from the bucket waits for the connection too, and says so, once. + let fgb = none.map().layers.iter().find_map(|l| l.source.as_ref().and_then(|src| src.path.clone())).unwrap(); + let scene = fgb.replace("grid.fgb", "scene.tif"); + assert_ne!(scene, fgb); + none.map_mut().add_raster("Scene".to_string(), rustgis_map::raster::RasterLayer::unread_in_place(scene)); + let text = serde_json::to_string(&none.project).unwrap(); + let mut none = session(Some(&empty)); + none.run("project.openText", &json!({"text": text})).unwrap(); + let scene = none.map().layers.iter().find(|l| l.name == "Scene").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(scene.data.is_none() && scene.error.as_deref() == Some(s3::UNRESOLVED), "{:?}", scene.error); + // One notice (the project's maps each hold the layer: one line naming them all). + let said = |s: &Session| { + s.notices.iter().filter(|n| n.error && n.text.contains("couldn't be read in place") && n.text.contains("Scene")).count() + }; + assert_eq!(said(&none), 1, "{:?}", none.notices); + none.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&none), 1); + assert_eq!(p.srv.requests(), from, "nothing fetched without the connection"); // An older project's bare address, with a session-only connection of the bucket: as before. let legacy = p.text.replace(&p.fgb, "s3://sv-proj-edge/grid.fgb"); assert!(legacy.contains("s3://sv-proj-edge/grid.fgb") && !legacy.contains(&p.fgb), "the Grid layer is an older project's"); diff --git a/crates/engine/src/service_logins.rs b/crates/engine/src/service_logins.rs index f539b570c..01dbca38e 100644 --- a/crates/engine/src/service_logins.rs +++ b/crates/engine/src/service_logins.rs @@ -416,6 +416,7 @@ fn layer_address(l: &rustgis_map::Layer) -> Option<&str> { fn rebind_connection(s: &mut Session, c: &Connection) -> usize { let binding = s.web_logins.active.get(&c.id).map(|a| a.binding.as_str()); let origin = login_origin(c); + let wait = s.in_place_wait; let mut failed = 0usize; let mut rewrite = |address: &mut String| { let (prefix, raw) = crate::sources::template_archive(address).unwrap_or(("", address.as_str())); @@ -477,8 +478,10 @@ fn rebind_connection(s: &mut Session, c: &Connection) -> usize { && let Some(path) = raster.path.as_mut() && rewrite(path) { - // Reload keeps the style and publishes any failure through raster.error. - let _ = raster.reload(); + // Reload keeps the style and publishes any failure through raster.error. A layer + // read in place lands as the map draws where the session doesn't wait for reads + // in place (the desktop), so a login made again doesn't hold the window. + let _ = if raster.in_place && !wait { raster.begin_reload() } else { raster.reload() }; } if let Some(path) = &mut l.dataset.source { rewrite(path); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index fde19769a..7f3edea40 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -15289,52 +15289,103 @@ fn serve_file(body: Vec, ranges: bool) -> (String, std::sync::Arc, n: usize) -> (String, std::sync::Arc) { + serve_with(body, true, None, n, 0) +} + +/// [`serve_ranges`], answering the first `n - 1` requests (a file's header) and refusing the rest +/// with a 500: a file whose header reads but whose tiles can't be. +fn serve_refusing_from(body: Vec, n: usize) -> (String, std::sync::Arc) { + serve_with(body, true, None, 0, n) +} + /// [`serve_file`] holding every answer until `gate` is set (a request still under way, for as long /// as a test needs). fn serve_gated( body: Vec, ranges: bool, gate: Option>, +) -> (String, std::sync::Arc) { + serve_with(body, ranges, gate, 0, 0) +} + +/// [`serve_gated`], refusing the first `refuse` requests (see [`serve_refusing_first`]), and every +/// request from the `refuse_from`th on (see [`serve_refusing_from`]; 0 for none). +fn serve_with( + body: Vec, + ranges: bool, + gate: Option>, + refuse: usize, + refuse_from: usize, ) -> (String, std::sync::Arc) { use std::io::{Read, Write}; + let refusals = std::sync::Arc::new(std::sync::Mutex::new(refuse)); + let seen = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let sent = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let count = sent.clone(); + let body = std::sync::Arc::new(body); std::thread::spawn(move || { + // Each connection on its own thread: a layer read in place asks for its tiles together, + // and answering them one by one would hold a fetch worker for each while the others wait + // (the fetch workers are shared by every test in the process). for stream in listener.incoming() { let Ok(mut s) = stream else { continue }; - let mut buf = [0u8; 4096]; - let read = s.read(&mut buf).unwrap_or(0); - if let Some(g) = &gate { - let start = std::time::Instant::now(); - while !g.load(std::sync::atomic::Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(60) { - std::thread::sleep(std::time::Duration::from_millis(5)); + let (body, count, gate, refusals, seen) = (body.clone(), count.clone(), gate.clone(), refusals.clone(), seen.clone()); + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + let read = s.read(&mut buf).unwrap_or(0); + // Refused from the `refuse_from`th request on: a file whose tiles can't be read. + let nth = seen.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + if refuse_from > 0 && nth >= refuse_from { + let _ = s.write_all(b"HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; } - } - let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); - // A HEAD request is answered with the length alone. - if req.starts_with("head ") { - let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); - continue; - } - let range = req.lines().filter(|_| ranges).find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { - let (a, b) = r.trim().split_once('-')?; - // A suffix (`-n`): the last n bytes. - if a.is_empty() { - let n = b.parse::().ok()?.min(body.len()); - return Some((body.len() - n, body.len().saturating_sub(1))); + // Refused while there are refusals left: busy, ask again in a second. + let refused = { + let mut left = refusals.lock().unwrap(); + let refused = *left > 0; + *left = left.saturating_sub(1); + refused + }; + if refused { + let _ = s + .write_all(b"HTTP/1.1 503 Service Unavailable\r\nRetry-After: 1\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + } + if let Some(g) = &gate { + let start = std::time::Instant::now(); + while !g.load(std::sync::atomic::Ordering::Relaxed) && start.elapsed() < std::time::Duration::from_secs(60) { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + } + let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); + // A HEAD request is answered with the length alone. + if req.starts_with("head ") { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + return; } - Some((a.parse::().ok()?, b.parse::().ok()?)) + let range = req.lines().filter(|_| ranges).find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + // A suffix (`-n`): the last n bytes. + if a.is_empty() { + let n = b.parse::().ok()?.min(body.len()); + return Some((body.len() - n, body.len().saturating_sub(1))); + } + Some((a.parse::().ok()?, b.parse::().ok()?)) + }); + let (status, part) = match range { + Some((a, b)) if a < body.len() => ("206 Partial Content", &body[a..=b.min(body.len() - 1)]), + Some(_) => ("416 Range Not Satisfiable", &body[..0]), + None => ("200 OK", &body[..]), + }; + count.fetch_add(part.len(), std::sync::atomic::Ordering::Relaxed); + let _ = write!(s, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); + let _ = s.write_all(part); }); - let (status, part) = match range { - Some((a, b)) if a < body.len() => ("206 Partial Content", &body[a..=b.min(body.len() - 1)]), - Some(_) => ("416 Range Not Satisfiable", &body[..0]), - None => ("200 OK", &body[..]), - }; - count.fetch_add(part.len(), std::sync::atomic::Ordering::Relaxed); - let _ = write!(s, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); - let _ = s.write_all(part); } }); (format!("http://{addr}"), sent) @@ -15924,7 +15975,8 @@ fn a_plugins_dated_cog_is_added_without_waiting() { } /// A GeoTIFF whose header was just read is framed even if the cache forgot it since (a login -/// ended), and one that can't be read again says why the map wasn't moved to it. +/// ended), and one that can't be read again says why the map wasn't moved to it: where the +/// session waits for reads in place, the command fails instead (see `add_in_place`). #[cfg(not(target_arch = "wasm32"))] #[test] fn a_probed_cog_is_framed_after_the_cache_forgets_it() { @@ -15937,16 +15989,21 @@ fn a_probed_cog_is_framed_after_the_cache_forgets_it() { let r = crate::cmd::layer::add_cog(&mut s, &json!({}), &url, true).unwrap(); assert!(r["bounds"].is_array(), "{r}"); assert!(r.get("warning").is_none(), "{r}"); - // Nothing answers at this address any more. + // Nothing answers at this address any more: waited for, the command fails. let gone = { let server = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); format!("http://{}/gone.tif", server.local_addr().unwrap()) }; + let e = crate::cmd::layer::add_cog(&mut s, &json!({}), &gone, true).unwrap_err(); + assert!(e.contains("gone.tif"), "{e}"); + // Not waited for (the desktop), the layer is added, its read failing as the map draws, and + // the answer says why the map wasn't moved to it: a header just read that can't be read + // again, and a failure recorded and not yet tried again. Another test's login ending forgets + // the record, so it is made again for each try. + s.in_place_wait = false; let r = crate::cmd::layer::add_cog(&mut s, &json!({}), &gone, true).unwrap(); let warning = r["warning"].as_str().unwrap_or_default(); assert!(warning.contains("couldn't be read (") && warning.contains("wasn't moved"), "{r}"); - // Not waited for, its failure (recorded, not yet tried again) is said too. Another test's - // login ending forgets the record, so it is made again for each try. let said = (0..3).any(|_| { let _ = crate::cmd::layer::add_cog(&mut s, &json!({}), &gone, true); let r = crate::cmd::layer::add_cog(&mut s, &json!({}), &gone, false).unwrap(); @@ -15966,17 +16023,43 @@ fn cloud_optimised_geotiffs_are_read_in_place() { let mut s = Session::new(Project::default()); let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif")})).unwrap(); assert_eq!(r["inPlace"], true, "{r}"); - let t = s.map().tiles.last().unwrap().clone(); - assert_eq!((t.kind, t.name.as_str()), (rustgis_map::TileKind::Cog, "scene")); + // A raster layer like a file's: it holds an overview (1024 pixels a side of the 2048), whose + // statistics its symbology reads. + let l = s.map().layers.iter().find(|l| l.name == "scene").unwrap().clone(); + let rl = l.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.path.as_deref() == Some(&*format!("{base}/scene.tif"))); + let d = rl.data.as_ref().unwrap(); + assert_eq!((d.width, d.overview), (1024, Some((2048, 2048)))); + assert!(d.bands[0].stats.count > 0 && d.bands[0].stats.max > 200.0); // Framed on the file: 10–11° E, 45–46° N. let b = r["bounds"].as_array().unwrap().iter().map(|v| v.as_f64().unwrap()).collect::>(); assert!((b[0] - 10.0).abs() < 1e-6 && (b[3] - 46.0).abs() < 1e-6, "{b:?}"); - // A map tile inside it (zoom 9) is drawn from a few of its tiles, not the whole file. + // Drawn zoomed in past the overview, the part in view is read at full resolution: a few of + // its tiles, not the whole file. Pixel (x, y) of the full image is (x + y) % 256: neighbouring + // screen pixels differ by one, finer than the overview's steps of two. + let before = sent.load(std::sync::atomic::Ordering::Relaxed); + let c = rustgis_geom::lonlat_to_merc(rustgis_geom::Coord::new(10.5, 45.5)); + s.map_mut().camera.center = c; + // A metre on the ground is about 1.43 Web Mercator units here: a third of a full pixel's. + s.map_mut().camera.res = 25.0; + s.view_size = (64.0, 64.0); + let png = std::env::temp_dir().join(format!("rustgis-cog-in-place-{}.png", std::process::id())); + s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap(); + let img = image::open(&png).unwrap().to_rgba8(); + let _ = std::fs::remove_file(&png); + let row: Vec = (0..64).map(|x| img.get_pixel(x, 32)[0]).collect(); + assert!(row.iter().any(|v| *v > 0), "drawn: {row:?}"); + assert!(row.windows(2).any(|w| w[1].abs_diff(w[0]) == 1), "full resolution (steps of one), not the overview's (of two): {row:?}"); + let window = sent.load(std::sync::atomic::Ordering::Relaxed) - before; + assert!(window > 0 && window < total / 8, "{window} of {total} bytes read for the view"); + // A map tile inside it (zoom 9) is drawn from a few of its tiles, not the whole file (a dated + // series of GeoTIFFs is drawn so, as a tile layer). let (z, lon, lat) = (9u32, 10.5f64, 45.5f64); let n = f64::from(1u32 << z); let x = ((lon + 180.0) / 360.0 * n) as u32; let y = ((1.0 - lat.to_radians().tan().asinh() / std::f64::consts::PI) / 2.0 * n) as u32; let tile = format!("cog:{z}/{x}/{y}@{base}/scene.tif"); + let before = sent.load(std::sync::atomic::Ordering::Relaxed); let mut got = None; for _ in 0..200 { match cache::get(&tile, Want::Image) { @@ -15992,13 +16075,13 @@ fn cloud_optimised_geotiffs_are_read_in_place() { let Payload::Image(pm) = &*p else { panic!("not a picture") }; assert_eq!((pm.width(), pm.height()), (256, 256)); assert!(pm.pixels().iter().any(|px| px.alpha() == 255)); - let read = sent.load(std::sync::atomic::Ordering::Relaxed); + let read = sent.load(std::sync::atomic::Ordering::Relaxed) - before; assert!(read < total / 4, "{read} of {total} bytes read"); // A tile outside it is empty. assert!(matches!(cache::get(&format!("cog:9/0/0@{base}/scene.tif"), Want::Image), State::Pending | State::Ready(_))); // The project keeps the address; asked to, the file is downloaded whole instead. let text = serde_json::to_string(&s.project).unwrap(); - assert!(text.contains("\"kind\":\"cog\"") && text.contains("scene.tif")); + assert!(text.contains("\"in_place\":true") && text.contains("scene.tif")); let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif"), "whole": true, "name": "Whole"})).unwrap(); assert!(r.get("inPlace").is_none(), "{r}"); let id = r["layer"].as_u64().unwrap() as u32; @@ -16063,8 +16146,9 @@ fn unnamed_cloud_optimised_geotiffs_are_read_in_place() { let mut s = Session::new(Project::default()); let r = s.run("layer.addUrl", &json!({"url": url, "name": "Ortho"})).unwrap(); assert_eq!(r["inPlace"], true, "{r}"); - let t = s.map().tiles.last().unwrap().clone(); - assert_eq!((t.kind, t.name.as_str(), t.url.as_str()), (rustgis_map::TileKind::Cog, "Ortho", url.as_str())); + let l = s.map().layers.iter().find(|l| l.name == "Ortho").unwrap(); + let rl = l.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.path.as_deref() == Some(url.as_str()), "{:?}", rl.path); let read = sent.load(std::sync::atomic::Ordering::Relaxed); assert!(read < total / 4, "{read} of {total} bytes read: only the header, not the whole file"); let r = s.run("layer.addUrl", &json!({"url": url, "name": "Probed", "cog": true})).unwrap(); @@ -16084,6 +16168,989 @@ fn unnamed_cloud_optimised_geotiffs_are_read_in_place() { assert!(e.contains("not a data format"), "{e}"); } +/// A cloud-optimised GeoTIFF read in place is a raster layer like a file's: its first look is what +/// its PhotometricInterpretation says (RGB in colour, a palette by its table, grey (MinIsBlack, as +/// most remote sensing images are) as its first band, but three grey bands as red, green and +/// blue), every band there for the raster commands and the Symbology pane to combine and stretch. +#[test] +fn cog_layers_are_rasters_styled_as_their_photometric_interpretation_says() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{RasterRenderer, Stretch}; + let e = Example { size: 512, tile: 256, ..Example::default() }; + let rgb = RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }; + let cases: [(&str, Example, usize, Option); 7] = [ + ("Pan", Example { bands: 1, ..e }, 1, None), + ("PanAlpha", Example { bands: 2, alpha: true, ..e }, 2, None), + ("Grey3", Example { bands: 3, ..e }, 3, Some(rgb.clone())), + ("Multispectral", Example { bands: 8, ..e }, 8, None), + ("Rgb", Example { bands: 3, photometric: 2, ..e }, 3, Some(rgb.clone())), + ("Rgbn", Example { bands: 4, photometric: 2, ..e }, 4, Some(rgb.clone())), + ("RgbaNir", Example { bands: 5, photometric: 2, alpha: true, planar: true, ..e }, 5, Some(rgb.clone())), + ]; + let mut s = Session::new(Project::default()); + for (name, ex, bands, renderer) in cases { + let (base, _) = serve_ranges(example_with(&ex)); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/{name}.tif"), "name": name})).unwrap(); + assert_eq!(r["inPlace"], true, "{name}: {r}"); + let id = r["layer"].as_u64().unwrap() as u32; + let rl = s.map().layer(id).and_then(|l| l.raster.clone()).unwrap(); + assert!(rl.in_place, "{name}"); + assert_eq!(rl.data.as_ref().map(|d| d.band_count()), Some(bands), "{name}"); + match renderer { + Some(want) => assert_eq!(rl.style.renderer, want, "{name}"), + None => assert!(matches!(rl.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{name}: {:?}", rl.style.renderer), + } + // The raster commands combine and stretch its bands, as a file's. + if bands >= 3 { + let last = bands as u64; + s.run("raster.bands", &json!({"layer": name, "red": last, "green": 2, "blue": 1})).unwrap(); + let st = s.map().layer(id).and_then(|l| l.raster.as_ref()).map(|r| r.style.clone()).unwrap(); + assert_eq!(st.renderer, RasterRenderer::Rgb { red: bands - 1, green: 1, blue: 0 }, "{name}"); + } + s.run("raster.stretch", &json!({"layer": name, "type": "standardDeviation", "n": 2})).unwrap(); + let st = s.map().layer(id).and_then(|l| l.raster.as_ref()).map(|r| r.style.clone()).unwrap(); + assert_eq!(st.stretch, Stretch::StandardDeviation { n: 2.0 }, "{name}"); + } + assert!(s.map().tiles.is_empty(), "no tile layers"); + // A palette image, by its table. + let (base, _) = serve_ranges(example_with(&Example { classes: true, ..e })); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/classes.tif")})).unwrap(); + let rl = s.map().layer(r["layer"].as_u64().unwrap() as u32).and_then(|l| l.raster.clone()).unwrap(); + assert_eq!(rl.style.renderer, RasterRenderer::Colormap); + // Saved and opened again, it is read in place again, its style kept. + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + let ms = t.map().layers.iter().find(|l| l.name == "Multispectral").and_then(|l| l.raster.clone()).unwrap(); + assert!(ms.in_place && ms.data.is_some() && ms.style.stretch == Stretch::StandardDeviation { n: 2.0 }); +} + +/// A view of a GeoTIFF read in place zoomed in past its overview shows the level zoomed from while +/// its own tiles are on their way: no holes, no blocks of the overview, then its own pixels. +#[test] +fn cog_windows_stand_in_the_level_kept_until_their_tiles_arrive() { + use crate::render::rustgis_tiles::cog::raster::window; + use std::sync::atomic::{AtomicBool, Ordering}; + let gate = std::sync::Arc::new(AtomicBool::new(true)); + let (base, _) = serve_gated(rustgis_data::raster::cog::example(2048, 256), true, Some(gate.clone())); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + // Its overview: level 1 (1024 pixels a side), whose tiles are kept. + s.run("layer.addUrl", &json!({"url": url})).unwrap(); + // Level 0's tiles held back: a window of it is whole, from level 1, and still to come. + gate.store(false, Ordering::Relaxed); + let (cols, rows) = ((1000, 1100), (900, 960)); + let gray = rustgis_data::raster::ColorHint::Gray; + let w = window(&url, 0, cols, rows, &[0], gray.clone()).unwrap(); + let part = w.raster.unwrap(); + assert!(w.pending); + let v = |r: &rustgis_data::raster::Raster, x: u32, y: u32| r.bands[0].data[((y - rows.0) * (cols.1 - cols.0) + (x - cols.0)) as usize]; + assert!(part.bands[0].data.iter().all(|v| v.is_finite()), "no holes"); + // Level 1's pixel under (1001, 901) is (500, 450): full pixel (1000, 900), (1000 + 900) % 256. + assert_eq!(v(&part, 1001, 901), ((1000 + 900) % 256) as f32); + // Its own tiles arrive: its own pixels, (x + y) % 256. + gate.store(true, Ordering::Relaxed); + let mut done = None; + for _ in 0..200 { + let w = window(&url, 0, cols, rows, &[0], gray.clone()).unwrap(); + if !w.pending { + done = w.raster; + break; + } + crate::render::rustgis_tiles::cache::wait(std::time::Duration::from_millis(50)); + } + let part = done.expect("its tiles never came"); + assert_eq!(v(&part, 1001, 901), ((1001 + 901) % 256) as f32); +} + +/// A tile a sparse file leaves out (GDAL's: no bytes) is empty, not waited for: a window over it +/// is whole at once and asks for nothing more, and an export of the view over it is written rather +/// than refused after waiting for the tile. +#[test] +fn a_tile_a_sparse_file_leaves_out_is_empty_not_waited_for() { + use crate::render::rustgis_tiles::cog::raster::window; + use rustgis_data::raster::cog::{Example, example_with}; + use std::sync::atomic::Ordering; + let (base, sent) = serve_ranges(example_with(&Example { size: 2048, tile: 256, sparse: true, ..Example::default() })); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + // Its overview is level 1 (1024 pixels a side); level 0's first tile has no bytes. + let r = s.run("layer.addUrl", &json!({"url": url.clone(), "name": "Scene"})).unwrap(); + assert_eq!(r["width"], 1024, "{r}"); + let before = sent.load(Ordering::Relaxed); + let gray = rustgis_data::raster::ColorHint::Gray; + for _ in 0..2 { + let w = window(&url, 0, (0, 64), (0, 64), &[0], gray.clone()).unwrap(); + assert!(!w.pending, "an empty tile isn't waited for"); + assert!(w.raster.unwrap().bands[0].data.iter().all(|v| v.is_nan()), "empty"); + } + assert_eq!(sent.load(Ordering::Relaxed), before, "nothing is fetched for it"); + // The view over it (the image's north-west corner), zoomed in past the overview: written. + s.view_size = (64.0, 64.0); + s.zoom_to(&rustgis_geom::Extent::new(10.02, 45.92, 10.08, 45.98)); + s.map_mut().camera.res = 12.0; + let png = std::env::temp_dir().join(format!("rustgis-sparse-{}.png", std::process::id())); + s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap(); + assert!(png.exists()); + let _ = std::fs::remove_file(png); +} + +/// A GeoTIFF whose server refuses its header with a 429 or 503 and a `Retry-After` fails to be +/// added, saying when to ask again, and is read again on a draw once the wait is over. (That it +/// isn't asked for before then, and that the map is asked to draw again by then, so the layer +/// doesn't stay failed until something else draws it: `rustgis_tiles::cog`'s +/// `a_header_refused_with_retry_after_has_the_map_drawn_again_then`, where the gate's state and +/// the alarm can be read; here another test may empty the cache meanwhile.) +#[test] +fn a_header_refused_with_retry_after_is_read_again_when_it_says() { + use crate::render::rustgis_tiles::{cache, cog}; + use std::sync::atomic::Ordering; + use std::time::Duration; + // Refused once (busy: ask again in a second), answered after. + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 1); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + let e = s.run("layer.addUrl", &json!({"url": url.clone(), "name": "Scene"})).unwrap_err(); + assert!(e.contains("retry after 1 s"), "{e}"); + assert_eq!(sent.load(Ordering::Relaxed), 0, "refused: nothing sent"); + // Its wait over, it is read again on a draw, and answered. + let start = std::time::Instant::now(); + let mut read = None; + while read.is_none() && start.elapsed() < Duration::from_secs(10) { + let _ = cache::generation(); + if let Ok(Some(info)) = cog::info(&url) { + read = Some(info); + } + cache::wait(Duration::from_millis(50)); + } + let width = read.as_ref().and_then(|i| i.layout.levels.first()).map(|l| l.width); + assert_eq!(width, Some(512), "not read again once its wait was over"); + assert!(sent.load(Ordering::Relaxed) > 0); +} + +/// A GeoTIFF's address look (`#bands=…&range=…`) is its layer's once its pixels land, read later +/// (as the browser and an import read them) as when read at once; a palette image stretched by a +/// range is drawn in grey, as its tile layer was, and without a look in its colours. +#[test] +fn an_address_look_holds_when_the_pixels_land_later() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{AddressLook, RasterLayer, RasterRenderer, Stretch, read_together}; + // A session installs the reader. + let _s = Session::new(Project::default()); + let (base, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, bands: 8, ..Example::default() })); + let look = AddressLook { bands: vec![4, 2, 1], range: Some((10.0, 200.0)) }; + let mut rl = RasterLayer::unread_in_place(format!("{base}/scene.tif")).with_look(look); + assert!(rl.data.is_none()); + assert!(read_together(&mut [&mut rl], true).is_empty()); + assert!(rl.data.is_some()); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + let (classes, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, classes: true, ..Example::default() })); + let mut stretched = + RasterLayer::unread_in_place(format!("{classes}/classes.tif")).with_look(AddressLook { bands: vec![], range: Some((0.0, 100.0)) }); + let mut own = RasterLayer::unread_in_place(format!("{classes}/classes.tif")); + assert!(read_together(&mut [&mut stretched, &mut own], true).is_empty()); + assert!(matches!(stretched.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", stretched.style.renderer); + assert_eq!(own.style.renderer, RasterRenderer::Colormap); + // Saved before its pixels landed (the browser, a read that failed), a layer opens in its look, + // or its own one, still. + let mut s = Session::new(Project::default()); + let look = AddressLook { bands: vec![4, 2, 1], range: Some((10.0, 200.0)) }; + s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place(format!("{base}/scene.tif")).with_look(look)); + s.map_mut().add_raster("Classes".to_string(), RasterLayer::unread_in_place(format!("{classes}/classes.tif"))); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let style = + |name: &str| s.map().layers.iter().find(|l| l.name == name).and_then(|l| l.raster.as_ref()).map(|r| r.style.clone()).unwrap(); + assert_eq!(style("Scene").renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); + assert_eq!(style("Scene").stretch, Stretch::custom(10.0, 200.0)); + assert_eq!(style("Classes").renderer, RasterRenderer::Colormap); +} + +/// An in-place address a project keeps is checked as a tile layer's is before anything is asked +/// for: a hand-edited one that isn't http(s) or s3 is refused, and said, on opening and on adding. +#[test] +fn an_unchecked_in_place_address_is_refused() { + use rustgis_map::raster::RasterLayer; + let bad = "ftp://127.0.0.1/scene.tif"; + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Odd".to_string(), RasterLayer::unread_in_place(bad)); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let rl = s.map().layers.iter().find(|l| l.name == "Odd").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(rl.data.is_none() && rl.error.as_deref().is_some_and(|e| e.contains("tile addresses start with")), "{:?}", rl.error); + assert_eq!(s.notices.iter().filter(|n| n.error && n.text.contains("Odd: couldn't be read in place")).count(), 1); + // Added directly (the browser's path), refused before the reader is asked. + let added = RasterLayer::opening_in_place(bad); + assert!(added.error.is_some() && added.data.is_none()); +} + +/// A PNG export of a raster read in place whose image can't be read (its tiles, past the overview +/// it holds) is refused, not written from the overview. +#[test] +fn a_png_export_of_an_unreadable_in_place_layer_is_refused() { + use rustgis_data::raster::{ColorHint, Crs, GeoTransform, Raster}; + let mut overview = Raster::new( + "gone", + 4, + 4, + vec![(0..16).map(|v| v as f32).collect()], + GeoTransform { origin_x: -180.0, origin_y: 80.0, pixel_width: 90.0, pixel_height: -40.0 }, + Crs::LonLat, + None, + ColorHint::Gray, + "Float32", + ) + .unwrap(); + overview.overview = Some((4096, 4096)); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Gone".to_string(), rustgis_map::raster::RasterLayer::in_place(overview, "http://127.0.0.1:9/gone.tif")); + // Zoomed in far past the overview: the image's tiles are needed, and can't be read. + s.map_mut().camera.center = rustgis_geom::Coord::new(0.0, 0.0); + s.map_mut().camera.res = 25.0; + s.view_size = (64.0, 64.0); + let png = std::env::temp_dir().join(format!("rustgis-in-place-refused-{}.png", std::process::id())); + let e = s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap_err(); + assert!(e.contains("isn't written") || e.contains("still on their way"), "{e}"); + assert!(!png.exists()); +} + +/// A turned layer read in place is drawn from its overview: zoomed in past it, an export is +/// refused rather than written from the overview; zoomed out, it is written. +#[test] +fn a_turned_in_place_layer_is_not_exported_past_its_overview() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(2048, 256)); + let mut s = Session::new(Project::default()); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif"), "name": "Scene"})).unwrap(); + assert_eq!(r["width"], 1024, "{r}"); + let id = s.map().layer_by_name("Scene").unwrap().id; + s.map_mut().layer_mut(id).and_then(|l| l.raster.as_mut()).unwrap().style.rotation = 15.0; + let png = std::env::temp_dir().join(format!("rustgis-turned-{}.png", std::process::id())); + // Zoomed in past the overview (its pixels are about 50 m): refused. + s.map_mut().camera.res = 12.0; + s.view_size = (64.0, 64.0); + let e = s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap_err(); + assert!(e.contains("turned") && e.contains("isn't written"), "{e}"); + // Zoomed out to the overview's resolution and beyond: written. + s.map_mut().camera.res = 200.0; + s.run("share.exportMap", &json!({"path": png.to_string_lossy(), "width": 64, "height": 64})).unwrap(); + assert!(png.exists()); +} + +/// An address look naming bands the file hasn't (`#bands=9,8,7` on a 3-band file), or two bands +/// (`#bands=1,2`, which no raster look draws), is said, and the layer draws in its own look; its +/// range is kept. +#[test] +fn a_look_naming_bands_the_file_hasnt_is_said() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{RasterRenderer, Stretch}; + let (base, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, bands: 3, ..Example::default() })); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif#bands=9,8,7&range=10,200"), "name": "Scene"})).unwrap(); + let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("asks for band(s) 9, 8, 7")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + let rl = s.map().layer_by_name("Scene").and_then(|l| l.raster.as_ref()).unwrap(); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, "the file's own look: three grey bands as RGB"); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), 1); + // Two bands: neither grey nor RGB, said as such. + s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif#bands=1,2&range=10,200"), "name": "Pair"})).unwrap(); + assert_eq!(s.notices.iter().filter(|n| n.text.contains("Pair") && n.text.contains("asks for 2 bands")).count(), 1, "{:?}", s.notices); + let rl = s.map().layer_by_name("Pair").and_then(|l| l.raster.as_ref()).unwrap(); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + // A palette image: in grey with the range, as its tile layer draws one whose address names + // bands (its colours are of one band alone), not in its colours without the range. + let (classes, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, classes: true, ..Example::default() })); + s.run("layer.addUrl", &json!({"url": format!("{classes}/classes.tif#bands=9,8,7&range=10,200"), "name": "Classes"})).unwrap(); + let said = s + .notices + .iter() + .filter(|n| n.text.contains("Classes") && n.text.contains("asks for band(s) 9, 8, 7") && n.text.contains("drawn in grey")); + assert_eq!(said.count(), 1, "{:?}", s.notices); + let rl = s.map().layer_by_name("Classes").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(matches!(rl.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", rl.style.renderer); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); +} + +/// A file without overviews whose only image has more tiles than the fetch queue holds (small +/// tiles) is read in place all the same: its overview is asked for a batch at a time, and a tile +/// the queue dropped is asked for again, so the layer lands rather than failing with "dropped". +#[test] +fn an_overview_of_more_tiles_than_the_fetch_queue_holds_lands() { + use rustgis_data::raster::cog::{Example, example_with}; + let (base, _) = serve_ranges(example_with(&Example { size: 1024, tile: 32, no_overviews: true, ..Example::default() })); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/flat.tif"), "name": "Flat"})).unwrap(); + let rl = s.map().layer_by_name("Flat").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(rl.in_place); + let d = rl.data.as_ref().unwrap(); + assert_eq!((d.width, d.height), (1024, 1024)); + // Every tile, not a stand-in: pixel (x, y) of band 0 is (x + y) % 256. + let at = |x: u32, y: u32| d.bands[0].data[(y * d.width + x) as usize]; + assert_eq!((at(0, 1), at(1023, 1023), at(500, 700)), (1.0, 254.0, 176.0)); +} + +/// A window over more tiles than the fetch queue holds (small tiles) asks for them a batch at a +/// time, the next as the map draws again, rather than all at once (which would push its own first +/// requests out of the queue, to ask for them all again at the next draw), and is pending until +/// the last lands. (The tile cache is one for the process: other tests' reads may evict some of +/// these tiles meanwhile, so how many are fetched in all isn't checked.) +#[test] +fn a_window_of_more_tiles_than_the_fetch_queue_holds_asks_for_them_in_batches() { + use crate::render::rustgis_tiles::{cache, cog}; + use rustgis_data::raster::ColorHint; + use rustgis_data::raster::cog::{Example, example_with}; + use std::time::Duration; + let _s = Session::new(Project::default()); + let (base, _) = serve_ranges(example_with(&Example { size: 1024, tile: 32, no_overviews: true, ..Example::default() })); + let url = format!("{base}/flat.tif"); + // The header first; then the window asks for its first batch. + let start = std::time::Instant::now(); + let mut w = cog::raster::window(&url, 0, (0, 1024), (0, 1024), &[0], ColorHint::Gray).unwrap(); + while w.raster.is_none() && start.elapsed() < Duration::from_secs(20) { + cache::wait(Duration::from_millis(20)); + w = cog::raster::window(&url, 0, (0, 1024), (0, 1024), &[0], ColorHint::Gray).unwrap(); + } + assert!(w.raster.is_some() && w.pending && !w.failed); + let asked = cog::raster::asked(&url, 0); + assert!(asked <= 256, "{asked} asked for at once"); + // Drawn again as tiles land, the window asks for the rest, and is whole in the end. + while w.pending && start.elapsed() < Duration::from_secs(60) { + cache::wait(Duration::from_millis(20)); + w = cog::raster::window(&url, 0, (0, 1024), (0, 1024), &[0], ColorHint::Gray).unwrap(); + } + assert!(!w.pending && !w.failed, "still pending after {:?}", start.elapsed()); + let r = w.raster.unwrap(); + let at = |x: u32, y: u32| r.bands[0].data[(y * r.width + x) as usize]; + assert_eq!((at(0, 1), at(1023, 1023), at(500, 700)), (1.0, 254.0, 176.0)); + assert!(r.bands[0].data.iter().all(|v| v.is_finite()), "a gap"); +} + +/// A palette image's overview without a colour table under a grey full image is read as the full +/// image is, in grey, and nothing is said: a file is what its full image says (as `main` reads a +/// grey full image with a stray colour table as grey), a window over it is read the same way, and +/// the overview's own say is the one thing the reader doesn't take over the full image's. +#[test] +fn a_palette_overview_under_a_grey_full_image_is_read_grey_as_the_full_image_is() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::RasterRenderer; + // 4096 × 4096 in 256 px tiles: the overview held is the 1024 × 1024 one (a palette image + // without a table), not the full image (grey, 16 M samples). + let file = example_with(&Example { size: 4096, tile: 256, classes: true, stray: true, no_table: true, ..Example::default() }); + let (base, _) = serve_ranges(file); + let mut s = Session::new(Project::default()); + s.run("layer.addUrl", &json!({"url": format!("{base}/stray.tif"), "name": "Stray"})).unwrap(); + let rl = s.map().layer_by_name("Stray").and_then(|l| l.raster.as_ref()).unwrap(); + let d = rl.data.as_ref().unwrap(); + assert_eq!((d.width, d.overview), (1024, Some((4096, 4096))), "the overview held"); + assert!(matches!(rl.style.renderer, RasterRenderer::Stretch { band: 0, .. }), "{:?}", rl.style.renderer); + assert!(!s.notices.iter().any(|n| n.text.contains("colour table")), "{:?}", s.notices); +} + +/// A layer read in place whose overview couldn't be read (its server refused the file's header, +/// saying when to ask again) is read again once its wait is over, as the app draws, and lands then: +/// not before, and failing was said once. +#[test] +fn an_in_place_layers_failed_overview_is_read_again_when_its_wait_is_over() { + use rustgis_map::raster::RasterLayer; + use std::sync::atomic::Ordering; + use std::time::Duration; + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 2); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place(format!("{base}/scene.tif"))); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let layer = |s: &Session| s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + let rl = layer(&s); + assert!(rl.data.is_none()); + assert!(rl.error.as_deref().is_some_and(|e| e.contains("retry after 1 s")), "{:?}", rl.error); + assert_eq!(rl.read_failure().map(|(_, _, failures)| failures), Some(1)); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // Not before its wait is over. + s.read_in_place_again(); + assert!(layer(&s).data.is_none()); + assert_eq!(sent.load(Ordering::Relaxed), 0, "read again before its wait was over"); + // Nor by a repair of the project (a layer file added, maps imported), which reads the layers + // not read yet together: a failed read is paced, not started over. + let at = layer(&s).read_failure().map(|(_, at, _)| at).unwrap(); + let mut other = rustgis_map::Map::new("Other"); + let id = other.add_dataset(crate::data::Dataset::new("Empty", crate::data::GeometryType::Point, Vec::new())); + crate::cmd::layer::add_layer_text(&mut s, &serde_json::to_string(other.layer(id).unwrap()).unwrap()).unwrap(); + assert_eq!( + layer(&s).read_failure().map(|(_, at, failures)| (at, failures)), + Some((at, 1)), + "started over by the repair: {:?}", + layer(&s).error + ); + assert_eq!(sent.load(Ordering::Relaxed), 0); + // Then read again, as the map draws: refused again, it waits twice as long this time (see + // `cache::backoff`), and isn't said again. + let view = rustgis_geom::Extent::new(10.0, 45.9, 10.1, 46.0); + let draw = |s: &mut Session| { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + s.read_in_place_again(); + s.map_mut().sync_sources(&view, 1000.0, false); + s.notice_landed_rasters(); + }; + let start = std::time::Instant::now(); + while layer(&s).read_failure().is_none_or(|(_, _, failures)| failures < 2) && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + let rl = layer(&s); + let (_, at, failures) = rl.read_failure().expect("refused again"); + assert_eq!(failures, 2); + assert_eq!(sent.load(Ordering::Relaxed), 0); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // Not before twice its wait is over (looked at while that is still ahead). + if at.elapsed() < Duration::from_millis(1800) { + s.read_in_place_again(); + assert!(layer(&s).read_failure().is_some(), "read again before twice its wait was over"); + } + // Then read a third time, landing as the map draws. + let start = std::time::Instant::now(); + while layer(&s).data.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + let rl = layer(&s); + assert!(rl.data.is_some(), "{:?}", rl.error); + assert!(rl.error.is_none() && rl.read_failure().is_none()); + assert!(sent.load(Ordering::Relaxed) > 0); + assert_eq!(said(&s), 1, "{:?}", s.notices); +} + +/// A cloud-optimised GeoTIFF added without waiting before its header is in (the browser, where +/// nothing may wait, so an address isn't probed first as it is natively) answers that it is still +/// being read, with no size yet rather than none; the size comes with the header (see +/// `a_cog_added_without_waiting_lands_as_the_map_draws`). +#[test] +fn a_cog_added_before_its_header_is_in_says_it_is_being_read() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; + let gate = std::sync::Arc::new(AtomicBool::new(false)); + let (base, _) = serve_gated(rustgis_data::raster::cog::example(512, 256), true, Some(gate.clone())); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + s.in_place_wait = false; + // As the browser adds it: straight in place, its header read with its overview. + let r = crate::cmd::layer::add_cog(&mut s, &json!({"name": "Scene"}), &url, false).unwrap(); + assert_eq!((r["inPlace"].clone(), r["reading"].clone()), (json!(true), json!(true)), "{r}"); + assert!(r["width"].is_null() && r["height"].is_null() && r["bands"].is_null(), "sized before its header is in: {r}"); + assert!(r.get("bounds").is_none(), "{r}"); + // Its header and overview arrive as the map draws. + gate.store(true, Ordering::Relaxed); + let layer = |s: &Session| s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + let view = rustgis_geom::Extent::new(10.0, 45.0, 11.0, 46.0); + let start = std::time::Instant::now(); + while layer(&s).data.is_none() && start.elapsed() < Duration::from_secs(20) { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + s.map_mut().sync_sources(&view, 1000.0, false); + s.notice_landed_rasters(); + } + let rl = layer(&s); + assert_eq!(rl.data.as_ref().map(|d| (d.width, d.band_count())), Some((512, 1)), "{:?}", rl.error); +} + +/// A raster layer read in place whose read can't start (a project edited by hand keeps an address +/// that can't be read) says so once, and a repair of the project (a layer file added, maps +/// imported) leaves it as it is: not started over, not said again. +#[test] +fn a_read_in_place_that_couldnt_start_isnt_started_over_by_a_repair() { + use rustgis_map::raster::RasterLayer; + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), RasterLayer::unread_in_place("ftp://files.example.org/scene.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let layer = |s: &Session| s.project.maps[0].layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + let rl = layer(&s); + assert!(rl.data.is_none() && rl.read_failure().is_none(), "{rl:?}"); + let error = rl.error.clone().expect("couldn't start"); + assert!(error.contains("start with https://"), "{error}"); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + let mut other = rustgis_map::Map::new("Other"); + let id = other.add_dataset(crate::data::Dataset::new("Empty", crate::data::GeometryType::Point, Vec::new())); + crate::cmd::layer::add_layer_text(&mut s, &serde_json::to_string(other.layer(id).unwrap()).unwrap()).unwrap(); + s.run("map.import", &json!({"text": serde_json::to_string(&rustgis_map::Map::new("Other")).unwrap()})).unwrap(); + s.notice_landed_rasters(); + let rl = layer(&s); + assert_eq!((rl.error.as_deref(), rl.read_failure().is_none()), (Some(error.as_str()), true), "started over: {rl:?}"); + assert_eq!(said(&s), 1, "said again: {:?}", s.notices); +} + +/// Where the session waits for reads in place (the command line, Python), a failed overview is +/// read again by the next command run once its wait is over, and waited for: a long-lived session +/// that doesn't draw has nothing else to ask again. +#[test] +fn a_waiting_sessions_next_command_reads_a_failed_overview_again() { + use std::sync::atomic::Ordering; + use std::time::Duration; + let (base, sent) = serve_refusing_first(rustgis_data::raster::cog::example(512, 256), 1); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), rustgis_map::raster::RasterLayer::unread_in_place(format!("{base}/scene.tif"))); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let layer = |s: &Session| s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + assert_eq!(layer(&s).read_failure().map(|(_, _, failures)| failures), Some(1), "{:?}", layer(&s).error); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Scene: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // A command run before its wait is over doesn't read it. + s.run("info.project", &json!({})).unwrap(); + assert!(layer(&s).data.is_none() && layer(&s).read_failure().is_some()); + assert_eq!(sent.load(Ordering::Relaxed), 0, "read again before its wait was over"); + // One run after does, waiting for it: the layer holds its overview as the command answers. + std::thread::sleep(Duration::from_millis(1100)); + s.run("info.project", &json!({})).unwrap(); + let rl = layer(&s); + assert!(rl.data.is_some(), "{:?}", rl.error); + assert!(rl.error.is_none() && rl.read_failure().is_none()); + assert!(sent.load(Ordering::Relaxed) > 0); + assert_eq!(said(&s), 1, "{:?}", s.notices); +} + +/// Where the session doesn't wait for reads in place (the desktop app, whose commands run on the +/// window's own thread), a cloud-optimised GeoTIFF is added at once, its header probed as the app +/// does, and its overview lands as the map draws; one that can't be read is added too, and says so +/// once its read fails. A project opens the same way. Waiting, the command fails instead. +#[test] +fn a_cog_added_without_waiting_lands_as_the_map_draws() { + use std::time::Duration; + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let url = format!("{base}/scene.tif"); + let mut s = Session::new(Project::default()); + s.in_place_wait = false; + let _ = crate::render::rustgis_tiles::cog::probe_blocking(&url); + let r = s.run("layer.addUrl", &json!({"url": url, "name": "Scene"})).unwrap(); + assert_eq!((r["inPlace"].as_bool(), r["kind"].as_str()), (Some(true), Some("raster")), "{r}"); + assert_eq!(r["bounds"], json!([10.0, 45.0, 11.0, 46.0]), "framed on the header read: {r}"); + // Sized by the header read, as the answer is once the overview has landed (512 a side: held + // whole, so not an overview of anything). + assert_eq!( + (r["width"].clone(), r["height"].clone(), r["bands"].clone(), r["reading"].clone(), r.get("overviewOf")), + (json!(512), json!(512), json!(1), json!(true), None), + "{r}" + ); + let layer = |s: &Session, name: &str| s.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); + let rl = layer(&s, "Scene"); + assert!(rl.in_place && rl.data.is_none() && rl.error.is_none(), "added before its overview is in: {rl:?}"); + let view = rustgis_geom::Extent::new(10.0, 45.0, 11.0, 46.0); + let draw = |s: &mut Session| { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + s.read_in_place_again(); + s.map_mut().sync_sources(&view, 1000.0, false); + s.notice_landed_rasters(); + }; + let start = std::time::Instant::now(); + while layer(&s, "Scene").data.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + let rl = layer(&s, "Scene"); + assert_eq!(rl.data.as_ref().map(|d| (d.width, d.height)), Some((512, 512)), "{:?}", rl.error); + // One whose header reads (the first request) but whose tiles can't be is added, and says so + // once its read fails. + let (broken, _) = serve_refusing_from(rustgis_data::raster::cog::example(512, 256), 2); + let broken = format!("{broken}/broken.tif"); + s.run("layer.addUrl", &json!({"url": broken, "name": "Broken"})).unwrap(); + let start = std::time::Instant::now(); + while layer(&s, "Broken").error.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut s); + } + assert!(layer(&s, "Broken").error.as_deref().is_some_and(|e| e.contains("500")), "{:?}", layer(&s, "Broken").error); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Broken: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices); + // Waiting, the same address fails the command. + s.in_place_wait = true; + assert!(s.run("layer.addUrl", &json!({"url": broken, "name": "Again"})).unwrap_err().contains("500")); + // A project opens the same way: its layers read in place land as the map draws. + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.in_place_wait = false; + t.run("project.openText", &json!({"text": text})).unwrap(); + assert!(layer(&t, "Scene").data.is_none(), "opened before its overview is in"); + let start = std::time::Instant::now(); + while layer(&t, "Scene").data.is_none() && start.elapsed() < Duration::from_secs(20) { + draw(&mut t); + } + assert!(layer(&t, "Scene").data.is_some(), "{:?}", layer(&t, "Scene").error); +} + +/// Binding a saved S3 connection to a project's rasters read in place reads their overviews +/// together, in the time of the slowest (see `s3_connections::reopen_where`), not one after the +/// other: the stand-in bucket answers nothing until both files have been asked for, so reads made +/// one at a time would never get their first answer. +#[test] +fn binding_a_connection_reads_its_in_place_rasters_together() { + use rustgis_map::raster::RasterLayer; + use std::time::Duration; + let base = serve_s3_together(vec![ + ("a.tif", rustgis_data::raster::cog::example(512, 256)), + ("b.tif", rustgis_data::raster::cog::example(256, 512)), + ]); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("A".to_string(), RasterLayer::unread_in_place("s3://s3-together-bucket/a.tif")); + s.map_mut().add_raster("B".to_string(), RasterLayer::unread_in_place("s3://s3-together-bucket/b.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run( + "connection.new", + &json!({"kind": "s3", "name": "Together", "address": base, "options": {"credentials": "none", "buckets": "s3-together-bucket"}}), + ) + .unwrap(); + let start = std::time::Instant::now(); + t.run("project.openText", &json!({"text": text})).unwrap(); + for (name, width) in [("A", 512), ("B", 256)] { + let rl = t.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); + assert_eq!(rl.data.as_ref().map(|d| d.width), Some(width), "{name}: {:?}", rl.error); + } + assert!(start.elapsed() < Duration::from_secs(60), "read one after the other: {:?}", start.elapsed()); + t.run("connection.remove", &json!({"id": "Together"})).unwrap(); +} + +/// Where the session doesn't wait for reads in place (the desktop), a saved S3 connection chosen +/// for a project's rasters read in place starts their overviews reading and says so: they count +/// as `reading` in `connection.resolveS3`'s answer, not as `reopened`, until they land as the map +/// draws. Waiting, the same choice reads them before it answers, and they are reopened. +#[test] +fn a_connection_chosen_without_waiting_counts_its_in_place_rasters_as_reading() { + use rustgis_map::raster::RasterLayer; + use std::time::Duration; + let base = serve_s3_together(vec![ + ("a.tif", rustgis_data::raster::cog::example(512, 256)), + ("b.tif", rustgis_data::raster::cog::example(256, 512)), + ]); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("A".to_string(), RasterLayer::unread_in_place("s3://s3-chosen-bucket/a.tif")); + s.map_mut().add_raster("B".to_string(), RasterLayer::unread_in_place("s3://s3-chosen-bucket/b.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let layer = |s: &Session, name: &str| s.map().layer_by_name(name).and_then(|l| l.raster.clone()).unwrap(); + // The connection lists no bucket, so the project's sources wait for the user's choice of it. + let open = |wait: bool| { + let mut t = Session::new(Project::default()); + t.in_place_wait = wait; + let c = + t.run("connection.new", &json!({"kind": "s3", "name": "Chosen", "address": base, "options": {"credentials": "none"}})).unwrap(); + let id = c["id"].as_str().unwrap_or_default().to_string(); + assert!(!id.is_empty(), "{c}"); + t.run("project.openText", &json!({"text": text})).unwrap(); + let source = t.s3_pending.first().map(|p| p.source.clone()).unwrap_or_default(); + assert!(!source.is_empty(), "nothing waited for a choice: {:?}", t.notices); + let r = t.run("connection.resolveS3", &json!({"source": source, "id": id})).unwrap(); + (t, r, id) + }; + let counts = |r: &Value| (r["reopened"].as_u64(), r["reading"].as_u64(), r["failed"].as_u64()); + let (mut t, r, id) = open(false); + assert_eq!(counts(&r), (Some(0), Some(2), Some(0)), "{r}"); + assert!(layer(&t, "A").data.is_none() && layer(&t, "B").data.is_none(), "chosen before the overviews are in"); + let view = rustgis_geom::Extent::new(10.0, 45.0, 11.0, 46.0); + let start = std::time::Instant::now(); + while (layer(&t, "A").data.is_none() || layer(&t, "B").data.is_none()) && start.elapsed() < Duration::from_secs(60) { + crate::render::rustgis_tiles::cache::wait(Duration::from_millis(50)); + t.read_in_place_again(); + t.map_mut().sync_sources(&view, 1000.0, false); + t.notice_landed_rasters(); + } + for (name, width) in [("A", 512), ("B", 256)] { + assert_eq!(layer(&t, name).data.as_ref().map(|d| d.width), Some(width), "{name}: {:?}", layer(&t, name).error); + } + t.run("connection.remove", &json!({"id": id})).unwrap(); + // Waiting, the choice answers once the overviews are in. + let (mut t, r, id) = open(true); + assert_eq!(counts(&r), (Some(2), Some(0), Some(0)), "{r}"); + assert!(layer(&t, "A").data.is_some() && layer(&t, "B").data.is_some()); + t.run("connection.remove", &json!({"id": id})).unwrap(); +} + +/// A stand-in S3 endpoint serving `files` (key, bytes) by range, answering nothing until each key +/// has been asked for once: reads made together get their answers, reads made one after the other +/// never get the first. +fn serve_s3_together(files: Vec<(&'static str, Vec)>) -> String { + use std::io::{Read, Write}; + use std::time::Duration; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + let files = std::sync::Arc::new(files); + let asked: std::sync::Arc>> = Default::default(); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut s) = stream else { continue }; + let (files, asked) = (files.clone(), asked.clone()); + std::thread::spawn(move || { + let mut buf = [0u8; 8192]; + let read = s.read(&mut buf).unwrap_or(0); + let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); + let path = req.split_whitespace().nth(1).unwrap_or("/").split(['?', '#']).next().unwrap_or("/").to_string(); + let Some((key, body)) = files.iter().find(|(k, _)| path.ends_with(&format!("/{k}"))) else { + let _ = s.write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + return; + }; + asked.lock().unwrap().insert(key); + let start = std::time::Instant::now(); + while asked.lock().unwrap().len() < files.len() && start.elapsed() < Duration::from_secs(150) { + std::thread::sleep(Duration::from_millis(5)); + } + if req.starts_with("head ") { + let _ = write!(s, "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()); + return; + } + let range = req.lines().find_map(|l| l.strip_prefix("range: bytes=")).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + if a.is_empty() { + let n = b.parse::().ok()?.min(body.len()); + return Some((body.len() - n, body.len().saturating_sub(1))); + } + Some((a.parse::().ok()?, b.parse::().ok()?)) + }); + let (status, part) = match range { + Some((a, b)) if a < body.len() => ("206 Partial Content", &body[a..=b.min(body.len() - 1)]), + Some(_) => ("416 Range Not Satisfiable", &body[..0]), + None => ("200 OK", &body[..]), + }; + let _ = write!(s, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", part.len()); + let _ = s.write_all(part); + }); + } + }); + base +} + +/// A COG tile layer opened as a raster layer takes a numbered name when another layer has its +/// name, as a layer added does. +#[test] +fn a_migrated_cog_layer_takes_a_free_name() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + let empty = + r#"{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}"#; + s.run("layer.addDataText", &json!({"fileName": "scene.geojson", "text": empty, "name": "Scene"})).unwrap(); + assert!(s.map().layer_by_name("Scene").is_some()); + let mut text: Value = serde_json::to_value(&s.project).unwrap(); + let tile = serde_json::to_value(rustgis_map::TileLayer::new("Scene", rustgis_map::TileKind::Cog, format!("{base}/scene.tif"))).unwrap(); + match text.pointer_mut("/maps").and_then(Value::as_array_mut) { + Some(maps) => maps[0]["tiles"] = json!([tile]), + None => text["map"]["tiles"] = json!([tile]), + } + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text.to_string()})).unwrap(); + let names: Vec<&str> = t.map().layers.iter().map(|l| l.name.as_str()).collect(); + assert!(names.contains(&"Scene") && names.contains(&"Scene (2)"), "{names:?}"); + assert!(t.map().layer_by_name("Scene (2)").and_then(|l| l.raster.as_ref()).is_some_and(|r| r.in_place)); +} + +/// A layer read in place that can't be read says so, once, as its project opens. +#[test] +fn an_in_place_layer_that_cant_be_read_says_so() { + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Gone".to_string(), rustgis_map::raster::RasterLayer::unread_in_place("http://127.0.0.1:9/gone.tif")); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + let said = |s: &Session| s.notices.iter().filter(|n| n.error && n.text.contains("Gone: couldn't be read in place")).count(); + assert_eq!(said(&s), 1, "{:?}", s.notices.last()); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), 1); +} + +/// A project an older RustGIS saved again (it didn't know `in_place`: the layer was missing there, +/// and saved without the key) reads its cloud-optimised GeoTIFF in place again: a raster layer +/// whose path is a plain web address is read no other way. +#[test] +fn a_project_an_older_rustgis_saved_again_reads_its_cog_in_place() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + s.map_mut().add_raster("Scene".to_string(), rustgis_map::raster::RasterLayer::unread_in_place(format!("{base}/scene.tif"))); + let mut text: Value = serde_json::to_value(&s.project).unwrap(); + // As the older RustGIS saved it: the path, without `in_place`. + let raster = text.pointer_mut("/maps/0/layers/0/raster").and_then(Value::as_object_mut).unwrap(); + assert_eq!(raster.remove("in_place"), Some(json!(true))); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text.to_string()})).unwrap(); + let rl = s.map().layer_by_name("Scene").and_then(|l| l.raster.clone()).unwrap(); + assert!(rl.in_place && rl.error.is_none(), "{:?}", rl.error); + assert_eq!(rl.data.as_ref().map(|d| d.width), Some(512)); +} + +/// A palette COG without its colour table is drawn in grey and says so, also when its pixels land +/// after the layer is added (a project opened, as in the browser and an import): once, and not for +/// one with its table. +#[test] +fn a_palette_without_its_table_says_so_when_its_pixels_land() { + use rustgis_data::raster::cog::{Example, example_with}; + let clean = example_with(&Example { size: 300, tile: 128, classes: true, ..Example::default() }); + let lossy = example_with(&Example { size: 300, tile: 128, classes: true, no_table: true, ..Example::default() }); + let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("no colour table")).count(); + for (file, want) in [(lossy, 1), (clean, 0)] { + let (base, _) = serve_ranges(file); + let mut s = Session::new(Project::default()); + let layer = rustgis_map::raster::RasterLayer::unread_in_place(format!("{base}/classes.tif")); + s.map_mut().add_raster("Classes".to_string(), layer); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + assert!(s.map().layers.iter().any(|l| l.raster.as_ref().is_some_and(|r| r.data.is_some())), "read on opening"); + assert_eq!(said(&s), want, "{:?}", s.notices.iter().map(|n| n.text.as_str()).collect::>()); + // Said once for these pixels, however often asked. + s.notice_landed_rasters(); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), want); + } +} + +/// A data cube's file (`"cube": true`) is a tile layer, and stays one when its project opens +/// again: its address keeps the marker, which the migration respects. +#[test] +fn a_data_cubes_file_stays_a_tile_layer_when_its_project_reopens() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/cube-2020.tif"), "cube": true})).unwrap(); + assert_eq!(r["kind"], "cog", "{r}"); + assert_eq!(s.map().tiles.len(), 1); + assert!(s.map().tiles[0].url.ends_with("#cube=1"), "{}", s.map().tiles[0].url); + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + assert_eq!((t.map().tiles.len(), t.map().layers.iter().filter(|l| l.raster.is_some()).count()), (1, 0)); +} + +/// Dated-series keys given as `null` (a script's unused ones) ask for nothing: the GeoTIFF is a +/// raster layer read in place, not a tile layer. +#[test] +fn null_series_keys_leave_a_geotiff_read_in_place() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let mut s = Session::new(Project::default()); + let p = json!({"url": format!("{base}/scene.tif"), "start": null, "end": null, "dates": null}); + let r = s.run("layer.addUrl", &p).unwrap(); + assert_eq!((r["kind"].clone(), r["inPlace"].clone()), (json!("raster"), json!(true)), "{r}"); + assert!(s.map().tiles.is_empty()); +} + +/// A layer read in place says its statistics and tools read an overview: once when added, once +/// when a project with it opens (its pixels landing then), and not again for the same pixels. +#[test] +fn in_place_layers_say_their_overview_once() { + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(2048, 256)); + let said = |s: &Session| s.notices.iter().filter(|n| n.text.contains("raster tools read a 1024 × 1024 overview")).count(); + let mut s = Session::new(Project::default()); + let r = s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif")})).unwrap(); + // The result names the full size, as a file opened from an overview's does. + assert_eq!((r["width"].clone(), r["overviewOf"].clone()), (json!(1024), json!([2048, 2048])), "{r}"); + s.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&s), 1, "{:?}", s.notices.last()); + let text = serde_json::to_string(&s.project).unwrap(); + let mut s = Session::new(Project::default()); + s.run("project.openText", &json!({"text": text})).unwrap(); + s.notice_landed_rasters(); + assert_eq!(said(&s), 1, "{:?}", s.notices.last()); +} + +/// A project saved when cloud-optimised GeoTIFFs read in place were tile layers opens them as +/// raster layers read in place, each where it was drawn, with its name, visibility, opacity, +/// look, connection and anonymity; a dated series stays a tile layer. +#[test] +fn cog_tile_layers_of_older_projects_open_as_rasters() { + use rustgis_data::raster::cog::{Example, example_with}; + use rustgis_map::raster::{RasterRenderer, Stretch}; + let (base, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, bands: 8, ..Example::default() })); + let mut s = Session::new(Project::default()); + s.run("layer.addData", &json!({"path": "points.geojson", "text": "{\"type\":\"FeatureCollection\",\"features\":[]}"})).ok(); + let mut text: Value = serde_json::to_value(&s.project).unwrap(); + let tile = |name: &str, url: String, opacity: f64, visible: bool| { + let mut l = rustgis_map::TileLayer::new(name, rustgis_map::TileKind::Cog, url); + // Scene's credit, zoom range, overlay flag and legend: what a raster layer hasn't. + if name == "Scene" { + l.attribution = "© Example".into(); + l.min_zoom = 5; + l.overlay = true; + l.legends = vec![rustgis_data::ogc::Legend { url: format!("{base}/legend.png"), min_scale: None, max_scale: None }]; + l.connection = "conn-scene".into(); + } + if name == "Aerial" { + l.anonymous = true; + } + let mut t = serde_json::to_value(l).unwrap(); + t["opacity"] = json!(opacity); + t["visible"] = json!(visible); + t + }; + // Tile layers list the top one first: Scene over Aerial, both under the map's layers. + let tiles = json!([ + tile("Scene", format!("{base}/scene.tif#bands=5,3,2&range=10,200"), 0.5, false), + tile("Series", format!("{base}/{{date:YYYY}}.tif"), 1.0, true), + tile("Aerial", format!("{base}/aerial.tif"), 1.0, true), + ]); + let maps = text.pointer_mut("/maps").and_then(Value::as_array_mut); + match maps { + Some(maps) => maps[0]["tiles"] = tiles, + None => text["map"]["tiles"] = tiles, + } + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text.to_string()})).unwrap(); + let scene = t.map().layers.iter().find(|l| l.name == "Scene").unwrap(); + let rl = scene.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.data.is_some() && rl.path.as_deref() == Some(&*format!("{base}/scene.tif")), "{:?}", rl.path); + assert_eq!(rl.style.renderer, RasterRenderer::Rgb { red: 4, green: 2, blue: 1 }); + assert_eq!(rl.style.stretch, Stretch::custom(10.0, 200.0)); + assert!(!scene.visible && (scene.opacity() - 0.5).abs() < 1e-6); + // Read again through the connection it came from; an anonymous layer stays one. + assert_eq!((scene.connection.as_str(), scene.anonymous), ("conn-scene", false)); + let aerial = t.map().layers.iter().find(|l| l.name == "Aerial").unwrap(); + assert!(aerial.anonymous && aerial.connection.is_empty(), "{aerial:?}"); + // Drawn under the map's layers, as they were, Scene over Aerial. + let names: Vec<&str> = t.map().layers.iter().map(|l| l.name.as_str()).collect(); + assert_eq!(names[names.len() - 2..], ["Scene", "Aerial"], "{names:?}"); + assert_eq!(t.map().tiles.iter().map(|x| x.name.as_str()).collect::>(), ["Series"]); + // Scene's settings a raster layer hasn't are said to be dropped, once; Aerial had none. + let said = |t: &Session| t.notices.iter().filter(|n| n.text.contains("which have no attribution, legend, overlay, zoom range")).count(); + assert_eq!(said(&t), 1, "{:?}", t.notices.last()); + assert!(t.notices.iter().any(|n| n.text.contains("not kept for Scene") && !n.text.contains("Aerial")), "{:?}", t.notices.last()); + t.run("map.zoomIn", &Value::Null).unwrap(); + assert_eq!(said(&t), 1); +} + +/// A plain COG tile layer added this session (`map.addTiles`, kind `cog`) stays a tile layer, with +/// what a raster layer hasn't, through a layer file added and maps imported (each repairs the +/// project): it opens as a raster layer read in place once the project is saved and opened again, +/// as an older project's does. A map file's plain COG tile layers open as raster layers as the map +/// is imported, as a project's do. +#[test] +fn a_cog_tile_layer_added_this_session_stays_one_until_the_project_opens_again() { + use rustgis_map::{TileKind, TileLayer}; + let (base, _) = serve_ranges(rustgis_data::raster::cog::example(512, 256)); + let url = format!("{base}/scene.tif"); + let _ = crate::render::rustgis_tiles::cog::probe_blocking(&url); + let mut s = Session::new(Project::default()); + s.run("map.addTiles", &json!({"url": url, "kind": "cog", "name": "Scene", "attribution": "© Example"})).unwrap(); + let scene = |s: &Session| s.project.maps[0].tiles.iter().find(|t| t.name == "Scene").map(|t| (t.kind, t.attribution.clone())); + assert_eq!(scene(&s), Some((TileKind::Cog, "© Example".to_string())), "{:?}", s.project.maps[0].tiles); + let mut other = rustgis_map::Map::new("Other"); + let id = other.add_dataset(crate::data::Dataset::new("Empty", crate::data::GeometryType::Point, Vec::new())); + crate::cmd::layer::add_layer_text(&mut s, &serde_json::to_string(other.layer(id).unwrap()).unwrap()).unwrap(); + s.run("map.import", &json!({"text": serde_json::to_string(&other).unwrap()})).unwrap(); + assert_eq!(scene(&s), Some((TileKind::Cog, "© Example".to_string())), "converted by a repair"); + assert!(s.project.maps[0].layer_by_name("Scene").is_none()); + assert!(!s.notices.iter().any(|n| n.text.contains("which have no attribution")), "{:?}", s.notices); + // An older map file's plain COG tile layer opens as a raster layer read in place as it is + // imported. + let mut old = serde_json::to_value(rustgis_map::Map::new("Old")).unwrap(); + old["tiles"] = json!([serde_json::to_value(TileLayer::new("Aerial", TileKind::Cog, format!("{base}/aerial.tif"))).unwrap()]); + s.run("map.import", &json!({"text": old.to_string()})).unwrap(); + let imported = s.project.maps.iter().find(|m| m.name == "Old").unwrap(); + let rl = imported.layer_by_name("Aerial").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(imported.tiles.is_empty() && rl.in_place && rl.data.is_some(), "{:?}", rl.error); + // Saved and opened again, Scene is a raster layer read in place too. + let text = serde_json::to_string(&s.project).unwrap(); + let mut t = Session::new(Project::default()); + t.run("project.openText", &json!({"text": text})).unwrap(); + let rl = t.project.maps[0].layer_by_name("Scene").and_then(|l| l.raster.as_ref()).unwrap(); + assert!(t.project.maps[0].tiles.is_empty() && rl.in_place && rl.data.is_some(), "{:?}", rl.error); +} + /// A server answering every request with `body`, logging each as `HEAD`, `GET` or `GET `. /// Its HEAD answers carry `announce` as `X-TIFF-Header-Length`, `kind` as `Content-Type` and, /// with `ranges`, `Accept-Ranges: bytes`. @@ -26988,18 +28055,21 @@ fn geolibre_group_without_id_does_not_claim_ungrouped_layers() { assert!(s.map().group_of(id).is_none()); } -/// Declared COGs stream even without a TIFF suffix; a tile's folder look is baked only once. +/// Declared COGs are raster layers read in place even without a TIFF suffix, kept when their server +/// can't be reached at import. #[test] -fn geolibre_extensionless_cog_uses_tile_storage_and_inherited_opacity() { +fn geolibre_extensionless_cog_is_a_raster_read_in_place() { let mut s = world(); let text = json!({"version":"0.2.0","name":"COG","mapView":{},"basemapStyleUrl":"","layerGroups":[{"id":"g","name":"Folder","opacity":0.5}],"layers":[{"id":"c","name":"Coverage","type":"cog","groupId":"g","opacity":0.6,"source":{"url":"http://127.0.0.1:9/extensionless-coverage"}}]}).to_string(); let out = s.run("project.importGeolibre", &json!({"text":text})).unwrap(); - assert_eq!(out["layers"], json!(["Coverage"])); - assert!(s.map().layers.is_empty()); - assert_eq!(s.map().tiles.len(), 1); - assert_eq!(s.map().tiles[0].kind, rustgis_map::tiles::TileKind::Cog); - assert!((s.map().tiles[0].opacity - 0.3).abs() < 1e-6); - assert!(s.map().groups.iter().all(|g| g.children.is_empty())); + assert_eq!(out["layers"], json!(["Coverage"]), "{out}"); + // A raster layer read in place, kept although its server can't be reached now (it says why). + assert!(s.map().tiles.is_empty()); + let l = s.map().layers.iter().find(|l| l.name == "Coverage").unwrap(); + let rl = l.raster.as_ref().unwrap(); + assert!(rl.in_place && rl.data.is_none() && rl.error.is_some(), "{:?}", rl.error); + // Drawn at its own opacity in its folder's: 0.6 × 0.5. + assert!((s.map().layer_opacity(l) - 0.3).abs() < 1e-6, "{}", s.map().layer_opacity(l)); assert!(s.take_url_requests().is_empty()); } @@ -27666,18 +28736,16 @@ fn a_cog_window_comes_in_as_a_raster() { (ids.rounded.as_slice(), ids.value(0, 0, 0), ids.value(0, 1, 0)), ([32768u64].as_slice(), Some(16_777_216.0), Some(16_777_216.0)) ); - // Drawn in place, a palette file added for the first time says it is grey, without its - // colour table and names (the header is read before it is added). + // Read in place, a palette file is a raster layer whose overview comes with its colour table + // and class names, as a window does: nothing is said to be left out. let (fresh, _) = serve_ranges(example_with(&Example { size: 512, tile: 256, classes: true, ..Example::default() })); let mark = s.notice_mark(); s.run("layer.addUrl", &json!({"url": format!("{fresh}/cover.tif"), "name": "In place"})).unwrap(); let said: Vec = s.notices_since(mark).iter().map(|n| n.text.clone()).collect(); - assert!( - said.iter() - .any(|n| n - == "In place: drawn in place in grey, without its colour table and band or class names: a window (\"bbox\") reads them"), - "{said:?}" - ); + assert!(said.iter().all(|n| !n.contains("drawn in place") && !n.contains("left out")), "{said:?}"); + let held = raster(&s, "In place"); + let ColorHint::Palette(table) = &held.color else { panic!("{:?}", held.color) }; + assert_eq!((table[3], held.bands[0].class_name(3.0)), ([3, 252, 0, 255], Some("Forest Formation"))); let mark = s.notice_mark(); s.run("layer.addUrl", &json!({"url": format!("{base}/scene.tif"), "name": "Plain"})).unwrap(); assert!(s.notices_since(mark).iter().all(|n| !n.text.contains("drawn in place")), "a grey file says nothing"); diff --git a/crates/engine/src/tests/service_logins.rs b/crates/engine/src/tests/service_logins.rs index b700d1f54..2fb107c60 100644 --- a/crates/engine/src/tests/service_logins.rs +++ b/crates/engine/src/tests/service_logins.rs @@ -44,7 +44,17 @@ struct LoginFile { #[cfg(not(target_arch = "wasm32"))] impl LoginFile { + /// A server that sends the whole file, whatever is asked for (as before ranges were answered). fn new(body: Vec) -> Self { + Self::with(body, false) + } + + /// A server that answers a range asked for with that part (a file read in place asks so). + fn ranged(body: Vec) -> Self { + Self::with(body, true) + } + + fn with(body: Vec, ranges: bool) -> Self { use std::io::{Read, Write}; use std::sync::atomic::{AtomicBool, Ordering}; let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); @@ -76,10 +86,30 @@ impl LoginFile { let request = String::from_utf8_lossy(&request).into_owned(); let authorized = request.to_ascii_lowercase().contains("\r\nx-key: fixture-login-757\r\n") || request.split_whitespace().nth(1).is_some_and(|p| p.starts_with("/public")); + // A range asked for (a file read in place): that part. + let range = + request.lines().find_map(|l| l.to_ascii_lowercase().strip_prefix("range: bytes=").map(str::to_string)).and_then(|r| { + let (a, b) = r.trim().split_once('-')?; + let last = body.len().checked_sub(1)?; + if a.is_empty() { + let n = b.parse::().ok()?.min(body.len()); + return Some((body.len() - n, last)); + } + Some((a.parse::().ok()?, b.parse::().ok()?.min(last))) + }); + let range = range.filter(|_| ranges); + let head = request.to_ascii_lowercase().starts_with("head "); log.lock().unwrap().push(request); - let (status, bytes) = if authorized { ("200 OK", body.as_slice()) } else { ("401 Unauthorized", &[][..]) }; + let (status, bytes) = match (authorized, range) { + (false, _) => ("401 Unauthorized", &[][..]), + (true, Some((a, b))) if a <= b => ("206 Partial Content", &body[a..=b]), + (true, _) => ("200 OK", body.as_slice()), + }; let _ = write!(stream, "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", bytes.len()); - let _ = stream.write_all(bytes); + // A HEAD request is answered with the length alone. + if !head { + let _ = stream.write_all(bytes); + } } }); Self { base, requests, stop, worker: Some(worker) } @@ -1401,3 +1431,41 @@ fn delayed_head_refusal_keeps_secrets_out_of_history_across_login_lifecycle() { } } } + +/// A cloud-optimised GeoTIFF added by address from a saved web-service connection is a raster +/// layer read in place that records the connection, as the tile layer it was did, its address +/// bound to the connection's login (never carrying the key) and read with it; the login made +/// again, its address is bound to the new login and it is read again through it. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_cog_read_in_place_from_a_connection_records_it_and_follows_its_login() { + let server = LoginFile::ranged(rustgis_data::raster::cog::example(512, 256)); + let mut s = session(None); + let id = new(&mut s, json!({"kind":"ogcapi","address":format!("{}/collections",server.base),"auth":"header","keyName":"X-Key"})); + s.run("connection.login", &json!({"id":id,"apiKey":"fixture-login-757","allowInsecure":true})).unwrap(); + let r = s.run("layer.addUrl", &json!({"url":format!("{}/scene.tif",server.base),"name":"Scene","connection":id})).unwrap(); + assert_eq!((r["inPlace"].as_bool(), r["width"].as_u64()), (Some(true), Some(512)), "{r}"); + let path = { + let layer = &s.map().layers[0]; + let raster = layer.raster.as_ref().unwrap(); + assert!(raster.in_place && raster.data.is_some(), "{:?}", raster.error); + assert_eq!(layer.connection, id); + raster.path.clone().unwrap() + }; + assert!(rustgis_data::login_address::is_wrapper(&path) && !path.contains("fixture-login-757"), "{path}"); + let carried = |server: &LoginFile| { + let requests = server.requests.lock().unwrap(); + !requests.is_empty() && requests.iter().all(|r| r.to_ascii_lowercase().contains("\r\nx-key: fixture-login-757\r\n")) + }; + assert!(carried(&server), "{:?}", server.requests.lock().unwrap()); + // Logged in again (a new binding): the layer's address follows it, and it is read through it. + server.requests.lock().unwrap().clear(); + s.run("connection.login", &json!({"id":id,"apiKey":"fixture-login-757","allowInsecure":true})).unwrap(); + let layer = &s.map().layers[0]; + let raster = layer.raster.as_ref().unwrap(); + let again = raster.path.clone().unwrap(); + assert!(again != path && rustgis_data::login_address::is_wrapper(&again), "{again}"); + assert!(raster.data.is_some() && raster.error.is_none(), "{:?}", raster.error); + assert_eq!(layer.connection, id); + assert!(carried(&server), "{:?}", server.requests.lock().unwrap()); +} diff --git a/crates/map/src/raster.rs b/crates/map/src/raster.rs index 5c31059b3..8d9235125 100644 --- a/crates/map/src/raster.rs +++ b/crates/map/src/raster.rs @@ -10,6 +10,7 @@ use std::sync::{Arc, Mutex, Weak}; use rustgis_data::raster::{BandStats, ColorHint, Raster}; use serde::{Deserialize, Serialize}; +use web_time::Instant; use crate::color::{Ramp, Rgba}; use crate::symbology::{ClassMethod, class_breaks, fmt_num}; @@ -20,6 +21,11 @@ pub struct RasterLayer { /// The file it was read from; reopened when the project loads. #[serde(default)] pub path: Option, + /// `path` is a cloud-optimised GeoTIFF's web address, read in place: the layer holds an + /// overview of it (its statistics, histograms and tools read that), and the map reads the part + /// in view as it draws (see [`set_in_place_reader`]). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub in_place: bool, /// The file's own name (for its format) when it is embedded. #[serde(default)] pub file_name: Option, @@ -55,6 +61,102 @@ pub struct RasterLayer { /// A remote reload shared with render snapshots, never serialized. #[serde(skip)] opening: Option, + /// The style is the file's own look once its pixels land (a layer added before they have; + /// saved with a project saved before they did, so it opens in that look still). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + restyle: bool, + /// The look its address gave a layer read in place, applied when its pixels land (saved, as + /// `restyle` is, until they have). + #[serde(default, skip_serializing_if = "Option::is_none")] + look: Option, + /// What reading its pixels lost has been said (see [`RasterLayer::pixels_unsaid`]). + #[serde(skip)] + pixels_said: bool, + /// Why it couldn't be read in place has been said (see [`RasterLayer::error_unsaid`]). + #[serde(skip)] + error_said: bool, + /// When reading its overview in place failed, for the read to be made again once its wait is + /// over (see [`RasterLayer::read_failure`]). + #[serde(skip)] + failed_at: Option, + /// How many reads of its overview in place have failed in a row: each waits longer than the + /// last (see [`RasterLayer::read_failure`]). + #[serde(skip)] + failures: u32, + /// What reading its pixels left out (`Raster::lost`) has been said (see + /// [`RasterLayer::lost_unsaid`]). + #[serde(skip)] + lost_said: bool, + /// What its address's look asked for that the file hasn't (bands past its own), said with + /// what the pixels lost. + #[serde(skip)] + look_lost: Option, + /// Settings of the tile layer this one was opened from that a raster layer hasn't (see + /// [`RasterLayer::dropped_unsaid`]). + #[serde(skip)] + dropped: Vec<&'static str>, +} + +/// Called once with a cloud-optimised GeoTIFF's overview, or why it couldn't be read. +pub type InPlaceRead = Box) + Send>; + +/// Reads the overviews of `layers` read in place and not read yet, all at once: a project of +/// several opens in the time of the slowest, not of them all. Waited for natively when `wait` +/// (at most two minutes in all); otherwise, as always in the browser, they land as the map draws +/// (see [`wait_for_reads`]). Other layers are left as they are. Returns each failure, by its +/// place in `layers`: without waiting, the reads that couldn't start. +pub fn read_together(layers: &mut [&mut RasterLayer], wait: bool) -> Vec<(usize, String)> { + let mut failed = Vec::new(); + let unread = |l: &RasterLayer| l.in_place && l.data.is_none() && l.opening.is_none() && l.time_zarr.is_none(); + for (i, l) in layers.iter_mut().enumerate() { + // Started without waiting: waited for below, together. + if unread(l) + && let Err(e) = l.begin_reload() + { + failed.push((i, e)); + } + } + if wait { + failed.extend(wait_for_reads(layers)); + } + failed +} + +/// Waits (natively; at most two minutes for all) for the overviews of `layers` being read in +/// place (see [`read_together`]), installing them. Returns each failure, by its place in +/// `layers`. In the browser, where nothing may wait, they land as the map draws. +pub fn wait_for_reads(layers: &mut [&mut RasterLayer]) -> Vec<(usize, String)> { + #[cfg(target_arch = "wasm32")] + { + let _ = layers; + Vec::new() + } + #[cfg(not(target_arch = "wasm32"))] + { + let deadline = Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; + let mut failed = Vec::new(); + for (i, l) in layers.iter_mut().enumerate() { + if l.in_place + && l.opening.is_some() + && let Err(e) = l.wait_for_read(deadline) + { + failed.push((i, e)); + } + } + failed + } +} +type InPlaceReader = Arc; + +fn in_place_reader() -> &'static Mutex> { + static R: std::sync::OnceLock>> = std::sync::OnceLock::new(); + R.get_or_init(Mutex::default) +} + +/// Sets how a cloud-optimised GeoTIFF's overview is read by its address (the tile cache's reader, +/// which checks addresses and signs requests; this crate does no networking). +pub fn set_in_place_reader(f: impl Fn(&str, InPlaceRead) + Send + Sync + 'static) { + *in_place_reader().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); } /// The renderer bands' statistics without the masked pixels, and what they were worked out for: @@ -80,6 +182,9 @@ impl RasterLayer { if self.embedded.is_some() { return Err("the raster's data is kept in the project: there's no file to repair".into()); } + if self.in_place { + return Err("the raster is read in place by its address: add the address again to change it".into()); + } let sub = subdataset.or(self.subdataset.as_deref()); let r = rustgis_data::raster::read_path_with(std::path::Path::new(path), sub, self.crs.as_deref()) .and_then(|r| placed(r, self.placement)) @@ -100,6 +205,7 @@ impl RasterLayer { style.repair(&r); Ok(RasterLayer { path: Some(path.into()), + in_place: false, file_name: None, embedded: None, subdataset: sub.map(String::from), @@ -111,6 +217,15 @@ impl RasterLayer { error: None, masked: MaskedStats::default(), opening: None, + restyle: false, + look: None, + pixels_said: false, + error_said: false, + failed_at: None, + failures: 0, + lost_said: false, + look_lost: None, + dropped: Vec::new(), }) } @@ -119,6 +234,7 @@ impl RasterLayer { let style = RasterStyle::default_for(&raster); RasterLayer { path: Some(path.into()), + in_place: false, file_name: None, embedded: None, subdataset: None, @@ -130,6 +246,15 @@ impl RasterLayer { error: None, masked: MaskedStats::default(), opening: None, + restyle: false, + look: None, + pixels_said: false, + error_said: false, + failed_at: None, + failures: 0, + lost_said: false, + look_lost: None, + dropped: Vec::new(), } } @@ -138,6 +263,7 @@ impl RasterLayer { let style = RasterStyle::default_for(&raster); RasterLayer { path: None, + in_place: false, file_name: Some(file_name.into()), embedded: Some(Arc::new(bytes)), subdataset: None, @@ -149,18 +275,174 @@ impl RasterLayer { error: None, masked: MaskedStats::default(), opening: None, + restyle: false, + look: None, + pixels_said: false, + error_said: false, + failed_at: None, + failures: 0, + lost_said: false, + look_lost: None, + dropped: Vec::new(), } } - /// Reads the pixels again from the path or the embedded file (after a project opens). An S3 - /// or login-wrapped address is downloaded whole, keeping its wrapper. A layer that follows the - /// time slider is left to the engine, which reads the step shown. + /// A cloud-optimised GeoTIFF read in place by its address, its overview `raster` read. + pub fn in_place(raster: Raster, address: impl Into) -> RasterLayer { + RasterLayer { in_place: true, ..RasterLayer::from_path(raster, address) } + } + + /// A cloud-optimised GeoTIFF read in place by its address, its overview not read yet: drawn, + /// in its own look, once [`RasterLayer::reload`] (or [`read_together`]) has read it. + pub fn unread_in_place(address: impl Into) -> RasterLayer { + RasterLayer { + path: Some(address.into()), + in_place: true, + file_name: None, + embedded: None, + subdataset: None, + crs: None, + time_zarr: None, + placement: None, + style: RasterStyle::placeholder(), + data: None, + error: None, + masked: MaskedStats::default(), + opening: None, + restyle: true, + look: None, + pixels_said: false, + error_said: false, + failed_at: None, + failures: 0, + lost_said: false, + look_lost: None, + dropped: Vec::new(), + } + } + + /// [`RasterLayer::unread_in_place`], its overview being read: it lands as the map draws + /// ([`RasterLayer::poll_reload`]), in the browser and on the desktop alike, and failing is + /// said then (see [`RasterLayer::error_unsaid`]). + pub fn opening_in_place(address: impl Into) -> RasterLayer { + let mut layer = RasterLayer::unread_in_place(address); + let _ = layer.begin_reload(); + layer + } + + /// The layer in the look its address gives (see [`AddressLook`]): now, when its pixels are + /// here, else once they land. + pub fn with_look(mut self, look: AddressLook) -> RasterLayer { + match self.data.clone() { + Some(d) => { + self.look_lost = RasterStyle::look_lost(&d, &look); + if let Some(style) = RasterStyle::for_look(&d, &look) { + self.style = style; + } + } + None => self.look = Some(look), + } + self + } + + /// Marks the layer as one that couldn't be read, and why (a project opened without the + /// connection its address needs), for [`RasterLayer::error_unsaid`] to say. + pub fn fail(&mut self, why: impl Into) { + self.error = Some(why.into()); + self.error_said = false; + self.failed_at = None; + } + + /// Starts reading its overview in place by the reader the app installed, which answers later + /// (installed by [`RasterLayer::poll_reload`] as the map draws, or waited for by + /// [`RasterLayer::reload`] natively). What it can't start (no address, an address that isn't a + /// tile layer's, no reader here) is the caller's to say. + fn begin_in_place_read(&mut self) -> Result<(), String> { + let Some(address) = self.path.clone() else { return Err("the raster has no source".to_string()) }; + // Checked as a tile layer's address is (http(s) or s3, a host, no control characters): a + // project edited by hand asks for nothing else. + crate::tiles::check_url(&address)?; + let Some(reader) = in_place_reader().lock().unwrap_or_else(|e| e.into_inner()).clone() else { + return Err(format!("{address}: can't be read in place here")); + }; + self.data = None; + let slot = Arc::new(Mutex::new(None)); + let land = slot.clone(); + self.opening = Some(slot); + reader( + &address, + Box::new(move |result| { + *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result.map(Arc::new)); + crate::source::wake(); + }), + ); + Ok(()) + } + + /// Why reading its overview in place failed, when, and how many reads in a row have failed: + /// for the read to be made again once its wait is over (see [`RasterLayer::read_again`]), as a + /// tile or a file's header is, waiting longer each time. `None` for a layer with its pixels, + /// one being read, and one that waits for something other than time (its bucket's connection: + /// see [`RasterLayer::fail`]). + pub fn read_failure(&self) -> Option<(&str, Instant, u32)> { + if !self.in_place || self.data.is_some() || self.opening.is_some() { + return None; + } + Some((self.error.as_deref()?, self.failed_at?, self.failures)) + } + + /// Reads its overview again after a read that failed (see [`RasterLayer::read_failure`]), + /// without waiting: it lands as the map draws, natively as in the browser (nothing may wait + /// each frame). The layer keeps its error until then, and failing the same way again isn't + /// said again. + pub fn read_again(&mut self) { + if self.read_failure().is_none() { + return; + } + if let Err(e) = self.begin_in_place_read() { + self.land(Err(e)); + } + } + + /// Reads the pixels again from the path or the embedded file (after a project opens). A + /// layer read in place is waited for natively (at most two minutes), as a file is read; + /// another layer's S3 or login-wrapped address is downloaded whole, keeping its wrapper. A + /// layer that follows the time slider is left to the engine, which reads the step shown. pub fn reload(&mut self) -> Result<(), String> { + self.reload_in(true) + } + + /// [`RasterLayer::reload`] without waiting for a layer read in place: its overview lands as + /// the map draws ([`RasterLayer::poll_reload`]), natively as in the browser, so a command on + /// the desktop's own thread isn't held by a slow host. The error is a read that can't start. + pub fn begin_reload(&mut self) -> Result<(), String> { + self.reload_in(false) + } + + fn reload_in(&mut self, wait: bool) -> Result<(), String> { if self.time_zarr.is_some() { return Ok(()); } self.opening = None; self.error = None; + self.failed_at = None; + self.failures = 0; + // Read in place: its overview, by the reader the app installed; waited for natively when + // asked to (as a file is read), else landing later, as always in the browser. + if self.in_place { + if let Err(e) = self.begin_in_place_read() { + self.error = Some(e.clone()); + self.error_said = false; + return Err(e); + } + #[cfg(not(target_arch = "wasm32"))] + if wait { + return self.wait_for_read(Instant::now() + rustgis_data::raster::IN_PLACE_WAIT); + } + #[cfg(target_arch = "wasm32")] + let _ = wait; + return Ok(()); + } if self.embedded.is_none() && let Some(path) = self.path.as_deref().filter(|p| { rustgis_data::login_address::is_wrapper(p) @@ -193,7 +475,7 @@ impl RasterLayer { .map_err(|e| e.to_string()) }); *land.lock().unwrap_or_else(|e| e.into_inner()) = Some(result); - crate::source::landed(); + crate::source::wake(); }), ); return Ok(()); @@ -211,6 +493,8 @@ impl RasterLayer { Ok(r) => { self.style.repair(&r); self.data = Some(Arc::new(r)); + self.pixels_said = false; + self.lost_said = false; Ok(()) } Err(e) => { @@ -221,19 +505,119 @@ impl RasterLayer { } } + /// Waits (natively, until `deadline`) for a read under way, then installs it. + #[cfg(not(target_arch = "wasm32"))] + fn wait_for_read(&mut self, deadline: Instant) -> Result<(), String> { + let Some(slot) = self.opening.clone() else { return Ok(()) }; + loop { + if let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() { + self.opening = None; + let error = result.as_ref().err().cloned(); + self.land(result); + return error.map_or(Ok(()), Err); + } + if Instant::now() >= deadline { + let address = self.path.clone().unwrap_or_default(); + let e = format!("{address}: no answer in two minutes"); + self.opening = None; + self.land(Err(e.clone())); + return Err(e); + } + std::thread::sleep(web_time::Duration::from_millis(10)); + } + } + + /// Its pixels, the first time they are asked for after they landed (added with them, or read + /// in place later: in the browser, a project, an import), so what reading them lost (a colour + /// table missing, an overview held) is said once for them. + pub fn pixels_unsaid(&mut self) -> Option> { + if self.pixels_said { + return None; + } + let d = self.data.clone()?; + self.pixels_said = true; + Some(d) + } + + /// The layer, opened from a tile layer whose `dropped` settings (`"attribution"`, `"legend"`, + /// `"zoom range"`, `"bounds"`, `"overlay"`) a raster layer hasn't, so they aren't kept: said + /// once, by [`RasterLayer::dropped_unsaid`]. + pub fn from_tiles_dropping(mut self, dropped: Vec<&'static str>) -> RasterLayer { + self.dropped = dropped; + self + } + + /// The tile layer settings dropped when this layer was opened from one, the first time they + /// are asked for. + pub fn dropped_unsaid(&mut self) -> Vec<&'static str> { + std::mem::take(&mut self.dropped) + } + + /// What reading its pixels left out (`Raster::lost`), the first time it is asked for after + /// they landed, so it is said once: as the layer is added, as its project opens, or when they + /// land later (the browser). + pub fn lost_unsaid(&mut self) -> Vec { + if self.lost_said { + // A look applied after the pixels' losses were said: what it asked for that the file + // hasn't, on its own, not the pixels' losses again. + return self.look_lost.take().into_iter().collect(); + } + let mut lost = self.data.as_ref().map(|d| d.lost.clone()).unwrap_or_default(); + lost.extend(self.look_lost.take()); + self.lost_said = self.data.is_some(); + lost + } + + /// Why a layer read in place couldn't be read, the first time it is asked for after the read + /// failed, so it is said once (a project, an import or the browser read it after adding it). + pub fn error_unsaid(&mut self) -> Option { + if !self.in_place || self.error_said { + return None; + } + let e = self.error.clone()?; + self.error_said = true; + Some(e) + } + /// Installs a completed remote read; each render copy can observe the same shared result. pub(crate) fn poll_reload(&mut self) -> bool { let Some(slot) = self.opening.as_ref() else { return false }; let Some(result) = slot.lock().unwrap_or_else(|e| e.into_inner()).clone() else { return true }; self.opening = None; + self.land(result); + // Installed while the map draws: a frame after it says what reading them lost. + crate::source::another_frame(); + false + } + + /// Installs pixels read (or why they couldn't be): in the file's own look for a layer added + /// before they arrived, else in its style, repaired for them. + fn land(&mut self, result: Result, String>) { match result { Ok(raster) => { - self.style.repair(&raster); + self.pixels_said = false; + self.lost_said = false; + if std::mem::take(&mut self.restyle) { + let look = self.look.take(); + self.look_lost = look.as_ref().and_then(|look| RasterStyle::look_lost(&raster, look)); + let look = look.and_then(|look| RasterStyle::for_look(&raster, &look)); + self.style = look.unwrap_or_else(|| RasterStyle::default_for(&raster)); + } else { + self.style.repair(&raster); + } self.data = Some(raster); + self.error = None; + self.failed_at = None; + self.failures = 0; + } + Err(error) => { + // A read made again that fails as before (see `read_again`) isn't said again. + self.error_said = self.error_said && self.error.as_deref() == Some(error.as_str()); + self.error = Some(error); + self.failed_at = Some(Instant::now()); + self.failures = self.failures.saturating_add(1); } - Err(error) => self.error = Some(error), } - false } /// Statistics of the renderer's bands (one per band, in its order) without the pixels the @@ -705,9 +1089,91 @@ fn one() -> f64 { 1.0 } +/// The look an address gives a GeoTIFF read in place (`…/scene.tif#bands=5,3,2&range=0,3000`, as +/// tile layers kept it): the bands drawn (from 0; one in grey, or red, green and blue) and a fixed +/// stretch. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct AddressLook { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub bands: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub range: Option<(f64, f64)>, +} + impl RasterStyle { - /// The default look: the file's colours, RGB for colour images, unique values for a few - /// categories, otherwise a grey stretch. + /// The style `look` gives `r`: its own look (see [`RasterStyle::default_for`]) with the bands + /// and stretch the look names (bands it hasn't are left as they are); `None` for a look that + /// names neither. A palette image stretched by a range, or whose address names bands, is + /// drawn in grey, as its tile layer was. + pub fn for_look(r: &Raster, look: &AddressLook) -> Option { + if look.bands.is_empty() && look.range.is_none() { + return None; + } + // Bands the file hasn't: its own look (said by `look_lost`); a palette image in grey, as + // its tile layer draws one whose address names bands (its colours are of one band alone). + if Self::look_lost(r, look).is_some() { + let mut style = RasterStyle::default_for(r); + if matches!(style.renderer, RasterRenderer::Colormap) { + style.renderer = RasterRenderer::Stretch { band: 0, ramp: Ramp::gray(), invert: false }; + style.stretch = Stretch::default(); + } + if let Some((lo, hi)) = look.range { + style.stretch = Stretch::custom(lo, hi); + } + return Some(style); + } + let mut style = RasterStyle::default_for(r); + let n = r.band_count(); + match look.bands[..] { + [b] if b < n => style.renderer = RasterRenderer::Stretch { band: b, ramp: Ramp::gray(), invert: false }, + [red, green, blue] if red < n && green < n && blue < n => style.renderer = RasterRenderer::Rgb { red, green, blue }, + [] if matches!(style.renderer, RasterRenderer::Colormap) => { + style.renderer = RasterRenderer::Stretch { band: 0, ramp: Ramp::gray(), invert: false }; + } + _ => {} + } + if let Some((lo, hi)) = look.range { + style.stretch = Stretch::custom(lo, hi); + } + Some(style) + } + + /// What a look asks for that `r` can't draw: bands past its own, or two bands (one is drawn + /// in grey, three as red, green and blue), so the file's own look is drawn instead; `None` + /// when the look fits. + pub fn look_lost(r: &Raster, look: &AddressLook) -> Option { + let n = r.band_count(); + // A palette image's colours are of one band alone: in grey, as its tile layer drew it. + let own = if matches!(r.color, ColorHint::Palette(_)) { "drawn in grey" } else { "drawn in its own look" }; + let past: Vec = look.bands.iter().filter(|b| **b >= n).map(|b| b.saturating_add(1).to_string()).collect(); + if !past.is_empty() { + return Some(format!("its address asks for band(s) {} (from 1), which it hasn't ({n} band(s)): {own}", past.join(", "))); + } + // An address written by hand: `map.setTiles` takes one band or three. + (!matches!(look.bands.len(), 0 | 1 | 3)) + .then(|| format!("its address asks for {} bands; one is drawn in grey, three as red, green and blue: {own}", look.bands.len())) + } + + /// A style for a layer whose pixels haven't arrived (its own look replaces it when they do). + pub fn placeholder() -> RasterStyle { + RasterStyle { + renderer: RasterRenderer::Stretch { band: 0, ramp: Ramp::gray(), invert: false }, + stretch: Stretch::default(), + dra: false, + resampling: Resampling::Nearest, + brightness: 0.0, + contrast: 0.0, + gamma: 1.0, + rotation: 0.0, + mask: RasterMask::default(), + } + } + + /// The default look, as the file's PhotometricInterpretation says: the file's colours, RGB for + /// colour images and for grey ones of exactly three bands (most remote sensing images are + /// grey, whatever their bands: three are taken as red, green and blue), unique values for a + /// few categories, otherwise a grey stretch of the first band (an image of more bands is + /// combined as the user picks). pub fn default_for(r: &Raster) -> RasterStyle { RasterStyle::default_with(r, &RasterDefaults::default()) } @@ -721,7 +1187,7 @@ impl RasterStyle { ColorHint::Rgb { .. } if r.band_count() >= 3 => { (RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, if byte_rgb { Stretch::None } else { Stretch::default() }) } - _ if r.band_count() >= 3 => (RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, Stretch::default()), + ColorHint::Gray if r.band_count() == 3 => (RasterRenderer::Rgb { red: 0, green: 1, blue: 2 }, Stretch::default()), _ => match r.bands.first().and_then(|b| b.stats.categories.as_ref()).filter(|c| c.len() <= 16 && r.data_type.contains("Int")) { Some(_) => (unique_values(r, 0, Ramp::Harbour), Stretch::None), None => (RasterRenderer::Stretch { band: 0, ramp: d.ramp.clone(), invert: d.invert }, d.stretch), @@ -1150,6 +1616,36 @@ mod tests { Raster::new("dem", 4, 4, vec![data], gt, Crs::LonLat, None, ColorHint::Gray, "Float32").unwrap() } + /// What reading the pixels lost is said once; a look applied after that (bands the file + /// hasn't) says what it lost on its own, not the pixels' losses again. + #[test] + fn a_look_applied_after_the_losses_were_said_says_its_own_alone() { + let mut r = dem(); + r.lost.push("its colour table is left out".into()); + let mut rl = RasterLayer::in_place(r, "test://look-later.tif"); + assert_eq!(rl.lost_unsaid(), vec!["its colour table is left out".to_string()]); + assert!(rl.lost_unsaid().is_empty()); + let mut rl = rl.with_look(AddressLook { bands: vec![9], range: None }); + let said = rl.lost_unsaid(); + assert_eq!(said.len(), 1, "{said:?}"); + assert!(!said[0].contains("colour table"), "{said:?}"); + assert!(rl.lost_unsaid().is_empty()); + // A look applied before they are said is said with them, once. + let mut r = dem(); + r.lost.push("its colour table is left out".into()); + let mut rl = RasterLayer::in_place(r, "test://look-first.tif").with_look(AddressLook { bands: vec![9], range: None }); + assert_eq!(rl.lost_unsaid().len(), 2); + assert!(rl.lost_unsaid().is_empty()); + } + + /// A project file may ask for any band number: the one past the last `usize` is said by + /// number, not overflowed. + #[test] + fn a_look_asking_for_the_band_past_the_last_usize_is_said() { + let lost = RasterStyle::look_lost(&dem(), &AddressLook { bands: vec![usize::MAX], range: None }).unwrap(); + assert!(lost.contains(&format!("band(s) {} (from 1)", usize::MAX)), "{lost}"); + } + #[test] fn defaults_and_colours() { let r = dem(); diff --git a/crates/map/src/source.rs b/crates/map/src/source.rs index 5d1af60e6..aded581c9 100644 --- a/crates/map/src/source.rs +++ b/crates/map/src/source.rs @@ -525,9 +525,9 @@ struct Pending { /// A source being opened off the frame. type Opening = Arc, String>>>>; -/// Tells the app something landed (a read, or a source opened by its address). -pub(crate) fn landed() { - GENERATION.fetch_add(1, Ordering::Relaxed); +/// Asks the app for a frame (the waker) with nothing new to draw: what landed while the map drew +/// is said a frame after (see [`wake`] for something new to draw). +pub(crate) fn another_frame() { if let Some(w) = waker().lock().ok().and_then(|w| w.clone()) { w(); } @@ -800,7 +800,7 @@ impl SourceLayer { if let Ok(mut s) = land.lock() { *s = Some(r.map(Arc::from).map_err(|e| e.to_string())); } - landed(); + wake(); }), ); Ok(()) @@ -833,7 +833,7 @@ impl SourceLayer { if let Ok(mut s) = land.lock() { *s = Some(r); } - landed(); + wake(); }), ); Ok(()) @@ -1015,7 +1015,7 @@ impl SourceLayer { if let Ok(mut slot) = p2.result.lock() { *slot = Some(r.map_err(|e| e.to_string())); } - landed(); + wake(); }), ); self.pending = Some(pending); @@ -1136,8 +1136,9 @@ fn waker() -> &'static Mutex>> { W.get_or_init(|| Mutex::new(None)) } -/// Something read in the background (outside this module: a Zarr step) has landed: the map -/// redraws ([`generation`]) and the UI is woken. +/// Something read in the background (a feature read, a source opened by its address, a raster's +/// pixels read in place; outside this module, a Zarr step) has landed: the map redraws +/// ([`generation`]) and the UI is woken. pub fn wake() { GENERATION.fetch_add(1, Ordering::Relaxed); if let Some(w) = waker().lock().ok().and_then(|w| w.clone()) { diff --git a/crates/render/src/gpu_scene.rs b/crates/render/src/gpu_scene.rs index 15a44c60f..bae92d1b3 100644 --- a/crates/render/src/gpu_scene.rs +++ b/crates/render/src/gpu_scene.rs @@ -57,6 +57,12 @@ pub struct GpuScene { pub data_pending: bool, /// Tile layers that couldn't be drawn, and why. pub tile_errors: Vec, + /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles + /// they couldn't read), by id, their errors among `tile_errors`: an export refuses them. + pub partial: Vec, + /// Raster layers read in place whose tiles are still on their way, by id: drawn again as + /// they land; an export that can't wait refuses them. + pub pending_in_place: Vec, /// A turned or tilted scene: the camera it was drawn for, which the vertex shader's full /// perspective takes, and a tilted one's sky and fog. `None` for a flat, north-up scene. pub tilt: Option, diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 7e7aaa3ef..5f1d9da01 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -278,6 +278,12 @@ pub struct Rendered { pub data_pending: bool, /// Tile layers that couldn't be drawn, and why. pub tile_errors: Vec, + /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles + /// they couldn't read), by id, their errors among `tile_errors`: an export refuses them. + pub partial: Vec, + /// Raster layers read in place whose tiles are still on their way, by id: drawn again as + /// they land; an export that can't wait refuses them. + pub pending_in_place: Vec, /// The frame kept as vectors, when it was recorded ([`render_complete_recorded`]). pub(crate) recorded: Option, } @@ -2019,6 +2025,7 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea let mut kept: Vec<(View, (f64, f64), Pixmap)> = Vec::new(); let last = bands_of.len().saturating_sub(1); let (mut attribution, mut tiles_pending, mut tile_errors) = (Vec::::new(), false, Vec::::new()); + let (mut partial, mut pending_in_place) = (Vec::::new(), Vec::::new()); for (i, b) in bands_of.into_iter().enumerate() { let v = band_view(view, &b); let Some(RenderOutput::Cpu(r)) = @@ -2054,6 +2061,16 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea tile_errors.push(e); } } + for id in r.partial { + if !partial.contains(&id) { + partial.push(id); + } + } + for id in r.pending_in_place { + if !pending_in_place.contains(&id) { + pending_in_place.push(id); + } + } tiles_pending |= r.tiles_pending; if ground.is_some() { // The nearest band reaches the bottom of the view, the farthest the sky. @@ -2090,6 +2107,8 @@ fn render_tilted(map: &mut Map, view: &View, full_res_rasters: bool, sources_rea tiles_pending, data_pending, tile_errors, + partial, + pending_in_place, recorded: None, }) } @@ -2283,6 +2302,7 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: let mut candidates = Vec::new(); let (mut cmds, mut clips) = (Vec::new(), Vec::new()); let (mut attribution, mut tiles_pending, mut tile_errors, mut clear) = (Vec::::new(), false, Vec::::new(), [0.0; 4]); + let (mut partial, mut pending_in_place) = (Vec::::new(), Vec::::new()); let mut plain = true; let last = bands.len().saturating_sub(1); for (i, b) in bands.iter().enumerate() { @@ -2325,6 +2345,16 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: tile_errors.push(e); } } + for id in scene.partial { + if !partial.contains(&id) { + partial.push(id); + } + } + for id in scene.pending_in_place { + if !pending_in_place.contains(&id) { + pending_in_place.push(id); + } + } tiles_pending |= scene.tiles_pending; // The nearest band reaches the bottom of the view and the farthest the sky, so no ground // falls between them. @@ -2359,7 +2389,20 @@ fn gpu_turned(map: &mut Map, view: &View, full_res_rasters: bool, sources_ready: clips.push(CmdClip { upright: true, ..CmdClip::ALL }); } let unplaced = if map.label_display.show_unplaced { placed.unplaced } else { Vec::new() }; - Some(GpuScene { clear, cmds, labels: placed.labels, unplaced, attribution, tiles_pending, data_pending, tile_errors, tilt, clips }) + Some(GpuScene { + clear, + cmds, + labels: placed.labels, + unplaced, + attribution, + tiles_pending, + data_pending, + tile_errors, + partial, + pending_in_place, + tilt, + clips, + }) } /// Places the labels a tilted view's bands offered, nearest band first. Non-repeated line @@ -3407,12 +3450,36 @@ fn render_frame( }; let stats = rl.masked_stats(); let stats = stats.as_ref().map(|s| s.as_slice()); + // Read in place: the part in view, at the level the view needs. + let address = rl.path.as_deref().filter(|_| rl.in_place); + let mut draw_it = |pm: &mut Pixmap, opacity: f32| match address { + // Drawn from less than the view needs (a coarser level, tiles that couldn't be + // read): shown, and said with the tile errors. + Some(a) => match raster::draw_in_place(pm, a, data, &rl.style, stats, view, opacity, full_res_rasters, place) { + Ok(pending) => { + tile_out.pending |= pending; + if pending && !tile_out.pending_in_place.contains(&layer.id) { + tile_out.pending_in_place.push(layer.id); + } + } + Err(e) => { + let e = format!("{}: {e}", layer.name); + if !tile_out.errors.contains(&e) { + tile_out.errors.push(e); + } + if !tile_out.partial.contains(&layer.id) { + tile_out.partial.push(layer.id); + } + } + }, + None => raster::draw(pm, data, &rl.style, stats, view, opacity, full_res_rasters, place), + }; if offscreen { let mut target = Pixmap::new(w, h)?; - raster::draw(&mut target, data, &rl.style, stats, view, 1.0, full_res_rasters, place); + draw_it(&mut target, 1.0); composite(&mut pm, &target); } else { - raster::draw(pm.pixels(), data, &rl.style, stats, view, opacity, full_res_rasters, place); + draw_it(pm.pixels(), opacity); } cpu_dirty = true; } @@ -4424,10 +4491,23 @@ fn render_frame( let attribution = tile_out.attribution; let tiles_pending = tile_out.pending; let tile_errors = tile_out.errors; + let partial = tile_out.partial; + let pending_in_place = tile_out.pending_in_place; match backend { Backend::Cpu => { let (pixmap, recorded) = if pm.is_recording() { (Pixmap::new(1, 1)?, Some(pm)) } else { (pm.into_pixmap(), None) }; - Some(RenderOutput::Cpu(Rendered { pixmap, labels, unplaced, attribution, tiles_pending, data_pending, tile_errors, recorded })) + Some(RenderOutput::Cpu(Rendered { + pixmap, + labels, + unplaced, + attribution, + tiles_pending, + data_pending, + tile_errors, + partial, + pending_in_place, + recorded, + })) } Backend::Gpu | Backend::GpuLive => { flush_acc(pm.pixels(), &mut cmds, &mut cpu_dirty)?; @@ -4446,6 +4526,8 @@ fn render_frame( tiles_pending, data_pending, tile_errors, + partial, + pending_in_place, tilt: None, clips: Vec::new(), })) @@ -5633,6 +5715,21 @@ pub fn render_drawing(map: &mut Map, view: &View) -> Option { /// Renders a map with its labels, a scale bar and a north arrow to PNG bytes. pub fn render_png(map: &mut Map, view: &View) -> Result, String> { let r = render_complete(map, view).ok_or("could not allocate the image")?; + // A raster read in place drawn from less than the view needs isn't written: its tiles still + // on their way (the browser can't wait; natively the wait ran out), or a coarser level or + // tiles it couldn't read (`partial`). + // Other layers' tiles pending export what has drawn, as before. + if let Some(l) = r.pending_in_place.first().and_then(|id| map.layers.iter().find(|l| l.id == *id)) { + return Err(format!("{}: its tiles are still on their way: export again once the map has drawn it", l.name)); + } + if let Some(l) = r.partial.first().and_then(|id| map.layers.iter().find(|l| l.id == *id)) { + // Its own error, named by the layer, says why; else the general one. + let why = r.tile_errors.iter().find(|e| e.starts_with(&format!("{}: ", l.name))).cloned(); + let why = why.unwrap_or_else(|| { + format!("{}: drawn from less than the view needs (a coarser level, or tiles that couldn't be read)", l.name) + }); + return Err(format!("{why}, so the image isn't written")); + } let mut pm = Canvas::from_pixmap(r.pixmap); for l in r.unplaced.iter().chain(&r.labels) { text::draw(&mut pm, l, view.pixels_per_point as f32); diff --git a/crates/render/src/raster.rs b/crates/render/src/raster.rs index 393cc92d8..9b357a231 100644 --- a/crates/render/src/raster.rs +++ b/crates/render/src/raster.rs @@ -102,6 +102,274 @@ pub(crate) fn draw( draw_with(pm, r, style, stats, view, opacity, full_res, place, threads); } +/// Screen points a side sampled to find the part of a raster read in place that is in view, and +/// how finely. +const WINDOW_GRID: usize = 17; +/// Why a part in view was drawn from a coarser level than it needs. +const TOO_LARGE: &str = "the view is too large a part of the image to read at the resolution it needs"; +/// Why a turned layer is drawn from its overview. +const TURNED: &str = "turned, it is drawn from its overview, not at the resolution the view needs"; + +/// Whether `rotation` (degrees) turns the raster: anything but a whole number of turns. A hair +/// either side of none (`-1e-7` as much as `1e-7`) isn't one. +fn turned(rotation: f64) -> bool { + let r = rotation.rem_euclid(360.0); + r.min(360.0 - r) > 1e-6 +} + +/// Why a see-through layer with tiles to come is drawn from its overview alone: no memory for +/// the image that blends the part in view with it. +const UNBLENDED: &str = "no memory to blend the part in view with its overview, so it is drawn from its overview"; + +/// Draws a raster layer read in place (a cloud-optimised GeoTIFF by its address, see +/// [`rustgis_map::raster::RasterLayer::in_place`]) as [`draw`] draws one held whole: its overview +/// `r` (the layer's pixels), or, zoomed in past it, the part in view read from the level the view +/// needs, drawn over the overview until its tiles arrive. `stats` are the renderer's bands' +/// statistics (the overview's when not given), so the part in view is stretched as the whole is. +/// Returns whether tiles are still to come (the map is drawn again as they land), or why it drew +/// less than the view needs (the overview, or other levels for tiles that couldn't be read): the +/// map shows what it has, an export refuses it. +#[allow(clippy::too_many_arguments)] +pub(crate) fn draw_in_place( + pm: &mut Pixmap, + address: &str, + r: &Raster, + style: &RasterStyle, + stats: Option<&[BandStats]>, + view: &View, + opacity: f32, + full_res: bool, + place: Place, +) -> Result { + let bands = style.renderer.bands(); + let own: Vec = match stats { + Some(_) => Vec::new(), + None => bands.iter().map(|b| r.bands.get(*b).map(|b| b.stats.clone()).unwrap_or_default()).collect(), + }; + let stats = stats.unwrap_or(&own); + let whole = |pm: &mut Pixmap| draw(pm, r, style, Some(stats), view, opacity, full_res, place); + let full = r.overview.unwrap_or((r.width, r.height)); + if full.0 <= r.width { + whole(pm); + return Ok(false); + } + // A rotated raster turns about the whole image's centre, not the part in view's: the + // overview is drawn, and zoomed in past it (a screen pixel under 0.8 of its pixels) that is + // less than the view needs: said, and refused by exports. + if turned(style.rotation) { + whole(pm); + let zoomed_in = in_view(r, view, place, pm.width(), pm.height(), full_res).is_some_and(|w| w.per_px < 0.8); + return if zoomed_in { Err(TURNED.into()) } else { Ok(false) }; + } + let info = match rustgis_tiles::cog::info(address) { + Ok(Some(info)) => info, + Ok(None) => { + whole(pm); + return Ok(true); + } + Err(e) => { + whole(pm); + return Err(e); + } + }; + let Some(want) = in_view(r, view, place, pm.width(), pm.height(), full_res) else { + whole(pm); + return Ok(false); + }; + // The coarsest level at least as fine as the view (within a quarter); zoomed in past the full + // image, the full image. + let levels = &info.layout.levels; + let full_px = want.per_px * f64::from(full.0) / f64::from(r.width.max(1)); + let fine_enough = |l: &rustgis_data::raster::cog::Level| f64::from(full.0) / f64::from(l.width.max(1)) <= full_px * 1.25; + let mut level = levels.iter().rposition(fine_enough).unwrap_or(0); + let needed = level; + // The renderer's bands, then the image's alpha band (again, when a colour draws it too), read + // as the window's own bands, as `draw` reads a raster held whole. + let mut read = bands.clone(); + let alpha = match (&style.renderer, &r.color) { + (RasterRenderer::Rgb { .. }, ColorHint::Rgb { alpha: true }) if r.band_count() >= 4 => Some(3), + (RasterRenderer::Stretch { band: 0, .. } | RasterRenderer::ShadedRelief { band: 0, .. }, ColorHint::GrayAlpha) => Some(1), + _ => None, + }; + read.extend(alpha); + // The strips in view: one, or two across a 0–360° grid's seam. + let strips: Vec<(f64, f64)> = std::iter::once(want.cols).chain(want.cols2).collect(); + let mut window = None; + while let Some(l) = levels.get(level) { + // The part in view, in the level's pixels, a pixel past each side (for bilinear sampling). + let (sx, sy) = (f64::from(l.width) / f64::from(r.width.max(1)), f64::from(l.height) / f64::from(r.height.max(1))); + let edge = |v: f64, k: f64, max: u32| ((v * k).floor() as i64).clamp(0, i64::from(max)) as u32; + let cols: Vec<(u32, u32)> = strips + .iter() + .map(|c| (edge(c.0, sx, l.width).saturating_sub(1), edge(c.1, sx, l.width).saturating_add(2).min(l.width))) + .collect(); + let rows = (edge(want.rows.0, sy, l.height).saturating_sub(1), edge(want.rows.1, sy, l.height).saturating_add(2).min(l.height)); + if cols.iter().all(|cols| rustgis_tiles::cog::raster::fits(address, level, *cols, rows, &read)) { + window = Some((level, cols, rows)); + break; + } + // Too large a part for the tiles kept (a view far wider than the screen, a file of many + // bands stored together): a coarser level. + level += 1; + } + let Some((level, cols, rows)) = window else { + whole(pm); + return Err(TOO_LARGE.into()); + }; + // The image at the overview's level is the overview: reached from a finer level the view + // needs (too large a part for the tiles kept), it is drawn but refused, as a coarser level is + // below. + if levels.get(level).is_some_and(|l| l.width <= r.width) { + whole(pm); + return if level > needed { Err(TOO_LARGE.into()) } else { Ok(false) }; + } + let color = match (&style.renderer, &r.color) { + (_, ColorHint::Palette(p)) => ColorHint::Palette(p.clone()), + (RasterRenderer::Rgb { .. }, _) => ColorHint::Rgb { alpha: alpha.is_some() }, + _ if alpha.is_some() => ColorHint::GrayAlpha, + _ => ColorHint::Gray, + }; + // Each strip read from the tiles kept, the missing ones asked for. + let (mut parts, mut pending, mut failed) = (Vec::new(), false, false); + for cols in cols { + match rustgis_tiles::cog::raster::window(address, level, cols, rows, &read, color.clone()) { + Ok(w) => { + pending |= w.pending; + failed |= w.failed; + parts.extend(w.raster); + } + Err(e) => { + whole(pm); + return Err(e); + } + } + } + if parts.is_empty() { + whole(pm); + return Ok(pending); + } + let mut own_style = style.clone(); + own_style.renderer = renumbered(&style.renderer); + // While it has gaps (tiles to come, or that couldn't be read and no level kept stands in for), + // the part in view goes over the overview: both drawn opaque, then blended once, so a + // see-through layer isn't darker where they overlap. + if pending || failed { + if opacity >= 1.0 { + whole(pm); + for part in &parts { + draw(pm, part, &own_style, Some(stats), view, 1.0, full_res, place); + } + } else if let Some(mut both) = Pixmap::new(pm.width(), pm.height()) { + draw(&mut both, r, style, Some(stats), view, 1.0, full_res, place); + for part in &parts { + draw(&mut both, part, &own_style, Some(stats), view, 1.0, full_res, place); + } + pm.draw_pixmap(0, 0, both.as_ref(), &PixmapPaint { opacity, ..PixmapPaint::default() }, Transform::identity(), None); + } else { + // No image to blend in: the overview alone, and said (the map shows it, an export + // refuses it), rather than nothing. + whole(pm); + return Err(UNBLENDED.into()); + } + } else { + for part in &parts { + draw(pm, part, &own_style, Some(stats), view, opacity, full_res, place); + } + } + match (pending, failed) { + (true, _) => Ok(true), + (false, true) => Err("some of its tiles couldn't be read".into()), + // Read coarser than the view needs (too large a part for the tiles kept): the map shows + // it, an export refuses it. + (false, false) if level > needed => Err(TOO_LARGE.into()), + (false, false) => Ok(false), + } +} + +/// A renderer drawing the same bands of a raster holding only those, in its order (see +/// [`RasterRenderer::bands`]). +fn renumbered(renderer: &RasterRenderer) -> RasterRenderer { + let mut out = renderer.clone(); + match &mut out { + RasterRenderer::Stretch { band, .. } + | RasterRenderer::Classify { band, .. } + | RasterRenderer::UniqueValues { band, .. } + | RasterRenderer::ShadedRelief { band, .. } => *band = 0, + RasterRenderer::Rgb { red, green, blue } => (*red, *green, *blue) = (0, 1, 2), + RasterRenderer::Colormap => {} + } + out +} + +/// The part of a raster in view: its columns and rows (half-open, in its own pixels, possibly +/// fractional), and how many of its pixels a pixel drawn spans. +#[derive(Debug, PartialEq)] +struct InView { + cols: (f64, f64), + /// A second run of columns: a geographic grid stored from 0 to 360° seen across its seam has + /// the places west of 0° at its far columns, so the part in view is two strips, not the width. + cols2: Option<(f64, f64)>, + rows: (f64, f64), + per_px: f64, +} + +/// [`InView`] of a raster drawn into a `width × height` pixmap, from a grid of screen points; +/// `None` when it isn't in view. +fn in_view(r: &Raster, view: &View, place: Place, width: u32, height: u32, full_res: bool) -> Option { + let k = if view.pixels_per_point >= 1.5 && !full_res { view.pixels_per_point } else { 1.0 }; + let (vw, vh) = (f64::from(width) / k, f64::from(height) / k); + let n = WINDOW_GRID; + let at = |i: usize, j: usize| source(r, view, place, k, vw * i as f64 / (n - 1) as f64, vh * j as f64 / (n - 1) as f64); + let grid: Vec> = (0..n * n).map(|p| at(p % n, p / n)).collect(); + let (mut c0, mut c1, mut r0, mut r1) = (f64::INFINITY, f64::NEG_INFINITY, f64::INFINITY, f64::NEG_INFINITY); + let (rw, rh) = (f64::from(r.width), f64::from(r.height)); + // A geographic grid stored from 0 to 360° has the places west of 0° at its far columns (see + // `Raster::wrap_col`): seen across that seam, the part in view is two strips, one at each end. + let (mut w0, mut w1) = (f64::INFINITY, f64::NEG_INFINITY); + for (c, row) in grid.iter().flatten() { + let col = r.wrap_col(*c); + if col != *c { + w0 = w0.min(col.clamp(0.0, rw)); + w1 = w1.max(col.clamp(0.0, rw)); + } else { + c0 = c0.min(col.clamp(0.0, rw)); + c1 = c1.max(col.clamp(0.0, rw)); + } + r0 = r0.min(row.clamp(0.0, rh)); + r1 = r1.max(row.clamp(0.0, rh)); + } + let mut cols2 = None; + if w0 < w1 { + if c0 < c1 { + // Both sides: the seam is in view, so each strip runs to it (the sample points stop + // short of it by up to a grid step). + cols2 = Some((w0, rw)); + c0 = 0.0; + } else { + (c0, c1) = (w0, w1); + } + } + if !(c0 < c1 && r0 < r1) { + return None; + } + // The raster's pixels a screen pixel spans: the smallest step between neighbouring points, + // where the view is finest (a tilted or curved view is finer near its centre). + let step = (vw / (n - 1) as f64).max(1.0); + let mut per_px = f64::INFINITY; + for j in 0..n { + for i in 0..n - 1 { + if let (Some(Some(a)), Some(Some(b))) = (grid.get(j * n + i), grid.get(j * n + i + 1)) { + let d = ((b.0 - a.0).powi(2) + (b.1 - a.1).powi(2)).sqrt() / step; + if d > 0.0 { + per_px = per_px.min(d); + } + } + } + } + // In the pixels drawn (a work pixel is `k` device pixels). + per_px.is_finite().then_some(InView { cols: (c0, c1), cols2, rows: (r0, r1), per_px }) +} + /// [`draw`], its rows spread over `threads` threads. #[allow(clippy::too_many_arguments)] fn draw_with( @@ -226,6 +494,7 @@ fn draw_with( relief, resampling, alpha_band, + alpha_max: rustgis_data::raster::alpha_max(&r.data_type), cell: r.cell_size_m(), mask: &style.mask, }; @@ -313,6 +582,8 @@ struct Pass1<'a> { relief: Option<(f64, f64, f64)>, resampling: Resampling, alpha_band: Option, + /// The alpha band's opaque value (its pixel type's: 65535 for 16 bits). + alpha_max: f32, cell: (f64, f64), /// Values drawn as no data. mask: &'a RasterMask, @@ -385,7 +656,7 @@ impl Pass1<'_> { last = Some((cell, dx)); } if let (Some(ab), Some(a)) = (self.alpha_band, alpha.get_mut(i)) { - *a = r.value(ab, cell.0, cell.1).map(|v| v.clamp(0.0, 255.0) as u8).unwrap_or(0); + *a = r.value(ab, cell.0, cell.1).map(|v| ((v / self.alpha_max).clamp(0.0, 1.0) * 255.0).round() as u8).unwrap_or(0); if *a == 0 { continue; } @@ -464,6 +735,18 @@ mod tests { use super::*; + /// A raster turned by a hair either side of none isn't turned: it is drawn at the view's + /// resolution, not from its overview. + #[test] + fn a_hair_of_rotation_either_way_isnt_a_turn() { + for r in [0.0, 1e-7, -1e-7, 360.0, -360.0, 720.0 + 1e-7, f64::NAN] { + assert!(!turned(r), "{r}"); + } + for r in [1.0, -1.0, 359.0, 180.0, 1e-5, -1e-5] { + assert!(turned(r), "{r}"); + } + } + fn sampling_grid(pixel_width: f64, crs: Crs, nodata: Option) -> Raster { let gt = GeoTransform { origin_x: 0.0, origin_y: 90.0, pixel_width, pixel_height: -60.0 }; let data = [0.0, 10.0, 20.0, 30.0].repeat(3); @@ -533,6 +816,31 @@ mod tests { assert_eq!([actual.red(), actual.green(), actual.blue(), actual.alpha()], expected); } + /// A 0–360° grid seen across its seam: the part in view is two strips, one at each end of + /// the grid, not the whole width. + #[test] + fn a_view_across_the_seam_of_a_full_turn_grid_is_two_strips() { + let gt = GeoTransform { origin_x: 0.0, origin_y: 90.0, pixel_width: 10.0, pixel_height: -10.0 }; + let r = Raster::new("turn", 36, 18, vec![vec![1.0; 36 * 18]], gt, Crs::LonLat, None, ColorHint::Gray, "Float32").unwrap(); + // About 90° to each side of 0°: the east strip from column 0, the west one to column 36. + let view = View { + width: 400.0, + height: 200.0, + pixels_per_point: 1.0, + camera: Camera { center: Coord::new(0.0, 0.0), res: 50_000.0, bearing: 0.0, pitch: 0.0 }, + sky: false, + }; + let w = in_view(&r, &view, Place::Flat(&Flat::Mercator, "mercator"), 400, 200, true).unwrap(); + assert!(w.cols.0 == 0.0 && w.cols.1 > 5.0 && w.cols.1 < 18.0, "{:?}", w.cols); + let (a, b) = w.cols2.unwrap(); + assert!(a > 18.0 && a < 31.0 && b == 36.0, "{:?}", (a, b)); + // East of 0° only: one strip. + let east = + View { camera: Camera { center: lonlat_to_merc(Coord::new(100.0, 0.0)), res: 50_000.0, bearing: 0.0, pitch: 0.0 }, ..view }; + let w = in_view(&r, &east, Place::Flat(&Flat::Mercator, "mercator"), 400, 200, true).unwrap(); + assert!(w.cols2.is_none() && w.cols.0 > 0.0, "{:?}", (w.cols, w.cols2)); + } + #[test] fn shaded_relief_is_unchanged_when_the_geographic_seam_moves() { let elevations: Vec = (0..36).map(|c| (((c as f64 + 0.5) * 10.0).to_radians().sin() * 1_000_000.0) as f32).collect(); diff --git a/crates/render/src/raster_export.rs b/crates/render/src/raster_export.rs index 627b1dc43..b23c50d8b 100644 --- a/crates/render/src/raster_export.rs +++ b/crates/render/src/raster_export.rs @@ -12,6 +12,8 @@ const MAX_TILES: u64 = 512; const MAX_BYTES: usize = 64 * 1024 * 1024; const TILE_SIZE: u32 = 256; const MAX_LAT: f64 = 85.051_128_779_806_6; +/// Waits in a row that bring nothing new before an export stops waiting for them. +const IDLE_WAITS: usize = 6; /// Half-open tile interval, retaining a tile for a zero-width extent. fn tile_range(start: f64, end: f64, n: u32) -> (u32, u32) { @@ -57,6 +59,8 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> ranges.push((z, x0, x1, y0, y1)); } let stats = layer.masked_stats(); + // A raster read in place waits for its tiles natively, two minutes at most in all. + let deadline = web_time::Instant::now() + rustgis_data::raster::IN_PLACE_WAIT; let mut tiles = Vec::new(); let mut bytes = 0usize; for (z, x0, x1, y0, y1) in ranges { @@ -66,17 +70,44 @@ pub fn export(layer: &RasterLayer, min_zoom: u8, max_zoom: u8, opacity: f32) -> let center = Coord::new(-half + (f64::from(x) + 0.5) * size, half - (f64::from(y) + 0.5) * size); let view = View { width: 256.0, height: 256.0, pixels_per_point: 1.0, camera: Camera::new(center, size / 256.0), sky: false }; + let place = raster::Place::Flat(&Flat::Mercator, "EPSG:3857"); + let stats = stats.as_deref().map(Vec::as_slice); let mut image = Pixmap::new(TILE_SIZE, TILE_SIZE).ok_or("couldn't allocate a raster tile")?; - raster::draw( - &mut image, - r, - &layer.style, - stats.as_deref().map(Vec::as_slice), - &view, - opacity, - true, - raster::Place::Flat(&Flat::Mercator, "EPSG:3857"), - ); + match layer.path.as_deref().filter(|_| layer.in_place) { + // Read in place: at the tile's level, once its parts have arrived (natively; + // the browser draws what has). + Some(address) => { + let mut idle = 0; + loop { + let drawn_at = crate::rustgis_tiles::cache::generation(); + match raster::draw_in_place(&mut image, address, r, &layer.style, stats, &view, opacity, true, place) { + Ok(false) => break, + Ok(true) => {} + // Drawn from less than the tile needs: not written in its place. + Err(e) => return Err(format!("{}: {e}, so the archive isn't written", r.name)), + } + // An archive of the overview where the image is wanted isn't written. + let left = deadline.saturating_duration_since(web_time::Instant::now()); + if cfg!(target_arch = "wasm32") { + return Err(format!( + "{}: its tiles are still being read (this export asked for the rest): export again once they are in", + r.name + )); + } + if left.is_zero() || idle >= IDLE_WAITS { + return Err(format!("{}: its tiles didn't all arrive, so the archive isn't written", r.name)); + } + crate::rustgis_tiles::cache::wait_for_news(drawn_at, left.min(web_time::Duration::from_secs(30))); + if crate::rustgis_tiles::cache::generation() == drawn_at { + idle += 1; + } else { + idle = 0; + } + image.fill(tiny_skia::Color::TRANSPARENT); + } + } + None => raster::draw(&mut image, r, &layer.style, stats, &view, opacity, true, place), + } if image.data().as_chunks::<4>().0.iter().all(|pixel| pixel[3] == 0) { continue; } @@ -123,6 +154,28 @@ mod tests { assert_eq!((last - first + 1).pow(2), 484); } + /// A raster read in place whose image can't be read isn't exported from its overview. + #[test] + #[allow(clippy::panic)] + fn an_in_place_raster_unread_is_not_exported_from_its_overview() { + let mut overview = Raster::new( + "gone", + 4, + 4, + vec![(0..16).map(|v| v as f32).collect()], + GeoTransform { origin_x: -180.0, origin_y: 80.0, pixel_width: 90.0, pixel_height: -40.0 }, + Crs::LonLat, + None, + ColorHint::Gray, + "Float32", + ) + .unwrap_or_else(|e| panic!("{e}")); + overview.overview = Some((4096, 4096)); + let layer = RasterLayer::in_place(overview, "http://127.0.0.1:9/gone.tif"); + let e = export(&layer, 2, 2, 1.0).err().unwrap_or_else(|| panic!("an archive of the overview")); + assert!(e.contains("the archive isn't written"), "{e}"); + } + #[test] #[allow(clippy::panic)] fn exports_png_tiles_and_rejects_unbounded_or_unsupported_requests() { diff --git a/crates/render/src/tiles.rs b/crates/render/src/tiles.rs index bbbc93173..6bc3fb359 100644 --- a/crates/render/src/tiles.rs +++ b/crates/render/src/tiles.rs @@ -27,6 +27,11 @@ pub(crate) struct TileOutput { pub attribution: Vec, pub pending: bool, pub errors: Vec, + /// Raster layers read in place drawn from less than the view needs (a coarser level, tiles + /// they couldn't read), by id: shown, with the error among `errors`; an export refuses them. + pub partial: Vec, + /// Raster layers read in place whose tiles are still on their way, by id. + pub pending_in_place: Vec, } impl TileOutput { diff --git a/crates/tiles/src/cache.rs b/crates/tiles/src/cache.rs index 5ce30d1fd..481ef2503 100644 --- a/crates/tiles/src/cache.rs +++ b/crates/tiles/src/cache.rs @@ -10,7 +10,7 @@ //! map stopped asking for is dropped unfetched ([`crate::schedule`]). Natively they are fetched on //! worker threads; the browser app installs its own fetcher with [`set_fetcher`]. -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; @@ -63,7 +63,7 @@ type ImageDecoder = Box, Decoded) + Send + Sync>; /// Bytes the cache keeps before dropping the least recently used entries. const BUDGET: usize = if cfg!(target_arch = "wasm32") { 160 << 20 } else { 384 << 20 }; /// A failed address is tried again after this long. -pub(crate) const RETRY: Duration = Duration::from_secs(60); +pub const RETRY: Duration = Duration::from_secs(60); /// The largest tile or document fetched. pub const MAX_BYTES: usize = 16 << 20; /// The error a fetcher gives a request it dropped unfetched (the view moved on): the address is @@ -191,11 +191,150 @@ pub fn set_waker(f: impl Fn() + Send + Sync + 'static) { *waker().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); } -/// Goes up each time a fetch finishes: a map drawn at an older generation may now draw more. +/// When the map is next to draw again by itself (a failed read may be tried again then): each +/// wait asked for ([`notify_at`]), in milliseconds after [`alarm_base`] plus one, the earliest +/// first. Every wait is kept (the earliest [`MAX_ALARMS`]), not the earliest alone: the draw when +/// it is over may not ask again for a later one (a style kept whole while its elevation source +/// waits). +fn alarms() -> &'static Mutex> { + static A: OnceLock>> = OnceLock::new(); + A.get_or_init(Mutex::default) +} + +/// The most waits kept at once: a later one past them all is left to its read, which asks again +/// on each draw while it waits. +const MAX_ALARMS: usize = 1024; + +fn alarm_base() -> Instant { + static B: OnceLock = OnceLock::new(); + *B.get_or_init(Instant::now) +} + +/// `when` as the alarms keep it. +fn alarm_at(when: Instant) -> u64 { + (when.saturating_duration_since(alarm_base()).as_millis() as u64).saturating_add(1) +} + +fn alarm_hook() -> &'static Mutex>> { + static A: OnceLock>>> = OnceLock::new(); + A.get_or_init(Mutex::default) +} + +/// Sets how the app is asked to draw again after a while (a repaint after it): a read that failed +/// is tried again once its wait is over, though nothing else draws the map. The request lasts +/// one frame (egui's `request_repaint_after`): it is made again on each draw while the wait +/// lasts (see [`generation`]). +pub fn set_alarm(f: impl Fn(Duration) + Send + Sync + 'static) { + *alarm_hook().lock().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(f)); +} + +/// Asks the app to draw again after `after` (see [`set_alarm`]). +fn ring(after: Duration) { + let hook = alarm_hook().lock().unwrap_or_else(|e| e.into_inner()).clone(); + if let Some(hook) = hook { + hook(after); + } +} + +/// Draws the map again at `when`: [`generation`] goes up then, so a map waiting on a failed read +/// asks for it again. The app is asked for the earliest wait pending; each draw asks it again +/// for the earliest left (see [`generation`]). +pub fn notify_at(when: Instant) { + let at = alarm_at(when); + let earliest = { + let mut a = alarms().lock().unwrap_or_else(|e| e.into_inner()); + if a.len() >= MAX_ALARMS { + // Full: a wait past every one kept is left to its read; an earlier one takes the + // latest's place. + if a.last().is_some_and(|last| at >= *last) { + return; + } + a.pop_last(); + } + a.insert(at); + a.first() == Some(&at) + }; + if earliest { + ring(when.saturating_duration_since(Instant::now())); + } +} + +/// Goes up each time a fetch finishes, and once a wait asked for ([`notify_at`]) is over: a map +/// drawn at an older generation may now draw more. Read as the map draws, it asks the app to draw +/// again at the earliest wait still pending: the app's request lasts one frame (egui's +/// `request_repaint_after`), and a draw before the wait was over (the pointer moved) used it up. pub fn generation() -> u64 { + let now = alarm_at(Instant::now()); + let next = { + let mut a = alarms().lock().unwrap_or_else(|e| e.into_inner()); + if a.first().is_some_and(|at| *at <= now) { + // Every wait over is one draw: the generation goes up once for them all. + let later = a.split_off(&now.saturating_add(1)); + *a = later; + GENERATION.fetch_add(1, Ordering::Relaxed); + } + a.first().copied() + }; + if let Some(at) = next { + ring(Duration::from_millis(at.saturating_sub(now))); + } GENERATION.load(Ordering::Relaxed) } +/// How long until the earliest wait pending; none when there is none. +#[cfg(test)] +pub(crate) fn next_alarm() -> Option { + let at = alarms().lock().unwrap_or_else(|e| e.into_inner()).first().copied()?; + Some(Duration::from_millis(at.saturating_sub(alarm_at(Instant::now())))) +} + +/// Forgets every wait pending. +#[cfg(test)] +pub(crate) fn clear_alarms() { + alarms().lock().unwrap_or_else(|e| e.into_inner()).clear(); +} + +/// Undoes [`set_alarm`]: a test done with its own. +#[cfg(test)] +pub(crate) fn clear_alarm() { + *alarm_hook().lock().unwrap_or_else(|e| e.into_inner()) = None; +} + +/// A test's turn at the process-wide alarms, held by the tests that arm, read or clear them, or +/// set what rings: they run in parallel threads of one process, and one's clearing would erase +/// another's. +#[cfg(test)] +pub(crate) fn alarm_turn() -> std::sync::MutexGuard<'static, ()> { + static L: Mutex<()> = Mutex::new(()); + L.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// How long a server that refused with 429 or 503 asked to be left before it is asked again, as +/// the error's text says it (`http status: 503 (retry after 30 s)`; see +/// [`crate::failure::with_retry_after`]). +pub fn retry_after(error: &str) -> Option { + let (_, rest) = error.rsplit_once("(retry after ")?; + let secs: u64 = rest.strip_suffix(" s)")?.parse().ok()?; + // Bounded as when written: a text from elsewhere (an address echoed) can't ask for years. + Some(Duration::from_secs(secs.clamp(1, crate::failure::MAX_RETRY_AFTER))) +} + +/// How long a failed read waits before it is tried again: what its server asked for (see +/// [`retry_after`]), else `default`. +pub fn retry_wait(error: &str, default: Duration) -> Duration { + retry_after(error).unwrap_or(default) +} + +/// How long a read that has failed `failures` times in a row waits before the next try: `wait`, +/// doubled for each failure past the first, within the hour (see [`retry_after`]'s bound), so a +/// file that never reads isn't asked for every minute all session. +pub fn backoff(wait: Duration, failures: u32) -> Duration { + let most = Duration::from_secs(crate::failure::MAX_RETRY_AFTER); + // Past twelve doublings the wait is over the hour whatever it was (it is a second at least). + let doublings = failures.saturating_sub(1).min(12); + wait.saturating_mul(1u32 << doublings).min(most) +} + /// How many fetches are under way. Once a fetch no longer counts, its [`generation`] shows. pub fn in_flight() -> usize { IN_FLIGHT.load(Ordering::Acquire) @@ -242,7 +381,15 @@ fn request(url: &str, want: Want, omit: &'static [u16]) -> State { match &slot.entry { Entry::Ready(p, _) => return State::Ready(p.clone()), Entry::Pending => return State::Pending, - Entry::Failed(e, at) if at.elapsed() < RETRY => return State::Failed(e.clone()), + Entry::Failed(e, at) if at.elapsed() < retry_wait(e, RETRY) => { + // A server that said when to come back (429 or 503 with `Retry-After`) is asked + // again then: the map draws again at that time, though nothing else draws it. + // One that didn't is asked again by the next draw after a minute, as before. + if let Some(wait) = retry_after(e) { + notify_at(*at + wait); + } + return State::Failed(e.clone()); + } Entry::Failed(..) => {} } } @@ -537,6 +684,10 @@ fn fetch_tile(url: String, host: String, want: Want, omit: &'static [u16]) { } Err(e) => { log::debug!("{}", crate::logins::scrub(&format!("{}: {e}", answer.key))); + // Refused until a time its server gave: the map draws again then. + if let Some(wait) = retry_after(&e) { + notify_at(Instant::now() + wait); + } Entry::Failed(e, Instant::now()) } }); @@ -773,8 +924,11 @@ impl<'a> RequestDiagnostics<'a> { /// another message (an address can hold "404"). pub fn http_status(error: &str) -> Option { let e = error.trim(); - let code = e.strip_prefix("http status: ").or_else(|| e.strip_prefix("HTTP "))?; - (code.len() == 3 && code.bytes().all(|b| b.is_ascii_digit())).then(|| code.parse().ok()).flatten() + let rest = e.strip_prefix("http status: ").or_else(|| e.strip_prefix("HTTP "))?; + // The status, and nothing after it but how long to wait (see [`retry_after`]). + let (code, after) = rest.split_at_checked(3)?; + let known = after.is_empty() || (after.starts_with(" (retry after ") && retry_after(after).is_some()); + (known && code.bytes().all(|b| b.is_ascii_digit())).then(|| code.parse().ok()).flatten() } /// Downloads a file the user named (data added by its web address): up to [`MAX_DOWNLOAD`] @@ -1499,7 +1653,11 @@ mod native { let status = resp.status().as_u16(); diagnostics.answered(status); if !(200..300).contains(&status) { - return Err(Failure::without_answer("", format!("http status: {status}"), Reason::Status(status))); + return Err(Failure::without_answer( + "", + crate::failure::with_retry_after(format!("http status: {status}"), status, retry_after_header(&resp)), + Reason::Status(status), + )); } let mut buf = Vec::new(); resp.body_mut() @@ -1591,6 +1749,11 @@ mod native { Failure::without_answer("", e.to_string(), reason) } + /// An answer's `Retry-After` header, as sent. + fn retry_after_header(resp: &ureq::http::Response) -> Option<&str> { + resp.headers().get("retry-after").and_then(|v| v.to_str().ok()) + } + /// An answer with an error status (#509): its kept headers and the start of its body. fn refused(resp: &mut ureq::http::Response) -> Failure { use std::io::Read; @@ -1735,7 +1898,11 @@ mod native { return Err(if detail && !prepared.signed && login.is_none() { refused(&mut resp) } else { - Failure::without_answer("", format!("http status: {status}"), Reason::Status(status)) + Failure::without_answer( + "", + crate::failure::with_retry_after(format!("http status: {status}"), status, retry_after_header(&resp)), + Reason::Status(status), + ) }); } if !resp.status().is_redirection() { @@ -2114,6 +2281,12 @@ mod tests { let records = network_records(&history, &url); assert_eq!(records.len(), 1); assert_eq!(records[0].network.as_ref().unwrap().status, Some(503)); + // One that says when to ask again: its wait goes with the status, read back as the wait. + let wait = "HTTP/1.1 503 Unavailable\r\nRetry-After: 30\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let base = serve(vec![wait.into()]); + let e = native::get(&native::agent(), &format!("{base}/tiles/0/0/0.png"), &[404]).unwrap_err(); + assert_eq!(e, "http status: 503 (retry after 30 s)"); + assert_eq!((http_status(&e), retry_wait(&e, RETRY)), (Some(503), Duration::from_secs(30))); // A download's absence names the address asked for, not the module's markers. let base = serve(vec![not_found.into()]); let url = format!("{base}/data.zarr/zarr.json"); @@ -2289,6 +2462,110 @@ mod tests { assert_eq!(http_status("https://x.org/404/a.zarr: timed out"), None); assert_eq!(http_status("HTTP 4040"), None); assert_eq!(http_status("io: connection reset (read 404 bytes)"), None); + // With how long the server asked to be left: the status, and the wait. + assert_eq!(http_status("http status: 503 (retry after 30 s)"), Some(503)); + assert_eq!(http_status("HTTP 429 (retry after 2 s)"), Some(429)); + assert_eq!(http_status("HTTP 429 (something else)"), None); + assert_eq!(retry_after("http status: 503 (retry after 30 s)"), Some(Duration::from_secs(30))); + assert_eq!(retry_after("http status: 503"), None); + assert_eq!(retry_after("x (retry after 18446744073709551615 s)"), Some(Duration::from_secs(3600)), "bounded, never overflowing"); + assert_eq!(retry_after("x (retry after 0 s)"), Some(Duration::from_secs(1))); + assert_eq!(retry_wait("HTTP 429 (retry after 2 s)", RETRY), Duration::from_secs(2)); + assert_eq!(retry_wait("http status: 500", RETRY), RETRY); + } + + /// A read that fails again waits twice as long each time, within the hour: a file that never + /// reads isn't asked for every minute all session. + #[test] + fn a_read_that_fails_again_waits_longer_each_time() { + let minute = Duration::from_secs(60); + assert_eq!(backoff(minute, 0), minute); + assert_eq!(backoff(minute, 1), minute); + assert_eq!(backoff(minute, 2), Duration::from_secs(120)); + assert_eq!(backoff(minute, 3), Duration::from_secs(240)); + assert_eq!(backoff(minute, 7), Duration::from_secs(3600), "within the hour"); + assert_eq!(backoff(minute, u32::MAX), Duration::from_secs(3600), "bounded, never overflowing"); + // A server's own wait is doubled the same way, within the hour. + assert_eq!(backoff(Duration::from_secs(1), 2), Duration::from_secs(2)); + assert_eq!(backoff(Duration::from_secs(3600), 2), Duration::from_secs(3600)); + } + + /// A tile its server refused saying when to come back (429 or 503 with `Retry-After`) has the + /// map asked to draw again then, from the gate, on each draw while it waits (the alarm is + /// one-shot, and another may have fired since; the failure arms it first). The alarm is the + /// earliest pending one, read here as the map reads it. The entry is stored as a failed fetch + /// stores it: the fetcher is one for the process, and another test's may be in. + #[test] + fn a_tile_refused_with_retry_after_has_the_map_drawn_again_then() { + // Its turn at the cache (the entry is planted in it, and another test's clearing would + // drop it) and at the alarms. + let _turn = super::turn(); + let _alarm = super::alarm_turn(); + let within = |secs: u64| super::next_alarm().is_some_and(|d| d <= Duration::from_secs(secs)); + let url = "test://cache/busy.json"; + store(url, Entry::Failed("http status: 503 (retry after 2 s)".into(), Instant::now())); + super::clear_alarms(); + let State::Failed(e) = get(url, Want::Json) else { panic!("not failed") }; + assert!(e.contains("retry after 2 s"), "{e}"); + assert!(within(2), "no alarm on a draw while waiting: {:?}", super::next_alarm()); + // Again on the next draw: the alarm fired (or was consumed) before the wait was over. + super::clear_alarms(); + assert!(matches!(get(url, Want::Json), State::Failed(_))); + assert!(within(2), "not armed again: {:?}", super::next_alarm()); + } + + /// A read that failed asks the map to draw again when it may be tried: the generation goes up + /// once that time has come. + #[test] + fn the_map_draws_again_when_a_wait_is_over() { + let _alarm = super::alarm_turn(); + let before = generation(); + notify_at(Instant::now() + Duration::from_millis(20)); + std::thread::sleep(std::time::Duration::from_millis(40)); + assert!(generation() > before); + } + + /// Every wait asked for has the map drawn again when it is over, not the earliest alone: the + /// draw when the earlier is over may not ask for the later again (a style kept whole while + /// its elevation source waits), so the later is kept, and the app asked for it then. + #[test] + fn a_later_wait_isnt_lost_behind_an_earlier_one() { + let _alarm = super::alarm_turn(); + let asked = std::sync::Arc::new(Mutex::new(Vec::new())); + let noted = asked.clone(); + super::set_alarm(move |after| noted.lock().unwrap_or_else(|e| e.into_inner()).push(after)); + super::clear_alarms(); + let before = generation(); + notify_at(Instant::now() + Duration::from_millis(20)); + notify_at(Instant::now() + Duration::from_millis(150)); + std::thread::sleep(std::time::Duration::from_millis(50)); + let earlier = generation(); + assert!(earlier > before, "the earlier wait over"); + // The later is still pending, and asked of the app as the earlier was over. + assert!(super::next_alarm().is_some_and(|d| d <= Duration::from_millis(150)), "{:?}", super::next_alarm()); + let later = |asked: &Vec| asked.iter().any(|d| (Duration::from_millis(30)..=Duration::from_millis(150)).contains(d)); + assert!(later(&asked.lock().unwrap_or_else(|e| e.into_inner())), "{:?}", asked.lock().unwrap_or_else(|e| e.into_inner())); + std::thread::sleep(std::time::Duration::from_millis(150)); + assert!(generation() > earlier, "the later wait over"); + super::clear_alarm(); + } + + /// A draw before a wait is over asks the app for it again: the app's request lasts one frame, + /// and a draw since (the pointer moved) used it up. + #[test] + fn a_draw_before_a_wait_is_over_asks_for_it_again() { + let _alarm = super::alarm_turn(); + let asked = std::sync::Arc::new(Mutex::new(Vec::new())); + let noted = asked.clone(); + super::set_alarm(move |after| noted.lock().unwrap_or_else(|e| e.into_inner()).push(after)); + super::clear_alarms(); + notify_at(Instant::now() + Duration::from_millis(300)); + asked.lock().unwrap_or_else(|e| e.into_inner()).clear(); + let _ = generation(); + let again = asked.lock().unwrap_or_else(|e| e.into_inner()).iter().any(|d| *d <= Duration::from_millis(300)); + assert!(again, "{:?}", asked.lock().unwrap_or_else(|e| e.into_inner())); + super::clear_alarms(); + super::clear_alarm(); } #[test] diff --git a/crates/tiles/src/cog.rs b/crates/tiles/src/cog.rs index 8e98a7378..f1dc6fe9a 100644 --- a/crates/tiles/src/cog.rs +++ b/crates/tiles/src/cog.rs @@ -3,24 +3,29 @@ //! tile (`cog:{z}/{x}/{y}@
`) is drawn from them in Web Mercator, as a picture, so a COG is //! a tile layer like a PMTiles archive: drawn as its tiles land, on the desktop and in the browser. //! -//! Grey images are stretched between their 2nd and 98th percentiles (from the coarsest -//! overview); 8-bit colour images are shown as they are. An address may end with a [`Look`] -//! (`#bands=2,17,10&range=-0.3,0.3`): the bands shown as red, green and blue (or one, grey) and -//! their stretch. Bands stored apart (planar files, such as AlphaEarth's 64 embedding bands) are -//! read a band at a time, only those shown. Files are hostile input: header reads, tile counts and -//! sizes are bounded (see [`rustgis_data::raster::cog`]). - -use std::collections::HashMap; +//! A file's own look is what its PhotometricInterpretation says: RGB images in colour, palette +//! images by their colour table, grey (MinIsBlack) ones grey, their first band, but for three bands +//! (taken as red, green and blue), each with its alpha band. Other bands of an image with more are +//! for the user to pick. Grey and colour images are stretched between their 2nd and 98th +//! percentiles (from the coarsest overview); 8-bit colour images are shown as they are. An address +//! may end with a [`Look`] (`#bands=2,17,10&range=-0.3,0.3`): the bands shown as red, green and +//! blue (or one, grey) and their stretch. Bands stored apart (planar files, such as AlphaEarth's 64 +//! embedding bands) are read a band at a time, only those shown. Files are hostile input: header +//! reads, tile counts and sizes are bounded (see [`rustgis_data::raster::cog`]). + +use std::collections::{BTreeMap, HashMap}; use std::sync::{Arc, Mutex, OnceLock}; use rustgis_data::raster::ColorHint; -use rustgis_data::raster::cog::{self, Layout, Parse, Sparse}; +use rustgis_data::raster::cog::{self, Layout, Level, Parse, Sparse}; use rustgis_data::raster::crs::Transformer; use rustgis_geom::Coord; use web_time::{Duration, Instant}; use crate::cache::{Done, MAX_BYTES, RANGE}; +pub mod raster; + /// The scheme of a map tile drawn from a COG. pub const SCHEME: &str = "cog:"; @@ -68,6 +73,9 @@ pub struct Look { pub range: Option<(f32, f32)>, /// Values are AlphaEarth's quantised embeddings, shown as `sign(v)·(v/127.5)²`. pub aef: bool, + /// A data cube's file (`stac.addCube`): a tile layer that follows the time slider, kept one + /// when its project opens again. + pub cube: bool, } impl Look { @@ -75,7 +83,7 @@ impl Look { pub fn suffix(&self) -> String { let mut parts = Vec::new(); if !self.bands.is_empty() { - parts.push(format!("bands={}", self.bands.iter().map(|b| (b + 1).to_string()).collect::>().join(","))); + parts.push(format!("bands={}", self.bands.iter().map(|b| b.saturating_add(1).to_string()).collect::>().join(","))); } if let Some((lo, hi)) = self.range { parts.push(format!("range={lo},{hi}")); @@ -83,6 +91,9 @@ impl Look { if self.aef { parts.push("dequantize=aef".into()); } + if self.cube { + parts.push("cube=1".into()); + } if parts.is_empty() { String::new() } else { format!("#{}", parts.join("&")) } } @@ -108,6 +119,7 @@ impl Look { look.range = Some((lo, hi)); } "dequantize" => look.aef = v == "aef", + "cube" => look.cube = v == "1", _ => return None, } } @@ -168,33 +180,52 @@ pub struct Info { /// Stretches whose read failed, and when: drawn with the type's full range meanwhile (not /// kept), read again after [`RETRY`]. stretch_failed: Mutex, bool), Instant>>, - /// The file's own bands shown as red, green, blue (or one, grey), and alpha. - rgb: bool, - alpha: bool, + /// The file's alpha band, shown with any look. + alpha: Option, } impl Info { - /// The bands a look shows (0-based): three as red, green and blue (and the file's alpha, as a - /// fourth, for its own), or one, grey. - pub fn shown(&self, look: &Look) -> Vec { + /// The bands a look colours the map with (0-based): three as red, green and blue, or one, + /// grey. Its own: an RGB image's colour bands, or a grey image's three bands (most remote + /// sensing images are grey, whatever their bands: three are taken as red, green and blue); + /// else the first band, the others for the user to pick. + pub fn colour(&self, look: &Look) -> Vec { let n = self.layout.bands.max(1); // Bands the file has, as many as asked (one or three); otherwise the file's own look. let picked: Vec = look.bands.iter().copied().filter(|b| *b < n).collect(); match picked.len() { 1 | 3 if picked.len() == look.bands.len() => picked, - _ if self.rgb && self.alpha => vec![0, 1, 2, 3], - _ if self.rgb => vec![0, 1, 2], + _ if matches!(self.layout.color, ColorHint::Rgb { .. }) && n >= 3 => vec![0, 1, 2], + _ if matches!(self.layout.color, ColorHint::Gray) && n == 3 => vec![0, 1, 2], _ => vec![0], } } + /// The bands a look reads: its colour bands, then the file's alpha band, if it has one. + pub fn shown(&self, look: &Look) -> Vec { + self.shown_from(self.colour(look)) + } + + /// [`Info::shown`], from the look's colours already found. + fn shown_from(&self, mut shown: Vec) -> Vec { + if let Some(a) = self.alpha.filter(|a| !shown.contains(a)) { + shown.push(a); + } + shown + } + + /// The colours of a palette image's values, when its own look shows them. + fn palette(&self, look: &Look) -> Option<&[[u8; 4]]> { + self.layout.palette.as_deref().filter(|_| look.bands.is_empty() && look.range.is_none() && !look.aef) + } + /// The bands' (low, high) values for a look, when known. - fn stretch_for(&self, look: &Look, shown: &[usize]) -> Option> { - let colour = &shown[..shown.len().min(3)]; + fn stretch_for(&self, look: &Look) -> Option> { + let colour = self.colour(look); if let Some(r) = look.range { return Some(vec![r; colour.len()]); } - self.stretch.lock().unwrap_or_else(|e| e.into_inner()).get(&(colour.to_vec(), look.aef)).cloned() + self.stretch.lock().unwrap_or_else(|e| e.into_inner()).get(&(colour, look.aef)).cloned() } } @@ -212,7 +243,9 @@ fn infos() -> &'static Mutex> { type Bands = Arc; enum TileState { - Ready(Bands, usize), + /// Decoded: its bands, how many samples they hold, and when it was last used (see + /// [`Tiles::clock`]). + Ready(Bands, usize, u64), Loading(Vec) + Send>>), } @@ -220,6 +253,52 @@ enum TileState { struct Tiles { map: HashMap<(String, usize, u32), TileState>, samples: usize, + /// Goes up with each use: past [`BUDGET`], the tile used longest ago makes room, never one + /// the map in view has just used (which would be read again, and make room for itself by + /// throwing out another in view: the view flickering between levels). + clock: u64, + /// The tiles ready, by when each was last used (the clock then, unique): the first is the + /// one to make room, found without a scan of them all. + order: BTreeMap, +} + +impl Tiles { + /// A tile ready, marked as just used. + fn used(&mut self, key: &(String, usize, u32)) -> Option { + self.clock += 1; + let now = self.clock; + match self.map.get_mut(key) { + Some(TileState::Ready(b, _, used)) => { + self.order.remove(used); + *used = now; + self.order.insert(now, key.clone()); + Some(b.clone()) + } + _ => None, + } + } + + /// Keeps a tile ready, making room for it past [`BUDGET`] (the tiles used longest ago go); + /// returns what was kept under its key before. + fn keep(&mut self, key: (String, usize, u32), bands: Bands, samples: usize) -> Option { + // Kept under this key already: counted once. + let before = self.map.remove(&key); + if let Some(TileState::Ready(_, n, used)) = &before { + self.samples = self.samples.saturating_sub(*n); + self.order.remove(used); + } + while self.samples + samples > BUDGET { + let Some((_, k)) = self.order.pop_first() else { break }; + if let Some(TileState::Ready(_, n, _)) = self.map.remove(&k) { + self.samples = self.samples.saturating_sub(n); + } + } + self.clock += 1; + self.samples += samples; + self.order.insert(self.clock, key.clone()); + self.map.insert(key, TileState::Ready(bands, samples, self.clock)); + before + } } fn tiles() -> &'static Mutex { @@ -227,24 +306,41 @@ fn tiles() -> &'static Mutex { T.get_or_init(Mutex::default) } -/// Forgets every file and tile, so they are read again (with [`crate::cache::clear`]). +/// Forgets every file and tile (those that failed too), so they are read again (with +/// [`crate::cache::clear`]). pub(crate) fn forget() { infos().lock().unwrap_or_else(|e| e.into_inner()).retain(|_, s| matches!(s, State::Loading(_))); + raster::forget_failed(None); let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); t.map.retain(|_, s| matches!(s, TileState::Loading(_))); + t.order.clear(); t.samples = 0; } -/// Forgets what was read of one file (its info and its tiles), but not a read under way: a -/// failed read is made again with whatever login now signs the file's address. +/// Forgets what was read of one file (its info and its tiles, those that failed too), but not a +/// read under way: a failed read is made again with whatever login now signs the file's address. pub(crate) fn forget_file(url: &str) { let mut all = infos().lock().unwrap_or_else(|e| e.into_inner()); if all.get(url).is_some_and(|s| !matches!(s, State::Loading(_))) { all.remove(url); } drop(all); - let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); - t.map.retain(|(u, _, _), s| u != url || matches!(s, TileState::Loading(_))); + raster::forget_failed(Some(url)); + forget_file_in(&mut tiles().lock().unwrap_or_else(|e| e.into_inner()), url); +} + +/// Forgets a file's tiles kept (not those loading) from `t`. +fn forget_file_in(t: &mut Tiles, url: &str) { + let mut freed = 0; + t.map.retain(|(u, _, _), s| match s { + TileState::Ready(_, n, _) if u == url => { + freed += *n; + false + } + _ => true, + }); + t.samples = t.samples.saturating_sub(freed); + t.order.retain(|_, (u, _, _)| u != url); } /// A file's info: `Ok(None)` while it is being read (asking for it starts the read). @@ -256,7 +352,14 @@ pub fn info(url: &str) -> Result>, String> { match i.get(url) { Some(State::Ready(info)) => return Ok(Some(info.clone())), Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => return Err(e.clone()), + Some(State::Failed(e, at)) => { + // Not read again until its wait is over: the map draws again then. + let wait = crate::cache::retry_wait(e, RETRY); + if at.elapsed() < wait { + crate::cache::notify_at(*at + wait); + return Err(e.clone()); + } + } _ => {} } } @@ -282,7 +385,8 @@ fn with_info(url: &str, then: Box, String>) + Send>) waiting.push(then); return; } - Some(State::Failed(e, at)) if at.elapsed() < RETRY => { + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { + crate::cache::notify_at(*at + crate::cache::retry_wait(e, RETRY)); let e = e.clone(); drop(i); then(Err(e)); @@ -305,7 +409,12 @@ fn with_info(url: &str, then: Box, String>) + Send>) let mut i = infos().lock().unwrap_or_else(|e| e.into_inner()); let next = match &r { Ok(info) => State::Ready(info.clone()), - Err(e) => State::Failed(e.clone(), Instant::now()), + Err(e) => { + // Read again once its wait is over: the map draws again then (a 429 + // or 503 says when). + crate::cache::notify_at(Instant::now() + crate::cache::retry_wait(e, RETRY)); + State::Failed(e.clone(), Instant::now()) + } }; match i.insert(key, next) { Some(State::Loading(w)) => w, @@ -494,8 +603,16 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box 0.0 { (2.0 * HALF / f64::from(SIZE) / res).log2().ceil().clamp(0.0, 24.0) as u8 } else { 18 }; - let rgb = layout.bands >= 3; - let alpha = matches!(layout.color, ColorHint::Rgb { alpha: true }); + let alpha = match layout.color { + ColorHint::Rgb { alpha: true } if layout.bands >= 4 => Some(3), + ColorHint::GrayAlpha if layout.bands >= 2 => Some(1), + _ => None, + }; + let rgb = match layout.color { + ColorHint::Rgb { .. } => layout.bands >= 3, + ColorHint::Gray => layout.bands == 3, + _ => false, + }; let bytes = layout.data_type == "UInt8"; let info = Arc::new(Info { layout, @@ -508,33 +625,37 @@ fn stats(url: String, header: Sparse, layout: Layout, done: Box done(Ok(i)), - Err(e) => done(Err(format!("{url}: {e}"))), - }), - ); + // Any other look's stretch is read when a tile first needs it (`with_stretch`): a header read + // for a window or an in-place overview reads nothing more. + done(Ok(info)); } -/// The image the stretch is read from: the coarsest at least 256 pixels wide (GDAL writes -/// overviews down to a pixel, too few to take percentiles of), else the full image. -fn stats_level(layout: &Layout) -> usize { - layout.levels.iter().rposition(|l| l.width >= 256 && l.height >= 256).unwrap_or(0) +/// The image the stretch is read from: of those few enough tiles to read for it, the coarsest of +/// at least 256 × 256 pixels (GDAL writes overviews down to a pixel, too few to take percentiles +/// of), else the finest. A long strip's overviews are narrow: one 256 pixels wide is too many tiles +/// long, a narrower one plenty of pixels. None small enough: the coarsest (the type's range is +/// shown). +fn stats_level(levels: &[Level]) -> usize { + let few = |l: &Level| (l.tiles() as usize) <= MAX_STATS_TILES; + let pixels = |l: &Level| u64::from(l.width) * u64::from(l.height); + levels + .iter() + .rposition(|l| few(l) && pixels(l) >= 256 * 256) + .or_else(|| levels.iter().position(few)) + .unwrap_or(levels.len().saturating_sub(1)) } /// The full range of a file's type of values, as a look shows them (floats have none: 0–255 is a @@ -554,7 +675,7 @@ fn type_range(layout: &Layout, aef: bool) -> (f32, f32) { /// Keeps a look's stretch: a few looks a file (a script trying many is hostile input); past them, /// the others are read again when asked for. fn keep_stretch(info: &Info, key: (Vec, bool), s: Vec<(f32, f32)>) { - let own = (info.shown(&Look::default()).into_iter().take(3).collect::>(), false); + let own = (info.colour(&Look::default()), false); let mut kept = info.stretch.lock().unwrap_or_else(|e| e.into_inner()); // Past the cap one other look goes, never the file's own (an 8-bit colour image's is "as it // is", not read). @@ -583,13 +704,18 @@ fn percentiles(mut v: Vec) -> (f32, f32) { type StretchThen = Box, String>) + Send>; fn with_stretch(info: Arc, file: String, look: Look, then: StretchThen) { - let shown = info.shown(&look); - if let Some(s) = info.stretch_for(&look, &shown) { + if let Some(s) = info.stretch_for(&look) { then(Ok(s)); return; } - let colour: Vec = shown.iter().copied().take(3).collect(); - let coarsest = stats_level(&info.layout); + // A palette image's own look colours its values by its table (see `paint`): no stretch to + // read, and none kept, which a look of its bands would take for its own. + if info.palette(&look).is_some() { + then(Ok(vec![(0.0, 255.0)])); + return; + } + let colour = info.colour(&look); + let coarsest = stats_level(&info.layout.levels); let n = info.layout.levels.get(coarsest).map(|l| l.tiles() as usize).unwrap_or(0); // Too many tiles to read for it: the values' full range for the type. if n == 0 || n > MAX_STATS_TILES { @@ -725,50 +851,144 @@ fn fetch_bands( type Gathered = Box, String>) + Send>; +/// How many times the fetch queue may drop the tiles of a joint read of `tiles` before it fails: +/// eight a tile, 64 at least, so a busy queue delays a read but one that never gets its turn ends +/// (the browser has no other end to it). +fn drops_allowed(tiles: usize) -> usize { + tiles.saturating_mul(8).max(64) +} + +/// Tiles a joint read ([`fetch_all`]) has under way at once: half the fetch queue, so a read of +/// many (the overview of a file with few overviews) doesn't push its own requests out of it. +pub(super) const AT_ONCE: usize = 256; + /// Reads tiles of a level (each once, shared by the map tiles that need it), then calls `then` -/// with their bands in the order asked. +/// with their bands in the order asked. [`AT_ONCE`] are under way at a time, and a tile the fetch +/// queue dropped (other reads filled it: see [`crate::cache::DROPPED`]) is asked for again, so a +/// drop is a delay, not a failure, until the drops pass [`drops_allowed`]: then the read fails, +/// saying so, rather than asking without end (the layer reads again later). fn fetch_all(info: Arc, url: String, level: usize, wanted: Vec, then: Gathered) { + fetch_all_with(Arc::new(get_tile), info, url, level, wanted, then) +} + +/// How [`fetch_all_with`] gets a tile: [`get_tile`], or a test's stand-in. +type GetTile = Arc, String, usize, u32, Box) + Send>) + Send + Sync>; + +/// [`fetch_all`] through `get`. +fn fetch_all_with(get: GetTile, info: Arc, url: String, level: usize, wanted: Vec, then: Gathered) { if wanted.is_empty() { then(Ok(Vec::new())); return; } struct Gather { + get: GetTile, + info: Arc, + url: String, + level: usize, + wanted: Vec, got: Vec>, - left: usize, + /// Places in `wanted` not asked for yet (or dropped, to ask for again), the next last. + to_ask: Vec, + in_flight: usize, + /// Someone is asking for the next tiles, and asks again after: a tile answered at once + /// leaves it to them (asking from the answer would nest as deep as the tiles kept). + asking: bool, + /// Tiles the fetch queue dropped so far, bounded by [`drops_allowed`]. + dropped: usize, failed: Option, then: Option, } - let g = Arc::new(Mutex::new(Gather { got: vec![None; wanted.len()], left: wanted.len(), failed: None, then: Some(then) })); - for (k, idx) in wanted.into_iter().enumerate() { - let g = g.clone(); - get_tile( - info.clone(), - url.clone(), - level, - idx, - Box::new(move |r| { + let n = wanted.len(); + let g = Arc::new(Mutex::new(Gather { + get, + info, + url, + level, + wanted, + got: vec![None; n], + to_ask: (0..n).rev().collect(), + in_flight: 0, + asking: false, + dropped: 0, + failed: None, + then: Some(then), + })); + ask(&g); + + /// Asks for the next tiles, [`AT_ONCE`] under way at most, until all are here or one failed + /// (then once the rest have answered), and answers `then` once. + fn ask(g: &Arc>) { + loop { + let (get, info, url, level, batch) = { let mut s = g.lock().unwrap_or_else(|e| e.into_inner()); - match r { - Ok(b) => { - if let Some(slot) = s.got.get_mut(k) { - *slot = Some(b); - } + if s.asking { + return; + } + if s.in_flight == 0 && (s.to_ask.is_empty() || s.failed.is_some()) { + if let Some(then) = s.then.take() { + let r = match s.failed.take() { + Some(e) => Err(e), + None => Ok(s.got.iter_mut().filter_map(Option::take).collect()), + }; + drop(s); + then(r); } - Err(e) => s.failed = Some(e), + return; } - s.left = s.left.saturating_sub(1); - if s.left == 0 - && let Some(then) = s.then.take() - { - let r = match s.failed.take() { - Some(e) => Err(e), - None => Ok(s.got.iter_mut().filter_map(Option::take).collect()), - }; - drop(s); - then(r); + let mut batch = Vec::new(); + while s.in_flight < AT_ONCE && s.failed.is_none() { + let Some(k) = s.to_ask.pop() else { break }; + let Some(&idx) = s.wanted.get(k) else { continue }; + s.in_flight += 1; + batch.push((k, idx)); } - }), - ); + if batch.is_empty() { + return; + } + s.asking = true; + (s.get.clone(), s.info.clone(), s.url.clone(), s.level, batch) + }; + for (k, idx) in batch { + let g = g.clone(); + get( + info.clone(), + url.clone(), + level, + idx, + Box::new(move |r| { + { + let mut s = g.lock().unwrap_or_else(|e| e.into_inner()); + s.in_flight = s.in_flight.saturating_sub(1); + match r { + Ok(b) => { + if let Some(slot) = s.got.get_mut(k) { + *slot = Some(b); + } + } + // Dropped: asked for again, within the drops allowed (a batch + // already asked for answers after the read failed: not counted). + Err(e) if e == crate::cache::DROPPED => { + if s.failed.is_none() { + s.dropped += 1; + if s.dropped > drops_allowed(s.wanted.len()) { + s.failed = Some(format!( + "the fetch queue dropped its tiles {} times (other reads filled it): busy", + s.dropped + )); + } else { + s.to_ask.push(k); + } + } + } + Err(e) => s.failed = Some(e), + } + } + ask(&g); + }), + ); + } + g.lock().unwrap_or_else(|e| e.into_inner()).asking = false; + } } } @@ -789,9 +1009,21 @@ fn get_tile(info: Arc, url: String, level: usize, idx: u32, then: Box b, + None => { + let b: Bands = Arc::new(vec![vec![f32::NAN; per_band]; nb].into()); + t.keep(key, b.clone(), samples); + b + } + }; + drop(t); + then(Ok(empty)); return; } // Bytes at offset 0 would be the file's header: a broken index, said rather than drawn blank. @@ -802,13 +1034,13 @@ fn get_tile(info: Arc, url: String, level: usize, idx: u32, then: Box { - let b = b.clone(); - drop(t); - then(Ok(b)); - return; - } + Some(TileState::Ready(..)) => {} Some(TileState::Loading(w)) => { w.push(then); return; @@ -830,19 +1062,8 @@ fn get_tile(info: Arc, url: String, level: usize, idx: u32, then: Box { - // Over the budget: tiles ready make room (the cache is a working set). - while t.samples + samples > BUDGET { - let Some(k) = t.map.iter().find(|(_, s)| matches!(s, TileState::Ready(..))).map(|(k, _)| k.clone()) else { - break; - }; - if let Some(TileState::Ready(_, n)) = t.map.remove(&k) { - t.samples = t.samples.saturating_sub(n); - } - } - t.samples += samples; - t.map.insert(key, TileState::Ready(b.clone(), samples)) - } + // Over the budget: the tiles used longest ago make room. + Ok(b) => t.keep(key, b.clone(), samples), // A failed tile is read again by the next map tile that needs it. Err(_) => t.map.remove(&key), }; @@ -993,8 +1214,18 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 done(Err("too many GeoTIFF tiles for one map tile".into())); return; } - let shown = info.shown(look); - let pen = Pen { rgb: shown.len() >= 3, alpha: shown.len() >= 4, aef: look.aef, stretch: stretch.to_vec() }; + let colour = info.colour(look); + let colours = colour.len(); + let shown = info.shown_from(colour); + let pen = Pen { + rgb: colours >= 3, + // The alpha band where it is among those shown: after the colours, or one of them. + alpha: info.alpha.and_then(|a| shown.iter().rposition(|b| *b == a)), + alpha_max: rustgis_data::raster::alpha_max(info.layout.data_type), + palette: info.palette(look).map(<[[u8; 4]]>::to_vec), + aef: look.aef, + stretch: stretch.to_vec(), + }; let w2 = wanted.clone(); fetch_bands( info, @@ -1011,9 +1242,13 @@ fn draw(info: Arc, url: String, look: &Look, stretch: &[(f32, f32)], z: u8 /// How a map tile's pixels are coloured from the bands shown. struct Pen { - /// Three bands as red, green and blue (else one, grey), and a fourth as alpha. + /// Three bands as red, green and blue (else one, grey). rgb: bool, - alpha: bool, + /// Where the alpha band is among those shown, after the colour ones, and its opaque value. + alpha: Option, + alpha_max: f32, + /// A palette image's colours, by value (no stretch). + palette: Option>, aef: bool, stretch: Vec<(f32, f32)>, } @@ -1031,23 +1266,38 @@ fn paint(pen: &Pen, spots: &[Option<(u32, usize)>], wanted: &[u32], tiles: &[Pic let Some((t, at)) = spot else { continue }; let Some(bands) = wanted.binary_search(t).ok().and_then(|i| tiles.get(i)) else { continue }; let v = |b: usize| bands.get(b).and_then(|(chunk, i)| chunk.get(*i)).and_then(|band| band.get(*at)).copied().unwrap_or(f32::NAN); + // The alpha band's value, 0 to 255 (opaque without one); none where it hides the pixel. + let a = match pen.alpha { + Some(i) => { + let a = v(i); + if a.is_nan() || a <= 0.0 { + continue; + } + ((a / pen.alpha_max).clamp(0.0, 1.0) * 255.0).round() as u8 + } + None => 255, + }; let px = if pen.rgb { let (r, g, b) = (v(0), v(1), v(2)); if !(r.is_finite() && g.is_finite() && b.is_finite()) { continue; } - let a = if pen.alpha { v(3) } else { 255.0 }; - if pen.alpha && (a.is_nan() || a <= 0.0) { + [scale(0, r), scale(1, g), scale(2, b), a] + } else if let Some(table) = &pen.palette { + let g = v(0); + // A value past the table: nothing. + let Some(c) = (g.is_finite() && g >= 0.0).then(|| table.get(g as usize)).flatten() else { continue }; + if c[3] == 0 { continue; } - [scale(0, r), scale(1, g), scale(2, b), a.clamp(0.0, 255.0) as u8] + [c[0], c[1], c[2], ((u16::from(c[3]) * u16::from(a)) / 255) as u8] } else { let g = v(0); if !g.is_finite() { continue; } let g = scale(0, g); - [g, g, g, 255] + [g, g, g, a] }; if let Some(out) = rgba.get_mut(k * 4..k * 4 + 4) { out.copy_from_slice(&px); @@ -1251,6 +1501,80 @@ pub fn read_values_counted_blocking( #[cfg(test)] mod tests { + /// A palette image's own look colours its values by the table the header keeps apart from its + /// colour hint (`Layout::palette`), with no stretch read or kept; a look of its bands doesn't. + #[test] + fn a_palette_images_own_look_has_its_table() { + use rustgis_data::raster::cog::{Parse, Sparse, example, layout}; + use std::sync::{Arc, Mutex}; + let Ok(Parse::Done(mut layout)) = layout(&Sparse::default().with(0, example(64, 16))) else { panic!("not a COG") }; + layout.palette = Some(vec![[1, 2, 3, 255]; 256]); + let transformer = layout.crs.transformer().unwrap(); + let info = Arc::new(super::Info { + layout: *layout, + header: Sparse::default(), + transformer, + bounds: [0.0; 4], + merc: [0.0; 4], + min_zoom: 0, + max_zoom: 0, + stretch: Mutex::default(), + stretching: Mutex::default(), + stretch_failed: Mutex::default(), + alpha: None, + }); + let own = super::Look::default(); + assert_eq!(info.palette(&own).map(<[[u8; 4]]>::len), Some(256)); + assert!(info.palette(&super::Look { bands: vec![0], ..super::Look::default() }).is_none()); + let (tx, rx) = std::sync::mpsc::channel(); + super::with_stretch(info.clone(), "test://palette.tif".into(), own.clone(), Box::new(move |r| tx.send(r).unwrap_or_default())); + let stretch = rx.recv_timeout(std::time::Duration::from_secs(5)).expect("answered at once").unwrap(); + assert_eq!(stretch, vec![(0.0, 255.0)]); + assert!(info.stretch_for(&own).is_none(), "kept, which a look of its bands would take for its own"); + } + + /// A joint read whose tiles the fetch queue keeps dropping ends, saying so, rather than + /// asking without end: within eight drops a tile (64 at least). + #[test] + fn a_read_the_queue_keeps_dropping_ends() { + use rustgis_data::raster::cog::{Parse, Sparse, example, layout}; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; + let Ok(Parse::Done(layout)) = layout(&Sparse::default().with(0, example(64, 16))) else { panic!("not a COG") }; + let transformer = layout.crs.transformer().unwrap(); + let info = Arc::new(super::Info { + layout: *layout, + header: Sparse::default(), + transformer, + bounds: [0.0; 4], + merc: [0.0; 4], + min_zoom: 0, + max_zoom: 0, + stretch: Mutex::default(), + stretching: Mutex::default(), + stretch_failed: Mutex::default(), + alpha: None, + }); + let asked = Arc::new(AtomicUsize::new(0)); + let a = asked.clone(); + let get: super::GetTile = Arc::new( + move |_: Arc, _: String, _: usize, _: u32, done: Box) + Send>| { + a.fetch_add(1, Ordering::Relaxed); + done(Err(crate::cache::DROPPED.into())); + }, + ); + let (tx, rx) = std::sync::mpsc::channel(); + let wanted: Vec = (0..16).collect(); + super::fetch_all_with(get, info, "test://dropping.tif".into(), 0, wanted, Box::new(move |r| tx.send(r).unwrap_or_default())); + let r = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("the read ended"); + let e = r.map(|_| ()).unwrap_err(); + // Eight drops a tile, then the read fails on the next; the rest of that batch isn't counted. + assert!(e.contains("dropped its tiles 129 times"), "{e}"); + let asked = asked.load(Ordering::Relaxed); + assert!((129..=129 + 15).contains(&asked), "{asked}"); + assert_eq!((super::drops_allowed(1), super::drops_allowed(16), super::drops_allowed(4096)), (64, 128, 32768)); + } + #[test] fn announced_header_lengths_and_types() { use super::{HEAD, header_length, named_tiff, not_tiff_type}; @@ -1282,7 +1606,28 @@ mod tests { let rustgis_data::raster::cog::Parse::Done(l) = cog::layout(&Sparse::default().with(0, file)).unwrap() else { panic!() }; // Overviews down to 64 pixels: the 256-pixel one is read. assert_eq!(l.levels.last().map(|l| l.width), Some(64)); - assert_eq!(l.levels[stats_level(&l)].width, 256); + assert_eq!(l.levels[stats_level(&l.levels)].width, 256); + // A long strip (a WorldView image 44 626 × 353 888 pixels, in tiles of 256): its overview + // 349 pixels wide is 22 tiles, too many; the one 175 wide, 6 tiles, still a quarter of a + // million pixels, is read. + let level = |width: u32, height: u32| Level { + ifd: 0, + width, + height, + tile_width: 256, + tile_height: 256, + offsets: Vec::new(), + counts: Vec::new(), + }; + let strip: Vec = [(44_626, 353_888), (5579, 44_236), (698, 5530), (349, 2765), (175, 1383), (88, 692)] + .into_iter() + .map(|(w, h)| level(w, h)) + .collect(); + assert_eq!(strip[stats_level(&strip)].width, 175); + // Only small ones: the finest of them; none: the coarsest. + assert_eq!(stats_level(&[level(4096, 4096), level(200, 200), level(100, 100)]), 1); + assert_eq!(stats_level(&[level(8192, 8192), level(4096, 4096)]), 1); + assert_eq!(stats_level(&[]), 0); } /// Reads `file`'s layout as [`load`] and [`step`] do, a range at a time: the layout, how many @@ -1392,7 +1737,7 @@ mod tests { #[test] fn looks_in_addresses() { let u = "https://x.org/a.tiff"; - let look = Look { bands: vec![1, 16, 9], range: Some((-0.3, 0.3)), aef: true }; + let look = Look { bands: vec![1, 16, 9], range: Some((-0.3, 0.3)), aef: true, cube: false }; let shown = with_look(u, &look); assert_eq!(shown, "https://x.org/a.tiff#bands=2,17,10&range=-0.3,0.3&dequantize=aef"); assert_eq!(split(&shown), (u, look)); @@ -1418,7 +1763,7 @@ mod tests { red[1] = f32::NAN; let tile: Picked = vec![(Arc::new(vec![red].into()), 0), (chunk(20.0), 0), (chunk(30.0), 0)]; let spots: Vec> = (0..(SIZE * SIZE) as usize).map(|k| if k < 2 { Some((7, k)) } else { None }).collect(); - let pen = Pen { rgb: true, alpha: false, aef: false, stretch: vec![(0.0, 40.0); 3] }; + let pen = Pen { rgb: true, alpha: None, alpha_max: 255.0, palette: None, aef: false, stretch: vec![(0.0, 40.0); 3] }; let out = paint(&pen, &spots, &[7], &[tile]).unwrap(); let px = &out[crate::cache::RAW.len() + 8..]; assert_eq!(&px[..4], &[64, 128, 191, 255]); @@ -1445,4 +1790,72 @@ mod tests { // Nothing there: no picture. assert!(paint(&pen, &vec![None; spots.len()], &[], &[]).unwrap().is_empty()); } + + #[test] + fn paints_grey_with_alpha_and_palettes() { + let chunk = |v: Vec| -> Bands { Arc::new(vec![v].into()) }; + let spots: Vec> = (0..(SIZE * SIZE) as usize).map(|k| (k < 3).then_some((7, k))).collect(); + let px = |out: Vec| out[crate::cache::RAW.len() + 8..crate::cache::RAW.len() + 8 + 12].to_vec(); + // A 16-bit pan band and its 16-bit alpha band: opaque, half, hidden. + let tile: Picked = vec![(chunk(vec![500.0, 1000.0, 1000.0, 0.0]), 0), (chunk(vec![65535.0, 32768.0, 0.0, 0.0]), 0)]; + let pen = Pen { rgb: false, alpha: Some(1), alpha_max: 65535.0, palette: None, aef: false, stretch: vec![(0.0, 1000.0)] }; + assert_eq!(px(paint(&pen, &spots, &[7], &[tile]).unwrap()), [128, 128, 128, 255, 255, 255, 255, 128, 0, 0, 0, 0]); + // A palette image: each value its colour, a transparent one or one past the table nothing. + let tile: Picked = vec![(chunk(vec![1.0, 2.0, 9.0, 0.0]), 0)]; + let table = vec![[0, 0, 0, 255], [200, 10, 20, 255], [0, 0, 0, 0]]; + let pen = Pen { rgb: false, alpha: None, alpha_max: 255.0, palette: Some(table), aef: false, stretch: vec![(0.0, 255.0)] }; + let out = px(paint(&pen, &spots, &[7], &[tile]).unwrap()); + assert_eq!(&out[..4], &[200, 10, 20, 255]); + assert_eq!(&out[4..8], &[0, 0, 0, 0], "a transparent colour"); + assert_eq!(&out[8..12], &[0, 0, 0, 0], "a value past the table"); + } + + #[test] + fn the_tiles_used_longest_ago_make_room() { + let mut t = Tiles::default(); + let key = |i: u32| ("https://x.org/a.tif".to_string(), 0usize, i); + let bands: Bands = Arc::new(cog::Tile::default()); + let half = BUDGET / 2; + t.keep(key(1), bands.clone(), half); + t.keep(key(2), bands.clone(), half); + // Tile 1 is used again (the view still shows it): tile 2 makes room for tile 3. + assert!(t.used(&key(1)).is_some()); + t.keep(key(3), bands.clone(), half); + assert!(t.used(&key(1)).is_some() && t.used(&key(3)).is_some()); + assert!(t.used(&key(2)).is_none()); + assert_eq!(t.samples, 2 * half); + // Kept again under its key: counted once, nothing else making room for it. + t.keep(key(3), bands.clone(), half); + assert!(t.used(&key(1)).is_some() && t.used(&key(3)).is_some()); + assert_eq!((t.samples, t.order.len()), (2 * half, 2)); + forget_file_in(&mut t, "https://x.org/a.tif"); + assert_eq!((t.samples, t.order.len(), t.map.len()), (0, 0, 0)); + } + + /// A file's header refused with a `Retry-After` (429 or 503) isn't asked for again before + /// then (the failure stands, no read starts), and the map is asked to draw again by then, on + /// each draw while it waits (as a tile's is), so a layer whose header failed doesn't stay + /// failed until something else draws the map. (That the file is read again then: the + /// engine's `a_header_refused_with_retry_after_is_read_again_when_it_says`.) + #[test] + fn a_header_refused_with_retry_after_has_the_map_drawn_again_then() { + // Its turn at the cache (another test's clearing would forget the failure) and the alarms. + let _turn = crate::cache::turn(); + let _alarm = crate::cache::alarm_turn(); + let file = "test://cog/busy.tif"; + let refused = || super::State::Failed("http status: 503 (retry after 2 s)".into(), Instant::now()); + super::infos().lock().unwrap().insert(file.to_string(), refused()); + crate::cache::clear_alarms(); + let within = || crate::cache::next_alarm().is_some_and(|d| d <= Duration::from_secs(2)); + let waiting = || matches!(super::infos().lock().unwrap().get(file), Some(super::State::Failed(..))); + let Err(e) = super::info(file) else { panic!("read while waiting") }; + assert!(e.contains("retry after 2 s"), "{e}"); + assert!(waiting(), "asked for again before its wait was over"); + assert!(within(), "no alarm on a draw while waiting: {:?}", crate::cache::next_alarm()); + // Again on the next draw: the alarm fired (or was consumed) before the wait was over. + crate::cache::clear_alarms(); + assert!(super::info(file).is_err() && waiting()); + assert!(within(), "not armed again: {:?}", crate::cache::next_alarm()); + super::infos().lock().unwrap().remove(file); + } } diff --git a/crates/tiles/src/cog/raster.rs b/crates/tiles/src/cog/raster.rs new file mode 100644 index 000000000..0555506c6 --- /dev/null +++ b/crates/tiles/src/cog/raster.rs @@ -0,0 +1,633 @@ +//! A cloud-optimised GeoTIFF as a raster layer's pixels, read in place: the same raster a file +//! gives, its symbology, statistics and tools alike, without downloading it. +//! +//! A layer holds an [`overview`]: every band of the finest overview that fits a budget, which its +//! statistics, histograms and tools read, as a file too large to hold whole is opened from an +//! overview. Drawn zoomed in past it, the part in view is read at the level the view needs as a +//! [`window`], a level's tiles at a time, from the tiles kept or fetched by range; the map draws +//! again as they land. + +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, OnceLock}; + +use rustgis_data::raster::{ColorHint, Raster}; +use web_time::Instant; + +use super::{AT_ONCE, BUDGET, Bands, Info, RETRY, TileState, fetch_all, get_tile, split, tiles, with_info}; + +/// The most samples (every band) of the overview a layer holds. +const OVERVIEW_SAMPLES: usize = if cfg!(target_arch = "wasm32") { 1 << 20 } else { 2 << 20 }; +/// The most chunks read for it. +const OVERVIEW_CHUNKS: usize = 512; +/// The most chunks of the coarsest level, held when no level fits [`OVERVIEW_SAMPLES`] (a file +/// with few or no overviews); its samples are bounded by half the tiles kept. +const FALLBACK_CHUNKS: usize = 4096; +/// The most samples of one band of a view's window (a large screen's worth, and more). +pub const WINDOW_SAMPLES: usize = 16 << 20; + +/// The level a layer holds as its overview: the finest whose every band fits [`OVERVIEW_SAMPLES`] +/// in at most [`OVERVIEW_CHUNKS`] chunks, else the coarsest when it fits half the tiles kept in at +/// most [`FALLBACK_CHUNKS`]; `None` when not even that does (a large image without overviews). +pub fn overview_level(info: &Info) -> Option { + let layout = &info.layout; + let bands = layout.bands.max(1); + let planes = if layout.planar { bands } else { 1 }; + // Samples decoded, whole tiles (an edge tile's padding too), and chunks read. + let size = |l: &rustgis_data::raster::cog::Level| { + let tile = (l.tile_width as usize).saturating_mul(l.tile_height as usize); + let samples = (l.tiles() as usize).saturating_mul(tile).saturating_mul(bands); + (samples, (l.tiles() as usize).saturating_mul(planes)) + }; + layout + .levels + .iter() + .position(|l| { + let (samples, chunks) = size(l); + samples <= OVERVIEW_SAMPLES && chunks <= OVERVIEW_CHUNKS + }) + .or_else(|| { + let last = layout.levels.len().checked_sub(1)?; + let (samples, chunks) = size(layout.levels.get(last)?); + (samples <= BUDGET / 2 && chunks <= FALLBACK_CHUNKS).then_some(last) + }) +} + +/// Reads a file's overview (see [`overview_level`]) as a raster, every band, then calls `done`. +/// The raster says it is an overview of the full image when it is one. +pub fn overview(url: &str, done: Box) + Send>) { + let file = split(url).0.to_string(); + let f = file.clone(); + with_info( + &file, + Box::new(move |r| match r { + Ok(info) => read_overview(info, f, done), + Err(e) => done(Err(e)), + }), + ); +} + +/// [`overview`], waited for (natively; at most two minutes). +#[cfg(not(target_arch = "wasm32"))] +pub fn overview_blocking(url: &str) -> Result { + let (tx, rx) = std::sync::mpsc::channel(); + overview(url, Box::new(move |r| tx.send(r).unwrap_or_default())); + rx.recv_timeout(rustgis_data::raster::IN_PLACE_WAIT).map_err(|_| format!("{}: no answer in two minutes", split(url).0))? +} + +fn read_overview(info: Arc, url: String, done: Box) + Send>) { + let Some(coarsest) = info.layout.levels.last() else { + done(Err(format!("{url}: the GeoTIFF has no image"))); + return; + }; + let Some((level, l)) = overview_level(&info).and_then(|i| Some((i, info.layout.levels.get(i)?))) else { + done(Err(format!( + "{url}: its smallest image, {} × {} with {} band(s), is too large to hold as the layer's overview: \ + a GeoTIFF with overviews (as GDAL's COG driver or gdaladdo make) is read in place", + coarsest.width, coarsest.height, info.layout.bands + ))); + return; + }; + let bands: Vec = (0..info.layout.bands.max(1)).collect(); + let planes = if info.layout.planar { bands.len() } else { 1 }; + let Some(n) = l.tiles().checked_mul(u32::try_from(planes).unwrap_or(u32::MAX)) else { + done(Err(format!("{url}: too many tiles"))); + return; + }; + let (w, h) = (l.width, l.height); + let i = info.clone(); + fetch_all( + info, + url.clone(), + level, + (0..n).collect(), + Box::new(move |r| { + done(r.and_then(|got| { + let (data, _) = assemble(&i, level, (0, w), (0, h), &bands, |c| got.get(c as usize).cloned()); + let l = i.layout.levels.get(level).ok_or("no such overview")?; + let mut r = Raster::new( + file_name(&url), + w, + h, + data, + i.layout.transform_of(l), + i.layout.crs, + i.layout.nodata, + i.layout.color.clone(), + i.layout.data_type, + ) + .map_err(|e| format!("{url}: {e}"))?; + if let Some(full) = i.layout.levels.first().filter(|_| level > 0) { + r.overview = Some((full.width, full.height)); + } + // What the header holds for a raster: its colour table (drawn grey in place; the + // layer's symbology colours it), the bands' and classes' names, and what was lost. + if let Some(table) = &i.layout.palette { + r.color = ColorHint::Palette(table.clone()); + } + if let Some(described) = &i.layout.described { + r.apply_pam(described, rustgis_data::raster::pam::GDAL_METADATA); + } + r.lost = i.layout.lost.clone(); + r.source = Some(url.clone()); + Ok(r) + })) + }), + ); +} + +/// A file's name, from its address (without its query). +fn file_name(url: &str) -> String { + let path = url.split(['?', '#']).next().unwrap_or(url).trim_end_matches('/'); + path.rsplit('/').next().filter(|n| !n.is_empty()).unwrap_or("raster").to_string() +} + +/// The chunks a window of a level needs, of the bands given (each band of a tile its own chunk +/// when they are stored apart, else one chunk a tile). +fn chunks_of(info: &Info, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize]) -> HashSet { + let mut chunks = HashSet::new(); + let Some(l) = info.layout.levels.get(level) else { return chunks }; + let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); + for ty in rows.0 / th..rows.1.min(l.height).div_ceil(th) { + for tx in cols.0 / tw..cols.1.min(l.width).div_ceil(tw) { + // A level too large for `u32` to count its tiles (a hostile header) has none past it. + let Some(tile) = ty.checked_mul(l.across()).and_then(|t| t.checked_add(tx)) else { continue }; + for &b in bands { + if let Some((c, _)) = info.layout.chunk(l, tile, b) { + chunks.insert(c); + } + } + } + } + chunks +} + +/// Whether a window of a level (see [`window`]) fits the tiles kept: at most half of them, so the +/// tiles of the level before stay while its own arrive. One that doesn't is read from a coarser +/// level. +pub fn fits(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize]) -> bool { + let Ok(Some(info)) = super::info(split(url).0) else { return true }; + let Some(l) = info.layout.levels.get(level) else { return false }; + let per_chunk = (l.tile_width as usize) * (l.tile_height as usize) * info.layout.chunk_bands(); + let n = chunks_of(&info, level, cols, rows, bands).len(); + ((cols.1.saturating_sub(cols.0)) as usize).saturating_mul(rows.1.saturating_sub(rows.0) as usize) <= WINDOW_SAMPLES + && n.saturating_mul(per_chunk) <= BUDGET / 2 +} + +/// The part of a level in view, as [`window`] has it now. +#[derive(Debug, Default)] +pub struct Window { + /// Its pixels (NaN where a tile hasn't arrived); `None` before the file's header has. + pub raster: Option, + /// Tiles (or the header) are still on their way: the map draws again as they land. + pub pending: bool, + /// Tiles in it couldn't be read lately (other levels stand in for them, and they are asked + /// for again when their wait is over). + pub failed: bool, +} + +/// Tiles whose read failed lately, and until when they aren't asked for again ([`RETRY`] after, or +/// as long as their server asked to be left), so a map drawn again and again doesn't ask for them +/// each time. The map is drawn again when that time comes. +pub(super) fn failed() -> &'static Mutex> { + static F: OnceLock>> = OnceLock::new(); + F.get_or_init(Mutex::default) +} + +/// The most failed tiles remembered. +const MAX_FAILED: usize = 4096; + +/// What a tile's arrival leaves: the map draws again. One that failed is remembered until its +/// wait is over (a minute, or what its server's `Retry-After` said), the map asked to draw again +/// then; one the fetch queue dropped (the view moved on, see [`crate::cache::DROPPED`]) is no +/// failure: not remembered, it is asked for again when it is drawn next, and the tiles that +/// pushed it out draw the map as they land. +fn tile_landed(key: (String, usize, u32), r: &Result) { + match r { + Err(e) if e == crate::cache::DROPPED => return, + Err(e) => { + let until = Instant::now() + crate::cache::retry_wait(e, RETRY); + remember_failed(&mut failed().lock().unwrap_or_else(|e| e.into_inner()), key, until); + crate::cache::notify_at(until); + } + Ok(_) => {} + } + crate::cache::notify(); +} + +/// How many tiles of level `level` of a file have been asked for and not forgotten (kept, or on +/// their way): for tests of how a window asks for them. +#[doc(hidden)] +pub fn asked(file: &str, level: usize) -> usize { + let t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + t.map.iter().filter(|((u, l, _), s)| u == file && *l == level && matches!(s, TileState::Ready(..) | TileState::Loading(_))).count() +} + +/// Forgets the failed tiles of `file` (`None`: of every file), so they are asked for at once: a +/// login or a connection changed (see [`crate::cache::forget`] and [`crate::cache::clear`]). +pub(super) fn forget_failed(file: Option<&str>) { + forget_failed_in(&mut failed().lock().unwrap_or_else(|e| e.into_inner()), file); +} + +fn forget_failed_in(f: &mut HashMap<(String, usize, u32), Instant>, file: Option<&str>) { + match file { + None => f.clear(), + Some(file) => f.retain(|(u, _, _), _| u != file), + } +} + +/// Remembers a failed tile until `until`. With [`MAX_FAILED`] already, those whose wait is over go +/// first; failing that, the one whose wait ends soonest (asked for again a little early, once) +/// makes room, rather than the new one, which would be asked for again at each draw, whatever its +/// server asked for. +fn remember_failed(f: &mut HashMap<(String, usize, u32), Instant>, key: (String, usize, u32), until: Instant) { + if f.len() >= MAX_FAILED && !f.contains_key(&key) { + let now = Instant::now(); + f.retain(|_, at| now < *at); + if f.len() >= MAX_FAILED + && let Some(soonest) = f.iter().min_by_key(|(_, at)| **at).map(|(k, _)| k.clone()) + { + f.remove(&soonest); + } + } + f.insert(key, until); +} + +/// The columns `cols` and rows `rows` (half-open, in the level's pixels) of level `level` of a +/// file, bands `bands` (0-based, in that order, as the raster's bands), coloured as `color` says: +/// from the tiles kept, without waiting. Those missing are fetched (each once at a time, and +/// [`AT_ONCE`] of them at a time: the rest as the map draws again); the tile cache's generation +/// goes up as each lands, so the map is drawn again. +pub fn window(url: &str, level: usize, cols: (u32, u32), rows: (u32, u32), bands: &[usize], color: ColorHint) -> Result { + let file = split(url).0; + let Some(info) = super::info(file)? else { return Ok(Window { raster: None, pending: true, failed: false }) }; + let l = info.layout.levels.get(level).ok_or("no such overview")?; + let (c0, c1) = (cols.0.min(l.width), cols.1.min(l.width)); + let (r0, r1) = (rows.0.min(l.height), rows.1.min(l.height)); + if c0 >= c1 || r0 >= r1 || bands.is_empty() { + return Ok(Window::default()); + } + if ((c1 - c0) as usize).saturating_mul((r1 - r0) as usize) > WINDOW_SAMPLES { + return Err("the view is too large a part of the GeoTIFF to read at once".into()); + } + if let Some(b) = bands.iter().find(|b| **b >= info.layout.bands.max(1)) { + return Err(format!("the GeoTIFF has {} bands, not {}", info.layout.bands, b.saturating_add(1))); + } + // The chunks the window needs: those kept (marked as used, so they stay), and the others + // asked for. + let chunks = chunks_of(&info, level, (c0, c1), (r0, r1), bands); + let mut kept: HashMap = HashMap::new(); + let mut ask = Vec::new(); + { + let mut t = tiles().lock().unwrap_or_else(|e| e.into_inner()); + for &c in &chunks { + let key = (file.to_string(), level, c); + match t.used(&key) { + Some(b) => { + kept.insert(c, b); + } + None if t.map.contains_key(&key) => {} + None => ask.push(c), + } + } + } + let waiting: bool; + { + let mut f = failed().lock().unwrap_or_else(|e| e.into_inner()); + let now = Instant::now(); + f.retain(|_, until| now < *until); + // Those failed lately wait; the map draws again when the first may be asked for. + let mut next: Option = None; + ask.retain(|c| match f.get(&(file.to_string(), level, *c)) { + Some(until) => { + next = Some(next.map_or(*until, |n| n.min(*until))); + false + } + None => true, + }); + if let Some(at) = next { + crate::cache::notify_at(at); + } + waiting = next.is_some(); + } + // Asked for `AT_ONCE` at a time (half the fetch queue, as a joint read asks): a view of small + // tiles needing more would push its own first requests out of the queue, to ask for them all + // again at the next draw. The rest are asked for as the map draws again (a tile landing draws + // it), and are on their way till then. + for c in ask.into_iter().take(AT_ONCE) { + let key = (file.to_string(), level, c); + get_tile(info.clone(), file.to_string(), level, c, Box::new(move |r| tile_landed(key, &r))); + // A tile the file leaves out (a sparse file's) is kept at once: not waited for. + if let Some(b) = tiles().lock().unwrap_or_else(|e| e.into_inner()).used(&(file.to_string(), level, c)) { + kept.insert(c, b); + } + } + // The tiles still missing are on their way (asked for, or to be asked for at the next draw; + // the map draws again as they land) or failed lately (not waited for: the map would wait for + // them for nothing; asked for again when their wait is over). + let missing: Vec = chunks.iter().copied().filter(|c| !kept.contains_key(c)).collect(); + let (pending, waiting) = { + let f = failed().lock().unwrap_or_else(|e| e.into_inner()); + let failed_now = |c: &u32| f.contains_key(&(file.to_string(), level, *c)); + (missing.iter().any(|c| !failed_now(c)), waiting || missing.iter().any(failed_now)) + }; + let (mut data, gaps) = assemble(&info, level, (c0, c1), (r0, r1), bands, |c| kept.get(&c).cloned()); + // Until they arrive, its missing tiles show the levels kept (the one zoomed from), not holes. + if !gaps.is_empty() { + // The tiles kept, looked up one at a time: the cache isn't held while the gaps fill. + let kept = |o: usize, c: u32| tiles().lock().unwrap_or_else(|e| e.into_inner()).used(&(file.to_string(), o, c)); + stand_in(&info, level, (c0, c1), (r0, r1), bands, &mut data, &gaps, kept); + } + let t = info.layout.transform_of(l); + let transform = rustgis_data::raster::GeoTransform { + origin_x: t.origin_x + f64::from(c0) * t.pixel_width, + origin_y: t.origin_y + f64::from(r0) * t.pixel_height, + ..t + }; + let raster = Raster::unmeasured( + file_name(file), + c1 - c0, + r1 - r0, + data, + transform, + info.layout.crs, + info.layout.nodata, + color, + info.layout.data_type, + ) + .map_err(|e| e.to_string())?; + Ok(Window { raster: Some(raster), pending, failed: waiting }) +} + +/// A part of a window a chunk not kept leaves empty: the band (its place among those read), and +/// its columns and rows (half-open, in the level's pixels). +struct Gap { + k: usize, + cols: (u32, u32), + rows: (u32, u32), +} + +/// Fills the `gaps` of a window of `level` from the other levels' tiles kept, the nearest first +/// (the finer, then the coarser, then further): the level zoomed from stands in for the one zoomed +/// to until its tiles arrive. Only the gaps are filled: a tile's own no-data pixels stay empty. +/// `kept` gives a level's chunk when it is kept (marking it used); nothing is fetched. +#[allow(clippy::too_many_arguments)] +fn stand_in( + info: &Info, + level: usize, + cols: (u32, u32), + rows: (u32, u32), + bands: &[usize], + data: &mut [Vec], + gaps: &[Gap], + mut kept: impl FnMut(usize, u32) -> Option, +) { + let levels = &info.layout.levels; + let Some(l) = levels.get(level) else { return }; + let ww = (cols.1 - cols.0) as usize; + // The gaps a level kept hasn't filled yet: walked a row at a time, nothing kept per pixel. + let mut open: Vec<&Gap> = gaps.iter().collect(); + // The other levels, nearest first. + let others: Vec = + (1..levels.len()).flat_map(|d| [level.checked_sub(d), Some(level + d)]).flatten().filter(|o| *o < levels.len()).collect(); + for o in others { + let Some(ol) = levels.get(o) else { continue }; + let (sx, sy) = (f64::from(ol.width) / f64::from(l.width.max(1)), f64::from(ol.height) / f64::from(l.height.max(1))); + let (tw, th) = (ol.tile_width.max(1), ol.tile_height.max(1)); + // The chunks of that level found so far (none: not kept). + let mut found: HashMap> = HashMap::new(); + open.retain(|g| { + let (Some(&b), Some(dst)) = (bands.get(g.k), data.get_mut(g.k)) else { return false }; + // Whether a pixel still empty found that level's tile not kept. + let mut short = false; + // The chunk the last pixel came from: its neighbours mostly share it. + let mut last: Option<(u32, Option)> = None; + for y in g.rows.0..g.rows.1 { + let oy = (((f64::from(y) + 0.5) * sy) as u32).min(ol.height.saturating_sub(1)); + let row = (y - rows.0) as usize * ww; + for x in g.cols.0..g.cols.1 { + let Some(v) = dst.get_mut(row + (x - cols.0) as usize).filter(|v| v.is_nan()) else { continue }; + // The pixel of that level under this one's centre. + let ox = (((f64::from(x) + 0.5) * sx) as u32).min(ol.width.saturating_sub(1)); + let tile = (oy / th) * ol.across() + ox / tw; + let Some((c, at)) = info.layout.chunk(ol, tile, b) else { + short = true; + continue; + }; + if last.as_ref().is_none_or(|(lc, _)| *lc != c) { + let chunk = found.entry(c).or_insert_with(|| kept(o, c)).clone(); + last = Some((c, chunk)); + } + let value = last + .as_ref() + .and_then(|(_, ch)| ch.as_ref()) + .and_then(|ch| ch.get(at)) + .and_then(|p| p.get(((oy % th) * tw + ox % tw) as usize)); + match value { + // That level's value, no data included: it is what the image has there. + Some(s) => *v = *s, + None => short = true, + } + } + } + short + }); + if open.is_empty() { + return; + } + } +} + +/// The pixels of columns `cols` and rows `rows` (half-open) of a level, bands `bands` in that +/// order, each row by row, from the chunks `chunk` gives (NaN where it gives none), and the gaps +/// the chunks it doesn't give leave. +fn assemble( + info: &Info, + level: usize, + cols: (u32, u32), + rows: (u32, u32), + bands: &[usize], + chunk: impl Fn(u32) -> Option, +) -> (Vec>, Vec) { + let (ww, wh) = ((cols.1 - cols.0) as usize, (rows.1 - rows.0) as usize); + let mut out = vec![vec![f32::NAN; ww * wh]; bands.len()]; + let mut gaps = Vec::new(); + let Some(l) = info.layout.levels.get(level) else { return (out, gaps) }; + let (tw, th) = (l.tile_width.max(1), l.tile_height.max(1)); + for ty in rows.0 / th..rows.1.div_ceil(th) { + for tx in cols.0 / tw..cols.1.div_ceil(tw) { + // A level too large for `u32` to count its tiles (a hostile header) has none past it: + // left empty, as a tile the header has no chunk for is. + let Some(tile) = ty.checked_mul(l.across()).and_then(|t| t.checked_add(tx)) else { continue }; + // The tile's part of the window, in the level's pixels. + // The tile's end saturates: a width near `u32::MAX` (the header bounds the tiles, + // not the width) stays a tile, not a wrapped, empty one. + let (x0, x1) = ((tx * tw).max(cols.0), tx.saturating_add(1).saturating_mul(tw).min(cols.1)); + let (y0, y1) = ((ty * th).max(rows.0), ty.saturating_add(1).saturating_mul(th).min(rows.1)); + for (k, &b) in bands.iter().enumerate() { + let Some((c, at)) = info.layout.chunk(l, tile, b) else { continue }; + let Some(data) = chunk(c) else { + gaps.push(Gap { k, cols: (x0, x1), rows: (y0, y1) }); + continue; + }; + let (Some(plane), Some(dst)) = (data.get(at), out.get_mut(k)) else { continue }; + for y in y0..y1 { + let src = ((y - ty * th) * tw + (x0 - tx * tw)) as usize; + let to = (y - rows.0) as usize * ww + (x0 - cols.0) as usize; + let n = (x1 - x0) as usize; + if let (Some(s), Some(d)) = (plane.get(src..src + n), dst.get_mut(to..to + n)) { + d.copy_from_slice(s); + } + } + } + } + } + (out, gaps) +} + +#[cfg(test)] +mod tests { + use super::*; + + use super::super::Tiles; + use rustgis_data::raster::cog::{Example, Level, Parse, Sparse, example_with, layout}; + + /// A file's info, as reading its header gives it. + fn info_of(file: Vec) -> Info { + let Ok(Parse::Done(layout)) = layout(&Sparse::default().with(0, file)) else { panic!("not a COG") }; + let transformer = layout.crs.transformer().unwrap(); + Info { + layout: *layout, + header: Sparse::default(), + transformer, + bounds: [0.0; 4], + merc: [0.0; 4], + min_zoom: 0, + max_zoom: 0, + stretch: Mutex::default(), + stretching: Mutex::default(), + stretch_failed: Mutex::default(), + alpha: None, + } + } + + /// The overview held is the finest small one; without one, the coarsest if it fits half the + /// tiles kept; else none, rather than reading a large image whole. + #[test] + fn the_overview_held_is_bounded() { + let mut info = info_of(example_with(&Example { size: 4096, tile: 256, ..Example::default() })); + // 4096, 2048, 1024 … a side: 1024 a side is a million values. + assert_eq!(overview_level(&info).and_then(|l| info.layout.levels.get(l)).map(|l| l.width), Some(1024)); + let first = info.layout.levels[0].clone(); + let level = |side: u32| Level { width: side, height: side, ..first.clone() }; + // Only the full image, 4096 a side (16 million values in 256 tiles): held. + info.layout.levels = vec![level(4096)]; + assert_eq!(overview_level(&info), Some(0)); + // One 20 000 a side (400 million values in 6241 tiles): none. + info.layout.levels = vec![level(20_000)]; + assert_eq!(overview_level(&info), None); + } + + /// A project file may ask a window for any band number (its layer's renderer): the one past + /// the last `usize` is refused by number, not overflowed. + #[test] + fn a_window_of_the_band_past_the_last_usize_is_refused() { + let file = "test://past-last-band.tif"; + let info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); + super::super::infos().lock().unwrap().insert(file.to_string(), super::super::State::Ready(Arc::new(info))); + let e = window(file, 0, (0, 16), (0, 16), &[usize::MAX], ColorHint::Gray).unwrap_err(); + assert_eq!(e, format!("the GeoTIFF has 1 bands, not {}", usize::MAX)); + } + + /// A window's missing tiles are filled from the level kept, but not its own no-data pixels. + #[test] + fn stand_ins_fill_only_the_tiles_missing() { + let info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); + let file = "test://stand-in.tif"; + // Level 1 (32 a side) kept: its tile 0 is level 0's pixels 0 to 32, all 7. + let mut t = Tiles::default(); + let _ = t.keep((file.to_string(), 1, 0), Arc::new(vec![vec![7.0; 16 * 16]].into()), 256); + // Level 0's tile 0 here, one pixel of it no data; tile 1, right of it, not yet. + let mut own = vec![3.0; 16 * 16]; + own[16 + 1] = f32::NAN; + let own: Bands = Arc::new(vec![own].into()); + let (cols, rows) = ((0, 32), (0, 16)); + let (mut data, gaps) = assemble(&info, 0, cols, rows, &[0], |c| (c == 0).then(|| own.clone())); + assert_eq!(gaps.len(), 1); + stand_in(&info, 0, cols, rows, &[0], &mut data, &gaps, |o, c| t.used(&(file.to_string(), o, c))); + let at = |x: usize, y: usize| data[0][y * 32 + x]; + assert_eq!((at(0, 0), at(20, 5), at(31, 15)), (3.0, 7.0, 7.0), "tile 1 from level 1"); + assert!(at(1, 1).is_nan(), "tile 0's no-data pixel stays empty"); + } + + /// A full memory of failed tiles makes room for a new one: the one whose wait ends soonest + /// goes, so every tile failing waits its turn. + #[test] + fn failed_tiles_make_room_for_the_newest() { + let mut f = HashMap::new(); + let now = Instant::now(); + let key = |i: u32| ("https://x.org/a.tif".to_string(), 0usize, i); + for i in 0..MAX_FAILED as u32 { + remember_failed(&mut f, key(i), now + std::time::Duration::from_secs(60 + u64::from(i))); + } + remember_failed(&mut f, key(u32::MAX), now + std::time::Duration::from_secs(30)); + assert_eq!(f.len(), MAX_FAILED); + assert!(f.contains_key(&key(u32::MAX)) && !f.contains_key(&key(0)), "the soonest gone, the newest kept"); + // One already remembered is waited for again, nothing else goes. + remember_failed(&mut f, key(5), now + std::time::Duration::from_secs(90)); + assert!(f.contains_key(&key(1)) && f.len() == MAX_FAILED); + } + + /// A tile the fetch queue dropped (the view moved on) is no failure: not remembered, so it is + /// asked for again when it is drawn next; one its server refused is, until its wait is over. + #[test] + fn a_dropped_tile_isnt_remembered_as_failed() { + let key = |c: u32| ("test://dropped.tif".to_string(), 0usize, c); + tile_landed(key(1), &Err(crate::cache::DROPPED.to_string())); + assert!(!failed().lock().unwrap().contains_key(&key(1))); + tile_landed(key(2), &Err("HTTP 503".to_string())); + assert!(failed().lock().unwrap().contains_key(&key(2))); + tile_landed(key(3), &Ok(Arc::new(vec![vec![0.0f32]].into()))); + assert!(!failed().lock().unwrap().contains_key(&key(3))); + } + + /// Failed tiles are forgotten with their file (a login or a connection changed: asked for at + /// once), and all of them when the cache is cleared. + #[test] + fn failed_tiles_are_forgotten_with_their_file() { + let until = Instant::now() + std::time::Duration::from_secs(60); + let a = ("test://forgotten-a.tif".to_string(), 0usize, 1u32); + let b = ("test://forgotten-b.tif".to_string(), 0usize, 1u32); + let mut f = HashMap::new(); + remember_failed(&mut f, a.clone(), until); + remember_failed(&mut f, b.clone(), until); + forget_failed_in(&mut f, Some("test://forgotten-a.tif")); + assert!(!f.contains_key(&a) && f.contains_key(&b)); + forget_failed_in(&mut f, None); + assert!(f.is_empty()); + } + + /// A level as wide as `u32` allows (a hostile header): its last tile's end doesn't wrap, so + /// the window is assembled, its missing tiles gaps. As tall too, its far tiles are more than + /// `u32` counts: a window there is assembled all the same (empty), and asks for no chunk. + #[test] + fn a_window_at_the_far_edge_of_a_huge_level_doesnt_overflow() { + let mut info = info_of(example_with(&Example { size: 64, tile: 16, ..Example::default() })); + info.layout.levels[0].width = u32::MAX; + let (data, gaps) = assemble(&info, 0, (u32::MAX - 15, u32::MAX), (0, 16), &[0], |_| None); + assert_eq!((data.len(), data[0].len()), (1, 16 * 15)); + assert!(gaps.iter().all(|g| g.cols.0 <= g.cols.1 && g.rows.0 <= g.rows.1)); + info.layout.levels[0].height = u32::MAX; + let far = ((u32::MAX - 15, u32::MAX), (u32::MAX - 15, u32::MAX)); + let (data, gaps) = assemble(&info, 0, far.0, far.1, &[0], |_| None); + assert_eq!((data.len(), data[0].len()), (1, 15 * 15)); + assert!(data[0].iter().all(|v| v.is_nan()) && gaps.is_empty()); + assert!(chunks_of(&info, 0, far.0, far.1, &[0]).is_empty()); + } + + #[test] + fn names_from_addresses() { + assert_eq!(file_name("https://x.org/a/scene.tif?sig=1"), "scene.tif"); + assert_eq!(file_name("https://api.maxar.com/analytics/v1/raster/ortho/geotiff?function=f"), "geotiff"); + assert_eq!(file_name("https://x.org/"), "x.org"); + } +} diff --git a/crates/tiles/src/embeddings.rs b/crates/tiles/src/embeddings.rs index dda9b6ee9..98611f969 100644 --- a/crates/tiles/src/embeddings.rs +++ b/crates/tiles/src/embeddings.rs @@ -238,7 +238,7 @@ pub const MAX_TILES: usize = 60; /// stretched over [−stretch, stretch]. pub fn aef_look(bands: [usize; 3], stretch: f32) -> Look { let s = if stretch.is_finite() && stretch > 0.0 { stretch.min(1.0) } else { AEF_DEFAULT_STRETCH }; - Look { bands: bands.iter().map(|b| (*b).min(63)).collect(), range: Some((-s, s)), aef: true } + Look { bands: bands.iter().map(|b| (*b).min(63)).collect(), range: Some((-s, s)), aef: true, cube: false } } /// A file's web address from the index's `s3://` one. diff --git a/crates/tiles/src/failure.rs b/crates/tiles/src/failure.rs index 33d50b0f5..0f5c94eef 100644 --- a/crates/tiles/src/failure.rs +++ b/crates/tiles/src/failure.rs @@ -88,10 +88,11 @@ impl Failure { } } let html = kept.iter().any(|(k, v)| k == "content-type" && v.to_ascii_lowercase().contains("html")); + let wait = kept.iter().find(|(k, _)| k == "retry-after").map(|(_, v)| v.as_str()); Failure { url: crate::logins::inner(url).to_string(), // As ureq has always said it, so text that reads the status back still finds it. - message: format!("http status: {status}"), + message: with_retry_after(format!("http status: {status}"), status, wait), reason: Reason::Status(status), answer: Some(Box::new(Answer { headers: kept, excerpt: excerpt(body, html), user_agent: None })), } @@ -194,6 +195,67 @@ impl std::fmt::Display for Failure { } } +/// The longest a server's `Retry-After` is waited for: a longer wait (or a hostile one) is cut +/// to it, so the read is tried again within the hour. +pub(crate) const MAX_RETRY_AFTER: u64 = 3600; + +/// A refusal's text (`http status: 503`, or the browser's `HTTP 503`), with how long the server +/// asked to be left (`… (retry after 30 s)`) when it refused with 429 or 503 and its `Retry-After` +/// header (`retry_after`) says so, in seconds or as a date. Read back by +/// [`crate::cache::retry_after`]. +pub fn with_retry_after(text: String, status: u16, retry_after: Option<&str>) -> String { + let now = web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + match retry_after.filter(|_| matches!(status, 429 | 503)).and_then(|v| retry_after_secs(v, now)) { + Some(secs) => format!("{text} (retry after {secs} s)"), + None => text, + } +} + +/// A `Retry-After` header's wait in seconds (at least one, at most [`MAX_RETRY_AFTER`]): a number +/// of seconds, or an HTTP date (`Wed, 21 Oct 2015 07:28:00 GMT`) after `now` (seconds since 1970). +fn retry_after_secs(value: &str, now: u64) -> Option { + let value = value.trim(); + let secs = match value.parse::() { + Ok(secs) => secs, + Err(_) => http_date(value)?.saturating_sub(now), + }; + Some(secs.clamp(1, MAX_RETRY_AFTER)) +} + +/// An HTTP date's seconds since 1970 (the IMF-fixdate form servers send: `Sun, 06 Nov 1994 +/// 08:49:37 GMT`). +fn http_date(s: &str) -> Option { + let mut parts = s.split_whitespace(); + let (_, day, month, year, time, zone) = (parts.next()?, parts.next()?, parts.next()?, parts.next()?, parts.next()?, parts.next()?); + if zone != "GMT" || parts.next().is_some() { + return None; + } + const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + let month = MONTHS.iter().position(|m| *m == month)? as i64 + 1; + let (day, year): (i64, i64) = (day.parse().ok()?, year.parse().ok()?); + // Each field unsigned: a sign isn't a time. + let mut hms = time.split(':').map(|p| p.parse::().ok().map(i64::from)); + let (h, m, sec) = (hms.next()??, hms.next()??, hms.next()??); + let leap = year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); + let days_in_month = match month { + 2 if leap => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + }; + if !(1..=days_in_month).contains(&day) || !(1970..=9999).contains(&year) || h > 23 || m > 59 || sec > 60 || hms.next().is_some() { + return None; + } + // Days since 1970 of a date in the proleptic Gregorian calendar (Howard Hinnant's algorithm). + let y = if month <= 2 { year - 1 } else { year }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (month + if month > 2 { -3 } else { 9 }) + 2) / 5 + day - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + let days = era * 146_097 + doe - 719_468; + u64::try_from(days * 86_400 + h * 3600 + m * 60 + sec).ok() +} + /// A status's usual name (`403` → `Forbidden`), for the common ones. pub fn status_name(status: u16) -> Option<&'static str> { Some(match status { @@ -394,4 +456,31 @@ mod tests { assert_eq!(Failure::without_answer("u", "timeout: global", Reason::Timeout).reason, Reason::Timeout); assert_eq!((status_name(403), status_name(299)), (Some("Forbidden"), None)); } + + /// A 429 or 503 refusal says how long its server asked to be left, from its `Retry-After` + /// header in seconds or as a date, within an hour; other refusals, and a header that can't be + /// read, say nothing more. + #[test] + fn retry_after_on_429_and_503() { + let base = || "http status: 503".to_string(); + assert_eq!(with_retry_after(base(), 503, Some("30")), "http status: 503 (retry after 30 s)"); + assert_eq!(with_retry_after("HTTP 429".into(), 429, Some(" 2 ")), "HTTP 429 (retry after 2 s)"); + assert_eq!(with_retry_after(base(), 503, Some("0")), "http status: 503 (retry after 1 s)"); + assert_eq!(with_retry_after(base(), 503, Some("86400")), "http status: 503 (retry after 3600 s)"); + assert_eq!(with_retry_after(base(), 503, Some("soon")), "http status: 503"); + assert_eq!(with_retry_after(base(), 503, None), "http status: 503"); + assert_eq!(with_retry_after("http status: 500".into(), 500, Some("30")), "http status: 500"); + // A date: the seconds until it (Sun, 06 Nov 1994 08:49:37 GMT is 784 111 777). + assert_eq!(http_date("Sun, 06 Nov 1994 08:49:37 GMT"), Some(784_111_777)); + assert_eq!(retry_after_secs("Sun, 06 Nov 1994 08:50:37 GMT", 784_111_777), Some(60)); + assert_eq!(retry_after_secs("Sun, 06 Nov 1994 08:49:37 GMT", 784_200_000), Some(1), "past: at once, after a second"); + assert_eq!(http_date("Sun, 06 Nov 1994 08:49:37 PST"), None); + assert_eq!(http_date("Sun, 32 Nov 1994 08:49:37 GMT"), None); + assert_eq!(http_date("Wed, 31 Feb 2024 00:00:00 GMT"), None, "no 31 February"); + assert_eq!(http_date("Sun, 06 Nov 1994 -8:49:37 GMT"), None, "no negative hour"); + assert!(http_date("Thu, 29 Feb 2024 00:00:00 GMT").is_some() && http_date("Sat, 29 Feb 2025 00:00:00 GMT").is_none()); + // A refused answer's own header. + let headers = vec![("Retry-After".to_string(), "12".to_string())]; + assert_eq!(Failure::answered("u", 429, &headers, b"").message, "http status: 429 (retry after 12 s)"); + } } diff --git a/crates/tiles/src/pmtiles.rs b/crates/tiles/src/pmtiles.rs index c6215d99c..1176b8548 100644 --- a/crates/tiles/src/pmtiles.rs +++ b/crates/tiles/src/pmtiles.rs @@ -418,7 +418,14 @@ pub fn info(archive: &str) -> Result>, String> { match all.get(archive) { Some(State::Ready(i)) => return Ok(Some(i.clone())), Some(State::Loading(_)) => return Ok(None), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => return Err(e.clone()), + Some(State::Failed(e, at)) => { + // Not read again until its wait is over: the map draws again then. + let wait = crate::cache::retry_wait(e, RETRY); + if at.elapsed() < wait { + crate::cache::notify_at(*at + wait); + return Err(e.clone()); + } + } _ => {} } all.insert(archive.to_string(), State::Loading(Vec::new())); @@ -450,7 +457,8 @@ fn with_info(archive: &str, f: Box, String>) + Send> f(Ok(i)); } Some(State::Loading(waiting)) => waiting.push(f), - Some(State::Failed(e, at)) if at.elapsed() < RETRY => { + Some(State::Failed(e, at)) if at.elapsed() < crate::cache::retry_wait(e, RETRY) => { + crate::cache::notify_at(*at + crate::cache::retry_wait(e, RETRY)); let e = e.clone(); drop(all); f(Err(e)); @@ -512,7 +520,12 @@ fn finish(archive: &str, r: Result) { // A read the busy queue dropped isn't the archive's fault: forgotten, so the next // draw reads it again, rather than kept as a failure. Err(e) if e == crate::cache::DROPPED => all.remove(archive), - Err(e) => all.insert(archive.to_string(), State::Failed(e.clone(), Instant::now())), + Err(e) => { + // Read again once its wait is over: the map draws again then (a 429 or 503 says + // when). + crate::cache::notify_at(Instant::now() + crate::cache::retry_wait(e, RETRY)); + all.insert(archive.to_string(), State::Failed(e.clone(), Instant::now())) + } }; match old { Some(State::Loading(w)) => w, diff --git a/crates/ui-egui/src/connectionui.rs b/crates/ui-egui/src/connectionui.rs index a67a9f0b0..1526015bd 100644 --- a/crates/ui-egui/src/connectionui.rs +++ b/crates/ui-egui/src/connectionui.rs @@ -1127,15 +1127,19 @@ fn resolve_waiting(app: &mut RustGisApp, id: &str) { fn choose_s3(app: &mut RustGisApp, source: &str, id: &str) -> Result<(), String> { let v = app.run("connection.resolveS3", json!({"source": source, "id": id}))?; let name = app.session.connections.find(id).map(|c| c.name.clone()).unwrap_or_else(|| id.to_string()); - let opened = v["reopened"].as_u64().unwrap_or(0).to_string(); + let opened = v["reopened"].as_u64().unwrap_or(0); let failed = v["failed"].as_u64().unwrap_or(0); - let text = if failed == 0 { - i18n::fmt("{name} now reads {n} project sources", &[("name", &name), ("n", &opened)]) - } else { - i18n::fmt( - "{name} now reads {n} project sources; {failed} could not be read", - &[("name", &name), ("n", &opened), ("failed", &failed.to_string())], - ) + // Rasters read in place whose overview is on its way: the connection reads them, but they + // aren't in yet (the desktop doesn't wait; they land as the map draws). + let reading = v["reading"].as_u64().unwrap_or(0); + let n = opened.saturating_add(reading).to_string(); + let (failed_n, reading_n) = (failed.to_string(), reading.to_string()); + let args: &[(&str, &str)] = &[("name", &name), ("n", &n), ("failed", &failed_n), ("reading", &reading_n)]; + let text = match (failed, reading) { + (0, 0) => i18n::fmt("{name} now reads {n} project sources", args), + (0, _) => i18n::fmt("{name} now reads {n} project sources; {reading} still being read", args), + (_, 0) => i18n::fmt("{name} now reads {n} project sources; {failed} could not be read", args), + _ => i18n::fmt("{name} now reads {n} project sources; {reading} still being read and {failed} could not be read", args), }; app.session.notify(text, failed > 0); Ok(()) diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 02351c8f5..c2e750df7 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -4511,6 +4511,10 @@ impl RustGisApp { // Plugin commands that waited for `host.fetch` answers (the browser) run again once // they are in; their errors are notices already. pluginstart::run_again(self, ctx); + // A raster read in place whose overview couldn't be read is read again once its wait is over. + self.session.read_in_place_again(); + // Rasters read in place whose pixels landed since (the browser) say what they lost. + self.session.notice_landed_rasters(); // A frame may not come until the user moves: look again while commands wait. if self.session.plugin_retries_waiting() { ctx.request_repaint_after(std::time::Duration::from_millis(100)); diff --git a/crates/ui-egui/src/mapgpu.rs b/crates/ui-egui/src/mapgpu.rs index ca94e8c70..4156e9414 100644 --- a/crates/ui-egui/src/mapgpu.rs +++ b/crates/ui-egui/src/mapgpu.rs @@ -1732,6 +1732,8 @@ mod tests { tiles_pending: false, data_pending: false, tile_errors: vec![], + partial: vec![], + pending_in_place: vec![], tilt: None, clips: vec![], }; @@ -2347,6 +2349,8 @@ mod tests { tiles_pending: false, data_pending: false, tile_errors: vec![], + partial: vec![], + pending_in_place: vec![], tilt: None, clips: vec![], }; diff --git a/crates/ui-egui/src/mapview.rs b/crates/ui-egui/src/mapview.rs index 9e81ace48..7d8a573e7 100644 --- a/crates/ui-egui/src/mapview.rs +++ b/crates/ui-egui/src/mapview.rs @@ -794,13 +794,21 @@ fn main_view(app: &mut RustGisApp, ui: &mut Ui, t: &Tokens) { WAKER.call_once(|| { let ctx = ui.ctx().clone(); rustgis_engine::render::rustgis_tiles::cache::set_waker(move || ctx.request_repaint()); + // A read that failed is tried again when its wait is over, though nothing else moves. + let ctx = ui.ctx().clone(); + rustgis_engine::render::rustgis_tiles::cache::set_alarm(move |after| ctx.request_repaint_after(after)); let ctx = ui.ctx().clone(); rustgis_engine::map::source::set_waker(move || ctx.request_repaint()); }); let mut key = key; // A scene whose tile layers are all live places them for every frame: tiles arriving need no // new scene (#635). - if gpu_ok && cam.bearing == 0.0 && cam.pitch == 0.0 && app.map.live_rev == Some(app.session.revision) { + if gpu_ok + && cam.bearing == 0.0 + && cam.pitch == 0.0 + && app.map.live_rev == Some(app.session.revision) + && !reads_in_place(app.session.map()) + { key.tiles = 0; } else if fetches_tiles(app.session.map()) { key.tiles = rustgis_engine::render::rustgis_tiles::cache::generation(); @@ -1278,6 +1286,13 @@ fn drape_state(app: &RustGisApp) -> (crate::globe_drape::DrapeKey, u64) { fn fetches_tiles(m: &rustgis_engine::map::Map) -> bool { !m.tiles.is_empty() || m.view.terrain.as_ref().is_some_and(|t| matches!(t.source, rustgis_engine::map::view::TerrainSource::Tiles { .. })) + || reads_in_place(m) +} + +/// Whether the map has raster layers read in place: the part in view arrives a tile at a time, +/// through the tile cache, and the map is drawn again as it does. +fn reads_in_place(m: &rustgis_engine::map::Map) -> bool { + m.layers.iter().any(|l| l.raster.as_ref().is_some_and(|r| r.in_place)) } /// Uploads a composed GPU scene, whether built in place or by the frame worker. diff --git a/crates/ui-egui/src/panes.rs b/crates/ui-egui/src/panes.rs index 33de04e6c..b9971026c 100644 --- a/crates/ui-egui/src/panes.rs +++ b/crates/ui-egui/src/panes.rs @@ -6256,20 +6256,36 @@ fn raster_legend_rows(ui: &mut Ui, entries: &[RasterLegend], indent: f32, t: &To fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens) { let Some(d) = &rl.data else { - ui.label(egui::RichText::new(tl!("The raster's data couldn't be read.")).color(t.red)); + // Read in place, its overview is on its way until it lands, or fails (said why). + match &rl.error { + None if rl.in_place => { + ui.label(egui::RichText::new(tl!("Reading in place…")).color(t.text_dim)); + } + error => { + ui.label(egui::RichText::new(tl!("The raster's data couldn't be read.")).color(t.red)); + if let Some(e) = error { + ui.label(egui::RichText::new(e).size(11.0)); + } + } + } if let Some(p) = &rl.path { ui.label(egui::RichText::new(p).monospace().size(11.0)); } return; }; let e = d.extent(); - let (cx, cy) = d.cell_size(); let unit = if d.crs.is_geographic() { "°" } else { " m" }; + // Read in place, the map draws the full image: its size and cell size, not the overview's. + let full = d.overview.filter(|_| rl.in_place); + let (w, h) = full.unwrap_or((d.width, d.height)); + let (cx, cy) = d.cell_size(); + let (cx, cy) = (cx * f64::from(d.width) / f64::from(w.max(1)), cy * f64::from(d.height) / f64::from(h.max(1))); + let f = rustgis_engine::map::symbology::fmt_num; let mut rows = vec![ - ("Size", format!("{} × {}", d.width, d.height)), + ("Size", format!("{w} × {h}")), ("Bands", d.band_count().to_string()), ("Pixel type", d.data_type.clone()), - ("Cell size", format!("{} × {}{unit}", rustgis_engine::map::symbology::fmt_num(cx), rustgis_engine::map::symbology::fmt_num(cy))), + ("Cell size", format!("{} × {}{unit}", f(cx), f(cy))), ("Coordinate system", d.crs.label()), ("NoData", d.nodata.map(|v| v.to_string()).unwrap_or_else(|| "—".into())), ("Symbology", tl!(rl.style.renderer.label()).to_string()), @@ -6280,8 +6296,23 @@ fn raster_details(ui: &mut Ui, rl: &rustgis_engine::map::RasterLayer, t: &Tokens ("South", format!("{:.4}°", e.min_y)), ("North", format!("{:.4}°", e.max_y)), ]; - if let Some((w, h)) = d.overview { - rows.insert(1, ("Full size", format!("{w} × {h}"))); + match d.overview { + // Read in place: the overview is what the layer holds. + Some(_) if rl.in_place => { + rows.insert( + 1, + ( + "Read", + crate::i18n::fmt( + "In place, holding a {w} × {h} overview", + &[("w", &d.width.to_string()), ("h", &d.height.to_string())], + ), + ), + ); + } + Some((fw, fh)) => rows.insert(1, ("Full size", format!("{fw} × {fh}"))), + None if rl.in_place => rows.insert(1, ("Read", tl!("In place").to_string())), + None => {} } let value_w = (ui.available_width() - 140.0).max(80.0); egui::Grid::new("raster_details").num_columns(2).striped(true).max_col_width(value_w).spacing(vec2(12.0, 4.0)).show(ui, |ui| { diff --git a/crates/ui-egui/tests/smoke.rs b/crates/ui-egui/tests/smoke.rs index a2b3fcf04..3bc59d663 100644 --- a/crates/ui-egui/tests/smoke.rs +++ b/crates/ui-egui/tests/smoke.rs @@ -2006,13 +2006,16 @@ fn data_from_a_web_address_lands_later() { assert!(h.state().ui.downloads.is_empty()); } -/// Add Data From URL reads a cloud-optimised GeoTIFF in place from an address that names no file -/// (an API such as Vantor's `…/geotiff?…`): its first bytes are probed off the frame, then the -/// layer reads its tiles by range, never downloading it whole. -#[test] -fn an_unnamed_cog_address_is_read_in_place() { +/// A local server answering with a small cloud-optimised GeoTIFF by range, its HEAD answers +/// announcing the TIFF header's length as Vantor's Transform API does: its address, the bytes it +/// has sent and the file's length. +fn serve_cog() -> (std::net::SocketAddr, std::sync::Arc, usize) { + serve_cog_bytes(rustgis_engine::data::raster::cog::example(2048, 256)) +} + +/// [`serve_cog`] serving `body`. +fn serve_cog_bytes(body: Vec) -> (std::net::SocketAddr, std::sync::Arc, usize) { use std::io::{Read, Write}; - let body = rustgis_engine::data::raster::cog::example(2048, 256); let total = body.len(); let sent = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let count = sent.clone(); @@ -2024,7 +2027,6 @@ fn an_unnamed_cog_address_is_read_in_place() { let mut buf = [0u8; 4096]; let read = s.read(&mut buf).unwrap_or(0); let req = String::from_utf8_lossy(&buf[..read]).to_ascii_lowercase(); - // Its headers announce the TIFF header's length, as Vantor's Transform API does. if req.starts_with("head ") { let _ = write!( s, @@ -2046,6 +2048,41 @@ fn an_unnamed_cog_address_is_read_in_place() { let _ = s.write_all(part); } }); + (addr, sent, total) +} + +/// A cloud-optimised GeoTIFF read in place is a raster layer like a file's: the Symbology pane +/// shows it with the same raster controls (its renderer, its bands), not COG ones of its own. +#[test] +fn a_cog_read_in_place_uses_the_raster_symbology_pane() { + use rustgis_engine::data::raster::cog::{Example, example_with}; + let (addr, _, _) = serve_cog_bytes(example_with(&Example { size: 512, tile: 256, bands: 8, ..Example::default() })); + let mut h = harness(); + h.state_mut().run("ui.language", json!({"code": "en"})).unwrap(); + h.state_mut().run("project.new", json!(null)).unwrap(); + let r = h.state_mut().run("layer.addUrl", json!({"url": format!("http://{addr}/multispectral.tif"), "name": "Multispectral"})).unwrap(); + assert_eq!(r["inPlace"], true, "{r}"); + assert!(h.state().session.map().tiles.is_empty(), "a raster layer, not a tile layer"); + h.state_mut().run("raster.bands", json!({"layer": "Multispectral", "red": 5, "green": 3, "blue": 2})).unwrap(); + h.state_mut().run("ui.pane.symbology", json!(null)).unwrap(); + for _ in 0..3 { + h.step(); + } + assert_eq!(h.state().ui.right, Some(rustgis_ui_egui::RightPane::Symbology)); + h.get_by_label("Primary symbology"); + // The band pickers a file raster's RGB symbology has, on the bands chosen. + for colour in ["Red", "Green", "Blue"] { + assert!(h.query_by_label(colour).is_some(), "{colour}"); + } + assert!(h.query_by_label("The raster's data couldn't be read.").is_none()); +} + +/// Add Data From URL reads a cloud-optimised GeoTIFF in place from an address that names no file +/// (an API such as Vantor's `…/geotiff?…`): its first bytes are probed off the frame, then the +/// layer reads its tiles by range, never downloading it whole. +#[test] +fn an_unnamed_cog_address_is_read_in_place() { + let (addr, sent, total) = serve_cog(); let mut h = harness(); h.step(); let url = format!("http://{addr}/analytics/v1/raster/ortho-with-overviews/geotiff?function=pansharp_ortho&key=k"); @@ -2053,13 +2090,13 @@ fn an_unnamed_cog_address_is_read_in_place() { assert!(h.state().ui.downloads.is_empty(), "probed first, not downloaded"); for _ in 0..200 { h.step(); - if h.state().session.map().tiles.iter().any(|t| t.name == "Ortho") { + if h.state().session.map().layers.iter().any(|l| l.name == "Ortho") { break; } std::thread::sleep(std::time::Duration::from_millis(20)); } - let t = h.state().session.map().tiles.iter().find(|t| t.name == "Ortho").cloned(); - assert!(t.is_some_and(|t| t.url == url), "added as a tile layer read in place"); + let t = h.state().session.map().layers.iter().find(|l| l.name == "Ortho").and_then(|l| l.raster.clone()); + assert!(t.is_some_and(|r| r.in_place && r.path.as_deref() == Some(url.as_str())), "added as a raster layer read in place"); assert!(h.state().ui.downloads.is_empty() && h.state().ui.cog_probes.is_empty()); let read = sent.load(std::sync::atomic::Ordering::Relaxed); assert!(read < total / 4, "{read} of {total} bytes read"); @@ -2069,13 +2106,13 @@ fn an_unnamed_cog_address_is_read_in_place() { assert!(h.state().ui.downloads.is_empty() && h.state().ui.cog_probes.len() == 1); for _ in 0..200 { h.step(); - if h.state().session.map().tiles.iter().any(|t| t.name == "Null") { + if h.state().session.map().layers.iter().any(|l| l.name == "Null") { break; } std::thread::sleep(std::time::Duration::from_millis(20)); } - let t = h.state().session.map().tiles.iter().find(|t| t.name == "Null").cloned(); - assert!(t.is_some_and(|t| t.url == null_url), "added as a tile layer read in place"); + let t = h.state().session.map().layers.iter().find(|l| l.name == "Null").and_then(|l| l.raster.clone()); + assert!(t.is_some_and(|r| r.in_place && r.path.as_deref() == Some(null_url.as_str())), "added as a raster layer read in place"); // A probe under way when another project opens is dropped: its layer never lands there. h.state_mut().run("ui.addUrl", json!({"url": format!("{url}&n=3"), "name": "Late"})).unwrap(); h.state_mut().run("project.new", json!({})).unwrap(); @@ -2084,7 +2121,7 @@ fn an_unnamed_cog_address_is_read_in_place() { h.step(); std::thread::sleep(std::time::Duration::from_millis(20)); } - assert!(!h.state().session.map().tiles.iter().any(|t| t.name == "Late")); + assert!(!h.state().session.map().layers.iter().any(|l| l.name == "Late")); } /// A service layer added from the app (Add From Service, the Connections tree) is read page by diff --git a/docs/user-guide/analysis.md b/docs/user-guide/analysis.md index c48c74a6d..176837a7b 100644 --- a/docs/user-guide/analysis.md +++ b/docs/user-guide/analysis.md @@ -179,7 +179,11 @@ bands, colors, mask and resampling. Set the minimum and maximum zoom from 0 to 1 transparency are included. The raster must be georeferenced and fit within one world. Turn off dynamic range adjustment and display rotation first. The in-memory export is limited to 512 tiles and 64 MiB of compressed tile data; reduce the extent or zoom range -if it exceeds a limit. Areas beyond the Web Mercator latitude limit are clipped. +if it exceeds a limit. Areas beyond the Web Mercator latitude limit are clipped. A cloud-optimised +GeoTIFF read in place is exported from its full-resolution tiles: the desktop waits for them (two +minutes at most); the web app can't wait, so an export whose tiles aren't all in says so and asks +for the rest: export again once they are in (a large export may take a few tries). Without them +nothing is written. **Vector to PMTiles** (`vectorToPmtiles`) writes a single-layer MVT archive in standard Web Mercator. Choose `minZoom` and `maxZoom` from 0 to 16; `sourceLayer` defaults to the input diff --git a/docs/user-guide/data-formats.md b/docs/user-guide/data-formats.md index 341b837f7..e78555ba9 100644 --- a/docs/user-guide/data-formats.md +++ b/docs/user-guide/data-formats.md @@ -111,16 +111,57 @@ unresponsive. The original JPEG 2000 byte buffer is released once it has been st before GDAL runs or the GeoTIFF is read back. The converted GeoTIFF and decoded raster still occupy memory together; the 512 MiB file limit is not a limit on total process memory. -A tiled GeoTIFF is read in place rather than downloaded: its header first, then, by HTTP range -requests, the tiles of the overview each part of the view needs. Overviews make zoomed-out reads -smaller. It is a tile layer (`map.addTiles` kind `cog`, saved by its address), drawn under the map's -own layers: RGB(A) 8-bit images in colour, other images stretched between the 2nd and 98th -percentiles of their coarsest overview. Planar files (bands stored apart), signed-byte data and -bottom-up images work too. `map.setTiles` selects one or three `"bands"` (from 1) and a fixed -`"range"`; see [Satellite embeddings](web-data.md#satellite-embeddings). -A GeoTIFF in strips, a rotated image or an unsupported tiled image downloads whole as a raster layer, as does a -GeoTIFF added with `"whole": true` (for raster tools and symbology). The web app needs the server -to allow range requests (CORS with the `Range` header). +A tiled GeoTIFF (a cloud-optimised one) is read in place rather than downloaded: its header +first, then, by HTTP range requests, only the parts the map needs. It is a raster layer like a +GeoTIFF file, saved by its address: the Symbology pane, the raster commands (`raster.symbology`, +`raster.bands`, `raster.stretch`, `raster.dra`, `raster.enhance`, `raster.mask`…), its legend, +identify and the raster tools work on it as on a file's. The layer holds an overview of the image +(every band, about two million values), from which its statistics, histograms and tools read, as a +file too large to hold whole is opened from an overview; zoomed in past it, the map reads the part +in view at the resolution it needs as it draws, and draws it with the layer's style and the +overview's statistics, so the image looks the same at every zoom. Until a part's tiles arrive, the +level zoomed from stands in for them. A turned layer (Rotation in the Symbology pane) is drawn +from the overview it holds: zoomed in past it, the map says so and an export refuses it. The +Details pane says the image's full size and that it is read in place. The desktop app adds the +layer at once and reads its overview as the map draws (the Details pane says so until it is in), +so a slow host doesn't hold the window; the command line and Python wait for it, up to two +minutes, so a command's result is complete. A saved S3 connection chosen for such layers (Choose +S3 connection…) counts them as still being read until their overview is in, apart from what it +opened. A file whose smallest image is too large to hold as the overview (a large image saved +without overviews) is refused where the session waits for it (the command line, Python); the desktop +app and the browser add the layer, which says why it can't be read. GDAL's COG driver or `gdaladdo` +makes the overviews. A tile a sparse file leaves out is empty. A tile, the file's header or the +overview a layer holds, that fails to arrive is asked for again a minute later, or when its server +says to in a `Retry-After` header (with a 429 or 503 answer, within the hour); the map draws again +then. An overview that fails again waits twice as long each time, up to an hour. On the command +line and in Python, which don't draw, the next command run once the wait is over reads a failed +overview again, waiting for it as the project did. +Exporting the layer (Raster to PMTiles) waits for its tiles on the desktop, up to two minutes, +and isn't written without them; the web app can't wait, so an export whose tiles aren't all in +says so and asks for the rest: export again once they are in (a large archive may take a few +tries). + +Its first look is what the file's PhotometricInterpretation tag says: + +- **RGB:** red, green and blue, with the alpha band if there is one. Bands past the colour ones + (an RGB and near-infrared orthophoto, say) are read too. +- **Palette:** its colour table (Colormap). +- **Grey (MinIsBlack, as most remote sensing images are):** the first band stretched in grey, or + three bands as red, green and blue. An alpha band is shown with it. + +In an image with more bands (an 8-band multispectral image, say) you pick the combination in the +Symbology pane, as for any raster. GeoTIFF files follow the same rule and have every band too (an +RGB and near-infrared file has four), and so do other rasters without colours of their own (Zarr +stores, files read through GDAL): four or more bands open as the first in grey, where any three +or more used to open as red, green and blue. A palette image without its colour table is drawn in +grey, and says so. Planar files (bands stored +apart), 8- to 64-bit integer and floating-point data, signed-byte data and bottom-up images work +too. A dated series of GeoTIFFs (an address with date placeholders), a data cube's files and +AlphaEarth's embeddings are tile layers that follow the time slider or dequantise their values; +see [Satellite embeddings](web-data.md#satellite-embeddings). +A GeoTIFF in strips, a rotated image or an unsupported tiled image downloads whole, as does a +GeoTIFF added with `"whole": true` (for raster tools that need every pixel at full resolution). +The web app needs the server to allow range requests (CORS with the `Range` header). On the desktop and the command line, `"bbox"` reads just an area of a cloud-optimised GeoTIFF as a raster layer, for files too large to download whole (a national land-cover map, a 1° elevation @@ -136,10 +177,10 @@ tile), without a STAC catalogue: - Only the tiles the window crosses are read, at full resolution. `"cellSize"` reads the coarsest overview whose cells are no larger, in the file's units (metres for a longitude/latitude file, a degree taken as 111.32 km); one finer than the full image's cells reads the full image, with a - notice. Without it the window takes no overview's values. Reading the header still reads up to - 16 tiles of the coarsest overview at least 256 pixels a side (the full image when none is), for - the stretch it is drawn with, as for any GeoTIFF drawn in place. An 8-bit colour file skips it, - and so does one whose level has more than 16 tiles: it is drawn with its value type's full range. + notice. Without it the window takes no overview's values. Reading the header reads nothing + more: a GeoTIFF read in place as a raster layer takes its stretch from the overview it holds, + and a tile layer (a dated series, a data cube's file) reads a few tiles of its coarsest + overview for its stretch when a tile first needs it. - A window holds 64 million values at most, all bands together (256 MB), and 4,096 of the file's tiles a band. A window over the first limit is refused, naming the cell size that fits when an overview does, or asking for a smaller box; one over the second asks for a coarser level or a @@ -177,9 +218,10 @@ desktop, while in the web app they need the file downloaded first. On desktop, `layer.addUrl` waits for downloads, up to a minute; the From URL… dialog (`ui.addUrl`) and a plugin's `layer.addUrl` work in the background. In the web app the same dialog handles asynchronous downloads. -`layer.addUrl` can add PMTiles and COG tile layers directly, but an ordinary download returns -`{"pending": true, "url": "…"}` and is added later by the app. To open a remote GeoParquet or -FlatGeobuf whose header hasn't been fetched yet, use the From URL… dialog or a `?data=` link. +`layer.addUrl` can add PMTiles archives and GeoTIFFs read in place directly, but an ordinary +download returns `{"pending": true, "url": "…"}` and is added later by the app. To open a remote +GeoParquet or FlatGeobuf whose header hasn't been fetched yet, use the From URL… dialog or a +`?data=` link. ## Map and feature services diff --git a/docs/user-guide/data.md b/docs/user-guide/data.md index 45a388cb9..fd083ea20 100644 --- a/docs/user-guide/data.md +++ b/docs/user-guide/data.md @@ -1041,7 +1041,9 @@ descriptions GDAL and rasterio write inside a GeoTIFF (its `GDAL_METADATA` tag) too, a sidecar's taking precedence. Exporting a raster opened from an overview writes that overview; the results of adding and of exporting it (`layer.addData`, `share.exportRaster`) give the full size as `overviewOf`, so scripts -and the MCP server see it too. +and the MCP server see it too. A GeoTIFF read in place that is added before its overview is in +(the desktop app, the browser) answers `"reading": true`, sized by its header when that is in, and +with `null` for its size and bands until then. `share.exportRaster` writes an untiled GeoTIFF; with `"cog": true` it writes a cloud-optimised one: 512-pixel tiles compressed with Deflate, overviews (each half the size of the last, down to one tile), and every directory ahead of the tiles, so a reader fetches the header once and then only the diff --git a/docs/user-guide/web-data.md b/docs/user-guide/web-data.md index d09d2b8d4..3ddf0e001 100644 --- a/docs/user-guide/web-data.md +++ b/docs/user-guide/web-data.md @@ -610,10 +610,11 @@ their footprints drawn as a layer. | **Tessera** (Cambridge; 128 values a 10 m pixel, 2017–2025) | **Add view** reads three bands (1, 2 and 3 by default) of the view from Tessera's [Zarr store](https://data.source.coop/tessera/tessera/zarr/v1.1-dclimate) in place (the shard's index, then the chunks the view needs: at most about 6 km a side, its centre when the view is larger), as a 32-bit raster in UTM. | | **Clay**, **Major TOM**, **Copernicus-Embed** | Listed with links to their data and papers; RustGIS doesn't read them. | -AlphaEarth's tiles keep their look in their layer's address (`#bands=2,17,10&range=-0.3,0.3&dequantize=aef`), -which `map.setTiles` changes for any cloud-optimised GeoTIFF read in place (`"bands"`, `"range"`, -`"dequantize"`); GeoTIFFs with their bands stored apart (planar), signed bytes and bottom-up images -are read in place too. The commands: `embeddings.datasets`, `embeddings.search` (`"dataset"`, +AlphaEarth's tiles are tile layers that keep their look in their address +(`#bands=2,17,10&range=-0.3,0.3&dequantize=aef`), which `map.setTiles` changes (`"bands"`, +`"range"`, `"dequantize"`); other cloud-optimised GeoTIFFs are raster layers read in place, styled +in the Symbology pane. GeoTIFFs with their bands stored apart (planar), signed bytes and bottom-up +images are read in place too. The commands: `embeddings.datasets`, `embeddings.search` (`"dataset"`, `"year"`, `"bbox"` or the view, `"footprints"`) and `embeddings.add` (`"dataset"`, `"url"`, `"bands"`, `"stretch"`, `"dimension"`, `"bbox"`, `"year"`):