diff --git a/docs/source/modules/index.md b/docs/source/modules/index.md index 7db4e77..46ea45c 100644 --- a/docs/source/modules/index.md +++ b/docs/source/modules/index.md @@ -4,3 +4,9 @@ Fetchez supports more than 100 distinct data modules. ```{module-table} ``` + +```{toctree} +:maxdepth: 1 + +tnm +``` diff --git a/docs/source/modules/tnm.md b/docs/source/modules/tnm.md new file mode 100644 index 0000000..332d502 --- /dev/null +++ b/docs/source/modules/tnm.md @@ -0,0 +1,37 @@ +# TNM elevation products + +Use `products` to request specific elevation products from The National Map: + +```bash +fetchez run --global-hook list-only -R -124.5/-124.0/41.5/44.0 tnm --products 1m/1_3as +``` + +Supported names are `s1m`, `1m`, `1_9as`, `1_3as`, `1_as`, `5m`, and +`2_as`. Each product is queried separately, in the requested order. Repeated +names are queried once. This order does not establish coverage precedence. + +Each entry includes `tnm_product` and `tnm_dataset`, along with the existing +source ID, publication date, update date, and metadata links supplied by TNM. +`tnm_project` is the decoded project directory from the download URL, when +present. It is not a verified WESM project identifier. No WESM lookup is made. + +Downloads requested through `products` use a product directory and a short +URL hash before the filename, so different endpoints with the same filename +do not overwrite each other. Overlapping entries remain available to hooks. + +Product queries raise an error if the API rejects the dataset, fails a request, +or returns incomplete or inconsistent pages. Entries collected during that +run are removed before the error is raised. A failed query must not be treated +as evidence that a higher-resolution product has no coverage. A successful +query returning zero products is allowed. + +The existing `datasets` argument and default one-arc-second query remain +available. Do not combine `products` and `datasets`. Use +`--strict-datasets true` with `datasets` to request the same error handling; +otherwise the existing broad-search fallback remains available. Results from +that fallback are not assigned a specific product label. + +Discovery provides API bounding boxes. Use footprint hooks to obtain more +precise geometry and `spatial_cull` when whole overlapping entries should be +removed. Partial coverage exclusions, resolution precedence, and WESM project +matching belong in the consuming application, such as Globato. diff --git a/src/fetchez/modules/tnm.py b/src/fetchez/modules/tnm.py index b518aef..a0bf771 100644 --- a/src/fetchez/modules/tnm.py +++ b/src/fetchez/modules/tnm.py @@ -11,16 +11,16 @@ :license: MIT, see LICENSE for more details. """ +import hashlib import logging +from collections.abc import Sequence from typing import Optional +from urllib.parse import unquote, urlsplit from shapely.geometry import box -from fetchez import core +from fetchez import cli, core, spatial, utils from fetchez.modules import FetchModule -from fetchez import utils -from fetchez import spatial -from fetchez import cli logger = logging.getLogger(__name__) @@ -57,12 +57,19 @@ "US Topo Historical", "Land Cover - Woodland", "3D Hydrography Program (3DHP)", + "Seamless 1-m DEM (S1M)", ] DATASET_ALIASES = { "1m": 2, "1_9as": 4, "1_3as": 3, "1_as": 1, + "2_as": 5, + "5m": 6, + "s1m": 29, +} +DATASET_PRODUCTS = { + DATASET_CODES[index]: alias for alias, index in DATASET_ALIASES.items() } @@ -77,6 +84,8 @@ q="Free text search query", date_start="Start date (YYYY-MM-DD)", date_end="End date (YYYY-MM-DD)", + products="Elevation products: s1m/1m/1_9as/1_3as/1_as/5m/2_as (strict queries)", + strict_datasets="Raise on rejected or incomplete queries instead of returning partial results", ) class TheNationalMap(FetchModule): name = "tnm" @@ -114,6 +123,8 @@ def __init__( date_type: Optional[str] = "dateCreated", date_start: Optional[str] = None, date_end: Optional[str] = None, + products: str | Sequence[str] | None = None, + strict_datasets: bool = False, **kwargs, ): super().__init__(name="tnm", **kwargs) @@ -124,6 +135,23 @@ def __init__( self.date_type = date_type self.date_start = date_start self.date_end = date_end + self.products = None + if products is not None: + if datasets is not None: + raise ValueError("Use either products or datasets, not both") + selected = utils.parse_arg_to_list( + products if isinstance(products, (str, list)) else list(products), str + ) + self.products = list( + dict.fromkeys(str(value).lower() for value in selected) + ) + if not self.products or any( + value not in DATASET_ALIASES for value in self.products + ): + raise ValueError(f"Unknown TNM products: {products}") + self.strict_datasets = self.products is not None or bool( + utils.str2bool(strict_datasets) + ) def run(self): """Run the TNM fetching module.""" @@ -131,15 +159,13 @@ def run(self): if self.wgs_region is None or not spatial.region_valid_p(self.wgs_region): return [] - w, e, s, n = self.wgs_region - bbox_str = f"{w},{s},{e},{n}" - - offset = 0 - total = 0 - # Determine Datasets to query dataset_names = [] - if self.datasets is not None: + if self.products is not None: + dataset_names = [ + DATASET_CODES[DATASET_ALIASES[value]] for value in self.products + ] + elif self.datasets not in (None, "None"): try: ds_indices = [] for x in self.datasets.split("/"): @@ -147,10 +173,16 @@ def run(self): ds_indices.append(DATASET_ALIASES[x.lower()]) else: ds_indices.append(int(x)) + if self.strict_datasets and any( + i < 0 or i >= len(DATASET_CODES) for i in ds_indices + ): + raise ValueError("Dataset index out of range") dataset_names = [ DATASET_CODES[i] for i in ds_indices if 0 <= i < len(DATASET_CODES) ] except (ValueError, IndexError): + if self.strict_datasets: + raise ValueError(f"Invalid TNM datasets: {self.datasets}") from None logger.warning( f"Could not parse datasets '{self.datasets}'. Using default." ) @@ -158,6 +190,28 @@ def run(self): if not dataset_names: dataset_names = ["National Elevation Dataset (NED) 1 arc-second"] + start = len(self.results) + try: + # Query products separately so each entry has an unambiguous product label. + if self.products is not None: + for dataset in dataset_names: + self._run_query([dataset]) + else: + self._run_query(dataset_names) + except Exception: + del self.results[start:] + raise + return self + + def _run_query(self, dataset_names): + w, e, s, n = self.wgs_region + bbox_str = f"{w},{s},{e},{n}" + offset = 0 + expected_total = None + seen_urls = set() + dataset = dataset_names[0] if len(dataset_names) == 1 else None + product = DATASET_PRODUCTS.get(dataset) + while True: params = { "bbox": bbox_str, @@ -183,10 +237,12 @@ def run(self): req = core.Fetch(TNM_API_PRODUCTS_URL).fetch_req(params=params) if ( - req + req is not None and "All dataset queries failed" in req.text and "datasets" in params ): + if self.strict_datasets: + raise RuntimeError("TNM API rejected the requested dataset") logger.warning( "USGS rejected the strict dataset strings. Retrying with broad text search..." ) @@ -200,14 +256,22 @@ def run(self): params["q"] = fallback_q req = core.Fetch(TNM_API_PRODUCTS_URL).fetch_req(params=params) + # A broad text search cannot establish a specific product identity. + product = None + dataset = None if req is None or req.status_code != 200: + if self.strict_datasets: + status = req.status_code if req is not None else "no response" + raise RuntimeError(f"TNM API request failed: {status}") logger.error( f"TNM API Failed: {req.status_code if req else 'No Response'}" ) break if req.text.strip().startswith("{errorMessage"): + if self.strict_datasets: + raise RuntimeError(f"TNM API error: {req.text}") logger.error(f"TNM API Error: {req.text}") break @@ -215,13 +279,39 @@ def run(self): data = req.json() total = data.get("total", 0) items = data.get("items", []) + if self.strict_datasets: + if ( + data.get("errorMessage") + or "total" not in data + or "items" not in data + or not isinstance(total, int) + or total < 0 + or not isinstance(items, list) + or offset + len(items) > total + or (offset < total and not items) + ): + raise ValueError( + "TNM API returned an incomplete or invalid page" + ) + if expected_total is not None and total != expected_total: + raise ValueError("TNM result total changed during pagination") + expected_total = total for item in items: url = item.get("downloadURL") if not url: + if self.strict_datasets: + raise ValueError("TNM product has no download URL") continue - - filename = url.split("/")[-1] + if self.strict_datasets: + if url in seen_urls: + raise ValueError( + "TNM API repeated a download URL during pagination" + ) + seen_urls.add(url) + + path = urlsplit(url).path + filename = path.rsplit("/", 1)[-1] fmt = item.get("format", "Unknown") item_bbox = item.get("boundingBox", {}) @@ -236,35 +326,22 @@ def run(self): ) geom = box(bounds[0], bounds[2], bounds[1], bounds[3]) - # Extract the tile footprint or project ID based on dataset type - # this is all redundant now, but keeping in case useful. - if ( - "ned19" in filename.lower() - or "opr" in filename.lower() - or "lpc" in filename.lower() - ): - _fn_bn = "_".join(filename.split("_")[:-1]) - elif bounds: - _fn_bn = f"{round(bounds[0], 4)}_{round(bounds[1], 4)}_{round(bounds[2], 4)}_{round(bounds[3], 4)}" - else: - _fn_bn = item.get("title", filename) - - date = item.get("publicationDate", "") - if bounds: - _bounds_str = f"{round(bounds[0], 4)}_{round(bounds[1], 4)}_{round(bounds[2], 4)}_{round(bounds[3], 4)}" - _project_id = "/".join(item.get("title", "")) - _fn_bn = f"{_bounds_str}_{_project_id}" - else: - _fn_bn = item.get("title", filename) - date = item.get("publicationDate", "") project = None - if "/Projects/" in url: - project = url.split("/Projects/", 1)[1].split("/", 1)[0] + if "/Projects/" in path: + project = unquote( + path.split("/Projects/", 1)[1].split("/", 1)[0] + ) + + dst_fn = filename + if self.products is not None: + # Different projects/versions can share a filename. + digest = hashlib.sha256(url.encode()).hexdigest()[:12] + dst_fn = f"{product}/{digest}/{filename}" self.add_entry_to_results( url=url, - dst_fn=filename, + dst_fn=dst_fn, data_type="tnm", format=fmt, bounds=bounds, @@ -273,6 +350,8 @@ def run(self): remote_size=item.get("sizeInBytes"), title=item.get("title"), tnm_project=project, + tnm_product=product, + tnm_dataset=dataset, tnm_source_id=item.get("sourceId"), tnm_publication_date=item.get("publicationDate"), tnm_last_updated=item.get("lastUpdated"), @@ -281,10 +360,12 @@ def run(self): ) except Exception as e: + if self.strict_datasets: + raise RuntimeError(f"Unable to complete TNM discovery: {e}") from e logger.exception(f"Error parsing TNM JSON: {e}") break - offset += 100 + offset += len(items) if self.strict_datasets else 100 if offset >= total: break diff --git a/tests/modules/test_tnm_provider_foundation.py b/tests/modules/test_tnm_provider_foundation.py index 773d1c8..e9947f2 100644 --- a/tests/modules/test_tnm_provider_foundation.py +++ b/tests/modules/test_tnm_provider_foundation.py @@ -68,6 +68,9 @@ def _item(title, url, publication_date, source_id): ("1_9as", tnm.DATASET_CODES[4]), ("1_3as", tnm.DATASET_CODES[3]), ("1_as", tnm.DATASET_CODES[1]), + ("s1m", tnm.DATASET_CODES[29]), + ("5m", tnm.DATASET_CODES[6]), + ("2_as", tnm.DATASET_CODES[5]), ("2", tnm.DATASET_CODES[2]), ("8/2", f"{tnm.DATASET_CODES[8]},{tnm.DATASET_CODES[2]}"), ], @@ -166,3 +169,169 @@ def test_spatial_cull_hook_retains_newest_tnm_product(): kept_mod, kept_entry = culled_results[0] assert kept_entry["tnm_source_id"] == "newer" assert kept_entry["metadata"]["date"] == "2025-02-01" + + +def _pages(monkeypatch, payloads): + pages = iter(payloads) + + def fetch_req(self, params=None): + FakeFetch.params_seen.append(dict(params or {})) + payload = next(pages) + return payload if isinstance(payload, FakeResponse) else FakeResponse(payload) + + monkeypatch.setattr(FakeFetch, "fetch_req", fetch_req) + + +@pytest.mark.parametrize( + "products", + ["1m/1_3as/1m", ["1m", "1_3as", "1m"]], +) +def test_products_query_separately_and_preserve_source_metadata(monkeypatch, products): + item = _item( + "Project tile", + "https://example.test/Projects/CA%20Survey/tile.tif", + "2024-01-01", + "source", + ) + _pages(monkeypatch, [{"total": 1, "items": [item]}, {"total": 1, "items": [item]}]) + mod = tnm.TheNationalMap( + src_region=SAMPLE_REGION, products=products, use_cache=False + ) + assert mod.products == ["1m", "1_3as"] + mod.run() + assert [p["datasets"] for p in FakeFetch.params_seen] == [ + tnm.DATASET_CODES[2], + tnm.DATASET_CODES[3], + ] + assert [entry["tnm_product"] for entry in mod.results] == ["1m", "1_3as"] + assert [entry["tnm_dataset"] for entry in mod.results] == [ + tnm.DATASET_CODES[2], + tnm.DATASET_CODES[3], + ] + assert mod.results[0]["tnm_project"] == "CA Survey" + assert mod.results[0]["tnm_source_id"] == "source" + assert mod.results[0]["tnm_publication_date"] == "2024-01-01" + assert mod.results[0]["tnm_vendor_meta_url"] == item["vendorMetaUrl"] + assert mod.results[0]["dst_fn"] != mod.results[1]["dst_fn"] + assert mod.results[0]["geometry"].bounds == (-118.65, 34.05, -118.60, 34.10) + + +def test_products_keep_distinct_urls_with_same_filename(): + FakeFetch.payload = { + "total": 2, + "items": [ + _item("old", "https://example.test/Projects/old/tile.tif", "2020", "old"), + _item("new", "https://example.test/Projects/new/tile.tif", "2021", "new"), + ], + } + mod = tnm.TheNationalMap(src_region=SAMPLE_REGION, products="1m", use_cache=False) + mod.run() + assert len({entry["dst_fn"] for entry in mod.results}) == 2 + assert len(mod.results) == 2 + + +@pytest.mark.parametrize("products", ["unknown", "1m/unknown", "", []]) +def test_invalid_products_never_fall_back(products): + with pytest.raises(ValueError, match="Unknown TNM products"): + tnm.TheNationalMap(src_region=SAMPLE_REGION, products=products) + assert not FakeFetch.params_seen + + +def test_products_and_datasets_are_mutually_exclusive(): + with pytest.raises(ValueError, match="either products or datasets"): + tnm.TheNationalMap(src_region=SAMPLE_REGION, products="1m", datasets="2") + + +def test_product_list_deduplicates_without_reordering(): + mod = tnm.TheNationalMap( + src_region=SAMPLE_REGION, products=["5m", "1m", "5m"], use_cache=False + ) + mod.run() + assert [p["datasets"] for p in FakeFetch.params_seen] == [ + tnm.DATASET_CODES[6], + tnm.DATASET_CODES[2], + ] + + +@pytest.mark.parametrize("selector", ["2/unknown", "99", "-1"]) +def test_strict_dataset_validation(selector): + mod = tnm.TheNationalMap( + src_region=SAMPLE_REGION, datasets=selector, strict_datasets=True + ) + with pytest.raises(ValueError, match="Invalid TNM datasets"): + mod.run() + assert not FakeFetch.params_seen + + +def test_strict_query_rejection_does_not_broaden(monkeypatch): + response = FakeResponse({}) + response.text = "All dataset queries failed" + response.status_code = 400 + _pages(monkeypatch, [response]) + mod = tnm.TheNationalMap(src_region=SAMPLE_REGION, products="s1m", use_cache=False) + with pytest.raises(RuntimeError, match="rejected"): + mod.run() + assert len(FakeFetch.params_seen) == 1 + assert mod.results == [] + + +@pytest.mark.parametrize( + "second", + [ + {"total": 2, "items": []}, + {"total": 3, "items": []}, + {"errorMessage": "failed"}, + {"total": 2, "items": [{"title": "missing URL"}]}, + ], +) +def test_incomplete_discovery_rolls_back_results(monkeypatch, second): + item = _item("tile", "https://example.test/tile.tif", "2020", "id") + _pages(monkeypatch, [{"total": 2, "items": [item]}, second]) + mod = tnm.TheNationalMap(src_region=SAMPLE_REGION, products="1m", use_cache=False) + with pytest.raises(RuntimeError): + mod.run() + assert mod.results == [] + assert FakeFetch.params_seen[1]["offset"] == 1 + + +def test_short_pages_are_not_skipped(monkeypatch): + items = [ + _item(str(i), f"https://example.test/{i}.tif", "2020", str(i)) for i in range(2) + ] + _pages(monkeypatch, [{"total": 2, "items": [item]} for item in items]) + mod = tnm.TheNationalMap(src_region=SAMPLE_REGION, products="1m", use_cache=False) + mod.run() + assert len(mod.results) == 2 + assert [p["offset"] for p in FakeFetch.params_seen] == [0, 1] + + +def test_repeated_page_is_rejected(monkeypatch): + item = _item("tile", "https://example.test/tile.tif", "2020", "id") + _pages(monkeypatch, [{"total": 2, "items": [item]}] * 2) + mod = tnm.TheNationalMap(src_region=SAMPLE_REGION, products="1m", use_cache=False) + with pytest.raises(RuntimeError, match="repeated"): + mod.run() + assert mod.results == [] + + +def test_later_product_failure_rolls_back_earlier_product(monkeypatch): + item = _item("tile", "https://example.test/tile.tif", "2020", "id") + _pages(monkeypatch, [{"total": 1, "items": [item]}, {"errorMessage": "failed"}]) + mod = tnm.TheNationalMap( + src_region=SAMPLE_REGION, products="1m/1_as", use_cache=False + ) + with pytest.raises(RuntimeError): + mod.run() + assert mod.results == [] + + +def test_legacy_fallback_does_not_assign_product_identity(monkeypatch): + rejected = FakeResponse({}) + rejected.text = "All dataset queries failed" + item = _item("tile", "https://example.test/tile.tif", "2020", "id") + _pages(monkeypatch, [rejected, {"total": 1, "items": [item]}]) + mod = tnm.TheNationalMap(src_region=SAMPLE_REGION, datasets="1m", use_cache=False) + mod.run() + assert "datasets" not in FakeFetch.params_seen[1] + assert mod.results[0]["tnm_product"] is None + assert mod.results[0]["tnm_dataset"] is None