From a7fcab874af52e3da99dbb92af005b05b6d1ffd2 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:26:27 -0400 Subject: [PATCH 01/14] fix: Add the HuBMAP DRS broker host to the outbound URL allowlist Every HuBMAP access_url resolves through drs.hubmapconsortium.org before bytes flow from assets.hubmapconsortium.org, but the worker outbound-URL allowlist only carried the asset host. Every HuBMAP file fetch failed with a 502, since the DRS resolution step itself was rejected as unsafe before it ever reached the asset host. --- src/cfdb/workflows/urlsafe.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cfdb/workflows/urlsafe.py b/src/cfdb/workflows/urlsafe.py index a42baaa..daec615 100644 --- a/src/cfdb/workflows/urlsafe.py +++ b/src/cfdb/workflows/urlsafe.py @@ -47,6 +47,10 @@ "encode-public.s3.amazonaws.com", # HuBMAP public assets. "assets.hubmapconsortium.org", + # HuBMAP DRS broker (GA4GH DRS metadata API that every HuBMAP + # ``access_url`` resolves through before bytes flow from + # assets.hubmapconsortium.org above). + "drs.hubmapconsortium.org", ) From 70087f985ad42baf4929c25103b04cb8acff1d3a Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:26:36 -0400 Subject: [PATCH 02/14] fix: Prune HuBMAP files matched via the persistent_id fallback join HuBMAP's C2M2 export never populates file_in_collection, so the Rust materializer falls back to matching file.persistent_id against collection.persistent_id to join a file to its dataset. The pruning step that removes files belonging to non-public datasets only knew about the file_in_collection join, so it was a complete no-op for HuBMAP: every non-public and dbGaP-controlled-access file survived pruning and was stamped data_access_level=public regardless of its true access level. --- src/cfdb/services/sync.py | 19 +++++++++++++++++++ tests/test_sync.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/src/cfdb/services/sync.py b/src/cfdb/services/sync.py index f9a76b8..0f506e2 100644 --- a/src/cfdb/services/sync.py +++ b/src/cfdb/services/sync.py @@ -869,6 +869,25 @@ async def _prune_non_public_hubmap_raw_records( f"HuBMAP pruning: deleted {fic_result.deleted_count} file_in_collection links" ) + # 4b. Delete files matched to a non-public dataset via the persistent_id + # (DOI) fallback join. HuBMAP's C2M2 export never populates + # file_in_collection (confirmed empty on every sync), so the + # materializer falls back to matching file.persistent_id against + # collection.persistent_id (see materialize/src/main.rs, "Fallback: + # look up collection by persistent_id"). Step 4 above is blind to that + # join, so without this a file whose own persistent_id names a + # non-public dataset survives pruning and gets the blanket + # data_access_level=public stamp below regardless of its true access + # level -- confirmed to leak files registered under dbGaP + # controlled-access studies. + doi_result = await api.db.file.delete_many( + {"submission": "hubmap", "persistent_id": {"$in": non_public_dois}} + ) + logger.info( + f"HuBMAP pruning: deleted {doi_result.deleted_count} file records via " + "persistent_id (DOI) fallback match" + ) + # 5. Find which candidate files still have links (shared with a public collection) if candidate_file_keys: still_linked: set[tuple[str, str]] = set() diff --git a/tests/test_sync.py b/tests/test_sync.py index e712609..3f1c4c8 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -534,6 +534,35 @@ async def test_missing_access_level_treated_as_non_public(self, mock_db): assert len(mock_db.file_in_collection.docs) == 0 assert len(mock_db.file.docs) == 0 + @pytest.mark.asyncio + async def test_file_matched_via_persistent_id_fallback(self, mock_db): + """ + GIVEN a non-public dataset whose files carry the dataset DOI as their + own persistent_id, with no file_in_collection entry at all -- + the shape HuBMAP's C2M2 export actually produces, since it + never populates file_in_collection and the materializer falls + back to matching file.persistent_id against + collection.persistent_id (materialize/src/main.rs) + WHEN _prune_non_public_hubmap_raw_records is called + THEN the file is deleted via the persistent_id fallback match, even + though the file_in_collection-based path finds nothing + """ + dataset_metadata = { + "https://doi.org/protected": {"data_access_level": "protected"}, + } + + mock_db.collection.docs = [ + {"submission": "hubmap", "persistent_id": "https://doi.org/protected", "id_namespace": "ns", "local_id": "prot-coll"}, + ] + # No file_in_collection entries at all -- HuBMAP never populates this table. + mock_db.file.docs = [ + {"submission": "hubmap", "id_namespace": "ns", "local_id": "f1", "persistent_id": "https://doi.org/protected"}, + ] + + await _prune_non_public_hubmap_raw_records(dataset_metadata) + + assert len(mock_db.file.docs) == 0 + # --------------------------------------------------------------------------- # _enrich_hubmap_collections_and_subjects From 22e1c73452b028d3ebb541e0d72b646abca0d4a8 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:26:55 -0400 Subject: [PATCH 03/14] fix: Boot a clean checkout to an empty, indexed database The mongodb image baked the sample dump in with COPY database/, but that directory has been gitignored since the 4DN backup was removed from the repo, so on a clean checkout the COPY failed the build outright. The build only appeared to work on machines where a stray file such as .DS_Store happened to keep the directory alive. The dump is now mounted read-only at run time instead: the image no longer varies with whatever a developer has on disk, and obtaining a dump later costs a container restart rather than a rebuild. A missing or empty database/ degrades to "empty database" with a pointer at /sync, while a dump that IS present and fails to restore still stops the container rather than leaving a half-loaded database looking healthy. An empty database then surfaced a second bootstrap break: getIndexes raises NamespaceNotFound on a collection that has never been written to, which under set -e took the whole container down. A namespace that does not exist trivially holds no conflicting index, so the helper treats it as the empty list and falls through to createIndex, which creates the collection along with the index. Only that one error code is swallowed -- anything else still propagates. --- Dockerfile.mongodb | 37 ++++- Makefile | 7 +- README.md | 8 +- scripts/create-indexes.js | 32 +++- tests/integration/test_mongo_bootstrap.py | 193 ++++++++++++++++++++++ tests/test_indexes.py | 45 +++++ 6 files changed, 313 insertions(+), 9 deletions(-) create mode 100644 tests/integration/test_mongo_bootstrap.py diff --git a/Dockerfile.mongodb b/Dockerfile.mongodb index 2dbe282..c2fd6c3 100644 --- a/Dockerfile.mongodb +++ b/Dockerfile.mongodb @@ -1,9 +1,26 @@ FROM mongo:latest -# Copy database dump and scripts -COPY database/ /data/database/ COPY scripts/create-indexes.js /scripts/create-indexes.js +# The sample dump is NOT baked into the image. It used to be — `COPY +# database/ /data/database/` — but `database/` is gitignored (the 4DN +# backup was removed from the repo in 2fc98f8), so on a clean checkout +# the directory does not exist at all and the COPY failed the build +# outright with `"/database": not found`. The build only appeared to +# work on machines where a stray file such as `.DS_Store` happened to +# keep the directory alive. +# +# Mounting the dump at run time instead (see the `mongodb` target in the +# Makefile) fixes that and is the better shape regardless: the image no +# longer varies with whatever a developer happens to have on disk, and +# obtaining a dump later is a container restart rather than a rebuild. +# The directory is created here so `mongorestore` always has a valid +# target — it exits 1 on a path that does not exist, which under +# `set -e` below would take the container down — and so running the +# image with no mount at all degrades to "empty database" rather than +# to a crash. +RUN mkdir -p /data/database + # Create startup script (development mode only) COPY <<'EOF' /startup.sh #!/bin/bash @@ -22,9 +39,19 @@ until mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; do done echo "MongoDB started." -# Restore database -echo "Restoring database..." -mongorestore --gzip /data/database +# Restore the dump, if one was mounted. `set -e` is deliberately kept: +# a failed restore of a dump that IS present should stop the container +# rather than leave a half-loaded database looking healthy. The empty +# case is not a failure though, so it is reported as such instead of +# being left to mongorestore's "0 document(s) restored" trace, which +# reads like something went wrong. +if [ -n "$(ls -A /data/database 2>/dev/null)" ]; then + echo "Restoring database from /data/database..." + mongorestore --gzip /data/database +else + echo "No dump at /data/database — starting with an empty database." + echo "Populate it with 'curl -X POST http://localhost:8000/sync'." +fi # Create indexes echo "Creating indexes..." diff --git a/Makefile b/Makefile index 79b0abb..a163976 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,13 @@ mongodb: @docker rm mongodb 2>/dev/null || true @echo "Building MongoDB image..." docker build -t cfdb-mongodb -f Dockerfile.mongodb . + @# database/ is gitignored and empty on a clean checkout. It is a + @# drop-in point for an optional mongodump, mounted read-only rather + @# than baked into the image (see Dockerfile.mongodb). Created here so + @# the mount source always exists and so the drop-in point is visible. + @mkdir -p database @echo "Starting MongoDB container..." - docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 cfdb-mongodb + docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 -v "$(CURDIR)/database:/data/database:ro" cfdb-mongodb @echo "MongoDB container starting on port 27017. Check logs with: docker logs -f mongodb" build-materialize: diff --git a/README.md b/README.md index f29c260..4d60761 100644 --- a/README.md +++ b/README.md @@ -35,13 +35,13 @@ Requires Python 3.11 or later. ### Docker Startup ```bash -# 1. Start MongoDB (restores sample data and creates indexes) +# 1. Start MongoDB (creates indexes, and restores a dump if one is present) make mongodb # 2. Start the API server make api -# 3. (Optional) Sync latest DCC metadata +# 3. Populate the database curl -X POST http://localhost:8000/sync ``` @@ -49,6 +49,10 @@ This starts: - MongoDB on port 27017 (with indexes) - GraphQL/REST API on port 8000 +**The database starts empty.** A 4DN sample dump used to be committed under `database/`, but it was removed and the directory gitignored, so a clean checkout has no data to restore and step 3 is how you get some — it is not optional. `make mongodb` still creates every index, so an empty database is a working one rather than a broken one. + +`database/` remains the drop-in point if you do have a dump: put a `mongodump --gzip` tree there (`database/cfdb/*.bson.gz`) and it is restored on the next `make mongodb`. It is mounted read-only at run time rather than baked into the image, so obtaining a dump later costs a container restart and not a rebuild. + ### Production (TLS/X.509) ```bash diff --git a/scripts/create-indexes.js b/scripts/create-indexes.js index c7ebff3..7ee6614 100644 --- a/scripts/create-indexes.js +++ b/scripts/create-indexes.js @@ -17,7 +17,37 @@ function ensureIndex(coll, keys, opts) { throw new Error("ensureIndex requires opts.name"); } var name = opts.name; - var existing = coll.getIndexes(); + // ``getIndexes`` raises NamespaceNotFound (code 26, "ns does not + // exist") when the collection has never been written to. On a + // database with no dump restored — the normal state of a clean + // checkout, since the sample dump is gitignored and no longer + // shipped — that is every collection this script touches. The bare + // ``createIndex`` calls elsewhere in this file are immune because + // ``createIndex`` creates the namespace implicitly; this helper is + // the only code path that reads the existing indexes first, which + // is why ``jobs`` was the single collection that aborted the run + // (and, under ``set -e`` in Dockerfile.mongodb's startup script, + // took the whole container down with it on exit 1). + // + // A namespace that does not exist trivially holds no index that + // could conflict, so treat it as the empty list and fall through to + // ``createIndex``, which creates the collection along with the + // index. This preserves the drop-and-recreate contract exactly: a + // collection with no indexes cannot produce a name match, so the + // IndexOptionsConflict path below is unreachable in this case and + // stays untouched for the case it was written for. Only + // NamespaceNotFound is swallowed — any other server error still + // propagates, so a genuine failure is not masked into a silently + // half-indexed database. + var existing; + try { + existing = coll.getIndexes(); + } catch (e) { + if (e.codeName !== "NamespaceNotFound" && e.code !== 26) { + throw e; + } + existing = []; + } var match = null; for (var i = 0; i < existing.length; i++) { if (existing[i].name === name) { diff --git a/tests/integration/test_mongo_bootstrap.py b/tests/integration/test_mongo_bootstrap.py new file mode 100644 index 0000000..91c8807 --- /dev/null +++ b/tests/integration/test_mongo_bootstrap.py @@ -0,0 +1,193 @@ +"""Clean-checkout boot of the MongoDB image, end to end through docker. + +Reproduces the exact failure shape that once took the container down on a +fresh clone: ``database/`` is gitignored, so a clean checkout has no dump +to restore, and the startup script's ``mongosh /scripts/create-indexes.js`` +run hit ``NamespaceNotFound`` on the never-written ``jobs`` collection — +which, under ``set -e``, exited the container. The unit-level lockstep +tests pin the JS source with regexes; only a real ``docker build`` + boot +proves the assembled image survives the empty-database path and still +lands the partial-unique mutex index. + +Docker-gated: the whole module skips when the docker CLI is absent or the +daemon is unreachable, mirroring how tool-dependent integration suites +skip without their binaries. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import time +import uuid +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# Generous ceilings: the first build may pull mongo:latest, and mongod plus +# mongorestore-skip plus index creation can take a while on a cold machine. +BUILD_TIMEOUT_S = 600 +BOOT_DEADLINE_S = 180 +SETTLE_SECONDS = 3 + + +def _docker_usable() -> bool: + """Return True when the docker CLI exists and its daemon answers.""" + if shutil.which("docker") is None: + return False + try: + probe = subprocess.run( + ["docker", "info"], + capture_output=True, + timeout=30, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + return probe.returncode == 0 + + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not _docker_usable(), + reason="docker CLI missing or daemon unreachable", + ), +] + + +def _docker(*args: str, timeout: int = 60) -> subprocess.CompletedProcess: + """Run a docker CLI command, capturing text output without raising.""" + return subprocess.run( + ["docker", *args], + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + +def _container_logs(name: str) -> str: + """Return the container's combined stdout/stderr log stream.""" + result = _docker("logs", name) + return result.stdout + result.stderr + + +def _container_running(name: str) -> bool: + """Return True while docker reports the container's state as running.""" + result = _docker("inspect", "--format", "{{.State.Running}}", name) + return result.returncode == 0 and result.stdout.strip() == "true" + + +class TestMongoCleanCheckoutBoot: + def test_should_boot_with_empty_dump_dir_and_create_jobs_indexes( + self, tmp_path + ): + """Test that the MongoDB image survives a clean-checkout boot. + + Given: + The image built from Dockerfile.mongodb, started with an + empty directory mounted read-only at /data/database — the + shape of a clean checkout, where the gitignored dump + directory has nothing to restore. + When: + The container starts and its logs are polled through the + startup script's initialization sequence. + Then: + It should stay running (no set -e exit), report the + "No dump at /data/database" guidance instead of a failed + restore, and hold the jobs partial-unique mutex index that + create-indexes.js lands on the never-written collection. + """ + # Arrange + suffix = uuid.uuid4().hex[:12] + image_tag = f"cfdb-mongodb-boot-test:{suffix}" + container = f"cfdb-mongodb-boot-{suffix}" + empty_dump_dir = tmp_path / "database" + empty_dump_dir.mkdir() + + try: + build = _docker( + "build", + "-f", + str(REPO_ROOT / "Dockerfile.mongodb"), + "-t", + image_tag, + str(REPO_ROOT), + timeout=BUILD_TIMEOUT_S, + ) + assert build.returncode == 0, ( + f"docker build failed:\n{build.stdout}\n{build.stderr}" + ) + + # Act + run = _docker( + "run", + "--detach", + "--name", + container, + "--volume", + f"{empty_dump_dir}:/data/database:ro", + image_tag, + ) + assert run.returncode == 0, ( + f"docker run failed:\n{run.stdout}\n{run.stderr}" + ) + + deadline = time.monotonic() + BOOT_DEADLINE_S + logs = "" + while time.monotonic() < deadline: + logs = _container_logs(container) + if "Development initialization complete" in logs: + break + assert _container_running(container), ( + "container exited before initialization completed " + f"(the pre-fix set -e failure shape); logs:\n{logs}" + ) + time.sleep(1) + else: + pytest.fail( + "initialization did not complete within " + f"{BOOT_DEADLINE_S}s; logs:\n{logs}" + ) + + # Assert + assert "No dump at /data/database" in logs + assert "Restoring database from /data/database" not in logs + + # A brief settle then a running re-check: the pre-fix failure + # was an exit 1 at the end of the startup script, so "still + # up after initialization" is the behavior under test. + time.sleep(SETTLE_SECONDS) + assert _container_running(container), ( + f"container exited after initialization; logs:\n" + f"{_container_logs(container)}" + ) + + indexes = _docker( + "exec", + container, + "mongosh", + "cfdb", + "--quiet", + "--eval", + "JSON.stringify(db.jobs.getIndexes())", + timeout=120, + ) + assert indexes.returncode == 0, ( + f"mongosh index query failed:\n{indexes.stdout}\n" + f"{indexes.stderr}" + ) + specs = { + spec["name"]: spec for spec in json.loads(indexes.stdout) + } + mutex = specs["workflow_key_active_unique"] + assert mutex["key"] == {"workflow_key": 1} + assert mutex["unique"] is True + assert mutex["partialFilterExpression"] == {"active": True} + finally: + _docker("rm", "--force", container) + _docker("rmi", "--force", image_tag) diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 97bfcc9..b7df787 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -415,6 +415,51 @@ def test_create_indexes_js_data_indexes_should_match_python(): assert js_set == py_set +def test_create_indexes_js_ensure_index_should_tolerate_only_namespace_not_found(): + """Test the JS ensureIndex helper swallows nothing but NamespaceNotFound. + + ``getIndexes`` raises NamespaceNotFound (code 26) on a collection + that has never been written to -- every collection on a clean + checkout, since the sample dump is gitignored. The helper must treat + that one error as "no indexes yet" and fall through to createIndex, + while rethrowing anything else so a genuine server failure cannot be + masked into a silently half-indexed database. + + Given: + The committed scripts/create-indexes.js source, comments + stripped. + When: + The ``ensureIndex`` try/catch around ``getIndexes`` is inspected. + Then: + Its catch should rethrow unless the error's codeName is + NamespaceNotFound or its code is 26, fall back to an empty index + list in that case, and be the file's only catch block. + """ + # Arrange + code = re.sub(r"//.*", "", _JS) + + # Act + match = re.search( + r"try\s*\{\s*existing\s*=\s*coll\.getIndexes\(\)\s*;\s*\}\s*" + r"catch\s*\(\s*(\w+)\s*\)\s*\{\s*" + r"if\s*\(\s*\1\.codeName\s*!==\s*\"NamespaceNotFound\"\s*" + r"&&\s*\1\.code\s*!==\s*26\s*\)\s*\{\s*" + r"throw\s+\1\s*;\s*\}\s*" + r"existing\s*=\s*\[\]\s*;\s*\}", + code, + ) + + # Assert + assert match is not None, ( + "ensureIndex must catch getIndexes failures, rethrow everything " + "but NamespaceNotFound (codeName or code 26), and fall back to []" + ) + assert code.count("catch") == 1, ( + "the NamespaceNotFound guard should be the only swallowed error " + "in create-indexes.js" + ) + + def _conflict_collection(mocker, *, existing_info: dict, create_side_effect): """Build a Mongo-shaped collection mock for conflict-recovery tests.""" collection = mocker.MagicMock() From 14c5014a22eeea74ef7db4a5498dbfe64e10d6eb Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:27:04 -0400 Subject: [PATCH 04/14] feat: Add a tileset artifact kind to the workflow layer Contact-map tile serving needs a third cached artifact kind alongside data and index: a tileset is a byte-identical local copy of the upstream .mcool, opened directly with h5py rather than streamed back over HTTP. This adds ArtifactKind.TILESET and the cache-key plumbing that keys it the same way as the other two, so the existing content-addressed cache invalidation (upstream md5, processor version) applies to it for free. --- src/cfdb/workflows/__init__.py | 19 +++++ src/cfdb/workflows/cache.py | 14 ++++ src/cfdb/workflows/models.py | 7 ++ tests/test_workflows/test_cache.py | 130 +++++++++++++++++++++++++++++ tests/test_workflows/test_init.py | 96 +++++++++++++++++++++ tests/test_workflows/test_keys.py | 21 +++-- 6 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 tests/test_workflows/test_init.py diff --git a/src/cfdb/workflows/__init__.py b/src/cfdb/workflows/__init__.py index 12ae181..27f5aed 100644 --- a/src/cfdb/workflows/__init__.py +++ b/src/cfdb/workflows/__init__.py @@ -60,6 +60,15 @@ retry scheduler re-attempts dispatch for a queued job awaiting capacity; a small random jitter is added per attempt. Default ``120`` (2 min). + +Matrix tile serving (issue #82): + +- ``CFDB_TILESET_MAX_SOURCE_BYTES`` — refuse to materialize a contact map + larger than this, before downloading a byte. Default ``21474836480`` + (20 GiB); ``0`` disables the guard. The tileset artifact for an + ``.mcool`` is a copy of the upstream file, so an oversized or + mislabelled source would otherwise fill the worker's disk and fail the + job somewhere less legible than at admission. """ from __future__ import annotations @@ -200,3 +209,13 @@ def _positive_int(name: str, value: str, *, minimum: int = 0) -> int: os.getenv("CFDB_WORKFLOW_RETRY_INTERVAL_S", "120"), minimum=1, ) + +# --- Matrix tile serving (issue #82) ---------------------------------------- + +# Upper bound on the source of a tileset artifact, checked before the +# download starts. ``minimum=0`` because 0 is a valid "no cap" sentinel. +TILESET_MAX_SOURCE_BYTES: Final = _positive_int( + "CFDB_TILESET_MAX_SOURCE_BYTES", + os.getenv("CFDB_TILESET_MAX_SOURCE_BYTES", str(20 * 1024**3)), + minimum=0, +) diff --git a/src/cfdb/workflows/cache.py b/src/cfdb/workflows/cache.py index fc9ebe4..ad4b651 100644 --- a/src/cfdb/workflows/cache.py +++ b/src/cfdb/workflows/cache.py @@ -131,6 +131,20 @@ def __init__(self, root: Path) -> None: def root(self) -> Path: return self._root + def path_for(self, key: str) -> Path: + """Return the on-disk path a key maps to, validated against traversal. + + Public because the tile server needs a real filesystem path: h5py + opens a file, it does not consume a byte stream, so ``get`` is the + wrong shape for it. On this backend the cached artifact already is + a local file, so the tile server can open it in place instead of + copying it — which is also what lets the whole tile subsystem be + unit-tested without S3. + + The path is not guaranteed to exist; call :meth:`head` for that. + """ + return _safe_key_path(self.root, key) + async def head(self, key: str) -> Optional[CacheEntry]: """Return size metadata for a key, or None if absent.""" path = _safe_key_path(self.root, key) diff --git a/src/cfdb/workflows/models.py b/src/cfdb/workflows/models.py index 6156b25..1212679 100644 --- a/src/cfdb/workflows/models.py +++ b/src/cfdb/workflows/models.py @@ -48,6 +48,13 @@ class ArtifactKind(str, Enum): DATA = "data" INDEX = "index" + #: A file prepared for random-access tile reads (issue #82) — for + #: contact maps, a multi-resolution cooler. Unlike DATA and INDEX it is + #: never streamed to a client: /data and /index would hand back bytes, + #: whereas this artifact exists to be opened locally by the tile server + #: and read a 256x256 block at a time. That is why no router serves it + #: and why the tile endpoints 404 rather than 202 when it is absent. + TILESET = "tileset" class JobRecord(BaseModel): diff --git a/tests/test_workflows/test_cache.py b/tests/test_workflows/test_cache.py index 0e866df..451cd5e 100644 --- a/tests/test_workflows/test_cache.py +++ b/tests/test_workflows/test_cache.py @@ -2,6 +2,7 @@ from __future__ import annotations +import tempfile from pathlib import Path import pytest @@ -395,6 +396,135 @@ async def test_get_should_yield_all_bytes_for_full_file_inclusive_range( # Assert assert collected == b"0123456789" + @pytest.mark.asyncio + async def test_path_for_should_return_committed_artifact_path_when_key_present( + self, tmp_path + ): + """Test that path_for agrees with put and head for a stored artifact. + + Given: + A LocalFsCache with an artifact committed via ``put`` under a + nested key. + When: + ``path_for`` is called with that key. + Then: + It should return an existing path whose bytes equal the + committed source and whose size matches what ``head`` + reports. + """ + # Arrange + cache = LocalFsCache(tmp_path) + source = tmp_path / "src" + payload = b"tileset artifact bytes" + _write(source, payload) + await cache.put("4dn/x/tileset/aa-v0", source) + + # Act + path = cache.path_for("4dn/x/tileset/aa-v0") + + # Assert + assert path.exists() + assert path.read_bytes() == payload + entry = await cache.head("4dn/x/tileset/aa-v0") + assert entry is not None + assert path.stat().st_size == entry.size == len(payload) + + @pytest.mark.asyncio + async def test_path_for_should_return_nonexistent_path_when_key_absent( + self, tmp_path + ): + """Test that path_for on an absent key neither raises nor creates. + + Given: + An empty LocalFsCache and a well-formed key that was never + written. + When: + ``path_for`` is called with that key. + Then: + It should return a non-existent path under the cache root + without raising or creating anything on disk, and ``head`` + should still miss. + """ + # Arrange + cache = LocalFsCache(tmp_path) + + # Act + path = cache.path_for("encode/x/data/aa-v0") + + # Assert + assert not path.exists() + assert tmp_path.resolve() in path.parents + assert list(tmp_path.rglob("*")) == [] + assert await cache.head("encode/x/data/aa-v0") is None + + @pytest.mark.parametrize( + "key", + ["../escape", "a/../../b", "/absolute", "", "///"], + ids=["dotdot", "nested-dotdot", "absolute", "empty", "slashes-only"], + ) + def test_path_for_should_raise_when_key_is_malformed(self, tmp_path, key): + """Test that path_for rejects traversal-shaped and degenerate keys. + + Given: + A LocalFsCache and a key that is empty, root-collapsing, or + carries a path-traversal segment. + When: + ``path_for`` is called with that key. + Then: + It should raise ValueError so the tile server's second public + door onto the key-to-path resolution cannot escape the cache + root. + """ + # Arrange + cache = LocalFsCache(tmp_path) + + # Act & assert + with pytest.raises(ValueError): + cache.path_for(key) + + @settings( + max_examples=50, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + key=st.one_of( + st.sampled_from( + ["..", "../x", "a/../../b", "/abs", "", "///", "a/..", "./."] + ), + st.text(alphabet="ab./_-", min_size=0, max_size=24), + st.text(min_size=0, max_size=24), + ) + ) + def test_path_for_property_contains_or_rejects_every_key(self, tmp_path, key): + """Test that path_for either rejects a key or contains it under root. + + Given: + An arbitrary Hypothesis-generated text key, biased toward + ``..`` segments, slashes, and empty strings alongside benign + keys. + When: + ``path_for`` is called on a fresh cache root. + Then: + It should either raise ValueError or return a path strictly + under the root, and it should never create anything on disk. + """ + # Arrange — a per-example root because hypothesis reuses tmp_path + # across examples within a single test method. + root = Path(tempfile.mkdtemp(dir=tmp_path)) + cache = LocalFsCache(root) + + # Act + try: + path = cache.path_for(key) + except ValueError: + path = None + + # Assert + if path is not None: + assert root.resolve() in path.parents + assert list(root.rglob("*")) == [] + @settings( max_examples=25, deadline=None, diff --git a/tests/test_workflows/test_init.py b/tests/test_workflows/test_init.py new file mode 100644 index 0000000..7b33bbf --- /dev/null +++ b/tests/test_workflows/test_init.py @@ -0,0 +1,96 @@ +"""Tests for env-knob parsing at ``cfdb.workflows`` import time. + +Only ``CFDB_TILESET_MAX_SOURCE_BYTES`` is exercised through a module +reload: its ``minimum=0`` no-cap sentinel is unique among the workflow +knobs, so a copy-paste refactor onto the ``minimum=1`` shape the runtime +caps use would silently break the documented "0 disables the guard" +contract. The reload pattern mirrors ``test_urlsafe.py``: mutate the env +via monkeypatch, reload the module to re-run its import-time parsing, +and reload again under a clean env in a ``finally`` so no other test +sees the mutated constants. +""" + +from __future__ import annotations + +import importlib + +import pytest + +import cfdb.workflows as workflows_module + +_TWENTY_GIB = 20 * 1024**3 + + +def test_tileset_max_source_bytes_should_parse_zero_as_no_cap_sentinel(monkeypatch): + """Test that "0" survives parsing as the documented no-cap sentinel. + + Given: + ``CFDB_TILESET_MAX_SOURCE_BYTES`` set to ``"0"`` and the module + reloaded so the value is re-parsed. + When: + ``TILESET_MAX_SOURCE_BYTES`` is read from the reloaded module. + Then: + It should equal 0 — the unique ``minimum=0`` sentinel that + disables the source-size guard rather than rejecting it. + """ + # Arrange + monkeypatch.setenv("CFDB_TILESET_MAX_SOURCE_BYTES", "0") + try: + # Act + reloaded = importlib.reload(workflows_module) + + # Assert + assert reloaded.TILESET_MAX_SOURCE_BYTES == 0 + finally: + # Restore the default module state. + monkeypatch.delenv("CFDB_TILESET_MAX_SOURCE_BYTES", raising=False) + importlib.reload(workflows_module) + + +def test_tileset_max_source_bytes_should_default_to_twenty_gib_when_unset(monkeypatch): + """Test that an unset env var yields the documented 20 GiB default. + + Given: + ``CFDB_TILESET_MAX_SOURCE_BYTES`` absent from the environment and + the module reloaded. + When: + ``TILESET_MAX_SOURCE_BYTES`` is read from the reloaded module. + Then: + It should equal 21474836480 (20 GiB), the README-documented + default cap. + """ + # Arrange + monkeypatch.delenv("CFDB_TILESET_MAX_SOURCE_BYTES", raising=False) + try: + # Act + reloaded = importlib.reload(workflows_module) + + # Assert + assert reloaded.TILESET_MAX_SOURCE_BYTES == _TWENTY_GIB + finally: + # Restore the default module state. + importlib.reload(workflows_module) + + +def test_tileset_max_source_bytes_should_raise_when_negative(monkeypatch): + """Test that a negative cap fails fast at import with the var named. + + Given: + ``CFDB_TILESET_MAX_SOURCE_BYTES`` set to ``"-1"``. + When: + The module is reloaded so import-time parsing re-runs. + Then: + It should raise ValueError naming + ``CFDB_TILESET_MAX_SOURCE_BYTES`` so a misconfigured deployment + fails at boot rather than admitting every source. + """ + # Arrange + monkeypatch.setenv("CFDB_TILESET_MAX_SOURCE_BYTES", "-1") + try: + # Act & assert + with pytest.raises(ValueError, match="CFDB_TILESET_MAX_SOURCE_BYTES"): + importlib.reload(workflows_module) + finally: + # Restore the default module state. + monkeypatch.delenv("CFDB_TILESET_MAX_SOURCE_BYTES", raising=False) + importlib.reload(workflows_module) diff --git a/tests/test_workflows/test_keys.py b/tests/test_workflows/test_keys.py index 6675a3d..a8336a6 100644 --- a/tests/test_workflows/test_keys.py +++ b/tests/test_workflows/test_keys.py @@ -315,22 +315,27 @@ def test_cache_key_should_return_artifact_scoped_key(self): assert key == f"encode/ENCFF123ABC/data/{FIXTURE_MD5}-v2" def test_cache_key_should_differ_between_artifact_kinds(self): - """Test that data and index artifact keys are distinct. + """Test that every artifact kind yields a distinct cache key. Given: - Two calls identical except for artifact_kind. + Calls identical except for artifact_kind, one per member of + ``ArtifactKind`` (DATA, INDEX, and TILESET). When: - cache_key is called for DATA and INDEX. + cache_key is called for each kind. Then: - It should return distinct keys so that the caches for the two - artifact kinds never alias. + It should return pairwise-distinct keys so that no two + artifact kinds — including the tileset artifact added for + matrix tile serving — ever alias the same cache slot. """ # Act - data_key = cache_key("encode", "x", ArtifactKind.DATA, FIXTURE_MD5, 0) - index_key = cache_key("encode", "x", ArtifactKind.INDEX, FIXTURE_MD5, 0) + keys = { + kind: cache_key("encode", "x", kind, FIXTURE_MD5, 0) + for kind in ArtifactKind + } # Assert - assert data_key != index_key + assert ArtifactKind.TILESET in keys + assert len(set(keys.values())) == len(ArtifactKind) def test_cache_key_should_raise_when_md5_empty(self): """Test that cache_key rejects an empty md5. From 6af9ad02a5a18af8eaca7db8b7aba4dd28ae9d34 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:27:12 -0400 Subject: [PATCH 05/14] feat: Build tileset artifacts with a matrix processor Adds MatrixTilesetProcessor, the workflow-layer piece that turns an upstream .mcool or .cool into a servable tileset artifact: an .mcool is fetched byte-identical into the cache, while a flat .cool is coarsened with cooler zoomify first, since clodius refuses a cooler with no resolutions group. Guards against an oversized or mislabelled source: CFDB_TILESET_MAX_SOURCE_BYTES is enforced before a byte is downloaded, including sources that report their size as a string rather than an int, so a bad record fails admission instead of filling the worker's disk. --- src/cfdb/workflows/processors/matrix.py | 239 ++++++ .../test_workflows/test_processors_matrix.py | 737 ++++++++++++++++++ 2 files changed, 976 insertions(+) create mode 100644 src/cfdb/workflows/processors/matrix.py create mode 100644 tests/test_workflows/test_processors_matrix.py diff --git a/src/cfdb/workflows/processors/matrix.py b/src/cfdb/workflows/processors/matrix.py new file mode 100644 index 0000000..a230cec --- /dev/null +++ b/src/cfdb/workflows/processors/matrix.py @@ -0,0 +1,239 @@ +"""Materialize a Hi-C contact map into a tile-servable cooler. + +Produces a single artifact kind, :attr:`~cfdb.workflows.models.ArtifactKind.TILESET` +— a multi-resolution cooler that the tile server opens locally and reads +256x256 blocks out of. Unlike DATA and INDEX, it is never streamed to a +client. + +Two source shapes reach this processor: + +- **Already multi-resolution** (the 773 4DN ``.mcool`` files). The upstream + file *is* the artifact; it only needs to be in the cache, on a filesystem + h5py can open. +- **Flat** (the 10 4DN ``.cool`` files). Coarsened into an mcool first, + because ``clodius.tiles_v2.cooler.CoolerTileset`` refuses a cooler with no + ``resolutions`` group. + +``.hic`` never reaches here: it is read in place over HTTP range requests +and so needs no artifact at all. See :mod:`cfdb.tilesets.formats`. +""" + +from __future__ import annotations + +import asyncio +import logging +import math +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any + +from cfdb.tilesets.formats import ( + MatrixSource, + matrix_source_kind, + needs_materialization, +) +from cfdb.workflows import TILESET_MAX_SOURCE_BYTES +from cfdb.workflows.cache import CacheBackend +from cfdb.workflows.events import Complete, StageComplete, WorkflowEvent +from cfdb.workflows.fetcher import download_source +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors.base import Processor + +logger = logging.getLogger(__name__) + +#: HiGlass tile dimension, in bins. The coarsest resolution worth +#: generating is the one that fits the whole genome into a single tile; +#: past that a zoom level adds nothing. Matches ``cooler.cli.zoomify``'s +#: ``HIGLASS_TILE_DIM`` and ``clodius.tiles_v2.cooler.TILE_SIZE``. +TILE_DIM = 256 + +#: Pixels processed per chunk during coarsening. Mirrors the +#: ``cooler zoomify`` CLI default. +ZOOMIFY_CHUNKSIZE = int(10e6) + + +def _has_resolutions_group(path: Path) -> bool: + """True when ``path`` is already a multi-resolution cooler. + + The filename is not authoritative — flat files named ``.mcool`` and + multi-resolution files named ``.cool`` both exist upstream — so the + coarsening decision is made by looking inside the container. This is + the same group ``CoolerTileset.file`` checks for before serving. + """ + import h5py + + with h5py.File(path, "r") as handle: + return "resolutions" in handle + + +def _zoom_ladder(path: Path) -> list[int]: + """Resolutions to coarsen a flat cooler onto. + + Reproduces what ``cooler zoomify`` computes for its default binary + progression: geometric steps of 2 from the base bin size up to the + resolution at which the whole genome fits in one HiGlass tile. + Deriving it here rather than passing ``resolutions=None`` is required + — ``cooler.zoomify_cooler`` takes the list as a mandatory argument. + """ + import cooler + from cooler.cli.zoomify import preferred_sequence + + clr = cooler.Cooler(str(path)) + binsize = clr.binsize + if not binsize: + # A variable-length bin table (restriction-fragment cooler) has no + # single base resolution to double from, so the standard ladder is + # undefined. Refuse rather than emit a ladder that misrepresents + # the bins. + raise RuntimeError( + "cannot build a zoom ladder for a cooler with variable-length " + "bins; only fixed-binsize coolers can be coarsened" + ) + + genome_length = int(clr.chromsizes.sum()) + coarsest = int(math.ceil(genome_length / TILE_DIM)) + ladder = preferred_sequence(binsize, coarsest, "binary") + + # ``preferred_sequence`` returns [] when the base resolution is + # already coarser than the whole-genome tile — a small genome, which + # every test fixture is. The base resolution is still a valid (and the + # only) zoom level. + return ladder or [binsize] + + +def _zoomify(source: Path, dest: Path) -> None: + """Coarsen a flat cooler at ``source`` into an mcool at ``dest``. + + Runs the Python API rather than shelling out to ``cooler zoomify``. + The console script is only incidentally on ``PATH`` — a consequence of + a transitive install rather than something either Dockerfile declares + — so a missing entry point would surface at job time as an opaque + ``FileNotFoundError`` instead of an ImportError at startup. The API + also raises real exceptions instead of stderr that has to be parsed. + """ + import cooler + + resolutions = _zoom_ladder(source) + logger.info(f"Coarsening {source.name} onto resolutions {resolutions}") + cooler.zoomify_cooler( + str(source), + str(dest), + resolutions, + ZOOMIFY_CHUNKSIZE, + # One worker: CFDB_WORKER_MAX_CONCURRENT_TASKS defaults to 1 on a + # 1-vCPU worker, so a process pool would only contend with itself. + nproc=1, + ) + + +class MatrixTilesetProcessor(Processor): + """Prepare a contact map for random-access tile reads.""" + + processor_version = 0 + + # Every contact-map container collapses onto this one EDAM term + # upstream, alongside AnnData and bare HDF5. Claiming the whole term + # here is safe because the real discrimination happens in + # ``needs_processing`` / ``artifact_kinds_produced`` below, and every + # consumer of ``ProcessorRegistry.lookup_for`` consults one or the + # other before acting. + supported_formats = frozenset({"HDF5"}) + artifact_kinds = (ArtifactKind.TILESET,) + + def needs_processing(self, file_meta: dict[str, Any]) -> bool: + """True only for coolers — the formats that need an artifact built. + + Returns False for ``.h5ad`` and bare ``.h5`` (not contact maps at + all) and for ``.hic`` (a contact map, but one served in place over + HTTP range requests). In every one of those cases the callers of + ``lookup_for`` fall through exactly as they did before this + processor existed. + """ + return needs_materialization(file_meta) + + def artifact_kinds_produced( + self, file_meta: dict[str, Any] | None = None + ) -> tuple[ArtifactKind, ...]: + """Artifact kinds this processor writes for ``file_meta``. + + Overridden because ``/index`` asks this question *without* first + consulting :meth:`needs_processing` — it uses an empty result to + mean "this format has no index in any state of the world". Without + the override, an ``.h5ad`` would advertise a TILESET it will never + produce. + """ + if file_meta is None: + return self.artifact_kinds + return self.artifact_kinds if needs_materialization(file_meta) else () + + async def run( + self, + file_meta: dict[str, Any], + workdir: Path, + cache: CacheBackend, + ) -> AsyncIterator[WorkflowEvent]: + """Fetch the contact map, coarsen it if flat, and cache it.""" + workdir.mkdir(parents=True, exist_ok=True) + tileset_key = self.cache_key_for(file_meta, ArtifactKind.TILESET) + + if await cache.head(tileset_key) is None: + artifact = await self._materialize(file_meta, workdir) + await cache.put(tileset_key, artifact) + yield StageComplete(kind=ArtifactKind.TILESET, key=tileset_key) + + yield Complete(artifacts={ArtifactKind.TILESET.value: tileset_key}) + + async def _materialize( + self, file_meta: dict[str, Any], workdir: Path + ) -> Path: + """Produce the multi-resolution cooler to be committed.""" + self._assert_source_within_size_cap(file_meta) + + source = workdir / "source.h5" + await download_source(file_meta, source) + + if await asyncio.to_thread(_has_resolutions_group, source): + # Already a pyramid. The upstream bytes are the artifact. + return source + + if matrix_source_kind(file_meta) is MatrixSource.MCOOL: + logger.info( + "%s is named .mcool but carries no resolutions group; " + "coarsening it as a flat cooler", + file_meta.get("filename"), + ) + + out = workdir / "tileset.mcool" + await asyncio.to_thread(_zoomify, source, out) + return out + + def _assert_source_within_size_cap(self, file_meta: dict[str, Any]) -> None: + """Refuse an oversized source before a byte is downloaded. + + The TILESET artifact for an ``.mcool`` is a copy of the upstream + file, so an oversized source — or a mislabelled one, such as a + multi-tens-of-GB ``.hic`` that reached this processor by a filename + that lies — would fill the worker's disk and fail somewhere far + less legible than here. + """ + if not TILESET_MAX_SOURCE_BYTES: + return + size = file_meta.get("size_in_bytes") + if isinstance(size, str): + # 4DN and HuBMAP load size_in_bytes as a string through the + # C2M2 TSV path (the README's BigInt note), and 4DN is the + # only DCC whose contact maps reach this processor — an + # int-only check leaves the cap inert for exactly the corpus + # it exists to bound. + try: + size = int(size) + except ValueError: + size = None + if not isinstance(size, int) or size <= TILESET_MAX_SOURCE_BYTES: + return + raise RuntimeError( + f"{file_meta.get('filename') or file_meta.get('local_id')} is " + f"{size} bytes, above the " + f"{TILESET_MAX_SOURCE_BYTES}-byte tileset source cap " + "(CFDB_TILESET_MAX_SOURCE_BYTES); refusing to materialize it" + ) diff --git a/tests/test_workflows/test_processors_matrix.py b/tests/test_workflows/test_processors_matrix.py new file mode 100644 index 0000000..bfd1b29 --- /dev/null +++ b/tests/test_workflows/test_processors_matrix.py @@ -0,0 +1,737 @@ +"""Tests for :class:`cfdb.workflows.processors.matrix.MatrixTilesetProcessor`.""" + +from __future__ import annotations + +import uuid +from pathlib import Path, PurePosixPath + +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from cfdb.workflows.cache import LocalFsCache +from cfdb.workflows.events import Complete, StageComplete +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors import matrix as matrix_module +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from tests.fixtures.coolers import build_cool, build_variable_bin_cool +from tests.test_workflows import FIXTURE_MD5 + + +def _file_doc(filename: str, **overrides) -> dict: + """Return a projected file document for an HDF5-labelled file.""" + doc = { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": FIXTURE_MD5, + "filename": filename, + "file_format": {"name": "HDF5"}, + "access_url": "https://data.4dnucleome.org/files/" + filename, + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + doc.update(overrides) + return doc + + +async def _drain(processor, file_doc, workdir, cache) -> list: + """Collect the processor's whole event stream.""" + return [event async for event in processor.run(file_doc, workdir, cache)] + + +def _resolutions_of(path: Path) -> list[int]: + """Read the sorted resolution ladder out of a committed artifact.""" + import h5py + + with h5py.File(path, "r") as handle: + if "resolutions" not in handle: + return [] + return sorted(int(name) for name in handle["resolutions"]) + + +#: Suffixes sharing the HDF5 EDAM term upstream, plus non-claimed noise. +_SUFFIX_POOL = (".mcool", ".cool", ".hic", ".h5ad", ".h5", ".txt", "") + + +@st.composite +def _hdf5_filenames(draw) -> str: + """Generate a filename from a free-text stem and a randomly-cased suffix.""" + stem = draw(st.text(max_size=16)) + suffix = draw(st.sampled_from(_SUFFIX_POOL)) + flips = draw( + st.lists(st.booleans(), min_size=len(suffix), max_size=len(suffix)) + ) + cased = "".join( + char.upper() if flip else char.lower() + for char, flip in zip(suffix, flips) + ) + return stem + cased + + +class TestApplicability: + """Which HDF5 files this processor claims, and what it advertises.""" + + @pytest.mark.parametrize( + ("filename", "expected"), + [ + ("sample.mcool", True), + ("sample.cool", True), + ("sample.hic", False), + ("matrix.h5ad", False), + ("raw.h5", False), + ], + ) + def test_needs_processing_should_claim_only_coolers(self, filename, expected): + """Test that only the formats needing an artifact are claimed. + + Given: + An HDF5-labelled document of each kind that shares the EDAM + term. + When: + needs_processing is called. + Then: + Only coolers should be claimed. ``.hic`` is a contact map but + is read in place, and AnnData is not a contact map at all. + """ + # Act + result = MatrixTilesetProcessor().needs_processing(_file_doc(filename)) + + # Assert + assert result is expected + + @pytest.mark.parametrize( + ("filename", "expected"), + [ + ("sample.mcool", (ArtifactKind.TILESET,)), + ("sample.cool", (ArtifactKind.TILESET,)), + ("sample.hic", ()), + ("matrix.h5ad", ()), + ], + ) + def test_artifact_kinds_produced_should_be_empty_for_unclaimed_files( + self, filename, expected + ): + """Test that an unclaimed file advertises no artifacts. + + Given: + An HDF5-labelled document of each kind. + When: + artifact_kinds_produced is called. + Then: + Only coolers should advertise a TILESET. ``/index`` asks this + question without first consulting needs_processing and reads + an empty result as "this format has no index in any state of + the world", so an .h5ad advertising a TILESET it will never + produce would be a lie with a visible consequence. + """ + # Act + result = MatrixTilesetProcessor().artifact_kinds_produced( + _file_doc(filename) + ) + + # Assert + assert result == expected + + def test_artifact_kinds_produced_should_fall_back_without_a_document(self): + """Test the generic-introspection call still answers. + + Given: + No file document, as a caller doing generic introspection + would pass. + When: + artifact_kinds_produced is called. + Then: + It should report the static class-level tuple. + """ + # Act + result = MatrixTilesetProcessor().artifact_kinds_produced() + + # Assert + assert result == (ArtifactKind.TILESET,) + + def test_cache_key_should_carry_the_tileset_kind_and_version(self): + """Test the artifact lands under a versioned, content-addressed key. + + Given: + A cooler document with an md5. + When: + cache_key_for is called for the TILESET kind. + Then: + The key should embed the DCC, local id, kind, md5, and + processor version — so an upstream byte change or a version + bump invalidates both the artifact and every tile cached + against it. + """ + # Act + key = MatrixTilesetProcessor().cache_key_for( + _file_doc("sample.mcool"), ArtifactKind.TILESET + ) + + # Assert — the DCC segment comes from ``dcc.dcc_abbreviation`` + # (normalized), not ``submission``; ``extract_identity`` is the + # single authority and every artifact kind shares its convention. + assert key == f"4dn_dcic/4DNFIABC123/tileset/{FIXTURE_MD5}-v0" + + @settings(max_examples=50) + @given(filename=_hdf5_filenames()) + def test_needs_processing_should_agree_with_artifact_kinds_produced( + self, filename + ): + """Test classification consistency over arbitrary HDF5 filenames. + + Given: + HDF5-labelled documents whose filenames combine free-text + stems with contact-map and non-contact-map suffixes under + random casing. + When: + needs_processing and artifact_kinds_produced are called. + Then: + It should never raise, claim exactly the filenames whose + final suffix is a cooler regardless of casing, and advertise + ``(TILESET,)`` if and only if the file is claimed — the + consistency ``/index`` depends on. + """ + # Arrange + doc = _file_doc(filename) + claimed = PurePosixPath(filename.lower()).suffix in {".mcool", ".cool"} + + # Act + needs = MatrixTilesetProcessor().needs_processing(doc) + kinds = MatrixTilesetProcessor().artifact_kinds_produced(doc) + + # Assert + assert needs is claimed + assert kinds == ((ArtifactKind.TILESET,) if needs else ()) + + +#: Sentinel distinguishing "size_in_bytes key absent" from "present as None". +_ABSENT = object() + + +class TestSourceSizeGuard: + """An oversized source is refused before a byte is downloaded.""" + + @pytest.mark.asyncio + async def test_should_refuse_a_source_above_the_cap_without_downloading( + self, mocker, tmp_path + ): + """Test that the guard runs ahead of the fetch. + + Given: + A cooler whose reported size exceeds the configured cap. + When: + The processor runs. + Then: + It should raise before calling download_source. The artifact + for an .mcool is a copy of the upstream file, so an oversized + or mislabelled source would otherwise fill the worker's disk + and fail somewhere far less legible. + """ + # Arrange + mocker.patch.object(matrix_module, "TILESET_MAX_SOURCE_BYTES", 1024) + download = mocker.patch.object(matrix_module, "download_source") + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("huge.mcool", size_in_bytes=2048) + + # Act + with pytest.raises(RuntimeError, match="tileset source cap"): + await _drain(MatrixTilesetProcessor(), doc, tmp_path / "wd", cache) + + # Assert + download.assert_not_called() + + @pytest.mark.asyncio + async def test_should_not_apply_the_guard_when_it_is_disabled( + self, mocker, tmp_path, tiny_mcool + ): + """Test that a zero cap means no cap. + + Given: + ``CFDB_TILESET_MAX_SOURCE_BYTES`` set to 0 and a source + reporting a very large size. + When: + The processor runs. + Then: + It should proceed to the download rather than refusing. + """ + # Arrange + mocker.patch.object(matrix_module, "TILESET_MAX_SOURCE_BYTES", 0) + mocker.patch.object( + matrix_module, + "download_source", + side_effect=_stage_copy(tiny_mcool), + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("huge.mcool", size_in_bytes=10**15) + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + assert isinstance(events[-1], Complete) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "size", + [1024, None, _ABSENT], + ids=["int-exactly-at-cap", "none", "absent"], + ) + async def test_should_admit_a_source_at_the_cap_or_of_unknown_size( + self, mocker, tmp_path, tiny_mcool, size + ): + """Test the cap's inclusive boundary and its permissive default. + + Given: + A 1024-byte cap and documents whose ``size_in_bytes`` is + exactly 1024, present as None, or absent entirely. + When: + The processor runs on each. + Then: + It should proceed to the download every time — the cap is + inclusive, and an unknown size admits rather than refuses. + """ + # Arrange + mocker.patch.object(matrix_module, "TILESET_MAX_SOURCE_BYTES", 1024) + download = mocker.patch.object( + matrix_module, + "download_source", + side_effect=_stage_copy(tiny_mcool), + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("ok.mcool") + if size is not _ABSENT: + doc["size_in_bytes"] = size + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + download.assert_called_once() + assert isinstance(events[-1], Complete) + + @pytest.mark.asyncio + async def test_should_refuse_a_string_size_above_the_cap( + self, mocker, tmp_path + ): + """Test that the guard coerces the 4DN string shape. + + Given: + A 1024-byte cap and a document reporting ``size_in_bytes`` as + the string ``"2048"`` — the shape 4DN's C2M2 TSV ingest + actually stores, per the README's BigInt note. + When: + The processor runs. + Then: + It should refuse before calling download_source. An int-only + check would leave the cap inert for exactly the DCC whose + contact maps reach this processor. + """ + # Arrange + mocker.patch.object(matrix_module, "TILESET_MAX_SOURCE_BYTES", 1024) + download = mocker.patch.object(matrix_module, "download_source") + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("huge.mcool", size_in_bytes="2048") + + # Act + with pytest.raises(RuntimeError, match="tileset source cap"): + await _drain(MatrixTilesetProcessor(), doc, tmp_path / "wd", cache) + + # Assert + download.assert_not_called() + + @pytest.mark.asyncio + async def test_should_admit_a_source_whose_size_string_is_not_numeric( + self, mocker, tmp_path, tiny_mcool + ): + """Test that an unparseable size degrades to admission. + + Given: + A 1024-byte cap and a document whose ``size_in_bytes`` is a + string that does not parse as an integer. + When: + The processor runs. + Then: + It should proceed to the download — a size the guard cannot + read is treated as unknown, not as a failure. + """ + # Arrange + mocker.patch.object(matrix_module, "TILESET_MAX_SOURCE_BYTES", 1024) + download = mocker.patch.object( + matrix_module, + "download_source", + side_effect=_stage_copy(tiny_mcool), + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("odd.mcool", size_in_bytes="not-a-number") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + download.assert_called_once() + assert isinstance(events[-1], Complete) + + +def _stage_copy(source: Path): + """Return a download_source stand-in that copies ``source`` into place.""" + + async def _download(file_meta, dest: Path) -> Path: + import shutil + + dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(source, dest) + return dest + + return _download + + +def _stage_bytes(payload: bytes): + """Return a download_source stand-in that writes ``payload`` into place.""" + + async def _download(file_meta, dest: Path) -> Path: + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(payload) + return dest + + return _download + + +class TestRun: + """The single materialization stage.""" + + @pytest.mark.asyncio + async def test_should_cache_a_multi_resolution_cooler_verbatim( + self, mocker, tmp_path, tiny_mcool + ): + """Test that an already-pyramided cooler is not coarsened. + + Given: + An upstream file that already carries a ``resolutions`` group. + When: + The processor runs. + Then: + It should commit the downloaded bytes as the artifact and + never invoke coarsening — the file is already a tile pyramid, + which is the whole reason issue #82 needs no pre-aggregation. + """ + # Arrange + mocker.patch.object( + matrix_module, "download_source", side_effect=_stage_copy(tiny_mcool) + ) + zoomify = mocker.patch.object(matrix_module, "_zoomify") + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.mcool") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + zoomify.assert_not_called() + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + assert await cache.head(key) is not None + assert events == [ + StageComplete(kind=ArtifactKind.TILESET, key=key), + Complete(artifacts={"tileset": key}), + ] + + @pytest.mark.asyncio + async def test_should_coarsen_a_flat_cooler_and_cache_the_output( + self, mocker, tmp_path, make_cool + ): + """Test that a flat cooler is coarsened before being cached. + + Given: + An upstream ``.cool`` with no ``resolutions`` group — the + shape clodius refuses to serve. + When: + The processor runs. + Then: + It should coarsen it and commit the *coarsened* file, so the + cached artifact is one clodius can open. + """ + # Arrange + flat = make_cool() + mocker.patch.object( + matrix_module, "download_source", side_effect=_stage_copy(flat) + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.cool") + + # Act + await _drain(MatrixTilesetProcessor(), doc, tmp_path / "wd", cache) + + # Assert — the committed artifact carries the ``resolutions`` + # group clodius requires, observed with h5py on the cached file. + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + cached = LocalFsCache(tmp_path / "cache").path_for(key) + assert _resolutions_of(cached) + + @pytest.mark.asyncio + async def test_should_coarsen_a_flat_file_despite_its_mcool_name( + self, mocker, tmp_path, make_cool + ): + """Test that content, not filename, decides the coarsening. + + Given: + An upstream file named ``sample.mcool`` whose bytes are a + flat, single-resolution cooler — the mislabelled shape that + exists in the wild. + When: + The processor runs. + Then: + It should coarsen it anyway and commit an artifact carrying a + ``resolutions`` group, so a lying filename cannot poison the + cache with a file clodius refuses to open. + """ + # Arrange + flat = make_cool() + mocker.patch.object( + matrix_module, "download_source", side_effect=_stage_copy(flat) + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.mcool") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + assert isinstance(events[-1], Complete) + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + cached = LocalFsCache(tmp_path / "cache").path_for(key) + assert _resolutions_of(cached) + + @pytest.mark.asyncio + async def test_should_cache_a_pyramided_file_despite_its_cool_name( + self, mocker, tmp_path, tiny_mcool + ): + """Test that a mislabelled pyramid is committed verbatim. + + Given: + An upstream file named ``sample.cool`` whose bytes already + carry a ``resolutions`` group — the reverse mislabelling. + When: + The processor runs, with coarsening spied at the module + boundary. + Then: + It should never coarsen and commit the downloaded bytes + byte-identical to the source. + """ + # Arrange + mocker.patch.object( + matrix_module, "download_source", side_effect=_stage_copy(tiny_mcool) + ) + zoomify = mocker.spy(matrix_module, "_zoomify") + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.cool") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + assert isinstance(events[-1], Complete) + zoomify.assert_not_called() + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + cached = LocalFsCache(tmp_path / "cache").path_for(key) + assert cached.read_bytes() == tiny_mcool.read_bytes() + + @pytest.mark.asyncio + async def test_should_refuse_a_cooler_with_variable_length_bins( + self, mocker, tmp_path + ): + """Test that a restriction-fragment cooler is refused, not mangled. + + Given: + An upstream flat cooler whose bin table is irregular + (``Cooler.binsize`` is None), so no doubling ladder exists. + When: + The processor runs. + Then: + It should raise a RuntimeError naming the variable-length + bins and commit nothing — a ladder over irregular bins would + misrepresent them. + """ + # Arrange + varbin = build_variable_bin_cool(tmp_path / "fragments.cool") + mocker.patch.object( + matrix_module, "download_source", side_effect=_stage_copy(varbin) + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("fragments.cool") + + # Act + with pytest.raises(RuntimeError, match="variable-length bins"): + await _drain(MatrixTilesetProcessor(), doc, tmp_path / "wd", cache) + + # Assert + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + assert await cache.head(key) is None + + @pytest.mark.asyncio + async def test_should_short_circuit_when_the_artifact_is_already_cached( + self, mocker, tmp_path, tiny_mcool + ): + """Test that a cached artifact is not rebuilt on retry. + + Given: + The TILESET artifact already present in the cache. + When: + The processor runs again, as it does after a crash-and-retry. + Then: + It should download nothing and still emit the full event + stream, so the executor records the artifact either way. + """ + # Arrange + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.mcool") + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + seeded = tmp_path / "seed.mcool" + seeded.write_bytes(tiny_mcool.read_bytes()) + await cache.put(key, seeded) + + download = mocker.patch.object(matrix_module, "download_source") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + download.assert_not_called() + assert events == [ + StageComplete(kind=ArtifactKind.TILESET, key=key), + Complete(artifacts={"tileset": key}), + ] + + @pytest.mark.asyncio + async def test_should_serve_from_cache_before_consulting_the_size_cap( + self, mocker, tmp_path, tiny_mcool + ): + """Test that the cached short-circuit precedes the size guard. + + Given: + The TILESET artifact already cached, and a cap set below the + document's reported source size. + When: + The processor runs again. + Then: + It should complete from the cache without downloading — the + guard exists to protect the worker's disk from a fetch, and + an artifact already committed needs no fetch to protect + against. + """ + # Arrange + mocker.patch.object(matrix_module, "TILESET_MAX_SOURCE_BYTES", 1024) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.mcool", size_in_bytes=10**9) + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + seeded = tmp_path / "seed.mcool" + seeded.write_bytes(tiny_mcool.read_bytes()) + await cache.put(key, seeded) + + download = mocker.patch.object(matrix_module, "download_source") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, tmp_path / "wd", cache + ) + + # Assert + download.assert_not_called() + assert events == [ + StageComplete(kind=ArtifactKind.TILESET, key=key), + Complete(artifacts={"tileset": key}), + ] + + @pytest.mark.asyncio + async def test_should_not_commit_an_artifact_when_the_source_is_not_hdf5( + self, mocker, tmp_path + ): + """Test that a garbage download poisons nothing. + + Given: + download_source staging bytes that are not an HDF5 container + at all — a truncated or misrouted upstream response. + When: + The processor runs. + Then: + It should raise and leave the cache key unwritten, so a + failed build can never be mistaken for a servable artifact. + """ + # Arrange + pytest.importorskip("h5py") + mocker.patch.object( + matrix_module, + "download_source", + side_effect=_stage_bytes(b"this is not an HDF5 container"), + ) + cache = LocalFsCache(tmp_path / "cache") + doc = _file_doc("sample.mcool") + + # Act + with pytest.raises(OSError): + await _drain(MatrixTilesetProcessor(), doc, tmp_path / "wd", cache) + + # Assert + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + assert await cache.head(key) is None + + @pytest.mark.asyncio + @settings( + max_examples=5, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given(binsize=st.sampled_from([2, 4, 6, 10, 1000])) + async def test_should_coarsen_onto_a_doubling_ladder_from_the_base_binsize( + self, mocker, tmp_path, binsize + ): + """Test the resolution ladder a flat cooler is coarsened onto. + + Given: + Flat coolers at sampled base bin sizes over the 3000 bp + fixture genome — including one so coarse the whole genome + already fits a single tile. + When: + The processor runs and the committed artifact's resolutions + are read back with h5py. + Then: + It should start the ladder at the base bin size and double + exactly at every step, never committing an empty ladder — + the same binary progression ``cooler zoomify`` derives, so a + cfdb-materialized mcool is indistinguishable from a + hand-zoomified one. + """ + # Arrange — a fresh root per example, so one example's committed + # artifact cannot short-circuit the next. + root = tmp_path / uuid.uuid4().hex + root.mkdir() + flat = build_cool(root / "flat.cool", binsize=binsize) + mocker.patch.object( + matrix_module, "download_source", side_effect=_stage_copy(flat) + ) + cache = LocalFsCache(root / "cache") + doc = _file_doc("sample.cool") + + # Act + events = await _drain( + MatrixTilesetProcessor(), doc, root / "wd", cache + ) + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + ladder = _resolutions_of(cache.path_for(key)) + + # Assert + assert isinstance(events[-1], Complete) + assert ladder, "a coarsened artifact must carry at least one resolution" + assert ladder[0] == binsize + assert all(b == 2 * a for a, b in zip(ladder, ladder[1:])) From 3022f3d2f98e3ac4858991ddcb615d6f6ef3e6f0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:27:28 -0400 Subject: [PATCH 06/14] feat: Add the tileset read stack Adds cfdb.tilesets: the read path that opens a cached tileset artifact and serves HiGlass tileset_info and tile payloads from it. - backend.py is the single import gate for the clodius tile backend, so a build without the tiles extra degrades to a clear 501 on the tile routes instead of an ImportError at startup. Its tile_error classification is bound to clodius's TilesetError root -- the base of the whole deliberate-error hierarchy, not just the narrower TileError node -- so a parse-time failure (a malformed id) and a read-time failure (an out-of-ladder position, an unusable transform) both degrade to a per-tile error payload rather than a bug. - service.py owns tileset lifetime: a bounded LRU of open h5py handles, reference-counted so an eviction never closes a handle a reader is still using, plus a byte-bounded tile cache. Reads run on a dedicated thread pool since clodius is synchronous h5py + numpy. - formats.py classifies a file as mcool, cool, or hic from its filename and file_format, since the C2M2 ontology collapses all of them onto one HDF5 term. - store.py, wire.py, and errors.py round out the artifact-hydration, HiGlass-payload-assembly, and failure-vocabulary pieces. --- src/cfdb/tilesets/__init__.py | 13 + src/cfdb/tilesets/backend.py | 139 +++ src/cfdb/tilesets/errors.py | 67 + src/cfdb/tilesets/formats.py | 96 ++ src/cfdb/tilesets/service.py | 450 +++++++ src/cfdb/tilesets/store.py | 205 +++ src/cfdb/tilesets/wire.py | 66 + tests/fixtures/__init__.py | 1 + tests/fixtures/coolers.py | 184 +++ tests/fixtures/remote_cache.py | 50 + tests/test_tilesets/__init__.py | 1 + tests/test_tilesets/test_backend.py | 215 ++++ tests/test_tilesets/test_errors.py | 56 + tests/test_tilesets/test_formats.py | 384 ++++++ .../test_hdf5_routing_regression.py | 251 ++++ tests/test_tilesets/test_service.py | 1108 +++++++++++++++++ tests/test_tilesets/test_store.py | 607 +++++++++ tests/test_tilesets/test_wire.py | 157 +++ 18 files changed, 4050 insertions(+) create mode 100644 src/cfdb/tilesets/__init__.py create mode 100644 src/cfdb/tilesets/backend.py create mode 100644 src/cfdb/tilesets/errors.py create mode 100644 src/cfdb/tilesets/formats.py create mode 100644 src/cfdb/tilesets/service.py create mode 100644 src/cfdb/tilesets/store.py create mode 100644 src/cfdb/tilesets/wire.py create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/coolers.py create mode 100644 tests/fixtures/remote_cache.py create mode 100644 tests/test_tilesets/__init__.py create mode 100644 tests/test_tilesets/test_backend.py create mode 100644 tests/test_tilesets/test_errors.py create mode 100644 tests/test_tilesets/test_formats.py create mode 100644 tests/test_tilesets/test_hdf5_routing_regression.py create mode 100644 tests/test_tilesets/test_service.py create mode 100644 tests/test_tilesets/test_store.py create mode 100644 tests/test_tilesets/test_wire.py diff --git a/src/cfdb/tilesets/__init__.py b/src/cfdb/tilesets/__init__.py new file mode 100644 index 0000000..a0fa0c2 --- /dev/null +++ b/src/cfdb/tilesets/__init__.py @@ -0,0 +1,13 @@ +"""Matrix tile serving for Hi-C contact maps (issue #82). + +Layered so that only ``service`` and ``store`` ever touch a tile backend: + +- :mod:`cfdb.tilesets.formats` — which files are contact maps, and of what + kind. Stdlib-only, so the worker-side processor can import it. +- :mod:`cfdb.tilesets.errors` — the failure vocabulary the routers translate + into status codes. +- :mod:`cfdb.tilesets.backend` — the single import gate for clodius. +- :mod:`cfdb.tilesets.store` — cached artifact to local filesystem path. +- :mod:`cfdb.tilesets.service` — open-tileset and tile caches, thread offload. +- :mod:`cfdb.tilesets.wire` — HiGlass response payload assembly. +""" diff --git a/src/cfdb/tilesets/backend.py b/src/cfdb/tilesets/backend.py new file mode 100644 index 0000000..4f88896 --- /dev/null +++ b/src/cfdb/tilesets/backend.py @@ -0,0 +1,139 @@ +"""The single import gate for the clodius tile backend. + +Every import of clodius in cfdb goes through here, so the whole subsystem +has exactly one on/off seam. That matters because clodius is currently an +optional extra installed from a sibling checkout (the fork is private, so +the Dockerfiles cannot fetch it) — which means production images do not +carry it. Concentrating the import lets that degrade to a clear 501 on the +tile routes instead of an ImportError at application startup or, worse, a +subsystem that half-works. + +The gate also resolves ``clodius.tiles_v2.hic`` separately from the cooler +tileset. The module is not part of the pinned build — cfdb does not serve +``.hic`` regardless (see :func:`load_hic_tileset` for why), so the two +capabilities are kept distinct rather than collapsed into one availability +flag. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from cfdb.tilesets.errors import TileBackendUnavailable + + +@dataclass(frozen=True) +class ClodiusBackend: + """The clodius names cfdb uses, resolved once. + + Held as data rather than imported at module scope so that a build + without clodius can still import every cfdb module. + """ + + #: The mcool tileset implementation. + cooler_tileset: type + #: Base of clodius's whole deliberate-error hierarchy (``TilesetError``, + #: not the narrower ``TileError``). Anything deriving from this is a + #: problem cfdb can render per-tile-id — a malformed id, an unusable + #: transform, an out-of-ladder position — rather than a bug; anything + #: else becomes a 500. Named ``tile_error`` for the call sites that + #: consult it, which all classify at tile-id granularity even though + #: some of what it now admits (e.g. ``MalformedTileId``) is raised + #: before a tile is ever read. + tile_error: type[Exception] + #: Raised for a tile position the ladder does not hold. ``CoolerTileset`` + #: catches it internally and renders it as that tile's payload — see + #: :meth:`~cfdb.tilesets.service.TilesetService._read_tiles` — but named + #: here so the contract is visible at the boundary. + tile_out_of_bounds: type[Exception] + + +def load_backend() -> ClodiusBackend: + """Resolve the clodius tile backend. + + Raises: + TileBackendUnavailable: when clodius is not installed. + """ + try: + from clodius.core.errors import TileOutOfBounds, TilesetError + from clodius.tiles_v2.cooler import CoolerTileset + except ImportError as exc: # pragma: no cover - exercised via monkeypatch + raise TileBackendUnavailable( + "Matrix tile serving requires the clodius tile backend, which " + "this build does not carry. Install the 'tiles' extra " + "(`uv sync --extra tiles`, which resolves clodius from the " + "sibling checkout) to enable it." + ) from exc + + return ClodiusBackend( + cooler_tileset=CoolerTileset, + tile_error=TilesetError, + tile_out_of_bounds=TileOutOfBounds, + ) + + +def is_available() -> bool: + """True when tiles can be served at all. + + Used by the lifespan to decide whether to construct the tile service, + and by the preparation channel to answer 503 rather than promising a + readiness it cannot deliver. + """ + try: + load_backend() + except TileBackendUnavailable: + return False + return True + + +def load_hic_tileset() -> type: + """Resolve the ``.hic`` tileset implementation. + + Resolving it is not the same as being able to use it. clodius's + ``.hic`` tileset is built on ``hictkpy``, whose entry points take + ``str | os.PathLike`` and nothing else — there is no URL, HTTP, or S3 + surface anywhere in the library. So a ``.hic`` can only be served from + a local copy, and cfdb does not make one: across the ENCODE corpus + those files total ~78 TB, with a median of 10 GB and a largest single + file of 315 GB, which exceeds the 200 GiB ceiling on Fargate ephemeral + storage and so could not be cached at any budget. + + That is a deliberate deferral rather than a missing import, which is + why the refusal lives at the call site in + :meth:`~cfdb.tilesets.service.TilesetService._open_hic` and not here. + This function still exists so a clodius that *loses* the module + reports the more accurate error. + + Raises: + TileBackendUnavailable: when clodius carries no ``.hic`` tileset. + """ + try: + from clodius.tiles_v2.hic import HicTileset + except ImportError as exc: + raise TileBackendUnavailable( + "Serving tiles from .hic files requires clodius.tiles_v2.hic, " + "which the installed clodius does not provide. 4DN .mcool " + "files are unaffected." + ) from exc + return HicTileset + + +def is_tile_error(backend: ClodiusBackend, exc: BaseException) -> bool: + """True when ``exc`` is a clodius per-tile error rather than a bug. + + Kept here so that ``except`` clauses elsewhere never need a + module-scope clodius import of their own. + """ + return isinstance(exc, backend.tile_error) + + +def datatype_of(tileset: Any) -> str: + """The HiGlass ``datatype`` a tileset serves. + + Read off the class rather than hardcoded to ``"matrix"``: it is + declared on the clodius ``Tileset`` protocol, and reading it is what + keeps every layer above ``resolve_tileset`` free of knowledge about + which concrete tileset it is holding. + """ + return type(tileset).datatype diff --git a/src/cfdb/tilesets/errors.py b/src/cfdb/tilesets/errors.py new file mode 100644 index 0000000..e4a0245 --- /dev/null +++ b/src/cfdb/tilesets/errors.py @@ -0,0 +1,67 @@ +"""Failure vocabulary for the tile subsystem. + +These are cfdb's own, distinct from clodius's ``TileError`` hierarchy. The +split is deliberate: clodius's errors describe things wrong with a *tile +request* against a tileset that exists, and are translated into per-tile +error payloads; these describe reasons cfdb cannot produce a tileset at +all, and become HTTP status codes. +""" + +from __future__ import annotations + + +class TilesetError(Exception): + """Base for every reason cfdb cannot serve tiles for a file.""" + + +class TilesetUnsupported(TilesetError): + """The file is not a contact map, so it has no tiles in any state. + + Rendered as 404 — permanently, not "not yet". + """ + + +class TilesetNotReady(TilesetError): + """The file is a contact map but its artifact has not been built. + + Rendered as 404 by the tile endpoints, which never dispatch. Callers + who want it built ask the preparation channel + (``POST /tilesets/{dcc}/{local_id}``); a heatmap track cannot be handed + a job id, so there is nothing useful to say here beyond "not here". + """ + + +class TilesetIdentityIncomplete(TilesetError): + """The document lacks the md5 the cache key is derived from. + + A stale or hand-edited record rather than a request to 500 on. + """ + + +class TileBackendUnavailable(TilesetError): + """The installed build carries no usable tile backend. + + Rendered as 501. Two distinct causes share it: clodius is absent + entirely (the images do not yet carry it — see the ``tiles`` extra in + ``pyproject.toml``), or the request is for a ``.hic``, which cfdb + deliberately does not serve because the backend reads local files only + and those files are too large to cache. + """ + + +class TilesetTooLarge(TilesetError): + """The artifact will not fit in the local disk budget. + + Rendered as 503 rather than 404: the file is genuinely tileable, and + a larger ``CFDB_TILESET_DISK_CACHE_BYTES`` would serve it. Raised + instead of evicting the entire cache for one oversized file, which + would thrash without ever succeeding. + """ + + +class TilesetHydrationTimeout(TilesetError): + """Pulling the artifact onto local disk exceeded its budget. + + Rendered as 503 with ``Retry-After``: the download may well finish, + and a retry will find it. + """ diff --git a/src/cfdb/tilesets/formats.py b/src/cfdb/tilesets/formats.py new file mode 100644 index 0000000..f89b579 --- /dev/null +++ b/src/cfdb/tilesets/formats.py @@ -0,0 +1,96 @@ +"""Which files are contact maps, and of what kind. + +The C2M2 ontology cannot answer this. ``services/ontology_mappings.py`` maps +``hdf5``, ``hic``, ``cool``, ``mcool`` **and** ``h5ad`` all onto the single +EDAM term ``format:3590`` / ``"HDF5"`` — correctly, since every one of them +really is an HDF5 container — and ``ProcessorRegistry.lookup_for`` keys on +that name alone. So format name distinguishes a contact map from a BAM but +not from an AnnData matrix, and the filename extension is the only +discriminator the file document carries. + +This module is deliberately **stdlib-only**. It is imported by both the API +and by :class:`~cfdb.workflows.processors.matrix.MatrixTilesetProcessor`, +which is cloudpickled across the wool boundary into a worker; pulling in +``processors.tools.format_name`` for the format check would drag +``workflows.cache`` and boto3 along with it. The check is three lines, +so it is inlined instead. +""" + +from __future__ import annotations + +from enum import Enum +from pathlib import PurePosixPath +from typing import Any + +#: The EDAM term every contact-map format collapses onto upstream. Matching +#: on it first means a filename that merely *ends* in ``.cool`` cannot make +#: a non-HDF5 record look like a contact map. +HDF5_FORMAT_NAME = "HDF5" + + +class MatrixSource(str, Enum): + """The contact-map container formats cfdb can serve tiles from.""" + + #: Multi-resolution cooler. Already a tile pyramid; needs only to be + #: fetched into the cache. + MCOOL = "mcool" + #: Single-resolution cooler. Must be coarsened into an mcool before it + #: can be served — clodius refuses a cooler with no ``resolutions`` + #: group. + COOL = "cool" + #: Juicer ``.hic``. Recognized so it can be refused specifically — + #: the tile backend reads local files only, and cfdb does not + #: materialize ``.hic`` artifacts, so these answer 501 rather than the + #: 404 a non-contact-map gets. + HIC = "hic" + + +#: Filename suffix to container format. Lower-cased before lookup. +_SUFFIX_TO_SOURCE = { + ".mcool": MatrixSource.MCOOL, + ".cool": MatrixSource.COOL, + ".hic": MatrixSource.HIC, +} + + +def matrix_source_kind(file_meta: dict[str, Any]) -> MatrixSource | None: + """Classify ``file_meta`` as a contact-map container, or ``None``. + + Args: + file_meta: A projected file document. Reads only ``file_format.name`` + and ``filename``, both of which ``FILE_DOC_PROJECTION`` selects. + + Returns: + The :class:`MatrixSource` this file is stored in, or ``None`` when it + is not a contact map — including for the other HDF5 formats + (``.h5ad``, bare ``.h5``) that share the same EDAM term. + + Note: + The extension decides **routing** only. Whether a cooler actually + needs coarsening is settled by looking for a ``resolutions`` group + in the file itself, because flat files named ``.mcool`` and + multi-resolution files named ``.cool`` both exist in the wild. + """ + file_format = file_meta.get("file_format") + if not isinstance(file_format, dict): + return None + if file_format.get("name") != HDF5_FORMAT_NAME: + return None + + filename = file_meta.get("filename") + if not isinstance(filename, str) or not filename: + return None + + suffix = PurePosixPath(filename.lower()).suffix + return _SUFFIX_TO_SOURCE.get(suffix) + + +#: Formats that must be materialized into the cache before tiles can be +#: read. ``HIC`` is absent because cfdb does not serve it at all — see +#: :class:`MatrixSource`. +MATERIALIZED_SOURCES = frozenset({MatrixSource.MCOOL, MatrixSource.COOL}) + + +def needs_materialization(file_meta: dict[str, Any]) -> bool: + """True when this file must be preprocessed before it can be tiled.""" + return matrix_source_kind(file_meta) in MATERIALIZED_SOURCES diff --git a/src/cfdb/tilesets/service.py b/src/cfdb/tilesets/service.py new file mode 100644 index 0000000..1a111e2 --- /dev/null +++ b/src/cfdb/tilesets/service.py @@ -0,0 +1,450 @@ +"""Open tilesets, read tiles, and cache both. + +Owns the three things a tile request needs that a single clodius call does +not provide: a local file to open, an open handle worth reusing across +thousands of requests, and somewhere to put the result so panning back +over the same region does not re-read it. + +Concurrency shape, since it is the part that is easy to get wrong: + +- clodius reads are synchronous h5py + numpy, so they run in a dedicated + thread pool. Not the default executor — the cache and processor helpers + already share that for byte streaming, and a multi-second tile read + there would stall ``/data``. +- An open tileset may be evicted while a thread is mid-read. Closing an + h5py handle under a live numpy read is a segfault, not an exception, so + handles are reference-counted and eviction defers the close to the last + reader. +""" + +from __future__ import annotations + +import asyncio +import logging +from collections import OrderedDict +from collections.abc import AsyncIterator, Callable, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from cfdb.tilesets.backend import load_backend, load_hic_tileset +from cfdb.tilesets.errors import ( + TileBackendUnavailable, + TilesetHydrationTimeout, + TilesetIdentityIncomplete, + TilesetNotReady, + TilesetUnsupported, +) +from cfdb.tilesets.formats import MatrixSource, matrix_source_kind +from cfdb.tilesets.store import LocalTilesetStore +from cfdb.tilesets.wire import error_payload, tileset_info_payload +from cfdb.workflows.cache import CacheBackend +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor + +logger = logging.getLogger(__name__) + + +@dataclass +class _OpenTileset: + """An open tileset handle and the readers currently using it.""" + + tileset: Any + #: Readers inside :meth:`TilesetService.checkout`. A handle with + #: readers is never closed, however urgently it is being evicted. + in_flight: int = 0 + #: Set when the entry has been evicted. The last reader out closes it. + closing: bool = False + #: Byte size of the tiles cached against this tileset, for accounting. + tile_bytes: int = field(default=0) + + +class TilesetService: + """Serves HiGlass tileset info and tiles from cached artifacts.""" + + def __init__( + self, + store: LocalTilesetStore, + *, + open_max: int, + tile_cache_bytes: int, + threads: int, + hydrate_timeout_s: float, + cache_provider: Callable[[], CacheBackend | None], + ) -> None: + self._store = store + self._open_max = open_max + self._tile_cache_bytes = tile_cache_bytes + self._hydrate_timeout_s = hydrate_timeout_s + self._cache_provider = cache_provider + self._backend = load_backend() + self._processor = MatrixTilesetProcessor() + self._pool = ThreadPoolExecutor( + max_workers=threads, thread_name_prefix="cfdb-tile" + ) + self._open: OrderedDict[str, _OpenTileset] = OrderedDict() + self._open_lock = asyncio.Lock() + #: ``(cache_key, tile_id.raw)`` to payload, least-recently-used + #: first. Keyed on the cache key rather than the uid because the + #: key embeds ``md5-v{processor_version}``, so an upstream byte + #: change or a processor bump invalidates every tile for free. + self._tiles: OrderedDict[tuple[str, str], dict[str, Any]] = OrderedDict() + self._tile_bytes = 0 + store.set_release_hook(self._release_for_eviction) + + # --- public API --------------------------------------------------------- + + async def tileset_info(self, uid: str, file_doc: dict[str, Any]) -> dict: + """Return the HiGlass tileset_info document for ``uid``.""" + async with self.checkout(uid, file_doc) as tileset: + return await self._run( + lambda: tileset_info_payload(uid, tileset, file_doc) + ) + + async def tiles( + self, uid: str, file_doc: dict[str, Any], raw_tile_ids: Sequence[str] + ) -> dict[str, dict[str, Any]]: + """Return tile payloads for ``raw_tile_ids``, keyed by request string. + + ``CoolerTileset.tiles`` returns exactly one entry per requested + position: a position the resolution ladder does not hold, an id that + fails to parse, or a modifier naming a balancing column the file + does not carry all render as a per-tile error object in that + position's slot rather than raising or being omitted — one bad id in + a batch must not blank the whole track. + """ + cache_key = self._cache_key(file_doc) + + payloads: dict[str, dict[str, Any]] = {} + pending: list[str] = [] + for raw in raw_tile_ids: + cached = self._tile_cache_get(cache_key, raw) + if cached is not None: + payloads[raw] = cached + else: + pending.append(raw) + + if not pending: + return payloads + + async with self.checkout(uid, file_doc) as tileset: + parsed = [] + for raw in pending: + try: + parsed.append(tileset.parse_tile_id(raw)) + except Exception as exc: + payloads[raw] = self._translate_tile_error(raw, exc) + + if parsed: + payloads.update( + await self._read_tiles(tileset, cache_key, parsed) + ) + + return payloads + + @asynccontextmanager + async def checkout( + self, uid: str, file_doc: dict[str, Any] + ) -> AsyncIterator[Any]: + """Borrow an open tileset for the duration of the block. + + The handle is reference-counted for as long as the block runs, so + an eviction that lands mid-read defers its ``close()`` until the + last reader leaves. + """ + entry = await self._acquire(uid, file_doc) + try: + yield entry.tileset + finally: + entry.in_flight -= 1 + if entry.closing and entry.in_flight == 0: + await self._close(entry) + + async def aclose(self) -> None: + """Close every open handle and shut the thread pool down.""" + async with self._open_lock: + entries = list(self._open.values()) + self._open.clear() + for entry in entries: + entry.closing = True + if entry.in_flight == 0: + await self._close(entry) + self._tiles.clear() + self._tile_bytes = 0 + await self._store.aclose() + self._pool.shutdown(wait=False, cancel_futures=True) + + # --- opening ------------------------------------------------------------ + + async def _acquire(self, uid: str, file_doc: dict[str, Any]) -> _OpenTileset: + """Return an open tileset for ``uid``, opening one if needed.""" + cache_key = self._cache_key(file_doc) + + async with self._open_lock: + entry = self._open.get(cache_key) + if entry is not None and not entry.closing: + self._open.move_to_end(cache_key) + entry.in_flight += 1 + return entry + + tileset = await self._open_tileset(uid, file_doc, cache_key) + + async with self._open_lock: + # Another request may have opened the same tileset while this + # one was hydrating. Keep the winner and close the loser rather + # than leaking a second h5py handle onto the same file. + existing = self._open.get(cache_key) + if existing is not None and not existing.closing: + await self._run(tileset.close) + self._open.move_to_end(cache_key) + existing.in_flight += 1 + return existing + + entry = _OpenTileset(tileset=tileset, in_flight=1) + self._open[cache_key] = entry + evictable = self._select_evictions() + + for victim in evictable: + await self._close(victim) + return entry + + def _select_evictions(self) -> list[_OpenTileset]: + """Drop over-cap entries from the table, returning ones safe to close. + + Called with ``_open_lock`` held. An entry with readers is marked + ``closing`` and removed from the table — so no new reader can find + it — but its ``close()`` is left to the last reader out. + """ + victims: list[_OpenTileset] = [] + while len(self._open) > self._open_max: + key, entry = self._open.popitem(last=False) + entry.closing = True + self._evict_tiles_for(key) + if entry.in_flight == 0: + victims.append(entry) + return victims + + async def _open_tileset( + self, uid: str, file_doc: dict[str, Any], cache_key: str + ) -> Any: + """Construct a tileset for ``file_doc``, without dispatching anything.""" + kind = matrix_source_kind(file_doc) + if kind is None: + raise TilesetUnsupported(f"{uid} is not a contact map") + if kind is MatrixSource.HIC: + return await self._open_hic(uid, file_doc) + return await self._open_cooler(uid, cache_key) + + async def _open_cooler(self, uid: str, cache_key: str) -> Any: + """Open the cached mcool artifact for ``uid``.""" + cache = self._cache_provider() + if cache is None: + raise TilesetNotReady(f"{uid} has no cache to read an artifact from") + + entry = await cache.head(cache_key) + if entry is None: + # Never dispatch from here. The tile endpoints are pure + # readers; building the artifact is the preparation channel's + # job, because a heatmap track cannot poll a job id. + raise TilesetNotReady(f"{uid} has no cached tileset artifact") + + try: + path = await asyncio.wait_for( + self._store.path_for(cache, cache_key, expected_size=entry.size), + timeout=self._hydrate_timeout_s, + ) + except asyncio.TimeoutError as exc: + raise TilesetHydrationTimeout( + f"pulling the tileset artifact for {uid} onto local disk " + f"exceeded {self._hydrate_timeout_s}s" + ) from exc + + return await self._run(lambda: self._backend.cooler_tileset(str(path))) + + async def _open_hic(self, uid: str, file_doc: dict[str, Any]) -> Any: + """Refuse a ``.hic`` — cfdb has no local artifact to open one from. + + The pinned clodius does not carry ``clodius.tiles_v2.hic`` at all + (see :func:`~cfdb.tilesets.backend.load_hic_tileset`), which makes + this refusal unconditional today. It would stay unconditional even + if a future clodius did carry the module: it is built on + ``hictkpy``, whose ``MultiResFile`` takes ``str | os.PathLike`` and + nothing else — there is no way to read one *remotely*, no URL + handling in it at all. + + That invalidates the assumption this endpoint was designed + around. ``.hic`` was to be read in place over HTTP range requests + precisely so the files would never have to be copied — and the + measurement behind that plan was taken with ``hicstraw``, a + different library, which does support range reads over HTTP. + Against ``hictkpy`` the only way to serve a ``.hic`` is to + materialize it first, and the corpus makes that a decision rather + than a detail: 3,903 files, 78 TB in total, a median of 10 GB, and + a largest single file of 315 GB — which exceeds the 200 GiB + ceiling on Fargate ephemeral storage, so it could not be cached on + an API task at any budget. + + Raising here keeps that decision visible instead of half-answering + it. 4DN ``.mcool`` files are unaffected. + """ + # Resolved for its side effect: surfaces the more specific + # "module missing" error when that's the case, even though this + # function refuses .hic regardless of whether the module is present. + load_hic_tileset() + raise TileBackendUnavailable( + f"{uid} is a .hic, which cfdb cannot serve tiles from yet. The " + "clodius .hic tileset reads local files only (hictkpy has no " + "remote support), and cfdb does not materialize .hic artifacts. " + "4DN .mcool contact maps are unaffected." + ) + + async def _close(self, entry: _OpenTileset) -> None: + """Close a handle off the event loop.""" + try: + await self._run(entry.tileset.close) + except Exception: + logger.exception("Failed to close a tileset handle") + + async def _release_for_eviction(self, cache_key: str) -> bool: + """Store hook: may this artifact's file be unlinked? + + Closes the tileset holding it open when nothing is reading, and + refuses otherwise. This is the coupling that keeps the disk store + from pulling a file out from under a live h5py handle. + """ + async with self._open_lock: + entry = self._open.get(cache_key) + if entry is None: + return True + if entry.in_flight: + return False + del self._open[cache_key] + entry.closing = True + self._evict_tiles_for(cache_key) + await self._close(entry) + return True + + # --- reading ------------------------------------------------------------ + + async def _read_tiles( + self, tileset: Any, cache_key: str, parsed: list[Any] + ) -> dict[str, dict[str, Any]]: + """Read tiles off the event loop and cache what comes back.""" + try: + returned = await self._run(lambda: tileset.tiles(parsed)) + except Exception as exc: + if not isinstance(exc, self._backend.tile_error): + raise + # A batch-level clodius error would apply to every id in the + # batch. In practice `CoolerTileset.tiles` no longer raises for + # this — it catches its own per-id and per-batch `TileError`s + # internally and renders them inline in `returned` below — so + # this branch guards a bug in the backend rather than an + # expected path. + message = str(exc) + return {tid.raw: error_payload(message) for tid in parsed} + + payloads: dict[str, dict[str, Any]] = {} + for tile_id, payload in returned: + # clodius renders its own per-tile failures as + # `{"error": ..., "error_type": ...}` (`TileError.to_dict`). + # Normalize through `error_payload` so every error on the wire + # — clodius's or cfdb's own parse-time translation below — has + # the same `{"error": str}` shape. + if "error" in payload and "dense" not in payload: + payload = error_payload(str(payload["error"])) + payloads[tile_id.raw] = payload + self._tile_cache_put(cache_key, tile_id.raw, payload) + return payloads + + def _translate_tile_error(self, raw: str, exc: Exception) -> dict[str, str]: + """Render a per-tile failure, or re-raise a genuine bug.""" + if isinstance(exc, self._backend.tile_error): + return error_payload(str(exc)) + raise exc + + # --- tile cache --------------------------------------------------------- + + def _tile_cache_get(self, cache_key: str, raw: str) -> dict | None: + entry = self._tiles.get((cache_key, raw)) + if entry is not None: + self._tiles.move_to_end((cache_key, raw)) + return entry + + def _tile_cache_put( + self, cache_key: str, raw: str, payload: dict[str, Any] + ) -> None: + size = _payload_bytes(payload) + if size > self._tile_cache_bytes: + return + key = (cache_key, raw) + if key in self._tiles: + self._tile_bytes -= _payload_bytes(self._tiles[key]) + self._tiles[key] = payload + self._tiles.move_to_end(key) + self._tile_bytes += size + while self._tile_bytes > self._tile_cache_bytes: + _, evicted = self._tiles.popitem(last=False) + self._tile_bytes -= _payload_bytes(evicted) + + def _evict_tiles_for(self, cache_key: str) -> None: + """Drop every cached tile belonging to one tileset.""" + for key in [k for k in self._tiles if k[0] == cache_key]: + self._tile_bytes -= _payload_bytes(self._tiles.pop(key)) + + @property + def tile_cache_bytes_used(self) -> int: + """Bytes currently held by the tile cache.""" + return self._tile_bytes + + # --- plumbing ----------------------------------------------------------- + + def _cache_key(self, file_doc: dict[str, Any]) -> str: + """The TILESET cache key for ``file_doc``. + + Derived through the processor rather than re-implemented, so the + writer, the readiness probe, and this reader agree by construction + rather than by three formulas kept in sync. + """ + try: + return self._processor.cache_key_for(file_doc, ArtifactKind.TILESET) + except ValueError as exc: + raise TilesetIdentityIncomplete(str(exc)) from exc + + async def _run(self, fn): + """Run a blocking tile-backend call on the dedicated pool.""" + return await asyncio.get_running_loop().run_in_executor(self._pool, fn) + + +def _payload_bytes(payload: dict[str, Any]) -> int: + """Approximate on-heap size of a tile payload. + + The base64 ``dense`` string dominates by orders of magnitude; the + handful of floats beside it are not worth measuring. + """ + dense = payload.get("dense") + return len(dense) if isinstance(dense, (str, bytes)) else 0 + + +def build_service( + *, + root: Path, + cache_provider: Callable[[], CacheBackend | None], + disk_cache_bytes: int, + open_max: int, + tile_cache_bytes: int, + threads: int, + hydrate_timeout_s: float, +) -> TilesetService: + """Construct a service and the disk store behind it.""" + store = LocalTilesetStore(root, max_bytes=disk_cache_bytes) + return TilesetService( + store, + open_max=open_max, + tile_cache_bytes=tile_cache_bytes, + threads=threads, + hydrate_timeout_s=hydrate_timeout_s, + cache_provider=cache_provider, + ) diff --git a/src/cfdb/tilesets/store.py b/src/cfdb/tilesets/store.py new file mode 100644 index 0000000..e35beff --- /dev/null +++ b/src/cfdb/tilesets/store.py @@ -0,0 +1,205 @@ +"""Cached tileset artifact to local filesystem path. + +The tile backend opens a file: ``CoolerTileset`` hands its path to +``h5py.File``, which needs random access to real bytes on a real +filesystem. ``CacheBackend.get`` yields a byte stream, which is the right +shape for ``/data`` and the wrong shape here. + +On the local profile that mismatch is free — a ``LocalFsCache`` artifact +already *is* a local file, so it is opened in place. On the S3 profile the +artifact has to be pulled down first, which this module does through a +single-flight, byte-budgeted LRU on the API task's disk. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +from collections import OrderedDict +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from pathlib import Path + +from cfdb.tilesets.errors import TilesetTooLarge +from cfdb.workflows.cache import CacheBackend, LocalFsCache +from cfdb.workflows.processors.tools import copy_from_cache + +logger = logging.getLogger(__name__) + +#: Asked before a hydrated file is unlinked. Returns True when the file is +#: no longer in use and may be removed. The tile service supplies one that +#: closes the corresponding open tileset first — unlinking a file whose +#: h5py handle is live is not an error you get to catch. +ReleaseHook = Callable[[str], Awaitable[bool]] + + +async def _always_releasable(_key: str) -> bool: + """Default release hook: nothing is holding anything open.""" + return True + + +@dataclass +class _Resident: + """A hydrated artifact on local disk.""" + + path: Path + size: int + + +class LocalTilesetStore: + """Gives the tile server a local path for a cached artifact. + + Never reaches upstream. A cache miss means the artifact has not been + built, which is the preparation channel's problem, not this one's. + """ + + def __init__( + self, + root: Path, + *, + max_bytes: int, + release_hook: ReleaseHook | None = None, + ) -> None: + self._root = Path(root) + self._root.mkdir(parents=True, exist_ok=True) + self._max_bytes = max_bytes + self._release = release_hook or _always_releasable + #: Hydrated files, least-recently-used first. + self._resident: OrderedDict[str, _Resident] = OrderedDict() + #: One lock per key, so N concurrent tile requests for a cold + #: dataset produce one download rather than N. + self._inflight: dict[str, asyncio.Lock] = {} + + def set_release_hook(self, hook: ReleaseHook) -> None: + """Attach the hook eviction consults before unlinking. + + Set after construction because the service and the store hold + references to each other: the service asks the store for a path, + and the store asks the service whether a file may go. + """ + self._release = hook + + @property + def resident_bytes(self) -> int: + """Total size of the hydrated files currently on disk.""" + return sum(entry.size for entry in self._resident.values()) + + async def path_for( + self, cache: CacheBackend, cache_key: str, *, expected_size: int + ) -> Path: + """Return a local path holding the artifact at ``cache_key``. + + Args: + cache: The configured cache backend. + cache_key: Key the artifact was committed under. + expected_size: The artifact's size, from ``cache.head``. Used + to make room before downloading rather than after. + + Raises: + TilesetTooLarge: when the artifact alone exceeds the whole + disk budget. + """ + if isinstance(cache, LocalFsCache): + # Already a local file. Opening it in place is not just an + # optimization: it is what lets the entire tile subsystem be + # exercised in tests without an S3 stand-in. + return cache.path_for(cache_key) + + if expected_size > self._max_bytes: + raise TilesetTooLarge( + f"artifact {cache_key} is {expected_size} bytes, which " + f"exceeds the whole {self._max_bytes}-byte local tileset " + "budget (CFDB_TILESET_DISK_CACHE_BYTES)" + ) + + lock = self._inflight.setdefault(cache_key, asyncio.Lock()) + async with lock: + resident = self._resident.get(cache_key) + if resident is not None and resident.path.exists(): + self._resident.move_to_end(cache_key) + return resident.path + return await self._hydrate(cache, cache_key, expected_size) + + async def _hydrate( + self, cache: CacheBackend, cache_key: str, expected_size: int + ) -> Path: + """Download the artifact onto local disk, evicting to make room.""" + await self._make_room(expected_size) + + dest = self._local_path(cache_key) + partial = dest.with_suffix(dest.suffix + ".part") + logger.info( + f"Hydrating tileset artifact {cache_key} ({expected_size} bytes)" + ) + try: + await copy_from_cache(cache, cache_key, partial) + # Rename rather than write in place, so a download interrupted + # by cancellation or a timeout can never be mistaken for a + # complete artifact and handed to h5py. + await asyncio.to_thread(os.replace, str(partial), str(dest)) + except BaseException: + partial.unlink(missing_ok=True) + raise + + self._resident[cache_key] = _Resident(path=dest, size=expected_size) + self._resident.move_to_end(cache_key) + return dest + + async def _make_room(self, incoming: int) -> None: + """Evict least-recently-used artifacts until ``incoming`` fits.""" + for key in list(self._resident): + if self.resident_bytes + incoming <= self._max_bytes: + return + if not await self._release(key): + # Something is reading it. Skip rather than unlink: closing + # an h5py handle out from under a numpy read in a worker + # thread is a segfault, not an exception. + logger.debug(f"Skipping eviction of in-use tileset {key}") + continue + self._evict(key) + + # Falling through means every remaining resident is pinned. Let the + # hydration proceed and overshoot the budget rather than failing a + # request; the overshoot is bounded by the open-tileset cap, and + # the next eviction pass reclaims it. + if self.resident_bytes + incoming > self._max_bytes: + logger.warning( + "Local tileset budget exceeded: %d resident bytes plus %d " + "incoming against a %d-byte budget, and every resident " + "artifact is in use", + self.resident_bytes, + incoming, + self._max_bytes, + ) + + def _evict(self, key: str) -> None: + """Drop one artifact from disk and from the residency table.""" + entry = self._resident.pop(key, None) + if entry is None: + return + entry.path.unlink(missing_ok=True) + self._inflight.pop(key, None) + logger.info(f"Evicted local tileset artifact {key}") + + def _local_path(self, cache_key: str) -> Path: + """Flat, collision-free filename for a cache key. + + Hashed rather than mirroring the key's directory structure: the + key is already validated against traversal by the cache layer, and + a flat namespace keeps eviction a single ``unlink`` with no empty + directories left behind. + """ + digest = hashlib.sha256(cache_key.encode()).hexdigest() + return self._root / f"{digest}.tileset" + + async def aclose(self) -> None: + """Drop every hydrated artifact. + + Called on lifespan teardown. The files are a cache of a cache; + leaving them behind would leak the API task's disk across + restarts, since the hashed names carry no expiry of their own. + """ + for key in list(self._resident): + self._evict(key) diff --git a/src/cfdb/tilesets/wire.py b/src/cfdb/tilesets/wire.py new file mode 100644 index 0000000..d2d342e --- /dev/null +++ b/src/cfdb/tilesets/wire.py @@ -0,0 +1,66 @@ +"""HiGlass response payload assembly. + +clodius produces tile payloads that are already JSON-ready and a +``TilesetInfo`` that is deliberately incomplete as a wire document: it +models what can be read out of the *file*, and leaves what only a server +knows — the dataset's name, its identifier, the coordinate system it is +annotated against — to the server. This module supplies exactly that +difference. +""" + +from __future__ import annotations + +from typing import Any + +from cfdb.tilesets.backend import datatype_of + + +def tileset_info_payload( + uid: str, tileset: Any, file_doc: dict[str, Any] +) -> dict[str, Any]: + """Build the ``tileset_info`` document HiGlass expects for ``uid``. + + Args: + uid: The tileset identifier, ``{dcc}/{local_id}``. + tileset: An open clodius tileset. + file_doc: The projected file document, for the display name and + the coordinate system. + + Returns: + clodius's own info fields, plus the four the library does not + model. + """ + payload = tileset.info().model_dump(exclude_none=True) + + # ``exclude_none`` above drops ``max_width`` and ``max_zoom``, both of + # which clodius leaves None for an explicit-ladder tileset on purpose: + # for a cooler the extent belongs to the zoom level, not the tileset, + # and a single value taken from the coarsest resolution over-reports + # the tile count at every finer zoom. Emitting them as JSON ``null`` + # would invite a client to do arithmetic on them; absence is what the + # explicit-ladder path expects. + payload["datatype"] = datatype_of(tileset) + payload["name"] = file_doc.get("filename") or file_doc.get("local_id") or uid + payload["uuid"] = uid + + # Omitted entirely when unknown — 21 of the 4DN contact maps carry no + # assembly. HiGlass matches ``coordSystem`` against chromosome-info + # tilesets, so a plausible-looking placeholder ("", "unknown") risks a + # silently misaligned track, whereas an absent key makes the client + # fall back to the ``chromsizes`` array that every tileset_info + # carries. The heatmap still renders either way. + coord_system = file_doc.get("genome_assembly") + if coord_system: + payload["coordSystem"] = coord_system + + return payload + + +def error_payload(message: str) -> dict[str, str]: + """A per-entry error object. + + HiGlass renders these in place of tile data rather than treating the + response as failed, which is what keeps one malformed tile id in a + batch of sixty from blanking the whole track. + """ + return {"error": message} diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000..6a911dc --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1 @@ +"""Synthetic binary fixtures shared across test modules.""" diff --git a/tests/fixtures/coolers.py b/tests/fixtures/coolers.py new file mode 100644 index 0000000..cd3ef96 --- /dev/null +++ b/tests/fixtures/coolers.py @@ -0,0 +1,184 @@ +"""Synthetic cooler builders for the matrix tile-serving tests. + +Ported from the clodius fork's ``test/harness/builders.py`` so cfdb's tile +tests run against a real file rather than a mock — the payload assertions +(a 256x256 dense block, base64-encoded, with no ``shape`` field) are only +meaningful against bytes clodius actually produced. + +Plain functions rather than pytest fixtures, so a test that needs a file +per case can call one directly; ``conftest.py`` wraps them for the ordinary +shared-file case. + +Two writer behaviours are not what the documentation suggests, and both +cost the clodius authors a debugging round: + +* ``cooler.create_cooler`` defaults to ``mode="w"``, which **truncates the + whole file**. Writing several resolutions in a loop silently leaves only + the last one, with no error at all. +* Bins must be declared as an ordered ``Categorical`` or the chromosome + order in the resulting file is not the order you passed. +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +#: Tiling genome, 3000 bp total. Deliberately tiny relative to the bin +#: sizes below: a cooler tile is 256 bins wide, so a realistic 1 kb binsize +#: on this genome would put the whole thing inside a single bin of a single +#: tile — one real value and 65,535 padding cells, exercising no geometry +#: at all. Three chromosomes of unequal length so tiles straddle boundaries. +CANONICAL_CHROMSIZES: list[tuple[str, int]] = [ + ("c1", 1000), + ("c2", 1500), + ("c3", 500), +] + +CANONICAL_TOTAL = 3000 + + +def build_cool( + path: Path, + chromsizes: list[tuple[str, int]] = CANONICAL_CHROMSIZES, + binsize: int = 4, + seed: int = 0, + weights: dict[str, float] | None = None, +) -> Path: + """A flat, single-resolution cooler — the shape that needs coarsening.""" + return _write_cooler(str(path), chromsizes, binsize, seed, weights, mode="w") + + +def build_mcool( + path: Path, + chromsizes: list[tuple[str, int]] = CANONICAL_CHROMSIZES, + resolutions: tuple[int, ...] = (1, 2, 4), + seed: int = 0, + weights: dict[str, float] | None = None, +) -> Path: + """A multi-resolution cooler, one group per entry in ``resolutions``. + + ``weights`` maps a bin-column name to a constant value, adding that + column at every resolution, which is what makes clodius's balancing + modifiers reachable. It takes a mapping rather than a single value + because a column named ``weight`` is also the one + ``resolve_balance`` falls back to by default — so a cooler carrying + only that column cannot distinguish honouring a named request from + ignoring it. Pass a second name to make the request observable. + """ + for i, binsize in enumerate(sorted(resolutions)): + _write_cooler( + f"{path}::/resolutions/{binsize}", + chromsizes, + binsize, + seed, + weights, + # The first call creates the file; every later one MUST append, + # or it truncates what the previous call just wrote. + mode="w" if i == 0 else "a", + ) + return path + + +def build_variable_bin_cool( + path: Path, + chromsizes: list[tuple[str, int]] = CANONICAL_CHROMSIZES, + seed: int = 0, +) -> Path: + """A flat cooler with an irregular bin table (``Cooler.binsize`` is None). + + The restriction-fragment shape the matrix processor refuses to coarsen: + ``cooler zoomify``'s binary ladder is meaningless over variable-width + bins, so ``run()`` must raise rather than emit a ladder that + misrepresents them. + """ + cooler = pytest.importorskip("cooler") + pd = pytest.importorskip("pandas") + rng = np.random.default_rng(seed) + + rows = [] + for name, length in chromsizes: + cuts = np.sort(rng.choice(np.arange(1, int(length)), size=7, replace=False)) + starts = np.concatenate(([0], cuts)) + ends = np.concatenate((cuts, [int(length)])) + rows.append(pd.DataFrame({"chrom": name, "start": starts, "end": ends})) + bins = pd.concat(rows, ignore_index=True) + bins["chrom"] = pd.Categorical( + bins["chrom"], + categories=[name for name, _ in chromsizes], + ordered=True, + ) + + n = len(bins) + b1 = rng.integers(0, n, size=100) + b2 = rng.integers(0, n, size=100) + pixels = ( + pd.DataFrame( + { + "bin1_id": np.minimum(b1, b2), + "bin2_id": np.maximum(b1, b2), + "count": 1, + } + ) + .groupby(["bin1_id", "bin2_id"], as_index=False)["count"] + .sum() + .sort_values(["bin1_id", "bin2_id"]) + .reset_index(drop=True) + ) + + cooler.create_cooler( + str(path), bins, pixels, mode="w", ordered=True, symmetric_upper=True + ) + return path + + +def _write_cooler( + uri: str, + chromsizes: list[tuple[str, int]], + binsize: int, + seed: int, + weights: dict[str, float] | None, + mode: str, +) -> Path: + """Write one cooler at ``uri`` with a deterministic random pixel table.""" + cooler = pytest.importorskip("cooler") + pd = pytest.importorskip("pandas") + rng = np.random.default_rng(seed) + + rows = [] + for name, length in chromsizes: + starts = np.arange(0, int(length), binsize) + ends = np.minimum(starts + binsize, int(length)) + rows.append(pd.DataFrame({"chrom": name, "start": starts, "end": ends})) + bins = pd.concat(rows, ignore_index=True) + bins["chrom"] = pd.Categorical( + bins["chrom"], + categories=[name for name, _ in chromsizes], + ordered=True, + ) + for column, value in (weights or {}).items(): + bins[column] = float(value) + + n = len(bins) + b1 = rng.integers(0, n, size=min(4 * n, 400)) + b2 = rng.integers(0, n, size=min(4 * n, 400)) + pixels = ( + pd.DataFrame( + { + "bin1_id": np.minimum(b1, b2), + "bin2_id": np.maximum(b1, b2), + "count": 1, + } + ) + .groupby(["bin1_id", "bin2_id"], as_index=False)["count"] + .sum() + .sort_values(["bin1_id", "bin2_id"]) + .reset_index(drop=True) + ) + + cooler.create_cooler( + uri, bins, pixels, mode=mode, ordered=True, symmetric_upper=True + ) + return Path(uri.split("::")[0]) diff --git a/tests/fixtures/remote_cache.py b/tests/fixtures/remote_cache.py new file mode 100644 index 0000000..5cb5358 --- /dev/null +++ b/tests/fixtures/remote_cache.py @@ -0,0 +1,50 @@ +"""A shared non-local cache stand-in for the tile-serving tests. + +Hoisted out of ``tests/test_tilesets/test_store.py`` so the service-layer +hydration tests and the router-level 503 tests can use the same fake +without importing from a test module. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path + +from cfdb.workflows.cache import CacheBackend, CacheEntry + + +class FakeRemoteCache(CacheBackend): + """A cache that is not a LocalFsCache, so it must be hydrated. + + Stands in for ``S3Cache`` without needing moto: the store branches on + the backend type, and everything past that branch is ``head``/``get``. + """ + + def __init__(self, blobs: dict[str, bytes]) -> None: + self.blobs = blobs + self.get_calls: list[str] = [] + self.gate: asyncio.Event | None = None + + async def head(self, key: str): + blob = self.blobs.get(key) + return None if blob is None else CacheEntry(key=key, size=len(blob)) + + def get(self, key: str, byte_range=None) -> AsyncIterator[bytes]: + self.get_calls.append(key) + blobs, gate = self.blobs, self.gate + + async def _stream(): + if gate is not None: + await gate.wait() + yield blobs[key] + + return _stream() + + async def put(self, key: str, source_path: Path) -> CacheEntry: + data = source_path.read_bytes() + self.blobs[key] = data + return CacheEntry(key=key, size=len(data)) + + async def delete(self, key: str) -> bool: + return self.blobs.pop(key, None) is not None diff --git a/tests/test_tilesets/__init__.py b/tests/test_tilesets/__init__.py new file mode 100644 index 0000000..9a090cb --- /dev/null +++ b/tests/test_tilesets/__init__.py @@ -0,0 +1 @@ +"""Tests for the matrix tile-serving subsystem.""" diff --git a/tests/test_tilesets/test_backend.py b/tests/test_tilesets/test_backend.py new file mode 100644 index 0000000..ffb7bde --- /dev/null +++ b/tests/test_tilesets/test_backend.py @@ -0,0 +1,215 @@ +"""Tests for the clodius import gate in ``cfdb.tilesets.backend``.""" + +from __future__ import annotations + +import sys + +import pytest + +from cfdb.tilesets.backend import ( + is_available, + is_tile_error, + load_backend, + load_hic_tileset, +) +from cfdb.tilesets.errors import TileBackendUnavailable + +#: The exact modules ``load_backend`` imports from. A ``None`` entry in +#: ``sys.modules`` makes ``from X import Y`` raise ImportError, which is how +#: a build without the ``tiles`` extra is simulated in an environment that +#: does carry clodius. +_ABSENT_BACKEND_MODULES = { + "clodius.core.errors": None, + "clodius.tiles_v2.cooler": None, +} + + +class TestLoadBackend: + """The one seam that turns the tile subsystem on or off.""" + + def test_should_raise_when_clodius_is_absent(self, mocker): + """Test the degraded import gate of a build without the tiles extra. + + Given: + ``sys.modules`` patched so both clodius modules the gate + imports from raise ImportError, simulating an image built + without the ``tiles`` extra. + When: + load_backend is called. + Then: + It should raise TileBackendUnavailable naming the ``tiles`` + extra, chained from the underlying ImportError. + """ + # Arrange + mocker.patch.dict(sys.modules, _ABSENT_BACKEND_MODULES) + + # Act & assert + with pytest.raises(TileBackendUnavailable, match="'tiles' extra") as excinfo: + load_backend() + assert isinstance(excinfo.value.__cause__, ImportError) + + def test_should_resolve_the_clodius_names(self): + """Test the resolved-names contract of a build with clodius. + + Given: + An environment where clodius is installed. + When: + load_backend is called. + Then: + It should return a backend whose ``cooler_tileset`` is + clodius's ``CoolerTileset`` and whose error attributes are the + clodius exception types the per-tile/500 split keys on. + """ + # Arrange + from clodius.core.errors import TileOutOfBounds, TilesetError + from clodius.tiles_v2.cooler import CoolerTileset + + # Act + backend = load_backend() + + # Assert + assert backend.cooler_tileset is CoolerTileset + assert backend.tile_error is TilesetError + assert backend.tile_out_of_bounds is TileOutOfBounds + assert issubclass(backend.tile_error, Exception) + assert issubclass(backend.tile_out_of_bounds, Exception) + + +class TestIsAvailable: + """The bool the lifespan and the 501/503 split gate on.""" + + def test_should_return_false_when_clodius_is_absent(self, mocker): + """Test the availability probe against a backendless build. + + Given: + ``sys.modules`` patched so the clodius imports raise + ImportError. + When: + is_available is called. + Then: + It should return False — the value that makes the lifespan + skip the tile service and the routers answer 501. + """ + # Arrange + mocker.patch.dict(sys.modules, _ABSENT_BACKEND_MODULES) + + # Act + result = is_available() + + # Assert + assert result is False + + def test_should_return_true_when_clodius_is_present(self): + """Test the availability probe against an installed backend. + + Given: + An environment where clodius is installed and unpatched. + When: + is_available is called. + Then: + It should return True. + """ + # Act + result = is_available() + + # Assert + assert result is True + + +class TestLoadHicTileset: + """The .hic module gate, resolved separately from the cooler tileset.""" + + def test_should_raise_when_the_hic_module_is_missing(self, mocker): + """Test the error reported by a clodius that lost its hic module. + + Given: + ``sys.modules`` patched so ``clodius.tiles_v2.hic`` raises + ImportError while the cooler tileset stays importable. + When: + load_hic_tileset is called. + Then: + It should raise TileBackendUnavailable stating that mcool + serving is unaffected, so the failure reads as the narrower + capability gap it is. + """ + # Arrange + mocker.patch.dict(sys.modules, {"clodius.tiles_v2.hic": None}) + + # Act & assert + with pytest.raises( + TileBackendUnavailable, match=r"\.mcool files are unaffected" + ) as excinfo: + load_hic_tileset() + assert isinstance(excinfo.value.__cause__, ImportError) + + +class TestIsTileError: + """The classification that splits per-tile payloads from 500s.""" + + def test_should_recognize_a_clodius_tile_error(self): + """Test that a clodius per-tile error is classified as such. + + Given: + A loaded backend and an instance of its ``tile_error`` type. + When: + is_tile_error is called with that instance. + Then: + It should return True — the branch that renders the failure + as a per-tile error payload instead of failing the request. + """ + # Arrange + backend = load_backend() + exc = backend.tile_error("bad tile position") + + # Act + result = is_tile_error(backend, exc) + + # Assert + assert result is True + + def test_should_recognize_a_malformed_tile_id_as_a_tile_error(self): + """Test that a parse-time clodius failure is classified as such too. + + Given: + A loaded backend and an instance of clodius's + ``MalformedTileId`` — raised by ``parse_tile_id`` before a tile + is ever read, and no longer a subclass of the narrower + ``TileError`` clodius restored, but still a ``TilesetError``. + When: + is_tile_error is called with that instance. + Then: + It should return True — a bad id must still degrade to a + per-tile error payload rather than failing the whole batch. + """ + # Arrange + from clodius.core.errors import MalformedTileId + + backend = load_backend() + exc = MalformedTileId("not an int") + + # Act + result = is_tile_error(backend, exc) + + # Assert + assert result is True + + def test_should_reject_a_non_tile_error(self): + """Test that an ordinary exception is not classified as a tile error. + + Given: + A loaded backend and a plain ValueError. + When: + is_tile_error is called with the ValueError. + Then: + It should return False — bugs must propagate and become 500s + rather than being swallowed into per-tile payloads. + """ + # Arrange + backend = load_backend() + exc = ValueError("a bug, not a tile problem") + + # Act + result = is_tile_error(backend, exc) + + # Assert + assert result is False diff --git a/tests/test_tilesets/test_errors.py b/tests/test_tilesets/test_errors.py new file mode 100644 index 0000000..e9aff7b --- /dev/null +++ b/tests/test_tilesets/test_errors.py @@ -0,0 +1,56 @@ +"""Tests for the failure vocabulary in ``cfdb.tilesets.errors``.""" + +from __future__ import annotations + +import inspect + +import cfdb.tilesets.errors as errors_module +from cfdb.tilesets.errors import TilesetError + + +class TestTilesetErrorHierarchy: + """The routers catch the subsystem's failures with one except clause.""" + + def test_should_derive_every_public_exception_from_tileset_error(self): + """Test that the hierarchy is closed under its declared base. + + Given: + Every public exception class defined in + ``cfdb.tilesets.errors``. + When: + Each is checked against TilesetError. + Then: + It should be a subclass — the routers' single + ``except TilesetError`` is the whole safety net, so an + exception defined here but outside the hierarchy would escape + it and surface as a 500. + """ + # Arrange + public_exceptions = [ + obj + for name, obj in vars(errors_module).items() + if inspect.isclass(obj) + and issubclass(obj, BaseException) + and obj.__module__ == errors_module.__name__ + and not name.startswith("_") + ] + + # Act + outsiders = [ + exc for exc in public_exceptions if not issubclass(exc, TilesetError) + ] + + # Assert + assert outsiders == [] + # Guard against vacuity: the module must actually define the + # vocabulary the routers translate into status codes. + names = {exc.__name__ for exc in public_exceptions} + assert { + "TilesetError", + "TilesetUnsupported", + "TilesetNotReady", + "TilesetIdentityIncomplete", + "TileBackendUnavailable", + "TilesetTooLarge", + "TilesetHydrationTimeout", + } <= names diff --git a/tests/test_tilesets/test_formats.py b/tests/test_tilesets/test_formats.py new file mode 100644 index 0000000..861c400 --- /dev/null +++ b/tests/test_tilesets/test_formats.py @@ -0,0 +1,384 @@ +"""Tests for contact-map classification in ``cfdb.tilesets.formats``.""" + +from __future__ import annotations + +from pathlib import PurePosixPath + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from cfdb.tilesets.formats import ( + MatrixSource, + matrix_source_kind, + needs_materialization, +) + +#: The suffixes the module claims, restated literally so the property +#: below asserts against the documented contract rather than against the +#: module's own lookup table. +_CONTACT_MAP_SUFFIXES = frozenset({".mcool", ".cool", ".hic"}) + +#: JSON-shaped scalars a stale or hand-edited document could carry in any +#: field. +_json_scalars = st.one_of( + st.none(), + st.booleans(), + st.integers(), + st.floats(allow_nan=False), + st.text(), +) + +#: Filename values biased toward the classification boundaries: real +#: contact-map names, casing variants, dotfiles, buried extensions, other +#: HDF5 formats, and outright non-strings. +_filename_values = st.one_of( + _json_scalars, + st.sampled_from( + [ + "sample.mcool", + "SAMPLE.MCOOL", + "x.cool", + "x.hic", + ".mcool", + "sample.mcool.gz", + "a.tar.mcool", + "dir/x.cool", + "", + "matrix.h5ad", + "raw.h5", + ] + ), + st.lists(st.text(), max_size=2), +) + +#: file_format values spanning the right shape, the right shape with the +#: wrong name (including the case-variant ``hdf5``), and non-dict garbage. +_file_format_values = st.one_of( + _json_scalars, + st.dictionaries(st.text(), _json_scalars, max_size=3), + st.just({"name": "HDF5"}), + st.just({"name": "hdf5"}), +) + +#: Arbitrary JSON-like file documents, each field independently absent or +#: malformed. +_documents = st.fixed_dictionaries( + {}, + optional={ + "filename": _filename_values, + "file_format": _file_format_values, + "extra": _json_scalars, + }, +) + + +@st.composite +def _cased_contact_map_cases(draw): + """Draw ``(filename, expected_kind)`` with a randomly-cased suffix.""" + stem = draw(st.text(min_size=1).filter(lambda s: "/" not in s)) + kind = draw(st.sampled_from(list(MatrixSource))) + suffix = "." + kind.value + upper_mask = draw( + st.lists(st.booleans(), min_size=len(suffix), max_size=len(suffix)) + ) + cased = "".join( + char.upper() if upper else char for char, upper in zip(suffix, upper_mask) + ) + return stem + cased, kind + + +def _hdf5_doc(filename: str | None) -> dict: + """Return a file document labelled HDF5 with the given filename.""" + doc: dict = {"file_format": {"name": "HDF5"}} + if filename is not None: + doc["filename"] = filename + return doc + + +class TestMatrixSourceKind: + """Extension is the only discriminator among HDF5-labelled files.""" + + @pytest.mark.parametrize( + ("filename", "expected"), + [ + ("sample.mcool", MatrixSource.MCOOL), + ("sample.cool", MatrixSource.COOL), + ("sample.hic", MatrixSource.HIC), + ], + ) + def test_should_classify_each_contact_map_container(self, filename, expected): + """Test that each supported container is recognized. + + Given: + An HDF5-labelled document whose filename carries a contact-map + extension. + When: + matrix_source_kind is called. + Then: + It should return the matching MatrixSource. + """ + # Act + result = matrix_source_kind(_hdf5_doc(filename)) + + # Assert + assert result is expected + + def test_should_classify_case_insensitively(self): + """Test that an upper-cased extension still classifies. + + Given: + A document named with an upper-cased ``.MCOOL`` extension. + When: + matrix_source_kind is called. + Then: + It should return MCOOL — upstream filename casing is not + something the DCC guarantees. + """ + # Act + result = matrix_source_kind(_hdf5_doc("SAMPLE.MCOOL")) + + # Assert + assert result is MatrixSource.MCOOL + + @pytest.mark.parametrize("filename", ["matrix.h5ad", "raw.h5", "data.hdf5"]) + def test_should_reject_other_hdf5_formats(self, filename): + """Test that the HDF5 ontology collision is actually escaped. + + Given: + A document sharing the ``HDF5`` EDAM term with contact maps — + an AnnData matrix or a bare HDF5 container. + When: + matrix_source_kind is called. + Then: + It should return None. This is the whole reason the module + exists: ``file_format.name`` is ``"HDF5"`` for all of these, + so the registry alone cannot tell them apart. + """ + # Act + result = matrix_source_kind(_hdf5_doc(filename)) + + # Assert + assert result is None + + def test_should_reject_a_non_hdf5_format_with_a_matching_extension(self): + """Test that the format gate runs before the extension check. + + Given: + A document named ``sample.mcool`` but labelled ``BED``. + When: + matrix_source_kind is called. + Then: + It should return None — a filename alone must not be able to + make a non-HDF5 record look like a contact map. + """ + # Arrange + doc = {"file_format": {"name": "BED"}, "filename": "sample.mcool"} + + # Act + result = matrix_source_kind(doc) + + # Assert + assert result is None + + @pytest.mark.parametrize( + "doc", + [ + {}, + {"filename": "sample.mcool"}, + {"file_format": None, "filename": "sample.mcool"}, + {"file_format": "HDF5", "filename": "sample.mcool"}, + {"file_format": {"name": "HDF5"}}, + {"file_format": {"name": "HDF5"}, "filename": ""}, + {"file_format": {"name": "HDF5"}, "filename": None}, + ], + ids=[ + "empty", + "no-format", + "null-format", + "format-not-a-dict", + "no-filename", + "empty-filename", + "null-filename", + ], + ) + def test_should_return_none_for_an_incomplete_document(self, doc): + """Test that a malformed document classifies as not-a-contact-map. + + Given: + A document missing or mistyping ``file_format`` or ``filename``. + When: + matrix_source_kind is called. + Then: + It should return None rather than raising, so a stale or + hand-edited record degrades to "not tileable" instead of + 500ing a request. + """ + # Act + result = matrix_source_kind(doc) + + # Assert + assert result is None + + @pytest.mark.parametrize( + "filename", + [".mcool", "sample.mcool.gz"], + ids=["dotfile", "buried-before-gz"], + ) + def test_should_return_none_when_the_extension_is_not_the_final_suffix( + self, filename + ): + """Test that only the final suffix can classify a file. + + Given: + An HDF5-labelled document named ``.mcool`` — a dotfile whose + whole name is the extension — or ``sample.mcool.gz``, where + the contact-map extension sits before a final ``.gz``. + When: + matrix_source_kind is called. + Then: + It should return None for both — a contact-map extension + anywhere but the final suffix position must not classify. + """ + # Act + result = matrix_source_kind(_hdf5_doc(filename)) + + # Assert + assert result is None + + @settings(max_examples=50) + @given(doc=_documents) + def test_should_classify_totally_and_soundly(self, doc): + """Test totality and soundness over arbitrary documents. + + Given: + An arbitrary JSON-like document — filename possibly a + non-string, ``file_format`` possibly a non-dict, either field + possibly absent, plus nested garbage. + When: + matrix_source_kind is called. + Then: + It should never raise, and return non-None exactly when the + format name is ``"HDF5"``, the filename is a non-empty + string, and its lower-cased final suffix is one of + ``.mcool``/``.cool``/``.hic`` — with the returned kind + matching that suffix. + """ + # Act + result = matrix_source_kind(doc) + + # Assert + file_format = doc.get("file_format") + filename = doc.get("filename") + classifiable = ( + isinstance(file_format, dict) + and file_format.get("name") == "HDF5" + and isinstance(filename, str) + and bool(filename) + and PurePosixPath(filename.lower()).suffix in _CONTACT_MAP_SUFFIXES + ) + if classifiable: + assert isinstance(result, MatrixSource) + assert "." + result.value == PurePosixPath(filename.lower()).suffix + else: + assert result is None + + @settings(max_examples=50) + @given(case=_cased_contact_map_cases()) + def test_should_classify_invariantly_of_stem_and_casing(self, case): + """Test stem- and casing-invariance of classification. + + Given: + An HDF5-labelled document whose filename joins an arbitrary + slash-free stem to a contact-map suffix under per-character + random casing. + When: + matrix_source_kind and needs_materialization are called. + Then: + It should classify as the suffix's kind regardless of stem or + casing, and needs_materialization should be True exactly for + MCOOL and COOL. + """ + # Arrange + filename, expected = case + doc = _hdf5_doc(filename) + + # Act + kind = matrix_source_kind(doc) + materialize = needs_materialization(doc) + + # Assert + assert kind is expected + assert materialize is (expected in {MatrixSource.MCOOL, MatrixSource.COOL}) + + +class TestNeedsMaterialization: + """Only coolers are materialized; .hic is refused with 501.""" + + @pytest.mark.parametrize( + ("filename", "expected"), + [ + ("sample.mcool", True), + ("sample.cool", True), + ("sample.hic", False), + ("matrix.h5ad", False), + ], + ) + def test_should_require_materialization_only_for_coolers( + self, filename, expected + ): + """Test which formats gate on a cached artifact. + + Given: + An HDF5-labelled document of each supported kind. + When: + needs_materialization is called. + Then: + Coolers should require it, while ``.hic`` should not — cfdb + does not serve tiles from ``.hic`` and never materializes an + artifact for it, so the endpoints refuse it with 501 instead + of gating on a cached artifact. + """ + # Act + result = needs_materialization(_hdf5_doc(filename)) + + # Assert + assert result is expected + + +class TestImportHygiene: + """The classifier crosses the wool pickle boundary, so it stays light.""" + + def test_should_not_import_the_workflow_or_aws_stack(self): + """Test that the module pulls in nothing heavy. + + Given: + ``cfdb.tilesets.formats``, imported by the worker-side + processor that is cloudpickled into a wool worker. + When: + Its module-level imports are inspected. + Then: + It should reference only the standard library — no boto3, no + cfdb.workflows.cache, no clodius. A convenience import of + ``processors.tools.format_name`` would drag the cache module + and boto3 across the boundary with it. + """ + # Arrange + import ast + import pathlib + + import cfdb.tilesets.formats as module + + source = pathlib.Path(module.__file__).read_text() + tree = ast.parse(source) + + # Act + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module.split(".")[0]) + + # Assert + assert imported <= {"__future__", "enum", "pathlib", "typing"}, imported diff --git a/tests/test_tilesets/test_hdf5_routing_regression.py b/tests/test_tilesets/test_hdf5_routing_regression.py new file mode 100644 index 0000000..e13ad34 --- /dev/null +++ b/tests/test_tilesets/test_hdf5_routing_regression.py @@ -0,0 +1,251 @@ +"""Pins what /data and /index do once a processor claims the HDF5 format. + +:class:`~cfdb.workflows.processors.matrix.MatrixTilesetProcessor` declares +``supported_formats = {"HDF5"}``, so ``ProcessorRegistry.lookup_for`` now +returns a processor for files it previously returned ``None`` for — every +``.h5ad``, every bare ``.h5``, and every contact map. Five call sites across +three modules read that result. These tests pin what each of them does +afterwards, because the change is invisible in the tile tests: it shows up +in endpoints that have nothing to do with tiles. +""" + +from __future__ import annotations + +import pytest +from fastapi import HTTPException + +from cfdb import api +from cfdb.api.routers.cache_stream import ( + probe_workflow_readiness, + serve_workflow_artifact_or_dispatch, +) +from cfdb.api.routers.data import stream_file_status +from cfdb.api.routers.index import stream_index_file +from cfdb.services import locks +from cfdb.workflows.cache import LocalFsCache +from cfdb.workflows.executor import WoolExecutor +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from cfdb.workflows.processors.registry import ProcessorRegistry +from tests.test_workflows import FIXTURE_MD5 + + +def _make_request(method: str = "GET"): + """Return a minimal mock request object.""" + + class FakeRequest: + def __init__(self): + self.method = method + + return FakeRequest() + + +def _hdf5_doc(filename: str) -> dict: + """Return an HDF5-labelled 4DN file document.""" + return { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": FIXTURE_MD5, + "filename": filename, + "file_format": {"name": "HDF5"}, + "access_url": "https://data.4dnucleome.org/files/" + filename, + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + + +@pytest.fixture() +def wired_subsystem(mocker, tmp_path, mock_db): + """Wire the workflow subsystem with only the matrix processor registered.""" + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + registry = ProcessorRegistry() + registry.register(MatrixTilesetProcessor()) + cache = LocalFsCache(tmp_path / "cache") + mocker.patch.object(api, "cache", cache) + mocker.patch.object(api, "processor_registry", registry) + mocker.patch.object( + api, + "executor", + WoolExecutor(mock_db, cache, registry, workdir_root=tmp_path / "jobs"), + ) + return cache + + +class TestDataFallThrough: + """/data must be unaffected: no HDF5 file gains a DATA artifact.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("filename", ["sample.mcool", "matrix.h5ad"]) + async def test_dispatch_helper_should_fall_through_for_hdf5( + self, wired_subsystem, filename + ): + """Test that /data still streams HDF5 files straight from upstream. + + Given: + An HDF5 file — a contact map or an AnnData matrix — with the + matrix processor registered. + When: + serve_workflow_artifact_or_dispatch is asked for a DATA + artifact. + Then: + It should return None so the caller falls through to upstream + streaming. A contact map is claimed by the processor but + produces only a TILESET, and AnnData is not claimed at all — + so neither can turn a /data GET into a 202. + """ + # Act + result = await serve_workflow_artifact_or_dispatch( + _hdf5_doc(filename), + ArtifactKind.DATA, + _make_request(), + None, + head_404_detail="unused", + ) + + # Assert + assert result is None + + @pytest.mark.asyncio + @pytest.mark.parametrize("filename", ["sample.mcool", "matrix.h5ad"]) + async def test_readiness_probe_should_report_ready_for_hdf5( + self, wired_subsystem, mock_db, filename + ): + """Test that /data/status still reports HDF5 files as ready. + + Given: + An HDF5 file with the matrix processor registered. + When: + The /data readiness probe runs. + Then: + It should report ready — /data serves these as raw upstream + bytes and always did. A contact map being separately tileable + is not a fact about /data. + """ + # Arrange + mock_db.files.docs = [] + mock_db.file.docs = [_hdf5_doc(filename)] + + # Act + result = await stream_file_status("4dn", "4DNFIABC123") + + # Assert + assert result == {"ready": True} + + @pytest.mark.asyncio + async def test_probe_should_not_claim_a_tileset_artifact_for_data( + self, wired_subsystem + ): + """Test that a TILESET artifact is invisible to the DATA probe. + + Given: + A contact map whose processor produces only a TILESET. + When: + probe_workflow_readiness is asked about DATA. + Then: + It should return None — "not a workflow question for this + artifact kind" — rather than False, which the caller would + render as ``ready: false`` and never become true. + """ + # Act + result = await probe_workflow_readiness( + _hdf5_doc("sample.mcool"), ArtifactKind.DATA + ) + + # Assert + assert result is None + + +class TestIndexFallThrough: + """/index must keep 404ing, but the detail string moves for some files.""" + + @pytest.mark.asyncio + async def test_should_404_a_contact_map_as_a_file_without_an_index( + self, wired_subsystem, mock_db + ): + """Test that a contact map still 404s from /index, unchanged. + + Given: + An ``.mcool`` with no sidecar and the subsystem wired. + When: + stream_index_file is called. + Then: + It should 404 with the original "for this file" detail. The + processor advertises a TILESET, so the + no-artifacts-for-this-format branch does not fire and the + terminal message is the one it always was. + """ + # Arrange + mock_db.files.docs = [] + mock_db.file.docs = [_hdf5_doc("sample.mcool")] + + # Act + with pytest.raises(HTTPException) as exc_info: + await stream_index_file("4dn", "4DNFIABC123", _make_request(), range=None) + + # Assert + assert exc_info.value.status_code == 404 + assert exc_info.value.detail == "No index file available for this file" + + @pytest.mark.asyncio + @pytest.mark.parametrize("filename", ["matrix.h5ad", "sample.hic"]) + async def test_should_404_an_unclaimed_hdf5_as_a_format_without_an_index( + self, wired_subsystem, mock_db, filename + ): + """Test the one detail string this change moves. + + Given: + An HDF5 file the matrix processor does not claim — AnnData, + or a ``.hic``, which cfdb does not serve tiles from and does + not materialize. + When: + stream_index_file is called. + Then: + It should still 404, but now with the "for this file format" + detail rather than "for this file". Before the processor + existed, ``lookup_for`` returned None and the + no-artifacts branch was unreachable; now a processor is + matched and advertises no artifacts, which is exactly what + that branch means. Same status code, different string — + pinned here so the change stays deliberate. + """ + # Arrange + mock_db.files.docs = [] + mock_db.file.docs = [_hdf5_doc(filename)] + + # Act + with pytest.raises(HTTPException) as exc_info: + await stream_index_file("4dn", "4DNFIABC123", _make_request(), range=None) + + # Assert + assert exc_info.value.status_code == 404 + assert exc_info.value.detail == "No index file available for this file format" + + +class TestRegistryLookup: + """The registry now matches HDF5, and the per-file hook does the rest.""" + + def test_lookup_should_match_every_hdf5_file(self): + """Test that format-name matching alone cannot discriminate. + + Given: + An AnnData matrix, which shares the ``HDF5`` EDAM term with + every contact map. + When: + ProcessorRegistry.lookup_for runs. + Then: + It should return the matrix processor — and needs_processing + should then decline it. This two-step is the whole escape from + the ontology collision: widening ``lookup_for``'s contract to + discriminate would have meant re-verifying five call sites + across three modules. + """ + # Arrange + registry = ProcessorRegistry() + registry.register(MatrixTilesetProcessor()) + doc = _hdf5_doc("matrix.h5ad") + + # Act + processor = registry.lookup_for(doc) + + # Assert + assert isinstance(processor, MatrixTilesetProcessor) + assert processor.needs_processing(doc) is False diff --git a/tests/test_tilesets/test_service.py b/tests/test_tilesets/test_service.py new file mode 100644 index 0000000..5dacc31 --- /dev/null +++ b/tests/test_tilesets/test_service.py @@ -0,0 +1,1108 @@ +"""Tests for :class:`cfdb.tilesets.service.TilesetService`.""" + +from __future__ import annotations + +import asyncio +import base64 + +import numpy as np +import pytest +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +from cfdb.tilesets.errors import ( + TilesetHydrationTimeout, + TilesetIdentityIncomplete, + TilesetNotReady, + TilesetUnsupported, +) +from cfdb.tilesets.service import build_service +from cfdb.workflows.cache import LocalFsCache +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from tests.fixtures.remote_cache import FakeRemoteCache +from tests.test_workflows import FIXTURE_MD5, FIXTURE_MD5_ALT + +UID = "4dn/4DNFIABC123" + + +def _file_doc(md5: str = FIXTURE_MD5, **overrides) -> dict: + """Return a projected document for a 4DN mcool.""" + doc = { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": md5, + "filename": "sample.mcool", + "file_format": {"name": "HDF5"}, + "genome_assembly": "GRCh38", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + doc.update(overrides) + return doc + + +def _tileset_key(doc: dict) -> str: + """The TILESET cache key for ``doc``, derived the way the service does.""" + return MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + + +async def _seed(cache: LocalFsCache, mcool, doc: dict, tmp_path) -> str: + """Commit ``mcool`` as the TILESET artifact for ``doc``; return the key.""" + key = _tileset_key(doc) + staged = tmp_path / f"staged-{key.replace('/', '_')}" + staged.write_bytes(mcool.read_bytes()) + await cache.put(key, staged) + return key + + +@pytest.fixture() +def service_factory(tmp_path): + """Build services sharing one LocalFsCache, and close them afterwards.""" + cache = LocalFsCache(tmp_path / "cache") + built = [] + + def _build(**overrides): + kwargs = { + "root": tmp_path / "tilesets", + "cache_provider": lambda: cache, + "disk_cache_bytes": 10**9, + "open_max": 2, + "tile_cache_bytes": 10**7, + "threads": 2, + "hydrate_timeout_s": 30, + } + kwargs.update(overrides) + service = build_service(**kwargs) + built.append(service) + return service + + yield cache, _build + + async def _close(): + for service in built: + await service.aclose() + + asyncio.run(_close()) + + +class TestResolution: + """What the service refuses to open, and why.""" + + @pytest.mark.asyncio + async def test_should_refuse_a_file_that_is_not_a_contact_map( + self, service_factory + ): + """Test that a non-matrix file is permanently unsupported. + + Given: + An AnnData matrix, which shares the HDF5 EDAM term with + contact maps. + When: + tileset_info is requested. + Then: + It should raise TilesetUnsupported — a permanent condition, + distinct from an artifact that has merely not been built yet. + """ + # Arrange + _, build = service_factory + service = build() + + # Act & Assert + with pytest.raises(TilesetUnsupported): + await service.tileset_info(UID, _file_doc(filename="matrix.h5ad")) + + @pytest.mark.asyncio + async def test_should_report_not_ready_when_no_artifact_is_cached( + self, service_factory + ): + """Test that an unbuilt artifact is not-ready, not unsupported. + + Given: + A contact map with nothing in the cache. + When: + tileset_info is requested. + Then: + It should raise TilesetNotReady and dispatch nothing. The tile + endpoints are pure readers; a heatmap track cannot be handed a + job id, so building is the preparation channel's job. + """ + # Arrange + _, build = service_factory + service = build() + + # Act & Assert + with pytest.raises(TilesetNotReady): + await service.tileset_info(UID, _file_doc()) + + @pytest.mark.asyncio + @pytest.mark.parametrize("method", ["tileset_info", "tiles"]) + async def test_should_refuse_a_document_with_no_md5( + self, service_factory, method + ): + """Test the identity seam behind the router's 409. + + Given: + A contact-map document carrying no md5, so no cache key can + be derived for it. + When: + tileset_info or tiles is requested. + Then: + It should raise TilesetIdentityIncomplete — the service's own + vocabulary, not the ValueError the key derivation raises — so + the router can render a 409 instead of a 500. + """ + # Arrange + _, build = service_factory + doc = _file_doc() + del doc["md5"] + service = build() + + # Act & Assert + with pytest.raises(TilesetIdentityIncomplete): + if method == "tileset_info": + await service.tileset_info(UID, doc) + else: + await service.tiles(UID, doc, [f"{UID}.0.0.0"]) + + @pytest.mark.asyncio + async def test_should_report_not_ready_when_no_cache_is_configured( + self, service_factory + ): + """Test the guard for a workflow subsystem that never came up. + + Given: + A cache provider that resolves to None — the shape of a + deployment with no cache configured. + When: + tileset_info is requested. + Then: + It should raise TilesetNotReady rather than an AttributeError + from calling head on None — the router turns this into a 404, + not a 500. + """ + # Arrange + _, build = service_factory + service = build(cache_provider=lambda: None) + + # Act & Assert + with pytest.raises(TilesetNotReady): + await service.tileset_info(UID, _file_doc()) + + +class TestTilesetInfo: + """The HiGlass info document.""" + + @pytest.mark.asyncio + async def test_should_serve_info_from_a_cached_artifact( + self, service_factory, tiny_mcool, tmp_path + ): + """Test the assembled tileset_info document. + + Given: + A cached mcool artifact. + When: + tileset_info is requested. + Then: + It should carry clodius's ladder fields plus the four the + library does not model, and omit the two clodius deliberately + leaves unset for an explicit ladder. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + info = await service.tileset_info(UID, doc) + + # Assert + assert info["datatype"] == "matrix" + assert info["name"] == "sample.mcool" + assert info["uuid"] == UID + assert info["coordSystem"] == "GRCh38" + assert info["resolutions"] == [1, 2, 4] + assert info["tile_size"] == 256 + assert "max_width" not in info + assert "max_zoom" not in info + + @pytest.mark.asyncio + async def test_should_omit_coord_system_when_no_assembly_is_recorded( + self, service_factory, tiny_mcool, tmp_path + ): + """Test the 21 4DN files with no genome assembly. + + Given: + A contact map whose document records no genome_assembly. + When: + tileset_info is requested. + Then: + ``coordSystem`` should be absent entirely rather than empty. + HiGlass matches it against chromosome-info tilesets, so a + placeholder risks a silently misaligned track, whereas an + absent key makes the client fall back to the chromsizes array + the document always carries. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + del doc["genome_assembly"] + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + info = await service.tileset_info(UID, doc) + + # Assert + assert "coordSystem" not in info + assert info["chromsizes"] + + +class TestHydrationTimeout: + """A hydration that overruns its budget must fail legibly.""" + + @pytest.mark.asyncio + async def test_should_raise_hydration_timeout_when_the_download_stalls( + self, service_factory, tiny_mcool + ): + """Test the translation of a stalled hydration into 503 vocabulary. + + Given: + A non-local cache whose download stream is gated shut, and a + hydrate timeout of a few milliseconds. + When: + tileset_info is requested. + Then: + It should raise TilesetHydrationTimeout chained from the + asyncio timeout — the type the router renders as 503 with + Retry-After, because the download may well finish and a + retry will find it. + """ + # Arrange + _, build = service_factory + doc = _file_doc() + remote = FakeRemoteCache({_tileset_key(doc): tiny_mcool.read_bytes()}) + remote.gate = asyncio.Event() # never opened: the download stalls + service = build(cache_provider=lambda: remote, hydrate_timeout_s=0.05) + + # Act & Assert + with pytest.raises(TilesetHydrationTimeout) as excinfo: + await service.tileset_info(UID, doc) + assert isinstance(excinfo.value.__cause__, asyncio.TimeoutError) + + +class TestTiles: + """Batch semantics, which are where the sharp edges are.""" + + @pytest.mark.asyncio + async def test_should_return_a_dense_payload_for_a_valid_tile( + self, service_factory, tiny_mcool, tmp_path + ): + """Test the tile payload shape. + + Given: + A cached mcool artifact. + When: + A valid tile is requested. + Then: + The payload should be keyed on the request string and carry a + base64 dense block with no ``shape`` field — the client + recovers the side length as sqrt(len), which is the contract + clodius emits. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + payloads = await service.tiles(UID, doc, [f"{UID}.0.0.0"]) + + # Assert + payload = payloads[f"{UID}.0.0.0"] + assert set(payload) == {"dense", "dtype", "min_value", "max_value", "size"} + flat = np.frombuffer(base64.b64decode(payload["dense"]), payload["dtype"]) + assert len(flat) == 256 * 256 + + @pytest.mark.asyncio + async def test_should_report_an_out_of_ladder_tile_as_a_per_tile_error( + self, service_factory, tiny_mcool, tmp_path + ): + """Test the one-entry-per-id contract for a position off the ladder. + + Given: + A batch pairing a valid tile with one past the resolution + ladder. + When: + tiles is called. + Then: + Both ids should appear in the response — the valid tile + returns tile data, and the out-of-ladder one gets a per-tile + ``{"error": ...}`` object rather than being omitted. clodius + now returns exactly one entry per requested id, so a response + built by zipping request to result is safe. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + payloads = await service.tiles( + UID, doc, [f"{UID}.0.0.0", f"{UID}.9.0.0"] + ) + + # Assert + assert set(payloads) == {f"{UID}.0.0.0", f"{UID}.9.0.0"} + assert "dense" in payloads[f"{UID}.0.0.0"] + assert set(payloads[f"{UID}.9.0.0"]) == {"error"} + + @pytest.mark.asyncio + async def test_should_report_a_malformed_id_per_tile( + self, service_factory, tiny_mcool, tmp_path + ): + """Test that one bad id does not blank the whole track. + + Given: + A batch pairing a valid tile id with a malformed one. + When: + tiles is called. + Then: + The malformed id should get an error object while its sibling + returns data, matching HiGlass's per-tile error convention. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + payloads = await service.tiles( + UID, doc, [f"{UID}.0.0.0", f"{UID}.notanint.0"] + ) + + # Assert + assert "error" in payloads[f"{UID}.notanint.0"] + assert "dense" in payloads[f"{UID}.0.0.0"] + + @pytest.mark.asyncio + async def test_should_report_an_unavailable_transform_per_tile( + self, service_factory, tiny_mcool, tmp_path + ): + """Test that a bad balancing column is a tile error, not a 500. + + Given: + A tile id naming a balancing column the cooler does not carry. + When: + tiles is called. + Then: + It should return an error object. clodius propagates this + rather than skipping it — a client asking for a column that + does not exist made a different kind of mistake than one + asking for a tile off the end of the genome — and the boundary + renders it rather than failing the request. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + payloads = await service.tiles(UID, doc, [f"{UID}.0.0.0.nosuchcolumn"]) + + # Assert + assert "error" in payloads[f"{UID}.0.0.0.nosuchcolumn"] + + @pytest.mark.asyncio + async def test_should_treat_two_spellings_of_one_tile_as_two_entries( + self, service_factory, tiny_mcool, tmp_path + ): + """Test that the response key is the request string, verbatim. + + Given: + The same tile requested twice, once with a trailing options + separator. + When: + tiles is called. + Then: + Both spellings should appear. clodius makes the raw request + string part of a tile id's identity precisely because the + client matches responses to requests by exact string. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + + # Act + payloads = await service.tiles(UID, doc, [f"{UID}.0.0.0", f"{UID}.0.0.0,"]) + + # Assert + assert set(payloads) == {f"{UID}.0.0.0", f"{UID}.0.0.0,"} + + @pytest.mark.asyncio + async def test_should_return_empty_without_opening_for_an_empty_batch( + self, service_factory, mocker + ): + """Test the zero-input short-circuit. + + Given: + A contact-map document and an empty tile-id sequence. + When: + tiles is called. + Then: + It should return an empty mapping without checking out a + tileset — there is nothing to read, so nothing should be + hydrated or opened, even when no artifact exists at all. + """ + # Arrange + _, build = service_factory + service = build() + checkout = mocker.spy(service, "checkout") + + # Act + payloads = await service.tiles(UID, _file_doc(), []) + + # Assert + assert payloads == {} + checkout.assert_not_called() + + @pytest.mark.asyncio + async def test_should_scope_a_missing_balancing_column_to_its_modifier_group( + self, service_factory, tiny_mcool, tmp_path + ): + """Test the blast radius of an unusable transform. + + Given: + A batch pairing a valid (untransformed) tile id with one + naming a balancing column the cooler does not carry — an id + that parses fine, so clodius raises TileError only once it + tries to resolve the transform for that id's (zoom, modifier) + group. + When: + tiles is called. + Then: + Only the id sharing the bad modifier's group should get an + error payload. clodius batches per (zoom, modifier) rather + than per whole request, so a valid sibling requesting a + different (in this case, no) transform is unaffected. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + valid = f"{UID}.0.0.0" + bad = f"{UID}.0.0.0.nosuchcolumn" + + # Act + payloads = await service.tiles(UID, doc, [valid, bad]) + + # Assert + assert "error" in payloads[bad] + assert "dense" in payloads[valid] + + @pytest.mark.asyncio + async def test_should_reraise_a_non_tile_error_from_the_backend( + self, service_factory, tiny_mcool, tmp_path, mocker + ): + """Test that a genuine bug is not laundered into a tile error. + + Given: + A tile backend whose batch read raises RuntimeError — not a + clodius TileError. + When: + tiles is called. + Then: + The RuntimeError should propagate. Per the errors contract, + only clodius's own per-tile hierarchy becomes error payloads; + anything else is a bug and must surface as a 500. + """ + # Arrange + from clodius.tiles_v2.cooler import CoolerTileset + + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + mocker.patch.object( + CoolerTileset, "tiles", side_effect=RuntimeError("h5py went away") + ) + + # Act & Assert + with pytest.raises(RuntimeError, match="h5py went away"): + await service.tiles(UID, doc, [f"{UID}.0.0.0"]) + + @pytest.mark.asyncio + async def test_should_serve_a_named_balancing_transform( + self, service_factory, balanced_mcool, tmp_path + ): + """Test the balancing happy path against a weighted cooler. + + Given: + A cached mcool carrying both a default ``weight`` column and + a distinct ``alt_weight`` column. + When: + The same tile is requested with no modifier and with the + ``alt_weight`` transform named. + Then: + Both should return dense payloads, and the transformed values + should differ from the default — proving the named column was + honoured rather than silently falling back to ``weight``. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, balanced_mcool, doc, tmp_path) + service = build() + default_id = f"{UID}.0.0.0" + alt_id = f"{UID}.0.0.0.alt_weight" + + # Act + payloads = await service.tiles(UID, doc, [default_id, alt_id]) + + # Assert + assert "dense" in payloads[default_id] + assert "dense" in payloads[alt_id] + default_block = np.frombuffer( + base64.b64decode(payloads[default_id]["dense"]), + payloads[default_id]["dtype"], + ).astype(np.float64) + alt_block = np.frombuffer( + base64.b64decode(payloads[alt_id]["dense"]), + payloads[alt_id]["dtype"], + ).astype(np.float64) + assert not np.array_equal(default_block, alt_block, equal_nan=True) + + @settings( + max_examples=10, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + positions=st.lists( + st.tuples( + st.integers(min_value=0, max_value=2), + st.integers(min_value=0, max_value=2), + st.integers(min_value=0, max_value=2), + ), + min_size=1, + max_size=4, + unique=True, + ) + ) + @pytest.mark.asyncio + async def test_should_emit_the_wire_contract_for_any_valid_batch( + self, service_factory, tiny_mcool, tmp_path, positions + ): + """Test the HiGlass wire-contract invariant over valid batches. + + Given: + Generated batches of (z, x, y) positions within tiny_mcool's + ladder — resolutions [1, 2, 4] over a 3000 bp genome, so z in + 0..2 and x, y in 0..2 are valid at every zoom. + When: + tiles is called. + Then: + It should key every payload on a requested id, carry exactly + the five contractual fields with no ``shape``, declare a + dtype of float16 or float32, and encode a base64 dense block + that decodes to 256x256 values of the declared dtype. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + ids = [f"{UID}.{z}.{x}.{y}" for z, x, y in positions] + + # Act + payloads = await service.tiles(UID, doc, ids) + + # Assert + assert set(payloads) <= set(ids) + for payload in payloads.values(): + assert set(payload) == { + "dense", + "dtype", + "min_value", + "max_value", + "size", + } + assert payload["dtype"] in {"float16", "float32"} + flat = np.frombuffer( + base64.b64decode(payload["dense"]), payload["dtype"] + ) + assert flat.shape == (256 * 256,) + + @settings( + max_examples=15, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + suffix=st.one_of( + st.text( + alphabet=st.sampled_from(list("0123456789.,:-_ weight")), + max_size=12, + ), + st.text(max_size=12), + ) + ) + @example(suffix="") + @example(suffix="0.0.0,") + @example(suffix=" 1.0.0") + @example(suffix="0.0.0.") + @pytest.mark.asyncio + async def test_should_contain_any_garbage_id_within_the_batch_contract( + self, service_factory, tiny_mcool, tmp_path, suffix + ): + """Test the parse-boundary robustness of a mixed batch. + + Given: + Batches pairing one valid tile id with a generated garbage + suffix appended to the valid uid prefix — empty, dotted, + comma-optioned, unicode, whitespace-padded, or naming a + transform. + When: + tiles is called. + Then: + No exception should escape; every returned key should be one + of the requested ids, and every payload either tile data or + an error object. This is a deliberate bug-hunter: a parse + failure outside clodius's TileError hierarchy would re-raise + out of the error translator and 500 the batch. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + valid = f"{UID}.0.0.0" + garbage = f"{UID}.{suffix}" + + # Act + payloads = await service.tiles(UID, doc, [valid, garbage]) + + # Assert + assert set(payloads) <= {valid, garbage} + for payload in payloads.values(): + assert "dense" in payload or "error" in payload + + +class TestTileCache: + """Repeat pans must not re-read the file.""" + + @pytest.mark.asyncio + async def test_should_serve_a_repeat_request_from_cache( + self, service_factory, tiny_mcool, tmp_path, mocker + ): + """Test that a second identical request does not reach the tileset. + + Given: + A tile already read once. + When: + The same tile is requested again. + Then: + The tileset should not be checked out again — h5py serializes + on a global lock, so re-reading is the cost the cache exists + to avoid. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + await service.tiles(UID, doc, [f"{UID}.0.0.0"]) + checkout = mocker.spy(service, "checkout") + + # Act + payloads = await service.tiles(UID, doc, [f"{UID}.0.0.0"]) + + # Assert + assert "dense" in payloads[f"{UID}.0.0.0"] + checkout.assert_not_called() + + @pytest.mark.asyncio + async def test_should_evict_by_bytes_rather_than_entry_count( + self, service_factory, tiny_mcool, tmp_path + ): + """Test that the tile cache respects a byte budget. + + Given: + A tile cache sized to hold roughly one tile. + When: + Several distinct tiles are read. + Then: + The cache should stay within its budget. Bounding by entry + count instead would let ~350 KiB base64 payloads accumulate + into gigabytes on a 2 GiB task. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build(tile_cache_bytes=200_000) + + # Act + await service.tiles( + UID, + doc, + [f"{UID}.2.0.0", f"{UID}.2.1.0", f"{UID}.2.0.1", f"{UID}.2.1.1"], + ) + + # Assert + assert 0 < service.tile_cache_bytes_used <= 200_000 + + @pytest.mark.asyncio + async def test_should_miss_when_the_source_bytes_change( + self, service_factory, tiny_mcool, tmp_path, mocker + ): + """Test that the tile cache key follows the artifact's identity. + + Given: + A tile cached for a document, and then the same file with a + different upstream md5. + When: + The same tile position is requested. + Then: + It should be re-read rather than served from cache. The tile + key embeds the artifact cache key, which embeds the md5 and + the processor version — so an upstream byte change or a + version bump invalidates every tile for free. + """ + # Arrange + cache, build = service_factory + original = _file_doc() + await _seed(cache, tiny_mcool, original, tmp_path) + revised = _file_doc(md5=FIXTURE_MD5_ALT) + await _seed(cache, tiny_mcool, revised, tmp_path) + service = build() + await service.tiles(UID, original, [f"{UID}.0.0.0"]) + checkout = mocker.spy(service, "checkout") + + # Act + await service.tiles(UID, revised, [f"{UID}.0.0.0"]) + + # Assert + checkout.assert_called_once() + + @pytest.mark.asyncio + async def test_should_serve_but_not_cache_an_oversized_payload( + self, service_factory, tiny_mcool, tmp_path, mocker + ): + """Test that a payload larger than the whole budget is skipped. + + Given: + A tile cache whose byte budget is smaller than a single tile + payload. + When: + The same tile is requested twice. + Then: + The first response should carry data while the cache stays + at zero bytes, and the second request should check the + tileset out again — served but uncacheable, rather than + evicting the world to admit one tile that cannot fit. + """ + # Arrange + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build(tile_cache_bytes=1_000) + tile = f"{UID}.0.0.0" + first = await service.tiles(UID, doc, [tile]) + checkout = mocker.spy(service, "checkout") + + # Act + second = await service.tiles(UID, doc, [tile]) + + # Assert + assert "dense" in first[tile] + assert "dense" in second[tile] + assert service.tile_cache_bytes_used == 0 + checkout.assert_called_once() + + @pytest.mark.asyncio + async def test_should_read_only_the_cold_tile_of_a_partial_hit_batch( + self, service_factory, tiny_mcool, tmp_path, mocker + ): + """Test that cached entries are merged with a narrowed file read. + + Given: + Tile A already cached, and a batch pairing A with a cold + tile B. + When: + tiles is called for both. + Then: + Both payloads should be returned, but only B should reach + the backend — the batch handed to the tileset carries + exactly the one id the cache could not answer. + """ + # Arrange + from clodius.tiles_v2.cooler import CoolerTileset + + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + tile_a = f"{UID}.0.0.0" + tile_b = f"{UID}.1.0.0" + await service.tiles(UID, doc, [tile_a]) + tiles_spy = mocker.spy(CoolerTileset, "tiles") + + # Act + payloads = await service.tiles(UID, doc, [tile_a, tile_b]) + + # Assert + assert "dense" in payloads[tile_a] + assert "dense" in payloads[tile_b] + tiles_spy.assert_called_once() + parsed = tiles_spy.call_args.args[1] + assert [tile_id.raw for tile_id in parsed] == [tile_b] + + @pytest.mark.asyncio + async def test_should_release_cached_tiles_with_their_evicted_handle( + self, service_factory, tiny_mcool, tmp_path + ): + """Test the coupling between the tile cache and the handle pool. + + Given: + An open-tileset cap of one, tiles cached for dataset A, and + a second dataset B. + When: + Opening B evicts A's handle past the cap. + Then: + The tile cache should drop by exactly A's bytes — cached + tiles are released with their handle, so an evicted dataset + cannot pin stale payloads in memory. + """ + # Arrange + cache, build = service_factory + doc_a = _file_doc() + doc_b = _file_doc(md5=FIXTURE_MD5_ALT, local_id="4DNFIXYZ789") + await _seed(cache, tiny_mcool, doc_a, tmp_path) + await _seed(cache, tiny_mcool, doc_b, tmp_path) + service = build(open_max=1) + await service.tiles(UID, doc_a, [f"{UID}.0.0.0"]) + cached_bytes = service.tile_cache_bytes_used + assert cached_bytes > 0 + + # Act + await service.tileset_info("4dn/4DNFIXYZ789", doc_b) + + # Assert — only A's tiles were cached, so dropping A's bytes is zero + assert service.tile_cache_bytes_used == 0 + + +class TestOpenTilesetLifetime: + """Handles are pooled, and closing one under a reader is a segfault.""" + + @pytest.mark.asyncio + async def test_should_close_a_handle_evicted_past_the_open_cap( + self, service_factory, make_mcool, tmp_path, mocker + ): + """Test that the open-handle pool is bounded. + + Given: + An open-tileset cap of one and two distinct contact maps. + When: + Both are opened. + Then: + The first handle should be closed — observed as exactly one + ``close()`` at the clodius boundary, on the handle the first + checkout yielded. Each is an open h5py handle over a + multi-GB file, so this is a file-descriptor bound. + """ + # Arrange + from clodius.tiles_v2.cooler import CoolerTileset + + cache, build = service_factory + first_doc = _file_doc() + second_doc = _file_doc(md5=FIXTURE_MD5_ALT, local_id="4DNFIXYZ789") + await _seed(cache, make_mcool(), first_doc, tmp_path) + await _seed(cache, make_mcool(), second_doc, tmp_path) + service = build(open_max=1) + close_spy = mocker.spy(CoolerTileset, "close") + + async with service.checkout(UID, first_doc) as first: + assert first.info() is not None # force the handle open + close_spy.assert_not_called() + + # Act + await service.tileset_info("4dn/4DNFIXYZ789", second_doc) + + # Assert + close_spy.assert_called_once() + assert close_spy.call_args.args[0] is first + + @pytest.mark.asyncio + async def test_should_defer_closing_a_handle_that_is_being_read( + self, service_factory, make_mcool, tmp_path, mocker + ): + """Test the reference-counted close that prevents a segfault. + + Given: + A tileset checked out by a reader, and a second tileset + opened that evicts it past the cap. + When: + The eviction lands while the first reader is still inside its + checkout. + Then: + The handle must stay open until that reader leaves — no + ``close()`` reaches the clodius boundary and the held handle + keeps serving reads — and be closed exactly once afterwards. + Closing an h5py handle under a live numpy read in a worker + thread is a segfault, not an exception, so this cannot be + left to chance. + """ + # Arrange + from clodius.tiles_v2.cooler import CoolerTileset + + cache, build = service_factory + first_doc = _file_doc() + second_doc = _file_doc(md5=FIXTURE_MD5_ALT, local_id="4DNFIXYZ789") + await _seed(cache, make_mcool(), first_doc, tmp_path) + await _seed(cache, make_mcool(), second_doc, tmp_path) + service = build(open_max=1) + close_spy = mocker.spy(CoolerTileset, "close") + + # Act & Assert + async with service.checkout(UID, first_doc) as held: + assert held.info() is not None # force the handle open + await service.tileset_info("4dn/4DNFIXYZ789", second_doc) + close_spy.assert_not_called() + assert held.info() is not None # still serving the reader + + # Assert — the last reader out performs the close + close_spy.assert_called_once() + assert close_spy.call_args.args[0] is held + + @pytest.mark.asyncio + async def test_should_share_one_handle_between_concurrent_cold_opens( + self, service_factory, tiny_mcool, mocker + ): + """Test the winner/loser dedup of a concurrent cold open. + + Given: + Two concurrent checkouts of the same cold uid over a gated + remote cache, so both miss the pool and both hydrate. + When: + The gate opens and both complete. + Then: + Both readers should be handed the identical tileset object, + and at most one close should reach the clodius boundary — + the loser's redundant handle, never the shared one — so no + second h5py handle leaks onto the same file. + """ + # Arrange + from clodius.tiles_v2.cooler import CoolerTileset + + _, build = service_factory + doc = _file_doc() + remote = FakeRemoteCache({_tileset_key(doc): tiny_mcool.read_bytes()}) + remote.gate = asyncio.Event() + service = build(cache_provider=lambda: remote) + close_spy = mocker.spy(CoolerTileset, "close") + seen = [] + + async def grab(): + async with service.checkout(UID, doc) as tileset: + seen.append(tileset) + + first = asyncio.create_task(grab()) + second = asyncio.create_task(grab()) + await asyncio.sleep(0.05) # let both miss the pool and reach the gate + + # Act + remote.gate.set() + await asyncio.gather(first, second) + + # Assert + assert seen[0] is seen[1] + assert close_spy.call_count <= 1 + assert all( + call.args[0] is not seen[0] for call in close_spy.call_args_list + ) + + @pytest.mark.asyncio + async def test_should_survive_aclose_while_a_reader_is_inside_checkout( + self, service_factory, tiny_mcool, tmp_path + ): + """Test that teardown under a live reader is orderly. + + Given: + A tile cached over a remote-style cache, and a reader inside + checkout when aclose is called. + When: + aclose completes while the reader is still in its block. + Then: + The handle should stay live for the reader, the tile cache + should be emptied, and the store's hydrated residue should + be gone — teardown must not yank the file out from under + the reader's h5py handle. + """ + # Arrange + _, build = service_factory + doc = _file_doc() + remote = FakeRemoteCache({_tileset_key(doc): tiny_mcool.read_bytes()}) + service = build(cache_provider=lambda: remote) + await service.tiles(UID, doc, [f"{UID}.0.0.0"]) + assert service.tile_cache_bytes_used > 0 + + # Act & Assert + async with service.checkout(UID, doc) as held: + await service.aclose() + assert held.info() is not None # the reader's handle stays live + + # Assert + assert service.tile_cache_bytes_used == 0 + assert not any((tmp_path / "tilesets").iterdir()) + + @pytest.mark.xfail( + strict=False, + reason=( + "aclose shuts the thread pool down before the last reader " + "leaves, so the deferred close is submitted to an " + "already-shutdown executor: _close swallows the RuntimeError " + "('cannot schedule new futures after shutdown') and the h5py " + "handle leaks at teardown instead of being closed" + ), + ) + @pytest.mark.asyncio + async def test_should_close_the_held_handle_after_aclose_under_a_reader( + self, service_factory, tiny_mcool, tmp_path, mocker + ): + """Test the deferred close of a handle held across aclose. + + Given: + A reader inside checkout when aclose is called. + When: + aclose completes and the reader then exits its block. + Then: + The held handle should be closed exactly once by the last + reader out — the same deferral contract eviction honours — + rather than leaking because the close could no longer be + scheduled. + """ + # Arrange + from clodius.tiles_v2.cooler import CoolerTileset + + cache, build = service_factory + doc = _file_doc() + await _seed(cache, tiny_mcool, doc, tmp_path) + service = build() + close_spy = mocker.spy(CoolerTileset, "close") + + # Act + async with service.checkout(UID, doc) as held: + await service.aclose() + close_spy.assert_not_called() + + # Assert + close_spy.assert_called_once() + assert close_spy.call_args.args[0] is held diff --git a/tests/test_tilesets/test_store.py b/tests/test_tilesets/test_store.py new file mode 100644 index 0000000..5cffbee --- /dev/null +++ b/tests/test_tilesets/test_store.py @@ -0,0 +1,607 @@ +"""Tests for :class:`cfdb.tilesets.store.LocalTilesetStore`.""" + +from __future__ import annotations + +import asyncio +from uuid import uuid4 + +import pytest +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st + +from cfdb.tilesets.errors import TilesetTooLarge +from cfdb.tilesets.store import LocalTilesetStore +from cfdb.workflows.cache import LocalFsCache +from tests.fixtures.remote_cache import FakeRemoteCache + + +class TestLocalFsFastPath: + """A local artifact is already a file; opening it in place is the point.""" + + @pytest.mark.asyncio + async def test_should_return_the_cache_path_without_copying(self, tmp_path): + """Test that a LocalFsCache artifact is not duplicated on disk. + + Given: + A LocalFsCache holding an artifact. + When: + path_for is called. + Then: + It should return the cache's own path and hydrate nothing. + Copying would double the disk cost of every dataset for no + gain, and it is this fast path that lets the whole tile + subsystem be tested without an S3 stand-in. + """ + # Arrange + cache = LocalFsCache(tmp_path / "cache") + source = tmp_path / "artifact.mcool" + source.write_bytes(b"payload") + await cache.put("4dn/x/tileset/abc-v0", source) + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + + # Act + path = await store.path_for( + cache, "4dn/x/tileset/abc-v0", expected_size=7 + ) + + # Assert + assert path == cache.path_for("4dn/x/tileset/abc-v0") + assert store.resident_bytes == 0 + + @pytest.mark.asyncio + async def test_should_serve_a_local_artifact_larger_than_the_budget( + self, tmp_path + ): + """Test that the disk budget does not apply to in-place artifacts. + + Given: + A LocalFsCache artifact larger than the store's entire + max_bytes budget. + When: + path_for is called. + Then: + It should return the in-place path without raising + TilesetTooLarge — the budget governs hydrated copies only, + and an artifact opened where it already lives costs the + store no disk at all. + """ + # Arrange + cache = LocalFsCache(tmp_path / "cache") + source = tmp_path / "artifact.mcool" + source.write_bytes(b"x" * 100) + await cache.put("4dn/x/tileset/abc-v0", source) + store = LocalTilesetStore(tmp_path / "local", max_bytes=10) + + # Act + path = await store.path_for( + cache, "4dn/x/tileset/abc-v0", expected_size=100 + ) + + # Assert + assert path == cache.path_for("4dn/x/tileset/abc-v0") + assert path.read_bytes() == b"x" * 100 + assert store.resident_bytes == 0 + + +class TestHydration: + """A remote artifact is pulled down once and reused.""" + + @pytest.mark.asyncio + async def test_should_hydrate_a_remote_artifact_onto_local_disk( + self, tmp_path + ): + """Test that a non-local cache is downloaded before being opened. + + Given: + A cache backend whose artifacts are not local files. + When: + path_for is called. + Then: + It should write the bytes to local disk and return that path, + because h5py opens a file rather than consuming a stream. + """ + # Arrange + cache = FakeRemoteCache({"k": b"contents"}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + + # Act + path = await store.path_for(cache, "k", expected_size=8) + + # Assert + assert path.read_bytes() == b"contents" + assert store.resident_bytes == 8 + + @pytest.mark.asyncio + async def test_should_reuse_a_hydrated_artifact(self, tmp_path): + """Test that a second request does not re-download. + + Given: + An artifact already hydrated. + When: + path_for is called again. + Then: + It should return the resident path without touching the cache + a second time. + """ + # Arrange + cache = FakeRemoteCache({"k": b"contents"}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + await store.path_for(cache, "k", expected_size=8) + + # Act + await store.path_for(cache, "k", expected_size=8) + + # Assert + assert cache.get_calls == ["k"] + + @pytest.mark.asyncio + async def test_should_download_once_under_concurrent_requests( + self, tmp_path + ): + """Test the single-flight guard around a cold dataset. + + Given: + Two concurrent requests for an artifact that is not yet + resident, with the download held open. + When: + Both call path_for. + Then: + Exactly one download should run. A heatmap track opens with a + burst of tile requests, so without this a cold multi-GB + dataset would be pulled down once per request in the burst. + """ + # Arrange + cache = FakeRemoteCache({"k": b"contents"}) + cache.gate = asyncio.Event() + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + + # Act + first = asyncio.create_task(store.path_for(cache, "k", expected_size=8)) + second = asyncio.create_task(store.path_for(cache, "k", expected_size=8)) + await asyncio.sleep(0) + cache.gate.set() + paths = await asyncio.gather(first, second) + + # Assert + assert cache.get_calls == ["k"] + assert paths[0] == paths[1] + + @pytest.mark.asyncio + async def test_should_not_leave_a_partial_file_behind_on_failure( + self, tmp_path + ): + """Test that an interrupted download cannot be mistaken for complete. + + Given: + A cache whose read raises partway through. + When: + path_for is called. + Then: + It should propagate and leave nothing resident, so a later + request re-downloads rather than handing h5py a truncated + file. + """ + # Arrange + class _Exploding(FakeRemoteCache): + def get(self, key, byte_range=None): + async def _stream(): + raise OSError("connection reset") + yield b"" # pragma: no cover + + return _stream() + + cache = _Exploding({"k": b"contents"}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + + # Act + with pytest.raises(OSError): + await store.path_for(cache, "k", expected_size=8) + + # Assert + assert store.resident_bytes == 0 + assert list((tmp_path / "local").iterdir()) == [] + + @pytest.mark.asyncio + async def test_should_refuse_an_artifact_larger_than_the_whole_budget( + self, tmp_path + ): + """Test that one oversized artifact fails instead of thrashing. + + Given: + An artifact bigger than the entire local disk budget. + When: + path_for is called. + Then: + It should raise TilesetTooLarge rather than evicting + everything and still not fitting. + """ + # Arrange + cache = FakeRemoteCache({"k": b"x" * 100}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=10) + + # Act + with pytest.raises(TilesetTooLarge): + await store.path_for(cache, "k", expected_size=100) + + # Assert + assert store.resident_bytes == 0 + + @pytest.mark.asyncio + async def test_should_rehydrate_when_the_resident_file_is_gone( + self, tmp_path + ): + """Test that residency is revalidated against the filesystem. + + Given: + A hydrated artifact whose file was deleted out-of-band. + When: + path_for is called again. + Then: + It should re-download and return a fresh readable path + rather than handing h5py a path that no longer exists. + """ + # Arrange + cache = FakeRemoteCache({"k": b"contents"}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + first = await store.path_for(cache, "k", expected_size=8) + first.unlink() + + # Act + second = await store.path_for(cache, "k", expected_size=8) + + # Assert + assert second.read_bytes() == b"contents" + assert cache.get_calls == ["k", "k"] + + @pytest.mark.asyncio + async def test_should_leave_no_residue_when_cancelled_mid_download( + self, tmp_path + ): + """Test that a cancelled hydration cleans up after itself. + + Given: + A path_for task cancelled while its download is blocked + mid-stream. + When: + The task is cancelled and awaited. + Then: + It should leave no .part residue and an empty store + directory, with resident_bytes at zero — cancellation runs + the same cleanup as any other failure, so a timed-out + request cannot poison the key. + """ + # Arrange + cache = FakeRemoteCache({"k": b"contents"}) + cache.gate = asyncio.Event() + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + task = asyncio.create_task( + store.path_for(cache, "k", expected_size=8) + ) + for _ in range(200): + if any((tmp_path / "local").iterdir()): + break + await asyncio.sleep(0.005) + else: # pragma: no cover - diagnostic guard + pytest.fail("download never reached the gated stream") + + # Act + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + # Assert + assert list((tmp_path / "local").iterdir()) == [] + assert store.resident_bytes == 0 + + @pytest.mark.asyncio + async def test_should_recover_after_a_failed_hydration(self, tmp_path): + """Test that one failed download does not poison the key's lock. + + Given: + A cache whose first read fails with OSError and whose + subsequent reads succeed. + When: + path_for is retried after the failure. + Then: + It should succeed and return the full bytes — the per-key + single-flight lock is released on failure rather than left + held or wedged in a failed state. + """ + # Arrange + class _FlakyOnce(FakeRemoteCache): + def __init__(self, blobs): + super().__init__(blobs) + self.failed = False + + def get(self, key, byte_range=None): + if not self.failed: + self.failed = True + + async def _stream(): + raise OSError("connection reset") + yield b"" # pragma: no cover + + return _stream() + return super().get(key, byte_range) + + cache = _FlakyOnce({"k": b"contents"}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=1024) + with pytest.raises(OSError): + await store.path_for(cache, "k", expected_size=8) + + # Act + path = await store.path_for(cache, "k", expected_size=8) + + # Assert + assert path.read_bytes() == b"contents" + assert store.resident_bytes == 8 + + @pytest.mark.asyncio + async def test_should_admit_at_the_budget_and_refuse_one_byte_over( + self, tmp_path + ): + """Test that the whole-budget guard is strictly greater-than. + + Given: + Artifacts sized exactly at max_bytes and at max_bytes + 1. + When: + path_for is called for each against a fresh store. + Then: + It should admit the exact-fit artifact and raise + TilesetTooLarge only for the one byte over — an artifact + that fills the budget precisely is still servable. + """ + # Arrange + cache = FakeRemoteCache({"fit": b"a" * 10, "over": b"b" * 11}) + exact = LocalTilesetStore(tmp_path / "exact", max_bytes=10) + over = LocalTilesetStore(tmp_path / "over", max_bytes=10) + + # Act + path = await exact.path_for(cache, "fit", expected_size=10) + with pytest.raises(TilesetTooLarge): + await over.path_for(cache, "over", expected_size=11) + + # Assert + assert path.read_bytes() == b"a" * 10 + assert exact.resident_bytes == 10 + assert over.resident_bytes == 0 + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + keys=st.lists( + st.text( + alphabet=st.sampled_from(list("ab_./\\-é🧬")), + min_size=1, + max_size=20, + ), + unique=True, + min_size=2, + max_size=6, + ) + ) + @example(keys=["a/b", "a_b", "..", "é"]) + @pytest.mark.asyncio + async def test_should_map_distinct_keys_to_distinct_paths_inside_root( + self, tmp_path, keys + ): + """Test that local naming is injective and contained. + + Given: + Generated sets of distinct path-ish cache keys — slashes, + dots, backslashes, unicode — each with its own content. + When: + Every key is hydrated through path_for. + Then: + It should return pairwise-distinct paths, all strictly + inside the store root, each reading back its own blob — + keys like "a/b" and "a_b" may never collide, and no key + shape may escape the root. + """ + # Arrange + root = tmp_path / uuid4().hex + blobs = {key: key.encode() for key in keys} + cache = FakeRemoteCache(blobs) + store = LocalTilesetStore(root, max_bytes=1 << 20) + + # Act + paths = { + key: await store.path_for( + cache, key, expected_size=len(blobs[key]) + ) + for key in keys + } + + # Assert + assert len(set(paths.values())) == len(keys) + for key, path in paths.items(): + assert path.is_relative_to(root) + assert path.read_bytes() == blobs[key] + + +#: Byte budget shared by the eviction property and its strategy, so every +#: generated artifact is guaranteed to fit the budget on its own. +_PROPERTY_BUDGET = 20 + + +@st.composite +def _access_sequences(draw): + """Draw an access sequence plus a fixed per-key artifact size.""" + sequence = draw( + st.lists( + st.sampled_from(["a", "b", "c", "d", "e"]), + min_size=1, + max_size=12, + ) + ) + sizes = { + key: draw(st.integers(min_value=1, max_value=_PROPERTY_BUDGET)) + for key in sorted(set(sequence)) + } + return sequence, sizes + + +class TestEviction: + """The disk budget is enforced, but never over a live handle.""" + + @pytest.mark.asyncio + async def test_should_evict_least_recently_used_to_make_room(self, tmp_path): + """Test that the budget is enforced in LRU order. + + Given: + Two resident artifacts filling the budget, with the first + touched more recently than the second. + When: + A third is hydrated. + Then: + The least recently used should be unlinked. + """ + # Arrange + cache = FakeRemoteCache({"a": b"1" * 10, "b": b"2" * 10, "c": b"3" * 10}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=20) + path_a = await store.path_for(cache, "a", expected_size=10) + await store.path_for(cache, "b", expected_size=10) + await store.path_for(cache, "a", expected_size=10) # touch a + + # Act + await store.path_for(cache, "c", expected_size=10) + + # Assert + assert path_a.exists() + assert store.resident_bytes == 20 + + @pytest.mark.asyncio + async def test_should_not_unlink_an_artifact_that_is_in_use(self, tmp_path): + """Test the guard that keeps eviction from segfaulting a reader. + + Given: + A resident artifact whose release hook reports it is in use. + When: + Room is needed for another artifact. + Then: + It should be left on disk and the incoming hydration should + still succeed, overshooting the budget rather than failing + the request. Unlinking a file whose h5py handle is live, or + closing that handle under a numpy read running in a worker + thread, is a segfault rather than an exception — so this is + a correctness guard, not an optimization. + """ + # Arrange + cache = FakeRemoteCache({"a": b"1" * 10, "b": b"2" * 10}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=10) + + async def _pinned(_key: str) -> bool: + return False + + store.set_release_hook(_pinned) + path_a = await store.path_for(cache, "a", expected_size=10) + + # Act + path_b = await store.path_for(cache, "b", expected_size=10) + + # Assert + assert path_a.exists() + assert path_b.read_bytes() == b"2" * 10 + # 20 resident bytes against a 10-byte budget: when every resident + # is pinned, the documented contract is overshoot, not failure. + assert store.resident_bytes == 20 + + @pytest.mark.asyncio + async def test_should_consult_the_release_hook_before_unlinking( + self, tmp_path + ): + """Test that eviction gives the service a chance to close first. + + Given: + A release hook that records which keys it was asked about. + When: + An eviction is required. + Then: + The hook should have been asked about the victim before the + file was removed. + """ + # Arrange + cache = FakeRemoteCache({"a": b"1" * 10, "b": b"2" * 10}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=10) + asked: list[str] = [] + + async def _hook(key: str) -> bool: + asked.append(key) + return True + + store.set_release_hook(_hook) + path_a = await store.path_for(cache, "a", expected_size=10) + + # Act + await store.path_for(cache, "b", expected_size=10) + + # Assert + assert asked == ["a"] + assert not path_a.exists() + + @pytest.mark.asyncio + async def test_aclose_should_drop_every_hydrated_file(self, tmp_path): + """Test that teardown does not leak the API task's disk. + + Given: + Hydrated artifacts on local disk. + When: + aclose runs, as it does on lifespan teardown. + Then: + The files should be removed. They are a cache of a cache, and + their hashed names carry no expiry of their own, so leaving + them would accumulate across restarts. + """ + # Arrange + cache = FakeRemoteCache({"a": b"1" * 10}) + store = LocalTilesetStore(tmp_path / "local", max_bytes=100) + path = await store.path_for(cache, "a", expected_size=10) + + # Act + await store.aclose() + + # Assert + assert not path.exists() + assert store.resident_bytes == 0 + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given(accesses=_access_sequences()) + @pytest.mark.asyncio + async def test_should_hold_resident_bytes_within_budget_for_any_sequence( + self, tmp_path, accesses + ): + """Test the LRU byte-budget invariant over arbitrary access orders. + + Given: + Generated access sequences over keys whose artifacts each + fit the budget, with every resident releasable. + When: + The sequence is replayed through path_for. + Then: + It should keep resident_bytes within max_bytes after every + call, and every returned path should hold its artifact's + exact bytes — no interleaving of hits, misses, and + evictions may overshoot the budget or serve a stale file. + """ + # Arrange + sequence, sizes = accesses + blobs = {key: key.encode() * size for key, size in sizes.items()} + cache = FakeRemoteCache(blobs) + store = LocalTilesetStore( + tmp_path / uuid4().hex, max_bytes=_PROPERTY_BUDGET + ) + + # Act & assert + for key in sequence: + path = await store.path_for( + cache, key, expected_size=sizes[key] + ) + assert path.read_bytes() == blobs[key] + assert store.resident_bytes <= _PROPERTY_BUDGET diff --git a/tests/test_tilesets/test_wire.py b/tests/test_tilesets/test_wire.py new file mode 100644 index 0000000..d09ba8b --- /dev/null +++ b/tests/test_tilesets/test_wire.py @@ -0,0 +1,157 @@ +"""Tests for HiGlass payload assembly in ``cfdb.tilesets.wire``.""" + +from __future__ import annotations + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from cfdb.tilesets.wire import tileset_info_payload + +UID = "4dn/4DNFIABC123" + +#: What the stub tileset's ``info()`` reports, mirroring the shape of +#: clodius's ``TilesetInfo`` for an explicit-ladder cooler: the extent +#: fields are None on purpose, and ``exclude_none`` must drop them. +_BASE_FIELDS = { + "resolutions": [1000, 2000, 4000], + "chromsizes": [["chr1", 3000], ["chr2", 3000]], + "max_pos": [6000, 6000], + "tile_size": 256, + "max_width": None, + "max_zoom": None, +} + + +class _StubInfo: + """Duck-type of the pydantic info model at the clodius boundary.""" + + def __init__(self, fields: dict) -> None: + self._fields = fields + + def model_dump(self, exclude_none: bool = False) -> dict: + if exclude_none: + return {k: v for k, v in self._fields.items() if v is not None} + return dict(self._fields) + + +class _StubTileset: + """Duck-type of an open clodius tileset.""" + + datatype = "matrix" + + def info(self) -> _StubInfo: + return _StubInfo(_BASE_FIELDS) + + +def _doc(**fields) -> dict: + """Return a projected file document carrying exactly ``fields``.""" + return dict(fields) + + +class TestTilesetInfoPayload: + """The four fields the server adds on top of what the file knows.""" + + @pytest.mark.parametrize( + ("doc", "expected_name"), + [ + (_doc(local_id="4DNFIABC123"), "4DNFIABC123"), + (_doc(), UID), + ], + ids=["local-id-fallback", "uid-fallback"], + ) + def test_should_fall_back_through_the_name_chain(self, doc, expected_name): + """Test the display-name fallback chain. + + Given: + A document missing ``filename`` but carrying ``local_id``, and + a document missing both. + When: + tileset_info_payload is called. + Then: + It should name the tileset after the local_id, and after the + uid when even that is absent — a tileset always has a name. + """ + # Act + payload = tileset_info_payload(UID, _StubTileset(), doc) + + # Assert + assert payload["name"] == expected_name + + def test_should_omit_coord_system_when_the_assembly_is_empty(self): + """Test that an empty assembly string is not emitted. + + Given: + A document whose ``genome_assembly`` is the empty string. + When: + tileset_info_payload is called. + Then: + It should leave ``coordSystem`` out entirely — an empty string + is exactly the plausible-looking placeholder that risks a + silently misaligned HiGlass track, whereas an absent key makes + the client fall back to ``chromsizes``. + """ + # Arrange + doc = _doc(filename="sample.mcool", genome_assembly="") + + # Act + payload = tileset_info_payload(UID, _StubTileset(), doc) + + # Assert + assert "coordSystem" not in payload + + _ABSENT = object() + _field = st.one_of( + st.just(_ABSENT), + st.none(), + st.just(""), + st.text(min_size=1, max_size=20), + ) + + @settings(max_examples=50) + @given(filename=_field, local_id=_field, assembly=_field) + def test_should_derive_every_server_field_from_the_document( + self, filename, local_id, assembly + ): + """Test the field-derivation contract over the document domain. + + Given: + Documents whose ``filename``, ``local_id``, and + ``genome_assembly`` are independently absent, None, empty, or + arbitrary text. + When: + tileset_info_payload is called. + Then: + It should set ``uuid`` to the uid, name the tileset with the + first truthy of filename, local_id, and uid, emit + ``coordSystem`` exactly when the assembly is truthy, and pass + the tileset's own non-None info fields through unchanged. + """ + # Arrange + doc: dict = {} + for key, value in ( + ("filename", filename), + ("local_id", local_id), + ("genome_assembly", assembly), + ): + if value is not self._ABSENT: + doc[key] = value + + # Act + payload = tileset_info_payload(UID, _StubTileset(), doc) + + # Assert + assert payload["uuid"] == UID + assert payload["name"] == ( + doc.get("filename") or doc.get("local_id") or UID + ) + if doc.get("genome_assembly"): + assert payload["coordSystem"] == doc["genome_assembly"] + else: + assert "coordSystem" not in payload + assert payload["datatype"] == "matrix" + for key, value in _BASE_FIELDS.items(): + if value is None: + assert key not in payload + else: + assert payload[key] == value From cb07bacb44ad550f99aebd92987baa12481ba5e7 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:27:36 -0400 Subject: [PATCH 07/14] refactor: Extract the shared request preamble into the router helpers /data and /index both open with the same sequence: normalize the DCC, look up the file document, and check consortium/protected access. Pulling that into _helpers.py lets the new tile and tileset-prep routers reuse it verbatim instead of restating the access-control logic a third and fourth time. --- src/cfdb/api/routers/_helpers.py | 107 ++++++++- src/cfdb/api/routers/data.py | 124 +++------- src/cfdb/api/routers/index.py | 81 ++----- tests/test_data.py | 2 +- tests/test_helpers.py | 390 +++++++++++++++++++++++++++++++ 5 files changed, 542 insertions(+), 162 deletions(-) create mode 100644 tests/test_helpers.py diff --git a/src/cfdb/api/routers/_helpers.py b/src/cfdb/api/routers/_helpers.py index ff608ff..e03ae93 100644 --- a/src/cfdb/api/routers/_helpers.py +++ b/src/cfdb/api/routers/_helpers.py @@ -1,24 +1,40 @@ -"""Shared primitives for /data and /index routers. +"""Shared primitives for the /data, /index, and tileset routers. Provides: -- ``FILE_DOC_PROJECTION``: the Mongo projection both routers use when reading +- ``FILE_DOC_PROJECTION``: the Mongo projection every router uses when reading a file document. Strips ``_id`` (so ``bson.ObjectId`` never crosses into the workflow subsystem) and limits the returned dict to the fields any router or workflow consumer reads. -- ``lookup_file_doc``: single canonical Mongo query so /data and /index hit +- ``lookup_file_doc``: single canonical Mongo query so the routers hit the same record for a given ``(dcc, local_id)`` pair — preserves the "exactly one workflow under contention" invariant. - ``enforce_hubmap_access``: defense-in-depth guard rejecting non-public HuBMAP files before any workflow dispatch or upstream stream. +- ``resolve_file_doc``: the whole per-request preamble the routers share — + DCC validation, database check, lookup, and access control — in the one + ordering all of them agree on. """ from __future__ import annotations +import logging from typing import Any, Final from fastapi import HTTPException, status +from cfdb import api +from cfdb.dcc_registry import get_all_dcc_names, normalize_dcc_name + +logger = logging.getLogger(__name__) + +#: Tight path-param constraint shared by every router that takes a +#: ``(dcc, local_id)`` pair. DCC accessions across ENCODE / 4DN / HuBMAP are +#: all subsets of ``[A-Za-z0-9._-]``; the length cap defends Mongo and log +#: lines from unbounded input. +PATH_PARAM_PATTERN: Final[str] = r"^[A-Za-z0-9._-]+$" +PATH_PARAM_MAX_LEN: Final[int] = 256 + FILE_DOC_PROJECTION: Final[dict[str, int]] = { "_id": 0, "local_id": 1, @@ -30,6 +46,13 @@ "filename": 1, "data_access_level": 1, "size_in_bytes": 1, + # Promoted to the top level by every DCC's sync: ENCODE via + # ``_add_extra(doc, "genome_assembly", ...)``, 4DN and HuBMAP via the + # ``top_level_map`` / direct ``update``. The ``extra.*.genome_assembly`` + # fields declared on the enriched models are never written, so this is + # the only path that carries an assembly. Read by the tileset routers to + # derive HiGlass's ``coordSystem``. + "genome_assembly": 1, "extra.extra_files": 1, "extra.fourdn.extra_files": 1, } @@ -85,3 +108,81 @@ def enforce_hubmap_access(normalized_dcc: str, file_doc: dict[str, Any]) -> None "https://portal.hubmapconsortium.org/" ), ) + + +async def resolve_file_doc( + dcc: str, local_id: str, *, require_access_url: bool = False +) -> tuple[str, dict[str, Any]]: + """Run the per-request preamble every file-serving router shares. + + In order: normalize and validate the DCC (400), check the database is + wired (500), look the document up (404), optionally require an access + method (501), and enforce the HuBMAP access guard (403). + + Args: + dcc: Raw DCC path parameter, normalized here. + local_id: The file's identifier within the DCC. + require_access_url: When True, reject a record with no + ``access_url`` with 501 *before* the HuBMAP guard runs. This + reproduces ``/data``'s ordering; ``/index`` and the tileset + routers do not check for an access URL at all and leave it + False. + + Returns: + ``(normalized_dcc, file_doc)``. + + Raises: + HTTPException: 400, 500, 404, 501, or 403 as described above. + + Note: + The caller keeps ``await locks.wait_for_cutover()``. It stays in + the handlers because that is where every route test patches it, + and because a cutover wait is about request admission rather than + about resolving a document. + """ + normalized_dcc = normalize_dcc_name(dcc) + valid_dccs = get_all_dcc_names() + + if normalized_dcc not in valid_dccs: + logger.warning(f"Invalid DCC requested: {dcc}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown DCC '{dcc}'. Valid DCCs: {', '.join(valid_dccs)}", + ) + + if api.db is None: + logger.error("Database not initialized") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Database not available", + ) + + logger.info(f"Looking up file: submission={normalized_dcc}, local_id={local_id}") + file_doc = await lookup_file_doc(api.db, normalized_dcc, local_id) + + if not file_doc: + logger.warning(f"File not found: {normalized_dcc}/{local_id}") + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="File not found" + ) + + # Checked before the HuBMAP guard, matching ``/data``'s existing + # ordering. Guarding here is what lets the caller treat + # ``file_doc["access_url"]`` as a plain str rather than an Optional + # that the call ordering happens to have narrowed. + if require_access_url and not file_doc.get("access_url"): + logger.warning(f"File has no access_url: {normalized_dcc}/{local_id}") + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail="File has no access URL", + ) + + # Defense-in-depth: reject any non-public HuBMAP file that survived + # pruning, before any caller streams upstream bytes, caches them, or + # dispatches a workflow. + # + # Runs after the access-url check so signed/private URLs for + # non-public HuBMAP files are never logged ahead of the 403. + enforce_hubmap_access(normalized_dcc, file_doc) + + return normalized_dcc, file_doc diff --git a/src/cfdb/api/routers/data.py b/src/cfdb/api/routers/data.py index 9923c3d..1759947 100644 --- a/src/cfdb/api/routers/data.py +++ b/src/cfdb/api/routers/data.py @@ -6,8 +6,15 @@ from fastapi import APIRouter, Header, HTTPException, Path, Request, status -from cfdb import api -from cfdb.api.routers._helpers import enforce_hubmap_access, lookup_file_doc +from cfdb.api.routers._helpers import ( + PATH_PARAM_MAX_LEN as _PATH_PARAM_MAX_LEN, +) +from cfdb.api.routers._helpers import ( + PATH_PARAM_PATTERN as _PATH_PARAM_PATTERN, +) +from cfdb.api.routers._helpers import ( + resolve_file_doc, +) from cfdb.api.routers.cache_stream import ( probe_workflow_readiness, serve_workflow_artifact_or_dispatch, @@ -24,12 +31,6 @@ ) from cfdb.workflows.models import ArtifactKind -#: Tight path-param constraint shared by /data and /index. DCC accessions -#: across ENCODE / 4DN / HuBMAP are all subsets of ``[A-Za-z0-9._-]``; -#: the length cap defends Mongo and log lines from unbounded input. -_PATH_PARAM_PATTERN = r"^[A-Za-z0-9._-]+$" -_PATH_PARAM_MAX_LEN = 256 - logger = logging.getLogger(__name__) router = APIRouter(prefix="/data", tags=["data"]) @@ -109,68 +110,29 @@ async def stream_file( await locks.wait_for_cutover() try: - # 1. Validate and normalize DCC name - from cfdb.dcc_registry import get_all_dcc_names, normalize_dcc_name - - normalized_dcc = normalize_dcc_name(dcc) - valid_dccs = get_all_dcc_names() - - if normalized_dcc not in valid_dccs: - logger.warning(f"Invalid DCC requested: {dcc}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown DCC '{dcc}'. Valid DCCs: {', '.join(valid_dccs)}", - ) - - # 2. Look up the file document via the shared helper so /data and - # /index converge on the same record for any given - # (normalized_dcc, local_id) pair. The helper uses - # FILE_DOC_PROJECTION, which strips _id and limits the doc to - # the fields routers + workflows actually read. - if api.db is None: - logger.error("Database not initialized") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Database not available", - ) - - logger.info( - f"Looking up file: submission={normalized_dcc}, local_id={local_id}" + # 1-4. DCC validation (400), database check (500), lookup (404), + # access-method check (501), and the HuBMAP access guard + # (403) — the preamble every file-serving router shares, in + # the one ordering they all agree on. ``require_access_url`` + # is what makes the 501 precede the 403 here, matching this + # handler's original ordering; /index leaves it off. + normalized_dcc, file_doc = await resolve_file_doc( + dcc, local_id, require_access_url=True ) - file_doc = await lookup_file_doc(api.db, normalized_dcc, local_id) - - if not file_doc: - logger.warning(f"File not found: {normalized_dcc}/{local_id}") - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="File not found" - ) - - # 3. Check the file has an access method, then reduce the document - # to what the streaming path reads. Guarding first is what lets - # _FileRef.access_url be a plain str rather than an Optional the - # call ordering happens to have narrowed. - access_url = file_doc.get("access_url") - if not access_url: - logger.warning(f"File has no access_url: {normalized_dcc}/{local_id}") - raise HTTPException( - status_code=status.HTTP_501_NOT_IMPLEMENTED, - detail="File has no access URL", - ) + # Reduce the document to what the streaming path reads. The 501 + # guard above is what lets ``_FileRef.access_url`` be a plain str + # rather than an Optional the call ordering happens to have + # narrowed. file_metadata = _FileRef( - access_url=access_url, + access_url=file_doc["access_url"], filename=file_doc.get("filename") or "file", size_in_bytes=file_doc.get("size_in_bytes"), ) - # 4. Defence-in-depth: reject any non-public HuBMAP file that somehow - # survived pruning. Placed before the workflow branch so protected - # files never enter the preprocessing pipeline. - # - # Log access_url AFTER the guard so signed/private URLs for - # non-public HuBMAP files don't leak to logs before the 403. - enforce_hubmap_access(normalized_dcc, file_doc) - + # Logged only after the HuBMAP guard inside the helper, so + # signed/private URLs for non-public files never reach the log + # ahead of the 403. logger.info(f"File access_url: {file_metadata.access_url}") # 5. Workflow path: if the client wants the preprocessed artifact @@ -326,39 +288,9 @@ async def stream_file_status( await locks.wait_for_cutover() try: - from cfdb.dcc_registry import get_all_dcc_names, normalize_dcc_name - - normalized_dcc = normalize_dcc_name(dcc) - valid_dccs = get_all_dcc_names() - - if normalized_dcc not in valid_dccs: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown DCC '{dcc}'. Valid DCCs: {', '.join(valid_dccs)}", - ) - - if api.db is None: - logger.error("Database not initialized") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Database not available", - ) - - file_doc = await lookup_file_doc(api.db, normalized_dcc, local_id) - if not file_doc: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="File not found" - ) - - # Mirror ``stream_file``'s ordering: the no-access-method 501 is - # checked before the HuBMAP access guard. - if not file_doc.get("access_url"): - raise HTTPException( - status_code=status.HTTP_501_NOT_IMPLEMENTED, - detail="File has no access URL", - ) - - enforce_hubmap_access(normalized_dcc, file_doc) + # Mirrors ``stream_file``'s preamble exactly, including that the + # no-access-method 501 is checked before the HuBMAP access guard. + _, file_doc = await resolve_file_doc(dcc, local_id, require_access_url=True) # Workflow cache state — never dispatches. ``None`` means the file # is served directly from upstream (passthrough format, a format diff --git a/src/cfdb/api/routers/index.py b/src/cfdb/api/routers/index.py index 6242320..e022936 100644 --- a/src/cfdb/api/routers/index.py +++ b/src/cfdb/api/routers/index.py @@ -28,13 +28,21 @@ from fastapi import APIRouter, Header, HTTPException, Path, Request, status from cfdb import api -from cfdb.api.routers._helpers import enforce_hubmap_access, lookup_file_doc +from cfdb.api.routers._helpers import ( + PATH_PARAM_MAX_LEN as _PATH_PARAM_MAX_LEN, +) +from cfdb.api.routers._helpers import ( + PATH_PARAM_PATTERN as _PATH_PARAM_PATTERN, +) +from cfdb.api.routers._helpers import ( + resolve_file_doc, +) from cfdb.api.routers.cache_stream import ( probe_workflow_readiness, serve_workflow_artifact_or_dispatch, stream_upstream_url, ) -from cfdb.dcc_registry import get_all_dcc_names, get_dcc_config, normalize_dcc_name +from cfdb.dcc_registry import get_dcc_config from cfdb.models import coerce_4dn_cv_token from cfdb.services import locks from cfdb.workflows.models import ArtifactKind @@ -44,12 +52,6 @@ router = APIRouter(prefix="/index", tags=["index"]) -#: Mirrors the constraint applied in ``routers/data.py`` so both endpoints -#: reject the same shape of path-param input. ENCODE/4DN/HuBMAP accessions -#: all live under ``[A-Za-z0-9._-]``. -_PATH_PARAM_PATTERN = r"^[A-Za-z0-9._-]+$" -_PATH_PARAM_MAX_LEN = 256 - #: Sidecar ``file_format`` tokens (4DN ``display_title`` values or bare #: strings) recognized as index artifacts in 4DN's ``extra.extra_files`` / #: ``extra.fourdn.extra_files`` arrays: ``bai``/``csi`` (BAM), ``tbi`` @@ -85,37 +87,13 @@ async def stream_index_file( await locks.wait_for_cutover() try: - normalized_dcc = normalize_dcc_name(dcc) - valid_dccs = get_all_dcc_names() - - if normalized_dcc not in valid_dccs: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown DCC '{dcc}'. Valid DCCs: {', '.join(valid_dccs)}", - ) - - if api.db is None: - logger.error("Database not initialized") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Database not available", - ) - - # Look up the file document via the shared helper so /data and - # /index converge on the same record for any given - # (normalized_dcc, local_id) pair. FILE_DOC_PROJECTION strips _id - # so bson.ObjectId never crosses into the workflow subsystem. - file_doc = await lookup_file_doc(api.db, normalized_dcc, local_id) - - if not file_doc: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="File not found" - ) - - # Defense-in-depth: reject any non-public HuBMAP file that survived - # pruning before either the sidecar fast path or the workflow path - # can stream upstream bytes or cache them. - enforce_hubmap_access(normalized_dcc, file_doc) + # DCC validation (400), database check (500), lookup (404), and + # the HuBMAP access guard (403) — shared with /data so both + # converge on the same record for a given (dcc, local_id) pair and + # the workflow_key mutex actually serializes them. No + # ``require_access_url``: /index has never demanded an access URL, + # because a sidecar-served file may not need one. + normalized_dcc, file_doc = await resolve_file_doc(dcc, local_id) # 1. Upstream sidecar (e.g., 4DN's extra_files) always takes # precedence when present — the upstream DCC pre-built the @@ -222,29 +200,8 @@ async def stream_index_file_status( await locks.wait_for_cutover() try: - normalized_dcc = normalize_dcc_name(dcc) - valid_dccs = get_all_dcc_names() - - if normalized_dcc not in valid_dccs: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail=f"Unknown DCC '{dcc}'. Valid DCCs: {', '.join(valid_dccs)}", - ) - - if api.db is None: - logger.error("Database not initialized") - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Database not available", - ) - - file_doc = await lookup_file_doc(api.db, normalized_dcc, local_id) - if not file_doc: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="File not found" - ) - - enforce_hubmap_access(normalized_dcc, file_doc) + # Mirrors ``stream_index_file``'s preamble exactly. + normalized_dcc, file_doc = await resolve_file_doc(dcc, local_id) # 1. Upstream sidecar present → a GET streams it immediately. # A malformed sidecar raises 502, mirroring the stream path. diff --git a/tests/test_data.py b/tests/test_data.py index d27bb08..869a3ed 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1239,7 +1239,7 @@ async def test_stream_file_should_log_an_error_when_it_genuinely_fails( # Arrange mocker.patch.object(locks, "wait_for_cutover", return_value=None) mocker.patch( - "cfdb.api.routers.data.lookup_file_doc", + "cfdb.api.routers._helpers.lookup_file_doc", side_effect=RuntimeError("lookup exploded"), ) mock_db.file.docs = [_make_encode_file_doc()] diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..fd0d3af --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,390 @@ +"""Tests for the shared router preamble in ``cfdb.api.routers._helpers``.""" + +from __future__ import annotations + +import asyncio +from itertools import cycle + +import pytest +from fastapi import HTTPException +from hypothesis import HealthCheck, assume, given, settings +from hypothesis import strategies as st + +from cfdb import api +from cfdb.api.routers._helpers import ( + FILE_DOC_PROJECTION, + enforce_hubmap_access, + lookup_file_doc, + resolve_file_doc, +) +from cfdb.dcc_registry import get_all_dcc_names, normalize_dcc_name + + +def _make_file_doc(**overrides): + """Return a minimal projected file document.""" + doc = { + "submission": "4dn", + "local_id": "4DNFIABC123", + "filename": "sample.mcool", + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "access_url": "https://data.4dnucleome.org/files/sample.mcool", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + "file_format": {"name": "HDF5"}, + } + doc.update(overrides) + return doc + + +class TestFileDocProjection: + """The projection is the only thing that limits what routers can read.""" + + def test_projection_should_select_genome_assembly(self): + """Test that the projection carries the field coordSystem derives from. + + Given: + ``FILE_DOC_PROJECTION``, the projection every router reads + through. + When: + The tileset routers derive HiGlass's ``coordSystem`` from + ``genome_assembly``. + Then: + The projection MUST select it. This is asserted structurally + rather than behaviourally because ``FakeCollection.find_one`` + ignores ``projection`` entirely, so a field omitted here still + reaches every unit test and only returns ``None`` in + production. + """ + # Assert + assert FILE_DOC_PROJECTION.get("genome_assembly") == 1 + + def test_projection_should_strip_the_mongo_object_id(self): + """Test that the document never carries a bson ObjectId. + + Given: + ``FILE_DOC_PROJECTION``. + When: + A projected document is handed to the workflow subsystem and + pickled across the wool boundary. + Then: + ``_id`` MUST be excluded, since ``bson.ObjectId`` is not part + of the workflow subsystem's contract. + """ + # Assert + assert FILE_DOC_PROJECTION["_id"] == 0 + + +class TestLookupFileDoc: + """The canonical query keeping every router on one record.""" + + @pytest.mark.asyncio + async def test_should_prefer_the_materialized_files_collection(self, mock_db): + """Test that the materialized document wins over the raw one. + + Given: + Distinguishable documents for the same ``(submission, + local_id)`` pair in both the materialized ``files`` collection + and the raw ``file`` collection. + When: + lookup_file_doc is called. + Then: + It should return the ``files`` document, so all four routers + resolve the same record and the workflow-key mutex derived + from it actually serializes concurrent requests. + """ + # Arrange + mock_db.files.docs = [_make_file_doc(filename="materialized.mcool")] + mock_db.file.docs = [_make_file_doc(filename="raw.mcool")] + + # Act + file_doc = await lookup_file_doc(mock_db, "4dn", "4DNFIABC123") + + # Assert + assert file_doc["filename"] == "materialized.mcool" + + +class TestEnforceHubmapAccess: + """The fail-closed HuBMAP access guard shared by every router.""" + + @given( + normalized_dcc=st.sampled_from(["hubmap", "4dn", "encode"]), + level=st.one_of( + st.none(), + st.just("public"), + st.sampled_from( + ["Public", "PUBLIC", " public", "public ", "protected", "consortium", ""] + ), + st.text(max_size=32), + ), + field_present=st.booleans(), + ) + @settings(max_examples=50) + def test_should_forbid_every_non_exact_public_level_under_hubmap( + self, normalized_dcc, level, field_present + ): + """Test that the guard is fail-closed and total over its domain. + + Given: + A normalized DCC drawn from the registry and a + ``data_access_level`` drawn from None, the exact string + ``"public"``, casing and padding variants of it, the known + non-public levels, arbitrary text, or an absent field. + When: + enforce_hubmap_access is called with a document carrying that + access level. + Then: + It should raise HTTP 403 for every HuBMAP document whose level + is not exactly ``"public"``, return None for an exact + ``"public"``, and never raise for any other DCC. + """ + # Arrange + file_doc = {"data_access_level": level} if field_present else {} + + # Act & assert + if normalized_dcc != "hubmap": + assert enforce_hubmap_access(normalized_dcc, file_doc) is None + elif field_present and level == "public": + assert enforce_hubmap_access(normalized_dcc, file_doc) is None + else: + with pytest.raises(HTTPException) as exc_info: + enforce_hubmap_access(normalized_dcc, file_doc) + assert exc_info.value.status_code == 403 + + +class TestResolveFileDoc: + """The preamble every file-serving router shares.""" + + @pytest.mark.asyncio + async def test_should_return_the_normalized_dcc_and_document(self, mock_db): + """Test the success path normalizes the DCC and returns the record. + + Given: + A file document present under submission ``4dn``. + When: + resolve_file_doc is called with a mixed-case DCC name. + Then: + It should return the lower-cased DCC and the projected + document. + """ + # Arrange + mock_db.file.docs = [_make_file_doc()] + + # Act + normalized_dcc, file_doc = await resolve_file_doc("4DN", "4DNFIABC123") + + # Assert + assert normalized_dcc == "4dn" + assert file_doc["local_id"] == "4DNFIABC123" + + @pytest.mark.asyncio + async def test_should_reject_an_unknown_dcc_with_400(self, mock_db): + """Test that an unrecognized DCC is a client error. + + Given: + A DCC name that is not in the registry. + When: + resolve_file_doc is called. + Then: + It should raise HTTP 400 naming the valid DCCs, before any + database access. + """ + # Act + with pytest.raises(HTTPException) as exc_info: + await resolve_file_doc("nosuchdcc", "abc") + + # Assert + assert exc_info.value.status_code == 400 + assert "nosuchdcc" in exc_info.value.detail + + @pytest.mark.asyncio + async def test_should_report_500_when_the_database_is_unwired(self, mocker): + """Test that a missing database is a server error, not a 404. + + Given: + ``api.db`` is None, as it is before the lifespan runs. + When: + resolve_file_doc is called with a valid DCC. + Then: + It should raise HTTP 500 rather than reporting the file + missing. + """ + # Arrange + mocker.patch.object(api, "db", None) + + # Act + with pytest.raises(HTTPException) as exc_info: + await resolve_file_doc("4dn", "4DNFIABC123") + + # Assert + assert exc_info.value.status_code == 500 + + @pytest.mark.asyncio + async def test_should_report_404_when_no_document_matches(self, mock_db): + """Test that an absent record is a 404. + + Given: + An empty file collection. + When: + resolve_file_doc is called. + Then: + It should raise HTTP 404. + """ + # Arrange + mock_db.file.docs = [] + + # Act + with pytest.raises(HTTPException) as exc_info: + await resolve_file_doc("4dn", "4DNFIABC123") + + # Assert + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_should_not_require_an_access_url_by_default(self, mock_db): + """Test that a record with no access URL resolves for /index. + + Given: + A document with no ``access_url`` — servable by /index from an + upstream sidecar, and tileable from a cached artifact. + When: + resolve_file_doc is called without ``require_access_url``. + Then: + It should return the document rather than raising 501. + """ + # Arrange + doc = _make_file_doc() + del doc["access_url"] + mock_db.file.docs = [doc] + + # Act + _, file_doc = await resolve_file_doc("4dn", "4DNFIABC123") + + # Assert + assert file_doc.get("access_url") is None + + @pytest.mark.asyncio + async def test_should_report_501_when_an_access_url_is_required(self, mock_db): + """Test that /data's no-access-method case stays a 501. + + Given: + A document with no ``access_url``. + When: + resolve_file_doc is called with ``require_access_url=True``. + Then: + It should raise HTTP 501, the code /data has always returned + for a file with no supported access method. + """ + # Arrange + doc = _make_file_doc() + del doc["access_url"] + mock_db.file.docs = [doc] + + # Act + with pytest.raises(HTTPException) as exc_info: + await resolve_file_doc("4dn", "4DNFIABC123", require_access_url=True) + + # Assert + assert exc_info.value.status_code == 501 + + @pytest.mark.asyncio + async def test_should_refuse_a_non_public_hubmap_file_with_403(self, mock_db): + """Test that the HuBMAP access guard runs on every resolution. + + Given: + A HuBMAP document whose ``data_access_level`` is not public. + When: + resolve_file_doc is called. + Then: + It should raise HTTP 403, so no caller can reach a protected + record regardless of which router asked. + """ + # Arrange + mock_db.file.docs = [ + _make_file_doc( + submission="hubmap", + local_id="HBM123", + data_access_level="consortium", + ) + ] + + # Act + with pytest.raises(HTTPException) as exc_info: + await resolve_file_doc("hubmap", "HBM123") + + # Assert + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_should_refuse_a_protected_hubmap_file_before_the_501( + self, mock_db + ): + """Test that /data's 501 is checked ahead of the access guard. + + Given: + A non-public HuBMAP document that also has no ``access_url``. + When: + resolve_file_doc is called with ``require_access_url=True``. + Then: + It should raise 501, not 403 — pinning the ordering /data has + always had, so that a signed URL is never logged before the + access guard runs. + """ + # Arrange + doc = _make_file_doc( + submission="hubmap", local_id="HBM123", data_access_level="protected" + ) + del doc["access_url"] + mock_db.file.docs = [doc] + + # Act + with pytest.raises(HTTPException) as exc_info: + await resolve_file_doc("hubmap", "HBM123", require_access_url=True) + + # Assert + assert exc_info.value.status_code == 501 + + @given( + dcc=st.sampled_from(get_all_dcc_names()), + upper_mask=st.lists(st.booleans(), min_size=1, max_size=8), + unknown=st.text(max_size=32), + ) + @settings( + max_examples=50, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + def test_should_normalize_any_casing_and_reject_unknown_dccs( + self, mock_db, dcc, upper_mask, unknown + ): + """Test DCC normalization and rejection across the input domain. + + Given: + A registered DCC name under a generated per-character casing, + and an arbitrary string that does not normalize to any + registered DCC, with a matching document seeded for the + registered one. + When: + resolve_file_doc is called with each. + Then: + It should resolve every casing variant to the lower-cased DCC + and return the seeded document, and reject every unknown + string with exactly HTTP 400. + """ + # Arrange + assume(normalize_dcc_name(unknown) not in get_all_dcc_names()) + variant = "".join( + ch.upper() if up else ch.lower() for ch, up in zip(dcc, cycle(upper_mask)) + ) + mock_db.file.docs = [ + _make_file_doc(submission=dcc, data_access_level="public") + ] + + # Act + normalized_dcc, file_doc = asyncio.run( + resolve_file_doc(variant, "4DNFIABC123") + ) + with pytest.raises(HTTPException) as exc_info: + asyncio.run(resolve_file_doc(unknown, "4DNFIABC123")) + + # Assert + assert normalized_dcc == dcc + assert file_doc["local_id"] == "4DNFIABC123" + assert exc_info.value.status_code == 400 From 5ff8d3b6e659d9f337c2f9ceea62428952a0832c Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:28:09 -0400 Subject: [PATCH 08/14] feat: Serve HiGlass tiles and the tileset preparation channel Gosling's MatrixData compiles to HiGlass's native heatmap track, which issues thousands of small tile requests and cannot be handed a job id to poll -- neither /data nor /index can satisfy it. Two channels, deliberately separate: - /tileset_info and /tiles speak the HiGlass wire protocol (repeatable d parameters, capped at 64; uid {dcc}/{local_id}) and are pure readers that never dispatch a workflow -- a contact map whose artifact has not been built is a 404. The response carries exactly one entry per requested id: a tile position the resolution ladder does not hold, or an id that fails to parse, gets a per-tile error object in its slot rather than being omitted or failing the whole batch. - POST /tilesets/{dcc}/{local_id} builds the artifact, and its /status probe mirrors the /data and /index readiness probes. POST is idempotent: an artifact that already exists answers 200 {"ready": true} rather than creating a no-op job. .hic is recognized and refused with a specific 501 rather than served: the tile backend reads local files only, and the ENCODE corpus (~78 TB, largest file 315 GB) cannot be materialized onto Fargate ephemeral storage at any budget. The lifespan wires the service only when the workflow subsystem is enabled and the build carries clodius, which is why the tiles extra exists as a separate, optional dependency group. --- pyproject.toml | 25 +- src/cfdb/api/__init__.py | 50 + src/cfdb/api/main.py | 69 ++ src/cfdb/api/routers/tiles.py | 246 +++++ src/cfdb/api/routers/tilesets.py | 258 ++++++ tests/conftest.py | 59 ++ tests/test_api/test_lifespan_tilesets.py | 347 +++++++ tests/test_tiles.py | 1052 ++++++++++++++++++++++ tests/test_tilesets_router.py | 748 +++++++++++++++ 9 files changed, 2853 insertions(+), 1 deletion(-) create mode 100644 src/cfdb/api/routers/tiles.py create mode 100644 src/cfdb/api/routers/tilesets.py create mode 100644 tests/test_api/test_lifespan_tilesets.py create mode 100644 tests/test_tiles.py create mode 100644 tests/test_tilesets_router.py diff --git a/pyproject.toml b/pyproject.toml index d6ce2bb..179cd0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,30 @@ readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" [project.optional-dependencies] -dev = ["allpairspy", "cloudpickle", "cryptography", "debugpy", "httpx", "hypothesis", "mongomock-motor", "moto[ecs,s3]", "pytest-asyncio", "pytest-mock", "ruff"] +dev = ["allpairspy", "clodius; python_version >= '3.12'", "cloudpickle", "cryptography", "debugpy", "httpx", "hypothesis", "mongomock-motor", "moto[ecs,s3]", "pytest-asyncio", "pytest-mock", "ruff"] +# Matrix tile serving (issue #82). Deliberately an extra rather than a +# runtime dependency, for two reasons that both bite if it moves. +# +# First, `clodius` on public PyPI is a different codebase from the fork +# this consumes — the public releases carry no ``tiles_v2`` — so a bare +# `pip install .` that resolved the name from PyPI would install a +# package that imports and then fails at the first tile. An extra makes +# that resolution impossible rather than merely unlikely. +# +# Second, the fork requires Python >= 3.12 while cfdb still supports +# 3.11. An extra alone does not buy that back — uv resolves universally +# across the whole `requires-python` range and rejects the lock outright +# — so the dependency carries an explicit `python_version >= '3.12'` +# marker. On 3.11 the requirement simply is not there, which is what lets +# the 3.11 support promise survive until the dependency is made +# unconditional. +# +# The consequence is that images built from the Dockerfiles do NOT carry +# clodius, so the tile routes answer 501 there. See ``cfdb.tilesets.backend`` +# for the single import gate that makes that degradation explicit rather +# than a partial failure. Promoting this to a real dependency is tracked +# as follow-up work, gated on the fork being published. +tiles = ["clodius; python_version >= '3.12'"] [project.scripts] cfdb = "cfdb.cli:cli" diff --git a/src/cfdb/api/__init__.py b/src/cfdb/api/__init__.py index 8bd1498..15a8bfb 100644 --- a/src/cfdb/api/__init__.py +++ b/src/cfdb/api/__init__.py @@ -10,6 +10,7 @@ from cfdb.workflows.constants import DEFAULT_TLS_IDENTITY, TLS_IDENTITY_ENV if TYPE_CHECKING: + from cfdb.tilesets.service import TilesetService from cfdb.workflows.cache import CacheBackend from cfdb.workflows.executor import JobExecutor from cfdb.workflows.processors.registry import ProcessorRegistry @@ -275,11 +276,60 @@ def _parse_assign_public_ip(name: str, default: str) -> str: #: profile (no provisioner exists in the local/LAN profile). ECS_MAX_WORKERS: Final = _parse_int_env("ECS_MAX_WORKERS", 16, minimum=0) +# --- Matrix tile serving (issue #82) ---------------------------------------- + +#: Byte budget for the API task's local copies of tileset artifacts. h5py +#: opens a file, so under the S3 profile a cached mcool has to be pulled +#: onto local disk before any tile can be read; this bounds how much of +#: that disk the tile server may hold. Keep it comfortably below the task's +#: ephemeral storage, which it shares with SYNC_DATA_DIR and the container +#: image. Default 8 GiB against a Fargate task sized at 50 GiB. +TILESET_DISK_CACHE_BYTES: Final = _parse_int_env( + "CFDB_TILESET_DISK_CACHE_BYTES", 8 * 1024**3, minimum=1 +) + +#: How many tilesets may be held open at once. Each is an open h5py handle +#: over a multi-GB file, so this is a file-descriptor and mmap bound, not a +#: memory one. +TILESET_OPEN_MAX: Final = _parse_int_env("CFDB_TILESET_OPEN_MAX", 8, minimum=1) + +#: Byte budget for the in-process tile cache. Bounded by bytes rather than +#: by entry count because a 256x256 float32 tile is ~350 KiB base64-encoded +#: — a plausible-sounding 20,000-entry cap would be ~7 GB on a 2 GiB task. +TILE_CACHE_BYTES: Final = _parse_int_env( + "CFDB_TILE_CACHE_BYTES", 256 * 1024**2, minimum=1 +) + +#: Threads used to run clodius tile reads off the event loop. +#: +#: This bounds concurrency; it does NOT buy read parallelism. h5py +#: serializes on a global lock unless linked against a thread-safe HDF5 +#: build, which the published wheels are not — so matrix tile throughput is +#: effectively serial per API process however high this is set. What it +#: does buy is that a multi-second tile read cannot block the event loop +#: and stall /data and /metadata. The real throughput levers are the tile +#: cache and horizontal scaling. +TILE_THREADS: Final = _parse_int_env("CFDB_TILE_THREADS", 2, minimum=1) + +#: Seconds to wait for a cached artifact to be pulled onto local disk +#: before giving up with 503. Bounds the cold-open cost of a multi-GB +#: download, which lands on the first /tileset_info for a dataset rather +#: than on any tile. +TILESET_HYDRATE_TIMEOUT_S: Final = _parse_int_env( + "CFDB_TILESET_HYDRATE_TIMEOUT_S", 60, minimum=1 +) + db: AsyncIOMotorDatabase | None = None cache: "CacheBackend | None" = None executor: "JobExecutor | None" = None processor_registry: "ProcessorRegistry | None" = None +#: Serves the HiGlass tile endpoints. ``None`` when the workflow subsystem +#: is disabled (no SYNC_DATA_DIR, hence no cache to read artifacts from) or +#: when the installed build carries no clodius — in both cases the tile +#: routes answer rather than half-working. See ``cfdb.tilesets.backend``. +tileset_service: "TilesetService | None" = None + #: Snapshot of the lifespan task's ``contextvars.Context()`` taken after #: ``wool.WorkerPool.__aenter__`` returns. uvicorn deliberately spawns #: each ASGI request handler inside a fresh empty ``Context()`` (see diff --git a/src/cfdb/api/main.py b/src/cfdb/api/main.py index b0c1160..7e76dab 100644 --- a/src/cfdb/api/main.py +++ b/src/cfdb/api/main.py @@ -21,6 +21,11 @@ from cfdb.api.routers.index import router as index_router from cfdb.api.routers.jobs import router as jobs_router from cfdb.api.routers.sync import router as sync_router +from cfdb.api.routers.tiles import router as tiles_router +from cfdb.api.routers.tilesets import router as tilesets_router +from cfdb.tilesets import backend as tileset_backend +from cfdb.tilesets.service import TilesetService +from cfdb.tilesets.service import build_service as build_tileset_service from cfdb.workflows.cache import ( CacheBackend, LocalFsCache, @@ -35,6 +40,7 @@ from cfdb.workflows.executor import WoolExecutor from cfdb.workflows.loadbalancer import PriorityLoadBalancer from cfdb.workflows.processors.bam import BamIndexProcessor +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor from cfdb.workflows.processors.registry import default_registry from cfdb.workflows.processors.tabix import TabixIntervalProcessor from cfdb.workflows.provisioner import EcsProvisioner @@ -93,7 +99,20 @@ async def _reset_api_globals() -> None: earlier steps fail. Tests rely on this so a lifespan exception doesn't leak the executor / cache / processor registry / wool context into a subsequent app instantiation. + + The tile service is *closed* before it is dropped. Nulling alone would + leak an open h5py handle per pooled tileset and a live thread pool + into the next app instantiation — precisely the leak this function + exists to prevent, just in a resource the others do not hold. """ + if api.tileset_service is not None: + try: + await api.tileset_service.aclose() + except Exception: + logging.getLogger(__name__).exception( + "Failed to close the tileset service during teardown" + ) + api.tileset_service = None api.executor = None api.cache = None api.processor_registry = None @@ -142,6 +161,49 @@ async def _warn_if_jobs_index_out_of_sync(db, log: logging.Logger) -> None: ) +def _build_tileset_service( + profile: WorkflowProfile, log: logging.Logger +) -> "TilesetService | None": + """Build the matrix tile service, or explain why there isn't one. + + Returns ``None`` when the installed build carries no clodius. That is + the expected state for images built today — the fork is private, so + the Dockerfiles cannot fetch it and it is installed only via the + ``tiles`` extra — and it must degrade to a clear 501 on the tile + routes rather than taking the whole application down at startup. + + The service reads ``api.cache`` through a callable rather than + capturing it, so a lifespan that rebuilds the cache (or a test that + patches it) is not left with a service pointed at a stale backend. + """ + if not tileset_backend.is_available(): + log.warning( + "clodius is not installed; matrix tile serving is disabled and " + "/tileset_info and /tiles will answer 501. Install the 'tiles' " + "extra to enable it." + ) + return None + + service = build_tileset_service( + root=profile.cache_root.parent / "tilesets", + cache_provider=lambda: api.cache, + disk_cache_bytes=api.TILESET_DISK_CACHE_BYTES, + open_max=api.TILESET_OPEN_MAX, + tile_cache_bytes=api.TILE_CACHE_BYTES, + threads=api.TILE_THREADS, + hydrate_timeout_s=api.TILESET_HYDRATE_TIMEOUT_S, + ) + log.info( + "Matrix tile serving enabled (disk budget %d bytes, %d open tilesets, " + "%d tile-cache bytes, %d threads)", + api.TILESET_DISK_CACHE_BYTES, + api.TILESET_OPEN_MAX, + api.TILE_CACHE_BYTES, + api.TILE_THREADS, + ) + return service + + async def _build_cache(profile: WorkflowProfile) -> CacheBackend: """Build the cache backend dictated by ``profile``. @@ -297,6 +359,8 @@ async def lifespan(_: FastAPI): api.processor_registry = default_registry() api.processor_registry.register(BamIndexProcessor()) api.processor_registry.register(TabixIntervalProcessor()) + api.processor_registry.register(MatrixTilesetProcessor()) + api.tileset_service = _build_tileset_service(profile, log) provisioner = _build_provisioner(profile) # Lease workers from the surrounding pool rather than spawning @@ -473,6 +537,11 @@ async def attach_wool_context(request, call_next): app.include_router(index_router) app.include_router(jobs_router) app.include_router(sync_router) +# Mounted at the root, without a prefix: a HiGlass client constructs +# ``{server}/tileset_info/?d=`` and ``{server}/tiles/?d=`` itself, so the +# paths are the wire protocol rather than a cfdb naming choice. +app.include_router(tiles_router) +app.include_router(tilesets_router) @app.get("/health") diff --git a/src/cfdb/api/routers/tiles.py b/src/cfdb/api/routers/tiles.py new file mode 100644 index 0000000..c4be393 --- /dev/null +++ b/src/cfdb/api/routers/tiles.py @@ -0,0 +1,246 @@ +"""HiGlass tile-serving endpoints for Hi-C contact maps. + +Two routes, both strictly read-only: + +- ``GET /tileset_info/?d=`` +- ``GET /tiles/?d=...`` + +They are mounted at the root, and take repeatable ``d`` query parameters, +because that is the wire protocol a HiGlass client constructs — Gosling's +``MatrixData.url`` is conventionally the full ``.../tileset_info/?d=`` +URL, which it splits into a server and a tileset uid. + +**These endpoints never dispatch a workflow.** A heatmap track issues +thousands of tile requests and has nowhere to put a job id, so there is no +useful 202 to return: a dataset whose artifact has not been built is a +404. Building it is the separate preparation channel's job — see +``cfdb.api.routers.tilesets``. +""" + +from __future__ import annotations + +import logging +import re +from collections import OrderedDict +from typing import Any + +from fastapi import APIRouter, HTTPException, Query, status + +from cfdb import api +from cfdb.api.routers._helpers import ( + PATH_PARAM_MAX_LEN, + PATH_PARAM_PATTERN, + resolve_file_doc, +) +from cfdb.services import locks +from cfdb.tilesets import backend +from cfdb.tilesets.errors import ( + TileBackendUnavailable, + TilesetError, + TilesetHydrationTimeout, + TilesetTooLarge, +) +from cfdb.tilesets.wire import error_payload + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["tiles"]) + +#: Ceiling on the number of ``d`` parameters one request may carry. A +#: HiGlass client batches on the order of dozens; anything far beyond that +#: is a way to turn one unauthenticated request into arbitrarily many h5py +#: reads. +MAX_TILE_IDS = 64 + +_UID_PATTERN = re.compile(PATH_PARAM_PATTERN) + + +def _split_uid(uid: str) -> tuple[str, str]: + """Split ``{dcc}/{local_id}`` into its two halves. + + Raises: + HTTPException: 400 when the uid is not the expected shape. + + Note: + A ``.`` in either half is rejected. clodius parses a tile id by + splitting on ``.`` and taking the first field as the uid, so a + dotted identifier would silently be read as a uid plus a zoom + level. No file in the corpus has one — 4DN uses UUIDs, ENCODE uses + ``ENCFF`` accessions — so this is a guard rather than a + restriction, but it is the kind of thing that would otherwise be + found by a user with a very confusing bug report. + """ + dcc, separator, local_id = uid.partition("/") + if not separator or not dcc or not local_id: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Malformed tileset uid '{uid}'; expected '/'", + ) + for part in (dcc, local_id): + if ( + len(part) > PATH_PARAM_MAX_LEN + or "." in part + or not _UID_PATTERN.fullmatch(part) + ): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Malformed tileset uid '{uid}'; each part must match " + f"{PATH_PARAM_PATTERN} and contain no '.'" + ), + ) + return dcc, local_id + + +def _require_service(): + """Return the tile service, or explain why there isn't one. + + Raises: + HTTPException: 501 when the build carries no tile backend, 503 + when the workflow subsystem (and hence the artifact cache) is + not wired. + """ + service = api.tileset_service + if service is not None: + return service + if not backend.is_available(): + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail=( + "Matrix tile serving is not available in this build — it " + "requires the clodius tile backend." + ), + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Tile serving is unavailable — set SYNC_DATA_DIR to enable the " + "artifact cache the tile server reads from." + ), + headers={"Retry-After": "30"}, + ) + + +def _validate_ids(values: list[str], parameter: str) -> list[str]: + """Reject an empty or oversized ``d`` list.""" + if not values: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"At least one '{parameter}' parameter is required", + ) + if len(values) > MAX_TILE_IDS: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Too many '{parameter}' parameters ({len(values)}); at most " + f"{MAX_TILE_IDS} may be requested at once" + ), + ) + return values + + +def _reraise_infrastructure(exc: TilesetError) -> None: + """Translate the tileset failures that are not per-dataset facts. + + A missing backend and a disk-budget or hydration problem are + conditions of the *server*, so they surface as status codes on the + whole request rather than as an error object beside a sibling uid that + happened to work. + """ + if isinstance(exc, TileBackendUnavailable): + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc) + ) + if isinstance(exc, (TilesetTooLarge, TilesetHydrationTimeout)): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=str(exc), + headers={"Retry-After": "30"}, + ) + + +def _render_dataset_failure( + uid: str, exc: TilesetError, *, solo: bool +) -> dict[str, Any]: + """Render a per-dataset failure as a 404 or an error object. + + ``solo`` decides which. A request naming one uid — what a Gosling + matrix track sends — gets the promised 404. A batch naming several + gets an error object beside its siblings, so one unbuilt dataset does + not discard the tiles that were available. + """ + _reraise_infrastructure(exc) + if solo: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail=str(exc) + ) + logger.info(f"Tileset {uid} unavailable: {exc}") + return error_payload(str(exc)) + + +@router.get("/tileset_info/") +@router.get("/tileset_info") +async def tileset_info( + d: list[str] = Query(default_factory=list), # noqa: B008 +) -> dict[str, Any]: + """Return HiGlass tileset metadata for each requested uid. + + Never dispatches: a contact map whose artifact has not been built is + reported as absent, not as pending. + """ + await locks.wait_for_cutover() + + uids = _validate_ids(d, "d") + service = _require_service() + solo = len(uids) == 1 + + response: dict[str, Any] = {} + for uid in uids: + dcc, local_id = _split_uid(uid) + _, file_doc = await resolve_file_doc(dcc, local_id) + try: + response[uid] = await service.tileset_info(uid, file_doc) + except TilesetError as exc: + response[uid] = _render_dataset_failure(uid, exc, solo=solo) + return response + + +@router.get("/tiles/") +@router.get("/tiles") +async def tiles( + d: list[str] = Query(default_factory=list), # noqa: B008 +) -> dict[str, Any]: + """Return tile payloads keyed by the exact tile id that was requested. + + The response carries exactly one entry per requested id — a response + may safely be built by zipping the request against the result. A tile + position the resolution ladder does not hold, a tile id that fails to + parse, or one that names a balancing column the file does not carry + all get an ``{"error": ...}`` object in place of data rather than being + omitted or failing the request. One malformed id in a batch of sixty + must not blank the track. + """ + await locks.wait_for_cutover() + + raw_ids = _validate_ids(d, "d") + service = _require_service() + + # Group by uid, preserving first-seen order, so one dataset is opened + # once per request however many of its tiles were asked for. + grouped: OrderedDict[str, list[str]] = OrderedDict() + for raw in raw_ids: + uid, _, _ = raw.partition(".") + grouped.setdefault(uid, []).append(raw) + solo = len(grouped) == 1 + + response: dict[str, Any] = {} + for uid, uid_tile_ids in grouped.items(): + dcc, local_id = _split_uid(uid) + _, file_doc = await resolve_file_doc(dcc, local_id) + try: + response.update(await service.tiles(uid, file_doc, uid_tile_ids)) + except TilesetError as exc: + failure = _render_dataset_failure(uid, exc, solo=solo) + for raw in uid_tile_ids: + response[raw] = failure + return response diff --git a/src/cfdb/api/routers/tilesets.py b/src/cfdb/api/routers/tilesets.py new file mode 100644 index 0000000..25c1f0f --- /dev/null +++ b/src/cfdb/api/routers/tilesets.py @@ -0,0 +1,258 @@ +"""Preparation channel for matrix tile serving. + +The HiGlass endpoints in ``cfdb.api.routers.tiles`` are pure readers: a +heatmap track cannot be handed a job id, so they 404 rather than 202 when +a contact map's artifact has not been built. This router is where that +artifact gets built. + +- ``GET /tilesets/{dcc}/{local_id}/status`` — side-effect-free, mirroring + the ``/data`` and ``/index`` readiness probes. +- ``POST /tilesets/{dcc}/{local_id}`` — dispatches, and hands back a job + to poll on the existing ``/jobs/{job_id}`` endpoint. + +A UI therefore probes, POSTs if needed, polls, and only then mounts the +Gosling spec — instead of mounting a track that fails until it doesn't. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from fastapi import APIRouter, HTTPException, Path, Response, status +from fastapi.responses import JSONResponse + +from cfdb import api +from cfdb.api.routers._helpers import ( + PATH_PARAM_MAX_LEN, + PATH_PARAM_PATTERN, + resolve_file_doc, +) +from cfdb.api.routers.cache_stream import probe_workflow_readiness +from cfdb.services import locks +from cfdb.tilesets import backend +from cfdb.tilesets.formats import MatrixSource, matrix_source_kind +from cfdb.workflows.executor import ( + AdmissionRejected, + ExecutorDraining, + WorkflowNotApplicable, +) +from cfdb.workflows.models import ArtifactKind + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/tilesets", tags=["tilesets"]) + + +async def _resolve_contact_map(dcc: str, local_id: str) -> tuple[dict, MatrixSource]: + """Resolve a document that is a contact map, or explain why it isn't. + + Raises: + HTTPException: 400/403/404/500 from the shared preamble, or 404 + when the file is not a contact map at all. + """ + _, file_doc = await resolve_file_doc(dcc, local_id) + + kind = matrix_source_kind(file_doc) + if kind is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=( + "This file is not a contact map; only .mcool, .cool, and " + ".hic files can be served as HiGlass tilesets." + ), + ) + return file_doc, kind + + +def _assert_subsystem_wired() -> None: + """Fail loudly when nothing could build or read an artifact. + + Raises: + HTTPException: 501 when the build carries no tile backend, 503 + when the workflow subsystem is disabled. + + Note: + Checked explicitly, ahead of ``probe_workflow_readiness``, because + that helper collapses three distinct conditions into ``None``. + Ruling two of them out here is what lets the third be handled + unambiguously. + """ + if not backend.is_available(): + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail=( + "Matrix tile serving is not available in this build — it " + "requires the clodius tile backend." + ), + ) + if api.processor_registry is None or api.cache is None or api.executor is None: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail=( + "Workflow subsystem disabled — set SYNC_DATA_DIR and start a " + "wool worker pool to prepare contact maps for tile serving." + ), + headers={"Retry-After": "30"}, + ) + + +def _assert_identity_complete(file_doc: dict[str, Any]) -> None: + """Reject a document with no md5 to build a cache key from. + + Raises: + HTTPException: 409 — a stale or hand-edited record, not something + a retry will fix, and not a server fault either. + """ + if not file_doc.get("md5"): + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "File metadata is incomplete (no md5), so no tileset " + "artifact can be addressed for it." + ), + ) + + +def _assert_hic_is_servable(kind: MatrixSource) -> None: + """Refuse a ``.hic``, which this build cannot tile. + + Raises: + HTTPException: 501 for every ``.hic``. + + Note: + Reported here and not only on the tile routes, because the whole + point of this channel is to tell a UI whether mounting a Gosling + spec will work. Answering ``ready: true`` — which is what an + in-place-read design would have meant — would send the client + straight into a 501 from ``/tileset_info``. A readiness probe that + promises something the tile endpoint then refuses is worse than no + probe at all. + """ + if kind is MatrixSource.HIC: + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, + detail=( + "ENCODE .hic contact maps cannot be tiled yet. The tile " + "backend reads local files only, and cfdb does not " + "materialize .hic artifacts — those files run to hundreds " + "of GB. 4DN .mcool contact maps are unaffected." + ), + ) + + +@router.get("/{dcc}/{local_id}/status") +async def tileset_status( + dcc: str = Path(..., max_length=PATH_PARAM_MAX_LEN, pattern=PATH_PARAM_PATTERN), + local_id: str = Path( + ..., max_length=PATH_PARAM_MAX_LEN, pattern=PATH_PARAM_PATTERN + ), +) -> dict[str, bool]: + """Report whether ``GET /tileset_info`` would serve this file now. + + Side-effect-free: it reads cache state and dispatches nothing, exactly + like the ``/data`` and ``/index`` probes it mirrors. + + ``ready: true`` means no preparation is required — the artifact is + already cached and ``GET /tileset_info`` will serve it. + + ``.hic`` answers 501 rather than a readiness value, matching the tile + routes. It is a contact map, but not one this build can serve. + """ + await locks.wait_for_cutover() + + try: + file_doc, kind = await _resolve_contact_map(dcc, local_id) + _assert_subsystem_wired() + _assert_hic_is_servable(kind) + + _assert_identity_complete(file_doc) + readiness = await probe_workflow_readiness(file_doc, ArtifactKind.TILESET) + # Both prior conditions that make this None are ruled out above, + # so None here can only mean the processor declined the file — + # which _resolve_contact_map has already excluded. + return {"ready": bool(readiness)} + + except HTTPException: + raise + except Exception: + logger.exception("Unexpected error in tileset_status") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal server error", + ) + + +@router.post("/{dcc}/{local_id}") +async def prepare_tileset( + dcc: str = Path(..., max_length=PATH_PARAM_MAX_LEN, pattern=PATH_PARAM_PATTERN), + local_id: str = Path( + ..., max_length=PATH_PARAM_MAX_LEN, pattern=PATH_PARAM_PATTERN + ), +) -> Response: + """Build the tileset artifact for a contact map, if it needs one. + + Returns: + ``200 {"ready": true}`` when the artifact is already cached, so + there is nothing to do. ``202`` with a ``Location`` header + pointing at ``/jobs/{id}`` when a workflow was claimed or + attached to. + """ + await locks.wait_for_cutover() + + try: + file_doc, kind = await _resolve_contact_map(dcc, local_id) + _assert_subsystem_wired() + _assert_hic_is_servable(kind) + + _assert_identity_complete(file_doc) + + if await probe_workflow_readiness(file_doc, ArtifactKind.TILESET): + # Idempotent: a second POST for a prepared file is not a job. + return JSONResponse(status_code=status.HTTP_200_OK, content={"ready": True}) + + return await _dispatch(file_doc) + + except HTTPException: + raise + except Exception: + logger.exception("Unexpected error in prepare_tileset") + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal server error", + ) + + +async def _dispatch(file_doc: dict[str, Any]) -> Response: + """Claim or attach to the workflow that builds this file's artifact. + + The exception ladder mirrors ``serve_workflow_artifact_or_dispatch``, + and its order is load-bearing: ``ExecutorDraining`` subclasses + ``WorkflowNotApplicable``, so it has to be caught first or a shutdown + would be reported as an inapplicable file. + """ + try: + record, _ = await api.executor.ensure_workflow(file_doc) + except ExecutorDraining: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service is shutting down; please retry", + headers={"Retry-After": "30"}, + ) + except AdmissionRejected as exc: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Too many active preprocessing jobs; please retry shortly", + headers={"Retry-After": str(exc.retry_after_seconds)}, + ) + except WorkflowNotApplicable as exc: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"No tileset can be prepared for this file: {exc}", + ) + + return JSONResponse( + status_code=status.HTTP_202_ACCEPTED, + content={"job_id": record.job_id, "status": record.status.value}, + headers={"Location": f"/jobs/{record.job_id}", "Retry-After": "5"}, + ) diff --git a/tests/conftest.py b/tests/conftest.py index 24769c1..bdc7874 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from __future__ import annotations +import itertools from typing import Any import pytest @@ -440,3 +441,61 @@ def mock_db(monkeypatch): db = FakeDB() monkeypatch.setattr(api, "db", db) return db + + +# --- Contact-map fixtures (issue #82) --------------------------------------- +# +# Session-scoped because building one costs a few seconds, almost all of it +# cooler's first-import cost, and every consumer opens it read-only. + + +@pytest.fixture(scope="session") +def tiny_mcool(tmp_path_factory): + """A real 3-resolution mcool over a 3000 bp, 3-chromosome genome.""" + from tests.fixtures.coolers import build_mcool + + return build_mcool(tmp_path_factory.mktemp("coolers") / "tiny.mcool") + + +@pytest.fixture(scope="session") +def balanced_mcool(tmp_path_factory): + """An mcool carrying two bin-weight columns. + + Two, not one: ``weight`` is also the column clodius falls back to by + default, so a cooler with only that column cannot distinguish honouring + a named transform request from ignoring it. + """ + from tests.fixtures.coolers import build_mcool + + return build_mcool( + tmp_path_factory.mktemp("coolers_balanced") / "balanced.mcool", + weights={"weight": 0.5, "alt_weight": 2.0}, + ) + + +@pytest.fixture() +def make_mcool(tmp_path): + """Factory building a fresh mcool per call, for tests that mutate one.""" + from tests.fixtures.coolers import build_mcool + + counter = itertools.count() + + def _make(**kwargs): + name = f"case{next(counter)}.mcool" + return build_mcool(tmp_path / name, **kwargs) + + return _make + + +@pytest.fixture() +def make_cool(tmp_path): + """Factory building a fresh flat (single-resolution) cooler per call.""" + from tests.fixtures.coolers import build_cool + + counter = itertools.count() + + def _make(**kwargs): + name = f"flat{next(counter)}.cool" + return build_cool(tmp_path / name, **kwargs) + + return _make diff --git a/tests/test_api/test_lifespan_tilesets.py b/tests/test_api/test_lifespan_tilesets.py new file mode 100644 index 0000000..89594d4 --- /dev/null +++ b/tests/test_api/test_lifespan_tilesets.py @@ -0,0 +1,347 @@ +"""Tests for the lifespan wiring of the matrix tile service (issue #82).""" + +from __future__ import annotations + +import contextlib + +import pytest + +from cfdb import api +from cfdb.api import main +from cfdb.api import profile as profile_mod + + +@pytest.fixture() +def stub_lifespan(request, mocker, tmp_path): + """Drive the enabled-workflow lifespan with everything else stubbed out. + + Indirectly parametrizable over the ``WorkflowProfile`` kind — + ``"local"`` (the default) or ``"s3-cached"`` — built through the real + ``from_env`` so the profile shape matches what a deployment gets. + Returns the profile in use so a test can assert on paths derived + from it. + """ + from mongomock_motor import AsyncMongoMockClient + + kind = getattr(request, "param", "local") + mocker.patch.object(api, "SYNC_DATA_DIR", str(tmp_path)) + mocker.patch.object(api, "ECS_CLUSTER", None) + mocker.patch.object( + api, + "WORKFLOW_S3_BUCKET", + "cfdb-test-artifacts" if kind == "s3-cached" else None, + ) + profile = profile_mod.WorkflowProfile.from_env() + assert profile is not None and profile.kind == kind + + @contextlib.asynccontextmanager + async def _fake_build_discovery(_profile): + yield object() + + class _PoolStub: + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_exc): + return None + + fake_executor = mocker.MagicMock() + fake_executor.drain = mocker.AsyncMock(return_value=0) + + mocker.patch.object( + main, "create_mongodb_client", return_value=AsyncMongoMockClient() + ) + mocker.patch.object(main.WorkflowProfile, "from_env", return_value=profile) + mocker.patch.object( + main, "_build_cache", new=mocker.AsyncMock(return_value=mocker.MagicMock()) + ) + mocker.patch.object(main, "_build_provisioner", return_value=None) + mocker.patch.object(main, "_build_discovery", new=_fake_build_discovery) + mocker.patch.object(main, "WoolExecutor", return_value=fake_executor) + mocker.patch.object(main.wool, "WorkerPool", _PoolStub) + return profile + + +@pytest.mark.asyncio +async def test_lifespan_should_build_the_tile_service_when_clodius_is_present( + stub_lifespan, +): + """Test that the tile service is wired on the enabled-workflow path. + + Given: + The workflow subsystem enabled and clodius installed. + When: + The app lifespan starts up. + Then: + ``api.tileset_service`` should be populated, so the tile routes + serve rather than answering 501. + """ + # Act + async with main.lifespan(main.app): + service = api.tileset_service + + # Assert + assert service is not None + + +@pytest.mark.asyncio +async def test_lifespan_should_register_the_matrix_processor(stub_lifespan): + """Test that contact maps become preparable. + + Given: + The workflow subsystem enabled. + When: + The app lifespan starts up. + Then: + The registry should match an HDF5 contact map. Without this + registration the preparation channel could never build an + artifact, and every tile request would 404 forever. + """ + # Arrange + from cfdb.workflows.processors.matrix import MatrixTilesetProcessor + + doc = {"file_format": {"name": "HDF5"}, "filename": "sample.mcool"} + + # Act + async with main.lifespan(main.app): + processor = api.processor_registry.lookup_for(doc) + + # Assert + assert isinstance(processor, MatrixTilesetProcessor) + + +@pytest.mark.asyncio +async def test_lifespan_should_close_and_clear_the_tile_service_on_teardown( + stub_lifespan, +): + """Test that teardown releases the service's resources. + + Given: + A lifespan that started the tile service. + When: + It tears down. + Then: + The service should be closed and the global nulled. Nulling alone + would leak an open h5py handle per pooled tileset and a live + thread pool into the next app instantiation — the exact class of + leak ``_reset_api_globals`` exists to prevent. + """ + # Arrange + closed: list[bool] = [] + + # Act + async with main.lifespan(main.app): + service = api.tileset_service + original = service.aclose + + async def _tracking_close(): + closed.append(True) + await original() + + service.aclose = _tracking_close + + # Assert + assert closed == [True] + assert api.tileset_service is None + + +@pytest.mark.asyncio +async def test_lifespan_should_leave_the_service_unset_without_clodius( + stub_lifespan, mocker +): + """Test the degraded path for images built without the tile backend. + + Given: + The workflow subsystem enabled but no clodius — the state of + every image built today, since the fork is private and the + Dockerfiles cannot fetch it. + When: + The app lifespan starts up. + Then: + Startup should succeed with the service unset, so the tile routes + answer 501 instead of the whole application failing to boot. + """ + # Arrange + mocker.patch.object(main.tileset_backend, "is_available", return_value=False) + + # Act + async with main.lifespan(main.app): + # Assert + assert api.tileset_service is None + + +@pytest.mark.asyncio +async def test_lifespan_should_leave_the_service_unset_without_a_profile(mocker): + """Test that a disabled workflow subsystem means no tile service. + + Given: + No ``WorkflowProfile`` — SYNC_DATA_DIR unset, so no artifact + cache is built. + When: + The app lifespan starts up. + Then: + ``api.tileset_service`` should stay None. There would be no cache + for it to read artifacts from, and the tile routes report that as + 503 rather than as a permanently empty dataset. + """ + # Arrange + from mongomock_motor import AsyncMongoMockClient + + mocker.patch.object( + main, "create_mongodb_client", return_value=AsyncMongoMockClient() + ) + mocker.patch.object(main.WorkflowProfile, "from_env", return_value=None) + + # Act + async with main.lifespan(main.app): + # Assert + assert api.tileset_service is None + + +@pytest.mark.asyncio +async def test_lifespan_should_propagate_config_to_the_tile_service( + stub_lifespan, mocker +): + """Test that the lifespan hands the service builder the configured knobs. + + Given: + The workflow subsystem enabled and clodius installed, with the + tile-service builder spied at the lifespan boundary. + When: + The app lifespan starts up. + Then: + It should call the builder exactly once with a store root of + ``tilesets`` beside the profile's cache root and the five + module-level knob values — the only observable pin for the + ``CFDB_TILESET_*`` / ``CFDB_TILE_*`` env knobs. + """ + # Arrange + spy = mocker.spy(main, "build_tileset_service") + + # Act + async with main.lifespan(main.app): + pass + + # Assert + spy.assert_called_once() + kwargs = spy.call_args.kwargs + assert kwargs["root"] == stub_lifespan.cache_root.parent / "tilesets" + assert kwargs["disk_cache_bytes"] == api.TILESET_DISK_CACHE_BYTES + assert kwargs["open_max"] == api.TILESET_OPEN_MAX + assert kwargs["tile_cache_bytes"] == api.TILE_CACHE_BYTES + assert kwargs["threads"] == api.TILE_THREADS + assert kwargs["hydrate_timeout_s"] == api.TILESET_HYDRATE_TIMEOUT_S + + +@pytest.mark.asyncio +async def test_lifespan_should_null_all_globals_when_service_close_raises( + stub_lifespan, +): + """Test that a failing service close cannot poison the teardown chain. + + Given: + A started lifespan whose tile service's ``aclose`` raises at + teardown. + When: + The lifespan exits. + Then: + It should swallow the failure and still null ``tileset_service`` + and every other ``cfdb.api`` global, so one bad h5py handle + cannot leak the executor, cache, registry, or wool context into + the next app instantiation. + """ + # Arrange + original_close = None + + # Act + async with main.lifespan(main.app): + service = api.tileset_service + original_close = service.aclose + + async def _exploding_close(): + raise RuntimeError("h5py refused to die") + + service.aclose = _exploding_close + + # Assert + assert api.tileset_service is None + assert api.executor is None + assert api.cache is None + assert api.processor_registry is None + assert api.wool_context is None + + # Release the real handles the exploding stub skipped, so this test + # does not itself leak a thread pool into the next one. + await original_close() + + +@pytest.mark.parametrize("stub_lifespan", ["s3-cached"], indirect=True) +@pytest.mark.asyncio +async def test_lifespan_should_build_the_tile_service_when_the_cache_is_s3( + stub_lifespan, +): + """Test the S3 cell of the profile matrix. + + Given: + The workflow subsystem enabled with an ``s3-cached`` profile and + clodius installed. + When: + The app lifespan starts up. + Then: + ``api.tileset_service`` should be wired exactly as on the local + profile — the S3 profile changes the artifact cache, not whether + tiles are served. + """ + # Arrange + assert stub_lifespan.kind == "s3-cached" + + # Act + async with main.lifespan(main.app): + service = api.tileset_service + + # Assert + assert service is not None + + +@pytest.mark.parametrize("stub_lifespan", ["local", "s3-cached"], indirect=True) +@pytest.mark.asyncio +async def test_lifespan_should_wire_the_cache_provider_to_late_bind( + stub_lifespan, mocker +): + """Test that the tile service reads ``api.cache`` at call time. + + Given: + A started lifespan whose ``api.cache`` is swapped for a fake + cache after startup, on both the local and s3-cached profiles. + When: + A tileset_info request resolves the service's cache provider. + Then: + It should consult the swapped object rather than the cache + captured at boot — the documented late-binding contract, so a + lifespan (or test) that replaces the cache never leaves the + service pointed at a stale backend. + """ + # Arrange + from cfdb.tilesets.errors import TilesetNotReady + + doc = { + "dcc": {"dcc_abbreviation": "4DN"}, + "local_id": "abc123", + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "sample.mcool", + "file_format": {"name": "HDF5"}, + } + swapped = mocker.MagicMock() + swapped.head = mocker.AsyncMock(return_value=None) + + # Act + async with main.lifespan(main.app): + api.cache = swapped + with pytest.raises(TilesetNotReady): + await api.tileset_service.tileset_info("4dn/abc123", doc) + + # Assert + swapped.head.assert_awaited_once() diff --git a/tests/test_tiles.py b/tests/test_tiles.py new file mode 100644 index 0000000..2e2560f --- /dev/null +++ b/tests/test_tiles.py @@ -0,0 +1,1052 @@ +"""Tests for the HiGlass tile endpoints in ``cfdb.api.routers.tiles``.""" + +from __future__ import annotations + +import asyncio +import base64 +import itertools + +import numpy as np +import pytest +from fastapi import HTTPException +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from cfdb import api +from cfdb.api.routers._helpers import PATH_PARAM_MAX_LEN +from cfdb.api.routers.tiles import MAX_TILE_IDS, tiles, tileset_info +from cfdb.services import locks +from cfdb.tilesets import backend +from cfdb.tilesets.errors import ( + TileBackendUnavailable, + TilesetHydrationTimeout, + TilesetTooLarge, +) +from cfdb.tilesets.service import build_service +from cfdb.workflows.cache import LocalFsCache +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from tests.test_workflows import FIXTURE_MD5 + +UID = "4dn/4DNFIABC123" + + +def _file_doc(**overrides) -> dict: + """Return a projected document for a 4DN mcool.""" + doc = { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": FIXTURE_MD5, + "filename": "sample.mcool", + "file_format": {"name": "HDF5"}, + "genome_assembly": "GRCh38", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + doc.update(overrides) + return doc + + +class _FailIfDispatched: + """An executor stand-in that fails the test if a tile route dispatches.""" + + async def ensure_workflow(self, _file_doc): + raise AssertionError("tile endpoints must never dispatch a workflow") + + +class _FailIfConsulted: + """A service stand-in that fails the test if a tile route reaches it.""" + + async def tileset_info(self, *_args, **_kwargs): + raise AssertionError("a malformed uid must never reach the service") + + async def tiles(self, *_args, **_kwargs): + raise AssertionError("a malformed uid must never reach the service") + + +#: Characters that are legal in both halves of a tileset uid, minus ``.`` +#: (rejected by the dot guard) — the alphabet well-formed parts draw from. +_PART_ALPHABET = ( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-" +) + +#: Characters outside ``PATH_PARAM_PATTERN`` (and not ``/``, which would +#: change how the uid splits rather than making a part illegal). +_ILLEGAL_CHARS = "!@#$%^&*()+= ~\t\n\"'<>?,;:|\\[]{}éλ" + +_wellformed_part = st.text(alphabet=_PART_ALPHABET, min_size=1, max_size=12) + + +@st.composite +def _malformed_uids(draw): + """Generate uid strings violating exactly one shape rule each.""" + part = draw(_wellformed_part) + other = draw(_wellformed_part) + kind = draw( + st.sampled_from( + [ + "no-slash", + "empty-dcc", + "empty-local-id", + "dotted-part", + "oversized-part", + "illegal-char", + ] + ) + ) + if kind == "no-slash": + return part + if kind == "empty-dcc": + return f"/{part}" + if kind == "empty-local-id": + return f"{part}/" + if kind == "dotted-part": + dotted = f"{part}.{other}" + return f"{dotted}/{other}" if draw(st.booleans()) else f"{part}/{dotted}" + if kind == "oversized-part": + oversized = part.ljust(PATH_PARAM_MAX_LEN + 1, "x") + return ( + f"{oversized}/{other}" if draw(st.booleans()) else f"{part}/{oversized}" + ) + position = draw(st.integers(min_value=0, max_value=len(part))) + bad = part[:position] + draw(st.sampled_from(_ILLEGAL_CHARS)) + part[position:] + return f"{bad}/{other}" if draw(st.booleans()) else f"{part}/{bad}" + + +@pytest.fixture() +def tile_env(mock_db, mocker, tmp_path): + """Wire a real tile service over a LocalFsCache, plus a seeded document. + + Returns a callable that seeds a document's artifact from an mcool. + """ + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + cache = LocalFsCache(tmp_path / "cache") + service = build_service( + root=tmp_path / "tilesets", + cache_provider=lambda: cache, + disk_cache_bytes=10**9, + open_max=4, + tile_cache_bytes=10**7, + threads=2, + hydrate_timeout_s=30, + ) + mocker.patch.object(api, "cache", cache) + mocker.patch.object(api, "tileset_service", service) + mocker.patch.object(api, "executor", _FailIfDispatched()) + + async def _seed(doc, mcool): + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + staged = tmp_path / f"staged-{key.replace('/', '_')}" + staged.write_bytes(mcool.read_bytes()) + await cache.put(key, staged) + + yield _seed, mock_db, service + + +class TestTilesetInfo: + """``GET /tileset_info/?d=``.""" + + @pytest.mark.asyncio + async def test_should_return_a_document_keyed_by_uid( + self, tile_env, tiny_mcool + ): + """Test the response envelope and the injected fields. + + Given: + A 4DN mcool whose tileset artifact is cached. + When: + tileset_info is requested for its uid. + Then: + The response should be keyed by the uid and carry the four + fields clodius does not model — datatype, name, uuid, and + coordSystem — alongside the resolution ladder. + """ + # Arrange + seed, mock_db, _ = tile_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + + # Act + response = await tileset_info(d=[UID]) + + # Assert + info = response[UID] + assert info["datatype"] == "matrix" + assert info["name"] == "sample.mcool" + assert info["uuid"] == UID + assert info["coordSystem"] == "GRCh38" + assert info["resolutions"] == [1, 2, 4] + + @pytest.mark.asyncio + async def test_should_404_a_contact_map_whose_artifact_is_absent( + self, tile_env + ): + """Test the not-yet-tileable answer. + + Given: + A contact map with nothing in the cache. + When: + tileset_info is requested. + Then: + It should raise 404 and dispatch nothing. A heatmap track has + nowhere to put a job id, so there is no useful 202 here — the + preparation channel owns building the artifact. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [_file_doc()] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=[UID]) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_should_404_a_file_that_is_not_a_contact_map(self, tile_env): + """Test that a non-matrix file is permanently untileable. + + Given: + An AnnData matrix, which shares the HDF5 EDAM term. + When: + tileset_info is requested. + Then: + It should raise 404. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [_file_doc(filename="matrix.h5ad")] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=[UID]) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_should_403_a_non_public_hubmap_file(self, tile_env): + """Test that the access guard covers this new egress path. + + Given: + A non-public HuBMAP contact map. + When: + tileset_info is requested. + Then: + It should raise 403. These endpoints are a new way for bytes + to leave the service, so they run the same guard /data and + /index do. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [ + _file_doc( + submission="hubmap", + local_id="HBM123", + data_access_level="consortium", + ) + ] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=["hubmap/HBM123"]) + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_should_400_an_unknown_dcc(self, tile_env): + """Test that an unrecognized DCC is rejected. + + Given: + A uid naming a DCC that does not exist. + When: + tileset_info is requested. + Then: + It should raise 400, mirroring /data and /index. + """ + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=["nosuchdcc/abc"]) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_should_render_a_failed_sibling_as_an_error_object( + self, tile_env, tiny_mcool + ): + """Test that a batch does not discard the datasets that worked. + + Given: + Two uids, one cached and one not. + When: + tileset_info is requested for both. + Then: + The cached one should return its document and the other an + error object — rather than the whole request 404ing and + throwing away work that succeeded. A single-uid request, which + is what a Gosling matrix track sends, still gets a clean 404. + """ + # Arrange + seed, mock_db, _ = tile_env + ready = _file_doc() + missing = _file_doc(local_id="4DNFIXYZ789", md5="0" * 32) + mock_db.file.docs = [ready, missing] + await seed(ready, tiny_mcool) + + # Act + response = await tileset_info(d=[UID, "4dn/4DNFIXYZ789"]) + + # Assert + assert response[UID]["datatype"] == "matrix" + assert "error" in response["4dn/4DNFIXYZ789"] + + _completeness_salt = itertools.count() + + @given(data=st.data()) + @settings( + max_examples=8, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + def test_should_answer_every_uid_of_a_wellformed_batch( + self, tile_env, tiny_mcool, tmp_path, data + ): + """Test the completeness invariant of a tileset_info batch. + + Given: + A batch of 2-8 well-formed uids, each resolving to a contact + map document, with a generated subset holding a cached + artifact. + When: + tileset_info is requested for the whole batch. + Then: + It should key the response by exactly the requested uids — + an info document for every cached dataset and an error object + for every uncached one, never omitting a uid the way /tiles + omits an out-of-ladder position. + """ + # Arrange + _, mock_db, _ = tile_env + salt = next(self._completeness_salt) + size = data.draw(st.integers(min_value=2, max_value=8), label="batch size") + seeded_mask = data.draw( + st.lists(st.booleans(), min_size=size, max_size=size), + label="has artifact", + ) + docs = [ + _file_doc( + local_id=f"4DNFIP{salt}N{index}", + md5=f"{salt * 4096 + index:032x}", + ) + for index in range(size) + ] + mock_db.file.docs = docs + uids = [f"4dn/{doc['local_id']}" for doc in docs] + + async def scenario(): + cache = LocalFsCache(tmp_path / f"p2-cache-{salt}") + service = build_service( + root=tmp_path / f"p2-tilesets-{salt}", + cache_provider=lambda: cache, + disk_cache_bytes=10**9, + open_max=4, + tile_cache_bytes=10**7, + threads=2, + hydrate_timeout_s=30, + ) + api.tileset_service = service + try: + processor = MatrixTilesetProcessor() + for doc, is_seeded in zip(docs, seeded_mask): + if not is_seeded: + continue + key = processor.cache_key_for(doc, ArtifactKind.TILESET) + staged = tmp_path / f"p2-staged-{key.replace('/', '_')}" + staged.write_bytes(tiny_mcool.read_bytes()) + await cache.put(key, staged) + # Act + return await tileset_info(d=uids) + finally: + await service.aclose() + + response = asyncio.run(scenario()) + + # Assert + assert set(response) == set(uids) + for uid, is_seeded in zip(uids, seeded_mask): + if is_seeded: + assert response[uid]["resolutions"] == [1, 2, 4] + else: + assert "error" in response[uid] + + +class TestTiles: + """``GET /tiles/?d=...``.""" + + @pytest.mark.asyncio + async def test_should_return_a_dense_payload_keyed_by_request_string( + self, tile_env, tiny_mcool + ): + """Test the tile response envelope. + + Given: + A cached contact map. + When: + One tile is requested. + Then: + The response should be keyed by the exact request string and + carry a base64 dense block that decodes to 256x256 — clodius + emits no ``shape`` field, so the client recovers the side as + sqrt(len). + """ + # Arrange + seed, mock_db, _ = tile_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + + # Act + response = await tiles(d=[f"{UID}.0.0.0"]) + + # Assert + payload = response[f"{UID}.0.0.0"] + assert "shape" not in payload + flat = np.frombuffer(base64.b64decode(payload["dense"]), payload["dtype"]) + assert len(flat) == 256 * 256 + + @pytest.mark.asyncio + async def test_should_report_an_out_of_ladder_tile_from_a_mixed_batch( + self, tile_env, tiny_mcool + ): + """Test the one-entry-per-id contract at the HTTP boundary. + + Given: + A batch pairing a valid tile with one past the ladder. + When: + tiles is requested. + Then: + Both ids should appear in the response — the valid tile with + real data, and the out-of-ladder one with a per-tile error + object. clodius returns exactly one entry per requested id, so + a response assembled by zipping request against result is + safe. + """ + # Arrange + seed, mock_db, _ = tile_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + + # Act + response = await tiles(d=[f"{UID}.0.0.0", f"{UID}.9.0.0"]) + + # Assert + assert set(response) == {f"{UID}.0.0.0", f"{UID}.9.0.0"} + assert "dense" in response[f"{UID}.0.0.0"] + assert "error" in response[f"{UID}.9.0.0"] + + @pytest.mark.asyncio + async def test_should_report_a_malformed_id_without_failing_the_batch( + self, tile_env, tiny_mcool + ): + """Test the per-tile error convention. + + Given: + A batch pairing a valid tile id with a malformed one. + When: + tiles is requested. + Then: + The malformed id should get an error object while its sibling + returns data, and the request should still be a 200. + """ + # Arrange + seed, mock_db, _ = tile_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + + # Act + response = await tiles(d=[f"{UID}.0.0.0", f"{UID}.notanint.0"]) + + # Assert + assert "error" in response[f"{UID}.notanint.0"] + assert "dense" in response[f"{UID}.0.0.0"] + + @pytest.mark.asyncio + async def test_should_404_every_tile_of_an_unbuilt_dataset(self, tile_env): + """Test that an unbuilt dataset 404s rather than dispatching. + + Given: + A contact map with no cached artifact. + When: + A tile is requested. + Then: + It should raise 404 and never reach the executor — which the + fixture's executor stand-in would fail the test over. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [_file_doc()] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tiles(d=[f"{UID}.0.0.0"]) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_should_open_a_dataset_once_for_a_multi_tile_batch( + self, tile_env, tiny_mcool, mocker + ): + """Test that tiles are grouped by dataset before being read. + + Given: + Four tiles of one dataset in a single request. + When: + tiles is requested. + Then: + The service should be asked once, not four times. A heatmap + track requests a screenful at a time, and h5py serializes on a + global lock, so re-opening per tile would be the dominant + cost. + """ + # Arrange + seed, mock_db, service = tile_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + spy = mocker.spy(service, "tiles") + + # Act + await tiles( + d=[ + f"{UID}.2.0.0", + f"{UID}.2.1.0", + f"{UID}.2.0.1", + f"{UID}.2.1.1", + ] + ) + + # Assert + spy.assert_called_once() + + @pytest.mark.asyncio + async def test_should_503_the_whole_batch_when_hydration_times_out( + self, tile_env, mocker + ): + """Test that an infrastructure failure is never a per-tile error. + + Given: + Tile ids spanning two resolvable datasets, with the service + raising TilesetHydrationTimeout. + When: + tiles is requested. + Then: + It should raise 503 with Retry-After for the whole request — + a hydration timeout is a condition of the server, not a fact + about one dataset, so it must not render as an error object + beside a sibling that happened to work. + """ + # Arrange + _, mock_db, service = tile_env + mock_db.file.docs = [ + _file_doc(), + _file_doc(local_id="4DNFIXYZ789", md5="1" * 32), + ] + mocker.patch.object( + service, + "tiles", + side_effect=TilesetHydrationTimeout("hydration exceeded its budget"), + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tiles(d=[f"{UID}.0.0.0", "4dn/4DNFIXYZ789.0.0.0"]) + assert exc_info.value.status_code == 503 + assert exc_info.value.headers["Retry-After"] == "30" + + @pytest.mark.asyncio + async def test_should_fan_one_error_object_across_an_unbuilt_datasets_ids( + self, tile_env, tiny_mcool + ): + """Test the per-uid failure fan-out in a mixed batch. + + Given: + Two datasets, one cached and one unbuilt, with two tile ids + for the unbuilt one and one for the cached one. + When: + tiles is requested for all three ids. + Then: + The cached dataset's tile should return data untouched, and + the same error object should sit under each of the unbuilt + dataset's ids — the failure is per dataset, fanned out to + every tile that named it. + """ + # Arrange + seed, mock_db, _ = tile_env + ready = _file_doc() + unbuilt = _file_doc(local_id="4DNFIXYZ789", md5="0" * 32) + mock_db.file.docs = [ready, unbuilt] + await seed(ready, tiny_mcool) + unbuilt_ids = ["4dn/4DNFIXYZ789.0.0.0", "4dn/4DNFIXYZ789.1.0.0"] + + # Act + response = await tiles(d=[*unbuilt_ids, f"{UID}.0.0.0"]) + + # Assert + assert "dense" in response[f"{UID}.0.0.0"] + assert "error" in response[unbuilt_ids[0]] + assert response[unbuilt_ids[0]] == response[unbuilt_ids[1]] + + @pytest.mark.asyncio + async def test_should_404_the_whole_batch_when_a_uid_matches_no_document( + self, tile_env, tiny_mcool + ): + """Test that database absence fails at the request level. + + Given: + One cached dataset plus a uid that matches no document at + all. + When: + tiles is requested for a tile of each. + Then: + It should raise 404 for the whole request — a uid with no + document fails before the per-dataset error rendering, unlike + an unbuilt dataset, which gets an error object beside its + siblings. This asymmetry is pinned as current behavior. + """ + # Arrange + seed, mock_db, _ = tile_env + ready = _file_doc() + mock_db.file.docs = [ready] + await seed(ready, tiny_mcool) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tiles(d=[f"{UID}.0.0.0", "4dn/4DNFINODOC01.0.0.0"]) + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_should_group_interleaved_ids_by_dataset_in_first_seen_order( + self, tile_env, tiny_mcool, mocker + ): + """Test the grouping contract for a multi-dataset batch. + + Given: + Two cached datasets with their tile ids interleaved A,B,A,B. + When: + tiles is requested. + Then: + The service should be asked exactly twice — once per dataset, + in first-seen order, each call carrying that dataset's ids in + request order — and the response should be keyed by the exact + raw ids. + """ + # Arrange + seed, mock_db, service = tile_env + first = _file_doc() + second = _file_doc(local_id="4DNFIDEF456", md5="f" * 32) + mock_db.file.docs = [first, second] + await seed(first, tiny_mcool) + await seed(second, tiny_mcool) + spy = mocker.spy(service, "tiles") + second_uid = "4dn/4DNFIDEF456" + ids = [ + f"{UID}.0.0.0", + f"{second_uid}.0.0.0", + f"{UID}.1.0.0", + f"{second_uid}.1.0.0", + ] + + # Act + response = await tiles(d=ids) + + # Assert + assert spy.call_count == 2 + (first_call, second_call) = spy.call_args_list + assert first_call.args[0] == UID + assert first_call.args[2] == [f"{UID}.0.0.0", f"{UID}.1.0.0"] + assert second_call.args[0] == second_uid + assert second_call.args[2] == [ + f"{second_uid}.0.0.0", + f"{second_uid}.1.0.0", + ] + assert set(response) == set(ids) + + @pytest.mark.asyncio + async def test_should_404_a_multi_tile_request_for_one_unbuilt_dataset( + self, tile_env + ): + """Test that the solo pivot counts datasets, not tile ids. + + Given: + Several tile ids all naming one unbuilt dataset. + When: + tiles is requested. + Then: + It should raise 404 for the whole request — the solo-vs-batch + pivot keys on the number of distinct uids, not the number of + ids, so three tiles of one dataset still get the clean 404 a + Gosling matrix track expects. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [_file_doc()] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tiles(d=[f"{UID}.0.0.0", f"{UID}.1.0.0", f"{UID}.2.0.0"]) + assert exc_info.value.status_code == 404 + + +class TestRequestValidation: + """Guards on the query parameters themselves.""" + + @pytest.mark.asyncio + async def test_should_400_a_request_with_no_ids(self, tile_env): + """Test that a bare request is rejected. + + Given: + No ``d`` parameter. + When: + tiles is requested. + Then: + It should raise 400 rather than returning an empty object, + which would look like a successful read of nothing. + """ + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tiles(d=[]) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_should_400_a_batch_beyond_the_ceiling(self, tile_env): + """Test the cap on tile ids per request. + + Given: + More tile ids than the ceiling allows. + When: + tiles is requested. + Then: + It should raise 400. Without a cap, one unauthenticated + request turns into arbitrarily many h5py reads. + """ + # Arrange + ids = [f"{UID}.0.0.{i}" for i in range(MAX_TILE_IDS + 1)] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tiles(d=ids) + assert exc_info.value.status_code == 400 + + @pytest.mark.asyncio + async def test_should_admit_a_batch_at_exactly_the_ceiling( + self, tile_env, tiny_mcool + ): + """Test that the tile-id cap is inclusive. + + Given: + A cached contact map and exactly MAX_TILE_IDS tile ids. + When: + tiles is requested. + Then: + It should be admitted rather than 400ed — the guard rejects + strictly more than the ceiling — and the in-ladder tile + should come back as data. + """ + # Arrange + seed, mock_db, _ = tile_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + ids = [f"{UID}.0.{i}.0" for i in range(MAX_TILE_IDS)] + + # Act + response = await tiles(d=ids) + + # Assert + assert "dense" in response[f"{UID}.0.0.0"] + assert set(response) <= set(ids) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "uid", + [ + "4DNFIABC123", + "4dn/", + "/4DNFIABC123", + "4dn/a/b", + "4dn/has.dot", + "4dn/" + "a" * (PATH_PARAM_MAX_LEN + 1), + "4dn/bad!id", + ], + ids=[ + "no-slash", + "no-local-id", + "no-dcc", + "two-slashes", + "dotted-id", + "over-length", + "illegal-char", + ], + ) + async def test_should_400_a_malformed_uid(self, tile_env, uid): + """Test uid shape validation, including the dot guard. + + Given: + A uid that is not ``/``, or whose local id + contains a dot, exceeds the path-param length cap, or carries + a character outside the allowed set. + When: + tileset_info is requested. + Then: + It should raise 400. The dot case matters specifically: + clodius parses a tile id by splitting on ``.`` and taking the + first field as the uid, so a dotted identifier would be + silently read as a uid plus a zoom level. + """ + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=[uid]) + assert exc_info.value.status_code == 400 + + @given(uid=_malformed_uids()) + @settings( + max_examples=50, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + def test_should_400_every_malformed_uid_before_any_lookup( + self, tile_env, uid + ): + """Test the totality of uid rejection. + + Given: + Generated uid strings each violating one shape rule — no + slash, an empty half, a dotted part, a part past the length + cap, or an illegal character. + When: + tileset_info is requested. + Then: + It should raise exactly 400 from the uid validator — never a + 500, and never reaching the database or the service, which + the stand-in service would fail the test over. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [] + api.tileset_service = _FailIfConsulted() + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + asyncio.run(tileset_info(d=[uid])) + assert exc_info.value.status_code == 400 + assert "Malformed tileset uid" in exc_info.value.detail + + +class TestHicIsDeferred: + """ENCODE .hic is recognized, and specifically refused.""" + + @pytest.mark.asyncio + async def test_should_501_a_hic_rather_than_404_it(self, tile_env): + """Test that .hic is refused as unimplemented, not as unknown. + + Given: + An ENCODE .hic — a genuine contact map, but one cfdb cannot + serve: the clodius tileset is built on hictkpy, which reads + local paths only, and cfdb does not materialize .hic + artifacts because they run to hundreds of GB. + When: + tileset_info is requested. + Then: + It should raise 501, not the 404 an unrecognized format gets. + The distinction is the whole reason MatrixSource still carries + a HIC member: "we do not support this yet" and "this is not a + contact map" are different answers, and a client can act on + the first. + """ + # Arrange + _, mock_db, _ = tile_env + mock_db.file.docs = [ + _file_doc( + submission="encode", local_id="ENCFF123ABC", filename="contact.hic" + ) + ] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=["encode/ENCFF123ABC"]) + assert exc_info.value.status_code == 501 + assert ".hic" in exc_info.value.detail + + +class TestInfrastructureFailures: + """The ``_reraise_infrastructure`` ladder: server conditions, not facts. + + A missing backend and a disk-budget or hydration problem describe the + server, so they fail the whole request as a status code — never an + error object rendered beside a sibling uid that happened to work. + """ + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("exc", "expected_status", "retry_after"), + [ + (TilesetHydrationTimeout("hydration exceeded its budget"), 503, "30"), + (TilesetTooLarge("artifact exceeds the disk budget"), 503, "30"), + (TileBackendUnavailable("this build carries no tile backend"), 501, None), + ], + ids=["hydration-timeout", "too-large", "backend-unavailable"], + ) + async def test_should_fail_the_whole_batch_on_an_infrastructure_error( + self, tile_env, mocker, exc, expected_status, retry_after + ): + """Test the infrastructure-error ladder against a healthy sibling. + + Given: + A two-uid batch whose first dataset answers cleanly and whose + second raises an infrastructure error — a hydration timeout, + an over-budget artifact, or a missing backend. + When: + tileset_info is requested for both. + Then: + It should raise for the whole request — 503 with Retry-After + for the operator conditions, 501 for the missing backend — + rather than rendering a per-entry error object beside the + healthy sibling. + """ + # Arrange + _, mock_db, service = tile_env + mock_db.file.docs = [ + _file_doc(), + _file_doc(local_id="4DNFIXYZ789", md5="1" * 32), + ] + mocker.patch.object( + service, + "tileset_info", + side_effect=[{"datatype": "matrix"}, exc], + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=[UID, "4dn/4DNFIXYZ789"]) + assert exc_info.value.status_code == expected_status + assert (exc_info.value.headers or {}).get("Retry-After") == retry_after + + +class TestOverHttp: + """The wire layer, which calling the handlers directly cannot exercise. + + The repeatable ``d`` parameter and the trailing slash are both parts of + the HiGlass protocol that a client constructs rather than cfdb, so they + are only really tested through ASGI. Follows ``tests/test_cors.py`` in + neutering the lifespan. + """ + + @pytest.fixture() + def client(self, mocker): + from mongomock_motor import AsyncMongoMockClient + from starlette.testclient import TestClient + + from cfdb.api import main + + mocker.patch.object( + main, "create_mongodb_client", return_value=AsyncMongoMockClient() + ) + mocker.patch.object(main.WorkflowProfile, "from_env", return_value=None) + with TestClient(main.app) as client: + yield client + + @pytest.mark.parametrize( + "url", + [ + "/tileset_info/?d=4dn/ABC&d=4dn/DEF", + "/tileset_info?d=4dn/ABC", + "/tiles/?d=4dn/ABC.0.0.0&d=4dn/ABC.0.1.0", + "/tiles?d=4dn/ABC.0.0.0", + ], + ) + def test_should_accept_repeated_d_parameters_with_or_without_a_slash( + self, client, url + ): + """Test that the HiGlass request shape reaches the handler. + + Given: + URLs in the shape a HiGlass client builds — repeated ``d`` + parameters, with and without the trailing slash. + When: + They are requested over HTTP. + Then: + They should reach the handler rather than 404ing on routing or + 422ing on parameter parsing. Here the lifespan is disabled, so + the handler answers 503; what matters is that it was reached. + """ + # Act + response = client.get(url) + + # Assert + assert response.status_code == 503 + + def test_should_400_a_request_carrying_no_d_parameter(self, client): + """Test that the empty-request guard survives the wire layer. + + Given: + A tileset_info URL with no ``d`` parameter at all. + When: + It is requested over HTTP. + Then: + It should be a 400 rather than an empty 200, which would look + like a successful read of nothing. + """ + # Act + response = client.get("/tileset_info/") + + # Assert + assert response.status_code == 400 + + +class TestDegradedModes: + """What the routes say when there is no service at all.""" + + @pytest.mark.asyncio + async def test_should_501_when_the_tile_backend_is_absent( + self, tile_env, mocker + ): + """Test the answer for an image built without clodius. + + Given: + No tile service, and no clodius to build one from — the state + of every image built today, since the fork is private. + When: + tileset_info is requested. + Then: + It should raise 501. The subsystem degrades to a clear "not + implemented here" rather than half-working. + """ + # Arrange + mocker.patch.object(api, "tileset_service", None) + mocker.patch.object(backend, "is_available", return_value=False) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=[UID]) + assert exc_info.value.status_code == 501 + + @pytest.mark.asyncio + async def test_should_503_when_the_workflow_subsystem_is_disabled( + self, tile_env, mocker + ): + """Test the answer when there is no artifact cache to read. + + Given: + clodius present but no tile service, as when SYNC_DATA_DIR is + unset and no cache was built. + When: + tileset_info is requested. + Then: + It should raise 503 with Retry-After — a deployment + configuration problem, distinct from an unsupported build. + """ + # Arrange + mocker.patch.object(api, "tileset_service", None) + mocker.patch.object(backend, "is_available", return_value=True) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_info(d=[UID]) + assert exc_info.value.status_code == 503 + assert exc_info.value.headers["Retry-After"] == "30" diff --git a/tests/test_tilesets_router.py b/tests/test_tilesets_router.py new file mode 100644 index 0000000..9772045 --- /dev/null +++ b/tests/test_tilesets_router.py @@ -0,0 +1,748 @@ +"""Tests for the preparation channel in ``cfdb.api.routers.tilesets``.""" + +from __future__ import annotations + +import json + +import pytest +from fastapi import HTTPException + +from cfdb import api +from cfdb.api.routers import tilesets +from cfdb.api.routers.tilesets import prepare_tileset, tileset_status +from cfdb.services import locks +from cfdb.tilesets import backend +from cfdb.workflows.cache import LocalFsCache +from cfdb.workflows.executor import ( + AdmissionRejected, + ExecutorDraining, + WorkflowNotApplicable, +) +from cfdb.workflows.models import ArtifactKind, JobStatus +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from cfdb.workflows.processors.registry import ProcessorRegistry +from tests.test_workflows import FIXTURE_MD5 + + +def _file_doc(**overrides) -> dict: + """Return a projected document for a 4DN mcool.""" + doc = { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": FIXTURE_MD5, + "filename": "sample.mcool", + "file_format": {"name": "HDF5"}, + "genome_assembly": "GRCh38", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + doc.update(overrides) + return doc + + +class _Record: + """Minimal stand-in for the JobRecord ``ensure_workflow`` returns.""" + + job_id = "job-abc-123" + status = JobStatus.PENDING + + +class _StubExecutor: + """Executor stand-in that returns a record or raises a chosen error.""" + + def __init__(self, error: Exception | None = None) -> None: + self.error = error + self.calls = 0 + + async def ensure_workflow(self, _file_doc): + self.calls += 1 + if self.error is not None: + raise self.error + return _Record(), True + + +@pytest.fixture() +def prep_env(mock_db, mocker, tmp_path): + """Wire the workflow subsystem with the matrix processor registered.""" + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + cache = LocalFsCache(tmp_path / "cache") + registry = ProcessorRegistry() + registry.register(MatrixTilesetProcessor()) + executor = _StubExecutor() + mocker.patch.object(api, "cache", cache) + mocker.patch.object(api, "processor_registry", registry) + mocker.patch.object(api, "executor", executor) + + async def _seed(doc, mcool): + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + staged = tmp_path / f"staged-{key.replace('/', '_')}" + staged.write_bytes(mcool.read_bytes()) + await cache.put(key, staged) + + return _seed, mock_db, executor, mocker + + +def _body(response) -> dict: + """Decode a JSONResponse body.""" + return json.loads(response.body) + + +class TestStatusProbe: + """``GET /tilesets/{dcc}/{local_id}/status``.""" + + @pytest.mark.asyncio + async def test_should_report_not_ready_when_no_artifact_is_cached( + self, prep_env + ): + """Test the pre-preparation state. + + Given: + A contact map whose artifact has not been built. + When: + The status probe runs. + Then: + It should report not ready and dispatch nothing — this probe + mirrors the /data and /index probes, which are documented as + never dispatching. + """ + # Arrange + _, mock_db, executor, _ = prep_env + mock_db.file.docs = [_file_doc()] + + # Act + result = await tileset_status("4dn", "4DNFIABC123") + + # Assert + assert result == {"ready": False} + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_report_ready_once_the_artifact_is_cached( + self, prep_env, tiny_mcool + ): + """Test the post-preparation state. + + Given: + A contact map whose tileset artifact is cached. + When: + The status probe runs. + Then: + It should report ready, which is a UI's signal that mounting + the Gosling spec will now work. + """ + # Arrange + seed, mock_db, _, _ = prep_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + + # Act + result = await tileset_status("4dn", "4DNFIABC123") + + # Assert + assert result == {"ready": True} + + @pytest.mark.asyncio + async def test_should_refuse_a_hic_rather_than_report_a_readiness( + self, prep_env + ): + """Test that .hic is refused here, not reported as ready. + + Given: + An ENCODE .hic, which cfdb does not serve — the tile backend + reads local files only, and these run to hundreds of GB. + When: + The status probe runs. + Then: + It should raise 501, matching the tile routes. Reporting + ``ready: true`` would be worse than useless: the whole point + of this probe is to tell a UI whether mounting a Gosling spec + will work, and a promise the tile endpoint then refuses sends + the client straight into a 501. + """ + # Arrange + _, mock_db, _, _ = prep_env + mock_db.file.docs = [ + _file_doc(submission="encode", local_id="ENCFF123ABC", filename="c.hic") + ] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("encode", "ENCFF123ABC") + assert exc_info.value.status_code == 501 + + @pytest.mark.asyncio + async def test_should_404_a_file_that_is_not_a_contact_map(self, prep_env): + """Test that non-matrix files are rejected here too. + + Given: + An AnnData matrix. + When: + The status probe runs. + Then: + It should raise 404 with a message naming the supported + formats. + """ + # Arrange + _, mock_db, _, _ = prep_env + mock_db.file.docs = [_file_doc(filename="matrix.h5ad")] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 404 + + @pytest.mark.asyncio + async def test_should_403_a_non_public_hubmap_file(self, prep_env): + """Test that access control precedes any readiness answer. + + Given: + A non-public HuBMAP contact map. + When: + The status probe runs. + Then: + It should raise 403 rather than disclosing readiness. + """ + # Arrange + _, mock_db, _, _ = prep_env + mock_db.file.docs = [ + _file_doc( + submission="hubmap", + local_id="HBM123", + data_access_level="protected", + ) + ] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("hubmap", "HBM123") + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_should_503_when_the_workflow_subsystem_is_disabled( + self, prep_env + ): + """Test degraded mode. + + Given: + No cache, as when SYNC_DATA_DIR is unset. + When: + The status probe runs. + Then: + It should raise 503 with Retry-After, matching how /index + signals the same condition. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object(api, "cache", None) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 503 + + @pytest.mark.asyncio + async def test_should_409_a_document_with_no_md5(self, prep_env): + """Test that an unaddressable document is a conflict, not a 500. + + Given: + A contact map whose document carries no md5, so no cache key + can be derived for it. + When: + The status probe runs. + Then: + It should raise 409 — a stale or hand-edited record, which no + retry will fix and which is not the server's fault. + """ + # Arrange + _, mock_db, _, _ = prep_env + doc = _file_doc() + del doc["md5"] + mock_db.file.docs = [doc] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 409 + + @pytest.mark.asyncio + async def test_should_501_when_the_tile_backend_is_absent( + self, prep_env, mocker + ): + """Test the answer for a build without clodius. + + Given: + A contact map, but a build whose tile backend is absent. + When: + The status probe runs. + Then: + It should raise 501 naming the tile backend and dispatch + nothing — a readiness answer would promise a tile route this + build can never serve. + """ + # Arrange + _, mock_db, executor, _ = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object(backend, "is_available", return_value=False) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 501 + assert "clodius" in exc_info.value.detail + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_503_a_hic_when_the_workflow_subsystem_is_disabled( + self, prep_env + ): + """Test guard ordering when two refusals apply at once. + + Given: + An ENCODE .hic document AND a disabled workflow subsystem + (api.cache is None), so both the 503 and the 501 branch match. + When: + The status probe runs. + Then: + It should raise 503, not 501 — the subsystem check precedes + the .hic refusal, so an operator condition is reported ahead + of a per-file limitation. This ordering is a visible decision: + reordering the guards would silently change the answer. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [ + _file_doc(submission="encode", local_id="ENCFF123ABC", filename="c.hic") + ] + mocker.patch.object(api, "cache", None) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("encode", "ENCFF123ABC") + assert exc_info.value.status_code == 503 + assert exc_info.value.headers["Retry-After"] == "30" + + @pytest.mark.asyncio + async def test_should_500_with_a_generic_detail_when_the_probe_raises( + self, prep_env + ): + """Test the status probe's catch-all wrapper. + + Given: + A readiness probe that raises RuntimeError — a bug, not a + modelled condition. + When: + The status probe runs. + Then: + It should raise 500 with a generic detail rather than leaking + the exception text to an unauthenticated caller. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object( + tilesets, + "probe_workflow_readiness", + side_effect=RuntimeError("disk exploded at /var/secret/path"), + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await tileset_status("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 500 + assert exc_info.value.detail == "Internal server error" + + +class TestPrepare: + """``POST /tilesets/{dcc}/{local_id}``.""" + + @pytest.mark.asyncio + async def test_should_dispatch_and_point_at_the_job_endpoint(self, prep_env): + """Test the dispatch response. + + Given: + A contact map with no cached artifact. + When: + prepare_tileset is called. + Then: + It should return 202 with a Location header pointing at the + existing /jobs endpoint, so the UI polls the same way it does + for /data and /index preprocessing. + """ + # Arrange + _, mock_db, executor, _ = prep_env + mock_db.file.docs = [_file_doc()] + + # Act + response = await prepare_tileset("4dn", "4DNFIABC123") + + # Assert + assert response.status_code == 202 + assert response.headers["location"] == "/jobs/job-abc-123" + assert response.headers["retry-after"] == "5" + assert _body(response)["job_id"] == "job-abc-123" + assert executor.calls == 1 + + @pytest.mark.asyncio + async def test_should_be_idempotent_once_the_artifact_exists( + self, prep_env, tiny_mcool + ): + """Test that re-preparing a prepared file is not a job. + + Given: + A contact map whose artifact is already cached. + When: + prepare_tileset is called. + Then: + It should return 200 ready and dispatch nothing. A UI that + POSTs before checking should not create a no-op job. + """ + # Arrange + seed, mock_db, executor, _ = prep_env + doc = _file_doc() + mock_db.file.docs = [doc] + await seed(doc, tiny_mcool) + + # Act + response = await prepare_tileset("4dn", "4DNFIABC123") + + # Assert + assert response.status_code == 200 + assert _body(response) == {"ready": True} + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_refuse_a_hic_without_dispatching(self, prep_env): + """Test that .hic cannot be queued for materialization. + + Given: + An ENCODE .hic. + When: + prepare_tileset is called. + Then: + It should raise 501 and dispatch nothing. Accepting the job + would queue a download of a file with a median size of 10 GB + and a corpus maximum of 315 GB — past the Fargate ephemeral + storage ceiling — to build an artifact no endpoint would then + serve. + """ + # Arrange + _, mock_db, executor, _ = prep_env + mock_db.file.docs = [ + _file_doc(submission="encode", local_id="ENCFF123ABC", filename="c.hic") + ] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("encode", "ENCFF123ABC") + assert exc_info.value.status_code == 501 + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_429_when_the_admission_ceiling_is_reached( + self, prep_env + ): + """Test load shedding. + + Given: + An executor at the active-workflow ceiling. + When: + prepare_tileset is called. + Then: + It should raise 429 carrying the executor's own Retry-After, + so tile preparation sheds under the same ceiling as /data and + /index rather than queuing past it. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object( + api, + "executor", + _StubExecutor( + AdmissionRejected(active=1024, ceiling=1024, retry_after_seconds=17) + ), + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 429 + assert exc_info.value.headers["Retry-After"] == "17" + + @pytest.mark.asyncio + async def test_should_503_while_the_executor_is_draining(self, prep_env): + """Test that shutdown is caught ahead of inapplicability. + + Given: + An executor that is draining for lifespan teardown. + When: + prepare_tileset is called. + Then: + It should raise 503, not 409. ExecutorDraining subclasses + WorkflowNotApplicable, so this only holds if the catch order + puts the subclass first — which is exactly the kind of thing + a reorder would silently break. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object( + api, "executor", _StubExecutor(ExecutorDraining("shutting down")) + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 503 + + @pytest.mark.asyncio + async def test_should_409_when_the_executor_declines_the_file(self, prep_env): + """Test the race where the executor disagrees with the router. + + Given: + An executor that reports the workflow inapplicable. + When: + prepare_tileset is called. + Then: + It should raise 409 rather than 500 — a disagreement about + this file, not a server fault. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object( + api, "executor", _StubExecutor(WorkflowNotApplicable("no processor")) + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 409 + + @pytest.mark.asyncio + async def test_should_501_without_dispatching_when_the_tile_backend_is_absent( + self, prep_env, mocker + ): + """Test that a backendless build refuses to queue work. + + Given: + A contact map, but a build whose tile backend is absent. + When: + prepare_tileset is called. + Then: + It should raise 501 naming the tile backend and dispatch + nothing — building an artifact no route in this build could + ever read would be pure waste. + """ + # Arrange + _, mock_db, executor, _ = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object(backend, "is_available", return_value=False) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 501 + assert "clodius" in exc_info.value.detail + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_409_a_document_with_no_md5_without_dispatching( + self, prep_env + ): + """Test the POST side of the unaddressable-document conflict. + + Given: + A contact map whose document carries no md5, so no cache key + can be derived for its artifact. + When: + prepare_tileset is called. + Then: + It should raise 409 and dispatch nothing — a workflow whose + output could never be addressed must not be queued. + """ + # Arrange + _, mock_db, executor, _ = prep_env + doc = _file_doc() + del doc["md5"] + mock_db.file.docs = [doc] + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 409 + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_503_when_the_workflow_subsystem_is_disabled( + self, prep_env + ): + """Test the POST side of degraded mode. + + Given: + The tile backend present but no cache, as when SYNC_DATA_DIR + is unset. + When: + prepare_tileset is called. + Then: + It should raise 503 with Retry-After and leave the executor + untouched — nothing could store the artifact a dispatch would + build. + """ + # Arrange + _, mock_db, executor, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object(api, "cache", None) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 503 + assert exc_info.value.headers["Retry-After"] == "30" + assert executor.calls == 0 + + @pytest.mark.asyncio + async def test_should_500_with_a_generic_detail_when_the_probe_raises( + self, prep_env + ): + """Test prepare_tileset's catch-all wrapper. + + Given: + A readiness probe that raises RuntimeError — a bug, not a + modelled condition. + When: + prepare_tileset is called. + Then: + It should raise 500 with a generic detail rather than leaking + the exception text to an unauthenticated caller. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object( + tilesets, + "probe_workflow_readiness", + side_effect=RuntimeError("disk exploded at /var/secret/path"), + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 500 + assert exc_info.value.detail == "Internal server error" + + @pytest.mark.asyncio + async def test_should_500_with_a_generic_detail_when_the_executor_raises( + self, prep_env + ): + """Test that a non-workflow executor error is not misclassified. + + Given: + An executor raising RuntimeError — outside the dispatch + ladder's modelled exceptions (draining, admission, not + applicable). + When: + prepare_tileset is called. + Then: + It should raise 500 with a generic detail: the ladder must + let a genuine bug escape to the catch-all rather than dress + it up as a 409 or 503 a client would retry forever. + """ + # Arrange + _, mock_db, _, mocker = prep_env + mock_db.file.docs = [_file_doc()] + mocker.patch.object( + api, "executor", _StubExecutor(RuntimeError("wool imploded")) + ) + + # Act & Assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", "4DNFIABC123") + assert exc_info.value.status_code == 500 + assert exc_info.value.detail == "Internal server error" + + +class TestOverHttp: + """The wire layer, which calling the handlers directly cannot exercise. + + The Path(...) length and pattern constraints are enforced by FastAPI's + validation layer, so they are only really tested through ASGI. Follows + ``tests/test_tiles.py::TestOverHttp`` in neutering the lifespan. + """ + + @pytest.fixture() + def client(self, mocker): + from mongomock_motor import AsyncMongoMockClient + from starlette.testclient import TestClient + + from cfdb.api import main + + mocker.patch.object( + main, "create_mongodb_client", return_value=AsyncMongoMockClient() + ) + mocker.patch.object(main.WorkflowProfile, "from_env", return_value=None) + with TestClient(main.app) as client: + yield client + + @pytest.mark.parametrize( + ("method", "url"), + [ + ("GET", "/tilesets/4dn/4DNFIABC123/status"), + ("POST", "/tilesets/4dn/4DNFIABC123"), + ], + ) + def test_should_mount_both_routes_on_the_app(self, client, method, url): + """Test that the router is wired into the application. + + Given: + Well-formed probe and prepare URLs for a file the (empty) + database does not hold. + When: + They are requested over HTTP. + Then: + They should reach the handlers — proven by the handlers' own + 404 "File not found" rather than a routing 404 or a 405. + """ + # Act + response = client.request(method, url) + + # Assert + assert response.status_code == 404 + assert response.json()["detail"] == "File not found" + + @pytest.mark.parametrize( + ("method", "url_template"), + [ + ("GET", "/tilesets/4dn/{local_id}/status"), + ("POST", "/tilesets/4dn/{local_id}"), + ], + ) + @pytest.mark.parametrize( + "local_id", + [ + pytest.param("a" * 257, id="overlong-257-chars"), + pytest.param("bad!id", id="illegal-character"), + ], + ) + def test_should_422_a_local_id_violating_the_path_constraints( + self, client, method, url_template, local_id + ): + """Test the Path(...) constraints at the wire. + + Given: + A local_id one character past PATH_PARAM_MAX_LEN, and one + carrying a character outside PATH_PARAM_PATTERN. + When: + The probe and prepare routes are requested over HTTP. + Then: + It should be rejected by FastAPI's validation layer with 422 + before any handler code runs. (The README's table says 400; + main.py registers no RequestValidationError handler, so the + actual behavior — pinned here — is FastAPI's default 422.) + """ + # Act + response = client.request(method, url_template.format(local_id=local_id)) + + # Assert + assert response.status_code == 422 From 0271711c880791cea014126722f5693a4d0e7d4a Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:28:29 -0400 Subject: [PATCH 09/14] build: Layer the clodius tile backend into the local dev images make api builds from Dockerfile.api, which carries no clodius, so its tile routes answer 501 -- correct while the tiles extra stays optional, but it leaves the whole tile chain untestable through the documented Docker flow. Dockerfile.api-tiles and the new api-tiles/worker-tiles Makefile targets layer the tile backend on and add the one piece make api does not need: a cache volume shared with a worker, since a tileset artifact is built by the worker and then opened locally by the API with h5py. Both are interim scaffolding, retired once tiles becomes an ordinary dependency. --- Dockerfile.api-tiles | 36 ++++++++++++++++++++++++++++ Makefile | 56 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 Dockerfile.api-tiles diff --git a/Dockerfile.api-tiles b/Dockerfile.api-tiles new file mode 100644 index 0000000..bb24df6 --- /dev/null +++ b/Dockerfile.api-tiles @@ -0,0 +1,36 @@ +# Local-development image: the API image plus the clodius tile backend. +# +# Interim scaffolding for issue #82. The clodius `tiles` extra is still +# optional (its fork requires Python 3.12 while cfdb supports 3.11), so +# `Dockerfile.api` does not carry it and the images it builds answer 501 +# on every tile route (see the `tiles` extra in pyproject.toml). That is +# the right production behaviour today, but it makes the tile endpoints +# untestable through the documented `make mongodb` / `make api` flow, +# which is the only place the whole chain -- workflow dispatch, artifact +# cache, h5py read -- runs together. +# +# So this layers clodius onto the image `make api` already built rather +# than duplicating that build, installing it straight from the public +# fork at the same sha `[tool.uv.sources]` pins. +# +# DELETE THIS FILE once the `tiles` extra becomes an ordinary dependency +# (which raises the Python floor to 3.12): `Dockerfile.api`'s existing +# `pip install .` then carries it. +FROM api + +USER root + +# Installed as its own layer, before the app, so that iterating on cfdb +# does not re-resolve clodius's 25-package closure (pysam, oxbow, apsw, +# polars, scipy, ...) on every rebuild. Keep this sha in lockstep with +# the `[tool.uv.sources]` pin in pyproject.toml. +RUN pip install --no-cache-dir \ + "clodius @ git+https://github.com/abdenlab/clodius@aac90219031e8c05cf6d5a916628531a72cfc461" + +# clodius imports matplotlib transitively, and the image's `app` user +# has no home directory, so matplotlib warns twice and rebuilds its font +# cache on every start. Point it at a writable path. +ENV MPLCONFIGDIR=/tmp/matplotlib + +USER app +WORKDIR /app/src diff --git a/Makefile b/Makefile index a163976..dbf0211 100644 --- a/Makefile +++ b/Makefile @@ -52,6 +52,62 @@ api: docker run -d --name api --network cvh-backend-network --network-alias cvh-backend -p 8000:8000 -e SYNC_DATA_DIR=/tmp/sync-data api @echo "API container is up and running on port 8000 (http://0.0.0.0:8000/metadata)." +# --- Local-dev matrix tile serving (issue #82) ----------------------- +# +# `make api` builds an image with no clodius, so its tile routes answer +# 501 -- correct for production while the `tiles` extra stays optional, +# but it leaves the whole tile chain untestable through the documented +# Docker flow. These targets layer clodius on (installed from the public +# fork at the sha pyproject.toml pins) and add the piece `make api` does +# not need: a cache volume shared with a worker, since a tileset +# artifact is built by the worker and then opened by the API. +# +# Retire both once `tiles` becomes an ordinary dependency (raising the +# Python floor to 3.12) -- `make api` then covers this on its own. + +api-tiles: + make network + @docker stop api 2>/dev/null || true + @docker rm api 2>/dev/null || true + @echo "Building the API image..." + docker build -t api -f Dockerfile.api . + @echo "Layering the clodius tile backend on top..." + docker build -t api-tiles -f Dockerfile.api-tiles . + @# The image runs as `app`, but a freshly created named volume is + @# owned by root, so the lifespan's mkdir under SYNC_DATA_DIR fails + @# with PermissionError and the API 500s on every workflow route. + @docker volume create cfdb-sync >/dev/null + @docker run --rm --user 0 --entrypoint chown -v cfdb-sync:/tmp/sync-data api-tiles -R app:app /tmp/sync-data + @echo "Starting the API container with matrix tile serving enabled..." + docker run -d --name api --network cvh-backend-network --network-alias cvh-backend -p 8000:8000 \ + -e SYNC_DATA_DIR=/tmp/sync-data \ + -e WORKFLOW_POOL_NAMESPACE=$${WORKFLOW_POOL_NAMESPACE:-cfdb-workers} \ + -v cfdb-sync:/tmp/sync-data \ + api-tiles + @echo "API up on port 8000." + @# Start the worker FIRST next time: the API dispatches on request, and a + @# dispatch that finds no worker is queued rather than failed, so the job + @# then waits out CFDB_WORKFLOW_RETRY_INTERVAL_S (2 min) before running. + @echo "Run 'make worker-tiles' before POSTing a tileset, or the first job waits ~2 min for the retry tick." + +# The worker that builds tileset artifacts. It reuses the API image +# rather than `cfdb-wool`, because the matrix processor needs cooler and +# h5py, which arrive with clodius and are absent from the worker image. +# It shares the API's cache volume: the worker writes the artifact, the +# API opens it locally with h5py. +worker-tiles: + make network + @docker stop worker 2>/dev/null || true + @docker rm worker 2>/dev/null || true + @echo "Starting a containerized LAN worker pool (namespace=$${WORKFLOW_POOL_NAMESPACE:-cfdb-workers})..." + docker run -d --name worker --network cvh-backend-network \ + -e SYNC_DATA_DIR=/tmp/sync-data \ + -e WORKFLOW_POOL_NAMESPACE=$${WORKFLOW_POOL_NAMESPACE:-cfdb-workers} \ + -e WORKFLOW_WORKER_COUNT=$${WORKFLOW_WORKER_COUNT:-1} \ + -v cfdb-sync:/tmp/sync-data \ + api-tiles python -m cfdb.workflows.worker_lan + @echo "Worker container is up. Check logs with: docker logs -f worker" + schema: @echo "Regenerating schema.graphql from the Strawberry schema..." uv run python scripts/export_schema.py From f3e1dd42fac0cd10c6934c06a4e4547632c26430 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:28:35 -0400 Subject: [PATCH 10/14] feat: Size the API task's disk for tileset copies h5py opens a file rather than consuming a byte stream, so under the S3 profile an mcool must be pulled onto local disk before any tile can be read from it. Adds ApiEphemeralStorageGiB and TilesetDiskCacheBytes parameters to backend.yml so the task's ephemeral storage is sized to hold the tileset disk cache alongside SYNC_DATA_DIR and the container image, instead of leaving the two budgets to silently compete for the same disk. --- cloudformation/backend.yml | 34 +++++++++++++ tests/test_cloudformation.py | 95 ++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/cloudformation/backend.yml b/cloudformation/backend.yml index 3515118..dcd9027 100644 --- a/cloudformation/backend.yml +++ b/cloudformation/backend.yml @@ -35,6 +35,31 @@ Parameters: Type: Number Default: 1 Description: Number of ECS tasks + ApiEphemeralStorageGiB: + Type: Number + Default: 50 + MinValue: 21 + MaxValue: 200 + Description: > + Fargate ephemeral storage (GiB) for the API task. Sized above the + 20 GiB Fargate default because matrix tile serving (issue #82) keeps + local copies of contact-map artifacts on this disk: h5py opens a + file, so an mcool cached in S3 has to be pulled down before any tile + can be read from it. That disk is shared with SYNC_DATA_DIR and the + container image, and the byte budget for the tile copies is set + independently by CFDB_TILESET_DISK_CACHE_BYTES — keep that value + comfortably below this one, or filling the tile cache takes /data + and /sync down with it. + TilesetDiskCacheBytes: + Type: Number + Default: 8589934592 + MinValue: 1 + Description: > + Byte budget for the API task's local copies of tileset artifacts + (CFDB_TILESET_DISK_CACHE_BYTES). Default 8 GiB against the 50 GiB + ephemeral storage above. Least-recently-used copies are evicted to + stay within it; a single artifact larger than the whole budget is + refused with 503 rather than thrashing. EcsMaxWorkers: Type: Number Default: 16 @@ -258,6 +283,8 @@ Resources: NetworkMode: awsvpc RequiresCompatibilities: - FARGATE + EphemeralStorage: + SizeInGiB: !Ref ApiEphemeralStorageGiB ExecutionRoleArn: !GetAtt ExecutionRole.Arn TaskRoleArn: !GetAtt TaskRole.Arn ContainerDefinitions: @@ -305,6 +332,13 @@ Resources: # rather than spawning past this. Bounds workers, not the queue. - Name: ECS_MAX_WORKERS Value: !Ref EcsMaxWorkers + # Byte budget for the API task's local copies of tileset + # artifacts (issue #82). Set explicitly rather than left to the + # application default so it moves together with + # ApiEphemeralStorageGiB — the two are a pair, and a disk + # resized without resizing this gains nothing. + - Name: CFDB_TILESET_DISK_CACHE_BYTES + Value: !Ref TilesetDiskCacheBytes # Only meaningful alongside the cert Secrets below, so it is # rendered away when mTLS is off rather than advertising a # setting that does nothing. The workers stack has no diff --git a/tests/test_cloudformation.py b/tests/test_cloudformation.py index 5fd8e20..d8be8fa 100644 --- a/tests/test_cloudformation.py +++ b/tests/test_cloudformation.py @@ -633,3 +633,98 @@ def test_backend_worker_tls_identity_should_only_render_when_mtls_is_enabled(): ] assert len(identity_entries) == 1 assert identity_entries[0]["Fn::If"][1]["Value"] == {"Ref": "WorkerTlsIdentity"} + + +def test_api_task_should_declare_ephemeral_storage_for_the_tile_cache(): + """Test that the API task gets a disk sized for local tileset copies. + + Given: + The backend template's API task definition, whose task now keeps + local copies of contact-map artifacts (issue #82) — h5py opens a + file, so an mcool cached in S3 has to be pulled onto disk before + any tile can be read. + When: + The task definition is inspected. + Then: + It should declare EphemeralStorage from a parameter sized above + the 20 GiB Fargate default and constrained to Fargate's valid + 21-200 GiB range, so an out-of-range override fails at parameter + validation rather than at RunTask. Without the storage, the tile + cache shares the default disk with SYNC_DATA_DIR and the + container image, and filling it takes /data and /sync down + alongside tile serving. + """ + # Arrange + template = _load_template("backend.yml") + task_definition = template["Resources"]["TaskDefinition"]["Properties"] + + # Act + storage = task_definition.get("EphemeralStorage") + + # Assert + assert storage == {"SizeInGiB": {"Ref": "ApiEphemeralStorageGiB"}} + parameter = template["Parameters"]["ApiEphemeralStorageGiB"] + assert parameter["Default"] > 20 + assert parameter["MinValue"] == 21 + assert parameter["MaxValue"] == 200 + + +def test_tileset_disk_budget_should_fit_inside_the_api_ephemeral_storage(): + """Test that the two disk parameters are consistent by default. + + Given: + The byte budget for local tileset copies and the ephemeral + storage backing it. + When: + Both defaults are compared. + Then: + The budget should sit well inside the disk, which it shares with + SYNC_DATA_DIR and the container image. Resizing one without the + other is the failure this pins: a larger disk with an unchanged + budget gains nothing, and a larger budget on an unchanged disk + fills it. + """ + # Arrange + template = _load_template("backend.yml") + parameters = template["Parameters"] + + # Act + budget_bytes = parameters["TilesetDiskCacheBytes"]["Default"] + disk_bytes = parameters["ApiEphemeralStorageGiB"]["Default"] * 1024**3 + + # Assert + assert budget_bytes < disk_bytes / 2, ( + "the tileset disk budget must leave room for SYNC_DATA_DIR and the " + "container image on the same volume" + ) + + +def test_api_container_should_wire_the_tileset_disk_budget(): + """Test that the disk budget reaches the application. + + Given: + The backend template's API container definition. + When: + Its Environment entries are searched. + Then: + CFDB_TILESET_DISK_CACHE_BYTES should be set from the parameter, so + the budget and the disk it lives on are resized together rather + than the budget silently keeping its application default. + """ + # Arrange + template = _load_template("backend.yml") + container = template["Resources"]["TaskDefinition"]["Properties"][ + "ContainerDefinitions" + ][0] + + # Act + entries = [ + entry + for entry in container["Environment"] + if isinstance(entry, dict) + and entry.get("Name") == "CFDB_TILESET_DISK_CACHE_BYTES" + ] + + # Assert + assert len(entries) == 1 + assert entries[0]["Value"] == {"Ref": "TilesetDiskCacheBytes"} From c15749da18aa5453417e8b10de52c75acb4a280b Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:28:54 -0400 Subject: [PATCH 11/14] build: Pin clodius to a public commit on main The fork was private during initial development, so clodius resolved from a sibling checkout on disk. Now that it is public, pin [tool.uv.sources] to a commit sha on the fork's main instead -- still a sha rather than a branch reference, since clodius considers tiles_v2 unstable and cfdb is its first consumer, so what runs must be exactly what was validated. main does not carry clodius.tiles_v2.hic -- load_hic_tileset degrades to TileBackendUnavailable for it, same as a build with no clodius at all. That is a no-op for cfdb today: .hic is refused at the call site regardless, because the module reads local files only, which does not fit this corpus at any disk budget. Serving .hic at all is tracked as separate follow-up work, not something this pin is expected to grow. --- pyproject.toml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 179cd0e..2a2945c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,32 @@ dev = ["allpairspy", "clodius; python_version >= '3.12'", "cloudpickle", "crypto # as follow-up work, gated on the fork being published. tiles = ["clodius; python_version >= '3.12'"] +[tool.uv.sources] +# Pinned to a sha, not a branch: clodius considers ``tiles_v2`` unstable +# and cfdb is its first consumer, so what runs must be exactly what was +# validated. This sha is the tip of the fork's ``main`` — the +# 6-restore-dropped-guards branch (restored ``tiles_v2`` validation +# guards, see ``cfdb.tilesets.backend.ClodiusBackend.tile_error``, now +# bound to ``TilesetError``, and the tile-omission contract change +# documented on ``cfdb.tilesets.service.TilesetService._read_tiles``) has +# been merged there. Even though ``main`` now carries ``tiles_v2``, stay +# pinned to a commit sha rather than a branch reference — a branch +# reference floats, and what runs must be exactly what was validated. +# +# ``main`` does not carry ``clodius.tiles_v2.hic`` — ``load_hic_tileset`` +# degrades to ``TileBackendUnavailable`` for it, same as a build with no +# clodius at all. That's a no-op for cfdb today: ``.hic`` is refused at +# the call site regardless (see ``TilesetService._open_hic``) because the +# module reads local files only, which does not fit this corpus at any +# disk budget. Serving ``.hic`` at all is tracked as separate follow-up +# work, not something this pin is expected to grow. +# +# The dependency stays an optional extra rather than becoming +# unconditional because the fork requires Python >= 3.12 while cfdb +# still supports 3.11; promoting it (and raising the floor) is tracked +# as follow-up. +clodius = { git = "https://github.com/abdenlab/clodius", rev = "07f043bea3380bbc38e76f0970d93abed0be4f69" } + [project.scripts] cfdb = "cfdb.cli:cli" cfdb-ensure-indexes = "cfdb.ensure_indexes:main" From 82a6eab219b517f1e7e4041bd495c294b4a0ad42 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:29:03 -0400 Subject: [PATCH 12/14] test: Exercise the tile pipeline end to end Covers the parts unit tests cannot: real wool worker dispatch building a tileset artifact and serving tiles from it over HTTP under concurrency and eviction pressure, the S3-profile tile chain under moto, and the lifespan wiring across every profile (local, S3, mTLS) booting real cache and discovery implementations rather than fakes. Extends the shared integration scenario model and its sample-file fixtures to cover the preparation channel alongside the existing data/index paths. --- tests/integration/conftest.py | 62 +- tests/integration/fixtures/make_samples.py | 29 + tests/integration/test_lifespan_profiles.py | 303 +++++++++ tests/integration/test_scenario.py | 112 ++++ tests/integration/test_tiles_e2e.py | 527 ++++++++++++++++ tests/integration/test_tiles_s3_profile.py | 308 +++++++++ tests/integration/test_tilesets_prep_e2e.py | 664 ++++++++++++++++++++ 7 files changed, 2001 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_lifespan_profiles.py create mode 100644 tests/integration/test_tiles_e2e.py create mode 100644 tests/integration/test_tiles_s3_profile.py create mode 100644 tests/integration/test_tilesets_prep_e2e.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index e3f58c5..126beb3 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -40,6 +40,7 @@ import datetime import http.server import os +import importlib.util import shutil import socketserver import threading @@ -69,6 +70,7 @@ from cfdb.workflows.lock import get_job from cfdb.workflows.models import ACTIVE_STATUSES from cfdb.workflows.processors.bam import BamIndexProcessor +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor from cfdb.workflows.processors.registry import ProcessorRegistry from cfdb.workflows.processors.tabix import TabixIntervalProcessor @@ -98,6 +100,17 @@ def tool_available(name: str) -> bool: return shutil.which(name) is not None +def cooler_available() -> bool: + """Return True when the ``cooler`` package is importable. + + The tiles extra is optional (the clodius fork is private), so the + contact-map rows are gated the way the gffread/bedToBigBed rows are + gated on PATH tools — probed with ``find_spec`` so collection never + pays the import. + """ + return importlib.util.find_spec("cooler") is not None + + def _require_tools() -> None: """Skip the integration session when core tools are absent.""" missing = [t for t in REQUIRED_TOOLS if shutil.which(t) is None] @@ -127,6 +140,8 @@ class Format(Enum): BIGBED = "bigBed" CSV = "CSV" BIGWIG = "bigWig" + MCOOL = "mcool" + COOL = "cool" class Endpoint(Enum): @@ -135,6 +150,7 @@ class Endpoint(Enum): DATA = "data" INDEX = "index" JOBS = "jobs" + TILESETS = "tilesets" class Method(Enum): @@ -287,7 +303,26 @@ def filter_func(row: list[Any]) -> bool: ): return False - # 5. mongomock-motor + N10 only runs when mongomock-motor is + # 5. Contact maps ride the preparation channel only: /data streams + # them from upstream and /index reports that they have no index + # (README, "Two channels, deliberately separate"), so TILESETS pairs + # exclusively with the cooler formats and the cooler formats with + # TILESETS. The prepare endpoint is POST-driven inside the tests and + # the probe has no HEAD contract, so a Method axis is meaningless on + # those rows. The tiles toolchain is an optional extra, so cooler + # rows are dropped when it is absent, mirroring the tool gates above. + if endpoint is Endpoint.TILESETS: + if fmt is not None and fmt not in (Format.MCOOL, Format.COOL): + return False + if method is Method.HEAD: + return False + if fmt in (Format.MCOOL, Format.COOL): + if endpoint is not None and endpoint is not Endpoint.TILESETS: + return False + if not cooler_available(): + return False + + # 6. mongomock-motor + N10 only runs when mongomock-motor is # importable; falls out as ``True`` when it is, ``False`` when it # is not, so the row is skipped on hosts without it. if mutex_backend is MutexBackend.MONGOMOCK and concurrency is Concurrency.N10: @@ -569,8 +604,18 @@ def integration_workdir_root(tmp_path) -> Path: @pytest.fixture() def install_jobs_index(mock_db): - """Seed the partial-unique mutex index on the FakeDB jobs collection.""" - mock_db.jobs.create_index( + """Seed the partial-unique mutex index on the FakeDB jobs collection. + + Uses ``register_index`` — the synchronous seam — because + ``create_index`` is a coroutine (Motor's is awaited in production) + and calling it from a sync fixture registers nothing: the coroutine + is dropped un-awaited and the mutex index silently never exists, so + concurrent claims insert duplicate active JobRecords instead of + funneling. That is exactly what happened here, masked for the + test_concurrency funnel tests by a KNOWN_BUGS retry predicate broad + enough to convert the assertion failure into an xfail. + """ + mock_db.jobs.register_index( {"workflow_key": 1}, unique=True, partialFilterExpression={"active": True}, @@ -594,6 +639,11 @@ async def integration_executor( registry = ProcessorRegistry() registry.register(BamIndexProcessor()) registry.register(TabixIntervalProcessor()) + # Registered unconditionally, mirroring the lifespan: the processor + # imports cooler/h5py lazily inside run(), so registration is safe on + # a checkout without the tiles extra — only dispatching to it is not, + # and the cooler rows are filtered out of the sweeps in that case. + registry.register(MatrixTilesetProcessor()) cache = LocalFsCache(integration_cache_root) executor = WoolExecutor( @@ -615,6 +665,8 @@ def make_file_meta( dcc: str = "ENCODE", local_id: str | None = None, extra_files: list[dict] | None = None, + filename: str | None = None, + file_format_name: str | None = None, ) -> dict[str, Any]: """Build a Mongo-style file_meta dict whose access_url resolves. @@ -632,8 +684,10 @@ def make_file_meta( "local_id": local_id or f"ENCFF-{sample.format}", "md5": sample.md5, "access_url": f"{base_url}/{sample.path.name}", - "file_format": {"name": sample.format}, + "file_format": {"name": file_format_name or sample.format}, } + if filename is not None: + meta["filename"] = filename if extra_files is not None: meta["extra"] = {"extra_files": list(extra_files)} return meta diff --git a/tests/integration/fixtures/make_samples.py b/tests/integration/fixtures/make_samples.py index ce0623a..7d92935 100644 --- a/tests/integration/fixtures/make_samples.py +++ b/tests/integration/fixtures/make_samples.py @@ -15,6 +15,7 @@ import gzip import hashlib +import importlib.util import random import shutil import subprocess @@ -302,6 +303,32 @@ def make_bigbed(root: Path) -> SampleFile | None: return SampleFile(path=bb_path, md5=_md5(bb_path), format="bigBed") +def make_mcool(root: Path) -> SampleFile | None: + """Emit a multi-resolution mcool, or None when cooler is unimportable. + + Delegates to the shared synthetic builders so the integration chain + and the unit suites tile the same bytes. The tiles extra is optional, + so absence of ``cooler`` degrades to None the way a missing + ``bedToBigBed`` does for bigBed. + """ + if importlib.util.find_spec("cooler") is None: + return None + from tests.fixtures.coolers import build_mcool + + path = build_mcool(root / "sample.mcool") + return SampleFile(path=path, md5=_md5(path), format="mcool") + + +def make_cool(root: Path) -> SampleFile | None: + """Emit a flat single-resolution cooler, or None without cooler.""" + if importlib.util.find_spec("cooler") is None: + return None + from tests.fixtures.coolers import build_cool + + path = build_cool(root / "sample.cool") + return SampleFile(path=path, md5=_md5(path), format="cool") + + def generate_all(root: Path) -> dict[str, SampleFile | None]: """Build every sample fixture under ``root``. @@ -323,5 +350,7 @@ def generate_all(root: Path) -> dict[str, SampleFile | None]: "NarrowPeak": make_narrowpeak_gz(root), "BroadPeak": make_broadpeak_gz(root), "bigBed": make_bigbed(root), + "mcool": make_mcool(root), + "cool": make_cool(root), } return samples diff --git a/tests/integration/test_lifespan_profiles.py b/tests/integration/test_lifespan_profiles.py new file mode 100644 index 0000000..39a4b12 --- /dev/null +++ b/tests/integration/test_lifespan_profiles.py @@ -0,0 +1,303 @@ +"""Lifespan profile matrix: real ``_build_cache`` composition per cell. + +Every other tile test hands the lifespan a pre-stubbed cache. These tests +run the real profile resolution instead — ``WorkflowProfile.from_env`` +over patched env globals, the real ``_build_cache`` (moto-backed for the +S3 cell), and the real ``_build_tileset_service`` — so what is pinned is +the *composition* the deployment actually gets: which cache class each +profile wires, where the tileset store lands relative to ``SYNC_DATA_DIR``, +and that a request served after boot reads the cache the lifespan +assigned rather than a stale capture. + +Only the boundaries the guide permits are stubbed: MongoDB (mongomock), +the wool discovery/pool/executor stack (no worker fleet exists here), and +— for the ABSENT rows — ``cfdb.tilesets.backend.is_available``, the one +axis that cannot vary for real within a single installed environment. +""" + +from __future__ import annotations + +import contextlib +from contextlib import asynccontextmanager +from enum import Enum +from pathlib import Path + +import pytest + +pytest.importorskip("cooler") +pytest.importorskip("clodius.tiles_v2") + +import boto3 +import httpx +from mongomock_motor import AsyncMongoMockClient +from moto import mock_aws + +from cfdb import api +from cfdb.api import main +from cfdb.tilesets import backend as tiles_backend +from cfdb.workflows.cache import LocalFsCache, S3Cache +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from tests.fixtures.coolers import build_mcool +from tests.test_workflows import FIXTURE_MD5 + +pytestmark = pytest.mark.integration + + +class Profile(Enum): + """Workflow profile axis of the lifespan matrix.""" + + DISABLED = "disabled" + LOCAL = "local" + S3_CACHED = "s3-cached" + + +class TileBackend(Enum): + """Clodius availability axis of the lifespan matrix.""" + + PRESENT = "present" + ABSENT = "absent" + + +#: The full factorial — six cells, kept as a flat parametrize per the +#: plan rather than a new Scenario dimension (the S3 axis cannot cross +#: the wool boundary, so it could never be swept anyway). +CELLS = [ + pytest.param(profile, backend, id=f"{profile.value}-{backend.value}") + for profile in Profile + for backend in TileBackend +] + +_BUCKET = "cfdb-lifespan-profiles" +_LOCAL_ID = "4DNFILP012AB" +UID = f"4dn/{_LOCAL_ID}" + + +def _file_doc() -> dict: + """Return a projected 4DN contact-map document for the served cells.""" + return { + "submission": "4dn", + "local_id": _LOCAL_ID, + "md5": FIXTURE_MD5, + "filename": "sample.mcool", + "file_format": {"name": "HDF5"}, + "genome_assembly": "GRCh38", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + "data_access_level": "public", + } + + +def _tileset_key() -> str: + """The cache key the serving chain derives for ``_file_doc()``.""" + return MatrixTilesetProcessor().cache_key_for(_file_doc(), ArtifactKind.TILESET) + + +@asynccontextmanager +async def _booted( + profile: Profile, + backend_state: TileBackend, + mocker, + tmp_path: Path, + *, + seed: Path | None = None, +): + """Enter the real lifespan for one (profile, backend) cell. + + Patches the env-derived ``cfdb.api`` globals so the lifespan's own + ``WorkflowProfile.from_env`` call resolves the requested profile, and + lets ``_build_cache`` run for real — against moto's in-process S3 for + the s3-cached cells. ``seed`` (a local mcool path) is committed to the + cell's real cache backend *before* boot under the tileset key for + ``_file_doc()``. + + Yields the ``SYNC_DATA_DIR`` root (``None`` for the disabled cell). + """ + if backend_state is TileBackend.ABSENT: + mocker.patch.object(tiles_backend, "is_available", return_value=False) + + sync_root: Path | None = None + mocker.patch.object(api, "ECS_CLUSTER", None) + mocker.patch.object(api, "AWS_ENDPOINT_URL", None) + mocker.patch.object(api, "AWS_REGION", "us-east-1") + mocker.patch.object(api, "WORKFLOW_S3_PREFIX", "") + if profile is Profile.DISABLED: + mocker.patch.object(api, "SYNC_DATA_DIR", None) + mocker.patch.object(api, "WORKFLOW_S3_BUCKET", None) + else: + sync_root = tmp_path / "sync" + mocker.patch.object(api, "SYNC_DATA_DIR", str(sync_root)) + mocker.patch.object( + api, + "WORKFLOW_S3_BUCKET", + _BUCKET if profile is Profile.S3_CACHED else None, + ) + + @contextlib.asynccontextmanager + async def _fake_build_discovery(_profile): + yield object() + + class _PoolStub: + def __init__(self, **_kwargs): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *_exc): + return None + + fake_executor = mocker.MagicMock() + fake_executor.drain = mocker.AsyncMock(return_value=0) + + mocker.patch.object( + main, "create_mongodb_client", return_value=AsyncMongoMockClient() + ) + mocker.patch.object(main, "_build_discovery", _fake_build_discovery) + mocker.patch.object(main, "WoolExecutor", return_value=fake_executor) + mocker.patch.object(main.wool, "WorkerPool", _PoolStub) + + with contextlib.ExitStack() as stack: + if profile is Profile.S3_CACHED: + stack.enter_context(mock_aws()) + boto3.client("s3", region_name="us-east-1").create_bucket( + Bucket=_BUCKET + ) + if seed is not None and profile is not Profile.DISABLED: + assert sync_root is not None + if profile is Profile.S3_CACHED: + seed_cache: LocalFsCache | S3Cache = S3Cache( + bucket=_BUCKET, + client=boto3.client("s3", region_name="us-east-1"), + ) + else: + seed_cache = LocalFsCache(sync_root / "cache") + await seed_cache.put(_tileset_key(), seed) + async with main.lifespan(main.app): + yield sync_root + + +class TestLifespanProfileMatrix: + """The six (profile x backend) cells of the lifespan composition.""" + + @pytest.mark.asyncio + @pytest.mark.parametrize("profile, backend_state", CELLS) + async def test_lifespan_should_compose_the_cell_profile( + self, profile, backend_state, mocker, tmp_path + ): + """Test the cache/service composition of one lifespan cell. + + Given: + Env globals resolving one workflow profile (disabled, local, + or s3-cached — the S3 cell against a real moto bucket) and a + tile backend that is present or patched absent. + When: + The real app lifespan is entered and exited, with only the + Mongo and wool boundaries stubbed. + Then: + It should wire the profile's real cache class (LocalFsCache + for local, S3Cache for s3-cached, none when disabled), build + the tile service only when a profile and the backend are both + present — rooting its store at SYNC_DATA_DIR/tilesets, the + sibling of cache/ — and on exit leave the service closed and + the global nulled. + """ + # Arrange + expect_cache_type = { + Profile.LOCAL: LocalFsCache, + Profile.S3_CACHED: S3Cache, + }.get(profile) + expect_service = ( + profile is not Profile.DISABLED + and backend_state is TileBackend.PRESENT + ) + build_spy = mocker.spy(main, "build_tileset_service") + closed: list[bool] = [] + + # Act + async with _booted(profile, backend_state, mocker, tmp_path) as sync_root: + wired_service = api.tileset_service + wired_cache = api.cache + if wired_service is not None: + original_aclose = wired_service.aclose + + async def _tracking_close(): + closed.append(True) + await original_aclose() + + wired_service.aclose = _tracking_close + + # Assert + if expect_cache_type is None: + assert wired_cache is None + else: + assert type(wired_cache) is expect_cache_type + if expect_service: + assert wired_service is not None + assert build_spy.call_args.kwargs["root"] == sync_root / "tilesets" + assert closed == [True] + else: + assert wired_service is None + build_spy.assert_not_called() + assert api.tileset_service is None + + @pytest.mark.asyncio + @pytest.mark.parametrize("profile, backend_state", CELLS) + async def test_tileset_info_should_serve_through_the_booted_profile( + self, profile, backend_state, mocker, tmp_path + ): + """Test serving /tileset_info through a really-booted lifespan. + + Given: + One lifespan cell booted over the real cache composition, + with a real mcool artifact pre-committed to the enabled + cells' backend (local disk or moto S3) and its document + seeded in the lifespan's database. + When: + GET /tileset_info/?d= is issued through the real router + over ASGI, after boot. + Then: + The two fully-present cells should answer 200 with the + file's real resolutions ladder — proving ``cache_provider`` + resolves the cache the lifespan actually assigned, the one + place the stale-capture hazard is observable — while the + backend-absent rows answer 501 and the disabled-with-backend + row answers 503 with Retry-After. + """ + # Arrange + if profile is Profile.DISABLED: + # The backend check outranks the missing-subsystem check in + # ``_require_service``, so disabled-without-clodius is a 501. + expected_status = ( + 501 if backend_state is TileBackend.ABSENT else 503 + ) + elif backend_state is TileBackend.ABSENT: + expected_status = 501 + else: + expected_status = 200 + seed = ( + build_mcool(tmp_path / "upstream.mcool") + if profile is not Profile.DISABLED + else None + ) + + # Act + async with _booted( + profile, backend_state, mocker, tmp_path, seed=seed + ): + await api.db.files.insert_one(dict(_file_doc())) + transport = httpx.ASGITransport(app=main.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://testserver" + ) as client: + response = await client.get( + "/tileset_info/", params=[("d", UID)] + ) + + # Assert + assert response.status_code == expected_status + if expected_status == 200: + info = response.json()[UID] + assert info["resolutions"] == [1, 2, 4] + assert info["uuid"] == UID + elif expected_status == 503: + assert response.headers["retry-after"] == "30" diff --git a/tests/integration/test_scenario.py b/tests/integration/test_scenario.py index d4143be..cd42ae1 100644 --- a/tests/integration/test_scenario.py +++ b/tests/integration/test_scenario.py @@ -21,6 +21,8 @@ MutexBackend, PickleBoundary, Scenario, + cooler_available, + filter_func, ) @@ -163,3 +165,113 @@ def test_str_should_return_empty_marker_when_no_fields_are_set(self): # Assert assert rendered == "EMPTY" + + +@pytest.mark.parametrize( + "fmt", + [f for f in Format if f not in (Format.MCOOL, Format.COOL)], + ids=lambda f: f.name, +) +def test_filter_func_should_reject_tilesets_rows_when_format_is_not_a_cooler(fmt): + """Test that filter_func drops TILESETS rows for non-contact-map formats. + + Given: + A pairwise row pairing the TILESETS endpoint with a format + other than MCOOL or COOL (the preparation channel serves only + contact maps — README, "Two channels, deliberately separate"). + When: + ``filter_func`` is applied to the row. + Then: + It should return False for every non-cooler format. + """ + # Arrange + row = [fmt, Endpoint.TILESETS, Method.GET, CacheState.COLD] + + # Act + accepted = filter_func(row) + + # Assert + assert accepted is False + + +@pytest.mark.parametrize( + "fmt", [Format.MCOOL, Format.COOL], ids=lambda f: f.name +) +@pytest.mark.parametrize( + "endpoint", + [e for e in Endpoint if e is not Endpoint.TILESETS], + ids=lambda e: e.name, +) +def test_filter_func_should_reject_cooler_formats_when_endpoint_is_not_tilesets( + fmt, endpoint +): + """Test that filter_func drops cooler-format rows off the prepare channel. + + Given: + A pairwise row pairing MCOOL or COOL with an endpoint other + than TILESETS (/data streams contact maps from upstream and + /index reports that they have no index, so those sweeps are + structurally meaningless for cooler formats). + When: + ``filter_func`` is applied to the row. + Then: + It should return False regardless of whether the tiles extra + is installed — the endpoint exclusion precedes the cooler gate. + """ + # Arrange + row = [fmt, endpoint, Method.GET, CacheState.COLD] + + # Act + accepted = filter_func(row) + + # Assert + assert accepted is False + + +@pytest.mark.parametrize( + "fmt", [Format.MCOOL, None], ids=["MCOOL", "no-format-axis"] +) +def test_filter_func_should_reject_head_method_when_endpoint_is_tilesets(fmt): + """Test that filter_func drops non-GET methods from TILESETS rows. + + Given: + A pairwise row pairing the TILESETS endpoint with the HEAD + method — with and without a format axis, since the readiness + probe has no HEAD contract either way. + When: + ``filter_func`` is applied to the row. + Then: + It should return False on both hosts with and without the + tiles extra — the method exclusion fires before the cooler gate. + """ + # Arrange + row = [v for v in (fmt, Endpoint.TILESETS, Method.HEAD) if v is not None] + + # Act + accepted = filter_func(row) + + # Assert + assert accepted is False + + +def test_filter_func_should_accept_mcool_tilesets_cold_row_when_cooler_imports(): + """Test that filter_func admits the canonical prepare-channel row. + + Given: + A pairwise row combining MCOOL, TILESETS, GET, and a cold + cache — the exact shape the tileset-prepare e2e sweep draws. + When: + ``filter_func`` is applied to the row. + Then: + It should return True exactly when ``cooler`` is importable: + admitted on a host with the tiles extra, dropped (like the + gffread/bedToBigBed tool gates) on a host without it. + """ + # Arrange + row = [Format.MCOOL, Endpoint.TILESETS, Method.GET, CacheState.COLD] + + # Act + accepted = filter_func(row) + + # Assert — guarded on host capability so the pin holds everywhere. + assert accepted is cooler_available() diff --git a/tests/integration/test_tiles_e2e.py b/tests/integration/test_tiles_e2e.py new file mode 100644 index 0000000..c83f1f6 --- /dev/null +++ b/tests/integration/test_tiles_e2e.py @@ -0,0 +1,527 @@ +"""End-to-end matrix tile serving over real coolers. + +Runs the whole chain against bytes rather than mocks: build a contact map, +put it through :class:`~cfdb.workflows.processors.matrix.MatrixTilesetProcessor` +into a real cache, then serve ``/tileset_info`` and ``/tiles`` from the +committed artifact and check the tile the client would decode against a +direct cooler read. + +Marked ``integration`` because the flat-cooler case runs a real +``cooler.zoomify_cooler``, which is the only place coarsening is exercised +for real. +""" + +from __future__ import annotations + +import asyncio +import base64 + +import numpy as np +import pytest +import pytest_asyncio + +from cfdb import api +from cfdb.api.routers.tiles import tiles, tileset_info +from cfdb.services import locks +from cfdb.tilesets.service import TilesetService, build_service +from cfdb.tilesets.store import LocalTilesetStore +from cfdb.workflows.cache import LocalFsCache +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors import matrix as matrix_module +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from tests.fixtures.coolers import CANONICAL_CHROMSIZES, build_cool, build_mcool +from tests.fixtures.remote_cache import FakeRemoteCache +from tests.test_workflows import FIXTURE_MD5, FIXTURE_MD5_ALT + +# None of the imports above pull in cooler or clodius at import time (the +# tile backend loads lazily), so on a checkout without the tiles extra the +# failure would otherwise land inside the first fixture as an error rather +# than a skip. Guard the whole module instead. +pytest.importorskip("cooler") +pytest.importorskip("clodius.tiles_v2.cooler") + +pytestmark = pytest.mark.integration + +UID = "4dn/4DNFIABC123" + + +def _file_doc(filename: str) -> dict: + """Return a projected 4DN document for a contact map.""" + return { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": FIXTURE_MD5, + "filename": filename, + "file_format": {"name": "HDF5"}, + "genome_assembly": "GRCh38", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + + +def _decode(payload: dict) -> np.ndarray: + """Decode a dense tile payload the way a HiGlass client would. + + clodius emits no ``shape`` field, so the side length is recovered as + the square root of the flat array's length. + """ + flat = np.frombuffer(base64.b64decode(payload["dense"]), dtype=payload["dtype"]) + side = int(round(len(flat) ** 0.5)) + return flat.reshape(side, side) + + +@pytest_asyncio.fixture() +async def served(mock_db, mocker, tmp_path): + """Materialize a contact map and stand up the routes over the result.""" + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + cache = LocalFsCache(tmp_path / "cache") + service = build_service( + root=tmp_path / "tilesets", + cache_provider=lambda: cache, + disk_cache_bytes=10**9, + open_max=4, + tile_cache_bytes=10**7, + threads=2, + hydrate_timeout_s=60, + ) + mocker.patch.object(api, "cache", cache) + mocker.patch.object(api, "tileset_service", service) + + async def _run(source, filename): + doc = _file_doc(filename) + mock_db.file.docs = [doc] + + async def _download(_file_meta, dest): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(source.read_bytes()) + return dest + + mocker.patch.object(matrix_module, "download_source", side_effect=_download) + + processor = MatrixTilesetProcessor() + events = [ + event + async for event in processor.run(doc, tmp_path / "workdir", cache) + ] + return doc, events, cache + + yield _run + + await service.aclose() + + +@pytest.mark.asyncio +async def test_should_serve_tiles_matching_a_direct_cooler_read(served, tmp_path): + """Test the full chain from upstream mcool to decoded tile. + + Given: + A multi-resolution cooler put through the tileset processor into + a real cache. + When: + /tileset_info and /tiles are served from the committed artifact. + Then: + The info document should describe the file's real ladder, and the + decoded tile should match a direct cooler fetch of the same + region. This is the only test that proves the artifact cfdb + commits is one clodius can actually serve — every other test + stops at one end or the other. + """ + # Arrange + import cooler + + source = build_mcool(tmp_path / "upstream.mcool") + doc, events, cache = await served(source, "sample.mcool") + + # Act + info = (await tileset_info(d=[UID]))[UID] + payload = (await tiles(d=[f"{UID}.0.0.0"]))[f"{UID}.0.0.0"] + block = _decode(payload) + + # Assert + assert info["datatype"] == "matrix" + assert info["resolutions"] == [1, 2, 4] + assert info["max_pos"] == [3000, 3000] + assert info["coordSystem"] == "GRCh38" + assert block.shape == (256, 256) + + # The coarsest resolution is 4 bp/bin, and tile (0, 0) at zoom 0 covers + # the first 256 bins of it — so the top-left corner is a direct read. + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + clr = cooler.Cooler(f"{cache.path_for(key)}::/resolutions/4") + expected = clr.matrix(balance=False).fetch("c1")[:8, :8] + np.testing.assert_allclose(block[:8, :8], expected.astype(np.float32)) + + +@pytest.mark.asyncio +async def test_should_coarsen_a_flat_cooler_into_a_servable_artifact( + served, tmp_path +): + """Test that the 10 flat 4DN coolers become tileable. + + Given: + A single-resolution ``.cool`` — the shape clodius refuses to serve, + because it carries no ``resolutions`` group. + When: + It is put through the processor and then served. + Then: + A real ``cooler.zoomify_cooler`` should have produced a ladder + whose base-resolution tile decodes to the same values a direct + cooler read of the source flat file yields. Without this the ten + flat 4DN files would be classified as contact maps, accepted for + preparation, and then serve tiles that misrepresent the data. + """ + # Arrange + import cooler + + source = build_cool(tmp_path / "upstream.cool", binsize=4) + + # Act + await served(source, "sample.cool") + info = (await tileset_info(d=[UID]))[UID] + # Resolutions are ascending and zoom levels count down from the + # coarsest, so the base resolution — the one that must reproduce the + # source verbatim — sits at the deepest zoom. + base_zoom = len(info["resolutions"]) - 1 + payload = (await tiles(d=[f"{UID}.{base_zoom}.0.0"]))[f"{UID}.{base_zoom}.0.0"] + block = _decode(payload) + + # Assert + assert info["resolutions"][0] == 4 + assert len(info["resolutions"]) >= 1 + assert block.shape == (256, 256) + + # Tile (0, 0) at the base zoom covers the first 256 bins of the + # 4 bp/bin base layer, so chromosome c1 (1000 bp = 250 bins) is a + # direct read of the *source* flat cooler — value fidelity, not just + # shape. + clr = cooler.Cooler(str(source)) + expected = clr.matrix(balance=False).fetch("c1") + np.testing.assert_allclose(block[:250, :250], expected.astype(block.dtype)) + + +@pytest.mark.asyncio +async def test_should_report_the_genome_the_fixture_actually_declares( + served, tmp_path +): + """Test that chromsizes survive the round trip intact. + + Given: + A contact map over a known three-chromosome genome. + When: + /tileset_info is served. + Then: + The chromsizes should match the source exactly. HiGlass positions + every tile against this array, so a reordering or a dropped + contig misplaces the whole heatmap rather than failing loudly. + """ + # Arrange + source = build_mcool(tmp_path / "upstream.mcool") + await served(source, "sample.mcool") + + # Act + info = (await tileset_info(d=[UID]))[UID] + + # Assert + assert [tuple(pair) for pair in info["chromsizes"]] == CANONICAL_CHROMSIZES + + +class TestTilesOverHttp: + """The full HTTP stack, which calling the handlers directly skips. + + FastAPI serializes the handler's return value to JSON on the way out, + so a clodius payload carrying a numpy scalar would 500 here first — + this is the only crossing of the "clodius payloads are JSON-ready" + seam. Follows ``tests/test_tiles.py::TestOverHttp`` in neutering the + lifespan, but patches the *real* service and cache into ``cfdb.api``. + """ + + @pytest.fixture() + def http_client(self, mock_db, mocker, tmp_path): + """A TestClient over the real app with a committed artifact behind it.""" + from mongomock_motor import AsyncMongoMockClient + from starlette.testclient import TestClient + + from cfdb.api import main + + source = build_mcool(tmp_path / "upstream.mcool") + doc = _file_doc("sample.mcool") + mock_db.file.docs = [doc] + cache = LocalFsCache(tmp_path / "cache") + + async def _download(_file_meta, dest): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(source.read_bytes()) + return dest + + mocker.patch.object(matrix_module, "download_source", side_effect=_download) + + async def _build(): + processor = MatrixTilesetProcessor() + async for _ in processor.run(doc, tmp_path / "workdir", cache): + pass + + asyncio.run(_build()) + + service = build_service( + root=tmp_path / "tilesets", + cache_provider=lambda: cache, + disk_cache_bytes=10**9, + open_max=4, + tile_cache_bytes=10**7, + threads=2, + hydrate_timeout_s=60, + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + mocker.patch.object(api, "cache", cache) + mocker.patch.object(api, "tileset_service", service) + mocker.patch.object( + main, "create_mongodb_client", return_value=AsyncMongoMockClient() + ) + mocker.patch.object(main.WorkflowProfile, "from_env", return_value=None) + + with TestClient(main.app) as client: + # Lifespan startup pointed api.db at the mongomock client; + # point it back at the fake holding the contact-map document. + # The lifespan teardown closes the patched service on the + # client's own event loop when the block exits. + mocker.patch.object(api, "db", mock_db) + yield client, doc, cache + + def test_should_serve_json_tiles_over_http_matching_a_direct_read( + self, http_client + ): + """Test the tile endpoints across the real ASGI wire. + + Given: + A committed tileset artifact with the real service patched + into the app and the lifespan otherwise neutered. + When: + /tileset_info and /tiles are requested over HTTP. + Then: + It should answer 200 with bodies that parse as JSON, and the + decoded dense tile should match a direct cooler read of the + committed artifact — proving the payloads survive FastAPI's + JSON serialization intact. + """ + # Arrange + import cooler + + client, doc, cache = http_client + tile_id = f"{UID}.0.0.0" + + # Act + info_response = client.get(f"/tileset_info/?d={UID}") + tiles_response = client.get(f"/tiles/?d={tile_id}") + + # Assert + assert info_response.status_code == 200 + assert tiles_response.status_code == 200 + info = info_response.json()[UID] + payload = tiles_response.json()[tile_id] + assert info["datatype"] == "matrix" + assert info["resolutions"] == [1, 2, 4] + + block = _decode(payload) + assert block.shape == (256, 256) + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + clr = cooler.Cooler(f"{cache.path_for(key)}::/resolutions/4") + expected = clr.matrix(balance=False).fetch("c1")[:8, :8] + np.testing.assert_allclose(block[:8, :8], expected.astype(np.float32)) + + +class TestConcurrentColdOpen: + """The ``_acquire`` winner/loser contract over real h5py handles.""" + + @pytest.mark.parametrize("n", [2, 10]) + @pytest.mark.asyncio + async def test_should_serve_identical_tiles_under_concurrent_cold_open( + self, served, tmp_path, n + ): + """Test that a cold dataset survives a burst of simultaneous opens. + + Given: + A committed tileset artifact behind a fresh service no request + has touched yet. + When: + N tile requests for the same tile are gathered concurrently. + Then: + It should return N identical, correct payloads — the open + race resolves to one shared handle with every loser closed — + and the service should close cleanly afterwards. + """ + # Arrange + import cooler + + source = build_mcool(tmp_path / "upstream.mcool") + doc, _, cache = await served(source, "sample.mcool") + tile_id = f"{UID}.0.0.0" + + # Act + responses = await asyncio.gather(*(tiles(d=[tile_id]) for _ in range(n))) + payloads = [response[tile_id] for response in responses] + + # Assert + assert len(payloads) == n + assert all(payload == payloads[0] for payload in payloads) + + block = _decode(payloads[0]) + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + clr = cooler.Cooler(f"{cache.path_for(key)}::/resolutions/4") + expected = clr.matrix(balance=False).fetch("c1")[:8, :8] + np.testing.assert_allclose(block[:8, :8], expected.astype(np.float32)) + + await api.tileset_service.aclose() + assert api.tileset_service.tile_cache_bytes_used == 0 + + +class TestEvictionUnderLiveRead: + """The store↔service release-hook coupling with real handles.""" + + @pytest.mark.asyncio + async def test_should_overshoot_for_a_pinned_artifact_and_reclaim_after_release( + self, mocker, tmp_path + ): + """Test disk eviction against live h5py handles on the S3-style path. + + Given: + Three tileset artifacts in a remote-style cache behind a disk + budget that fits exactly one, with a checkout of dataset A + held open. + When: + B is served while A is pinned, then A is released and a third + dataset forces an eviction pass, then B is served again. + Then: + It should serve B correctly both times, keep A's hydrated + file on disk while it is pinned (the budget overshoots rather + than unlinking under a live handle), and reclaim back inside + the budget once A is released. + """ + # Arrange + import cooler + + third_md5 = "5d41402abc4b2a76b9719d911017c592" + specs = [ + ("IDA", FIXTURE_MD5, 0), + ("IDB", FIXTURE_MD5_ALT, 1), + ("IDC", third_md5, 2), + ] + remote = FakeRemoteCache({}) + processor = MatrixTilesetProcessor() + docs, sources, keys = {}, {}, {} + for local_id, md5, seed in specs: + doc = _file_doc("sample.mcool") + doc["local_id"] = local_id + doc["md5"] = md5 + docs[local_id] = doc + sources[md5] = build_mcool(tmp_path / f"upstream-{local_id}.mcool", seed=seed) + keys[local_id] = processor.cache_key_for(doc, ArtifactKind.TILESET) + + async def _download(file_meta, dest): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(sources[file_meta["md5"]].read_bytes()) + return dest + + mocker.patch.object(matrix_module, "download_source", side_effect=_download) + for local_id, _, _ in specs: + async for _ in processor.run( + docs[local_id], tmp_path / "workdir" / local_id, remote + ): + pass + + sizes = {local_id: len(remote.blobs[keys[local_id]]) for local_id in docs} + budget = max(sizes.values()) + # The premise: any one artifact fits, no two do. + assert min(sizes.values()) * 2 > budget + + store_root = tmp_path / "tilesets" + store = LocalTilesetStore(store_root, max_bytes=budget) + service = TilesetService( + store, + open_max=4, + tile_cache_bytes=0, # force every serve through a real read + threads=2, + hydrate_timeout_s=60, + cache_provider=lambda: remote, + ) + tile_b = "4dn/IDB.0.0.0" + + # Act & assert + async with service.checkout("4dn/IDA", docs["IDA"]): + # Open A's h5py file for real, so the pinned handle is live. + await service.tileset_info("4dn/IDA", docs["IDA"]) + + first = (await service.tiles("4dn/IDB", docs["IDB"], [tile_b]))[tile_b] + + # B hydrated beside pinned A: both files on disk, over budget. + assert store.resident_bytes == sizes["IDA"] + sizes["IDB"] + assert store.resident_bytes > budget + assert len(list(store_root.iterdir())) == 2 + + # A released; a third hydration triggers the eviction pass, which + # now closes A's (and B's) real handle through the release hook. + await service.tiles("4dn/IDC", docs["IDC"], ["4dn/IDC.0.0.0"]) + assert store.resident_bytes == sizes["IDC"] + assert store.resident_bytes <= budget + + second = (await service.tiles("4dn/IDB", docs["IDB"], [tile_b]))[tile_b] + + assert second == first + clr = cooler.Cooler(f"{sources[FIXTURE_MD5_ALT]}::/resolutions/4") + expected = clr.matrix(balance=False).fetch("c1")[:8, :8] + np.testing.assert_allclose( + _decode(second)[:8, :8], expected.astype(np.float32) + ) + # B was re-hydrated after its eviction, and back inside the budget. + assert remote.get_calls.count(keys["IDB"]) == 2 + assert store.resident_bytes == sizes["IDB"] + + await service.aclose() + + +class TestBalancedServe: + """Balancing transforms through the processor into a served tile.""" + + @pytest.mark.asyncio + async def test_should_serve_a_balanced_tile_matching_a_direct_balanced_read( + self, served, tmp_path + ): + """Test that a named balancing transform reaches the pixels. + + Given: + An mcool carrying two weight columns — two, because ``weight`` + is also clodius's default fallback, so only a second named + column makes honouring the request observable — put through + the processor into a real cache. + When: + A tile naming the ``alt_weight`` transform is requested. + Then: + It should decode to the values a direct balanced cooler fetch + with that column yields, within dtype quantization, and + differ from the default-weight balance — proving the named + request was honoured rather than ignored. + """ + # Arrange + import cooler + + source = build_mcool( + tmp_path / "upstream.mcool", + weights={"weight": 0.5, "alt_weight": 2.0}, + ) + doc, _, cache = await served(source, "sample.mcool") + tile_id = f"{UID}.0.0.0.alt_weight" + + # Act + payload = (await tiles(d=[tile_id]))[tile_id] + block = _decode(payload) + + # Assert + key = MatrixTilesetProcessor().cache_key_for(doc, ArtifactKind.TILESET) + clr = cooler.Cooler(f"{cache.path_for(key)}::/resolutions/4") + # Compare the whole of c1 (250 bins at 4 bp/bin) rather than a + # corner, so the comparison actually crosses nonzero pixels. + expected = clr.matrix(balance="alt_weight").fetch("c1") + default = clr.matrix(balance=True).fetch("c1") + assert (expected > 0).any() + np.testing.assert_allclose( + block[:250, :250], expected.astype(block.dtype), rtol=1e-3 + ) + assert not np.allclose(block[:250, :250], default.astype(block.dtype)) diff --git a/tests/integration/test_tiles_s3_profile.py b/tests/integration/test_tiles_s3_profile.py new file mode 100644 index 0000000..8337fdc --- /dev/null +++ b/tests/integration/test_tiles_s3_profile.py @@ -0,0 +1,308 @@ +"""Matrix tile serving over the S3 profile, moto-backed and in-process. + +The local-profile e2e chain (``test_tiles_e2e.py``) opens the committed +artifact in place, so the hydration path — pull the artifact from S3 onto +the API task's disk before h5py can touch it — is never exercised there. +These tests commit a real mcool through the real processor into a +moto-backed :class:`~cfdb.workflows.cache.S3Cache` and serve tiles through +the store's single-flight LRU, covering the three behaviours only the S3 +profile has: hydrate-once-then-serve-warm, the disk-budget refusal, and +the hydration timeout with clean recovery. + +In-process rather than through a wool worker, deliberately: a moto mock +cannot cross the subprocess boundary, and nothing S3-specific lives on the +worker side of it. +""" + +from __future__ import annotations + +import asyncio +import base64 + +import numpy as np +import pytest +import pytest_asyncio + +pytest.importorskip("cooler") +pytest.importorskip("moto") + +import boto3 +from fastapi import HTTPException +from moto import mock_aws + +from cfdb import api +from cfdb.api.routers.tiles import tiles, tileset_info +from cfdb.services import locks +from cfdb.tilesets.service import TilesetService +from cfdb.tilesets.store import LocalTilesetStore +from cfdb.workflows.cache import CacheBackend, S3Cache +from cfdb.workflows.models import ArtifactKind +from cfdb.workflows.processors import matrix as matrix_module +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor +from tests.fixtures.coolers import build_mcool +from tests.test_workflows import FIXTURE_MD5 + +pytestmark = pytest.mark.integration + +_BUCKET = "cfdb-tiles-test-cache" + +UID = "4dn/4DNFIABC123" + + +def _file_doc(filename: str) -> dict: + """Return a projected 4DN document for a contact map.""" + return { + "submission": "4dn", + "local_id": "4DNFIABC123", + "md5": FIXTURE_MD5, + "filename": filename, + "file_format": {"name": "HDF5"}, + "genome_assembly": "GRCh38", + "dcc": {"dcc_abbreviation": "4DN_DCIC"}, + } + + +def _decode(payload: dict) -> np.ndarray: + """Decode a dense tile payload the way a HiGlass client would.""" + flat = np.frombuffer(base64.b64decode(payload["dense"]), dtype=payload["dtype"]) + side = int(round(len(flat) ** 0.5)) + return flat.reshape(side, side) + + +class DelegatingS3Cache(CacheBackend): + """Delegate every call to a real S3Cache, observing ``get``. + + The store branches only on ``isinstance(cache, LocalFsCache)``, so a + wrapper keeps the hydration path honest while letting a test count S3 + GETs — via the delegation seam rather than moto internals — and, + when ``stall`` is set, hold the byte stream open past the service's + hydration timeout. + """ + + def __init__(self, inner: S3Cache) -> None: + self.inner = inner + self.get_calls: list[str] = [] + #: When set to an un-set event, ``get`` streams nothing until the + #: event fires — the shape of a download that outlives its budget. + self.stall: asyncio.Event | None = None + + async def head(self, key: str): + return await self.inner.head(key) + + def get(self, key: str, byte_range=None): + self.get_calls.append(key) + inner, stall = self.inner, self.stall + + async def _stream(): + if stall is not None: + await stall.wait() + async for chunk in inner.get(key, byte_range): + yield chunk + + return _stream() + + async def put(self, key: str, source_path): + return await self.inner.put(key, source_path) + + async def delete(self, key: str) -> bool: + return await self.inner.delete(key) + + +@pytest.fixture() +def s3_client(): + """Return a moto-backed boto3 S3 client with one created bucket.""" + with mock_aws(): + client = boto3.client("s3", region_name="us-east-1") + client.create_bucket(Bucket=_BUCKET) + yield client + + +@pytest.fixture() +def s3_cache(s3_client) -> S3Cache: + """Return an S3Cache wired up to the moto-backed client.""" + return S3Cache(bucket=_BUCKET, client=s3_client) + + +@pytest_asyncio.fixture() +async def committed(s3_cache, mock_db, mocker, tmp_path): + """Commit a real mcool through the processor into the S3 cache. + + Returns the projected file document, the tileset cache key, and the + committed artifact's CacheEntry (whose size the store budgets by). + """ + source = build_mcool(tmp_path / "upstream.mcool") + doc = _file_doc("sample.mcool") + mock_db.file.docs = [doc] + + async def _download(_file_meta, dest): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(source.read_bytes()) + return dest + + mocker.patch.object(matrix_module, "download_source", side_effect=_download) + + processor = MatrixTilesetProcessor() + async for _event in processor.run(doc, tmp_path / "workdir", s3_cache): + pass + + key = processor.cache_key_for(doc, ArtifactKind.TILESET) + entry = await s3_cache.head(key) + assert entry is not None + return doc, key, entry + + +@pytest_asyncio.fixture() +async def serve(mocker, tmp_path): + """Return a builder wiring a TilesetService over a given cache. + + The store is built by hand rather than through ``build_service`` so + tests can assert against its public residency accounting and pick a + per-test disk budget; teardown closes every service that was built. + """ + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + services: list[TilesetService] = [] + store_root = tmp_path / "tilesets" + + def _build(cache, *, max_bytes: int = 10**9, hydrate_timeout_s: float = 60.0): + store = LocalTilesetStore(store_root, max_bytes=max_bytes) + service = TilesetService( + store, + open_max=4, + tile_cache_bytes=10**7, + threads=2, + hydrate_timeout_s=hydrate_timeout_s, + cache_provider=lambda: cache, + ) + mocker.patch.object(api, "cache", cache) + mocker.patch.object(api, "tileset_service", service) + services.append(service) + return service, store, store_root + + yield _build + + for service in services: + await service.aclose() + + +class TestS3ProfileTileChain: + @pytest.mark.asyncio + async def test_should_hydrate_once_and_serve_warm_without_a_second_get( + self, committed, s3_cache, serve + ): + """Test the cold-to-warm tile chain over a moto-backed S3 cache. + + Given: + A real mcool committed by the processor into a moto-backed + S3Cache, and a tileset store with an empty root. + When: + /tileset_info and /tiles are served cold, then the same tile + is requested again warm. + Then: + It should hydrate exactly once — one file under the store + root, resident_bytes equal to the artifact's size, and no + .part residue — with the decoded tile matching a direct + cooler read of the hydrated copy, and the warm request + should make no second S3 GET. + """ + # Arrange + import cooler + + doc, key, entry = committed + counting = DelegatingS3Cache(s3_cache) + service, store, store_root = serve(counting) + + # Act + info = (await tileset_info(d=[UID]))[UID] + cold = (await tiles(d=[f"{UID}.0.0.0"]))[f"{UID}.0.0.0"] + gets_after_cold = list(counting.get_calls) + warm = (await tiles(d=[f"{UID}.0.0.0"]))[f"{UID}.0.0.0"] + + # Assert + assert info["datatype"] == "matrix" + assert info["resolutions"] == [1, 2, 4] + + hydrated = sorted(store_root.iterdir()) + assert [p.suffix for p in hydrated] == [".tileset"] + assert not list(store_root.glob("*.part")) + assert store.resident_bytes == entry.size + assert hydrated[0].stat().st_size == entry.size + + assert gets_after_cold == [key] + + block = _decode(cold) + assert block.shape == (256, 256) + clr = cooler.Cooler(f"{hydrated[0]}::/resolutions/4") + expected = clr.matrix(balance=False).fetch("c1")[:8, :8] + np.testing.assert_allclose(block[:8, :8], expected.astype(np.float32)) + + assert counting.get_calls == [key] + assert warm == cold + + @pytest.mark.asyncio + async def test_should_answer_503_when_artifact_exceeds_disk_budget( + self, committed, s3_cache, serve + ): + """Test the disk-budget refusal against a real committed artifact. + + Given: + A store whose max_bytes sits below the committed artifact's + size, with nothing injected — the store learns the size from + cache.head itself. + When: + /tileset_info is requested. + Then: + It should answer 503 with a Retry-After header, not 404 — + the budget is an operator condition, not "dataset does not + exist". + """ + # Arrange + doc, key, entry = committed + serve(s3_cache, max_bytes=entry.size - 1) + + # Act & assert + with pytest.raises(HTTPException) as excinfo: + await tileset_info(d=[UID]) + assert excinfo.value.status_code == 503 + assert excinfo.value.headers is not None + assert "Retry-After" in excinfo.value.headers + + @pytest.mark.asyncio + async def test_should_leave_no_residue_and_recover_after_hydration_timeout( + self, committed, s3_cache, serve + ): + """Test that a timed-out hydration cleans up and does not poison. + + Given: + A delegating cache whose byte stream stalls past the + service's hydrate_timeout_s, then has the stall removed. + When: + /tileset_info is requested, and then requested again. + Then: + It should first answer 503 with Retry-After leaving no .part + file or store residue — the timeout's cancellation propagates + into hydration cleanup — and the retry should succeed, + proving the timed-out key was not poisoned. + """ + # Arrange + doc, key, entry = committed + stalled = DelegatingS3Cache(s3_cache) + stalled.stall = asyncio.Event() + service, store, store_root = serve(stalled, hydrate_timeout_s=0.2) + + # Act & assert — the stalled hydration times out cleanly + with pytest.raises(HTTPException) as excinfo: + await tileset_info(d=[UID]) + assert excinfo.value.status_code == 503 + assert excinfo.value.headers is not None + assert "Retry-After" in excinfo.value.headers + assert list(store_root.iterdir()) == [] + assert store.resident_bytes == 0 + + # Act — remove the delay and retry the same key + stalled.stall.set() + info = (await tileset_info(d=[UID]))[UID] + + # Assert — the retry hydrated and served + assert info["resolutions"] == [1, 2, 4] + assert store.resident_bytes == entry.size + assert stalled.get_calls == [key, key] diff --git a/tests/integration/test_tilesets_prep_e2e.py b/tests/integration/test_tilesets_prep_e2e.py new file mode 100644 index 0000000..451cc5d --- /dev/null +++ b/tests/integration/test_tilesets_prep_e2e.py @@ -0,0 +1,664 @@ +"""End-to-end tileset preparation over a real wool worker. + +Exercises the ``POST /tilesets/{dcc}/{local_id}`` preparation channel all +the way through: the router claims a workflow, the ``WoolExecutor`` +dispatches it across the cloudpickle boundary into a real +``wool.WorkerPool`` worker, ``MatrixTilesetProcessor`` downloads the +contact map from the sample HTTP server and commits (or refuses) the +TILESET artifact, and the readiness probe flips. Every other tile test +runs the processor in-process with ``download_source`` patched — this +module is where the worker subprocess actually builds the artifact. +""" + +from __future__ import annotations + +import asyncio +import json +import re +from datetime import datetime, timezone + +import pytest + +pytest.importorskip("cooler") + +import numpy as np +from allpairspy import AllPairs +from fastapi import HTTPException +from fastapi.responses import JSONResponse + +from cfdb import api +from cfdb.api.routers.tilesets import prepare_tileset, tileset_status +from cfdb.services import locks +from cfdb.workflows import TILESET_MAX_SOURCE_BYTES, WORKFLOW_MAX_ACTIVE +from cfdb.workflows.lock import JOBS_COLLECTION, get_job +from cfdb.workflows.models import ArtifactKind, JobStatus +from cfdb.workflows.processors.matrix import MatrixTilesetProcessor + +from tests.integration.conftest import ( + CacheState, + Concurrency, + Endpoint, + Format, + PickleBoundary, + Scenario, + _wait_for_terminal, + filter_func, + make_file_meta, +) +from tests.integration.fixtures.make_samples import SampleFile, _md5 + +pytestmark = pytest.mark.integration + +#: Multi-segment absolute paths must never survive error scrubbing into a +#: persisted JobRecord (mirrors the /jobs router e2e pin). +_PATH_LEAK = re.compile(r"/[A-Za-z][A-Za-z0-9_.-]*/[A-Za-z]") + + +@pytest.fixture() +def jobs_mutex_index(mock_db): + """Install the partial-unique mutex index for real. + + The shared ``install_jobs_index`` fixture calls the FakeCollection's + *async* ``create_index`` without awaiting it, so the coroutine never + runs and the mutex index is never installed — concurrent claims then + insert duplicate active rows instead of funneling. Registered here + through the synchronous ``register_index`` seam so the funnel tests + in this module exercise the real DuplicateKeyError attach path. + """ + mock_db.jobs.register_index( + {"workflow_key": 1}, + unique=True, + partialFilterExpression={"active": True}, + ) + return mock_db + + +@pytest.fixture() +def wired_api(jobs_mutex_index, integration_executor, mocker): + """Bind integration executor / cache / registry onto the api module.""" + mocker.patch.object(api, "cache", integration_executor._cache) + mocker.patch.object( + api, "processor_registry", integration_executor._registry + ) + mocker.patch.object(api, "executor", integration_executor) + return integration_executor + + +def _job_id_from_202(resp) -> str: + assert isinstance(resp, JSONResponse) + assert resp.status_code == 202 + payload = json.loads(bytes(resp.body).decode()) + return payload["job_id"] + + +def _seed_contact_map( + mock_db, + sample: SampleFile, + sample_server: str, + *, + local_id: str, + filename: str, +) -> dict: + """Insert a 4DN contact-map file_meta into the file collections.""" + meta = make_file_meta( + sample, + base_url=sample_server, + dcc="4DN", + local_id=local_id, + filename=filename, + file_format_name="HDF5", + ) + doc = {**meta, "id_namespace": "tag:4dn.example,2020:"} + mock_db.files.docs = [doc] + mock_db.file.docs = [doc] + return doc + + +def _records_for(mock_db, local_id: str) -> list[dict]: + return [ + d + for d in mock_db[JOBS_COLLECTION].docs + if d.get("local_id") == local_id + ] + + +def _decode(payload: dict) -> np.ndarray: + """Decode a dense tile payload the way a HiGlass client would.""" + import base64 + + flat = np.frombuffer( + base64.b64decode(payload["dense"]), dtype=payload["dtype"] + ) + side = int(round(len(flat) ** 0.5)) + return flat.reshape(side, side) + + +class TestTilesetPrepareE2E: + @pytest.mark.asyncio + async def test_prepare_tileset_should_build_a_servable_mcool_artifact_end_to_end( + self, + samples, + sample_server, + wired_api, + mock_db, + mocker, + tmp_path, + xfail_known_bugs, + ): + """Test the full POST → worker build → probe → idempotent-POST cycle. + + Given: + A real multi-resolution mcool served over the sample HTTP + server, a 4DN contact-map document in the DB, and a cold + cache — the first scenario to carry MatrixTilesetProcessor + across the wool cloudpickle boundary. + When: + POST /tilesets claims a workflow, the job is polled to a + terminal state, the readiness probe is called, and a second + POST is issued. + Then: + It should answer 202 with a /jobs Location; complete with + stages_done == ["tileset"] and the tileset cache key; commit + bytes that open as a cooler whose decoded tile matches a + direct read; flip the probe to ready:true; and answer the + second POST with 200 while exactly one JobRecord exists. + """ + pytest.importorskip("clodius.tiles_v2.cooler") + import cooler + + from cfdb.api.routers.tiles import tiles + from cfdb.tilesets.service import build_service + + scenario = Scenario( + format=Format.MCOOL, + endpoint=Endpoint.TILESETS, + cache_state=CacheState.COLD, + pickle_boundary=PickleBoundary.WOOL_WORKER, + ) + + async def _body(): + # Arrange + sample = samples["mcool"] + assert sample is not None + doc = _seed_contact_map( + mock_db, + sample, + sample_server, + local_id="4DNFI-prep-mcool", + filename="sample.mcool", + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + assert await tileset_status("4dn", doc["local_id"]) == { + "ready": False + } + + # Act — POST dispatches; poll the job to terminal. + first = await prepare_tileset("4dn", doc["local_id"]) + job_id = _job_id_from_202(first) + assert first.headers["location"] == f"/jobs/{job_id}" + await _wait_for_terminal(mock_db, job_id) + + # Assert — the record carries the tileset stage and key. + final = await get_job(mock_db, job_id) + assert final is not None + assert final.status is JobStatus.COMPLETED + assert final.stages_done == ["tileset"] + key = final.artifact_cache_keys["tileset"] + + # Assert — the committed bytes open as a cooler and a decoded + # tile matches a direct read of the same corner. + cached_path = api.cache.path_for(key) + clr = cooler.Cooler(f"{cached_path}::/resolutions/4") + service = build_service( + root=tmp_path / "tilesets", + cache_provider=lambda: api.cache, + disk_cache_bytes=10**9, + open_max=4, + tile_cache_bytes=10**7, + threads=2, + hydrate_timeout_s=60, + ) + mocker.patch.object(api, "tileset_service", service) + try: + uid = f"4dn/{doc['local_id']}" + payload = (await tiles(d=[f"{uid}.0.0.0"]))[f"{uid}.0.0.0"] + block = _decode(payload) + assert block.shape == (256, 256) + expected = clr.matrix(balance=False).fetch("c1")[:8, :8] + np.testing.assert_allclose( + block[:8, :8], expected.astype(np.float32) + ) + finally: + await service.aclose() + + # Assert — the probe flips and a second POST is idempotent. + assert await tileset_status("4dn", doc["local_id"]) == { + "ready": True + } + second = await prepare_tileset("4dn", doc["local_id"]) + assert isinstance(second, JSONResponse) + assert second.status_code == 200 + assert json.loads(bytes(second.body).decode()) == {"ready": True} + assert len(_records_for(mock_db, doc["local_id"])) == 1 + + await xfail_known_bugs(scenario, _body) + + @pytest.mark.asyncio + async def test_prepare_tileset_should_coarsen_a_flat_cool_inside_the_worker( + self, + samples, + sample_server, + wired_api, + mock_db, + mocker, + xfail_known_bugs, + ): + """Test that a flat .cool is zoomified by the worker subprocess. + + Given: + A real single-resolution .cool served over the sample HTTP + server — the shape clodius refuses without a resolutions + group — and the same live wool stack. + When: + POST /tilesets runs the same lifecycle to completion. + Then: + It should complete, and the committed artifact should carry + a resolutions ladder starting at the source binsize — + proving ``cooler.zoomify_cooler`` ran inside the worker. + """ + import h5py + + scenario = Scenario( + format=Format.COOL, + endpoint=Endpoint.TILESETS, + cache_state=CacheState.COLD, + pickle_boundary=PickleBoundary.WOOL_WORKER, + ) + + async def _body(): + # Arrange + sample = samples["cool"] + assert sample is not None + doc = _seed_contact_map( + mock_db, + sample, + sample_server, + local_id="4DNFI-prep-cool", + filename="sample.cool", + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + + # Act + resp = await prepare_tileset("4dn", doc["local_id"]) + job_id = _job_id_from_202(resp) + await _wait_for_terminal(mock_db, job_id) + + # Assert + final = await get_job(mock_db, job_id) + assert final is not None + assert final.status is JobStatus.COMPLETED + assert final.stages_done == ["tileset"] + key = final.artifact_cache_keys["tileset"] + cached_path = api.cache.path_for(key) + with h5py.File(cached_path, "r") as handle: + assert "resolutions" in handle + ladder = sorted(int(r) for r in handle["resolutions"]) + # The fixture cooler is built at binsize 4, so the worker's + # ladder must start there. + assert ladder[0] == 4 + assert await tileset_status("4dn", doc["local_id"]) == { + "ready": True + } + + await xfail_known_bugs(scenario, _body) + + @pytest.mark.asyncio + async def test_prepare_tileset_should_fail_cleanly_when_upstream_bytes_are_not_hdf5( + self, + sample_data_root, + sample_server, + wired_api, + mock_db, + mocker, + xfail_known_bugs, + ): + """Test that upstream garbage fails the job without poisoning the cache. + + Given: + Non-HDF5 bytes named sample.mcool served upstream, so the + worker's h5py open fails after a real download. + When: + POST /tilesets runs to a terminal state, the probe is + called, and a second POST is issued. + Then: + It should record FAILED with a path-scrubbed error, leave + ``cache.head`` None for the tileset key (garbage never lands + in cache), report ready:false, and let the second POST claim + a fresh job because the mutex was released. + """ + scenario = Scenario( + format=Format.MCOOL, + endpoint=Endpoint.TILESETS, + cache_state=CacheState.COLD, + pickle_boundary=PickleBoundary.WOOL_WORKER, + ) + + async def _body(): + # Arrange — write garbage into the served sample directory. + garbage_path = sample_data_root / "garbage-not-hdf5.mcool" + garbage_path.write_bytes( + b"cfdb-integration: definitely not HDF5 bytes\n" * 8 + ) + garbage = SampleFile( + path=garbage_path, md5=_md5(garbage_path), format="mcool" + ) + doc = _seed_contact_map( + mock_db, + garbage, + sample_server, + local_id="4DNFI-prep-garbage", + filename="sample.mcool", + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + + # Act + first = await prepare_tileset("4dn", doc["local_id"]) + first_job_id = _job_id_from_202(first) + await _wait_for_terminal(mock_db, first_job_id) + + # Assert — FAILED, scrubbed, nothing committed. + final = await get_job(mock_db, first_job_id) + assert final is not None + assert final.status is JobStatus.FAILED + assert final.error + assert _PATH_LEAK.search(final.error) is None + key = MatrixTilesetProcessor().cache_key_for( + doc, ArtifactKind.TILESET + ) + assert await api.cache.head(key) is None + assert await tileset_status("4dn", doc["local_id"]) == { + "ready": False + } + + # Act & assert — the mutex is released, so a retry claims a + # NEW job rather than attaching to the failed one. + second = await prepare_tileset("4dn", doc["local_id"]) + second_job_id = _job_id_from_202(second) + assert second_job_id != first_job_id + await _wait_for_terminal(mock_db, second_job_id) + + await xfail_known_bugs(scenario, _body) + + @pytest.mark.asyncio + async def test_prepare_tileset_should_refuse_an_oversized_source_named_by_the_cap( + self, + samples, + sample_server, + wired_api, + mock_db, + mocker, + xfail_known_bugs, + ): + """Test that the source-size cap fails the job before downloading. + + Given: + A contact-map document whose integer ``size_in_bytes`` + exceeds ``CFDB_TILESET_MAX_SOURCE_BYTES``, dispatched through + the real wool stack. + When: + POST /tilesets runs to a terminal state and is then retried. + Then: + It should record FAILED with an error naming the source cap + and leaking no path, keep the probe at ready:false, and let + the retry claim a fresh job. + """ + if not TILESET_MAX_SOURCE_BYTES: + pytest.skip( + "CFDB_TILESET_MAX_SOURCE_BYTES=0 disables the cap in this " + "environment" + ) + + scenario = Scenario( + format=Format.MCOOL, + endpoint=Endpoint.TILESETS, + cache_state=CacheState.COLD, + pickle_boundary=PickleBoundary.WOOL_WORKER, + ) + + async def _body(): + # Arrange + sample = samples["mcool"] + assert sample is not None + doc = _seed_contact_map( + mock_db, + sample, + sample_server, + local_id="4DNFI-prep-oversize", + filename="sample.mcool", + ) + doc["size_in_bytes"] = TILESET_MAX_SOURCE_BYTES + 1 + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + + # Act + first = await prepare_tileset("4dn", doc["local_id"]) + first_job_id = _job_id_from_202(first) + await _wait_for_terminal(mock_db, first_job_id) + + # Assert + final = await get_job(mock_db, first_job_id) + assert final is not None + assert final.status is JobStatus.FAILED + assert final.error is not None + assert "CFDB_TILESET_MAX_SOURCE_BYTES" in final.error + assert _PATH_LEAK.search(final.error) is None + assert await tileset_status("4dn", doc["local_id"]) == { + "ready": False + } + + # Act & assert — retry claims a fresh job. + second = await prepare_tileset("4dn", doc["local_id"]) + second_job_id = _job_id_from_202(second) + assert second_job_id != first_job_id + await _wait_for_terminal(mock_db, second_job_id) + + await xfail_known_bugs(scenario, _body) + + +# Pairwise (Format × Concurrency) rows for the preparation-channel funnel +# test. filter_func drops the cooler rows when the tiles extra is absent, +# mirroring the tool gates on the tabix sweeps. +def _prep_concurrency_scenarios() -> list[Scenario]: + rows = AllPairs( + [[Format.MCOOL, Format.COOL], list(Concurrency)], + filter_func=filter_func, + ) + scenarios: list[Scenario] = [] + for row in rows: + fmt = next(v for v in row if isinstance(v, Format)) + concurrency = next(v for v in row if isinstance(v, Concurrency)) + scenarios.append( + Scenario( + format=fmt, + endpoint=Endpoint.TILESETS, + concurrency=concurrency, + ) + ) + return scenarios + + +_PREP_CONCURRENCY_SCENARIOS = _prep_concurrency_scenarios() + +_SAMPLE_KEY_BY_FORMAT: dict[Format, str] = { + Format.MCOOL: "mcool", + Format.COOL: "cool", +} + +_FILENAME_BY_FORMAT: dict[Format, str] = { + Format.MCOOL: "sample.mcool", + Format.COOL: "sample.cool", +} + + +class TestTilesetPrepareConcurrency: + @pytest.mark.parametrize( + "scenario", _PREP_CONCURRENCY_SCENARIOS, ids=str + ) + @pytest.mark.asyncio + async def test_prepare_tileset_should_funnel_concurrent_posts_to_single_workflow_pairwise( + self, + samples, + sample_server, + wired_api, + mock_db, + mocker, + scenario: Scenario, + xfail_known_bugs, + ): + """Test that N concurrent POSTs funnel onto one workflow per source. + + Given: + A pairwise sweep over ``(Format ∈ {MCOOL, COOL}, + Concurrency ∈ {N2, N10})`` filtered through ``filter_func``; + one contact-map document per scenario over a cold cache. + When: + N concurrent ``prepare_tileset`` coroutines are awaited via + ``asyncio.gather`` and the workflow runs to terminal. + Then: + It should answer every caller 202 with one identical + job_id, persist exactly one JobRecord, and complete — + the mutex funnel extended to the preparation channel. + """ + + async def _body(): + # Arrange + sample = samples[_SAMPLE_KEY_BY_FORMAT[scenario.format]] + if sample is None: + pytest.skip( + f"Sample for {scenario.format.value} unavailable on " + "this host" + ) + local_id = ( + f"4DNFI-conc-{scenario.format.name}-{scenario.concurrency.name}" + ) + doc = _seed_contact_map( + mock_db, + sample, + sample_server, + local_id=local_id, + filename=_FILENAME_BY_FORMAT[scenario.format], + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + n = scenario.concurrency.value + + # Act + responses = await asyncio.gather( + *[prepare_tileset("4dn", doc["local_id"]) for _ in range(n)] + ) + + # Assert + job_ids = {_job_id_from_202(r) for r in responses} + assert len(job_ids) == 1, ( + f"Expected one job_id across {n} concurrent POSTs; " + f"got {job_ids!r}" + ) + assert len(_records_for(mock_db, doc["local_id"])) == 1 + + job_id = next(iter(job_ids)) + await _wait_for_terminal(mock_db, job_id, timeout=120.0) + final = await get_job(mock_db, job_id) + assert final is not None + assert final.status is JobStatus.COMPLETED + + await xfail_known_bugs(scenario, _body) + + +class TestTilesetPrepareAdmission: + @pytest.mark.asyncio + async def test_prepare_tileset_should_return_429_when_admission_ceiling_reached( + self, samples, sample_server, wired_api, mock_db, mocker + ): + """Test that the admission ceiling sheds a POST with 429 Retry-After. + + Given: + The jobs collection seeded with ``WORKFLOW_MAX_ACTIVE`` + active job documents — the data the admission count reads — + and a contact-map document over a cold cache. + When: + ``prepare_tileset`` is awaited. + Then: + It should raise HTTPException 429 carrying a numeric + Retry-After header, and persist no JobRecord for the file. + """ + # Arrange — fill the active backlog to the public ceiling. + sample = samples["mcool"] + assert sample is not None + doc = _seed_contact_map( + mock_db, + sample, + sample_server, + local_id="4DNFI-prep-admission", + filename="sample.mcool", + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + now = datetime.now(timezone.utc) + mock_db[JOBS_COLLECTION].docs.extend( + { + "job_id": f"seed-{i}", + "workflow_key": f"seed/{i}/{'0' * 32}/v1", + "status": JobStatus.PENDING.value, + "active": True, + "dcc": "seed", + "local_id": f"seed-{i}", + "md5": "0" * 32, + "pipeline_version": 1, + "submitted_at": now, + "updated_at": now, + } + for i in range(WORKFLOW_MAX_ACTIVE) + ) + + # Act & assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", doc["local_id"]) + assert exc_info.value.status_code == 429 + assert exc_info.value.headers["Retry-After"].isdigit() + assert _records_for(mock_db, doc["local_id"]) == [] + + @pytest.mark.asyncio + async def test_prepare_tileset_should_return_503_when_executor_drained( + self, samples, sample_server, wired_api, mock_db, mocker + ): + """Test that a drained executor answers 503, not 409. + + Given: + A contact-map document over a cold cache and a really-drained + ``WoolExecutor`` (``drain`` awaited, not patched). + When: + ``prepare_tileset`` is awaited. + Then: + It should raise HTTPException 503 with a Retry-After header + — pinning the catch order against the real hierarchy, where + ``ExecutorDraining`` subclasses ``WorkflowNotApplicable`` and + a mis-ordered ladder would report 409 — and persist no + JobRecord. + """ + # Arrange + sample = samples["mcool"] + assert sample is not None + doc = _seed_contact_map( + mock_db, + sample, + sample_server, + local_id="4DNFI-prep-draining", + filename="sample.mcool", + ) + mocker.patch.object(locks, "wait_for_cutover", return_value=None) + await wired_api.drain(timeout=5.0) + + # Act & assert + with pytest.raises(HTTPException) as exc_info: + await prepare_tileset("4dn", doc["local_id"]) + assert exc_info.value.status_code == 503 + assert exc_info.value.status_code != 409 + assert exc_info.value.headers["Retry-After"].isdigit() + assert _records_for(mock_db, doc["local_id"]) == [] From 66d74e6239efe3890bf35698998357aad4681e19 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 28 Sep 2026 14:29:12 -0400 Subject: [PATCH 13/14] docs: Document matrix tile serving Adds the Matrix tile serving section: the two-channel design (pure readers vs. the preparation channel that builds artifacts), the wire protocol and status codes, the tileset environment variables, and the Dockerfile.api-tiles / make api-tiles / make worker-tiles local-dev flow. Also records the documented limitations -- .hic is recognized and refused rather than served, the tileset artifact's disk footprint per API task, and the per-tile dtype and interchromosomal-tile behavior verified against live HiGlass reference servers. --- CLAUDE.md | 1 + README.md | 142 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..1571c04 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +Start with @README.md \ No newline at end of file diff --git a/README.md b/README.md index 4d60761..6001dae 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,14 @@ pip install git+https://github.com/abdenlab/cfdb.git Requires Python 3.11 or later. +Matrix tile serving for Hi-C contact maps is an optional extra, because the [`abdenlab/clodius`](https://github.com/abdenlab/clodius) fork it depends on requires Python 3.12 while cfdb still supports 3.11. The dependency is pinned to a commit sha of the public fork, so enabling it needs nothing beyond the extra: + +```bash +uv sync --extra tiles +``` + +Without it the API runs unchanged and the tile endpoints answer `501`. See [Matrix tile serving](#matrix-tile-serving-hi-c-contact-maps). + ### Environment Variables | Variable | Description | Default | @@ -25,6 +33,12 @@ Requires Python 3.11 or later. | `CFDB_WORKER_TLS_CERT` | Path to this process's PEM certificate on the worker gRPC channel — the worker leaf cert on a worker (`worker_main`/`worker_lan`), the API client cert on the API. Must be signed by `CFDB_WORKER_TLS_CA`. | - | | `CFDB_WORKER_TLS_KEY` | Path to this process's PEM private key paired with `CFDB_WORKER_TLS_CERT`. | - | | `CFDB_WORKER_TLS_IDENTITY` | Logical name the **API** verifies worker certificates against, in place of the address it dialed. Workers answer on addresses assigned at launch — an awsvpc IP on Fargate, a bridge IP in local containers — that no certificate minted ahead of time can name, so without this the handshake cannot succeed. The worker leaf must carry this value as a SAN; `certs/generate-worker-certs.sh` mints the default. Set to the empty string to verify against the dialed address instead. Read by every process that dials: the API on the dispatch channel, and each worker on the graceful-stop channel wool opens back to its own subprocess — so the API and the workers MUST agree on this value, or drain fails its name check and in-flight work is lost with no TLS error anywhere. Ignored while mTLS is off. See [Worker mTLS](#worker-mtls). | `cfdb-worker` | +| `CFDB_TILESET_DISK_CACHE_BYTES` | Byte budget for the API task's local copies of contact-map artifacts. h5py opens a file rather than consuming a byte stream, so under the S3 profile an mcool must be pulled onto local disk before any tile can be read from it. Least-recently-used copies are evicted to stay inside the budget, and a copy currently being read is never removed. Keep this comfortably below the task's ephemeral storage, which it shares with `SYNC_DATA_DIR` and the container image — `cloudformation/backend.yml` sizes that disk with `ApiEphemeralStorageGiB` and passes this value with `TilesetDiskCacheBytes`, so the two move together. | `8589934592` (8 GiB) | +| `CFDB_TILESET_OPEN_MAX` | How many tilesets may be held open at once. Each is an open h5py handle over a potentially multi-GB file, so this is a file-descriptor bound rather than a memory one. A handle evicted while a request is still reading it is closed only once that reader finishes. | `8` | +| `CFDB_TILE_CACHE_BYTES` | Byte budget for the in-process tile cache. Bounded by bytes rather than entry count because a 256×256 float32 tile is roughly 350 KiB base64-encoded, so a plausible-sounding 20,000-entry cap would be several GB on a 2 GiB task. | `268435456` (256 MiB) | +| `CFDB_TILE_THREADS` | Threads used to run tile reads off the event loop. **This bounds concurrency; it does not buy read parallelism** — h5py serializes on a global lock unless linked against a thread-safe HDF5 build, which the published wheels are not, so matrix tile throughput is effectively serial per API process however high this is set. What it does buy is that a multi-second tile read cannot stall `/data` and `/metadata`. The real throughput levers are the tile cache and horizontal scaling. | `2` | +| `CFDB_TILESET_HYDRATE_TIMEOUT_S` | Seconds to wait for a cached artifact to be pulled onto local disk before answering `503 Retry-After`. This cost lands on the first `/tileset_info` for a dataset, not on any tile. | `60` | +| `CFDB_TILESET_MAX_SOURCE_BYTES` | Refuse to materialize a contact map larger than this, checked before a byte is downloaded. `0` disables the guard. Set on the **worker**. The artifact for an `.mcool` is a copy of the upstream file, so an oversized — or mislabelled — source would otherwise fill the worker's disk and fail somewhere far less legible than at admission. | `21474836480` (20 GiB) | | `CFDB_API_URL` | Base URL for the cfdb API | `http://localhost:8000` | | `DATABASE_URL` | MongoDB connection string | `mongodb://127.0.0.1:27017` | | `DATABASE_NAME` | Name of the MongoDB database to use | `cfdb` | @@ -86,6 +100,8 @@ Run `./certs/generate-certs.sh --help` for full usage information. |--------|-------------| | `make mongodb` | Build and start MongoDB with sample data and indexes | | `make api` | Build and start the API container | +| `make api-tiles` | Build and start the API container **with matrix tile serving enabled**, on a cache volume shared with the worker (local dev; see [Serving tiles locally](#serving-tiles-locally)) | +| `make worker-tiles` | Start a containerized LAN worker pool on that same cache volume | | `make materialize-files` | Manually materialize all file metadata (usually done via sync) | | `make materialize-dcc DCC=hubmap` | Materialize a single DCC | | `make schema` | Regenerate the checked-in `schema.graphql` from the Strawberry schema | @@ -866,12 +882,138 @@ curl http://localhost:8000/index/encode/ENCFF123ABC/status # {"ready": false} ``` +### Matrix tile serving (Hi-C contact maps) + +Gosling's `MatrixData` compiles to HiGlass's **native `heatmap` track**, not to `gosling-track`, because HiGlass is better optimized for matrix rendering. The consequence is that `MatrixData.url` must point at a live HiGlass tile API rather than at a file: a heatmap issues thousands of small tile requests and cannot be handed a job id to poll. Neither `/data` nor `/index` can satisfy that, so cfdb serves contact maps through a dedicated pair of endpoints that speak the HiGlass wire protocol directly. `higlass-server` itself is deliberately not deployed — it is a Django application with its own database and tileset registry, which would duplicate what the `files` collection already is. + +No pre-aggregation is required. Both `.mcool` and `.hic` are already multi-resolution pyramids built at creation time (a 4DN mcool typically carries 13 resolutions from 1 kb to 10 Mb; an ENCODE hic carries 17 from 10 bp to 2.5 Mb), so the only preparation is getting the bytes into a shape the tile reader can open. + +**Two channels, deliberately separate.** The HiGlass endpoints are pure readers that **never dispatch a workflow**; a contact map whose artifact has not been built is a `404`. Building it belongs to a separate preparation channel, which a UI drives before it mounts the Gosling spec. + +| Endpoint | Purpose | Dispatches? | +|---|---|---| +| `GET /tileset_info/?d=` | HiGlass tileset metadata | Never | +| `GET /tiles/?d=...` | Tile data | Never | +| `GET /tilesets/{dcc}/{local_id}/status` | Side-effect-free readiness probe | Never | +| `POST /tilesets/{dcc}/{local_id}` | Build the artifact | Yes | + +The tileset **uid is `{dcc}/{local_id}`** — for example `4dn/12a5bc67-e358-42ab-96e1-7893c556ded4`. Both halves must be free of `.`, because a tile id is parsed by splitting on `.` and taking the first field as the uid; no accession in the corpus contains one (4DN uses UUIDs, ENCODE uses `ENCFF…`), and a uid that does is rejected with `400`. + +Per format: + +| Extension | Count | DCC | Handling | +|---|---|---|---| +| `.mcool` | 773 | 4DN | Fetched into the cache as a `tileset` artifact, then read locally | +| `.cool` | 10 | 4DN | Coarsened with `cooler zoomify` first — a flat cooler has no `resolutions` group and cannot be tiled as-is | +| `.hic` | 3,903 | ENCODE | **Not served.** Recognized and refused with `501` — see the limitation below | + +Only 4DN contact maps are servable today. `.hic` is recognized as a contact map — so it is refused with a specific `501` rather than the `404` an unrecognized format gets — but no endpoint serves it. The reasoning is in the limitations below. + +#### Serving tiles + +```bash +curl 'http://localhost:8000/tileset_info/?d=4dn/' +# {"4dn/": {"resolutions": [1000, 2000, ...], "chromsizes": [["chr1", 248956422], ...], +# "max_pos": [3088269832, 3088269832], "tile_size": 256, +# "datatype": "matrix", "name": "4DNFI3EWJW3N.mcool", +# "uuid": "4dn/", "coordSystem": "GRCh38"}} + +curl 'http://localhost:8000/tiles/?d=4dn/.0.0.0&d=4dn/.0.1.0' +# {"4dn/.0.0.0": {"dense": "", "dtype": "float32", "min_value": 0.0, "max_value": 12.0, "size": 1}, ...} +``` + +Both endpoints accept repeatable `d` parameters, capped at 64 per request. The response carries exactly one entry per requested id — a response may safely be built by zipping request against result. A tile position the resolution ladder does not hold, and a tile id that fails to parse, both get an `{"error": ...}` object in their slot instead of tile data, so one bad id cannot blank the whole track. Tile payloads carry no `shape` field — the side length is `sqrt(len(dense))`, which is 256. + +| Code | Description | +|------|-------------| +| 200 | Tileset info or tile data returned | +| 400 | Malformed uid, no `d` parameter, more than 64 `d` parameters, or an unknown DCC | +| 403 | File requires consortium/protected access (HuBMAP) | +| 404 | File not found, not a contact map, or its tileset artifact has not been built yet | +| 501 | This build carries no tile backend (see the `tiles` extra below), or the file is a `.hic` | +| 503 | Workflow subsystem disabled, or the artifact could not be pulled onto local disk in time (`Retry-After`) | + +#### Preparing a contact map + +```bash +curl http://localhost:8000/tilesets/4dn//status # {"ready": false} +curl -X POST http://localhost:8000/tilesets/4dn/ # 202, Location: /jobs/ +curl http://localhost:8000/jobs/ # poll to completion +curl http://localhost:8000/tilesets/4dn//status # {"ready": true} +``` + +`POST` is idempotent: a file whose artifact already exists returns `200 {"ready": true}` rather than creating a no-op job. The probe never dispatches, mirroring the `/data` and `/index` readiness probes. Both answer `501` for a `.hic`, deliberately — a readiness probe that promised `ready: true` for a file the tile endpoint then refuses would be worse than no probe at all. + +| Code | Description | +|------|-------------| +| 200 | Ready — nothing to prepare | +| 202 | Workflow accepted; `Location` points at `/jobs/{id}` | +| 400 | Invalid DCC or path-param shape | +| 403 | File requires consortium/protected access (HuBMAP) | +| 404 | File not found, or not a contact map | +| 409 | File metadata is incomplete (no md5), so no artifact can be addressed for it | +| 429 | Active-workflow ceiling reached (`CFDB_WORKFLOW_MAX_ACTIVE`); `Retry-After` set | +| 501 | This build carries no tile backend, or the file is a `.hic` | +| 503 | Workflow subsystem disabled, or shutting down (`Retry-After`) | + +#### Serving tiles locally + +`make api` builds from `Dockerfile.api`, which carries no clodius, so its tile routes answer `501` — correct while the `tiles` extra stays optional, but it leaves the whole chain untestable through the documented Docker flow. `make api-tiles` layers the tile backend on (installed from the public fork at the same sha `pyproject.toml` pins) and adds the one piece `make api` does not need: a cache volume shared with a worker, because a tileset artifact is built by the worker and then opened locally by the API with h5py. Both targets are interim scaffolding and are retired once `tiles` becomes an ordinary dependency, at which point `make api` covers this on its own. + +The worker reuses the API image rather than `cfdb-wool`: the matrix processor needs `cooler` and `h5py`, which arrive with clodius and are absent from the worker image. + +```bash +make mongodb # database +make worker-tiles # worker FIRST — see below +make api-tiles # API with the tile backend +``` + +Start the worker **before** POSTing a tileset. A dispatch that finds no worker is queued rather than failed, so the job then waits out `CFDB_WORKFLOW_RETRY_INTERVAL_S` (2 min) before the retry tick picks it up — correct behaviour, but a confusing two minutes of `pending` if you were not expecting it. + +The database starts empty, so seed a contact-map document before requesting anything: `/tileset_info` and `/tiles` resolve a file by `{submission, local_id}` and need `filename` (the `.mcool` suffix is what routes it), `md5` (the cache key is content-addressed on it), `access_url`, `file_format.name: "HDF5"`, `dcc.dcc_abbreviation`, and `data_access_level: "public"`, plus `genome_assembly` if you want `coordSystem` in the payload. Note that 4DN's `@@download` URLs answer `403` without credentials; use the open-data S3 mirror (`4dn-open-data-public.s3.amazonaws.com`), which the URL allowlist already covers. + +#### Rendering in Gosling + +```js +{ data: { type: "matrix", url: "http://localhost:8000/tileset_info/?d=4dn/" } } +``` + +Gosling splits that URL into a server and a tileset uid, then drives `/tiles` itself. + +#### Wire compatibility with HiGlass + +The endpoints were checked against two live HiGlass servers — `https://higlass.io/api/v1` and `https://server.gosling-lang.org/api/v1` — and, more usefully, against higlass-server's own tile implementation (`clodius.tiles.cooler`, the v1 module, which the fork still ships alongside `tiles_v2`) run locally over the same `.mcool`. The `tileset_info` and tile payload shapes match: tiles carry exactly `dense`, `dtype`, `min_value`, `max_value` and `size`, 256x256, with no `shape` field. Per-tile `dtype` variation between `float16` and `float32` is not a cfdb quirk — both reference servers do the same thing. + +One representational difference is worth recording, because it looks alarming and is not. **higlass-server sends half of a symmetric matrix; cfdb sends the whole thing.** For a diagonal tile higlass-server emits only the lower triangle, and of each mirrored off-diagonal pair it populates one position and returns zeros for the other. cfdb populates every position with the full correct block. The values themselves agree — cfdb's tiles are bit-identical to a direct `cooler` read at every intrachromosomal resolution. + +This is invisible to the client, for two reasons that were verified rather than assumed: + +- The HiGlass client only ever *requests* one triangle. `tilesToId` asks for the tile at the transposed position with a `mirrored` flag and derives the other half itself, so the mirror-position tiles cfdb would happily serve are never fetched. For every off-diagonal tile the client actually requests, cfdb's bytes are identical to higlass-server's. +- For diagonal tiles the client symmetrizes the payload itself, in `tileDataToPixData`, by copying the lower triangle over the upper one. That is an assignment rather than an accumulation, so on an already-symmetric tile it writes each value back over an identical value and changes nothing. Applying that same step to both servers' diagonal tiles yields identical pixel arrays. The three `extent` modes take different paths through this code and all three agree. + +**Do not start emitting `mirror_tiles`.** The client's `mirrorTiles()` reads `!(mirror_tiles && (mirror_tiles === false || mirror_tiles === 'false'))`, so mirroring is on unless the field is the literal *string* `"false"` — a boolean `false` does not disable it, because `false && ...` short-circuits before the equality test runs. Omitting the field, which is what cfdb does, selects the on behaviour that makes the equivalence above hold, and is what every other server in the ecosystem does. + +**One deliberate divergence from higlass-server: an out-of-ladder tile is a per-tile error object, not an absent key.** `clodius.tiles.cooler` (the v1 module higlass-server runs) silently omits a position past the resolution ladder, returning a batch response shorter than the request. `tiles_v2`'s `CoolerTileset` — what cfdb runs — instead returns exactly one entry per requested id, rendering the out-of-ladder position as `{"error": ..., "error_type": "TileOutOfBounds"}` in that slot. The HiGlass client does not appear to treat the two differently — it draws nothing at a position whether the key is absent or holds an error object — so this has not been observed to change what renders. It is called out because it means a response can now be built by zipping request against result, which used to be unsafe. + +#### Limitations + +- **The clodius tile backend is an optional extra, so deployed images do not carry it.** Tile reading is backed by the [`abdenlab/clodius`](https://github.com/abdenlab/clodius) fork's `tiles_v2` subpackage, installed via `uv sync --extra tiles` from a commit-pinned git reference on the fork's `main` (a sha, deliberately, rather than a floating branch reference — the fork treats `tiles_v2` as unstable and cfdb is its first consumer, so what runs must be exactly what was validated). The extra stays optional because clodius requires Python 3.12 while cfdb still supports 3.11, so `Dockerfile.api` and `Dockerfile.wool` do not install it and images built today answer `501` on the tile routes. Promoting the dependency to an unconditional one — which raises the floor to 3.12 and lets the images carry the backend — is tracked as follow-up work. +- **ENCODE `.hic` contact maps are not served.** The pinned clodius does not carry a `.hic` tileset at all, which makes this an unconditional `501` today. It would stay unconditional even if a future clodius did carry one: `.hic` support (tracked separately, on a different path than the tiles_v2 backend above) is built on `hictkpy`, whose entry points take `str | os.PathLike` and nothing else. There is no URL, HTTP, or S3 surface in the library, so a `.hic` can only be read from a local copy. Making that copy is not viable at this corpus's scale: 3,903 files totalling ~78 TB, a median of 10.3 GB, and a largest single file of 315 GB — past the 200 GiB ceiling on Fargate ephemeral storage, so it could not be cached on an API task at any budget. Serving them needs either a remote-capable reader (hictk tracks S3 support in [paulsengroup/hictk#395](https://github.com/paulsengroup/hictk/issues/395), currently scoped to Cooler rather than `.hic`) or an explicit size cap accepting partial coverage — an 8 GiB cap would reach 43% of the files, 20 GiB 67%, 50 GiB 91%. That is deferred rather than decided. 4DN `.mcool` files are unaffected. +- **`coordSystem` is omitted for the 21 4DN contact maps that carry no `genomeAssembly`.** It is left out entirely rather than emitted as an empty string: HiGlass matches `coordSystem` against chromosome-info tilesets, so a plausible-looking placeholder risks a silently misaligned track, whereas an absent key makes the client fall back to the `chromsizes` array every tileset info carries. The heatmap still renders. +- **ICE-balanced heatmaps are unavailable for the 10 flat `.cool` files.** Coarsening runs without `--balance` because balancing a large flat cooler costs minutes to hours; clodius falls back cleanly to unbalanced when no `weight` column exists. +- **Interchromosomal tiles are served as the concatenated-genome lattice clodius produces**, with no special-casing. +- **Tile payloads are quantized per tile.** clodius emits `float16` whenever a tile's values fit that range and `float32` otherwise, so `dtype` varies from tile to tile within one tileset and a client must read it rather than assume. Verified against direct `cooler` reads: intrachromosomal tiles are bit-identical to the source data round-tripped through the declared dtype, so the only deviation is float16's ~1e-3 relative quantization on tiles that use it. That is well inside a heatmap's colour resolution, but it means tile values are not a substitute for reading the cooler when exact counts matter. +- **The `tileset` artifact for an `.mcool` is a byte-identical copy of the upstream file.** Across all 773 4DN mcools that is a material S3 footprint if every one is ever browsed; the workers stack's `CacheArtifactExpirationDays` (30 days) trims it, at the cost of re-materializing anything revisited after a month. +- **The local tileset copies are per-API-task.** `DesiredCount` is 1 today; scaling the API horizontally multiplies both the disk footprint and the cold-open cost, and an ALB without session affinity scatters one user's tiles across tasks. + ### Preprocessing & indexing workflow Many upstream files are not directly consumable by Gosling Designer without preprocessing (e.g., sort+index for BAM, bgzip+tabix for VCF/GFF/BED). When `/data` or `/index` is called for a format that needs preprocessing and the processed artifact is not yet in cache, the API dispatches a workflow and returns `202 Accepted` with a `Location` header pointing to a job status endpoint. A subsequent call, after the workflow completes, streams the processed artifact from cache (with `Range` support). Both endpoints share a single workflow per source file via a Mongo-backed mutex. The preprocessed artifact is the default response. Clients that want the raw upstream file instead can pass `?raw=true`; on `/index`, `raw=true` serves only an upstream sidecar (e.g., 4DN's `extra_files`) and 404s when none exists. `HEAD` requests never dispatch preprocessing — on cache miss they return 404 so monitoring probes and prefetch tools cannot trigger workflows as a side-effect. Issue a `GET` when you actually want the artifact. +Contact maps add a third artifact kind, `tileset`, alongside `data` and `index`. It is not streamable over `/data` or `/index` and no router serves it: unlike the other two it exists to be *opened* locally by the tile server and read a 256×256 block at a time, not handed back as bytes. It is produced only by `POST /tilesets/{dcc}/{local_id}` — see [Matrix tile serving](#matrix-tile-serving-hi-c-contact-maps). Because contact maps produce no `data` or `index` artifact, `/data` continues to stream them straight from upstream and `/index` continues to report that they have no index. + | Format | Workflow | Cached artifacts | |--------|----------|------------------| | CSV, TSV, bigWig | passthrough — served directly | — | From 726d4df94285102748de3acbb515e45b94c66d31 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 29 Sep 2026 10:25:12 -0400 Subject: [PATCH 14/14] test: Skip clodius-dependent tile tests when clodius is absent CI's Python 3.11 leg installs the dev extra without tiles, since clodius is gated python_version >= '3.12' -- by design, cfdb still supports 3.11 without the tile backend. Two gaps let that leg fail outright instead of skipping cleanly: - Five test modules imported numpy unconditionally at module scope (tests/fixtures/coolers.py and its transitive importers, plus test_tiles.py and test_tilesets/test_service.py directly), so collection itself raised ModuleNotFoundError before any skip logic could run. Two of the five already guarded on cooler/clodius but not numpy, so the bare numpy import failed first regardless. - 25 tests across test_tilesets/test_backend.py, test_api/test_lifespan_tilesets.py, and test_tilesets_router.py exercise the "clodius present" path against the real backend rather than a mock, and were never actually reached on 3.11 before now -- pytest aborts the whole run on collection errors, which the numpy ImportError always triggered first, masking these underneath it. Guard both classes of test with pytest.importorskip("clodius") (or "numpy" where that is the more precise dependency), leaving the absent-clodius degradation tests -- the ones that matter most for a 3.11 build -- to keep running unconditionally. --- tests/fixtures/coolers.py | 7 ++++++- tests/integration/test_tiles_e2e.py | 19 +++++++++++-------- tests/integration/test_tiles_s3_profile.py | 5 ++++- tests/test_api/test_lifespan_tilesets.py | 8 ++++++++ tests/test_tiles.py | 6 +++++- tests/test_tilesets/test_backend.py | 17 ++++++++++++++++- tests/test_tilesets/test_service.py | 6 +++++- tests/test_tilesets_router.py | 13 +++++++++++++ 8 files changed, 68 insertions(+), 13 deletions(-) diff --git a/tests/fixtures/coolers.py b/tests/fixtures/coolers.py index cd3ef96..ed5e82e 100644 --- a/tests/fixtures/coolers.py +++ b/tests/fixtures/coolers.py @@ -23,9 +23,14 @@ from pathlib import Path -import numpy as np import pytest +#: numpy arrives transitively with clodius/cooler, which are only +#: installed for the ``tiles`` extra (Python >= 3.12). Guard here rather +#: than let a bare ``import numpy`` fail collection outright on 3.11, +#: since every caller of this module already expects a skip in that case. +np = pytest.importorskip("numpy") + #: Tiling genome, 3000 bp total. Deliberately tiny relative to the bin #: sizes below: a cooler tile is 256 bins wide, so a realistic 1 kb binsize #: on this genome would put the whole thing inside a single bin of a single diff --git a/tests/integration/test_tiles_e2e.py b/tests/integration/test_tiles_e2e.py index c83f1f6..db16a04 100644 --- a/tests/integration/test_tiles_e2e.py +++ b/tests/integration/test_tiles_e2e.py @@ -16,10 +16,20 @@ import asyncio import base64 -import numpy as np import pytest import pytest_asyncio +# numpy arrives transitively with clodius/cooler, which are only installed +# for the tiles extra (Python >= 3.12). tests.fixtures.coolers guards its +# own numpy import the same way, but this module's own bare import would +# fail collection before ever reaching that one, so it needs the same +# guard directly. The tile backend itself loads lazily, so nothing else +# below pulls in cooler or clodius at import time -- guard the whole +# module here instead. +np = pytest.importorskip("numpy") +pytest.importorskip("cooler") +pytest.importorskip("clodius.tiles_v2.cooler") + from cfdb import api from cfdb.api.routers.tiles import tiles, tileset_info from cfdb.services import locks @@ -33,13 +43,6 @@ from tests.fixtures.remote_cache import FakeRemoteCache from tests.test_workflows import FIXTURE_MD5, FIXTURE_MD5_ALT -# None of the imports above pull in cooler or clodius at import time (the -# tile backend loads lazily), so on a checkout without the tiles extra the -# failure would otherwise land inside the first fixture as an error rather -# than a skip. Guard the whole module instead. -pytest.importorskip("cooler") -pytest.importorskip("clodius.tiles_v2.cooler") - pytestmark = pytest.mark.integration UID = "4dn/4DNFIABC123" diff --git a/tests/integration/test_tiles_s3_profile.py b/tests/integration/test_tiles_s3_profile.py index 8337fdc..aa3777c 100644 --- a/tests/integration/test_tiles_s3_profile.py +++ b/tests/integration/test_tiles_s3_profile.py @@ -19,10 +19,13 @@ import asyncio import base64 -import numpy as np import pytest import pytest_asyncio +# numpy arrives transitively with clodius/cooler, which are only installed +# for the tiles extra (Python >= 3.12); guard it alongside the others +# rather than let a bare import fail collection outright on 3.11. +np = pytest.importorskip("numpy") pytest.importorskip("cooler") pytest.importorskip("moto") diff --git a/tests/test_api/test_lifespan_tilesets.py b/tests/test_api/test_lifespan_tilesets.py index 89594d4..77d65c9 100644 --- a/tests/test_api/test_lifespan_tilesets.py +++ b/tests/test_api/test_lifespan_tilesets.py @@ -79,6 +79,9 @@ async def test_lifespan_should_build_the_tile_service_when_clodius_is_present( ``api.tileset_service`` should be populated, so the tile routes serve rather than answering 501. """ + # Arrange + pytest.importorskip("clodius") + # Act async with main.lifespan(main.app): service = api.tileset_service @@ -130,6 +133,7 @@ async def test_lifespan_should_close_and_clear_the_tile_service_on_teardown( leak ``_reset_api_globals`` exists to prevent. """ # Arrange + pytest.importorskip("clodius") closed: list[bool] = [] # Act @@ -219,6 +223,7 @@ async def test_lifespan_should_propagate_config_to_the_tile_service( ``CFDB_TILESET_*`` / ``CFDB_TILE_*`` env knobs. """ # Arrange + pytest.importorskip("clodius") spy = mocker.spy(main, "build_tileset_service") # Act @@ -254,6 +259,7 @@ async def test_lifespan_should_null_all_globals_when_service_close_raises( the next app instantiation. """ # Arrange + pytest.importorskip("clodius") original_close = None # Act @@ -296,6 +302,7 @@ async def test_lifespan_should_build_the_tile_service_when_the_cache_is_s3( tiles are served. """ # Arrange + pytest.importorskip("clodius") assert stub_lifespan.kind == "s3-cached" # Act @@ -325,6 +332,7 @@ async def test_lifespan_should_wire_the_cache_provider_to_late_bind( service pointed at a stale backend. """ # Arrange + pytest.importorskip("clodius") from cfdb.tilesets.errors import TilesetNotReady doc = { diff --git a/tests/test_tiles.py b/tests/test_tiles.py index 2e2560f..d60c49e 100644 --- a/tests/test_tiles.py +++ b/tests/test_tiles.py @@ -6,12 +6,16 @@ import base64 import itertools -import numpy as np import pytest from fastapi import HTTPException from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st +# numpy arrives transitively with clodius, which is only installed for the +# tiles extra (Python >= 3.12). Guard rather than let a bare import fail +# collection outright on 3.11. +np = pytest.importorskip("numpy") + from cfdb import api from cfdb.api.routers._helpers import PATH_PARAM_MAX_LEN from cfdb.api.routers.tiles import MAX_TILE_IDS, tiles, tileset_info diff --git a/tests/test_tilesets/test_backend.py b/tests/test_tilesets/test_backend.py index ffb7bde..f9ee5b2 100644 --- a/tests/test_tilesets/test_backend.py +++ b/tests/test_tilesets/test_backend.py @@ -3,6 +3,7 @@ from __future__ import annotations import sys +from importlib.util import find_spec import pytest @@ -61,6 +62,7 @@ def test_should_resolve_the_clodius_names(self): clodius exception types the per-tile/500 split keys on. """ # Arrange + pytest.importorskip("clodius") from clodius.core.errors import TileOutOfBounds, TilesetError from clodius.tiles_v2.cooler import CoolerTileset @@ -109,6 +111,9 @@ def test_should_return_true_when_clodius_is_present(self): Then: It should return True. """ + # Arrange + pytest.importorskip("clodius") + # Act result = is_available() @@ -144,7 +149,17 @@ def test_should_raise_when_the_hic_module_is_missing(self, mocker): class TestIsTileError: - """The classification that splits per-tile payloads from 500s.""" + """The classification that splits per-tile payloads from 500s. + + Every test here loads a real backend, so the whole class needs clodius + installed — unlike TestLoadBackend's absent-module tests, there is no + sys.modules trick that stands in for a real error hierarchy to + classify against. + """ + + pytestmark = pytest.mark.skipif( + find_spec("clodius") is None, reason="requires the clodius tile backend" + ) def test_should_recognize_a_clodius_tile_error(self): """Test that a clodius per-tile error is classified as such. diff --git a/tests/test_tilesets/test_service.py b/tests/test_tilesets/test_service.py index 5dacc31..480bcb2 100644 --- a/tests/test_tilesets/test_service.py +++ b/tests/test_tilesets/test_service.py @@ -5,11 +5,15 @@ import asyncio import base64 -import numpy as np import pytest from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st +# numpy arrives transitively with clodius, which is only installed for the +# tiles extra (Python >= 3.12). Guard rather than let a bare import fail +# collection outright on 3.11. +np = pytest.importorskip("numpy") + from cfdb.tilesets.errors import ( TilesetHydrationTimeout, TilesetIdentityIncomplete, diff --git a/tests/test_tilesets_router.py b/tests/test_tilesets_router.py index 9772045..1a64f6e 100644 --- a/tests/test_tilesets_router.py +++ b/tests/test_tilesets_router.py @@ -105,6 +105,7 @@ async def test_should_report_not_ready_when_no_artifact_is_cached( never dispatching. """ # Arrange + pytest.importorskip("clodius") _, mock_db, executor, _ = prep_env mock_db.file.docs = [_file_doc()] @@ -232,6 +233,7 @@ async def test_should_503_when_the_workflow_subsystem_is_disabled( signals the same condition. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object(api, "cache", None) @@ -255,6 +257,7 @@ async def test_should_409_a_document_with_no_md5(self, prep_env): retry will fix and which is not the server's fault. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, _ = prep_env doc = _file_doc() del doc["md5"] @@ -310,6 +313,7 @@ async def test_should_503_a_hic_when_the_workflow_subsystem_is_disabled( reordering the guards would silently change the answer. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [ _file_doc(submission="encode", local_id="ENCFF123ABC", filename="c.hic") @@ -338,6 +342,7 @@ async def test_should_500_with_a_generic_detail_when_the_probe_raises( the exception text to an unauthenticated caller. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object( @@ -370,6 +375,7 @@ async def test_should_dispatch_and_point_at_the_job_endpoint(self, prep_env): for /data and /index preprocessing. """ # Arrange + pytest.importorskip("clodius") _, mock_db, executor, _ = prep_env mock_db.file.docs = [_file_doc()] @@ -454,6 +460,7 @@ async def test_should_429_when_the_admission_ceiling_is_reached( /index rather than queuing past it. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object( @@ -485,6 +492,7 @@ async def test_should_503_while_the_executor_is_draining(self, prep_env): a reorder would silently break. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object( @@ -509,6 +517,7 @@ async def test_should_409_when_the_executor_declines_the_file(self, prep_env): this file, not a server fault. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object( @@ -563,6 +572,7 @@ async def test_should_409_a_document_with_no_md5_without_dispatching( output could never be addressed must not be queued. """ # Arrange + pytest.importorskip("clodius") _, mock_db, executor, _ = prep_env doc = _file_doc() del doc["md5"] @@ -591,6 +601,7 @@ async def test_should_503_when_the_workflow_subsystem_is_disabled( build. """ # Arrange + pytest.importorskip("clodius") _, mock_db, executor, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object(api, "cache", None) @@ -618,6 +629,7 @@ async def test_should_500_with_a_generic_detail_when_the_probe_raises( the exception text to an unauthenticated caller. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object( @@ -650,6 +662,7 @@ async def test_should_500_with_a_generic_detail_when_the_executor_raises( it up as a 409 or 503 a client would retry forever. """ # Arrange + pytest.importorskip("clodius") _, mock_db, _, mocker = prep_env mock_db.file.docs = [_file_doc()] mocker.patch.object(