Add HiGlass-compatible tileset endpoints for Hi-C contact maps — Closes #82

CLAUDE.md

@@ -0,0 +1 @@
+Start with @README.md

Dockerfile.api-tiles

@@ -0,0 +1,36 @@
+# Local-development image: the API image plus the clodius tile backend.
+#
+# Interim scaffolding for issue #82. The clodius `tiles` extra is still
+# optional (its fork requires Python 3.12 while cfdb supports 3.11), so
+# `Dockerfile.api` does not carry it and the images it builds answer 501
+# on every tile route (see the `tiles` extra in pyproject.toml). That is
+# the right production behaviour today, but it makes the tile endpoints
+# untestable through the documented `make mongodb` / `make api` flow,
+# which is the only place the whole chain -- workflow dispatch, artifact
+# cache, h5py read -- runs together.
+#
+# So this layers clodius onto the image `make api` already built rather
+# than duplicating that build, installing it straight from the public
+# fork at the same sha `[tool.uv.sources]` pins.
+#
+# DELETE THIS FILE once the `tiles` extra becomes an ordinary dependency
+# (which raises the Python floor to 3.12): `Dockerfile.api`'s existing
+# `pip install .` then carries it.
+FROM api
+
+USER root
+
+# Installed as its own layer, before the app, so that iterating on cfdb
+# does not re-resolve clodius's 25-package closure (pysam, oxbow, apsw,
+# polars, scipy, ...) on every rebuild. Keep this sha in lockstep with
+# the `[tool.uv.sources]` pin in pyproject.toml.
+RUN pip install --no-cache-dir \
+    "clodius @ git+https://github.com/abdenlab/clodius@aac90219031e8c05cf6d5a916628531a72cfc461"
+
+# clodius imports matplotlib transitively, and the image's `app` user
+# has no home directory, so matplotlib warns twice and rebuilds its font
+# cache on every start. Point it at a writable path.
+ENV MPLCONFIGDIR=/tmp/matplotlib
+
+USER app
+WORKDIR /app/src

Dockerfile.mongodb

@@ -1,9 +1,26 @@
 FROM mongo:latest
 
-# Copy database dump and scripts
-COPY database/ /data/database/
 COPY scripts/create-indexes.js /scripts/create-indexes.js
 
+# The sample dump is NOT baked into the image. It used to be — `COPY
+# database/ /data/database/` — but `database/` is gitignored (the 4DN
+# backup was removed from the repo in 2fc98f8), so on a clean checkout
+# the directory does not exist at all and the COPY failed the build
+# outright with `"/database": not found`. The build only appeared to
+# work on machines where a stray file such as `.DS_Store` happened to
+# keep the directory alive.
+#
+# Mounting the dump at run time instead (see the `mongodb` target in the
+# Makefile) fixes that and is the better shape regardless: the image no
+# longer varies with whatever a developer happens to have on disk, and
+# obtaining a dump later is a container restart rather than a rebuild.
+# The directory is created here so `mongorestore` always has a valid
+# target — it exits 1 on a path that does not exist, which under
+# `set -e` below would take the container down — and so running the
+# image with no mount at all degrades to "empty database" rather than
+# to a crash.
+RUN mkdir -p /data/database
+
 # Create startup script (development mode only)
 COPY <<'EOF' /startup.sh
 #!/bin/bash
@@ -22,9 +39,19 @@ until mongosh --eval "db.adminCommand('ping')" >/dev/null 2>&1; do
 done
 echo "MongoDB started."
 
-# Restore database
-echo "Restoring database..."
-mongorestore --gzip /data/database
+# Restore the dump, if one was mounted. `set -e` is deliberately kept:
+# a failed restore of a dump that IS present should stop the container
+# rather than leave a half-loaded database looking healthy. The empty
+# case is not a failure though, so it is reported as such instead of
+# being left to mongorestore's "0 document(s) restored" trace, which
+# reads like something went wrong.
+if [ -n "$(ls -A /data/database 2>/dev/null)" ]; then
+    echo "Restoring database from /data/database..."
+    mongorestore --gzip /data/database
+else
+    echo "No dump at /data/database — starting with an empty database."
+    echo "Populate it with 'curl -X POST http://localhost:8000/sync'."
+fi
 
 # Create indexes
 echo "Creating indexes..."

Makefile

@@ -13,8 +13,13 @@ mongodb:
 	@docker rm mongodb 2>/dev/null || true
 	@echo "Building MongoDB image..."
 	docker build -t cfdb-mongodb -f Dockerfile.mongodb .
+	@# database/ is gitignored and empty on a clean checkout. It is a
+	@# drop-in point for an optional mongodump, mounted read-only rather
+	@# than baked into the image (see Dockerfile.mongodb). Created here so
+	@# the mount source always exists and so the drop-in point is visible.
+	@mkdir -p database
 	@echo "Starting MongoDB container..."
-	docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 cfdb-mongodb
+	docker run -d --name mongodb --network cvh-backend-network --network-alias cvh-backend -p 27017:27017 -v "$(CURDIR)/database:/data/database:ro" cfdb-mongodb
 	@echo "MongoDB container starting on port 27017. Check logs with: docker logs -f mongodb"
 
 build-materialize:
@@ -47,6 +52,62 @@ api:
 	docker run -d --name api --network cvh-backend-network --network-alias cvh-backend -p 8000:8000 -e SYNC_DATA_DIR=/tmp/sync-data api
 	@echo "API container is up and running on port 8000 (http://0.0.0.0:8000/metadata)."
 
+# --- Local-dev matrix tile serving (issue #82) -----------------------
+#
+# `make api` builds an image with no clodius, so its tile routes answer
+# 501 -- correct for production while the `tiles` extra stays optional,
+# but it leaves the whole tile chain untestable through the documented
+# Docker flow. These targets layer clodius on (installed from the public
+# fork at the sha pyproject.toml pins) and add the piece `make api` does
+# not need: a cache volume shared with a worker, since a tileset
+# artifact is built by the worker and then opened by the API.
+#
+# Retire both once `tiles` becomes an ordinary dependency (raising the
+# Python floor to 3.12) -- `make api` then covers this on its own.
+
+api-tiles:
+	make network
+	@docker stop api 2>/dev/null || true
+	@docker rm api 2>/dev/null || true
+	@echo "Building the API image..."
+	docker build -t api -f Dockerfile.api .
+	@echo "Layering the clodius tile backend on top..."
+	docker build -t api-tiles -f Dockerfile.api-tiles .
+	@# The image runs as `app`, but a freshly created named volume is
+	@# owned by root, so the lifespan's mkdir under SYNC_DATA_DIR fails
+	@# with PermissionError and the API 500s on every workflow route.
+	@docker volume create cfdb-sync >/dev/null
+	@docker run --rm --user 0 --entrypoint chown -v cfdb-sync:/tmp/sync-data api-tiles -R app:app /tmp/sync-data
+	@echo "Starting the API container with matrix tile serving enabled..."
+	docker run -d --name api --network cvh-backend-network --network-alias cvh-backend -p 8000:8000 \
+		-e SYNC_DATA_DIR=/tmp/sync-data \
+		-e WORKFLOW_POOL_NAMESPACE=$${WORKFLOW_POOL_NAMESPACE:-cfdb-workers} \
+		-v cfdb-sync:/tmp/sync-data \
+		api-tiles
+	@echo "API up on port 8000."
+	@# Start the worker FIRST next time: the API dispatches on request, and a
+	@# dispatch that finds no worker is queued rather than failed, so the job
+	@# then waits out CFDB_WORKFLOW_RETRY_INTERVAL_S (2 min) before running.
+	@echo "Run 'make worker-tiles' before POSTing a tileset, or the first job waits ~2 min for the retry tick."
+
+# The worker that builds tileset artifacts. It reuses the API image
+# rather than `cfdb-wool`, because the matrix processor needs cooler and
+# h5py, which arrive with clodius and are absent from the worker image.
+# It shares the API's cache volume: the worker writes the artifact, the
+# API opens it locally with h5py.
+worker-tiles:
+	make network
+	@docker stop worker 2>/dev/null || true
+	@docker rm worker 2>/dev/null || true
+	@echo "Starting a containerized LAN worker pool (namespace=$${WORKFLOW_POOL_NAMESPACE:-cfdb-workers})..."
+	docker run -d --name worker --network cvh-backend-network \
+		-e SYNC_DATA_DIR=/tmp/sync-data \
+		-e WORKFLOW_POOL_NAMESPACE=$${WORKFLOW_POOL_NAMESPACE:-cfdb-workers} \
+		-e WORKFLOW_WORKER_COUNT=$${WORKFLOW_WORKER_COUNT:-1} \
+		-v cfdb-sync:/tmp/sync-data \
+		api-tiles python -m cfdb.workflows.worker_lan
+	@echo "Worker container is up. Check logs with: docker logs -f worker"
+
 schema:
 	@echo "Regenerating schema.graphql from the Strawberry schema..."
 	uv run python scripts/export_schema.py