diff --git a/.dockerignore b/.dockerignore index 725bc7f..2e5d239 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,9 @@ # Gitignored directories node_modules/ +# Native kernel libraries built in place (the image compiles its own) +/src/foldmatch/_native/_fmsw_* + # Directories /dist/ /tests/ diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index da502a0..8577a73 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -14,6 +14,10 @@ jobs: name: Test on Python ${{ matrix.python-version }} runs-on: ["self-hosted", "buildchain"] timeout-minutes: 120 + env: + # The editable install compiles the native kernel in place (hatch_build.py); a failed compile is an error here, + # never a pure-Python install or skipped kernel tests. + FOLDMATCH_REQUIRE_NATIVE: "1" strategy: matrix: python-version: ["3.12"] @@ -26,6 +30,18 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential pkg-config libzstd-dev + # fmsw.c needs GCC >= 12 or Clang >= 14 (older GCC stops at its #error). The self-hosted runner's gcc is not + # pinned, so an older one is replaced by gcc-12 for this job; CC reaches the editable install's build hook. + - name: C compiler for the native kernel (GCC >= 12) + run: | + cc_bin=gcc + if [ "$(gcc -dumpversion | cut -d. -f1)" -lt 12 ]; then + sudo apt-get install -y gcc-12 + cc_bin=gcc-12 + fi + "$cc_bin" --version + echo "CC=$cc_bin" >> "$GITHUB_ENV" + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: @@ -37,10 +53,142 @@ jobs: - name: Run tests run: hatch test + # Native Smith-Waterman kernel (PLAN section 7, P3.7) on every supported compiler and ISA. The DP golden hash (from + # q fp16 through the fixed-order reference similarity) must be identical in every job: tests/test_swkern.py checks + # it for every ISA library of the machine. The production BLAS path (OpenBLAS on Linux, Accelerate on macOS) must + # stay within the S16 tolerance of it: the test module checks the dispatched ISA, selftest() every other one. + kernel: + name: ${{ matrix.job }} (${{ matrix.cc }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - { job: kernel-x86, os: ubuntu-latest, cc: gcc, isas: "sse2 avx2 generic" } + - { job: kernel-x86, os: ubuntu-latest, cc: clang, isas: "sse2 avx2 generic" } + - { job: kernel-arm64-linux, os: ubuntu-24.04-arm, cc: gcc, isas: "neon generic" } + - { job: kernel-macos, os: macos-15, cc: cc, isas: "neon generic" } + env: + CC: ${{ matrix.cc }} + PYTHONPATH: src + FOLDMATCH_REQUIRE_NATIVE: "1" + FOLDMATCH_SLOW_TESTS: "1" + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install test dependencies + run: pip install "numpy>=2,<3" pytest==9.0.2 threadpoolctl==3.6.0 importlib-metadata==8.7.1 + + - name: Compiler version + run: $CC --version + + - name: Require AVX2 on the x86 runner (the avx2 library must run, not be skipped) + if: matrix.job == 'kernel-x86' + run: grep -qw avx2 /proc/cpuinfo + + - name: Build every ISA library of this machine + run: python -m foldmatch._native.build + + - name: Kernel tests + run: python -m pytest tests/test_swkern.py -rs + + # The esm-codes registry golden digests (plan P4 acceptance: identical on Linux and macOS). The test module needs + # only the standard library and foldmatch.search.scoring_system, so it runs on every kernel runner. + - name: Scoring-system golden digests + run: python -m pytest tests/test_scoring_system_golden.py -rs + + - name: selftest() for each ISA + run: | + for isa in ${{ matrix.isas }}; do + FOLDMATCH_SW_ISA=$isa python -m foldmatch.search.swkern + done + + # The kernel test module against ASan + UBSan builds of the sse2, avx2 and generic libraries (undersized workspaces + # and bad arguments included). The sanitizer runtime is preloaded because python itself is not instrumented. + kernel-sanitize: + name: kernel-sanitize (ASan + UBSan, gcc) + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + CC: gcc + PYTHONPATH: src + FOLDMATCH_REQUIRE_NATIVE: "1" + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install test dependencies + run: pip install "numpy>=2,<3" pytest==9.0.2 threadpoolctl==3.6.0 importlib-metadata==8.7.1 + + - name: Build the libraries with ASan and UBSan + run: | + python - <<'EOF' + from foldmatch._native import build + build.build(extra_flags=["-fsanitize=address,undefined", "-fno-sanitize-recover=all", + "-fno-omit-frame-pointer", "-g"]) + EOF + + - name: Kernel tests under the sanitizers + run: | + export LD_PRELOAD="$(gcc -print-file-name=libasan.so)" + export ASAN_OPTIONS=detect_leaks=0:abort_on_error=1 + export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1 + python -m pytest tests/test_swkern.py -rs + + # Every PR: the sdist, a wheel built from it, the wheel and sdist content tests, selftest() from the wheel in a clean + # venv, and the build-hook tests (pure-wheel fallback, FOLDMATCH_REQUIRE_NATIVE, editable build, platform tag). + packaging: + name: packaging (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-15] + env: + FOLDMATCH_REQUIRE_NATIVE: "1" + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install build and test tools + run: >- + pip install build hatchling editables packaging "numpy>=2,<3" pytest==9.0.2 + importlib-metadata==8.7.1 + + - name: Build the sdist, then the wheel from the sdist + run: python -m build --outdir dist . + + - name: Packaging tests + env: + FOLDMATCH_DIST_DIR: dist + PYTHONPATH: src + run: python -m pytest tests/test_packaging.py -rs + hatch-build: - name: Build to PyPI + name: Build the sdist needs: - hatch-test + - kernel + - kernel-sanitize + - packaging runs-on: ubuntu-latest if: github.event_name == 'release' steps: @@ -55,8 +203,10 @@ jobs: - name: Install Hatch run: pip install hatch + # Wheels come from the cibuildwheel matrix below: a plain `hatch build` wheel would carry a linux_x86_64 tag, + # which PyPI refuses. - name: Build distribution - run: hatch build + run: hatch build -t sdist - name: Store the distribution packages uses: actions/upload-artifact@v4 @@ -64,9 +214,87 @@ jobs: name: python-package-distributions path: dist/ + # Platform wheels (D19): Linux x86_64 and aarch64 in manylinux_2_28 images (auditwheel repairs and retags them), + # macOS arm64 and x86_64 (cross-built on the arm64 runner; delocate checks and retags them). The build hook tags the + # raw wheel py3-none-; no manylinux tag is hard-coded. + build-wheels: + name: Wheels (${{ matrix.name }}) + needs: + - hatch-test + - kernel + - kernel-sanitize + - packaging + if: github.event_name == 'release' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - { name: linux-x86_64, os: ubuntu-latest, archs: x86_64 } + - { name: linux-aarch64, os: ubuntu-24.04-arm, archs: aarch64 } + - { name: macos, os: macos-15, archs: "arm64 x86_64" } + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@v3.1.4 + env: + CIBW_BUILD: "cp312-*" + CIBW_SKIP: "*-musllinux_*" + CIBW_ARCHS: ${{ matrix.archs }} + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_ENVIRONMENT: FOLDMATCH_REQUIRE_NATIVE=1 + # gcc = the image's gcc-toolset compiler (GCC >= 12), chosen explicitly over any system cc + CIBW_ENVIRONMENT_LINUX: FOLDMATCH_REQUIRE_NATIVE=1 CC=gcc + CIBW_ENVIRONMENT_MACOS: FOLDMATCH_REQUIRE_NATIVE=1 MACOSX_DEPLOYMENT_TARGET=11.0 + with: + output-dir: wheelhouse + + # The wheels are checked here, not by cibuildwheel's test step, which would install the full dependency set + # (torch, esm): the kernel check needs only numpy and importlib-metadata (imported by foldmatch/__init__), so the + # wheel goes into a clean venv with --no-deps that borrows those two. + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Wheel contents and selftest() in a clean venv + env: + FOLDMATCH_DIST_DIR: wheelhouse + run: | + pip install packaging "numpy>=2,<3" pytest==9.0.2 importlib-metadata==8.7.1 + python -m pytest tests/test_packaging.py -rs -k "test_wheel_contents or test_wheel_selftest_in_clean_venv" + + - name: Set up an x86_64 Python 3.12 (Rosetta) + if: runner.os == 'macOS' + id: python-x64 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + architecture: x64 + + - name: selftest() of the macOS x86_64 wheel under Rosetta + if: runner.os == 'macOS' + env: + FOLDMATCH_DIST_DIR: wheelhouse + run: | + PY="${{ steps.python-x64.outputs.python-path }}" + "$PY" -m pip install packaging "numpy>=2,<3" pytest==9.0.2 importlib-metadata==8.7.1 + "$PY" -m pytest tests/test_packaging.py -rs -k "test_wheel_selftest_in_clean_venv" + + - name: Store the wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.name }} + path: wheelhouse/*.whl + push-image: needs: - hatch-build + - build-wheels name: Push image to harbor uses: ./.github/workflows/_workflow-docker.yaml @@ -84,10 +312,16 @@ jobs: id-token: write steps: - - name: Download all the dists + - name: Download the sdist uses: actions/download-artifact@v4 with: name: python-package-distributions path: dist/ + - name: Download the wheels + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + merge-multiple: true + path: dist/ - name: Publish distribution 📦 to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index a1cd894..9fa1cd4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,9 @@ /.pypi.rc __pycache__ /tests/resources/tmp -/.claude \ No newline at end of file +/.claude +# native kernel libraries built in place (python -m foldmatch._native.build) +/src/foldmatch/_native/_fmsw_* +/src/foldmatch/_native/._fmsw_* +# cibuildwheel output +/wheelhouse/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ca4fba..e551ea1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,13 +10,1362 @@ While the major version is `0`, breaking changes are released in a minor bump. ### Added -- **`--max-evalue`** on `fm-search query sequences` and `fm-search query db`: - drops Stage-2 hits whose database E-value is above the threshold. Defaults to - `1e-3`, so a default search now reports only statistically significant - alignments; pass `--max-evalue inf` to keep every hit. Because the E-value is - a significance quantity, the option turns the significance pass on even when - neither `evalue` nor `bits` is in `--format-output` (and so, under - `--significance-mode default`, requires the 11/1 gap penalties). +- **`--leiden-iterations`** on `fm-search cluster`: the number of Leiden + iterations, a fixed count (default 10, at least 1; below 1 it is refused + with exit status 2 before the database is loaded). Fewer iterations are + faster, with somewhat noisier partitions: on a 1M-chain graph + (`--max-neighbors` 50), the Leiden step took 26.9-28.4 s at 2 iterations + against 77.7-78.5 s at 10 (one thread, NERSC CPU nodes), and two seeds + agreed at a mean AMI of 0.7548 against 0.7659 (10 pairs of 5 seeds). + `--method leiden` only: refused under `--method greedy`, with + `--resolution` and `--seed`, in one message. +- **`--cleanup-embeddings`** on `fm-search build structures` and `fm-search + build sequences`: deletes the build's intermediate embedding shards from + `--tmp-embedding-folder` once the database is built. Off by default, so the + shards are kept as before. +- **ESM3 residue-code scoring: the building blocks.** The parts of the + opt-in Stage-2 scoring `esm-codes-v1`, an alternative to BLOSUM62: the + residue-code encoders + (`foldmatch.model.residue_code_encoder`), which turn ESM3 per-residue + embeddings into 8-byte residue codes and query encodings with the files of + the Hugging Face repo `rcsb/foldmatch-esm-codes`; a native C + Smith-Waterman kernel with its Python wrapper (`foldmatch.search.swkern`); + and the versioned scoring system (`foldmatch.search.scoring_system`), a + frozen registry entry `esm-codes-v1` that names every number and + convention behind the scores, including the bundle's repo, pinned revision + and file sha256s (about 26 MB), with its E-value statistics + (`foldmatch.search.code_statistics`). A changed value will always be a new + system name. The database build (`--residue-codes`) and the query + (`fm-search query sequences --scoring esm-codes`), both below, use them; + existing commands, BLOSUM62 scoring and its output are unchanged; and the + Stage-2 aligner is `foldmatch.search.code_alignment` + (`align_candidates_esm_codes`). `python -m foldmatch.search.swkern` runs the kernel's + self-test (reference hash and the BLAS similarity path) and prints a JSON + report. +- **`fm-search build sequences --residue-codes`** (opt-in, off by default). + The ESM3 pass that computes the chain embeddings also computes an 8-byte + ESM3 code per residue with the `esm-codes-v1` encoders, and the build + stores them next to the database: `.rescodes.bin` (8 bytes per + residue), a SQLite index `.rescodes.idx` (about 60 bytes per + entry) and a manifest `.rescodes.json`, written last, that + records the scoring system, the ESM3 and bundle pins, the entry and + residue counts, the sha256 of the codes, and how the pass computed them + (device type and ESM3 precision, number of ranks, software versions; + the embedding shards record the same). The first such build downloads + the scoring bundle (about 26 MB; `download-models` fetches it in advance). + Before ESM3 is loaded, and on every rank alike, the build refuses FASTA + input the codes cannot be lined up with (duplicate ids, `<` or `>` in a + sequence, non-ASCII characters), any existing output (`.rescodes.*` + included) and a bundle that cannot be fetched or verified. After the pass + the codes are checked against the sequence store, the pre-flight counts + and the FAISS index; if a check fails, only the code store is removed, so + the database still serves BLOSUM62 searches, and the embedding shards are + kept for `build residue-codes`. FASTA (sequence) databases only. Such a + database is searched with its codes by `query sequences --scoring + esm-codes` (below), and with BLOSUM62 as before. Without the option the + build and all its files are unchanged. +- **`fm-search build residue-codes DB_FOLDER INDEX_NAME --from-shards DIR`** + rebuilds and validates the residue-code store of a database from the + embedding shards its `build sequences --residue-codes` run kept in + `--tmp-embedding-folder` (they are kept unless `--cleanup-embeddings`), + without running ESM3 again; the codes are byte-identical to the original + build's, and the manifest's `min_res_n` and build information come from + the shards, so they describe the pass that computed the codes. `--run-id` + picks the run when `DIR` holds the shards of several. `--force` replaces + an existing store, but only once the new codes have passed validation: the + new store is written next to the old one (room for both is needed + meanwhile), and a refused or interrupted rebuild leaves the old one as it + was. Shards without codes, of another encoding, or whose entry count + differs from the database's are refused before anything is changed. +- **A split sequence build for multi-node GPU runs: `fm-search build + sequences-embed`, `build sequences-index` and `build sequences-seal`.** + After its ESM3 pass, `build sequences` builds the FAISS index, the sequence + store and the residue-code store in one process, while the other nodes of + the job stay allocated: in a build of 30 million sequences on 8 GPU nodes + this phase took 7.4 of the 20.6 hours. The split build ends the GPU job with + the pass and builds the database in a job of its own on one CPU node; `build + sequences` itself is unchanged. Step 1, `build sequences-embed`, takes the + input and pass options of `build sequences` (`--fasta-file`, `--min-res-n`, + `--accelerator`, `--devices`, `--num-nodes`, `--strategy`, `--batch-size`, + `--num-workers`, `--residue-codes`), runs its pre-flight and ESM3 pass on + every rank and keeps the embedding shards in `--shards-folder` (the + `--tmp-embedding-folder` of `build sequences`, renamed); rank 0 then checks + them and writes, last, a run record `emb_.run.json` beside them, + which holds the FASTA's size, sha256 and pre-flight counts, each shard's + rows and size, the pass and `--output-db`, the database step 2 is to build + (refused if any of its files exists). Step 1 writes no database file and + logs the command of step 2. Step 2, `build sequences-index`, runs in one + process, without ESM3, a GPU or a download: it builds from the run in + `--from-shards` (`--run-id` picks one when the folder holds more than one + run, unfinished ones included), with the index options of `build sequences` + (`--index-type`, `--ivf-nlist`, `--ivf-nprobe`, `--load-batch-size`, + `--use-gpu-index`), into `--output-db`, which may differ from the recorded + one. Its `--residue-codes` defaults to what the run computed; + `--no-residue-codes` leaves the code store out (`build residue-codes + --from-shards --run-id` adds it later), and `--cleanup-embeddings` deletes + the record, then the shards, once the database is built. Before it writes + any database file, step 2 checks the record, that the shards are the + recorded ones with their sizes and rows, their schema (with codes, those of + the system the record names, with this FoldMatch's digests), the FASTA + against the record (its size, then its sha256, then its pre-flight counts) + and the id of every shard row against the FASTA entry the pass gave it. Step + 1 takes a stat of the FASTA on every rank before its pre-flight and after + its pass, and step 2 around its own reads, so a FASTA that changes during + either step is refused (step 2 then removes the files it wrote). A rank may + read a node-local copy of the FASTA, which must be a byte copy of rank 0's: + only rank 0's copy is hashed. A run embedded by another FoldMatch version is + built with a warning. A FAISS refusal (an `ivf_pq` setting the data cannot + take, say) costs no GPU time: rerun step 2 with other options. From the same + shards, the two steps write the files of `build sequences` byte for byte, + except `build.created_utc` and `build.run_id` in the residue-code manifest + (and the copy in its index) and the added `build.split: true`; `hnsw` and + `ivf_pq` indexes are byte-identical, as between any two builds, only at one + thread, and an `ivf_pq` index holds the path of its `.invlists`, so only at + the same `--output-db`. Every refusal of the three commands is an error + with exit status 1 (2 for a usage error), with or without + `--residue-codes`. The recipe: run `fm-search download-models` once on a + login node first (no rank of a multi-node `--residue-codes` run downloads); + run step 1 in the GPU job as `srun --kill-on-bad-exit=1 ... fm-search build + sequences-embed ...`, with a generous wall time (step 1 cannot resume) and + one shards folder per build on storage that every node can write and that + outlives the job (such as `$SCRATCH`); and run step 2 as its own job on one + CPU node, submitted with `sbatch -C cpu -N 1 --dependency=afterok: + --kill-on-invalid-dep=yes` and started without srun, with a generous wall + time too. At 30.4 million sequences with `--residue-codes`, step 2 with the + `faiss-cpu` 1.15.1 wheel FoldMatch now pins (see *Changed*) took 3 h 33 min + on one CPU node (128 cores and FAISS threads): 2 h 37 min for the `ivf_pq` + index, 16 min for the sequence store, 30 min for the residue-code store and + 4 min for the shard id check. On one GPU node (64 cores and threads) it took + 2 h 47 min, 1 h 45 min of it for the index, and with FAISS 1.15.1 built on + MKL (the official `pytorch` conda channel's build; see the README) 2 h 02 + min, 59 min of it for the index: ask for 6 hours. The `faiss-cpu` 1.13.2 + wheel's single-threaded BLAS made the index take about 7 to 8 hours whatever + the thread count; the index phase logs a line, a warning except on macOS, + when FAISS's BLAS is single-threaded. A step 2 killed by its wall time + cannot resume: it leaves `.build.lock` + and partial database files, which the next step 2 lists and which must be + deleted before FAISS is rebuilt from scratch. A project with GPU hours only + may run step 2 on one GPU node, where `ivf_pq` fits (`flat` and `hnsw` may + not). To keep several builds in one folder, set `FOLDMATCH_RUN_ID` in job 1 + and pass the same value to step 2 as its `--run-id`. Step 2 and the seal + refuse to run as one of several ranks, or in the batch step of a job of more + than one node, and warn in a one-task srun step of such a job and in a + one-node job with GPUs. They refuse before Lightning and torch are + imported (which can take minutes on a compute node), and so do step 2's + other refusals up to its index phase (the run record, the residue-code + options, the build lock, existing database files, the shards and the FASTA + against the record) and every refusal of the seal, which never imports + them: within seconds of the launch once the file system has the + environment's files cached (a first launch on a node can take a minute). + The refusal in the batch step, like the srun step's warning, ends with the + command line to run in the one-node CPU job, + as it was invoked (`fm-search ...` through the console script), after a + `cd` to the folder it ran in when the line depends on it (a relative path, + or `python -m`); the refusal as one of several ranks ends with the command + to launch once. The command is on a line of its own, indented, so that it + can be copied whole. Step 2 sets FAISS's OpenMP threads to the + physical cores it may use (and the threads of the BLAS libraries + threadpoolctl finds, which it logs), unless `OMP_NUM_THREADS` (for BLAS also + `OPENBLAS_NUM_THREADS`, `MKL_NUM_THREADS` or `VECLIB_MAXIMUM_THREADS`) is + set; on macOS it leaves them as they are. On Linux, FAISS's own OpenBLAS, + in the `faiss-cpu` wheel, is one of those libraries: an OpenMP build, it + runs FAISS's OpenMP threads (at most 128 in the x86_64 wheel), so + `OMP_NUM_THREADS` sets it and `OPENBLAS_NUM_THREADS` does not (the log line + says so). With the wheel, the OPQ step of an `ivf_pq` training does not gain + from more threads, which is most of why the CPU node above took longer than + the GPU node. With the wheel, run step 2 with `OMP_NUM_THREADS=64` on a node + of more than 64 physical cores (it sets the threads of the whole step 2; not + timed at this scale on a 128-core node), and expect the warning that FAISS + has fewer threads than cores; with the MKL build, leave it unset. It holds + a lock, `.build.lock`, while it builds, so a second step 2 into + the same database is refused; a lock left by a killed job is reported with + its content, the database files present and what to delete. `build + sequences-seal --fasta-file F --from-shards S --run-id X` recovers shards + that have no record: those of a step 1 whose rank 0 stopped after the pass + (wall time, out of memory, a full disk; when step 1 itself refuses then, its + error gives the command), and those a `build sequences` kept when its index + phase failed. In one process and without ESM3, it checks them against the + FASTA (the entry count, each rank's rows, the shard schema and the id of + every row, and with residue codes each row's `seq_sha1` against the sha1 of + its sequence) and writes their record, marked as sealed (step 2 then warns + that the shards are bound by these checks, not by the pass, and takes the + FoldMatch version of the pass from residue-code shards, the only ones that + record it). A run without residue codes needs the `--min-res-n` it used, + which its shards do not record. Shards written by FoldMatch 0.8.0 and + earlier embedded the FASTA's sequences before they were normalised, so a + database sealed from them matches a fresh build only when the FASTA's + sequences are uppercase with no whitespace inside their lines; otherwise + embed them again. +- **`fm-search query sequences --scoring esm-codes`** (opt-in; `--scoring + blosum62`, the BLOSUM62 Stage 2 as before, stays the default). Stage 2 + aligns each query with its Stage-1 candidates through their ESM3 residue + codes, on a database built with `build sequences --residue-codes`, using + the scoring system its residue-code manifest names (`esm-codes-v1`). The + ESM3 pass that embeds the queries also computes their residue encodings; + the native kernel scores every candidate, the E-value of a hit is + calibrated against a fixed panel of 1,000 UniRef50 sequences and computed + over the database's residue count, and only the hits that pass + `--max-evalue` are traced back for the alignment columns. Hits are ranked + by score. One scoring per run. Under esm-codes, `--min-seq-identity` and + `--min-coverage` default to 0 and `--max-evalue` to `1e-3`, as under + BLOSUM62 (explicit values are honoured; the help gives both scorings' + defaults), and the BLOSUM62 options `--gap-open`, `--gap-extend`, + `--comp-bias-corr`, `--comp-bias-corr-scale`, `--significance-mode`, + `--significance-sample-size`, as well as `--no-seq-identity`, are refused + (exit status 2), whatever their value. Before ESM3 is loaded, and on every + rank alike, the query is refused with a one-line message (exit status 1) + when its FASTA has duplicate ids, `<` or `>` in a sequence, or non-ASCII + characters; when the database was built without `--residue-codes`, or + its codes are a partial build; when the codes were built for a scoring + system this FoldMatch does not know (a database from a newer FoldMatch) + or do not match the system or the database's sequence store; when the + native kernel cannot be loaded (the message gives the command that builds + it); and when the scoring bundle cannot be fetched or verified (only the + process that starts a run downloads it). Once the index is loaded, codes + whose entry count differs from the FAISS index are refused. A hit is + inside the calibrated domain of the E-values (`ecal=1`) when the query + has 21 to 966 residues, the target at most 3,000, both at most 1% + non-standard residues and the target at least 2 distinct codes; other + hits are still reported, with `ecal=0`, and one warning per run gives the + counts. A query of one residue, or whose encoding rows are all identical, + reports no hits. Stage 2 runs in a pool of worker processes that load + neither torch nor ESM3, after ESM3 and the aggregator have been freed: + `--align-workers N` uses N workers; by default the pool has up to one + worker per physical core (SMT siblings count once) and starts only as many + as the work needs, so a small query does not start a worker on every CPU + (a later batch of queries with more work grows the pool). Its output does + not depend on the number of workers. A worker needs about 5 bytes per + residue pair of the pair it aligns (up to 1.3 GiB for the longest pairs it + aligns, 2^28 residue pairs; longer ones are skipped and counted) and keeps + at most 128 MiB of it between tasks; a worker that is killed (for example + by the out-of-memory killer) stops the run with an error. The query + encodings are written next to the query's embedding shards in + `--tmp-embedding-folder` and deleted with them at the end of the run, also + when it fails (a process whose ESM3 pass fails deletes the shard it wrote). +- **New `--format-output` fields `rawrev`, `q90`, `rawadj` and `ecal`**, filled + by `query sequences --scoring esm-codes`: the score of the reversed query, + the query's 0.9 quantile over the scoring panel, the adjusted score behind + `evalue` and `bits` (`raw - rawrev - q90`), and whether the hit is inside + the calibrated domain (`1`/`0`). Under esm-codes `raw` is the forward score + with three decimals, `evalue` is computed in log space, so it never + underflows, and `bits` is rounded half up (`floor(x + 0.5)`, also for a + negative bit score). The four fields render empty under BLOSUM62 and in + `query db`. BLOSUM62 columns, `bits` included, and the default columns are + unchanged. +- **esm-codes Stage 2 speed** (measured on one AMD EPYC 7763 node, OpenBLAS, + one BLAS thread per worker; the output does not depend on any of the + settings below). On one core, scoring a 300-residue query against 1,000 + candidates and tracing back the hits that pass takes 0.25 s at + `--max-evalue 1e-3` (none passed in this benchmark) and 0.49 s at + `--max-evalue inf` (all 1,000 traced back); BLOSUM62's Stage 2 takes + 2.65 s on the same targets. The per-query panel pass that calibrates the + E-value adds 0.11 s at 71 residues, 0.38 s at 300 and 1.17 s at 966. With + 100 candidates a query takes 0.15 / 0.40 / 1.29 s at 100 / 300 / 1,000 + residues on one core, 1.5-1.6x the BLOSUM62 time (codes are faster from + about 160 candidates on), and 0.03 / 0.07 / 0.20 s on 8 workers. On 64 + workers, 640 queries x 1,000 candidates run at 128,000-154,000 candidate + pairs per second at `1e-3` and 58,000-67,000 at `inf` (the lower ends + with the workers pinned to 65 cores, the higher with the default + affinity). A worker's memory stays under 70 MiB at 1,000 residues x 1,000 + candidates. +- **`fm-search download-models`**: the same command, with the same `--system` + option and help, as `fm-embedding download-models` and `inference + download-models`, since `fm-search` now uses the scoring bundle too. +- **`fm-search cluster --method greedy`**: greedy representatives in one + streamed pass over the database, for collections whose similarity graph does + not fit in memory. Chains are taken in database order; a chain in no cluster + yet becomes a representative and takes the chains among its + `--max-neighbors` nearest that score at least `--threshold` and are in no + cluster yet. Each block's search results are dropped once used, so memory + grows with the number of chains, not with the number of edges, and only + chains no representative has taken are searched. Deterministic (no seed). + Clusters have at most `--max-neighbors` + 1 chains and, when most + representatives have `--max-neighbors` neighbours above `--threshold`, are + bounded by `--max-neighbors` rather than by `--threshold` (the statistics + report that share, with a warning when it is above half). Results depend on + database order. The output gains a 4th column, `representative` (the + representative's chain id), and cluster ids are numbered by decreasing size + under both methods (`cluster_id` 0 is the largest cluster; Leiden's already + were). `--method leiden`, the graph and Leiden as before, stays the default. + Measured with this command: 30,430,281 chains, `ivf_pq`, `--max-neighbors` + 1000, `--threshold` 0.8, one NERSC m4468 CPU node (2 x AMD EPYC 7763, 128 + cores / 256 threads, `OMP_NUM_THREADS` 128): 53.4 min from the load to the + last block, 54:11 min wall for the whole command (with the start-up, the + statistics and the export of the 1.65 GB file), peak RSS 5.57 GiB. + +### Changed + +- **`fm-search cluster` and `fm-search similarity-graph` build the graph many + chains per FAISS call.** They searched one chain per FAISS call and went + through its neighbours in Python; they now read and search blocks of at most + 16,384 chains (whose results take about 200 MB at the default + `--max-neighbors` 1000) and keep the edges with NumPy, which is much faster. + The edge rule is unchanged: an edge comes from the search of its + lower-numbered chain, in chain order, then in FAISS's rank order. On `hnsw` + the graph is the same, and so are the files of `cluster` (for the same + `--seed`) and `similarity-graph`. On `flat` it is the same up to float + rounding (scores within about 2e-6 at 1536 dimensions), so a pair at + `--threshold` or tied at a chain's `--max-neighbors`-th score can gain or lose + its edge, and near-tied neighbours of a chain can change order, which can + change the clusters of the same `--seed`; the scores now depend on FAISS's + thread count, as batched `flat` searches already do. Both commands log the + time of the graph build, and `cluster` that of the Leiden clustering. +- **`fm-search cluster` and `fm-search similarity-graph` need much less memory + per edge to build the similarity graph.** The kept edges are no longer + concatenated into one array, they are handed to igraph without a temporary + NumPy array conversion, and equal edge weights are stored once. The graph, + its weights, the clusters and the output files are unchanged. +- **BREAKING — `fm-search cluster --method greedy` refuses `--resolution` and + `--seed`**, the Leiden options (greedy has no resolution and is + deterministic), in one message. **`--max-neighbors` below 1 is refused** by + `cluster` and `similarity-graph` before the database is loaded (it raised a + bare FAISS `AssertionError` after the load). `--resolution`'s default, 1.0, + is now stated in its help instead of Typer's `[default: 1.0]`. +- **BREAKING — `fm-search cluster`'s Leiden step runs on python-igraph's + `Graph.community_leiden`** (modularity, the same objective at + `--resolution`, a fixed 10 iterations by default, set by the new + `--leiden-iterations`) instead of `leidenalg` run to convergence; + `leidenalg` is no longer a dependency. `--resolution` works as before: at + 0.5 and 2.0 on a 1M-chain graph (`--max-neighbors` 50) the new partitions' + largest cluster, cluster count and modularity overlap `leidenalg`'s across + seeds, and the new partitions agree with `leidenalg`'s about as well as two + seeds of either library agree. **Partitions and cluster labels change for + the same `--seed`**, by about as much as a change of seed: on a 1M-chain + graph (`hnsw`, `--threshold` 0.8, `--max-neighbors` 1000), the new + partitions agree with `leidenalg`'s converged ones at AMI 0.857 (ARI 0.778; + 3 seeds against 2), while two `leidenalg` seeds agree at AMI 0.854 (ARI + 0.763) and two new seeds at 0.859 (ARI 0.790); modularity 0.6663-0.6687 + against 0.6653-0.6696. The Leiden step took 679-852 s instead of + 6,514-7,671 s on that graph (one thread, NERSC CPU and GPU nodes), and + needed about 5.1 GiB of working memory instead of 11.8 GiB in a standalone + run of the Leiden step on that graph (`cluster`'s own `leidenalg` Leiden + phase measured about 8.7 GiB there). In `cluster` itself (seeds 42-44, + `--threshold` 0.8, `--max-neighbors` 1000, Linux x86_64 at 128 threads), + the Leiden phase needed 5.1 GiB above the built graph instead of 8.7 GiB + on that `hnsw` database, and the run peaked at 20.2 GiB instead of 23.79 + GiB (VmHWM); on a 1M-chain `ivf_pq` database (447,547,205 edges), 10.3 + GiB instead of 17.1 GiB, a peak of 28.16 GiB instead of 34.99 GiB, and a + Leiden step of 1,714-1,936 s instead of 17,945 s (new: a NERSC GPU node; + old: a CPU node). Cluster ids stay by decreasing size (0 the largest), + ties now by the smallest chain index, as under greedy. **`--seed` below 0 + is refused** (exit status 2, before the load): the new generator gives + seeds n and -n the same partition. `--seed` 2**31 and above now work + (`leidenalg` raised `OverflowError`). **`--threshold` below 0 with + `--method leiden` is refused when a kept edge score is negative** (exit + status 2, after the graph is built; previously `leidenalg` raised a bare + `BaseException` then). A negative threshold whose kept scores are all at + least 0 worked and still works. **Library**: + `EmbeddingClusterer.cluster_leiden` returns a NumPy array (it returned a + list); its `n_iterations` defaults to 10 and must be at least 1 (it + defaulted to -1, run to convergence); it raises `ValueError` for a seed + below 0 (`leidenalg` accepted negative seeds; the new generator gives seeds + n and -n the same partition) and for a graph built with a negative kept + weight. It seeds python-igraph's random generator privately + (`random.Random(seed)`), leaving Python's global `random` state untouched + as `leidenalg` did, and restores python-igraph's generator to the `random` + module afterwards (a caller's own igraph generator is replaced). Seeds are + integers of at least 0 (`operator.index`: NumPy integers accepted, floats + refused). +- **Structure-based chain selection keeps chains with at least `min_res_n` + residues** (was more than `min_res_n`). This covers `fm-search build + structures`, `fm-embedding` and `inference`, and matches `FoldMatch(min_res=...)` + at query time and the FASTA paths, which already used "at least". Databases and + embeddings built with a non-zero minimum now also include chains of exactly + that length. +- **The ESM3 weights are pinned.** FoldMatch now downloads `rcsb/rcsb-esm` at + revision `636acb420daf674df37ab0220f56061a5de2f169` instead of whatever its + `main` branch points to, and refuses any `esm3_sm_open_v1.pth` other than the + one with sha256 + `5ead5a135c658068db6a4f1b933e72d6110992c4668822e1c0e2dcc53e38acd9`. These are + the weights FoldMatch already used, so embeddings do not change. In the usual + Hugging Face cache layout the check reads the file's blob name, which is its + sha256, and costs nothing; when the layout hides it (a `local_dir` download or + a copied snapshot), the 2.8 GB file is hashed once per process. An offline + installation (`HF_HUB_OFFLINE=1`) whose cache holds only an older + `rcsb/rcsb-esm` snapshot must run `download-models` once online. + Every `torch.load` now passes `weights_only=True` (already PyTorch's default + from 2.6), so model and residue-embedding files cannot run pickled code. +- **FASTA sequences are normalised before ESM3.** ASCII whitespace inside + sequence lines is removed and ASCII letters are uppercased before a sequence + is embedded, stored in the sequence store or aligned in Stage 2, on every + FASTA path (`fm-search build sequences`, `fm-search query sequences`, + `fm-embedding`, `inference`) and in `FoldMatch.sequence_embedding` (Python + API). ESM3 used to read lowercase letters, and spaces or tabs inside a line, + as unknown residues, so the chain embeddings of such records were wrong + (cosine 0.31-0.34 to the correct ones on the test fixture): rebuild databases + and redo embeddings made from lowercase or soft-masked FASTA. `--min-res-n`, + and `min_res`/`max_res` of `FoldMatch.sequence_embedding`, now count residues + without whitespace. Uppercase FASTA without whitespace inside its sequence + lines gives exactly the same results as before. +- **FASTA problems and existing outputs are reported before the ESM3 pass.** + `fm-search build sequences` now refuses an existing `.index`, `.metadata`, + `.invlists`, `.sequences` or `.rescodes.*` file before computing any + embedding; before, the existing database was only refused after the whole + pass, and a leftover `.sequences` file made the build fail at the end. + `fm-search build structures` checks `.index`, `.metadata` and `.invlists` up + front in the same way. `build sequences` and `query sequences` scan the FASTA + once on every rank before ESM3 and log a warning for duplicate ids, sequences + containing `<` or `>`, and non-ASCII sequences, then go ahead as before. If + the sequence store written after the pass disagrees with that scan (the FASTA + changed during the build), the build fails and removes the database files it + wrote, so it can simply be run again. A FASTA with no sequence of at least + `--min-res-n` residues (or an empty one) is refused before ESM3 too, instead + of failing after the pass with "No embeddings provided". `fm-search query + sequences` and `fm-search query db` decide whether Stage 2 runs before ESM3 + or the search, and check the sequence stores it will read: `--seq-identity` + without a store, an incomplete store and an E-value filter over an empty + database are reported there. +- **An unknown `--format-output` field is refused before the ESM3 pass** + by `fm-search query sequences --scoring blosum62` too, as under esm-codes: + whenever Stage 2 runs, an empty list or an unknown field is now a usage + error (exit status 2) reported before the queries are embedded. Before, the + queries were embedded first and the command then failed with exit status 1 + (and the Python function `query_database_from_fasta` raised `ValueError`; + it now raises `typer.BadParameter`). An embedding-only search does not use + the option and still ignores it. `fm-search query db` now checks it too, + before its search (see *BREAKING — the BLOSUM62 Stage 2 needs the native + kernel* below). +- **An interrupted sequence-store build is detected.** A `.sequences` file + whose build stopped part way (it now records its totals last) is refused + with an "incomplete" error instead of being read as the whole database: + delete the database and rebuild it. +- **Embedding shards are named per run.** The intermediate files in + `--tmp-embedding-folder` are now `emb_-.parquet`. The run id is + `FOLDMATCH_RUN_ID` when set; in a task of an `srun` step it is built from the + job id, step id, restart count and a digest of the command's arguments, so + the tasks of a step agree and the commands it runs one after another do not; + otherwise, including the batch step of an `sbatch` script, the launching + process draws a random id and passes it to the ranks it starts. `fm-search + query sequences` deletes its query shards after the search; build shards are + kept unless `--cleanup-embeddings` is given. A rank above 0 started without a + shared id refuses to start: set `FOLDMATCH_RUN_ID` to the same value for every + rank whenever something other than `srun` or Lightning starts the ranks, that + is `torchrun`, or a multi-node run outside SLURM started on each node with + `NODE_RANK` (set it on every node: otherwise the other nodes refuse and node 0 + waits for them until the one-day process-group timeout). Every rank also + refuses to start when shards with its run id are already in + `--tmp-embedding-folder` (the same command run twice in one `srun` step, or a + reused `FOLDMATCH_RUN_ID`). After a FASTA pass, rank 0 checks that the shards + hold exactly one embedding per entry, and a missing rank is now an error + instead of being dropped silently. +- **`fm-search build sequences` logs its index phase.** When rank 0 starts the + FAISS index it logs the FAISS version and OpenMP threads, how many of the + node's CPUs (and physical cores) the process may use, and `OMP_NUM_THREADS`, + with one line per thread pool threadpoolctl finds; a line naming FAISS's + BLAS library when it is single-threaded, as the `faiss-cpu` 1.13.2 wheel's + was (a warning, except on macOS: the `ivf_pq` training and adds then run on + one core; the line says to install the pinned `faiss-cpu` 1.15.1, whose + OpenBLAS is multi-threaded, or a FAISS built on MKL); and, except on + macOS, a warning when FAISS has fewer threads than usable physical cores, + naming the cause: `OMP_NUM_THREADS`, which Lightning's DDP launcher sets + during a multi-device pass when it is unset (to the CPUs the rank may use + divided by the devices per node; the lines say so, and exporting + `OMP_NUM_THREADS` before the run chooses FAISS's threads); then the time + taken by the FAISS index, the sequence store and the residue-code store. A + build on more than one node (`--num-nodes`) also logs that the other nodes + stay allocated through this one-process phase, and that `build + sequences-embed` and `build sequences-index` (see *Added*) release them. The + outputs are unchanged. +- **The `--load-batch-size` help of the build commands is corrected.** It said + that larger batches (65536 and more) use many-core nodes better during an + IVF-PQ build, but FAISS adds IVF-PQ vectors in chunks of 32768 whatever the + batch: a larger one only takes more memory. +- **`--max-evalue` against an empty database is an error.** With an E-value + filter (including the default `--max-evalue 1e-3`), `fm-search query + sequences`, `fm-search query db` and `align_candidates` now refuse a database + whose residue count is 0 or negative, with a message that names the likely + cause (an empty or unbuilt sequence store) and the two ways out: delete the + database and rebuild it with `fm-search build sequences`, or pass + `--max-evalue inf` (`max_evalue=None`). The CLI reports it before the ESM3 + pass or the search. Before, a query against an empty sequence store returned + no hits with only "missing from sequence store" warnings, and + `align_candidates` failed with an unrelated error from deep inside the + alignment loop. +- **`--min-coverage` help text corrected** on `fm-search query sequences` and + `fm-search query db`: a hit is kept only if the query coverage and the subject + coverage both reach the value, so it is dropped when either side is below. + The old text suggested the opposite. Filtering itself is unchanged. +- **The BLOSUM62 Stage 2 aligns on FoldMatch's native SIMD kernel: the same + output, about 23 times faster.** `fm-search query sequences` + (`--scoring blosum62`, the default), `fm-search query db` and + `align_candidates` align each (query, candidate) pair with `fmswi_align`, an + integer Smith-Waterman added to FoldMatch's compiled library (the one of the + esm-codes kernel), instead of biotite's `align_optimal`. It uses a striped + query profile in 16-bit SIMD lanes (SSE2, AVX2, NEON, or a portable build) and + redoes a pair in 32-bit lanes when its score could overflow them; the scores + are exact integers, so every build gives the same results. It reproduces + biotite 1.7.1's `align_optimal` with `local=True` and affine gaps, ties + included: the same score, end cell and path, hence the same value in every + output column. On the full nbt.3988 benchmark, the Stage-2 tables were + byte-identical: 17,733,246 rows at `--threshold 0.7` and 9,503,077 at `0.8`. + Stage 2 of the 0.7 search took 486 s instead of 11,403 s (468 to 486 s over + the runs with the slices described below), on the CPU of one Perlmutter GPU + node (a 64-core AMD EPYC 7763) with 128 workers. The kernel alone aligned the + benchmark's largest query in 72 s instead of 9,864 s, and cut the end of the + run that waited on the longest queries from 8,459 s to 4 s. Pairs are now also + handed to the workers in slices of about a quarter of each worker's share of + the DP cells (a pair larger than that is a slice of its own), the largest + first, which shortens that end of the run to under 1 s. The composition-bias + correction (`--comp-bias-corr`) is part of the query profile and costs nothing + more. The traceback needs one byte per DP cell (359 MB for the benchmark's + largest pair, where biotite took 4.87 GB). A worker process that cannot load + the kernel now fails Stage 2 instead of hanging it. + `python -m foldmatch.search.swkern`, the kernel's self-test, also checks + `fmswi_align` against a golden digest and reports it under `fmswi`. biotite + remains a dependency: it supplies the BLOSUM62 matrix and the + `--significance-mode sampled` statistics. +- **BREAKING — the BLOSUM62 Stage 2 needs the native kernel, and its gap costs + are checked before Stage 1.** A search that runs Stage 2 under BLOSUM62 now + needs FoldMatch's compiled kernel, as `--scoring esm-codes` already did. An + install without it (the pure-Python package a build from source falls back to + when it finds no C compiler; see *Platform wheels* below) aligned with + biotite; `fm-search query sequences` and `fm-search query db` now refuse it + before Stage 1, with exit status 1 and the command that rebuilds or reinstalls + the kernel, and `align_candidates` raises `KernelError`. There is no fallback. + Embedding-only searches (`--no-seq-identity`, or a database without a sequence + store) do not need the kernel. `--gap-open` must now be at least 1 (0 was + accepted) and `--gap-extend` at least 0, both at most 1,048,576; the sign is + still ignored. Called from Python, a gap cost must be a whole number: a float + such as `11.0` is still taken as 11, but `10.5`, which biotite truncated to + 10, is now refused. Under `--significance-mode default`, gap costs other than + 11/1 are refused, as before, whenever the run needs significance (an `evalue` + or `bits` column in `--format-output`, or a finite `--max-evalue`, `1e-3` by + default). `--significance-sample-size` must now be at least 2 (the Python + functions check it when they compute the sampled significance): under + `--significance-mode sampled`, 1 and 0 gave infinite or NaN bit scores and + E-values, and a negative size failed after Stage 1. These refusals now come + before Stage 1, before the queries are embedded (`query sequences`, on every + rank) or the database is loaded (`query db`), as usage errors (exit status 2) + that name the options and, for 11/1, the ways out + (`--gap-open 11 --gap-extend 1`; `--significance-mode sampled`; or + `--max-evalue inf` with neither column); the 11/1 refusal came after Stage 1, + with exit status 1 (the Python functions `query_database_from_fasta` and + `query_database_from_database` now raise `typer.BadParameter` for them). + `align_candidates` checks the gap costs the same way before it reads a + sequence or starts a worker (`ValueError`, or `TypeError` for one that is not + a whole number). A gap cost of `-11`/`-1` now counts as 11/1 under + `--significance-mode default` (it was refused there). Whenever Stage 2 runs, + `query db` also refuses an unknown or empty `--format-output` field before its + search, as a usage error (exit status 2); it failed with exit status 1 after + the search. +- **Long FASTA sequences need far less ESM3 memory.** For sequence-only input + (the FASTA paths of `fm-search`, `fm-embedding` and `inference`, and + `FoldMatch.sequence_embedding` in the Python API), ESM3 now runs without its + geometric attention, which has no coordinates to attend to there but + allocated a tensor that grows with the square of the sequence length. ESM3 + ran out of memory at 2,750 residues on a 40 GB A100; 16,000 residues now fit + in about 10 GiB. On a CPU node a 4,000-residue sequence needed 83 GiB and now + needs 8.3 GiB, and a 1,500-residue sequence takes 10 s instead of 56 s on an + M1 Max CPU. Embeddings are bit-identical. Structure inputs are unchanged. In + the Python API, ESM3 calls on one model from several threads now run one at + a time, so that no structure embedding runs while another thread has + geometric attention switched off. +- **Platform wheels; installing from source needs a C compiler.** FoldMatch now + contains a small compiled library (the Smith-Waterman kernel above), so the + release publishes platform wheels for Linux x86_64 and aarch64 (built in + manylinux_2_28 images) and macOS arm64 and x86_64 (macOS 11 or later), and + the classifiers name Linux and macOS instead of "OS Independent". Anything + that builds FoldMatch from source compiles the kernel and needs GCC 12 or + later, or Clang 14 or later: an install from the sdist (another platform, or + `pip install --no-binary foldmatch`), an editable install (`pip install -e + .`, `uv sync`) and `hatch test`. Without a usable compiler the install still + succeeds, with a warning, as a pure-Python package: everything works as + before except the Stage 2 of a search (under either scoring, and in `query + db`: the BLOSUM62 Stage 2 aligns on the kernel too, see above), which refuses + to run and prints the reinstall command, `pip install --no-cache-dir --force-reinstall + --no-binary foldmatch foldmatch` (pip caches the pure-Python wheel, hence + `--no-cache-dir`). Set `FOLDMATCH_REQUIRE_NATIVE=1` to make a failed compile + an error instead; CI, the release wheels and the Docker image do. In a + source checkout, rebuild the kernel after editing its C source with `python + -m foldmatch._native.build`; a library built from other source is refused. +- **`threadpoolctl` is a declared dependency** (`threadpoolctl==3.6.0`, + exactly pinned like the others). The esm-codes Stage 2 uses it to limit to + one thread the BLAS of its worker processes and of an in-process run, which + keeps reruns bit-identical; it already came with `esm` (through + scikit-learn), and the code still works without it. threadpoolctl cannot + limit Apple Accelerate (numpy's BLAS on macOS): an in-process run there + keeps Accelerate's own thread count and says so once in the log, while the + worker processes are limited through `VECLIB_MAXIMUM_THREADS=1` either way. +- **`faiss-cpu` 1.15.1** (was 1.13.2). On Linux its wheels bundle an OpenMP + build of OpenBLAS, where 1.13.2's OpenBLAS was single-threaded. The OPQ and + k-means training of an `ivf_pq` index, the IVF assignment of its adds and + batched flat searches therefore run on FAISS's OpenMP threads, at most 128 + for BLAS in the x86_64 wheel (see the README, *FAISS threads*): an `ivf_pq` + index of 30 million sequences, about 7 hours with 1.13.2 on any node, took + 1 h 45 min on a 64-core node at 64 threads and 2 h 37 min on a 128-core node + at 128, where the wheel's OPQ training runs slower than at fewer threads. + `OMP_NUM_THREADS` sets those threads; `OPENBLAS_NUM_THREADS` does not. The + one-thread BLAS limit of an in-process esm-codes Stage 2 now reaches that + OpenBLAS too, and so FAISS's OpenMP threads (and torch's, which share them + when torch is loaded first) while it runs; it then sets every OpenMP runtime + back to its thread count, where threadpoolctl's restore alone left them at + the OpenBLAS's own, older count. Index files written by either version are + read by both, and the two search one database alike up to float rounding, + which can reorder tied hits (with `hnsw`, 5 to 7% of searches changed a + hit). With the wheel, rounding and so the order of tied hits also depend on + FAISS's thread count: set `OMP_NUM_THREADS` to the same value for + bit-identical reruns. Any rebuild of an `ivf_pq` index gives a different, + equally valid index (at 30 million entries, an MKL build below shared 57% of + its top-100 hits with the 1.13.2 index, wheel builds 58 to 59% with an MKL + build, and two wheel builds at 64 and 128 threads 59% with each other), so + compare only searches made on the same database. + A FAISS built on MKL, from the official `pytorch` conda channel (Linux + x86_64, Python 3.12), is faster still for large `ivf_pq` builds; the README + gives the install and its caveats. On macOS the wheel still uses Accelerate + and bundles its own OpenMP runtime, so FoldMatch still sets + `OMP_NUM_THREADS=1` there by default. On Linux x86_64, `import faiss` now + logs at INFO that it "Could not load library with AVX2 support": expected, + the wheel has one library that picks the CPU's instructions at run time. Its + Linux wheels need glibc 2.27 (FoldMatch's own wheels need 2.28), its macOS + wheels macOS 14 on arm64, as before, and macOS 15 on x86_64 (was 14); + `faiss-cpu` still publishes no sdist. + `uv.lock` is regenerated, and now also records FoldMatch 0.10.0 and its + `threadpoolctl` pin. +- **`download-models` also fetches the esm-codes scoring bundle.** `fm-embedding + download-models` and `inference download-models` now download, besides ESM3 + and the aggregator, the bundle of every registered esm-codes scoring system + (one, about 26 MB, from `rcsb/foldmatch-esm-codes` at the revision pinned in + FoldMatch) and check the sha256 of every file. `--system NAME` (repeatable) + limits this to the named systems; an unknown name is refused before any + download. A cached bundle file whose sha256 differs (a truncated or + corrupted download) is downloaded again. The files go into the Hugging Face + cache like the models, so later runs work with `HF_HUB_OFFLINE=1` and from + a read-only cache; nothing else is written there. +- **`download-models` downloads only.** It fetches and checks the pinned files + (the ESM3 snapshot, whose weights it hashes in full, the aggregator weights, + now pinned by sha256 too, and the esm-codes bundles) without building ESM3 + or the aggregator, importing torch or touching a GPU. Before, it loaded both + models, on the GPU when one was visible, which ran out of memory on shared + GPU login nodes; the `CUDA_VISIBLE_DEVICES=` workaround is no longer + needed. A cached ESM3 or aggregator file whose content differs from its pin + is downloaded again, and the command prints one line per model with the + verified pin. +- **The `--format-output` field list** in `fm-search query sequences --help` + and `fm-search query db --help` names the four new esm-codes fields + (`rawrev`, `q90`, `rawadj`, `ecal`); in `query db` they are always empty. +- **Docker image.** The image is built with a regular (non-editable) install + that must compile the kernel, runs the kernel self-test at build time and + again in the compiler-free runtime stage, and no longer copies the source + tree into `/app`. +- **A `.pt` or `.csv` embedding file holds one vector: residue embeddings are + refused.** `fm-search build embeddings` and `fm-search query embedding` + read one embedding vector per `.pt` or `.csv` file: a tensor of shape + `[D]`, `[1, D]` or `[D, 1]`, or a CSV of one column (as `fm-embedding` and + `inference` write chain and assembly embeddings) or of one line. A file of + residue embeddings (one row per residue, as `fm-embedding from-structures + residue`, `fm-embedding from-sequences residue` and `inference + residue-embedding` write them), or any other table, now stops the command + with an error that names the file and its shape. Such a file was averaged + over its rows without a message, and the mean indexed or queried as the + chain's embedding, which it is not: on the test chains its cosine to the + chain embedding is -0.03 to 0.01, and the means of any two chains, related + or not, score 0.86 to 0.97. Compute chain embeddings with `fm-embedding + from-structures chain` or `fm-embedding from-sequences chain`, or from + residue files `NAME.CHAIN.pt` or `NAME.CHAIN.csv` with `inference + chain-embedding --src-file LIST --res-embedding-location DIR + --res-embedding-format pt|csv --output-folder OUT`, `LIST` holding one line + `NAME,,CHAIN` per file. **Rebuild databases built from residue + embeddings**: the mean has the width of the residue embeddings, which for + FoldMatch's ESM3 ones is the dimension of its chain embeddings (1536), so + `fm-search stats` cannot tell such a database apart, but building it again + from the same folder now stops with this error, which identifies it. + Results of such databases, and of residue embeddings used as queries, are + meaningless. A `[D, 1]` tensor is now read as its `D` values; it was read + as their mean, one value. A NaN or infinite value is now refused too (such + an embedding was never found, and as a query it found nothing), and so are + a file that cannot be read or holds no values, a `.pt` that does not hold a + dense tensor, and a CSV that is not numbers (its error named no file and + could quote a whole line), each with the file's name. +- **BREAKING — residue embeddings in Parquet keep their shape.** With + `--output-format parquet`, `fm-embedding from-structures residue`, + `fm-embedding from-sequences residue` and `inference residue-embedding` + write one row per chain whose `embedding` holds one list of `D` values per + residue (ESM3's BOS and EOS rows included, as before): its type is + `list>`, so the file stores `D`. They wrote each + chain's residue table flattened into one row of `list`, which + stores no shape. The Python readers `ResidueEmbeddingFromParquet` and + `ResidueAssemblyEmbeddingFromParquet`, and `chain_inference.predict`, lose + their `embedding_dim` argument, so a call that passes it now fails (for + `chain_inference.predict`, whatever its `src_from`), and the arguments of + `ResidueAssemblyEmbeddingFromParquet` after `parquet_path`, and + `return_predictions` of `chain_inference.predict`, are now keyword-only, so + that an old positional call fails too. The readers take `D` from the file, + read only the new layout and refuse a flat file, or one that cannot be read, + naming it. They read one file, or a list of files + (`ResidueEmbeddingFromParquet` only), such as the shards of a multi-device + run: `sorted(Path(OUT).glob("*.parquet"))`. A folder, which they read + before, is refused. `fm-search build embeddings` and `fm-search query + embedding` refuse the new files as residue embeddings, whatever their row + lengths. The files are written in row groups of about 256 MB, so a residue + run no longer holds up to 16,384 chains in memory. `ParquetBatchWriter` + writes the new layout with `residue_embeddings=True` (not with a residue + output); without it, it now refuses a tensor with more than one axis longer + than 1, and the chain, assembly and sequence embeddings and the residue-code + shards it writes are unchanged byte for byte. Recompute the residue Parquet + files of earlier versions, or convert one with this function (pyarrow only), + giving the number of values per residue (1536 for FoldMatch's ESM3), as in + `nest("inference-0.parquet", "inference-0-nested.parquet", 1536)`: + + ```python + import pyarrow as pa + import pyarrow.parquet as pq + + def nest(src, dst, d): + table = pq.read_table(src) + chunks = [] + for flat in table.column("embedding").chunks: + offsets = flat.offsets.to_numpy() - flat.offsets[0].as_py() + if (offsets % d).any(): + raise ValueError(f"{src}: a row is not a multiple of {d} values") + values = flat.flatten().cast(pa.float32()) + offsets = pa.array(offsets // d, pa.int32()) # in residues + residues = pa.FixedSizeListArray.from_arrays(values, d) + chunks.append(pa.ListArray.from_arrays(offsets, residues)) + kind = pa.list_(pa.list_(pa.float32(), d)) + nested = pa.chunked_array(chunks, type=kind) + i = table.column_names.index("embedding") + pq.write_table(table.set_column(i, "embedding", nested), dst) + ``` +- **BREAKING — `--min-res-n` only where it selects chains.** `inference + residue-embedding`, `inference structure-embedding` and `inference + chain-embedding` took `--min-res-n`, whose help said that it keeps the + chains of a structure with at least that many residues, but each row of + their chain list names one chain, embedded as listed, so it did nothing. + They no longer take it: a command line that passes it stops with `No such + option` (exit status 2); leave it out. `inference complete-embedding` + keeps it, with the help of `inference assembly-embedding` ("Consider only + assembly chains with at least residues."), and passes it to + its assembly step only, where it selects the chains of each assembly. In + the library, `esm_inference.predict` and `structure_inference.predict` + given chains (`src_from=SrcProteinFrom.chain`, their default), and + `chain_inference.predict` given residue embedding files (`src_from` file, + its default) or Parquet, or an `inference_set`, now refuse a `min_res_n` + other than 0 (`RefusalError`) before anything is read: `min_res_n` selects + the protein chains of structures read whole (`src_from` structure). Leave + it at 0 in these calls, or list only the chains to embed. Their signatures + are unchanged. +- **BREAKING — one chain list, one name per row.** The chain lists of + `inference residue-embedding`, `structure-embedding`, `chain-embedding` and + `complete-embedding` (`--src-file`, `--src-chain-file`), and the file or + stream of the library's `EsmProtFromChain`, are read one way: CSV without a + header (a quoted field read as CSV reads it), of structure name, structure + file or URL, chain ID and embedding name. The embedding name now defaults + to `.` when the 4th column is missing or empty: + for such a row, `residue-embedding` stopped with torch's `invalid file + name: .../.pt` (with `--output-format csv`, it wrote a hidden `.csv` that + the next such row replaced; with `parquet` or `json`, an empty id), and + `structure-embedding` wrote an empty id. In the stream of + `EsmProtFromChain`, a row of fewer than four elements, which pandas refused + unless another row had four (its missing elements then read as `nan`), is + now read like a short line of a file, and a `None` name, read as `nan`, + gets the default too. `chain-embedding` read the residue embedding + `.` from `--res-embedding-location` and wrote the + chain embedding under that name, whatever the 4th column said, so it could + not find what `residue-embedding` wrote under another name, and it skipped + lines of one or two fields without a word. It now reads `.`, writes under the embedding name and does not + read the structure file column; a line of one or two fields gets an empty + chain ID; and a missing residue file is reported with its path and the + row's embedding name. Its old list reader, + `adapt_csv_to_embedding_chain_stream` in `foldmatch.utils.data`, is + removed. Lists that name each row `.`, and the + `NAME,,CHAIN` lists of residue files above, give the same files as before, + except that: (1) a space, a tab or other whitespace at a line's start or + end is part of a field (`chain-embedding` stripped each line; `x,,A ` now + names `x.A `; and `complete-embedding`, which gave ` x,x.cif,A,x.A` the + files of `x,x.cif,A,x.A`, refuses it, as its structure name is ` x`: remove + the whitespace); (2) a quoted field, or a byte-order mark at the start of + the file, is read as CSV (`chain-embedding` kept the quotes or the mark in + the names, and stopped on the missing file; the new reading works); (3) a + row of more than 4 fields is refused (`RefusalError`, naming the file or + the stream and the row) by the four commands, before any model is loaded, + and by `EsmProtFromChain`: `chain-embedding` ignored the fields after the + 3rd, and `residue-embedding`, `structure-embedding` and `EsmProtFromChain` + misread them, or stopped on a bare pandas error (but read right a list + whose every row started with an index column, as pandas' + `to_csv(header=False)` writes it); remove the extra fields (a comma after + the embedding name makes a 5th; write such a list with + `to_csv(header=False, index=False)`); (4) a list whose file name + ends in `.gz`, `.bz2`, `.zip`, `.xz`, `.zst` or `.tar` (letter case + ignored) is read as compressed, as `residue-embedding` and + `structure-embedding` read it (`chain-embedding` read it as plain text): + rename a plain-text list so named. Any other pandas `ParserError` (an + unclosed quote, say) is refused as in (3), with pandas' message. + `complete-embedding`, whose assembly step reads residue embeddings as + `.`, as `inference assembly-embedding` does, now + refuses a chain list with other embedding names before any model is loaded, + naming up to 5 rows (it failed, or ran on the residue embeddings found + under `.`: the row's own, as in (1), an earlier + run's, or another row's, as when two rows swap names): name them + `.`, or leave their 4th column out. The help of + `--src-file` and `--src-chain-file` describes the columns and the names. +- **BREAKING — an assembly list row of more than 4 fields is refused.** The + assembly lists of `inference assembly-embedding` (`--src-file`) and + `inference complete-embedding` (`--src-assembly-file`), and the file or + stream of the library's `ResidueAssemblyEmbeddingFromTensorFile` and + `ResidueAssemblyEmbeddingFromParquet`, hold 4 fields per row: structure + name, structure file or URL, assembly ID and embedding name. A row of more + is now refused (`RefusalError`, naming the file or the stream and the row), + by both commands before any model is loaded (by `complete-embedding`, for + an assembly list in a regular file), as a chain list row of more than 4 + fields is (above): remove the extra fields (a comma after the embedding + name makes a 5th). Before, pandas read the leading fields of a first such + row as the index, which shifted the columns of every row: a trailing comma + or a 5th field ended in a misleading error, such as `Structure file source + is neither a recognized URL nor file: '1'` (the assembly ID read as the + file) or the refusal of a structure name given to two files. A list whose + every row started with one or more extra fields, such as the index column + that pandas' `to_csv` writes unless `index=False`, worked (pandas took them + as the index, so the columns lined up) and is now refused: write it without + them (`to_csv(header=False, index=False)`). A longer later row stopped with + a bare pandas `ParserError` that named no file, and a stream row of 5 + elements with pandas' `ValueError`. Any other pandas `ParserError` (an + unclosed quote, say) is refused the same way, with pandas' message. Lists + of at most 4 fields per row are read as before. `complete-embedding` now + reads its assembly list before its residue step, so such a list, or one + that gives one structure name to two files, is refused before the ESM3 + step instead of after its chain step, which had written the residue and + chain embeddings. A list given as a pipe (a named pipe, or `<(...)` on + macOS), or on macOS as `/dev/stdin` or `/dev/fd/N` (even of a regular + file), which can be read once only, or at any other path under `/dev` (a + file in `/dev/shm`, say), is read once, into memory, and its text given to + the assembly step. The help + of `--src-file` and `--src-assembly-file` describes the columns. +- **BREAKING — a structure list or residue file list row of too many fields + is refused.** A structure list holds 3 fields per row (structure name, + structure file or URL, embedding name): it is the file or stream of the + library's `EsmProtFromStructure`, `EsmAssemblyFromStructure`, + `ResidueEmbeddingFromStructure` and + `ResidueAssemblyDatasetFromStructure`. A residue file list holds 2 (residue + embedding file, embedding name): it is the file or stream of + `ResidueEmbeddingFromTensorFile`. These lists are given in a file through + the library only, to `predict` of `esm_inference`, `structure_inference`, + `chain_inference`, `assembly_inferece`, `chain_complete_inference` and + `assembly_complete_inference`. The commands that use these readers + (`fm-embedding from-structures`, `fm-search build structures`, + `inference chain-embedding` and `complete-embedding`) give them streams + they build themselves, of the right length, and are unaffected. A row of + more fields is now refused (`RefusalError`, naming the file or the stream + and the row), before any model is loaded, as a chain or assembly list row + is (above): remove the extra fields (a comma after the embedding name makes + a 4th in a structure list, a 3rd in a residue file list). Before, pandas + read the leading fields of a first such row as the index, which shifted the + columns of every row (after a trailing comma, the embedding name was read + as the file). A list whose every row started with one or more extra fields, + such as the index column that pandas' `to_csv` writes unless `index=False`, + worked (pandas took them as the index, so the columns lined up) and is now + refused: write it without them (`to_csv(header=False, index=False)`). A + longer later row stopped with a bare pandas `ParserError` that named no + file, and a longer stream row with pandas' `ValueError`. Any other pandas + `ParserError` (an unclosed quote, say) is refused the same way, with + pandas' message. A residue file list in a file is now read as strings, as + its stream and every other list are: otherwise pandas 3 reads an index + column `0, 1, 2` like no index at all, and the refusal would miss it. So + the names of a list whose names all look like numbers, or all like booleans + (any of them may be empty), which pandas read as such, keep their spelling: + `007` and `1.50` give `007.csv` and `1.50.csv` (or `.pt`) where they gave + `7.0.csv` and `1.5.csv`, `TRUE` gives `TRUE.csv` where it gave `True.csv`, + and `1` next to an empty name gives `1.csv` where it gave `1.0.csv`; `1` + and `2` give `1.csv` and `2.csv`, as before. A space around a number, which + pandas dropped, now stays in the name (`a.pt, 1` gives ` 1.csv`). In JSON + output their ids are strings (`"007"`, `"1"`, `"TRUE"`) where they were + numbers or booleans (`7.0`, `1`, `true`), and Parquet output, which stopped + on them with pyarrow's `ArrowTypeError` and wrote nothing, now holds them + as written. An empty name or `NA` stays NaN, as before: `nan.csv`, `null` + in JSON, and that `ArrowTypeError` in Parquet. Lists of at most that many + fields per row are otherwise read as before. + +- **BREAKING — `inference chain-embedding` takes no `--structure-format`.** It + reads residue embeddings and no structure file, so the option did nothing. A + command line that passes it now stops with `No such option` (exit status 2); + leave it out. `inference complete-embedding` keeps the option, for its + residue and assembly steps, which read structures. In Python, + `foldmatch.cli.inference.chain_embedding` no longer takes `structure_format` + (`TypeError`); `chain_inference.predict` keeps it, for `src_from` structure. +- **BREAKING — `inference complete-embedding` names its assembly file + `assembly-inference` by default.** `--output-assembly-name` defaulted to + `chain-inference`, the default of `--output-chain-name`, so that with + `--output-format parquet` or `json` and one folder for + `--output-chain-path` and `--output-assembly-path`, the assembly file + replaced the chain file. Both are now kept. In any folder, the default + assembly file is now `assembly-inference-.parquet` or `.json.gz`: read + it under that name. Output in `csv` or `pt` does not use the name. +- **BREAKING — every row of a list has an embedding name of its own.** The + chain lists, assembly lists, structure lists and residue file lists of + `inference` and of the library (`read_chain_list`, `read_assembly_list`, + `read_structure_list`, `read_residue_file_list` and the readers built on + them) read `NA`, `nan`, `null` and pandas' other NA words as text, and a + missing or empty field as `''`, as chain lists already did; and a missing or + empty embedding name is a default, as in chain lists + (`.`): `-` in an + assembly list, the structure name in a structure list, the residue file's + name without its extension in a residue file list (`x` for `x.pt`). Before, + such a name was NaN, which gave the file `nan.csv` (every unnamed row of an + assembly list into that one file), the id `nan` in JSON, an + `ArrowTypeError` in Parquet, or, in a structure list, hidden files `.A.csv` + shared by every unnamed structure. A name given to more than one row is now + refused (`RefusalError`, naming the names and their rows), in every output + format and before any model is built: with `csv` or `pt` each row replaced + the one before, and in Parquet and JSON the rows could not be told apart + (the assembly readers keep one). `read_structure_list` no longer takes + `keep_default_na`. A list that relied on NaN names, or repeated a name, must + give each row its own name. +- **BREAKING — an `inference` run refuses outputs that would replace each + other or its inputs, before any model is built** (`RefusalError`, exit + status 1). With `--output-format csv` or `pt`, an embedding name is a file + name: one that is empty, `.` or `..`, holds `/` or NUL, or makes a file + name over 255 bytes is refused (`../x` wrote outside the output folder), and + so are two names that the output folder takes for one file (`A` and `a`, + or `straße` and `strasse`, on macOS's case-insensitive file system; a + case-sensitive folder keeps both), the names of a structure list's rows + (`.`) included. A folder that cannot be written is probed + on the names it holds. + `--output-name`, `--output-chain-name` and `--output-assembly-name` follow + the same rules in every format (`-.parquet`, a 6-digit rank + reserved). No output may be an input of its run: the list file, a structure + file of the list, or a residue embedding the run reads. + `chain-embedding` refuses an `--output-folder` that is + `--res-embedding-location` when `--output-format` is + `--res-embedding-format` (each chain embedding replaced the residue + embedding it was computed from); `assembly-embedding` refuses, in that same + case, an assembly name `.`, which can be a residue + embedding's name (`1acb-1` is fine); `chain_inference.predict` refuses a + Parquet input in `out_folder` named like its Parquet output. Folders, and + inputs and output files joined by a link, are compared as files: a + symbolic link to the folder is the folder, and a residue embedding linked + (symbolic or hard link) from another folder to an output's file, or an + output's file linked to it, is that file. + `complete-embedding` checks all three steps before its residue step, and + refuses two steps that would write one file: the residue and chain + embeddings in one folder with `--output-format pt` (the chain step replaced + its own input and the assembly step failed), or one name for + `--output-chain-name` and `--output-assembly-name` with Parquet or JSON in + one folder; and an output of any step over either list or a structure file + of either. An assembly list that can be read once (a pipe, `/dev/fd/N`) is + read once, before the residue step, and checked as one in a regular file is + (before, it was not checked against the other steps). Files of earlier runs + are not checked: a rerun into the same folders replaces its own earlier + files, as before. +- **BREAKING — `fm-embedding from-sequences` refuses FASTA ids that conflict + in its files, before ESM3.** A record id (the first word of a header) + repeated among the records kept (`--min-res-n`) is refused in every format, + as the residue-code pass of `fm-search build sequences --residue-codes` + refuses it: in `csv` and `pt` the last record replaced the others, in + Parquet and JSON two rows had one id. With `csv` or `pt`, an id that cannot + name a file (`../x` wrote outside the output folder, `Q9/1-100` stopped the + run with a traceback), ids that the output folder takes for one file (`abc` + and `ABC` on macOS), and a `--fasta-file` among the output files are + refused too; Parquet and JSON keep any id. `--output-name` follows the file + name rules in every `fm-embedding` command. `fm-search` keeps accepting a + repeated query id under BLOSUM62, with its warning. +- **BREAKING — a csv or pt writer refuses to write one file twice in a run.** + `CsvBatchWriter` and `TensorBatchWriter` (`fm-embedding`, `inference` and + the library with `csv` or `pt` output) keep the files they have written in + the run, compared as the output folder compares names, and stop the run + with a `RefusalError` naming the file at a second write of one: the first + file is kept, the second item is not written. It catches what is known only + while structures are read whole, such as chains `A` and `a` of one structure + on macOS's case-insensitive file system (one of them was lost), and two + structures whose `.` coincide. The check is per process, so + two ranks that write one file are not caught; it also refuses a name that + cannot be a file name there. +- **BREAKING — `fm-search` refuses an `--output-file` that is one of its + inputs, before any work.** `query structure`, `query embedding`, `query + sequences`, `query db`, `cluster` and `similarity-graph` refuse + (`RefusalError`, exit status 1) an output file that is a file of a + database they read (`.index`, `.metadata`, `.invlists`, + `.sequences` or `.rescodes.*`), the query FASTA or structure file, or a + query embedding file that `query embedding` reads. Compared as files: a + symbolic link to such a file, or on macOS another case of its name, is + refused too. An output file written by an earlier command is not an input, + even one named `.hits.tsv` beside the database, and is replaced as + before. +- **BREAKING — `fm-search query embedding` refuses query embeddings that + share an id**, before the database is loaded, naming the ids and their + files: two rows of one id in a Parquet file, one id in two Parquet files, + or a Parquet id that is also a `.pt` or `.csv` file's name. The results are + kept by query id, so all but one of them disappeared from the output. Give + each embedding its own id, or query the files apart (`--file-extension`). +- **`click` pinned at 8.3.1** (Typer's dependency, as in `uv.lock`): click + 8.5 words its usage errors differently. The tests no longer depend on + that wording. + +### Fixed + +- **`ivf_pq` searches of several queries at once could silently return wrong + hits.** With `faiss-cpu` 1.14.2 to 1.15.1, a FAISS call with 2 or more queries + on an `ivf_pq` database of `--ivf-nlist` 10000 or more searched only the first + `nlist // T` coarse cells for the queries in a slice of fewer than T queries, + where T is the number of FAISS threads and `nlist >= 1024 * T` + (facebookresearch/faiss#5824). With the default `--ivf-nlist` 16384 that means + 2 to 16 threads and fewer than T\*T queries per FAISS call: `query sequences` + makes one call per query shard (one shard per rank), and `query db` one per + 4096 queries, so its last call is usually short. No error was raised. + FoldMatch now sets FAISS's `parallel_mode` 3 on these databases, which gives + the correct result at every thread count, and no rebuild is needed. Results in + that range should be run again. A single-query call, an `nlist` under 10000 or + under 1024 times the thread count (so 17 or more threads at the default + 16384), and `flat` or `hnsw` databases were not affected. The defect is likely + solved in the next FAISS release: the upstream fix + (facebookresearch/faiss#5824) was merged on 2026-10-31, but it is not in + 1.15.1, the latest release and FoldMatch's pin. When the `faiss-cpu` pin moves + to a release that has it, this workaround is to be re-examined. +- **`fm-search cluster` and `fm-search similarity-graph` run on `ivf_pq` + databases.** They failed on every `ivf_pq` database, at the first chain, with + FAISS's `direct map not initialized`. The chains' embeddings are now read with + `reconstruct_n`, which needs no direct map. On `ivf_pq` a score is the inner + product of the chain's normalised decoded embedding with each stored PQ code, + as in `query db`: an estimate of the cosine, so two identical chains score the + norm of their code, close to but not exactly 1 (0.84 to 1.03 on a 30M-chain + database, 98% of chains between 0.94 and 1.01). +- **Assembly embeddings of entries with more than one assembly.** Structures + were cached under the entry name alone, so assemblies of an entry read one + after another all got the first one's chains. `fm-search build structures + --granularity assembly` and `fm-embedding from-structures assembly` gave + every assembly of an entry the chains of its first; `inference + assembly-embedding` and `inference complete-embedding` did the same for + assemblies of one entry read in a row by one process (input rows are sorted + by entry). `fm-search query structure --granularity assembly` read the + requested assembly, so such queries were compared with wrong database + vectors. The cache now keys on the assembly too, and each dataset has its + own. An assembly with no protein chain long enough (`--min-res`, + `--min-res-n`) is now skipped with a warning by the first two commands + (before, it got the first assembly's chains, and a first assembly without + one stopped the build); the `inference` commands refuse it when their input + lists it (`No chains found for ...`), as they did for a first assembly. + Given a URL, the assembly inference of the library + (`foldmatch.inference.assembly_complete_inference`) read the download to its + end while listing its assemblies, then failed on every entry; it now reads + it from its start. **Rebuild assembly databases, and recompute assembly + embeddings, of entries with more than one assembly.** +- **Assembly embeddings of assemblies with copies of a chain.** The copies of + a chain that an assembly's operators make keep the chain's ID, and + `fm-search build structures --granularity assembly` and `fm-embedding + from-structures assembly`, with the assembly inference of the library + (`foldmatch.inference.assembly_complete_inference`), read each protein + chain of an assembly by its ID, from the first stretch of atoms with that + ID: in mmCIF, BinaryCIF and PDB files alike, every copy of a chain was + embedded with the first copy's coordinates. Each copy is now read from its + own atoms, as `fm-search query structure --granularity assembly` already + read it. ESM reads each chain as PDB text, which cannot hold a coordinate + of -1000 Å or less, or of 10000 Å or more: a copy with one, embedded before + with its first copy's coordinates, now stops the first two commands and the + library (`Error while creating ESMProtein from structure chain ...`), as it + already stopped the query (none of the 1,341 assemblies with copies + checked, in 1,083 entries, has one). In mmCIF and BinaryCIF files laid out + as the PDB archive's are (each ligand with an asym ID of its own, each + chain's records in one block, with residue numbers that never decrease), + the number and order of the chains are unchanged, and the first copy of + each chain is read as before, bit for bit. In PDB files, which name chains + by their author IDs, and in any other mmCIF or BinaryCIF file (one whose + asym IDs are author chains, ligands included, as biotite writes from a PDB + structure, say), one ID can cover several pieces of a chain: how such a + chain is now read is in the next entry. `inference assembly-embedding` and + `inference complete-embedding`, which repeat a chain's stored residue + embeddings for each of its copies, and the chain-level commands are + unaffected. ESM3 embeds each chain on its own, and a copy differs from the + chain it copies only by a rigid motion and by the rounding of its + coordinates to 0.001 Å (after the motion, each atom within 0.0015 Å of its + first copy's, on 3,720 copies in 346 assemblies of 241 entries), so its + embeddings stay close to its first copy's, which it was given before: on 10 + assemblies with copies, from C2 dimers to a 60-copy capsid, 30 of the + copies' 17,875 structure tokens changed, and the assembly embeddings kept a + cosine of at least 0.99987 to the old ones (at least 0.99999999 for 8 of + them), no value moving by more than 0.31% of the vector's norm. Rebuilding + assembly databases, or recomputing assembly embeddings, for the copies is + therefore optional (for chains that one ID splits, see the next entry). +- **Chain and assembly embeddings of chains that one chain ID splits.** In + PDB files, which name chains by their author IDs and numbers, and in mmCIF + and BinaryCIF files not laid out as the archive's, one chain ID can cover + several pieces of a chain: where its residue numbers decrease (a tag + numbered 998-999 before residue 1, an insert numbered from 1001) and where + records of other chains lie between its records (its free amino-acid + ligands, HETATM groups listed after every chain, say). `fm-search build + structures --granularity chain`, `fm-embedding from-structures residue` and + `fm-embedding from-structures chain`, with the library's + `chain_complete_inference`, `esm_inference` and `structure_inference` given + structures (`foldmatch.inference`), listed such a chain once for each piece + long enough and embedded, each time, the first piece of the atoms with its + ID, in which blocks that only other chains separate run together, with the + chain's free amino-acid ligands listed later: a database could hold the + chain's id more than once, and only one piece of the chain, or all of it + with its free amino acids appended. `inference residue-embedding`, + `inference structure-embedding` and `inference complete-embedding`, with + `esm_inference` and `structure_inference` given chains, embedded that first + piece too; `fm-search query structure --granularity chain` + (`FoldMatch.residue_embedding_by_chain`), with or without `--chain-id`, + embedded every piece of at least `--min-res` amino acids and kept the last + one, the others embedded for nothing; `fm-search query structure + --granularity assembly` (`residue_embedding_by_assembly`), and since the + fix above the assembly builds, embedded the pieces of each copy apart; + `FoldMatch.residue_embedding` and `structure_embedding` embedded the pieces + apart; and the datasets of stored residue embeddings (`inference + assembly-embedding` and `inference complete-embedding`, and the library's + `chain_inference` given structures, `assembly_inferece` and + `ResidueAssemblyEmbeddingFromParquet`) took the chain's stored file once + for each piece. Every path now reads a protein chain the same way: the + pieces with its ID (in an assembly, of one copy), their amino-acid atoms + joined in file order into one protein, embedded in one pass, leaving out + the pieces whose amino acids are all HETATM records when it has amino acids + in ATOM records (a chain made only of HETATM amino acids, a peptide of + modified residues, say, keeps them all; HETATM groups within a piece of the + chain, right after its records with rising numbers, stay in it, as before). + It is a chain if its polymer amino-acid residues, counted piece by piece as + before, reach `--min-res-n` (`--min-res` for `fm-search`) together, so a + chain whose pieces are each shorter is now kept. Each chain is listed once + (in an assembly, once per copy): the database and the query embed the same + residues, each chain once (with `--chain-id`, that chain once), and the + datasets of stored residue embeddings take each chain's file once, as the + build writes it. A chain ID with no protein chain (water, or a lone free + amino acid with an asym ID of its own) is refused by the `inference` + commands above with an `OSError` that names the structure (the input's + first column), the chain and the file (`No protein chain found for ...`); a + lone amino acid was embedded before as a protein of one residue. In an + assembly of a PDB file, each copy of a chain is a chain of its own: biotite + numbers the copies made by each REMARK 350 list from 0, so FoldMatch now + numbers them itself as biotite builds them, and the two copies of a chain + in two lists no longer run together where its numbering rises from its last + residue to its first (a tag numbered 998-999 before residue 1, say). In + mmCIF and BinaryCIF files laid out as the archive's, every protein chain is + one piece: the chains listed and their residues are unchanged, bit for bit, + for every build; the query's threshold now counts polymer amino-acid + residues, as the build does, instead of every amino acid, which changes + only chains near `--min-res` and, at `--min-res` 0 or 1, free amino acids + with an asym ID of their own, no longer embedded as chains. + `FoldMatch.residue_embedding` and `structure_embedding` now read each chain + as the other paths do: HETATM residues within a chain (selenomethionine, + say), which ESM's PDB reader left out, are read, as X; a chain ID of two + characters is accepted; each chain is embedded in one pass, with BOS and + EOS rows at its ends only; and when no chain qualifies they raise `No + protein chain ... found in the structure`, and + `residue_embedding_by_assembly` returns an empty dict, as + `residue_embedding_by_chain` does (so `fm-search query structure + --granularity assembly` refuses the assembly with its own message), instead + of torch's error. In any file, archive mmCIF included, the results of + `residue_embedding` and `structure_embedding` for chains with HETATM + residues therefore change, and now agree with the database: recompute them. + `get_structure_from_src` and `StructureProvider.get_structure` + (`foldmatch.utils`) no longer take a `chain_id`: a selection by chain ID + joins the pieces that other chains separate; their `assembly_id`, which + took its place, is now keyword-only, so that an old positional call fails + too. In the database and the query, a chain that one ID does not split is + embedded as before (on 420 archive entries, every build and the query at + `--min-res` 10 gave the same chains and residues, bit for bit). On PDB + files written from 206 of them, every path now reads each of the 546 chains + that one ID splits once, as its archive mmCIF chain, bit for bit (before, + the database had 13 of them right and listed 23 more than once at + `--min-res` 10, 49 at 0, and the query kept the last piece of 26 at 10, and + a wrong piece of every one at 0), and in 180 of their 206 assemblies each + copy as in the mmCIF assembly, residue for residue (164 before; in the 26 + others, a chain keeps the free amino acids that the assembly leaves right + after it, as before, while its stored file, which `inference + assembly-embedding` and `inference complete-embedding` give each copy, now + lacks them). With ESM3, the database vector of such a chain now equals the + mmCIF chain's, bit for bit, and scores 1.0000 against the query vector of + the same chain (13 chains of 8 entries; before, 0.19 to 0.83 for 9 chains + cut where their numbering drops, and 0.999 for 2 with free amino acids + appended). **Rebuild chain and assembly databases, and recompute chain, + assembly and residue embeddings, made from PDB files, or from mmCIF and + BinaryCIF files not laid out as the archive's, with chains that one ID + splits**; a database built from them may hold a chain's id more than once. +- **`max_res` counts the residues of the chains a call embeds, as `min_res` + counts them.** `FoldMatch.residue_embedding_by_chain` and + `residue_embedding_by_assembly` (`fm-search query structure --max-res`, + `--granularity chain` and `assembly`) counted every amino-acid residue of + the structure or assembly, with its free amino acids, its chains under + `min_res` and the pieces that a chain leaves out, even with `--chain-id`, + so a chain within the limit was refused in a structure above it; + `residue_embedding` and `structure_embedding` counted the same, or the + atoms with the chain ID, their free amino acids included. Each now counts + the protein chains it is about to embed, as `min_res` counts them (polymer + amino-acid residues, piece by piece, summed over the pieces kept), all of + them together: those with `chain_id` (in an assembly, one per copy), else + every protein chain of the structure or assembly. Over `max_res`, the call + raises `ValueError` before any ESM3 pass, with the count, the limit and + what was counted. With no chain to embed, it gives what it gives without + `max_res` (`residue_embedding` and `structure_embedding` their + `ValueError`, the other two `{}`, and `fm-search query structure` its error + that no chain qualifies), where the old count could refuse it as too long. + Structures and chains refused before may now be embedded. The `--max-res` + help says what is counted. +- **Chain and assembly embeddings saved as CSV.** `fm-embedding` and + `inference` write a chain or assembly embedding to CSV (their default + `--output-format`) as one column, one value per line. `fm-search build + embeddings` and `fm-search query embedding` read a CSV of several rows as + residue embeddings and averaged its rows, so each vector became one number: + a database built from such files had dimension 1 (every score was +1 or -1), + and a query with them failed. A one-column CSV is now read as one vector + (and residue embeddings are refused: see *Changed*). **Rebuild + databases built from CSV chain or assembly embeddings**: `fm-search stats` + shows them as `Dimension: 1`, and the `fm-search query embedding`, + `query db`, `cluster` and `similarity-graph` results made with them are + meaningless (every score is 1 or -1). +- **Parquet embedding files, embedding dimensions and repeated ids.** + `fm-search build embeddings` and `fm-search query embedding` read one + embedding vector per row of a Parquet file's `embedding` column, a list of + floating-point values, and every row of a file must hold as many values as + its first. A file of residue embeddings, each chain's residue table + flattened into one row as FoldMatch wrote them, was cut into equal slices + that mixed chains (without a message when the values divided evenly) or + stopped on a numpy error that named no file. It is now refused at any + `--load-batch-size`, naming the file: compute chain embeddings with + `fm-embedding from-structures chain` or `fm-embedding from-sequences chain`. + A row with no embedding, whose batch was sliced the same way, a row with a + NaN or infinite value (never found, and as a query it found nothing) and an + `embedding` column of another type (a list of integers too, read before as + floats) are refused, naming the row or the type. An unreadable Parquet file + (empty, cut short by an interrupted run, or corrupted) and a file without + the `id` or `embedding` column are now refused, naming the file; they + stopped with pyarrow's message, which named no file, or with a traceback. + The embeddings read must have one dimension: files of two dimensions + stopped a build with a numpy error that named no file (or a FAISS error + with `--index-type ivf_pq`), and query embeddings of another dimension than + the database's stopped `fm-search query embedding` with an empty FAISS + `AssertionError`. Both are now refused as they are read, before FAISS, + naming the files (a Parquet file with its row) and the dimensions. A `.pt` + and a `.csv` file of one name in one folder (left, for example, by a + residue run and a chain run that wrote to one folder) were read as two + embeddings of one id; they are now refused, and `--file-extension .pt` or + `--file-extension .csv` reads one kind. A flat row stores no shape, so residue + embeddings whose rows all have one length (a single chain, or chains of one + length) still build a database, whose dimension is that length, but are + refused as queries. **Rebuild databases built from Parquet residue + embeddings** (`fm-search stats` shows a dimension other than that of the + chain embeddings, such as 236544) **or from a folder that held a `.pt` and + a `.csv` of one id.** +- **A structure name that names two files or URLs is refused.** The commands + that read structures read a structure once for all the input rows that name + it (one per chain or assembly), from a file or a download alike, so rows + that gave one name to two files got the first file's structure for both. + Such an input now stops with an error that names the files when the command + reads its input list. A structure folder that holds one entry in two + formats or compressions (`1acb.cif` and `1acb.cif.gz` both take the name + `1acb`): keep one copy, or choose one with `--structure-file-extension` + (`fm-embedding`) or `--file-extension` (`fm-search build structures`). An + `inference` input that gives one name to several files, such as models of + one target: give each its own name. One file under two spellings (a + relative path, a link, a case variant on a case-insensitive volume) is one + file. **Recompute embeddings, and rebuild databases, made from inputs that + gave one name to more than one file or URL.** +- **`--log-level warn` sets up logging.** It set up none in every command of + `fm-search`, `fm-embedding` and `inference` (the option's value is `warn`, + the set-up looked for `warning`), so a warning printed in Python's default + format (`WARNING:root:...`, `WARNING:foldmatch.:...` or the bare + message). It now prints in the format of `--log-level debug` + (`> time - [proc-N] - WARNING - file:line - message`), at WARNING level. As + at `debug`, Lightning's own INFO lines (`GPU available: ...`) still show; + only `info` hides them. +- **`fm-search build structures --granularity assembly` now honours `--min-res` + and `--batch-size`.** Both were dropped for assembly builds, so every chain + went into the assembly embedding, while `fm-search query structure + --granularity assembly` skips chains shorter than `--min-res`; database and + query embeddings were computed over different chains. Assembly databases + built with the default `--min-res 10` now leave out chains shorter than 10 + residues, so rebuild existing ones to match queries. +- **Multi-GPU runs without SLURM lost the embeddings of every rank but rank 0.** + Each process drew its own shard-name prefix, so rank 0 built the database, or + ran the query, from its own shard only and silently dropped the others. All + ranks now share one run id (see *Changed*). Rebuild databases that were built + outside SLURM on more than one device (`--devices` greater than 1, or the + default `--devices auto` on a machine with several GPUs). +- **Apple Silicon GPU (MPS).** On Apple Silicon, where MPS is the default + accelerator, embedding failed in the chain aggregator with + `NotImplementedError: aten::_nested_tensor_from_mask_left_aligned`. The + `PYTORCH_ENABLE_MPS_FALLBACK=1` workaround made it run, but gave wrong chain + embeddings for chains padded within a batch, that is with `--batch-size` + greater than 1. **Rebuild databases and redo embeddings computed on MPS with + `PYTORCH_ENABLE_MPS_FALLBACK=1` and `--batch-size` greater than 1.** FASTA + input now runs on MPS without the fallback flag and agrees with CPU to about + 1e-6 (relative), and the MPS memory cache is released after every batch. + Structure input on MPS still needs `PYTORCH_ENABLE_MPS_FALLBACK=1`, because + ESM3's structure encoder uses an operator MPS lacks; its chain and assembly + embeddings are now correct at any batch size as well. Chain and assembly + embeddings on CPU and CUDA are byte-identical to before (CUDA checked on an + A100). Code that calls `ResidueEmbeddingAggregator` directly with a padding + mask outside the batched inference fast path (autograd enabled, training, or + an unbatched input with a mask) now also leaves the padded rows out of the + sum, which changes, and corrects, its output. +- **`EmbeddingComputer.compute_from_fasta` (Python API) in a multi-process + run** now returns on every rank only once every rank has finished the pass. + Rank 0 could read the shards right after it returned and find another + rank's shard not yet written ("Found 1 embeddings in 1 shard(s) ... a + rank's shard is missing"). `fm-search` already waited and is unaffected. +- **A FASTA header without an id** (a line holding only `>`) is now refused + before the ESM3 pass with a one-line error naming the file and position; + it used to stop the build with an `IndexError` traceback. +- **A multi-rank `fm-search query sequences` writes the queries in input + order.** With more than one rank (`--devices 2`, several GPUs, or the tasks + of a multi-task `srun` step), rank 0 read the query embeddings rank by rank, + so the output file listed rank 0's queries first, then rank 1's, and so on, + under `--scoring blosum62` (with or without Stage 2) and `--scoring + esm-codes` alike. The queries are now written in the order of the FASTA + file, so a multi-rank run writes the file a 1-rank run writes from the same + embeddings; each query's hits were already the same. A 1-rank run's output + is unchanged. A repeated query id (accepted, with a warning, only under + BLOSUM62) is reported once, at its first record, with the hits of its last + record, as on one rank; on several ranks it could take its embedding scores + and candidates from one record and its alignments from the other. Stage 2 + now also aligns the record that Stage 1 searched when the last record of a + repeated id is shorter than `--min-res-n`: before, that short record's + sequence was aligned, even on one rank (the one case in which a 1-rank + run's output changes). + +- **The help of `inference` and `fm-search`.** Only help text changes. + `--log-level` read "Number of nodes to use for inference of embeddings." in + every `inference` embedding command and in `fm-search stats`, `cluster`, + `build structures`, `query structure` and `query db`. `inference + structure-embedding` named a pandas DataFrame file, an option `out-df-name` + that does not exist and a JSON file; it writes + `-.parquet`. `inference residue-embedding`, + `chain-embedding`, `assembly-embedding` and `complete-embedding` named one + output format, whatever `--output-format` said (`complete-embedding` a + single JSON file, where its default is one CSV file per chain or assembly); + they now name the default and say what each format writes. `chain-embedding` + and `assembly-embedding` also name the CSV residue embeddings they read + (`--res-embedding-format`). +- **`inference` runs that destroyed their own output or input, with exit + status 0.** Two rows of one embedding name kept the last row's embedding + only; unnamed assemblies all went to `nan.csv`; an embedding name `../x` + replaced a file outside the output folder; `chain-embedding` into its + residue folder, in the residue format, replaced every residue embedding by a + chain vector; `complete-embedding` with `--output-format pt` and one folder + for residues and chains did the same, then failed in its assembly step; and + the recipe of P9.H2, `--output-assembly-name chain-inference` with one + folder for chains and assemblies, brought back the loss it fixed. All are + now refused before any model is built (see *BREAKING — an `inference` run + refuses outputs that would replace each other or its inputs* and *BREAKING + — every row of a list has an embedding name of its own*). +- **`fm-embedding` runs that lost their own output or input, with exit status + 0.** `from-sequences` kept the last of the records of one id, wrote a record + `../x` outside the output folder, replaced its own `--fasta-file` when it was + named like an output, and on macOS kept one of two ids that differ in case; + `from-structures` kept one of two chains `A` and `a` of a structure on + macOS. Each is now refused (see the two *BREAKING* entries on + `fm-embedding from-sequences` and on the csv and pt writers). +- **`fm-search` commands that destroyed their own input, with exit status 0.** + `--output-file .index` (or `.metadata`, `.sequences`, ...) + replaced a file of the database the command had just searched, so every + later command on it failed until a rebuild; an output file named like the + query FASTA, structure or embedding file replaced it; and `query embedding` + dropped every query but one of a repeated id. Each is now refused (see the + two *BREAKING* entries on `fm-search`). +- **A step-2 build could delete another build's lock on Linux.** At its + end, `fm-search build sequences-index` removed `.build.lock` + when the file there had the inode of its own lock; Linux can give that + inode to a lock created right after the first was deleted, so a lock + deleted by hand during a build, then taken by a second build, was removed + by the first. The lock now holds a random `lock_id`, and a build removes + only the lock with its own. +- **`complete-embedding --src-assembly-file /dev/fd/N` on Linux** named, in + its refusals, the file the descriptor is open on instead of the path given, + and `/dev/stdin` from a pipe was refused as a file that does not exist. A + path under `/dev` is now taken as given (other paths are resolved as + before). + +## [0.10.0] - 2026-09-15 + +### Added + - **Compositional bias correction** (`--comp-bias-corr`, `--comp-bias-corr-scale`) on `fm-search query sequences` and `fm-search query db`, and as `comp_bias_corr` / `comp_bias_corr_scale` on `align_candidates`. A port of @@ -31,6 +1380,18 @@ While the major version is `0`, breaking changes are released in a minor bump. that benchmark) and alignments over biased regions get shorter, lowering coverage. Pass `--comp-bias-corr` to compare with a default mmseqs run. +## [0.9.0] - 2026-08-27 + +### Added + +- **`--max-evalue`** on `fm-search query sequences` and `fm-search query db`: + drops Stage-2 hits whose database E-value is above the threshold. Defaults to + `1e-3`, so a default search now reports only statistically significant + alignments; pass `--max-evalue inf` to keep every hit. Because the E-value is + a significance quantity, the option turns the significance pass on even when + neither `evalue` nor `bits` is in `--format-output` (and so, under + `--significance-mode default`, requires the 11/1 gap penalties). + ### Changed - **`--min-coverage` now defaults to `0.8`** (was `0.0`) on `fm-search query @@ -44,22 +1405,6 @@ While the major version is `0`, breaking changes are released in a minor bump. `min_coverage=0.0, max_evalue=None`; `max_evalue=None` is also required when no `subject_db_size` is available, since an E-value threshold needs a search space. -- **Structure-based chain selection keeps chains with at least `min_res_n` - residues** (was more than `min_res_n`). This covers `fm-search build - structures`, `fm-embedding` and `inference`, and matches `FoldMatch(min_res=...)` - at query time and the FASTA paths, which already used "at least". Databases and - embeddings built with a non-zero minimum now also include chains of exactly - that length. - -### Fixed - -- **`fm-search build structures --granularity assembly` now honours `--min-res` - and `--batch-size`.** Both were dropped for assembly builds, so every chain - went into the assembly embedding, while `fm-search query structure - --granularity assembly` skips chains shorter than `--min-res`; database and - query embeddings were computed over different chains. Assembly databases - built with the default `--min-res 10` now leave out chains shorter than 10 - residues, so rebuild existing ones to match queries. ## [0.8.0] - 2026-07-21 @@ -158,4 +1503,6 @@ explicit `--format-output` to pin the column order you expect. Not tracked in this file; see the git history and release tags. +[0.10.0]: https://github.com/rcsb/foldmatch/releases/tag/v0.10.0 +[0.9.0]: https://github.com/rcsb/foldmatch/releases/tag/v0.9.0 [0.8.0]: https://github.com/rcsb/foldmatch/releases/tag/v0.8.0 diff --git a/Dockerfile b/Dockerfile index 922a865..72ca72a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,17 +3,25 @@ FROM python:3.12-slim AS builder WORKDIR /app -# Copy only necessary files for installation -COPY pyproject.toml README.md LICENSE.md ./ +# Copy only necessary files for installation (hatch_build.py compiles the native Smith-Waterman kernel) +COPY pyproject.toml README.md LICENSE.md hatch_build.py ./ COPY src/ ./src/ -# Install build dependencies, build, strip, and cleanup in one layer +# A failed kernel compile fails the image build instead of producing a pure-Python install +ENV FOLDMATCH_REQUIRE_NATIVE=1 + +# Install build dependencies, build (non-editable; the hook compiles the kernel) and run the kernel selftest +# (DP golden hash, BLAS path within the S16 tolerance). Kept apart from the cleanup below, whose `|| true` would +# otherwise swallow a failed install or selftest. # hadolint ignore=DL3008 RUN apt-get update && \ apt-get install -y --no-install-recommends gcc g++ binutils && \ - pip install --no-cache-dir -e . && \ - # Strip binaries immediately to reduce size before copying - find /usr/local/lib/python3.12 -name "*.so" -exec strip --strip-unneeded {} \; 2>/dev/null || true && \ + pip install --no-cache-dir . && \ + python -m foldmatch.search.swkern + +# Strip, clean up and remove the build dependencies in one layer +# hadolint ignore=SC2015 +RUN find /usr/local/lib/python3.12 -name "*.so" -exec strip --strip-unneeded {} \; 2>/dev/null || true && \ find /usr/local/lib/python3.12 -type f -executable -exec strip --strip-unneeded {} \; 2>/dev/null || true && \ # Remove unnecessary files in builder stage find /usr/local/lib/python3.12 -type d -name "tests" -exec rm -rf {} + 2>/dev/null || true && \ @@ -31,10 +39,9 @@ FROM python:3.12-slim WORKDIR /app -# Copy the cleaned packages from builder +# Copy the cleaned packages from builder (a regular install: the source tree in /app is not needed at runtime) COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages COPY --from=builder /usr/local/bin /usr/local/bin -COPY --from=builder /app /app # Final cleanup for runtime image # hadolint ignore=SC2015 @@ -42,4 +49,7 @@ RUN find /usr/share/locale -mindepth 1 -maxdepth 1 ! -name 'en*' -exec rm -rf {} find /usr/share/i18n/locales -mindepth 1 -maxdepth 1 ! -name 'en_*' -exec rm -rf {} + 2>/dev/null || true && \ rm -rf /usr/share/doc/* /usr/share/man/* /usr/share/info/* 2>/dev/null || true +# The stripped kernel libraries work in the runtime image, which has no compiler +RUN python -m foldmatch.search.swkern > /dev/null + ENTRYPOINT ["inference"] diff --git a/README.md b/README.md index 9c2c1ce..52b6fc9 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ If you are interested in training a new model with a new structure dataset, visi - **Sequence-based embeddings** from FASTA files without requiring 3D structures - **Structure-level embeddings** aggregated via a transformer-based aggregator network - **Fast and efficient** FAISS-based similarity search -- **Two-stage sequence search** — an embedding prefilter followed by exact pairwise Smith-Waterman alignment, reporting sequence identity, coverage, and calibrated E-values over selectable output columns -- **Structural clustering** using the Leiden algorithm for biological assembly identification +- **Two-stage sequence search** — an embedding prefilter followed by exact pairwise Smith-Waterman alignment on FoldMatch's native SIMD kernel, reporting sequence identity, coverage, and calibrated E-values over selectable output columns +- **Structural clustering**: Leiden (python-igraph) on the similarity graph, or greedy representatives in one streamed pass for collections whose graph does not fit in memory - **Command-line interface** implemented with Typer for high-throughput inference workflows - **Python API** for interactive embedding computation and integration into analysis pipelines - **High-performance inference** leveraging PyTorch Lightning, with multi-node and multi-GPU support @@ -50,15 +50,107 @@ pip install -e . - ESM 3.2.3 - Lightning 2.6.1 - Typer 0.24.1 -- Biotite 1.6.0 -- FAISS 1.13.2 +- Biotite 1.7.1 +- FAISS 1.15.1 (the `faiss-cpu` wheel) - igraph 1.0.0 -- leidenalg 0.11.0 - PyTorch with CUDA support (recommended for GPU acceleration) +- A C compiler (GCC 12 or later, or Clang 14 or later) when FoldMatch is built from source (an install from the + sdist, `pip install -e .`): it compiles the native Smith-Waterman kernel of the sequence search's Stage 2. The + platform wheels (Linux x86_64 and aarch64, macOS arm64 and x86_64) include it. Without a compiler the install + still succeeds, with a warning, as a pure-Python package in which Stage 2 refuses to run and prints the reinstall + command; everything else works. **Optional Dependencies:** - `faiss-gpu` for GPU-accelerated similarity search (instead of `faiss-cpu`) +- a FAISS built on MKL, for faster `ivf_pq` builds of large databases on Linux x86_64 with Python 3.12 (see *FAISS + threads* below) + +### FAISS threads + +FoldMatch installs the `faiss-cpu` 1.15.1 wheel. On Linux the wheel bundles an OpenMP build of OpenBLAS +(`faiss_cpu.libs/libopenblaso-*.so`) and its own `libgomp`. FAISS's BLAS work therefore runs on FAISS's OpenMP +threads: the k-means and OPQ steps of an `ivf_pq` training, the IVF assignment of its adds, and batched flat searches. +Earlier FoldMatch versions pinned `faiss-cpu` 1.13.2, whose OpenBLAS is single-threaded. With it, a 30-million-entry +`ivf_pq` index took about 7 hours on any node. + +- **Choosing the threads.** Export `OMP_NUM_THREADS` to set FAISS's threads. It also sets the threads of FAISS's + OpenBLAS. `OPENBLAS_NUM_THREADS` has no effect on FAISS's OpenBLAS; it changes only the OpenBLAS copies of numpy + and scipy. When `OMP_NUM_THREADS` is unset, `fm-search build sequences-index` gives FAISS the physical cores it may + use and logs the count. +- **Build times.** On NERSC Perlmutter, `fm-search build sequences-index --residue-codes` over 30.4 million sequences + built the `ivf_pq` index in 1 h 45 min on a GPU node (one 64-core AMD EPYC 7763, 64 threads; 2 h 47 min for the + whole command) and in 2 h 37 min on a CPU node (two 64-core EPYC 7763, 128 threads; 3 h 33 min). Most of the + difference is the training, 40 min against 1 h 16 min: the OPQ step of the wheel's training does not gain from + more threads. In a benchmark on the CPU node, two OPQ iterations took 66 to 70 s at 4 to 16 threads and 173 s at + 128, while 10 iterations of the IVF k-means gained up to about 32 threads. +- **Recommended on large nodes.** With the wheel, run `build sequences-index` with `OMP_NUM_THREADS=64` on a node + with more than 64 physical cores, such as a 128-core CPU node. The setting applies to the whole command. This + advice rests on the 64-thread GPU-node build above and on the OPQ benchmark (121 s at 64 threads against 173 s at + 128); a full build at 64 threads on a 128-core node was not timed. With it, `build sequences-index` warns that FAISS + has fewer threads than physical cores; with the wheel that warning is expected. With the MKL build below, leave + `OMP_NUM_THREADS` unset: it built a 30-million-entry index in 43 minutes at 128 threads. +- **128 BLAS threads at most.** The x86_64 wheel's OpenBLAS is built with `MAX_THREADS=128`, so FAISS's BLAS work + runs on at most 128 threads. That OpenBLAS also lowers a higher OpenMP thread count to 128 when FAISS calls it. On + a node with more than 128 physical cores, FAISS therefore runs 128 threads, and `build sequences-index` warns that + FAISS has fewer threads than physical cores. +- **Reproducibility.** Two `hnsw` or `ivf_pq` builds are byte-identical only at `OMP_NUM_THREADS=1`. The wheel's + k-means results also change with the thread count. `OPENBLAS_NUM_THREADS=1` alone does not make FAISS + single-threaded. With the wheel, search results can change with FAISS's thread count too, in the last digits of + the scores and so in the order of tied hits: between 32 and 256 threads, about a quarter of 1,000 batched flat + searches and 3 to 5 `ivf_pq` searches were reordered. For bit-identical reruns, set `OMP_NUM_THREADS` to the same + value (unset, FAISS gets each node's core count). +- **Existing databases.** FAISS 1.13.2 and 1.15.1 read each other's index files. Searching one database with either + version gives the same hits up to float rounding (flat and `ivf_pq` scores within about 3e-7), which can reorder + tied hits and swap hits tied at the cut-off; with `hnsw`, 5 to 7% of 1,000 searches changed at least one hit. A + rebuilt `ivf_pq` index is a different, equally valid index: at 30 million entries, rebuilds shared 57 to 59% of + their top-100 hits with each other (1.13.2 against the MKL build below, the wheel against the MKL build, and two + wheel builds at 64 and 128 threads), with about the same self-hit rates. Searches whose results you compare should + all use the same database. +- **Start-up log.** On Linux x86_64, `import faiss` logs at INFO `Could not load library with AVX2 support due to: + ModuleNotFoundError("No module named 'faiss.swigfaiss_avx2'")`, then `Successfully loaded faiss.`. This is + expected: the 1.15.1 wheel has one library, which picks the CPU's instructions at run time (its compile options + include `DD`). +- **macOS.** The macOS wheel uses Apple Accelerate for BLAS and bundles its own `libomp`, so FoldMatch keeps + `OMP_NUM_THREADS=1` there by default (see *macOS notes*). + +**Optional: FAISS with MKL.** For Linux x86_64 and Python 3.12 only, the official FAISS conda channel, `pytorch`, +publishes a `faiss-cpu` 1.15.1 build on MKL. It builds large `ivf_pq` indexes faster than the wheel, mostly because +its OPQ training is faster. On one NERSC Perlmutter GPU node (64 cores), `build sequences-index --residue-codes` over +30.4 million sequences took 2 h 02 min with it, of which 59 min was the index (the wheel: 2 h 47 min and 1 h 45 min). +In a FAISS-only script on one CPU node (2× AMD EPYC 7763), with 128 threads bound to cores (`OMP_PROC_BIND=spread +OMP_PLACES=cores`; FoldMatch does not bind threads), it built a 30-million-entry `ivf_pq` index in 43 minutes. The +commands below replace the wheel with the conda build. They were tested on NERSC, in a conda env (Python 3.12) in +which FoldMatch and its dependencies had been installed with `pip`: + +```bash +python -m pip uninstall -y faiss-cpu +conda install -y --no-deps --override-channels -c pytorch -c conda-forge \ + "pytorch::faiss-cpu=1.15.1=py3.12_hf65b397_0_cpu" "pytorch::libfaiss=1.15.1=py3.12_hf65b397_0_cpu" \ + "conda-forge::mkl=2025.3.1=h0e700b2_12" "conda-forge::onemkl-license=2025.3.1=hf2ce2f3_12" \ + "conda-forge::tbb=2023.1.0=haa8c3ee_2" "conda-forge::libhwloc=2.15.0=default_hd7c4d0c_1001" \ + "conda-forge::libxml2=2.15.4=h7df9aa5_1" "conda-forge::libxml2-16=2.15.4=hf3af7cc_1" \ + "conda-forge::libiconv=1.18=h0cb94f2_3" "conda-forge::llvm-openmp=23.1.4=h7148c6a_0" \ + "conda-forge::_openmp_mutex=4.5=8_kmp_llvm" "conda-forge::libsvs-runtime=0.4.0=g7f7c639_0_0" \ + "conda-forge::python_abi=3.12=9_cp312" "conda-forge::libgcc=16.2.0=ha9f2e26_7" \ + "conda-forge::libgcc-ng=16.2.0=h69a702a_7" "conda-forge::libgomp=16.2.0=he0feb66_7" \ + "conda-forge::libstdcxx=16.2.0=h934c35e_7" +python -c "import faiss; print(faiss.__version__, faiss.get_compile_options())" # 1.15.1 OPTIMIZE AVX2 ... +``` + +- **Why `--no-deps`.** It keeps conda from replacing pip's numpy and its BLAS, but conda then checks nothing. The + listed packages are what `libfaiss` and MKL needed in that env, which had the rest; `libfaiss` needs `libgcc` 16 or + later. In another env, a library still missing makes `import faiss` fail with an `ImportError` that names it. +- **Reinstalling FoldMatch.** The conda package registers as the distribution `faiss`, not `faiss-cpu`, so `pip check` + reports `foldmatch ... requires faiss-cpu, which is not installed`. That is expected. Reinstall or upgrade FoldMatch + in this env only with `pip install --no-deps ...`. Without `--no-deps`, pip installs the `faiss-cpu` wheel over the + conda FAISS. +- **Updating the env.** Never run `conda update`, or a `conda install` without `--no-deps`, in this env: conda's view + of it is inconsistent on purpose. Repeat the `--no-deps` form instead. +- **OpenMP runtime.** It depends on the import order. FoldMatch's commands load torch first, so torch's `libgomp` + serves FAISS and MKL. A script that imports `faiss` first gets LLVM's `libomp`. Both work. +- **Thread logs.** threadpoolctl 3.6.0 does not list this MKL, so the index phase's `Thread pool:` lines do not show + FAISS's BLAS. @@ -72,21 +164,24 @@ The package provides two main interfaces: ## Command-Line Interface (CLI) -The toolkit ships three CLIs. Each is invoked with `--help` for full option documentation; the canonical examples below are enough to get started. +The toolkit ships two CLIs, `fm-embedding` and `fm-search`. Each is invoked with `--help` for full option documentation; the canonical examples below are enough to get started. ### `fm-embedding` — compute embeddings Two subcommand groups reflect input modality: ```bash +# Output folders must exist; give each kind of embedding its own (residue and chain files share names) +mkdir -p out/residue out/chain out/assembly out/seq-residue out/seq-chain + # Residue / chain / assembly embeddings from a folder of 3D structures -fm-embedding from-structures residue --src-folder data/pdb --output-path out --structure-format mmcif -fm-embedding from-structures chain --src-folder data/pdb --output-path out --structure-format mmcif -fm-embedding from-structures assembly --src-folder data/pdb --output-path out --structure-format mmcif +fm-embedding from-structures residue --src-folder data/pdb --output-folder out/residue --structure-format mmcif +fm-embedding from-structures chain --src-folder data/pdb --output-folder out/chain --structure-format mmcif +fm-embedding from-structures assembly --src-folder data/pdb --output-folder out/assembly --structure-format mmcif # Residue / chain embeddings from protein sequences in a FASTA file (no 3D required) -fm-embedding from-sequences residue --fasta-file seqs.fasta --output-path out -fm-embedding from-sequences chain --fasta-file seqs.fasta --output-path out +fm-embedding from-sequences residue --fasta-file seqs.fasta --output-folder out/seq-residue +fm-embedding from-sequences chain --fasta-file seqs.fasta --output-folder out/seq-chain # One-shot model download fm-embedding download-models @@ -94,6 +189,8 @@ fm-embedding download-models Assembly-level embeddings are only available under `from-structures` — there is no assembly concept for a bare sequence. +Each FASTA record's embedding is stored under its id, the first word of its header, so `from-sequences` refuses a repeated id before ESM3 is loaded. With `--output-format csv` or `pt` (one file per record or chain), an id is a file name: one that holds `/` or cannot name a file, ids that the output folder takes for one file (`abc` and `ABC` on macOS's default file system), and a `--fasta-file` named like an output are refused too, and a run stops at a second write of one file (chains `A` and `a` of one structure on macOS). Parquet and JSON keep any id. A rerun into the same folder replaces its own files, as before. + Run `fm-embedding [from-structures|from-sequences] [command] --help` for full options (batch size, accelerator, devices, output format, distributed settings, etc.). ### `fm-search` — build and query FAISS databases @@ -102,26 +199,33 @@ Run `fm-embedding [from-structures|from-sequences] [command] --help` for full op # Build a similarity-search database from structures, FASTA, or pre-computed embeddings fm-search build structures --structure-folder data/pdb --output-db dbs/my_db --tmp-embedding-folder tmp fm-search build sequences --fasta-file seqs.fasta --output-db dbs/my_db --tmp-embedding-folder tmp -fm-search build embeddings --embedding-folder out --output-db dbs/my_db +fm-search build embeddings --embedding-folder out/chain --output-db dbs/my_db # Query the database (--output-file is required) fm-search query structure --db-path dbs/my_db --query-structure q.cif --output-file hits.tsv fm-search query sequences --db-path dbs/my_db --fasta-file q.fasta --tmp-embedding-folder tmp --output-file hits.tsv -fm-search query embedding --db-path dbs/my_db --embedding-folder out --output-file hits.tsv +fm-search query embedding --db-path dbs/my_db --embedding-folder out/chain --output-file hits.tsv fm-search query db --query-db-path dbs/queries --subject-db-path dbs/my_db --output-file hits.tsv # Inspect, cluster, export fm-search stats --db-path dbs/my_db fm-search cluster --db-path dbs/my_db --output-file clusters.tsv +fm-search cluster --db-path dbs/my_db --output-file clusters.tsv --method greedy fm-search similarity-graph --db-path dbs/my_db --output-file graph.graphml ``` All `build` commands accept `--index-type [auto|flat|hnsw|ivf_pq]` and IVF-PQ tuning flags (`--ivf-nlist`, `--ivf-nprobe`). See `fm-search --help` for the full surface. +`cluster` and `similarity-graph` work on every database type. On `ivf_pq` a score is estimated from the PQ codes (the chain's decoded embedding, normalised, against each stored code, as in `query db`), so two identical chains score close to, but not exactly, 1. With `--method leiden` (the default) the k-NN search takes time in proportion to the number of chains and `--max-neighbors`, whatever `--threshold`; `--method greedy` searches only chains no representative has taken, so its time also depends on `--threshold` and on database order. The graph's memory grows with the number of edges, which `--threshold` and `--max-neighbors` bound (at most `--max-neighbors` edges from each chain's search), and the Leiden step's memory also grows with the number of chains. The Leiden step of `cluster` runs on one CPU thread. It runs a fixed number of iterations of python-igraph's Leiden (modularity at `--resolution`), 10 unless `--leiden-iterations` says otherwise; `--seed` makes it repeatable. + +`cluster --method greedy` clusters collections whose similarity graph does not fit in memory. When most chains have `--max-neighbors` neighbours above `--threshold`, the Leiden route keeps about `n x --max-neighbors / 2` edges for `n` chains: 447,547,205 edges for 1,000,000 chains in an `ivf_pq` database (`--max-neighbors 1000`, `--threshold 0.8`), about 15.1e9 for 30.4M chains (`ivf_pq`); with fewer neighbours above the threshold, fewer (219,292,519 for the same 1,000,000 chains on `hnsw`, where few chains reach `--max-neighbors`). Leiden's memory grows with the edges (on that 1M-chain `ivf_pq` database, `cluster` peaked at 28.16 GiB, about 66 bytes an edge above the memory before the load, on Linux x86_64 at 128 threads), greedy's with the chains: it picks representatives in one streamed pass over the database, dropping each block's search results once used. A cluster is a representative and the still-unclustered chains among its `--max-neighbors` nearest that score at least `--threshold` (MMseqs2/CD-HIT style): chains are taken in database order, and a chain in no cluster yet becomes a representative. When most representatives have `--max-neighbors` neighbours above the threshold (the statistics report the share; 98.96% of rows in the first 10 blocks of a 30.4M-chain `ivf_pq` database), clusters are bounded by `--max-neighbors`, and a representative or singleton can score above the threshold with an earlier representative. Greedy clusters are smaller and tighter than Leiden's few large communities at 0.8: higher family purity, lower completeness and family ARI (measured at 100k and 1M chains). Results depend on database order (the order chains were added), as CD-HIT's depend on length order. `--max-neighbors` caps cluster size at `--max-neighbors` + 1 chains (on `hnsw`, FAISS's `efSearch` of 16 returns fewer: 481.7 of 1000 on average on a 100k-chain database, so a larger value adds few). Greedy needs no graph and computes no modularity; it has no resolution or iteration count and is deterministic, so `--resolution`, `--leiden-iterations` and `--seed` are refused with it. + #### Output format Every command writes **tab-separated rows** to `--output-file` — one record per line. `query` results are ordered best-first per query, so rank is simply the row order. +`--output-file` cannot be one of the command's inputs: a file of a database it reads (`.index`, `.metadata`, `.invlists`, `.sequences` or `.rescodes.*`), the query FASTA or structure file, or a query embedding file; such a run is refused before any work. A file written by an earlier command is replaced. `query embedding` refuses query embeddings that share an id (results are kept by query id). + `--format-mode` selects the file layout: | Mode | Layout | @@ -153,11 +257,21 @@ A **sequence-identity** search (Stage 2, below) emits the columns chosen with `- chain_id cluster_id cluster_size ``` +and, with `--method greedy`, a fourth column, the representative's chain id (equal to `chain_id` on the representative's own row): + +``` +chain_id cluster_id cluster_size representative +``` + +`cluster_id` 0 is the largest cluster, under both methods (ties under greedy in representative order). Rows are in database order; `--min-cluster-size` drops the rows of smaller clusters. + (`similarity-graph` is the one exception: GraphML is a graph format, not tabular.) #### Two-stage sequence search (exact identity) -`build sequences` also writes a sidecar `{db}.sequences` store next to the FAISS index. This lets sequence-built databases report **exact** sequence identity, not just embedding similarity: when you run `query sequences` (or `query db`) against such a database, a second stage pairwise-aligns each embedding hit (local Smith-Waterman, BLOSUM62) and re-ranks the surviving hits by identity. +`build sequences` also writes a sidecar `{db}.sequences` store next to the FAISS index. This lets sequence-built databases report **exact** sequence identity, not just embedding similarity: when you run `query sequences` (or `query db`) against such a database, a second stage pairwise-aligns each embedding hit (local Smith-Waterman, BLOSUM62) and re-ranks the surviving hits by bitscore. + +Stage 2 aligns on FoldMatch's native SIMD Smith-Waterman kernel, a compiled library (see *Requirements*). It reproduces biotite 1.7.1's `align_optimal` (local, affine gaps), ties included, which Stage 2 used before: the same scores, coordinates and alignments, so the same output. Without the kernel, `query sequences` and `query db` refuse to run Stage 2 before the queries are embedded or the database is searched, and print the command that rebuilds it; `--no-seq-identity` still runs an embedding-only search. Stage-2 hits use the same **tab-separated, no-header** format as every other query output. Pick the columns with `--format-output`: @@ -192,19 +306,15 @@ Supported `--format-output` fields: - **Auto by default**: Stage 2 runs when the database(s) carry a sequence store and falls back to embedding-only otherwise. Force it with `--seq-identity` (errors if no store is present) or disable with `--no-seq-identity`. `query db` requires **both** databases to have sequence stores. - Hits below `--min-seq-identity` (default `0.3`) or `--min-coverage` (default `0.8`) are dropped, as are hits above `--max-evalue` (default `1e-3`; pass `inf` to keep every hit). `--max-evalue` turns the significance pass on even when neither `evalue` nor `bits` is in `--format-output`. -- Tuning: `--gap-open`, `--gap-extend`, and `--align-workers` (defaults to all CPUs on the node). +- Tuning: `--gap-open` (default `11`) and `--gap-extend` (default `1`), where a gap of length L costs gap-open + (L − 1) × gap-extend, and `--align-workers` (defaults to all CPUs on the node). `--gap-open` must be at least 1 and `--gap-extend` at least 0, both at most 1,048,576; the sign is ignored. With `--significance-mode default`, a run that needs significance (an `evalue` or `bits` column, or a finite `--max-evalue`, which the default `1e-3` is) needs the gap costs 11/1: for other gap costs use `--significance-mode sampled`, or `--max-evalue inf` with neither column in `--format-output`. Both rules are checked before the queries are embedded (`query sequences`) or the database is searched (`query db`). - Compositional bias correction: off by default, so alignments are scored with plain BLOSUM62. `--comp-bias-corr` (scaled by `--comp-bias-corr-scale`, default `1`) applies mmseqs's correction for locally biased amino-acid composition, which mmseqs enables by default; use it when comparing with `mmseqs search`. With the same setting on both tools, raw scores, bit scores and E-values match `mmseqs search` (18-8cc5c) on the same pairs, except for alignments containing a non-standard residue (`X`, `B`, `Z`, `J`, `U`, `O`, `*`), which the two tools score differently. The correction can shorten alignments over biased regions, so coordinates, identity and coverage change too. - `evalue`/`bits` use calibrated Karlin–Altschul statistics by default (`--significance-mode default`). `--significance-mode sampled` estimates λ/K by sampling for other gap penalties, but its absolute magnitudes are relative-only. The significance pass is skipped automatically when neither `evalue` nor `bits` is in `--format-output`. -### `inference` — low-level inference subcommands - -Lower-level entry point exposing individual inference passes (`residue-embedding`, `structure-embedding`, `chain-embedding`, `assembly-embedding`, `complete-embedding`). Mostly useful for advanced workflows that compose inference stages explicitly. Run `inference --help` for the command list. - --- ## Python API -The `RcsbStructureEmbedding` class provides methods for computing embeddings programmatically. +The `FoldMatch` class provides methods for computing embeddings programmatically. ### Basic Usage @@ -221,6 +331,8 @@ model.load_models() # Auto-detects CUDA # model.load_models(device=torch.device("cuda:0")) ``` +A protein chain is embedded only if it has at least `min_res` residues (default 10), counted as polymer amino-acid residues: free amino acids are not counted. With `max_res` (default `None`, no limit), a structure method raises `ValueError`, before any ESM3 pass, when the protein chains it is about to embed have more than `max_res` residues together, counted the same way: the chains with `chain_id` (in an assembly, one per copy), else every protein chain of the structure or assembly. `sequence_embedding` raises `ValueError` when the sequence is shorter than `min_res` or longer than `max_res`. + ### Methods #### `load_models(device=None)` @@ -261,10 +373,10 @@ Compute per-residue embeddings for a structure. **Parameters:** - `src_structure`: File path, URL, or file-like object - `structure_format`: `'mmcif'`, `'binarycif'`, or `'pdb'` -- `chain_id`: Specific chain ID (optional, uses all chains if None) +- `chain_id`: Chain ID (optional): the protein chains with this ID (in an assembly, one per copy); every protein chain if None - `assembly_id`: Assembly ID for biological assembly (optional) -**Returns:** `torch.Tensor` of shape `[num_residues, embedding_dim]` +**Returns:** `torch.Tensor` of shape `[L + 2, 1536]` for one chain of `L` residues: one row per residue, between ESM3's BOS and EOS rows. Several chains give their tensors one after another, each with its BOS and EOS rows. Raises `ValueError` when no protein chain qualifies, or when `max_res` is exceeded. ```python # Single chain @@ -294,7 +406,7 @@ assembly_residues = model.residue_embedding( Compute per-residue embeddings separately for each chain. -**Returns:** `dict[str, torch.Tensor]` mapping chain IDs to embeddings +**Returns:** `dict[str, torch.Tensor]` mapping each chain ID to its embeddings, of shape `[L + 2, 1536]`; `{}` when no chain qualifies ```python chain_embeddings = model.residue_embedding_by_chain( @@ -316,7 +428,7 @@ chain_a = model.residue_embedding_by_chain( Compute residue embeddings for an assembly. -**Returns:** `dict[str, torch.Tensor]` mapping assembly ID to concatenated embeddings +**Returns:** `dict[str, torch.Tensor]` mapping the assembly ID (`'0'` for the asymmetric unit when `assembly_id` is None) to the tensors of its protein chains (one per copy) one after another, each with its BOS and EOS rows; `{}` when no chain qualifies ```python assembly_emb = model.residue_embedding_by_assembly( @@ -336,7 +448,7 @@ Compute residue embeddings from amino acid sequence (no structural information). **Parameters:** - `sequence`: Amino acid sequence string (plain or FASTA format) -**Returns:** `torch.Tensor` of shape `[sequence_length, embedding_dim]` +**Returns:** `torch.Tensor` of shape `[L + 2, 1536]` for a sequence of `L` residues (ESM3's BOS and EOS rows included) ```python # Plain sequence @@ -356,7 +468,7 @@ seq_emb = model.sequence_embedding(fasta) Aggregate residue embeddings into a single structure-level vector. **Parameters:** -- `residue_embedding`: `torch.Tensor` from residue embedding methods +- `residue_embedding`: `torch.Tensor` from any of the residue embedding methods (for `residue_embedding_by_chain` and `residue_embedding_by_assembly`, a value of their dict) **Returns:** `torch.Tensor` of shape `[1536]` @@ -451,7 +563,7 @@ pytest ### macOS notes -**The problem.** PyPI wheels for `faiss-cpu` and `torch` (pulled in via `lightning`) each bundle their own copy of `libomp.dylib`. On macOS, both copies get loaded into the same Python process. Whenever FAISS enters an OpenMP-parallel section (batched search with more than one query vector, `IndexHNSWFlat` graph construction, IVF-PQ training) the second OpenMP runtime fails to `pthread_mutex_init` and the call deadlocks — the CLI appears to hang indefinitely. Linux installs are unaffected because both libraries share a single OpenMP runtime. +**The problem.** PyPI wheels for `faiss-cpu` and `torch` (pulled in via `lightning`) each bundle their own copy of `libomp.dylib`. On macOS, both copies get loaded into the same Python process. Whenever FAISS enters an OpenMP-parallel section (batched search with more than one query vector, `IndexHNSWFlat` graph construction, IVF-PQ training) the second OpenMP runtime fails to `pthread_mutex_init` and the call deadlocks — the CLI appears to hang indefinitely — or the process crashes (a segmentation fault at `OMP_NUM_THREADS=4` on an M1 Max, with torch 2.11 and `faiss-cpu` 1.13.2 or 1.15.1). Linux installs are unaffected: there the copies are GNU `libgomp` builds, which can be loaded together, and FoldMatch's commands, which load torch first, run FAISS on torch's copy. **Affected commands** on macOS without mitigation: @@ -459,6 +571,7 @@ pytest - `fm-search query embedding` with a multi-row `.parquet` file. - `fm-search query sequences` with more than one input sequence. - `fm-search query db` (database-to-database). +- `fm-search cluster` and `fm-search similarity-graph`, which search many chains per FAISS call (safe at the default `OMP_NUM_THREADS=1` pin). Single-query paths (`fm-search query structure`, small `--index-type flat` builds) are unaffected. diff --git a/benchmarks/fmsw_probe.c b/benchmarks/fmsw_probe.c new file mode 100644 index 0000000..ef227ec --- /dev/null +++ b/benchmarks/fmsw_probe.c @@ -0,0 +1,92 @@ +/* Benchmark-only probe of the native kernel (benchmarks/stage2_esm_codes.py, mode 'components'; plan P7 profiling + * gate): the per-pair loops of fmsw_score_fwd_rev and fmsw_align with a clock around each stage, so the worker time + * the kernel call reports as one number splits into rowcol z, DP and traceback. + * + * The shipped kernel source is #included UNCHANGED and this file is compiled with the shipped build's flags for the + * ISA (foldmatch._native.build.compile_command), so the stage functions are the shipped ones; the probe's scores, + * best cells and paths are checked bit for bit against the shipped library by the benchmark. Never shipped. + */ +#define _POSIX_C_SOURCE 200809L +#include +#include "fmsw.c" + +static double probe_now(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + 1e-9 * (double)ts.tv_nsec; +} + +/* fmsw_score_fwd_rev's loop; t[0] += rowcol z (fwd + rev), t[1] += score-only DP (fwd + rev) */ +FMSW_API int probe_score_fwd_rev(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, + float offset, float go, float ge, float *s_fwd, float *s_rev, float *ws, + size_t ws_len, double *t) +{ + int64_t mmax = 0; + int st = check_pairs(sim, n, pair_off, m, K, offset, go, ge, &mmax); + if (st) return st; + if (K > 0 && (!s_fwd || !s_rev || !ws || !t)) return FMSW_ERR_ARGS; + if (K > 0 && ws_len < fmsw_ws_floats(n, mmax)) return FMSW_ERR_WORKSPACE; + float *ra, *rc, *cb, *cd, *dws; + ws_layout(ws, n, mmax, &ra, &rc, &cb, &cd, &dws); + for (int k = 0; k < K; k++) { + const int mk = m[k]; + int bi, bj; + if (mk == 0) { s_fwd[k] = s_rev[k] = 0.0f; continue; } + const float *fwd = sim + pair_off[k]; + const float *rev = fwd + (ptrdiff_t)(n - 1) * mk; + double a = probe_now(); + rowcol_coef4(fwd, n, mk, mk, offset, ra, rc, cb, cd); + double b = probe_now(); + s_fwd[k] = ad_score(fwd, n, mk, mk, ra, rc, cb, cd, go, ge, dws, &bi, &bj); + double c = probe_now(); + rowcol_coef4(rev, n, mk, -(int64_t)mk, offset, ra, rc, cb, cd); + double d = probe_now(); + s_rev[k] = ad_score(rev, n, mk, -(int64_t)mk, ra, rc, cb, cd, go, ge, dws, &bi, &bj); + double e = probe_now(); + t[0] += (b - a) + (d - c); + t[1] += (c - b) + (e - d); + } + return FMSW_OK; +} + +/* fmsw_align's loop with traceback; t[0] += rowcol z, t[1] += DP with traceback flags, t[2] += traceback */ +FMSW_API int probe_align_tb(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, float offset, + float go, float ge, float *scores, int32_t *ends, int32_t *path_buf, size_t path_cap, + int64_t *path_off, float *ws, size_t ws_len, uint8_t *flags, size_t flags_len, double *t) +{ + int64_t mmax = 0; + int st = check_pairs(sim, n, pair_off, m, K, offset, go, ge, &mmax); + if (st) return st; + if (K > 0 && (!scores || !ends || !ws || !path_off || !path_buf || !flags || !t)) return FMSW_ERR_ARGS; + if (K > 0 && ws_len < fmsw_ws_floats(n, mmax)) return FMSW_ERR_WORKSPACE; + if (K > 0 && flags_len < fmsw_flag_bytes(n, mmax)) return FMSW_ERR_WORKSPACE; + float *ra, *rc, *cb, *cd, *dws; + ws_layout(ws, n, mmax, &ra, &rc, &cb, &cd, &dws); + int64_t po = 0; + path_off[0] = 0; + for (int k = 0; k < K; k++) { + const int mk = m[k]; + int bi = 0, bj = 0, plen = 0; + float sc = 0.0f; + if (mk > 0) { + const float *fwd = sim + pair_off[k]; + if ((size_t)po + (size_t)n + (size_t)mk > path_cap) return FMSW_ERR_WORKSPACE; + double a = probe_now(); + rowcol_coef4(fwd, n, mk, mk, offset, ra, rc, cb, cd); + double b = probe_now(); + sc = ad_tb(fwd, n, mk, mk, ra, rc, cb, cd, go, ge, dws, flags, &bi, &bj); + double c = probe_now(); + plen = traceback(flags, mk, bi, bj, path_buf + 2 * po); + double d = probe_now(); + t[0] += b - a; + t[1] += c - b; + t[2] += d - c; + } + scores[k] = sc; + ends[2 * k] = bi; ends[2 * k + 1] = bj; + po += plen; + path_off[k + 1] = po; + } + return FMSW_OK; +} diff --git a/benchmarks/stage2_capture.py b/benchmarks/stage2_capture.py new file mode 100644 index 0000000..ce3ce03 --- /dev/null +++ b/benchmarks/stage2_capture.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +"""Run ``fm-search ARGS...`` (``foldmatch.cli.search.main``, what the console script calls) and save the exact inputs +of its esm-codes Stage 2 to ``$FM_CAPTURE_DIR`` for ``stage2_esm_codes.py replay`` (plan section 8 P7, profiling +gate on the real pipeline: real Stage-1 candidates, so the real E-filter survivor fraction). + +Saved: ``queries.f16`` (a copy of the run's query-encoding memmap, before the CLI deletes it) and ``capture.json``: +its entries (offset, length, seq_sha1), the Stage-1 candidates ``{query: [[ids], [embedding scores]]}`` exactly as +Stage 2 received them, the query sequences, N, the code store's ``.bin`` path, the keyword options of the call (the +output fields as a list), the hits' signature (``stage2_esm_codes.signature``) and the Stage-2 ``stats`` (with the +profile when ``FOLDMATCH_STAGE2_PROFILE=1``); and the wall time of the CLI's phases ``stage2`` and ``output`` +(``output.write_aligned_results``). The wrappers pass arguments and results through unchanged. Several calls in +one run (not the CLI's case) get ``capture.json``, ``capture_1.json``, ... +""" +import dataclasses +import json +import os +import shutil +import sys +import time +from pathlib import Path + +T0 = time.perf_counter() +sys.path.insert(0, str(Path(__file__).resolve().parent)) + + +def main(): + out = Path(os.environ["FM_CAPTURE_DIR"]) + out.mkdir(parents=True, exist_ok=True) + import foldmatch.cli.search as cli + import foldmatch.search.code_alignment as ca + import foldmatch.search.output as output + from stage2_esm_codes import signature + + rec = dict(argv=sys.argv[1:], phases={}, calls=0, import_s=time.perf_counter() - T0) + orig_s2, orig_out = ca.align_candidates_esm_codes, output.write_aligned_results + + def stage2(query_index, prefilter_results, code_store, fetch, prepared, db_residues, pool, **kw): + n = rec["calls"] + rec["calls"] += 1 + suffix = "" if n == 0 else f"_{n}" + shutil.copyfile(query_index.path, out / f"queries{suffix}.f16") + if kw.get("stats") is None: + kw["stats"] = {} + t0 = time.perf_counter() + res = orig_s2(query_index, prefilter_results, code_store, fetch, prepared, db_residues, pool, **kw) + dt = time.perf_counter() - t0 + opts = {} + for k, v in kw.items(): + if k in ("stats", "query_sequences"): + continue + if k == "output_needs" and not isinstance(v, (list, tuple)): + v = dataclasses.asdict(v) + opts[k] = list(v) if isinstance(v, tuple) else v + cap = dict( + rows=query_index.rows, width=query_index.width, system_name=getattr(query_index, "system_name", ""), + entries={q: [int(e.offset), int(e.length), bytes(e.seq_sha1).hex()] for q, e in query_index.entries.items()}, + prefilter_results={q: [list(v[0]), [float(s) for s in v[1]]] for q, v in prefilter_results.items()}, + query_sequences=dict(kw.get("query_sequences") or {}), db_residues=int(db_residues), + code_store_bin=str(code_store.bin_path), pool=dict(repr=repr(pool), workers=getattr(pool, "workers", None)), + options=opts, stage2_s=dt, signature=signature(res), hits=sum(len(v) for v in res.values()), + stats=kw["stats"]) + (out / f"capture{suffix}.json").write_text(json.dumps(cap, default=str)) + rec["phases"].setdefault("stage2", []).append(dt) + return res + + def write(*args, **kwargs): + t0 = time.perf_counter() + try: + return orig_out(*args, **kwargs) + finally: + rec["phases"].setdefault("output", []).append(time.perf_counter() - t0) + + ca.align_candidates_esm_codes, output.write_aligned_results = stage2, write + sys.argv = ["fm-search"] + sys.argv[1:] + rc = 0 + try: + cli.main() + except SystemExit as e: + rc = e.code if isinstance(e.code, int) else (0 if e.code is None else 1) + raise + finally: + rec["rc"] = rc + rec["total_s"] = time.perf_counter() - T0 + (out / "cli_phases.json").write_text(json.dumps(rec, indent=1, default=str)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/stage2_esm_codes.py b/benchmarks/stage2_esm_codes.py new file mode 100644 index 0000000..8a0a374 --- /dev/null +++ b/benchmarks/stage2_esm_codes.py @@ -0,0 +1,1444 @@ +#!/usr/bin/env python +"""esm-codes Stage-2 benchmark and profiling gate (not a test): plan section 7 P3 acceptance (e) and section 8 P7. + +Data (all real; md5-checked): + chains ``L2m_B8_s0.chains.npz`` (md5 5519866044ab0f348c74b9319125acd2): the 559 chains of the full-chain + evaluation set, target codes (and chain query encodings for the research queries of ``kernel``); + chain seqs ``chains_seq.tsv`` (md5 7996b1728759fe8424366843f95d8961): their sequences (id, sequence, ...), for + BLOSUM62, the round-2 metrics and the sequence store; every length equals the codes'; + queries ``queries.L2m_B8_s0.npz`` (md5 e821497f59dda06a3899c18dce37286c): the 1,465 real benchmark queries (ESM3 + query encodings q fp16 and sequences; 21-966 aa, median 71). +Candidates: the research throughput draw ``np.random.default_rng(20260925).integers(0, 559, 10000)``; candidate k +is ``#`` (a database of 10,000 entries, 2,218,273 residues); one query gets the first K, query j of a +batch the slice ``[(617 j) mod (10000 - K), + K)`` (research bench_batch). Fixed-length queries follow the research +rule on the real queries: the shortest real query with length >= Lq, cut to its first Lq residues; Lq = 1000 (> 966) +is the longest real query + the first residues of the next longest. N (the E-value database size) is +``--db-residues`` (default 15,889,271, the residues of the research E-value benchmark's 50,000 targets), so the +E-value filter keeps the real survivor fraction of these pairs. + +FoldMatch runs the production Stage 2 (``code_alignment.align_candidates_esm_codes``) on a real sequence store and +residue-code store built in ``--work``, with the registry bundle's panel and statistics (HF cache), default output +fields; 1 core = the serial in-process path (BLAS limited to one thread), pin it with ``--core`` / taskset. +BLOSUM62 is ``alignment.align_candidates`` with the CLI defaults on the same target sequences. + +Modes (each writes ``--out`` JSON and prints a summary): + kernel P3 gate: kernel-level fwd+rev vs the research production path (unchanged from P3). + components 1 core: per-stage split of the worker work, ns per DP cell: query prep, similarity (decode + + per-pair GEMM), rowcol z, DP (score only, S_f + S_r), DP with traceback flags, traceback (the probe + library ``fmsw_probe.c``: the shipped kernel source compiled with the shipped flags plus clocks; + checked bit for bit against the shipped library), path metrics and CIGAR (parent). + grid 1 core, Lq x K grid: Stage 2 at max_evalue 1e-3 and inf (the panel on, default fields), the panel + alone, rounds 1+2 without the panel (= total - panel alone; also measured with panel_policy never), + BLOSUM62, survivors; plus one profiled 1e-3 run per cell. + panel panel pass, 1 core, at the real 71-aa and 966-aa queries: FoldMatch (aligner, 1 candidate; and + kernel-level over the panel codes) vs the research path (``--research-src``); and the panel wall + time of one 300-aa query with 1 vs ``--workers`` workers (warm persistent pool; cold reported). + perquery Stage 2 per query at K = 100: serial and a warm pool of each ``--workers``; BLOSUM62 likewise. + node ``--nq`` real queries x K on a persistent pool of ``--workers``: 1e-3, inf and no panel; pairs/s; + a profiled 1e-3 run; ``--serial-nq`` of them serially for the per-core rate (Stage B). + sweep block_floats and task_cells sweeps on ``--nq`` x K; every configuration's output hashed (S16: bit + for bit the same). + profile the profiling gate: ``--nq`` x K with FOLDMATCH_STAGE2_PROFILE=1 for each ``--workers`` (0/1 = + serial), plus profile-off runs (overhead, identical output) and the output writing time. + rss worker RSS at Lq = 1000, K = 1000, default block, ``--workers`` workers (profiled). + replay the real pipeline's Stage 2, as captured by ``benchmarks/stage2_capture.py`` around ``fm-search + query sequences --scoring esm-codes`` (real Stage-1 candidates, so the real E-filter survivor + fraction): cold / warm profiled calls and profile-off calls per ``--workers``, BLOSUM62 on the same + candidates; the output signature is checked against the CLI's. + research-batch the research Stage-2 driver (``codepath.align_candidates_x`` of the research throughput bench, + with the research ``swfast`` / ``libswkern``) on the same ``--nq`` x K workload, or the research P64 + single-query workload (``--research-query chain:300 --nq 1 --k 10000``), at ``--workers``; modes + s0 (score only), s10 (score + traceback of the top 10 %), tb (traceback all, vector identity). + Run with ``--research-bench`` (codepath.py), ``--research-build`` (swfast.py + libswkern) and + optionally ``--research-fm-src`` (the FoldMatch snapshot the research ran with; put first on + sys.path). Never imports this tree's swkern. + bandwidth STREAM-like triad with ``--workers`` processes (DRAM-sized and L2-sized arrays), GB/s. + +Examples (M1: through bin/heavy.sh, one at a time): + python benchmarks/stage2_esm_codes.py grid --chains C --chain-seqs S --queries Q --work W --out grid.json + python benchmarks/stage2_esm_codes.py profile --nq 64 --k 1000 --workers 1 8 ... --out profile.json + python benchmarks/stage2_esm_codes.py node --nq 640 --k 1000 --workers 64 ... --out node.json + +Gate of mode ``kernel`` (EPYC 7763, NERSC): foldmatch@2^17 >= 0.9 x research@2^20 at Lq=300, K=1,000 and >= 2,500 +pairs/s; the P7 targets are in PLAN section 8 P7 and are checked from these JSONs. +""" +from __future__ import annotations + +import argparse +import ctypes as C +import hashlib +import json +import math +import os +import platform +import shlex +import shutil +import socket +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +if __name__ in ("__main__", "__mp_main__"): # as a script (and its spawn workers): BLAS on one thread + for _v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", + "NUMEXPR_NUM_THREADS", "BLIS_NUM_THREADS"): + os.environ[_v] = "1" +import numpy as np # noqa: E402 + +HERE = Path(__file__).resolve().parent +CHAINS_MD5 = "5519866044ab0f348c74b9319125acd2" +CHAIN_SEQS_MD5 = "7996b1728759fe8424366843f95d8961" +QUERIES_MD5 = "e821497f59dda06a3899c18dce37286c" +RESEARCH_MD5 = {"swkern.c": "438d5a9811153f61e66cee845f1ed037", "swmulti.c": "b12fbc2c03638659440b12282a4d8670"} +WORKLOAD_SEED, POOL, DRAW = 20260925, 559, 10000 +QUERY_SEED = 20261019 +BENCH_N = 15_889_271 +QUERIES = {100: [("1j2vA", 0, 100)], 300: [("4xrtA", 0, 300)], 600: [("3zukA", 0, 600)], + 1000: [("4j5tA", 0, None), ("3dwbA", 0, 212)]} +OFF, GO, GE = 1.0, 5.0, 0.5 +ESM_REV = "636acb420daf674df37ab0220f56061a5de2f169" +PROFILE_ENV = "FOLDMATCH_STAGE2_PROFILE" # foldmatch.search.stage2_profile.PROFILE_ENV +_vp = C.c_void_p + + +def _swkern(): + from foldmatch.search import swkern + return swkern + + +def md5(path) -> str: + h = hashlib.md5() + with open(path, "rb") as f: + for b in iter(lambda: f.read(1 << 22), b""): + h.update(b) + return h.hexdigest() + + +def _check_md5(path: Path, want: str): + got = md5(path) + if got != want: + raise SystemExit(f"{path}: md5 {got} differs from {want}") + + +def host_info(with_kernel: bool = True) -> dict: + info = dict(host=socket.gethostname(), platform=platform.platform(), machine=platform.machine(), + python=platform.python_version(), cpus=os.cpu_count(), numpy=np.__version__) + if with_kernel: + sk = _swkern() + info["blas"] = sk.blas_info() + info["kernel"] = sk.load().info() + try: + if sys.platform == "darwin": + info["cpu"] = subprocess.check_output(["sysctl", "-n", "machdep.cpu.brand_string"]).decode().strip() + else: + for line in open("/proc/cpuinfo"): + if line.startswith("model name"): + info["cpu"] = line.split(":", 1)[1].strip() + break + aff = sorted(os.sched_getaffinity(0)) + info["affinity"] = dict(count=len(aff), first=aff[:8], last=aff[-4:]) + except Exception: + pass + for k in ("SLURM_JOB_ID", "SLURMD_NODENAME", "SLURM_CPUS_ON_NODE"): + if os.environ.get(k): + info[k.lower()] = os.environ[k] + return info + + +def median_rounds(f, rounds, warmup=True): + if warmup: + f() + ts = [] + for _ in range(rounds): + t0 = time.perf_counter() + f() + ts.append(time.perf_counter() - t0) + return float(np.median(ts)), [round(t, 6) for t in ts] + + +def _timed(f): + t0 = time.perf_counter() + out = f() + return out, time.perf_counter() - t0 + + +# ===================================================================================================================== +# data (numpy only) +# ===================================================================================================================== +class ChainSet: + """The 559 chains: codes, chain query encodings, sequences.""" + + def __init__(self, chains: Path, chain_seqs: Path = None): + _check_md5(chains, CHAINS_MD5) + z = np.load(chains) + self.q, self.codes, self.offsets = z["q"], z["codes"], z["offsets"].astype(np.int64) + self.ids = [str(x) for x in z["ids"]] + self.index = {s: i for i, s in enumerate(self.ids)} + if len(self.ids) != POOL: + raise SystemExit(f"expected {POOL} chains, found {len(self.ids)}") + self.lens = np.diff(self.offsets) + self.seqs = None + if chain_seqs is not None: + _check_md5(chain_seqs, CHAIN_SEQS_MD5) + seq = {} + for line in open(chain_seqs): + f = line.rstrip("\n").split("\t") + seq.setdefault(f[0], f[1]) + self.seqs = [seq[c] for c in self.ids] + bad = [c for c, s, m in zip(self.ids, self.seqs, self.lens) if len(s) != m] + if bad: + raise SystemExit(f"{len(bad)} chain sequences differ in length from their codes: {bad[:5]}") + self.draw = np.random.default_rng(WORKLOAD_SEED).integers(0, POOL, DRAW) + self.cand_ids = [f"{self.ids[c]}#{k}" for k, c in enumerate(self.draw.tolist())] + + def chain_codes(self, i: int) -> np.ndarray: + return self.codes[self.offsets[i]:self.offsets[i + 1]] + + def research_query(self, Lq: int): + """The research throughput query of length Lq (chain-set q16 and sequence).""" + rows, seqs, names = [], [], [] + for cid, a, b in QUERIES[Lq]: + i = self.index[cid] + rows.append(self.q[self.offsets[i]:self.offsets[i + 1]][a:b]) + if self.seqs is not None: + seqs.append(self.seqs[i][a:b]) + names.append(f"{cid}[{a}:{b if b is not None else ''}]") + q16 = np.ascontiguousarray(np.concatenate(rows), np.float16) + assert len(q16) == Lq, (Lq, len(q16)) + return (f"chain{Lq}", "".join(seqs) if seqs else None, q16, "+".join(names)) + + def slice_ids(self, j: int, K: int): + start = 0 if K >= DRAW else (j * 617) % (DRAW - K) + return self.cand_ids[start:start + K] + + +class RealQueries: + """The 1,465 real benchmark queries (q fp16, sequences).""" + + def __init__(self, path: Path): + _check_md5(path, QUERIES_MD5) + z = np.load(path) + self.ids = [str(x) for x in z["ids"]] + self.off = z["offsets"].astype(np.int64) + self.lens = np.diff(self.off) + self.q = z["q"] + sc = z["seq_concat"] + self.seqs = [sc[self.off[i]:self.off[i + 1]].tobytes().decode("ascii") for i in range(len(self.ids))] + + def get(self, i: int): + return (self.ids[i], self.seqs[i], np.ascontiguousarray(self.q[self.off[i]:self.off[i + 1]]), self.ids[i]) + + def exact(self, L: int): + """The first real query of length exactly L.""" + idx = np.flatnonzero(self.lens == L) + if not idx.size: + raise SystemExit(f"no real query of length {L}") + return self.get(int(idx[0])) + + def fixed(self, Lq: int): + """Research rule on the real queries: the shortest with length >= Lq cut to Lq; longer than every query: the + longest + the first residues of the next longest.""" + order = np.argsort(self.lens, kind="stable") + ok = [int(i) for i in order if self.lens[i] >= Lq] + if ok: + i = ok[0] + qid, seq, q16, _ = self.get(i) + return (f"{qid}@1-{Lq}", seq[:Lq], np.ascontiguousarray(q16[:Lq]), f"{qid} (length {self.lens[i]}) 1-{Lq}") + i1, i2 = int(order[-1]), int(order[-2]) + a, b = self.get(i1), self.get(i2) + rest = Lq - len(a[1]) + return (f"{a[0]}+{b[0]}@{Lq}", a[1] + b[1][:rest], np.ascontiguousarray(np.concatenate([a[2], b[2][:rest]])), + f"{a[0]} (all {len(a[1])}) + {b[0]} 1-{rest}") + + def sample(self, nq: int, seed: int = QUERY_SEED): + perm = np.random.default_rng(seed).permutation(len(self.ids)) + if nq > len(perm): + raise SystemExit(f"--nq {nq} > {len(perm)} real queries") + return [self.get(int(i)) for i in perm[:nq]] + + +def make_queries(a, cs: ChainSet): + """The query set of a mode from --query-set: 'real' (--nq sampled real queries), 'fixed:' (one fixed-length + real query), 'chain:' (the research chain-set query).""" + spec = a.query_set + if spec == "real": + return RealQueries(a.queries).sample(a.nq) + kind, _, lq = spec.partition(":") + if kind == "fixed": + return [RealQueries(a.queries).fixed(int(lq))] + if kind == "chain": + return [cs.research_query(int(lq))] + raise SystemExit(f"unknown --query-set {spec!r}") + + +def batch_prefilter(cs: ChainSet, queries, K: int): + emb = [float(x) for x in np.linspace(0.99, 0.5, K)] + return {q[0]: (cs.slice_ids(j, K), emb) for j, q in enumerate(queries)} + + +# ===================================================================================================================== +# FoldMatch world: a real sequence store + residue-code store of the 10,000 candidates, and a query memmap +# ===================================================================================================================== +class World: + def __init__(self, work: Path, cs: ChainSet, queries, db_residues: int = BENCH_N, n_cand: int = DRAW): + from foldmatch.search import scoring_system as ss + from foldmatch.search.query_encodings import QueryEncodingEntry, QueryEncodingIndex + from foldmatch.search.residue_code_store import ResidueCodeStore + from foldmatch.search.sequence_store import SequenceStore + from foldmatch.utils.sequence import seq_sha1 + if cs.seqs is None: + raise SystemExit("--chain-seqs is required") + self.system = ss.get(ss.DEFAULT_ESM_CODES) + self.prepared = ss.prepare(self.system, local_files_only=bool(os.environ.get("HF_HUB_OFFLINE"))) + work.mkdir(parents=True, exist_ok=True) + self.folder = Path(tempfile.mkdtemp(prefix="world_", dir=work)) + ids = cs.cand_ids[:n_cand] + idx = cs.draw[:n_cand].tolist() + t0 = time.perf_counter() + fasta = self.folder / "db.fasta" + with open(fasta, "w") as f: + for tid, i in zip(ids, idx): + f.write(f">{tid}\n{cs.seqs[i]}\n") + self.seq_store = SequenceStore(self.folder, "db") + self.seq_store.create(fasta) + self.store = ResidueCodeStore(self.folder, "db") + self.store.create([(ids, [cs.chain_codes(i) for i in idx], [seq_sha1(cs.seqs[i]) for i in idx])], + self.system, esm_revision=ESM_REV) + self.store.validate(self.seq_store, len(ids), preflight=None) + self.build_s = time.perf_counter() - t0 + self.store_residues = int(self.store.manifest()["total_residues"]) + self.db_residues = int(db_residues) + path = self.folder / "queries.f16" + entries, rows = {}, 0 + with open(path, "wb") as f: + for qid, seq, q16, _src in queries: + f.write(np.ascontiguousarray(q16, " str: + """sha256 over every metrics field (floats as hex) and the rendered row of every hit, in order.""" + import dataclasses + from foldmatch.search import output + fields = list(output.SUPPORTED_OUTPUT_FIELDS) if fields is None else fields + h = hashlib.sha256() + for qid, hits in results.items(): + h.update(qid.encode() + b"\0") + for hit in hits: + vals = [float(v).hex() if isinstance(v, float) else repr(v) + for v in (getattr(hit.metrics, f.name) for f in dataclasses.fields(hit.metrics))] + h.update(("\t".join([hit.subject_id, float(hit.emb_score).hex(), *vals]) + "\n").encode()) + h.update((output.format_row(qid, hit, fields) + "\n").encode()) + return h.hexdigest() + + +def _pool_size(W: int): + """--workers value -> CodeAlignmentPool num_workers: -1 is the default pool (None: sized by the work, up to every + CPU), 0 / 1 serial, N > 1 N workers.""" + return None if W < 0 else max(W, 1) + + +def _pool_label(W: int) -> str: + return "default" if W < 0 else str(W) + + +def _counts(stats): + keys = ("queries", "candidate_pairs", "panel_pairs", "survivors", "hits", "round1_tasks", "round2_tasks", + "workers", "batches", "skipped_pairs") + return {k: stats.get(k) for k in keys} + + +def _profile_brief(p: dict) -> dict: + """The profile summary without the per-task list.""" + return {k: v for k, v in p.items() if k != "tasks"} + + +# ===================================================================================================================== +# research path (unmodified research kernels) for kernel / panel +# ===================================================================================================================== +def research_flags() -> list: + m = platform.machine().lower() + if sys.platform == "darwin" and m in ("arm64", "aarch64"): + arch = ["-mcpu=apple-m1"] + elif m in ("x86_64", "amd64"): + arch = ["-march=znver3"] + else: + arch = ["-mcpu=native"] + return ["-O3", *arch, "-ffp-contract=off", "-shared", "-fPIC"] + + +def build_research(src: Path, work: Path, cc: str, cflags: list) -> Path: + for name, want in RESEARCH_MD5.items(): + _check_md5(src / name, want) + work.mkdir(parents=True, exist_ok=True) + ext = "dylib" if sys.platform == "darwin" else "so" + for name in ("swkern", "swmulti"): + cmd = [*shlex.split(cc), *cflags, f"-I{src}", "-o", str(work / f"lib{name}.{ext}"), str(src / f"{name}.c"), + "-lm"] + subprocess.run(cmd, check=True) + return work + + +def research_from_args(a): + if a.research_lib_dir: + return Research(a.research_lib_dir), dict(lib_dir=str(a.research_lib_dir)) + if not a.research_src: + raise SystemExit("--research-src or --research-lib-dir is required") + flags = shlex.split(a.research_cflags) if a.research_cflags else research_flags() + R = Research(build_research(a.research_src, a.work / "research_lib", a.research_cc, flags)) + return R, dict(src=str(a.research_src), cc=a.research_cc, cflags=flags, sources_md5=RESEARCH_MD5, + cc_version=subprocess.run(shlex.split(a.research_cc) + ["--version"], capture_output=True, + text=True).stdout.splitlines()[0]) + + +class Research: + _LUT = np.unpackbits(np.arange(256, dtype=np.uint8)[:, None], axis=1).astype(np.float32) * 2 - 1 + + def __init__(self, libdir: Path): + ext = "dylib" if sys.platform == "darwin" else "so" + self.k = C.CDLL(str(libdir / f"libswkern.{ext}")) + self.m = C.CDLL(str(libdir / f"libswmulti.{ext}")) + self.k.sw_ad_ws_floats.restype = C.c_size_t + self.k.sw_ad_ws_floats.argtypes = [C.c_int, C.c_int] + self.k.swkern_vector_lanes.restype = C.c_int + self.m.swz_ad_multi_f32.restype = None + self.m.swz_ad_multi_f32.argtypes = [_vp, _vp, C.c_int, C.c_int, _vp, C.c_int, C.c_int, _vp, _vp, _vp, _vp, + _vp, _vp] + self.R = (np.array([OFF, OFF], np.float32), np.array([GO, GO], np.float32), np.array([GE, GE], np.float32), + np.array([0, 1], np.int32)) + self.ws = np.empty(0, np.float32) + self.libdir = libdir + + @staticmethod + def query_prep(q16): + q = np.asarray(q16, dtype=np.float32) + return q / (8.0 * np.linalg.norm(q, axis=1, keepdims=True)) + + @staticmethod + def blocks(lengths, n, block_floats): + """research codepath._blocks: a candidate opens a new block when the current one is non-empty and would + exceed block_floats similarity floats.""" + out, k0, acc = [], 0, 0 + for k, m in enumerate(lengths): + if k > k0 and acc + n * m > block_floats: + out.append((k0, k)) + k0, acc = k, 0 + acc += n * int(m) + out.append((k0, len(lengths))) + return out + + def multi_fwd_rev(self, sim, simr, col_off, out2): + n, ld = sim.shape + K = len(col_off) - 1 + mmax = int(np.max(np.diff(col_off))) + need = 2 * (n + mmax) + int(self.k.sw_ad_ws_floats(n, mmax)) + if self.ws.size < need: + self.ws = np.empty(need, np.float32) + o, g, e, r = self.R + self.m.swz_ad_multi_f32(sim.ctypes.data, simr.ctypes.data, n, ld, col_off.ctypes.data, K, 2, o.ctypes.data, + g.ctypes.data, e.ctypes.data, r.ctypes.data, out2.ctypes.data, self.ws.ctypes.data) + + def runner(self, codes, offsets, q16, block_floats, reuse=False): + lens = np.diff(offsets) + K = len(lens) + blk = self.blocks(lens.tolist(), len(q16), block_floats) + cols = [np.ascontiguousarray(offsets[k0:k1 + 1] - offsets[k0]) for k0, k1 in blk] + out = np.empty((2, K), np.float32) + bufs = {} + + def run(): + qp = self.query_prep(q16) + n = len(qp) + for (k0, k1), col in zip(blk, cols): + j0, j1 = int(offsets[k0]), int(offsets[k1]) + w = j1 - j0 + o2 = np.empty((2, k1 - k0), np.float32) + if reuse: + for key, size in (("T", w * 64), ("sim", n * w), ("simr", n * w)): + if bufs.get(key) is None or bufs[key].size < size: + bufs[key] = np.empty(size, np.float32) + T = bufs["T"][:w * 64].reshape(w, 8, 8) + np.take(self._LUT, codes[j0:j1], axis=0, out=T, mode="clip") + sim = bufs["sim"][:n * w].reshape(n, w) + np.matmul(qp, T.reshape(w, 64).T, out=sim) + simr = bufs["simr"][:n * w].reshape(n, w) + np.copyto(simr, sim[::-1]) + else: + T = np.unpackbits(codes[j0:j1], axis=1)[:, :64].astype(np.float32) * 2 - 1 + sim = qp @ T.T + simr = np.ascontiguousarray(sim[::-1]) + self.multi_fwd_rev(sim, simr, col, o2) + out[:, k0:k1] = o2 + return out + return run + + +def foldmatch_runner(codes, offsets, q16, block_floats): + sk = _swkern() + k = sk.load() + ws = sk.default_workspace() + + def run(): + qp = sk.query_prep(q16, k) + return sk.score_fwd_rev(qp, codes, offsets, sk.SETTING_D, block_floats=block_floats, kernel=k, workspace=ws) + return run + + +def kernel_workload(cs: ChainSet, K: int): + cand = cs.draw[:K] + parts = [cs.chain_codes(int(c)) for c in cand] + codes = np.ascontiguousarray(np.concatenate(parts), np.uint8) + offsets = np.concatenate([[0], np.cumsum([len(p) for p in parts])]).astype(np.int64) + return codes, offsets + + +# ===================================================================================================================== +# mode: kernel (P3) +# ===================================================================================================================== +def kernel_mode(a) -> dict: + sk = _swkern() + cs = ChainSet(a.chains) + codes, offsets = kernel_workload(cs, a.k) + K, residues = len(offsets) - 1, int(offsets[-1]) + R, research_build = research_from_args(a) + res = dict(mode="kernel", host=host_info(), research=research_build, + research_vector_lanes=int(R.k.swkern_vector_lanes()), K=K, residues=residues, + workload=dict(chains=str(a.chains), seed=WORKLOAD_SEED, queries={str(k): v for k, v in QUERIES.items()}), + rounds=a.rounds, results={}) + for Lq in a.lq: + q16 = cs.research_query(Lq)[2] + r = {} + fm = foldmatch_runner(codes, offsets, q16, 1 << 17)() + rs = R.runner(codes, offsets, q16, 1 << 20)() + bound = sk.s16_score_bound(rs) + got = np.stack([fm.s_fwd, fm.s_rev]) + r["s16"] = dict(max_ratio=float(np.max(np.abs(got - rs) / bound)), + bit_equal_fraction=float(np.mean(got.view(np.int32) == rs.view(np.int32)))) + variants = [(f"foldmatch.bf2^{e}", foldmatch_runner(codes, offsets, q16, 1 << e)) for e in a.blocks] + for e in a.research_blocks: + variants.append((f"research.bf2^{e}", R.runner(codes, offsets, q16, 1 << e))) + variants.append((f"research.reuse.bf2^{e}", R.runner(codes, offsets, q16, 1 << e, reuse=True))) + for name, f in variants: + m, ts = median_rounds(f, a.rounds) + r[name] = dict(s_med=m, s_rounds=ts, pairs_per_s=K / m, ns_per_cell=1e9 * m / (Lq * residues)) + print(f"Lq={Lq:5d} K={K} {name:28s} {K / m:10.1f} pairs/s {1e9 * m / (Lq * residues):6.3f} ns/cell", + flush=True) + fm_def = r["foldmatch.bf2^17"]["pairs_per_s"] if "foldmatch.bf2^17" in r else None + if fm_def and "research.bf2^20" in r: + r["ratio.foldmatch_bf2^17_over_research_bf2^20"] = fm_def / r["research.bf2^20"]["pairs_per_s"] + best_res = max(v["pairs_per_s"] for k, v in r.items() if k.startswith("research")) + best_fm = max(v["pairs_per_s"] for k, v in r.items() if k.startswith("foldmatch")) + r["ratio.best_foldmatch_over_best_research"] = best_fm / best_res + print(f"Lq={Lq} " + ", ".join(f"{k} {v:.3f}" for k, v in r.items() if k.startswith("ratio")) + + f"; S16 max ratio {r['s16']['max_ratio']:.3g}", flush=True) + res["results"][str(Lq)] = r + return res + + +# ===================================================================================================================== +# mode: components (probe) +# ===================================================================================================================== +def build_probe(work: Path, cc=None) -> Path: + """The probe library for this machine's kernel ISA: benchmarks/fmsw_probe.c (#includes the shipped fmsw.c) with + the shipped build's exact flags for that ISA.""" + from foldmatch._native import SOURCE_DIR, library_filename + from foldmatch._native import build as nb + sk = _swkern() + isa = sk.load().isa + work.mkdir(parents=True, exist_ok=True) + out = work / f"probe_{library_filename(isa)}" + cmd = nb.compile_command(isa, out, cc=cc) + # the shipped command ends with "-o OUT SRC/fmsw.c -lm": compile the probe instead, with the source dir on -I + i = cmd.index("-o") + cmd = cmd[:i] + [f"-I{SOURCE_DIR}", "-o", str(out), str(HERE / "fmsw_probe.c"), "-lm"] + subprocess.run(cmd, check=True) + return out + + +class Probe: + def __init__(self, path: Path): + lib = C.CDLL(str(path)) + lib.probe_score_fwd_rev.restype = C.c_int + lib.probe_score_fwd_rev.argtypes = [_vp, C.c_int, _vp, _vp, C.c_int, C.c_float, C.c_float, C.c_float, + _vp, _vp, _vp, C.c_size_t, _vp] + lib.probe_align_tb.restype = C.c_int + lib.probe_align_tb.argtypes = [_vp, C.c_int, _vp, _vp, C.c_int, C.c_float, C.c_float, C.c_float, _vp, _vp, + _vp, C.c_size_t, _vp, _vp, C.c_size_t, _vp, C.c_size_t, _vp] + self.lib = lib + + +def components_mode(a) -> dict: + """1 core: the worker's stages for one query x K candidates, timed block by block exactly as swkern runs them + (same blocks, same per-pair GEMMs), with the kernel call replaced by the probe.""" + sk = _swkern() + from foldmatch.search import code_alignment as ca + cs = ChainSet(a.chains, a.chain_seqs) + rq = RealQueries(a.queries) + probe = Probe(build_probe(a.work, a.cc)) + kern = sk.load() + params = sk.SETTING_D + codes, offsets = kernel_workload(cs, a.k) + lens = np.diff(offsets) + starts = offsets[:-1] + t_seqs = [cs.seqs[int(c)] for c in cs.draw[:a.k]] + res = dict(mode="components", host=host_info(), K=a.k, residues=int(offsets[-1]), rounds=a.rounds, + block_floats=a.block_floats, results={}) + for Lq in a.lq: + qid, qseq, q16, src = rq.fixed(Lq) + cells = Lq * int(offsets[-1]) + ws = sk.Workspace() + qp = sk.query_prep(q16, kern) + skipped = np.zeros(len(lens), bool) + blocks = sk.plan_blocks(Lq, lens.tolist(), skipped.tolist(), a.block_floats) + ref1 = sk.score_fwd_rev(qp, codes, offsets, params, block_floats=a.block_floats, kernel=kern) + ref2 = sk.align(qp, codes, offsets, params, block_floats=a.block_floats, kernel=kern) + + def one_round(): + t = dict(prep=0.0, sim=0.0, rowcol_r1=0.0, dp_r1=0.0, rowcol_r2=0.0, dp_tb_r2=0.0, traceback_r2=0.0, + metrics=0.0, cigar=0.0) + t0 = time.perf_counter() + qp_ = sk.query_prep(q16, kern) + t["prep"] += time.perf_counter() - t0 + s_f = np.full(len(lens), np.nan, np.float32) + s_r = np.full(len(lens), np.nan, np.float32) + sc2 = np.full(len(lens), np.nan, np.float32) + ends = np.zeros((len(lens), 2), np.int32) + paths, poffs = [], [] + for k0, k1 in blocks: + t0 = time.perf_counter() + b = sk._Block(kern, ws, qp_, codes, starts, lens, k0, k1, False) + t["sim"] += time.perf_counter() - t0 + mmax = int(b.m.max()) + kw = ws.kernel_floats(kern.ws_floats(Lq, mmax)) + tt = np.zeros(3, np.float64) + st = probe.lib.probe_score_fwd_rev(b.sim.ctypes.data, Lq, b.pair_off.ctypes.data, b.m.ctypes.data, + b.K, params.offset, params.gap_open, params.gap_extend, + s_f[k0:].ctypes.data, s_r[k0:].ctypes.data, kw.ctypes.data, + kw.size, tt.ctypes.data) + assert st == 0, st + t["rowcol_r1"] += tt[0] + t["dp_r1"] += tt[1] + fl = ws.flags(kern.flag_bytes(Lq, mmax)) + cap = int(b.K * Lq + int(b.m.sum())) + pbuf = np.empty(2 * max(cap, 1), np.int32) + poff = np.empty(b.K + 1, np.int64) + tt[:] = 0 + st = probe.lib.probe_align_tb(b.sim.ctypes.data, Lq, b.pair_off.ctypes.data, b.m.ctypes.data, b.K, + params.offset, params.gap_open, params.gap_extend, + sc2[k0:].ctypes.data, ends[k0:].ctypes.data, pbuf.ctypes.data, cap, + poff.ctypes.data, kw.ctypes.data, kw.size, fl.ctypes.data, fl.size, + tt.ctypes.data) + assert st == 0, st + t["rowcol_r2"] += tt[0] + t["dp_tb_r2"] += tt[1] + t["traceback_r2"] += tt[2] + paths.append(pbuf[:2 * int(poff[-1])].reshape(-1, 2)) + poffs.append(np.diff(poff)) + path = np.concatenate(paths) + path_off = np.zeros(len(lens) + 1, np.int64) + np.cumsum(np.concatenate(poffs), out=path_off[1:]) + t0 = time.perf_counter() + q_bytes = np.frombuffer(ca.sanitized_bytes(qseq), np.uint8) + t_clean = [ca.sanitized_bytes(s) for s in t_seqs] + t_starts = np.zeros(len(t_clean), np.int64) + np.cumsum(lens[:-1], out=t_starts[1:]) + ca.path_columns(path, path_off, q_bytes, np.frombuffer(b"".join(t_clean), np.uint8), t_starts) + t["metrics"] += time.perf_counter() - t0 + t0 = time.perf_counter() + for k in range(len(lens)): + sk.cigar_from_path(path[path_off[k]:path_off[k + 1]]) + t["cigar"] += time.perf_counter() - t0 + return t, s_f, s_r, sc2, ends, path, path_off + + t, s_f, s_r, sc2, ends, path, path_off = one_round() + same = (np.array_equal(s_f.view(np.uint32), ref1.s_fwd.view(np.uint32)) + and np.array_equal(s_r.view(np.uint32), ref1.s_rev.view(np.uint32)) + and np.array_equal(sc2.view(np.uint32), ref2.scores.view(np.uint32)) + and np.array_equal(ends, ref2.ends) and np.array_equal(path, ref2.path) + and np.array_equal(path_off, ref2.path_off)) + rounds = [one_round()[0] for _ in range(a.rounds)] + med = {k: float(np.median([r[k] for r in rounds])) for k in rounds[0]} + # production wrappers for comparison: the whole score_fwd_rev / align calls + m1, _ = median_rounds(lambda: sk.score_fwd_rev(sk.query_prep(q16, kern), codes, offsets, params, + block_floats=a.block_floats, kernel=kern, workspace=ws), + a.rounds) + m2, _ = median_rounds(lambda: sk.align(sk.query_prep(q16, kern), codes, offsets, params, + block_floats=a.block_floats, kernel=kern, workspace=ws), a.rounds) + ns = {k: 1e9 * v / cells for k, v in med.items()} + r = dict(query=src, Lq=Lq, cells=cells, probe_equals_shipped=bool(same), seconds=med, ns_per_cell=ns, + score_fwd_rev_call_s=m1, align_call_s=m2, score_fwd_rev_ns_per_cell=1e9 * m1 / cells, + align_ns_per_cell=1e9 * m2 / cells, blocks=len(blocks)) + res["results"][str(Lq)] = r + print(f"Lq={Lq} K={a.k}: probe == shipped {same}; ns/cell " + ", ".join(f"{k} {v:.3f}" for k, v in ns.items()) + + f"; score_fwd_rev call {1e9 * m1 / cells:.3f}, align call {1e9 * m2 / cells:.3f}", flush=True) + return res + + +# ===================================================================================================================== +# mode: grid (1 core) +# ===================================================================================================================== +def grid_mode(a) -> dict: + from foldmatch.search import code_alignment as ca + cs = ChainSet(a.chains, a.chain_seqs) + rq = RealQueries(a.queries) + queries = [rq.fixed(Lq) for Lq in a.lq] + world = World(a.work, cs, queries, a.db_residues) + res = dict(mode="grid", host=host_info(), db_residues=world.db_residues, store_residues=world.store_residues, + rounds=a.rounds, blosum_rounds=a.blosum_rounds, results=[]) + serial = ca.CodeAlignmentPool(1) # one in-process worker state for the run, as the CLI's serial Stage 2 + try: + for (qid, seq, q16, src), Lq in zip(queries, a.lq): + for K in a.ks: + ids = cs.cand_ids[:K] + emb = [float(x) for x in np.linspace(0.99, 0.5, K)] + pf = {qid: (ids, emb)} + pf1 = {qid: (ids[:1], emb[:1])} + cell = dict(Lq=Lq, K=K, query=src, residues=int(sum(cs.lens[cs.draw[:K]]))) + runs = {} + rounds = a.rounds if K < a.big_k else a.big_k_rounds + cell["rounds"] = rounds + for name, kw, p in (("e1e-3", dict(max_evalue=1e-3), pf), ("inf", dict(max_evalue=None), pf), + ("nopanel_inf", dict(max_evalue=None, panel_policy="never"), pf), + ("panel_only", dict(max_evalue=1e-3), pf1)): + walls, stats = [], None + if K < a.big_k or p is pf1: + world.align(p, serial, **kw) # warm-up + for _ in range(rounds if p is not pf1 else a.rounds): + _r, stats, w = world.align(p, serial, **kw) + walls.append(w) + runs[name] = dict(wall=float(np.median(walls)), walls=walls, counts=_counts(stats)) + # profiled runs: rounds 1+2 without the panel = wall - the panel's worker seconds (1 core, in-process) + world.align(pf1, serial, max_evalue=1e-3, profile=True) # profiled worker state + prof_runs = {} + for name, me in (("e1e-3", 1e-3), ("inf", None)): + excl, walls, last = [], [], None + for _ in range(rounds): + _r, pst, pw = world.align(pf, serial, max_evalue=me, profile=True) + last = _profile_brief(pst["profile"]) + excl.append(pw - last["workers"].get("panel", {}).get("seconds", 0.0)) + walls.append(pw) + prof_runs[name] = dict(wall=float(np.median(walls)), walls=walls, + excl_panel=float(np.median(excl)), excl_panel_rounds=excl, + parent=last["parent"], workers=last["workers"]) + cell["profiled"] = prof_runs + br = a.blosum_rounds if K < a.big_k else 1 + bw = [] + for _ in range(br): + bres, w = world.blosum(pf, 1) + bw.append(w) + runs["blosum62"] = dict(wall=float(np.median(bw)), walls=bw, + hits=sum(len(v) for v in bres.values())) + panel = runs["panel_only"]["wall"] + b = runs["blosum62"]["wall"] + ex3, exi = prof_runs["e1e-3"]["excl_panel"], prof_runs["inf"]["excl_panel"] + sub3, subi = runs["e1e-3"]["wall"] - panel, runs["inf"]["wall"] - panel + cell.update(runs=runs, panel_alone=panel, + panel_worker_s=prof_runs["e1e-3"]["workers"].get("panel", {}).get("seconds"), + excl_panel_e1e3=ex3, excl_panel_inf=exi, + excl_panel_e1e3_by_subtraction=sub3, excl_panel_inf_by_subtraction=subi, + survivor_fraction_e1e3=runs["e1e-3"]["counts"]["survivors"] / K, + ratio_blosum_over_code_excl_panel_e1e3=b / ex3, + ratio_blosum_over_code_excl_panel_inf=b / exi, + ratio_blosum_over_code_nopanel_inf=b / runs["nopanel_inf"]["wall"], + ratio_blosum_over_code_total_e1e3=b / runs["e1e-3"]["wall"], + ratio_blosum_over_code_total_inf=b / runs["inf"]["wall"], + pairs_per_s=dict(code_total_e1e3=K / runs["e1e-3"]["wall"], + code_excl_panel_e1e3=K / ex3, code_excl_panel_inf=K / exi, + code_nopanel_inf=K / runs["nopanel_inf"]["wall"], blosum62=K / b)) + res["results"].append(cell) + print(f"Lq={Lq:5d} K={K:6d}: e1e-3 {runs['e1e-3']['wall']:.3f}s (survivors " + f"{runs['e1e-3']['counts']['survivors']}), inf {runs['inf']['wall']:.3f}s, no-panel inf " + f"{runs['nopanel_inf']['wall']:.3f}s, panel alone {panel:.3f}s, BLOSUM62 {b:.3f}s -> " + f"BLOSUM62 / code (excl. panel): 1e-3 {cell['ratio_blosum_over_code_excl_panel_e1e3']:.2f}x, " + f"inf {cell['ratio_blosum_over_code_excl_panel_inf']:.2f}x (no-panel run " + f"{cell['ratio_blosum_over_code_nopanel_inf']:.2f}x); with the panel: 1e-3 " + f"{cell['ratio_blosum_over_code_total_e1e3']:.2f}x, inf " + f"{cell['ratio_blosum_over_code_total_inf']:.2f}x", flush=True) + _dump_partial(a, res) + finally: + serial.close() + world.close() + return res + + +def _dump_partial(a, res): + a.out.parent.mkdir(parents=True, exist_ok=True) + a.out.with_suffix(".partial.json").write_text(json.dumps(res, indent=1, default=str)) + + +# ===================================================================================================================== +# mode: panel +# ===================================================================================================================== +def panel_mode(a) -> dict: + sk = _swkern() + from foldmatch.search import code_alignment as ca + cs = ChainSet(a.chains, a.chain_seqs) + rq = RealQueries(a.queries) + queries = [rq.exact(L) for L in a.panel_lengths] + [rq.fixed(a.pool_query_length)] + world = World(a.work, cs, queries, a.db_residues) + R, research_build = research_from_args(a) + spec = world.prepared.for_worker() + pcodes, poff = spec.load_panel() + pcodes = np.ascontiguousarray(pcodes) + poff = np.asarray(poff, np.int64) + res = dict(mode="panel", host=host_info(), research=research_build, panel_members=len(poff) - 1, + panel_residues=int(poff[-1]), rounds=a.rounds, results={}, pool={}) + serial = ca.CodeAlignmentPool(1) + try: + for qid, seq, q16, src in queries[:len(a.panel_lengths)]: + L = len(seq) + pf1 = {qid: (cs.cand_ids[:1], [0.99])} + m_al, ts_al = median_rounds(lambda: world.align(pf1, serial, max_evalue=1e-3), a.rounds) + m_fm, ts_fm = median_rounds(foldmatch_runner(pcodes, poff, q16, 1 << 17), a.rounds) + m_rs, ts_rs = median_rounds(R.runner(pcodes, poff, q16, 1 << 20), a.rounds) + m_rs17, _ = median_rounds(R.runner(pcodes, poff, q16, 1 << 17), a.rounds) + fm = foldmatch_runner(pcodes, poff, q16, 1 << 17)() + rs = R.runner(pcodes, poff, q16, 1 << 20)() + got = np.stack([fm.s_fwd, fm.s_rev]) + s16 = float(np.max(np.abs(got - rs) / sk.s16_score_bound(rs))) + world.align(pf1, serial, max_evalue=1e-3, profile=True) + _r, pst, _w = world.align(pf1, serial, max_evalue=1e-3, profile=True) + pw = pst["profile"]["workers"].get("panel", {}) + r = dict(query=src, length=L, aligner_panel_plus_1_s=m_al, aligner_rounds=ts_al, kernel_level_s=m_fm, + kernel_rounds=ts_fm, research_bf2_20_s=m_rs, research_rounds=ts_rs, research_bf2_17_s=m_rs17, + ratio_aligner_over_research=m_al / m_rs, ratio_kernel_over_research=m_fm / m_rs, + s16_max_ratio=s16, profile_panel=pw) + res["results"][str(L)] = r + print(f"panel pass L={L}: aligner (panel + 1 candidate) {m_al:.4f}s, kernel-level {m_fm:.4f}s, research " + f"{m_rs:.4f}s (2^17 {m_rs17:.4f}s) -> aligner / research {m_al / m_rs:.3f}, kernel / research " + f"{m_fm / m_rs:.3f}; S16 {s16:.3g}", flush=True) + # one 300-aa query, panel wall time: 1 worker (serial) vs warm pools + qid, seq, q16, src = queries[-1] + pf1 = {qid: (cs.cand_ids[:1], [0.99])} + m1, ts1 = median_rounds(lambda: world.align(pf1, serial, max_evalue=1e-3), a.rounds) + res["pool"]["query"] = src + res["pool"]["serial"] = dict(wall=m1, walls=ts1) + for W in a.workers: + if 0 <= W <= 1: + continue + with ca.CodeAlignmentPool(_pool_size(W)) as pool: + _r, st0, cold = world.align(pf1, pool, max_evalue=1e-3) + mW, tsW = median_rounds(lambda: world.align(pf1, pool, max_evalue=1e-3), a.rounds, warmup=False) + _r, pst, pwall = world.align(pf1, pool, max_evalue=1e-3, profile=True) + prof = _profile_brief(pst["profile"]) + res["pool"][_pool_label(W)] = dict(cold_first_call=cold, warm=mW, warm_walls=tsW, speedup_warm=m1 / mW, + speedup_cold=m1 / cold, tasks=st0.get("round1_tasks"), + workers=st0.get("workers"), profile=prof) + print(f"panel wall, one {len(seq)}-aa query: serial {m1:.4f}s; {_pool_label(W)} workers warm {mW:.4f}s " + f"({m1 / mW:.2f}x), cold first call {cold:.4f}s ({m1 / cold:.2f}x); {st0.get('round1_tasks')} " + f"round-1 tasks, {st0.get('workers')} worker processes", flush=True) + finally: + serial.close() + world.close() + return res + + +# ===================================================================================================================== +# mode: perquery (K = 100) +# ===================================================================================================================== +def perquery_mode(a) -> dict: + from foldmatch.search import code_alignment as ca + cs = ChainSet(a.chains, a.chain_seqs) + rq = RealQueries(a.queries) + queries = [rq.fixed(Lq) for Lq in a.lq] + if a.nq: + queries += rq.sample(a.nq) + world = World(a.work, cs, queries, a.db_residues) + K = a.k + res = dict(mode="perquery", host=host_info(), K=K, rounds=a.rounds, results=[]) + try: + pools = {} + for W in a.workers: + pools[W] = ca.CodeAlignmentPool(_pool_size(W)) + world.align({queries[0][0]: (cs.cand_ids[:K], [0.9] * K)}, pools[W], max_evalue=1e-3) # start + for j, (qid, seq, q16, src) in enumerate(queries): + pf = {qid: (cs.slice_ids(j, K), [float(x) for x in np.linspace(0.99, 0.5, K)])} + row = dict(query=src, Lq=len(seq), code={}, blosum62={}) + for W in a.workers: + pool = pools.get(W) + m, ts = median_rounds(lambda: world.align(pf, pool, max_evalue=1e-3), a.rounds) + _r, st, _w = world.align(pf, pool, max_evalue=1e-3) + row["code"][_pool_label(W)] = dict(wall=m, walls=ts, counts=_counts(st)) + if 0 <= W <= 1 or (a.blosum_pool and W > 1): + mb, tb = median_rounds(lambda: world.blosum(pf, max(W, 1)), a.blosum_rounds) + row["blosum62"][str(W)] = dict(wall=mb, walls=tb) + res["results"].append(row) + print(f"{src} (Lq {len(seq)}) K={K}: " + ", ".join( + f"code W={w} {v['wall']:.3f}s" for w, v in row["code"].items()) + "; " + ", ".join( + f"BLOSUM62 W={w} {v['wall']:.3f}s" for w, v in row["blosum62"].items()), flush=True) + for p in pools.values(): + p.close() + finally: + world.close() + return res + + +# ===================================================================================================================== +# mode: node / sweep / profile / rss (multi-query) +# ===================================================================================================================== +def _workload(a): + cs = ChainSet(a.chains, a.chain_seqs) + queries = make_queries(a, cs) + world = World(a.work, cs, queries, a.db_residues) + pf = batch_prefilter(cs, queries, a.k) + cells = sum(len(q[1]) * int(sum(cs.lens[cs.index[t.split("#")[0]]] for t in pf[q[0]][0])) for q in queries) + info = dict(nq=len(queries), K=a.k, query_set=a.query_set, query_lengths=dict( + min=min(len(q[1]) for q in queries), median=float(np.median([len(q[1]) for q in queries])), + mean=float(np.mean([len(q[1]) for q in queries])), max=max(len(q[1]) for q in queries)), + candidate_pairs=len(queries) * a.k, candidate_cells=int(cells), db_residues=world.db_residues, + store_residues=world.store_residues) + return cs, queries, world, pf, info + + +def node_mode(a) -> dict: + from foldmatch.search import code_alignment as ca + cs, queries, world, pf, info = _workload(a) + res = dict(mode="node", host=host_info(), workload=info, rounds=a.rounds, results={}) + configs = (("e1e-3", dict(max_evalue=1e-3)), ("inf", dict(max_evalue=None)), + ("nopanel_inf", dict(max_evalue=None, panel_policy="never"))) + try: + for W in a.workers: + out = res["results"][str(W)] = {} + with ca.CodeAlignmentPool(W) as pool: + # the first call starts the persistent pool (cold): the whole workload + _r, _s, cold = world.align(pf, pool, max_evalue=1e-3) + out["pool_start_call_s"] = cold + out["pool_workers"] = pool.workers + out["pool_processes"] = getattr(pool, "processes", None) + for name, kw in configs: + walls, st, sig = [], None, None + for _ in range(a.rounds): + r_, st, w = world.align(pf, pool, **kw) + walls.append(w) + s_ = signature(r_) + if sig is not None and s_ != sig: + raise SystemExit(f"node {name}: output differs between repeats") + sig = s_ + m = float(np.median(walls)) + pairs = info["candidate_pairs"] + out[name] = dict(wall=m, walls=walls, counts=_counts(st), signature=sig, + candidate_pairs_per_s=pairs / m, + all_pairs_per_s=(pairs + (st.get("panel_pairs") or 0)) / m, + gcells_per_s=info["candidate_cells"] / m / 1e9) + print(f"node W={W} {name}: {m:.3f}s, {pairs / m:,.0f} candidate pairs/s " + f"({out[name]['all_pairs_per_s']:,.0f} incl. panel), survivors " + f"{st.get('survivors')}", flush=True) + if a.profile_run: + for name, kw in configs[:2]: + world.align(pf, pool, profile=True, **kw) # the profiled pool (a new worker setup) + _r, pst, pwall = world.align(pf, pool, profile=True, **kw) + out[f"profile_{name}"] = dict(wall=pwall, **_profile_brief(pst["profile"])) + out[f"profile_{name}_tasks"] = pst["profile"]["tasks"] if a.keep_tasks else None + _dump_partial(a, res) + if a.serial_nq: + sub = {q[0]: pf[q[0]] for q in queries[:a.serial_nq]} + npairs = sum(len(v[0]) for v in sub.values()) + ser = res["results"]["serial_subset"] = dict(nq=len(sub), pairs=npairs) + serial = ca.CodeAlignmentPool(1) + for name, kw in configs: + world.align({queries[0][0]: (pf[queries[0][0]][0][:50], pf[queries[0][0]][1][:50])}, serial, **kw) + _r, st, w = world.align(sub, serial, **kw) + ser[name] = dict(wall=w, pairs_per_s=npairs / w, counts=_counts(st)) + print(f"serial subset {len(sub)} queries {name}: {w:.3f}s, {npairs / w:,.0f} pairs/s", flush=True) + for W in a.workers: + full = res["results"][str(W)][name]["candidate_pairs_per_s"] + res["results"][str(W)][name]["linear_fraction"] = full / (W * npairs / w) + serial.close() + finally: + world.close() + return res + + +def sweep_mode(a) -> dict: + from foldmatch.search import code_alignment as ca + cs, queries, world, pf, info = _workload(a) + res = dict(mode="sweep", host=host_info(), workload=info, rounds=a.rounds, block_floats={}, task_cells={}) + sigs = set() + try: + for W in a.workers: + pool = ca.CodeAlignmentPool(max(W, 1)) + try: + world.align(pf, pool) # starts the persistent pool with the whole workload + res.setdefault("pool_workers", {})[str(W)] = pool.workers + for e in a.block_exps: + walls, sig = [], None + # a block size is part of the worker setup: the pool restarts at this first, untimed call + world.align(pf, pool, max_evalue=1e-3, block_floats=1 << e) + for _ in range(a.rounds): + r_, st, w = world.align(pf, pool, max_evalue=1e-3, block_floats=1 << e) + walls.append(w) + sig = signature(r_) + sigs.add(sig) + m = float(np.median(walls)) + res["block_floats"][f"{W}:2^{e}"] = dict(workers=W, block_floats=1 << e, wall=m, walls=walls, + signature=sig, pairs_per_s=info["candidate_pairs"] / m) + print(f"sweep W={W} block_floats 2^{e}: {m:.3f}s {info['candidate_pairs'] / m:,.0f} pairs/s " + f"sig {sig[:12]}", flush=True) + world.align(pf, pool) # back to the default block size (a restart), untimed + for e in a.task_exps: + walls, sig, st = [], None, None + for _ in range(a.rounds): + r_, st, w = world.align(pf, pool, max_evalue=1e-3, task_cells=1 << e) + walls.append(w) + sig = signature(r_) + sigs.add(sig) + m = float(np.median(walls)) + res["task_cells"][f"{W}:2^{e}"] = dict(workers=W, task_cells=1 << e, wall=m, walls=walls, + signature=sig, pairs_per_s=info["candidate_pairs"] / m, + round1_tasks=st.get("round1_tasks")) + print(f"sweep W={W} task_cells 2^{e}: {m:.3f}s {info['candidate_pairs'] / m:,.0f} pairs/s " + f"({st.get('round1_tasks')} round-1 tasks) sig {sig[:12]}", flush=True) + finally: + pool.close() + _dump_partial(a, res) + finally: + world.close() + res["all_outputs_identical"] = len(sigs) == 1 + res["signatures"] = sorted(sigs) + print(f"sweep: {len(sigs)} distinct output signature(s)", flush=True) + return res + + +def profile_mode(a) -> dict: + """For each worker count (0/1 = serial in-process): a profiled call on a fresh pool (cold: with the worker + start-up), a profiled call on the warm pool, then ``--rounds`` profile-off and profile-on calls (each set after + one untimed call, since switching the mode restarts the workers) for the overhead and the identical output, and + the time ``output.write_aligned_results`` takes for these hits (default fields and every field).""" + from foldmatch.search import code_alignment as ca + from foldmatch.search import output + cs, queries, world, pf, info = _workload(a) + res = dict(mode="profile", host=host_info(), workload=info, rounds=a.rounds, max_evalue=a.max_evalue, + results={}) + me = a.max_evalue if math.isfinite(a.max_evalue) else None + try: + for W in a.workers: + out = res["results"][str(W)] = {} + with ca.CodeAlignmentPool(max(W, 1)) as pool: + for label in ("cold", "warm"): + r_on, pst, w = world.align(pf, pool, max_evalue=me, profile=True) + prof = pst["profile"] + out[label] = dict(wall=w, counts=_counts(pst), **_profile_brief(prof)) + if a.keep_tasks: + out[f"{label}_tasks"] = prof["tasks"] + sig_on = signature(r_on) + walls = {} + for mode, flag in (("off", False), ("on", True)): + world.align(pf, pool, max_evalue=me, profile=flag) + walls[mode] = [] + for _ in range(a.rounds): + r_, _s, w = world.align(pf, pool, max_evalue=me, profile=flag) + walls[mode].append(w) + if mode == "off": + r_off, sig_off = r_, signature(r_) + out["off_walls"], out["on_walls"] = walls["off"], walls["on"] + out["off_median"], out["on_median"] = float(np.median(walls["off"])), float(np.median(walls["on"])) + out["profile_overhead_fraction"] = out["on_median"] / out["off_median"] - 1.0 + out["identical_output_on_off"] = sig_on == sig_off + out["signature"] = sig_off + with tempfile.TemporaryDirectory(dir=a.work) as td: + for label, fields in (("default", list(output.DEFAULT_OUTPUT_FIELDS)), + ("all", list(output.SUPPORTED_OUTPUT_FIELDS))): + r_w = r_off if label == "default" else world.align(pf, pool, max_evalue=me, fields=fields)[0] + t0 = time.perf_counter() + output.write_aligned_results(r_w, fields, str(Path(td) / f"{label}.tsv")) + out[f"output_{label}_s"] = time.perf_counter() - t0 + out[f"output_{label}_bytes"] = (Path(td) / f"{label}.tsv").stat().st_size + out[f"output_{label}_rows"] = sum(len(v) for v in r_w.values()) + for label in ("cold", "warm"): + p = out[label] + print(f"profile W={W} {label}: wall {p['wall']:.3f}s; parent " + ", ".join( + f"{k} {v:.3f}" for k, v in p["parent"].items()) + "; workers " + "; ".join( + f"{k}: {v['tasks']} tasks {v['seconds']:.3f}s (sim {v['sim']:.3f} kernel {v['kernel']:.3f})" + for k, v in p["workers"].items()), flush=True) + print(f"profile W={W}: off {out['off_median']:.3f}s on {out['on_median']:.3f}s (overhead " + f"{100 * out['profile_overhead_fraction']:.1f} %), identical output {out['identical_output_on_off']}" + f"; output default {out['output_default_s']:.3f}s ({out['output_default_rows']} rows), all fields " + f"{out['output_all_s']:.3f}s", flush=True) + _dump_partial(a, res) + finally: + world.close() + return res + + +def rss_mode(a) -> dict: + from foldmatch.search import code_alignment as ca + a.query_set = f"fixed:{a.rss_lq}" + cs, queries, world, pf, info = _workload(a) + res = dict(mode="rss", host=host_info(), workload=info, results={}) + try: + for W in a.workers: + with ca.CodeAlignmentPool(W) as pool: + for name, kw in (("e1e-3", dict(max_evalue=1e-3)), ("inf", dict(max_evalue=None))): + _r, pst, w = world.align(pf, pool, profile=True, **kw) + mem = pst["profile"]["memory"] + first = {} + for t in pst["profile"]["tasks"]: + first.setdefault(t["pid"], t) + res["results"][f"{W}:{name}"] = dict(wall=w, memory=mem, counts=_counts(pst), + worker_rss_first_task_bytes=sorted( + v.get("rss_max_bytes", 0) for v in first.values())) + print(f"rss W={W} {name}: worker RSS max {mem['worker_rss_max_bytes'] / 2**20:.1f} MiB, parent " + f"{mem['parent_rss_max_bytes'] / 2**20:.1f} MiB", flush=True) + finally: + world.close() + return res + + +# ===================================================================================================================== +# mode: replay (the real pipeline's Stage-2 inputs, captured by stage2_capture.py) +# ===================================================================================================================== +class Replay: + """A captured Stage-2 call: the database's stores, the query memmap, the Stage-1 candidates, N and options.""" + + def __init__(self, capture: Path, db_dir: Path = None, lq_range=None, max_candidates=None): + from foldmatch.search import scoring_system as ss + from foldmatch.search.query_encodings import QueryEncodingEntry, QueryEncodingIndex + from foldmatch.search.residue_code_store import RESCODES_BIN_SUFFIX, ResidueCodeStore + from foldmatch.search.sequence_store import SequenceStore + cap = json.loads((capture / "capture.json").read_text()) + self.cap = cap + bin_path = Path(cap["code_store_bin"]) + name = bin_path.name[:-len(RESCODES_BIN_SUFFIX)] + folder = Path(db_dir) if db_dir else bin_path.parent + self.store = ResidueCodeStore(folder, name) + self.seq_store = SequenceStore(folder, name) + manifest = self.store.manifest() + self.system = ss.get(manifest["system_name"]) + self.prepared = ss.prepare(self.system, local_files_only=bool(os.environ.get("HF_HUB_OFFLINE"))) + entries = {q: QueryEncodingEntry(int(o), int(n), bytes.fromhex(h)) for q, (o, n, h) in cap["entries"].items()} + self.query_index = QueryEncodingIndex(capture / "queries.f16", cap["rows"], cap["width"], entries, + cap["system_name"]) + self.query_sequences = cap["query_sequences"] + pf = {q: (list(v[0]), list(v[1])) for q, v in cap["prefilter_results"].items()} + if lq_range: + lo, hi = lq_range + pf = {q: v for q, v in pf.items() if lo <= len(self.query_sequences[q]) <= hi} + if max_candidates: + pf = {q: (v[0][:max_candidates], v[1][:max_candidates]) for q, v in pf.items()} + self.prefilter = pf + self.db_residues = int(cap["db_residues"]) + self.options = dict(cap["options"]) + self.fields = self.options.pop("output_needs") + self.options.pop("max_evalue", None) + self.max_evalue = cap["options"].get("max_evalue") + + def align(self, pool, *, max_evalue="capture", profile=False, prefilter=None, **kw): + from foldmatch.search import code_alignment as ca + me = self.max_evalue if max_evalue == "capture" else max_evalue + opts = dict(self.options) + opts.update(kw) + stats = {} + old = os.environ.get(PROFILE_ENV) + os.environ[PROFILE_ENV] = "1" if profile else "" + try: + t0 = time.perf_counter() + res = ca.align_candidates_esm_codes( + self.query_index, prefilter if prefilter is not None else self.prefilter, self.store, + self.seq_store.fetch, self.prepared, self.db_residues, pool, query_sequences=self.query_sequences, + max_evalue=me, output_needs=self.fields, stats=stats, **opts) + wall = time.perf_counter() - t0 + finally: + if old is None: + os.environ.pop(PROFILE_ENV, None) + else: + os.environ[PROFILE_ENV] = old + return res, stats, wall + + def blosum(self, num_workers=1, prefilter=None): + from foldmatch.search import alignment + t0 = time.perf_counter() + res = alignment.align_candidates(self.query_sequences, prefilter if prefilter is not None else self.prefilter, + self.seq_store.fetch, num_workers=num_workers, + subject_db_size=self.db_residues, output_fields=None) + return res, time.perf_counter() - t0 + + +def replay_mode(a) -> dict: + """The captured real-pipeline Stage 2 for each ``--workers``: a cold profiled call (fresh pool), a warm profiled + call, ``--rounds`` profile-off calls (wall, output signature: equal to the CLI's when the capture's machine has + the same BLAS and CPU kernel), the same at ``--max-evalue`` inf when ``--also-inf``; BLOSUM62 on the same + candidates with ``--blosum-workers``.""" + from foldmatch.search import code_alignment as ca + rp = Replay(a.capture, a.db_dir, a.lq_range, a.max_candidates) + pf = rp.prefilter + lens = [len(rp.query_sequences[q]) for q in pf] + npairs = sum(len(v[0]) for v in pf.values()) + info = dict(capture=str(a.capture), queries=len(pf), queries_with_candidates=sum(1 for v in pf.values() if v[0]), + candidate_pairs=npairs, query_lengths=dict(min=min(lens), median=float(np.median(lens)), + max=max(lens)), + db_residues=rp.db_residues, max_evalue=rp.max_evalue, options=rp.options, fields=rp.fields, + cli_signature=rp.cap["signature"], cli_stage2_s=rp.cap["stage2_s"], cli_pool=rp.cap["pool"], + lq_range=a.lq_range, max_candidates=a.max_candidates) + res = dict(mode="replay", host=host_info(), workload=info, rounds=a.rounds, results={}, blosum62={}) + settings = [("capture", "capture")] + ([("inf", None)] if a.also_inf else []) + for W in a.workers: + out = res["results"][str(W)] = {} + with ca.CodeAlignmentPool(max(W, 1)) as pool: + for label in (("cold",) if a.no_warm_profile else ("cold", "warm")): + _r, pst, w = rp.align(pool, profile=True) + out[label] = dict(wall=w, counts=_counts(pst), **_profile_brief(pst["profile"])) + if a.keep_tasks: + out[f"{label}_tasks"] = pst["profile"]["tasks"] + for name, me in settings: + if not a.no_warmup: + rp.align(pool, max_evalue=me) + walls, sig, st = [], None, None + for _ in range(a.rounds): + r_, st, w = rp.align(pool, max_evalue=me) + walls.append(w) + sig = signature(r_) + out[name] = dict(wall=float(np.median(walls)), walls=walls, signature=sig, counts=_counts(st), + pairs_per_s=npairs / float(np.median(walls))) + if name == "inf": + _r, pst, w = rp.align(pool, max_evalue=None, profile=True) + out["inf_profiled"] = dict(wall=w, counts=_counts(pst), **_profile_brief(pst["profile"])) + out["identical_to_cli"] = (out["capture"]["signature"] == rp.cap["signature"] + if a.lq_range is None and a.max_candidates is None else None) + c = out.get("warm", out["cold"]) + print(f"replay W={W}: cold {out['cold']['wall']:.3f}s, warm {c['wall']:.3f}s, off {out['capture']['wall']:.3f}s " + f"({out['capture']['counts']['survivors']} survivors of {npairs}); identical to the CLI " + f"{out['identical_to_cli']}; parent " + ", ".join(f"{k} {v:.3f}" for k, v in c["parent"].items()) + + "; workers " + "; ".join(f"{k}: {v['tasks']} tasks {v['seconds']:.3f}s" for k, v in + c["workers"].items()), flush=True) + _dump_partial(a, res) + for W in a.blosum_workers: + walls = [] + for _ in range(a.blosum_rounds): + _r, w = rp.blosum(W) + walls.append(w) + res["blosum62"][str(W)] = dict(wall=float(np.median(walls)), walls=walls, pairs_per_s=npairs / np.median(walls)) + print(f"replay BLOSUM62 W={W}: {np.median(walls):.3f}s", flush=True) + return res + + +# ===================================================================================================================== +# mode: research-batch (the research driver; never imports this tree's swkern) +# ===================================================================================================================== +def research_batch_mode(a) -> dict: + """The research throughput bench's FoldMatch-shaped driver (codepath.align_candidates_x, scorer 'code': per block + decode (np.unpackbits) + ONE block GEMM + rowcol z folded into the research strip SW; research setting offset + 0.5, gaps 3.0 / 0.3, cost-neutral vs setting D) on the same workload as the FoldMatch modes.""" + for p in (a.research_bench, a.research_build): + if p is None: + raise SystemExit("--research-bench and --research-build are required") + paths = [str(a.research_build), str(a.research_bench)] + if a.research_fm_src: + paths.insert(0, str(a.research_fm_src)) + for p in reversed(paths): + sys.path.insert(0, p) + import foldmatch.search.alignment as FA # noqa: F401 (the snapshot when --research-fm-src) + import codepath as CP + import swfast + cs = ChainSet(a.chains, a.chain_seqs) + queries = make_queries(a, cs) + pf = batch_prefilter(cs, queries, a.k) + seq_of = {t: cs.seqs[int(cs.draw[k])] for k, t in enumerate(cs.cand_ids)} + code_of = {t: np.ascontiguousarray(cs.chain_codes(int(cs.draw[k]))) for k, t in enumerate(cs.cand_ids)} + qseq = {q[0]: q[1] for q in queries} + qside = {q[0]: np.ascontiguousarray(swfast.query_prep(q[2].astype(np.float32))) for q in queries} + fetch = (lambda ids: {i: seq_of[i] for i in ids}) + codes = (lambda ids: {i: code_of[i] for i in ids}) + import multiprocessing as mp + prewarmed = False + if mp.get_start_method() == "fork": + CP.worker_init(dict(scorer="code", mode="tb", tb_fraction=0.0, lam=CP.CODE_LAM, k=CP.CODE_K, gaps=None, + db_residues=100_000_000, needs=None, block_cells=CP.BLOCK_CELLS), {}) + prewarmed = True + import foldmatch + res = dict(mode="research-batch", host=host_info(with_kernel=False), foldmatch_imported_from=foldmatch.__file__, + research_bench=str(a.research_bench), research_build=str(a.research_build), + swfast_vector_lanes=int(swfast.VECTOR_LANES), start_method=mp.get_start_method(), + parent_prewarmed=prewarmed, nq=len(queries), K=a.k, query_set=a.query_set, rounds=a.rounds, + results={}) + npairs = len(queries) * a.k + modes = dict(s0=dict(mode="score", tb_fraction=0.0, identity="vector"), + s10=dict(mode="score", tb_fraction=0.1, identity="vector"), + tb=dict(mode="tb", identity="vector")) + for W in a.workers: + for name in a.research_modes: + kw = modes[name] + q0 = queries[0][0] + + def call(sub=None): + p = pf if sub is None else sub + return CP.align_candidates_x({k: qseq[k] for k in p}, p, fetch, codes, {k: qside[k] for k in p}, + scorer="code", min_seq_identity=0.0, num_workers=W, min_coverage=0.8, + max_evalue=1e-3, subject_db_size=100_000_000, **kw) + call({q0: (pf[q0][0][:100], pf[q0][1][:100])}) + walls, meta = [], None + for _ in range(a.rounds): + t0 = time.perf_counter() + _r, meta = call() + walls.append(time.perf_counter() - t0) + m = float(np.median(walls)) + res["results"][f"{W}:{name}"] = dict(workers=W, mode=name, wall=m, walls=walls, pairs=npairs, + pairs_per_s=npairs / m, + meta={k: v for k, v in (meta or {}).items() if k != "wall"}) + print(f"research W={W} {name}: {m:.3f}s {npairs / m:,.0f} pairs/s", flush=True) + _dump_partial(a, res) + return res + + +# ===================================================================================================================== +# mode: bandwidth +# ===================================================================================================================== +def _triad_worker(args): + n, seconds, cpu = args + if cpu is not None and hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(0, {cpu}) + a_ = np.ones(n, np.float32) + b_ = np.ones(n, np.float32) + c_ = np.ones(n, np.float32) + s = np.float32(1.5) + reps, t0 = 0, time.perf_counter() + while time.perf_counter() - t0 < seconds: + np.multiply(c_, s, out=a_) + np.add(a_, b_, out=a_) + reps += 1 + dt = time.perf_counter() - t0 + return reps * 4 * n * 4 / dt # bytes: read c, write a, read a, read b, write a ~ 4 streams per rep + + +def bandwidth_mode(a) -> dict: + import multiprocessing as mp + res = dict(mode="bandwidth", host=host_info(with_kernel=False), results={}) + ctx = mp.get_context("spawn") + cpus = sorted(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else list(range(os.cpu_count())) + for W in a.workers: + for label, n in (("dram_64MiB", 1 << 24), ("l2_256KiB", 1 << 16)): + pin = [cpus[i % len(cpus)] for i in range(W)] if a.pin else [None] * W + with ctx.Pool(W) as pool: + bw = pool.map(_triad_worker, [(n, a.seconds, c) for c in pin]) + res["results"][f"{W}:{label}"] = dict(workers=W, floats=n, gbytes_per_s_total=sum(bw) / 1e9, + gbytes_per_s_per_worker=float(np.median(bw)) / 1e9) + print(f"bandwidth W={W} {label}: {sum(bw) / 1e9:.1f} GB/s total, {np.median(bw) / 1e9:.2f} per worker", + flush=True) + return res + + +# ===================================================================================================================== +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("mode", choices=["kernel", "components", "grid", "panel", "perquery", "node", "sweep", "profile", + "rss", "replay", "research-batch", "bandwidth"]) + ap.add_argument("--chains", type=Path) + ap.add_argument("--chain-seqs", type=Path) + ap.add_argument("--queries", type=Path, help="queries.L2m_B8_s0.npz (real benchmark queries)") + ap.add_argument("--work", type=Path, default=Path("stage2_bench_work")) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--db-residues", type=int, default=BENCH_N, help="N for the E-values (default: the research " + "E-value benchmark's 15,889,271 residues)") + ap.add_argument("--lq", type=int, nargs="+", default=None) + ap.add_argument("--k", type=int, default=1000) + ap.add_argument("--ks", type=int, nargs="+", default=[100, 1000, 10000]) + ap.add_argument("--nq", type=int, default=None, help="queries (default 64; perquery: 0 extra real queries)") + ap.add_argument("--query-set", default="real", help="real | fixed: | chain:") + ap.add_argument("--workers", type=int, nargs="+", default=[1], + help="worker counts; panel / perquery: -1 is the default pool (num_workers=None)") + ap.add_argument("--rounds", type=int, default=3) + ap.add_argument("--blosum-rounds", type=int, default=3) + ap.add_argument("--blosum-pool", action="store_true", help="perquery: also BLOSUM62 with each worker count") + ap.add_argument("--max-evalue", type=float, default=1e-3) + ap.add_argument("--warm", action="store_true", help="profile: start the pool before the profiled run") + ap.add_argument("--keep-tasks", action="store_true", help="keep the per-task profile records in the JSON") + ap.add_argument("--profile-run", action="store_true", help="node: one profiled 1e-3 run per worker count") + ap.add_argument("--serial-nq", type=int, default=0, help="node: queries of the serial per-core reference") + ap.add_argument("--block-floats", type=int, default=1 << 17) + ap.add_argument("--block-exps", type=int, nargs="+", default=[14, 15, 16, 17, 18, 19, 20]) + ap.add_argument("--task-exps", type=int, nargs="+", default=[20, 21, 22, 23, 24, 25, 26]) + ap.add_argument("--panel-lengths", type=int, nargs="+", default=[71, 966]) + ap.add_argument("--pool-query-length", type=int, default=300) + ap.add_argument("--rss-lq", type=int, default=1000) + ap.add_argument("--cc", default=None) + # replay + ap.add_argument("--capture", type=Path, help="replay: a stage2_capture.py directory") + ap.add_argument("--db-dir", type=Path, help="replay: the database folder (default: the capture's)") + ap.add_argument("--lq-range", type=int, nargs=2, default=None, help="replay: only queries of these lengths") + ap.add_argument("--max-candidates", type=int, default=None, help="replay: the first K candidates per query") + ap.add_argument("--also-inf", action="store_true", help="replay: also at max_evalue inf") + ap.add_argument("--blosum-workers", type=int, nargs="*", default=[], help="replay: BLOSUM62 worker counts") + ap.add_argument("--no-warm-profile", action="store_true", help="replay: only the cold profiled call") + ap.add_argument("--no-warmup", action="store_true", help="replay: no untimed call before the timed ones") + ap.add_argument("--big-k", type=int, default=10000, help="grid: K from which --big-k-rounds apply") + ap.add_argument("--big-k-rounds", type=int, default=1, help="grid: rounds (no warm-up) for K >= --big-k") + # research + ap.add_argument("--research-src", type=Path, help="directory with the research swkern.c and swmulti.c") + ap.add_argument("--research-lib-dir", type=Path, help="prebuilt research libswkern / libswmulti") + ap.add_argument("--research-cc", default=os.environ.get("CC", "cc")) + ap.add_argument("--research-cflags", default=None) + ap.add_argument("--research-bench", type=Path, help="research throughput bench dir (codepath.py)") + ap.add_argument("--research-build", type=Path, help="research build dir (swfast.py + libswkern)") + ap.add_argument("--research-fm-src", type=Path, help="the FoldMatch src snapshot of the research runs") + ap.add_argument("--research-modes", nargs="+", default=["s0", "s10", "tb"]) + ap.add_argument("--blocks", type=int, nargs="+", default=[17, 20], help="kernel: FoldMatch block exponents") + ap.add_argument("--research-blocks", type=int, nargs="+", default=[20, 17]) + ap.add_argument("--seconds", type=float, default=3.0) + ap.add_argument("--pin", action="store_true") + ap.add_argument("--core", type=int, default=None, help="Linux: pin this process to one core") + a = ap.parse_args() + if a.lq is None: + a.lq = [300] if a.mode == "kernel" else [100, 300, 1000] + if a.nq is None: + a.nq = 0 if a.mode == "perquery" else 64 + if a.core is not None and hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(0, {a.core}) + t0 = time.time() + fn = dict(kernel=kernel_mode, components=components_mode, grid=grid_mode, panel=panel_mode, + perquery=perquery_mode, node=node_mode, sweep=sweep_mode, profile=profile_mode, rss=rss_mode, + replay=replay_mode, bandwidth=bandwidth_mode) + fn["research-batch"] = research_batch_mode + res = fn[a.mode](a) + res["argv"] = sys.argv + res["seconds"] = round(time.time() - t0, 1) + a.out.parent.mkdir(parents=True, exist_ok=True) + a.out.write_text(json.dumps(res, indent=1, default=str)) + part = a.out.with_suffix(".partial.json") + if part.exists(): + part.unlink() + print(f"wrote {a.out}") + + +if __name__ == "__main__": + main() diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..df944d6 --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,180 @@ +"""Hatchling build hook: compile the native Smith-Waterman kernel ``fmsw`` into the wheel (PLAN section 7, P3.2). + +Standard wheels + Every ISA library of the target machine (``sse2``, ``avx2``, ``generic`` on x86_64; ``neon``, ``generic`` on + arm64; ``generic`` elsewhere) is compiled into a fresh temporary directory, never reusing libraries built in place, + and shipped through ``build_data["force_include"]`` as ``foldmatch/_native/_fmsw_``. The wheel is + tagged ``py3-none-``: the sysconfig platform on Linux (``linux_x86_64``, ``linux_aarch64``), and on + macOS ``macosx__`` with the deployment target ``$MACOSX_DEPLOYMENT_TARGET`` (default 11.0, passed + to the compiler as ``-mmacosx-version-min``). No manylinux tag is written here: auditwheel (Linux) and delocate + (macOS) check the libraries and retag the wheel. + +Editable installs + The libraries are compiled in place into ``src/foldmatch/_native/`` (gitignored), exactly as the developer build + ``python -m foldmatch._native.build`` does; the editable wheel itself carries none. + +Compile failures + ``FOLDMATCH_REQUIRE_NATIVE=1`` (CI, Docker, cibuildwheel) makes a missing or failing C compiler fatal. Without it + the hook warns and the build falls back to a pure-Python wheel (``py3-none-any``): the Stage 2 of a search + (``--scoring blosum62`` and ``esm-codes``) refuses with the reinstall command (pip caches that wheel, hence + ``--no-cache-dir``); everything else works. + +Every library embeds the sha256 of the kernel source it was compiled from (``fmsw_source_sha256()``), which the +loader compares with the installed source. The compile step itself is ``foldmatch._native.build.compile_library``, +loaded by file path because the build environment has neither numpy nor the foldmatch dependencies. +""" +from __future__ import annotations + +import importlib.util +import os +import platform +import re +import shutil +import sys +import sysconfig +import tempfile +from pathlib import Path +from types import ModuleType +from typing import Any, Dict, List, Optional, Tuple + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +REQUIRE_NATIVE_ENV = "FOLDMATCH_REQUIRE_NATIVE" +DEFAULT_MACOS_TARGET = "11.0" +NATIVE_SRC = Path("src", "foldmatch", "_native") # relative to the project root +NATIVE_WHEEL = "foldmatch/_native" # inside the wheel + + +class _TargetError(RuntimeError): + """The build target cannot get native libraries (e.g. a universal2 macOS build).""" + + +def load_native_build(root: str) -> ModuleType: + """``src/foldmatch/_native/build.py`` of the project at ``root``, loaded by file path (standard library only).""" + path = Path(root) / NATIVE_SRC / "build.py" + spec = importlib.util.spec_from_file_location("_foldmatch_native_build_hook", path) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load {path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _macos_arch() -> str: + """Target architecture of a macOS build: a single ``-arch`` in ``$ARCHFLAGS`` (cross builds, cibuildwheel), else + the architecture in ``$_PYTHON_HOST_PLATFORM``, else this machine.""" + archs = sorted(set(re.findall(r"-arch\s+(\S+)", os.environ.get("ARCHFLAGS", "")))) + if len(archs) > 1: + raise _TargetError(f"ARCHFLAGS={os.environ['ARCHFLAGS']!r} asks for a multi-architecture (universal2) " + "build; the native kernel is built per architecture, so build one wheel per architecture") + if archs: + return archs[0] + host = os.environ.get("_PYTHON_HOST_PLATFORM", "") + if host.startswith("macosx-"): + arch = host.rsplit("-", 1)[-1] + if arch in ("arm64", "x86_64"): + return arch + return platform.machine() + + +def _macos_target(arch: str) -> Tuple[str, str]: + """(``-mmacosx-version-min`` value, wheel tag version ``major_minor``) for the deployment target.""" + target = (os.environ.get("MACOSX_DEPLOYMENT_TARGET") or DEFAULT_MACOS_TARGET).strip() + m = re.fullmatch(r"(\d+)(?:\.(\d+))?(?:\.\d+)?", target) + if not m: + raise _TargetError(f"MACOSX_DEPLOYMENT_TARGET={target!r} is not a macOS version") + major, minor = int(m.group(1)), int(m.group(2) or 0) + if arch == "arm64" and major < 11: + target, major, minor = "11.0", 11, 0 # the first macOS on Apple silicon + if major >= 11: + minor = 0 # pip only knows macosx__0 tags from macOS 11 on + return target, f"{major}_{minor}" + + +def target_platform() -> Tuple[str, str, List[str]]: + """(wheel platform tag, target machine, extra compiler flags) of this build.""" + if sys.platform == "darwin": + arch = _macos_arch() + version_min, tag_version = _macos_target(arch) + flags = [f"-mmacosx-version-min={version_min}"] + if arch != platform.machine(): + flags[:0] = ["-arch", arch] # cross build (clang); a native build needs no -arch (gcc has none) + return f"macosx_{tag_version}_{arch}", arch, flags + plat = sysconfig.get_platform() # honours _PYTHON_HOST_PLATFORM + tag = re.sub(r"[^A-Za-z0-9_]", "_", plat) + machine = plat.split("-", 1)[1] if "-" in plat else platform.machine() + return tag, machine, [] + + +class FoldMatchNativeBuildHook(BuildHookInterface): + """Compiles the fmsw kernel libraries for the wheel target (see the module docstring).""" + + PLUGIN_NAME = "custom" + + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self._tmp: Optional[str] = None + + def initialize(self, version: str, build_data: Dict[str, Any]) -> None: + if self.target_name != "wheel": + return + required = os.environ.get(REQUIRE_NATIVE_ENV) == "1" + try: + native = load_native_build(self.root) + except Exception as e: # a broken tree is never acceptable, but report it like a compile failure + self._failed(f"cannot load the kernel build module: {e}", required, None) + return + try: + if version == "editable": + self._build_in_place(native) + else: + self._build_into_wheel(native, build_data) + except (native.NativeBuildError, _TargetError) as e: + self._cleanup() + self._failed(str(e), required, native) + + # ----------------------------------------------------------------------------------------------------------------- + def _build_in_place(self, native: ModuleType) -> None: + out = Path(self.root) / NATIVE_SRC + paths = native.build(out_dir=out, quiet=True) + self.app.display_info(f"fmsw: built {', '.join(p.name for p in paths)} in place in {out} " + f"(source sha256 {native.source_sha256()})") + + def _build_into_wheel(self, native: ModuleType, build_data: Dict[str, Any]) -> None: + plat_tag, machine, flags = target_platform() + isas = native.machine_isas(machine) + self._tmp = tempfile.mkdtemp(prefix="foldmatch-fmsw-") + built = [] + for isa in isas: + path = native.compile_library(isa, Path(self._tmp), extra_flags=flags, machine=machine, quiet=True) + if not native.library_is_current(path): + raise native.NativeBuildError(f"{path.name} does not embed the source sha256 of this tree") + built.append(path) + # build_data changes only once every library is built: a failure part-way (one ISA does not compile) leaves + # it untouched, so the fallback is the pure wheel, never a wheel pointing at deleted temporary files + for path in built: + build_data["force_include"][str(path)] = f"{NATIVE_WHEEL}/{path.name}" + build_data["pure_python"] = False + build_data["infer_tag"] = False + build_data["tag"] = f"py3-none-{plat_tag}" + self.app.display_info(f"fmsw: built {', '.join(p.name for p in built)} for {machine} (tag {build_data['tag']}, " + f"compiler {native.find_compiler()}, source sha256 {native.source_sha256()})") + + def _failed(self, reason: str, required: bool, native: Optional[ModuleType]) -> None: + reinstall = native.REINSTALL_COMMAND if native is not None else \ + "pip install --no-cache-dir --force-reinstall --no-binary foldmatch foldmatch" + if required: + raise RuntimeError(f"FoldMatch native kernel build failed and {REQUIRE_NATIVE_ENV}=1: {reason}") + self.app.display_warning( + f"FoldMatch native kernel not built ({reason}). Building a pure-Python package: the Stage 2 of a search " + f"(sequence identity, under --scoring blosum62 and esm-codes) will refuse to run; everything else works. " + f"Install a C compiler and reinstall with `{reinstall}`; set {REQUIRE_NATIVE_ENV}=1 to make this an " + f"error.") + + def _cleanup(self) -> None: + if self._tmp is not None: + shutil.rmtree(self._tmp, ignore_errors=True) + self._tmp = None + + def finalize(self, version: str, build_data: Dict[str, Any], artifact_path: str) -> None: + self._cleanup() diff --git a/pyproject.toml b/pyproject.toml index 94d7d30..98f8a08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,8 @@ readme = "README.md" requires-python = ">=3.12" classifiers = [ "Programming Language :: Python :: 3", - "Operating System :: OS Independent", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", ] license = {file = "LICENSE.md"} @@ -19,13 +20,14 @@ dependencies=[ "biotite==1.7.1", "lightning==2.6.1", "typer==0.24.1", + "click==8.3.1", "hf-xet==1.4.2", "huggingface-hub==0.36.2", "httpx==0.28.1", - "faiss-cpu==1.13.2", + "faiss-cpu==1.15.1", "igraph==1.0.0", - "leidenalg==0.11.0", "pyarrow==24.0.0", + "threadpoolctl==3.6.0", ] [project.urls] @@ -43,6 +45,19 @@ requires = [ ] build-backend = "hatchling.build" +[tool.hatch.build] +# Native kernel libraries built in place (developer build, editable installs) never ship: wheels get libraries +# freshly compiled by the build hook, and the sdist carries only the C source. +exclude = [ + "/src/foldmatch/_native/_fmsw_*", + "/src/foldmatch/_native/._fmsw_*", +] + +[tool.hatch.build.targets.wheel.hooks.custom] +# hatch_build.py compiles the fmsw Smith-Waterman kernel (one library per ISA) and tags the wheel +# py3-none-; FOLDMATCH_REQUIRE_NATIVE=1 makes a compile failure fatal (otherwise: pure wheel + warning). +path = "hatch_build.py" + [project.scripts] inference = "foldmatch.cli.inference:main" fm-embedding = "foldmatch.cli.embedding:main" diff --git a/src/foldmatch/_native/__init__.py b/src/foldmatch/_native/__init__.py new file mode 100644 index 0000000..9a90721 --- /dev/null +++ b/src/foldmatch/_native/__init__.py @@ -0,0 +1,114 @@ +"""Native Smith-Waterman kernel (``fmsw``): library discovery. + +The kernel source lives in ``fmsw/fmsw.c``, ``fmsw/fmsw.h`` and ``fmsw/fmswi_body.h`` (the integer Smith-Waterman +template that ``fmsw.c`` includes twice). It is compiled into one shared library per +instruction set, named ``_fmsw_`` and loaded with ctypes by :mod:`foldmatch.search.swkern`: + +========== ===================== ====================================== +ISA built on extra compile flags +========== ===================== ====================================== +``sse2`` x86_64 none (the x86_64 baseline) +``avx2`` x86_64 ``-mavx2`` (no ``-mfma``) +``neon`` aarch64 / arm64 none +``generic`` every platform ``-DFMSW_GENERIC`` (vector extensions) +========== ===================== ====================================== + +Every x86_64 library is compiled with ``-march=x86-64 -mtune=generic`` first, so a compiler that defaults to a newer +``-march`` cannot put AVX2 code into the sse2 or generic library (see :mod:`foldmatch._native.build`). + +Wheels ship the libraries next to this file. Editable installs and source checkouts build them in place with +``python -m foldmatch._native.build`` (the libraries are gitignored). This module only locates files; it never +compiles and never imports numpy or torch. +""" +from __future__ import annotations + +import hashlib +import platform +import sys +from pathlib import Path +from typing import Iterable, Optional, Tuple + +NATIVE_DIR = Path(__file__).resolve().parent +SOURCE_DIR = NATIVE_DIR / "fmsw" +SOURCE_FILES = ("fmsw.h", "fmsw.c", "fmswi_body.h") +ISAS = ("sse2", "avx2", "neon", "generic") + +REBUILD_COMMAND = "python -m foldmatch._native.build" +REINSTALL_COMMAND = "pip install --no-cache-dir --force-reinstall --no-binary foldmatch foldmatch" + + +def library_suffix() -> str: + """Shared-library suffix of this platform.""" + return ".dylib" if sys.platform == "darwin" else ".so" + + +def library_filename(isa: str) -> str: + if isa not in ISAS: + raise ValueError(f"unknown fmsw ISA {isa!r}; expected one of {', '.join(ISAS)}") + return f"_fmsw_{isa}{library_suffix()}" + + +def normalized_machine(machine: Optional[str] = None) -> str: + """``x86_64``, ``arm64`` or the raw lower-cased ``platform.machine()`` for anything else.""" + m = (machine or platform.machine() or "").lower() + if m in ("x86_64", "amd64", "x64"): + return "x86_64" + if m in ("arm64", "aarch64", "armv8l", "arm64e"): + return "arm64" + return m + + +def machine_isas(machine: Optional[str] = None) -> Tuple[str, ...]: + """The ISA libraries built (and loadable) on a machine, in dispatch-preference order.""" + m = normalized_machine(machine) + if m == "x86_64": + return ("sse2", "avx2", "generic") + if m == "arm64": + return ("neon", "generic") + return ("generic",) + + +def find_library(isa: str, search_dirs: Optional[Iterable[Path]] = None) -> Optional[Path]: + """Path of the ``isa`` library in ``search_dirs`` (default: this package directory), or None.""" + name = library_filename(isa) + for d in (search_dirs if search_dirs is not None else (NATIVE_DIR,)): + p = Path(d) / name + if p.is_file(): + return p + return None + + +def source_sha256(source_dir: Optional[Path] = None) -> Optional[str]: + """sha256 identifying the kernel source: over each of ``SOURCE_FILES`` (in that order) the bytes + `` NUL NUL ``. The build injects it into the library (``fmsw_source_sha256()``); + a library whose value differs from the tree's is stale. None when the source files are not present.""" + d = Path(source_dir) if source_dir is not None else SOURCE_DIR + h = hashlib.sha256() + for name in SOURCE_FILES: + p = d / name + if not p.is_file(): + return None + data = p.read_bytes() + h.update(name.encode("ascii") + b"\0" + str(len(data)).encode("ascii") + b"\0") + h.update(data) + return h.hexdigest() + + +def in_source_tree(native_dir: Optional[Path] = None) -> bool: + """True for a source checkout or an editable install (``/src/foldmatch/_native`` next to + ``/pyproject.toml``), False for an installed wheel.""" + src = Path(native_dir if native_dir is not None else NATIVE_DIR).parent.parent + return src.name == "src" and (src.parent / "pyproject.toml").is_file() + + +def rebuild_hint(missing: bool = False, native_dir: Optional[Path] = None) -> str: + """How to get working libraries, for refusal messages. ``missing``: no library at all was found (for an installed + package that means a pure-Python wheel, built from the sdist without a C compiler).""" + if in_source_tree(native_dir): + return (f"build the native kernel with `{REBUILD_COMMAND}` (needs a C compiler); for a pip install from an " + f"sdist, reinstall with a C compiler available: `{REINSTALL_COMMAND}`") + if missing: + return ("this FoldMatch was installed without its native kernel: a pure-Python wheel, built from the sdist " + "where no C compiler was available (pip caches and reuses that wheel). Install a C compiler (gcc or " + f"clang) and reinstall: `{REINSTALL_COMMAND}`") + return f"reinstall FoldMatch with a C compiler available: `{REINSTALL_COMMAND}`" diff --git a/src/foldmatch/_native/build.py b/src/foldmatch/_native/build.py new file mode 100644 index 0000000..ce35376 --- /dev/null +++ b/src/foldmatch/_native/build.py @@ -0,0 +1,176 @@ +"""Developer build of the native Smith-Waterman kernel. + + python -m foldmatch._native.build [--isa ISA ...] [--out DIR] [--cc CC] + +compiles ``fmsw/fmsw.c`` into one shared library per ISA of this machine (``sse2``, ``avx2`` and ``generic`` on +x86_64; ``neon`` and ``generic`` on arm64; ``generic`` elsewhere), in place into ``src/foldmatch/_native/`` (the +libraries are gitignored). Rebuild after every C edit: :mod:`foldmatch.search.swkern` refuses a library whose +embedded source sha256 differs from the tree. + +Every build uses exactly ``-O3 -std=c11 -fPIC -shared -ffp-contract=off -fno-fast-math -fvisibility=hidden`` +plus the ISA flag (``-mavx2`` for avx2, ``-DFMSW_GENERIC`` for generic) and the injected source sha256; never +``-march=native`` and never ``-mfma``. Every x86_64 library is compiled for the plain x86-64 baseline +(``-march=x86-64 -mtune=generic`` before the ISA flag), whatever ``-march`` the compiler or a wrapper defaults to +(e.g. ``x86-64-v3`` distribution compilers, ``-march=znver3`` from the Cray ``cc`` wrapper): the sse2 and generic +libraries must not contain AVX2 code, and avx2 adds exactly ``-mavx2``. On macOS the install name is set to +``@rpath/`` (a linker flag only). + +:func:`compile_library` is the single compile step, also used by the packaging build hook. This module uses the +standard library only and can be loaded by file path (without importing the foldmatch package). +""" +from __future__ import annotations + +import argparse +import importlib.util +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Iterable, List, Optional, Sequence + +try: # imported as foldmatch._native.build + from . import (ISAS, NATIVE_DIR, REINSTALL_COMMAND, SOURCE_DIR, library_filename, machine_isas, + normalized_machine, source_sha256) +except ImportError: # loaded by file path (packaging build hook): load the sibling __init__.py the same way + _spec = importlib.util.spec_from_file_location("_foldmatch_native_init", Path(__file__).with_name("__init__.py")) + _init = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(_init) + ISAS, NATIVE_DIR, SOURCE_DIR = _init.ISAS, _init.NATIVE_DIR, _init.SOURCE_DIR + REINSTALL_COMMAND = _init.REINSTALL_COMMAND + library_filename, machine_isas, source_sha256 = _init.library_filename, _init.machine_isas, _init.source_sha256 + normalized_machine = _init.normalized_machine + +BASE_FLAGS = ("-O3", "-std=c11", "-fPIC", "-shared", "-ffp-contract=off", "-fno-fast-math", "-fvisibility=hidden") +ISA_FLAGS = {"sse2": (), "avx2": ("-mavx2",), "neon": (), "generic": ("-DFMSW_GENERIC",)} +#: Target-architecture baseline, placed before the ISA flag (see the module docstring). +MACHINE_FLAGS = {"x86_64": ("-march=x86-64", "-mtune=generic")} +_ISA_MACHINE = {"sse2": "x86_64", "avx2": "x86_64", "neon": "arm64"} + + +class NativeBuildError(RuntimeError): + """The C compiler is missing or failed.""" + + +def find_compiler(cc: Optional[str] = None) -> str: + """``cc`` argument, else ``$CC``, else the first of cc / clang / gcc on PATH.""" + cand = cc or os.environ.get("CC") + if cand: + exe = shlex.split(cand)[0] + if shutil.which(exe) is None and not Path(exe).is_file(): + raise NativeBuildError(f"C compiler {cand!r} not found") + return cand + for name in ("cc", "clang", "gcc"): + if shutil.which(name): + return name + raise NativeBuildError("no C compiler found (tried $CC, cc, clang, gcc)") + + +def machine_flags(isa: str, machine: Optional[str] = None) -> List[str]: + """Baseline flags of the library's target architecture: the ISA's own architecture for sse2 / avx2 / neon, else + ``machine`` (default: this machine; the packaging hook passes the wheel's target).""" + target = _ISA_MACHINE.get(isa) or normalized_machine(machine) + return list(MACHINE_FLAGS.get(target, ())) + + +def compile_command(isa: str, output: Path, *, cc: Optional[str] = None, source_dir: Optional[Path] = None, + extra_flags: Sequence[str] = (), machine: Optional[str] = None) -> List[str]: + """The exact compiler command line for one ISA library (the source sha256 is injected as a macro). ``machine`` + is the target architecture of a generic library (default: this machine).""" + if isa not in ISA_FLAGS: + raise ValueError(f"unknown fmsw ISA {isa!r}; expected one of {', '.join(ISAS)}") + src_dir = Path(source_dir) if source_dir is not None else SOURCE_DIR + sha = source_sha256(src_dir) + if sha is None: + raise NativeBuildError(f"kernel source not found in {src_dir}") + return [*shlex.split(find_compiler(cc)), *BASE_FLAGS, *machine_flags(isa, machine), *ISA_FLAGS[isa], + *extra_flags, *link_flags(isa), + f'-DFMSW_SOURCE_SHA256="{sha}"', "-o", str(output), str(src_dir / "fmsw.c"), "-lm"] + + +def link_flags(isa: str) -> List[str]: + """Linker-only flags. On macOS the library's install name is ``@rpath/`` rather than the + temporary output path, so the binary never records a build directory (reproducible builds; delocate).""" + if sys.platform == "darwin": + return [f"-Wl,-install_name,@rpath/{library_filename(isa)}"] + return [] + + +def compile_library(isa: str, out_dir: Path, *, cc: Optional[str] = None, source_dir: Optional[Path] = None, + extra_flags: Sequence[str] = (), machine: Optional[str] = None, quiet: bool = False) -> Path: + """Compile the ``isa`` library into ``out_dir`` and return its path. + + The library is written to a temporary file in ``out_dir`` and moved into place atomically, so a process that has + the previous library loaded keeps a valid mapping and a concurrent loader never sees a partial file.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + final = out_dir / library_filename(isa) + fd, tmp = tempfile.mkstemp(prefix=f".{final.name}.", suffix=".tmp", dir=out_dir) + os.close(fd) + try: + cmd = compile_command(isa, Path(tmp), cc=cc, source_dir=source_dir, extra_flags=extra_flags, + machine=machine) + if not quiet: + print(" ".join(shlex.quote(c) for c in cmd), flush=True) + try: + res = subprocess.run(cmd, capture_output=True, text=True) + except OSError as e: + raise NativeBuildError(f"cannot run the C compiler: {e}") from e + if res.returncode != 0: + raise NativeBuildError(f"compiling {final.name} failed (exit {res.returncode}):\n" + f"{res.stdout}{res.stderr}") + os.chmod(tmp, 0o755) + os.replace(tmp, final) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + return final + + +def library_is_current(path: Path, source_dir: Optional[Path] = None) -> bool: + """True when ``path`` exists and embeds the tree's source sha256 (checked on the file bytes, without loading).""" + sha = source_sha256(source_dir) + p = Path(path) + return sha is not None and p.is_file() and sha.encode("ascii") in p.read_bytes() + + +def build(isas: Optional[Iterable[str]] = None, out_dir: Optional[Path] = None, *, cc: Optional[str] = None, + extra_flags: Sequence[str] = (), only_if_stale: bool = False, quiet: bool = False) -> List[Path]: + """Build the libraries of ``isas`` (default: every ISA of this machine) into ``out_dir`` (default: in place). + With ``only_if_stale`` a library that already embeds the tree's source sha256 is kept.""" + out = Path(out_dir) if out_dir is not None else NATIVE_DIR + paths = [] + for isa in (tuple(isas) if isas is not None else machine_isas()): + target = out / library_filename(isa) + if only_if_stale and library_is_current(target): + paths.append(target) + continue + paths.append(compile_library(isa, out, cc=cc, extra_flags=extra_flags, quiet=quiet)) + return paths + + +def main(argv: Optional[Sequence[str]] = None) -> int: + ap = argparse.ArgumentParser(prog="python -m foldmatch._native.build", description=__doc__.split("\n\n")[0]) + ap.add_argument("--isa", action="append", choices=ISAS, + help="ISA to build (repeatable; default: every ISA of this machine: " + f"{', '.join(machine_isas())})") + ap.add_argument("--out", type=Path, default=None, help=f"output directory (default: {NATIVE_DIR})") + ap.add_argument("--cc", default=None, help="C compiler (default: $CC, else cc / clang / gcc)") + ap.add_argument("--if-stale", action="store_true", help="skip libraries that already match the source") + ap.add_argument("--quiet", action="store_true") + a = ap.parse_args(argv) + try: + paths = build(a.isa, a.out, cc=a.cc, only_if_stale=a.if_stale, quiet=a.quiet) + except NativeBuildError as e: + print(f"error: {e}", file=sys.stderr) + return 1 + print(f"fmsw source sha256 {source_sha256()}") + for p in paths: + print(f"built {p}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/foldmatch/_native/fmsw/fmsw.c b/src/foldmatch/_native/fmsw/fmsw.c new file mode 100644 index 0000000..3d040aa --- /dev/null +++ b/src/foldmatch/_native/fmsw/fmsw.c @@ -0,0 +1,872 @@ +/* + * fmsw.c -- FoldMatch native Smith-Waterman kernel (ABI 1, DP id fmsw/1). API, semantics and build: fmsw.h. + * + * Provenance. Derived from the research kernel embcomp/throughput/build/swkern.c (md5 438d5a9811153f61e66cee845f1ed037) + * and its R = 2 driver embcomp/evalue/score/swmulti.c (md5 b12fbc2c03638659440b12282a4d8670), through the P0.4 spike + * rcsb-embedding-search src/scripts/esm_codes/spike/fmsw_spike.c. Kept from the research kernel: the rowcol + * coefficient two-pass structure, the anti-diagonal strip DP (interleave_strip, ad_core, ad_score, ad_tb), traceback() + * and the workspace sizing. Removed: the research-only sw_tb_f64_exact, sw_tb_f32, sw_score_f32, rowcol_z_fast_f32, + * rowcol_z_f32 and swz_batch_f32, and the spike's fixed-order production similarity and block drivers. + * Changes (PLAN section 7, P3.1): + * 1. rowcol reductions use a fixed 4-lane order on every ISA (AVX2 builds use the same 4-lane reductions); + * 2. query_prep in C in a fixed order (S5); the similarity is the caller's per-pair BLAS GEMM (S6, D2) into + * per-pair contiguous matrices at 64-byte-aligned offsets, read forward and through a row-reversed view; + * fmsw_sim_ref is the fixed-order test reference only; + * 3. S7 degenerate rule (std < 1e-6 -> the row / column contributes exactly 0); + * 4. generic vector-extension path (-DFMSW_GENERIC, and the #else of the ISA selection), vmax = a > b ? a : b; + * 5. int64 pair offsets, int32 lengths, size_t workspace sizes; + * 6. every entry point returns a status (0 ok, 1 workspace too small, 2 bad arguments); + * 7. ABI and provenance exports (the source sha256 is injected by the build); + * 8. (P7.2) the vector arrays of the workspace (cb, cd, Sv, P) start on 64-byte boundaries: a layout change only, + * the values and the DP golden hash are unchanged. + * -0 results are canonicalised to +0 (S8). + */ +#define _POSIX_C_SOURCE 200112L /* posix_memalign under -std=c11 (swi_alloc) */ + +#include "fmsw.h" + +#include +#include +#include +#include +#include + +#define F_HY 1 /* H[i,j] = Y[i,j] */ +#define F_VX 2 /* V[i,j] = X[i,j] */ +#define F_XE 4 /* X[i,j] extends X[i-1,j] */ +#define F_YE 8 /* Y[i,j] extends Y[i,j-1] */ +#define F_ST 16 /* V[i,j] <= 0: start here */ +#define WS 16 /* rows per anti-diagonal strip */ + +#ifndef FMSW_SOURCE_SHA256 +#define FMSW_SOURCE_SHA256 "unknown" +#endif + +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ < 12) +#error "fmsw needs GCC >= 12 or Clang >= 14 (__builtin_shufflevector on vector extensions)" +#endif + +/* ------------------------------------------------------------------------------------------------ */ +/* ISA selection (the DP vector width; the rowcol reductions always use 4-lane f4 vectors) */ +/* ------------------------------------------------------------------------------------------------ */ +#if !defined(FMSW_GENERIC) && defined(__ARM_NEON) +#include +#define VL 4 +#define ISA_NAME "neon" +typedef float vf __attribute__((vector_size(16))); +typedef int32_t vi __attribute__((vector_size(16))); +typedef uint8_t vb __attribute__((vector_size(4))); +static inline vf vmax(vf a, vf b) { return (vf)vmaxq_f32((float32x4_t)a, (float32x4_t)b); } +#define SHIFTIN(p, c) __builtin_shufflevector((p), (c), 3, 4, 5, 6) /* (p3, c0, c1, c2) */ +#define LANES {0, 1, 2, 3} +#elif !defined(FMSW_GENERIC) && defined(__AVX2__) +#include +#define VL 8 +#define ISA_NAME "avx2" +typedef float vf __attribute__((vector_size(32))); +typedef int32_t vi __attribute__((vector_size(32))); +typedef uint8_t vb __attribute__((vector_size(8))); +static inline vf vmax(vf a, vf b) { return (vf)_mm256_max_ps((__m256)a, (__m256)b); } +#define SHIFTIN(p, c) __builtin_shufflevector((p), (c), 7, 8, 9, 10, 11, 12, 13, 14) +#define LANES {0, 1, 2, 3, 4, 5, 6, 7} +#elif !defined(FMSW_GENERIC) && (defined(__SSE2__) || defined(__x86_64__)) +#include +#define VL 4 +#define ISA_NAME "sse2" +typedef float vf __attribute__((vector_size(16))); +typedef int32_t vi __attribute__((vector_size(16))); +typedef uint8_t vb __attribute__((vector_size(4))); +static inline vf vmax(vf a, vf b) { return (vf)_mm_max_ps((__m128)a, (__m128)b); } +#define SHIFTIN(p, c) __builtin_shufflevector((p), (c), 3, 4, 5, 6) +#define LANES {0, 1, 2, 3} +#else /* -DFMSW_GENERIC, or no known vector ISA: vector extensions only */ +#define VL 4 +#define ISA_NAME "generic" +typedef float vf __attribute__((vector_size(16))); +typedef int32_t vi __attribute__((vector_size(16))); +typedef uint8_t vb __attribute__((vector_size(4))); +/* (a > b) ? a : b per lane: the operand order of _mm_max_ps(a, b) */ +static inline vf vmax(vf a, vf b) { vi m = (vi)(a > b); return (vf)(((vi)a & m) | ((vi)b & ~m)); } +#define SHIFTIN(p, c) __builtin_shufflevector((p), (c), 3, 4, 5, 6) +#define LANES {0, 1, 2, 3} +#endif +#define NVEC (WS / VL) + +typedef float f4 __attribute__((vector_size(16))); +static inline vf vload(const float *p) { vf v; memcpy(&v, p, sizeof(vf)); return v; } +static inline vf vbc(float x) { vf z = {0}; return z + x; } +static inline vi vsel_i(vi mask, vi a, vi b) { return (a & mask) | (b & ~mask); } +static inline f4 ld4(const float *p) { f4 v; memcpy(&v, p, 16); return v; } +static inline void st4(float *p, f4 v) { memcpy(p, &v, 16); } +static inline f4 bc4(float x) { f4 z = {0, 0, 0, 0}; return z + x; } +static inline float hsum4(f4 v) { float s = 0.0f; s += v[0]; s += v[1]; s += v[2]; s += v[3]; return s; } + +/* ------------------------------------------------------------------------------------------------ */ +/* provenance and dispatch */ +/* ------------------------------------------------------------------------------------------------ */ +FMSW_API int fmsw_abi_version(void) { return FMSW_ABI_VERSION; } +FMSW_API const char *fmsw_dp_id(void) { return FMSW_DP_ID; } +FMSW_API const char *fmsw_isa(void) { return ISA_NAME; } +FMSW_API const char *fmsw_source_sha256(void) { return FMSW_SOURCE_SHA256; } +FMSW_API const char *fmsw_compiler(void) +{ +#if defined(__clang__) + return "clang " __clang_version__; +#elif defined(__GNUC__) + return "gcc " __VERSION__; +#else + return "unknown"; +#endif +} + +#if defined(__x86_64__) || defined(__i386__) +#include +static unsigned int fmsw_xcr0_low(void) +{ + unsigned int eax, edx; + __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(0)); /* xgetbv(0) */ + (void)edx; + return eax; +} +FMSW_API int fmsw_cpu_has_avx2(void) +{ + unsigned int a, b, c, d; + if (!__get_cpuid(1, &a, &b, &c, &d)) return 0; + if (!((c >> 27) & 1u) || !((c >> 28) & 1u)) return 0; /* OSXSAVE, AVX */ + if ((fmsw_xcr0_low() & 6u) != 6u) return 0; /* OS saves XMM and YMM state */ + if (__get_cpuid_max(0, NULL) < 7u) return 0; + __cpuid_count(7, 0, a, b, c, d); + return (int)((b >> 5) & 1u); /* AVX2 */ +} +#else +FMSW_API int fmsw_cpu_has_avx2(void) { return 0; } +#endif + +/* ------------------------------------------------------------------------------------------------ */ +/* S5 query_prep */ +/* ------------------------------------------------------------------------------------------------ */ +FMSW_API int fmsw_query_prep(const float *q, int n, float *qp) +{ + if (n < 0 || n > FMSW_MAX_LEN || (n > 0 && (!q || !qp))) return FMSW_ERR_ARGS; + for (int i = 0; i < n; i++) { + const float *r = q + (size_t)i * 64; + float *o = qp + (size_t)i * 64; + double ss = 0.0; + for (int k = 0; k < 64; k++) { + const double x = (double)r[k]; + if (!isfinite(x)) return FMSW_ERR_ARGS; + ss += x * x; /* exact product (24 x 24 bits), one rounding per add, k ascending */ + } + const double nrm = sqrt(ss); + if (nrm < 1e-12) { memset(o, 0, 64 * sizeof(float)); continue; } + const double den = 8.0 * nrm; + for (int k = 0; k < 64; k++) o[k] = (float)((double)r[k] / den); + } + return FMSW_OK; +} + +/* ------------------------------------------------------------------------------------------------ */ +/* S6 test reference similarity: per cell, a float32 accumulator from +0 adds (bit ? qp_ik : -qp_ik) */ +/* for k = 0..63 in order. The loops run k outside and j inside so that independent cells can share */ +/* vector lanes; each cell's own sequence of additions is the fixed one above. */ +/* ------------------------------------------------------------------------------------------------ */ +FMSW_API int fmsw_sim_ref(const float *qp, int n, const uint8_t *codes, int m, float *sim) +{ + if (n < 0 || m < 0 || n > FMSW_MAX_LEN || m > FMSW_MAX_LEN) return FMSW_ERR_ARGS; + if (n == 0 || m == 0) return FMSW_OK; + if (!qp || !codes || !sim) return FMSW_ERR_ARGS; + for (int i = 0; i < n; i++) { + const float *qr = qp + (size_t)i * 64; + float *s = sim + (size_t)i * (size_t)m; + for (int j = 0; j < m; j++) s[j] = 0.0f; + for (int k = 0; k < 64; k++) { + const float pos = qr[k], neg = -qr[k]; + const int byte = k >> 3, shift = 7 - (k & 7); + for (int j = 0; j < m; j++) s[j] += ((codes[(size_t)j * 8 + byte] >> shift) & 1) ? pos : neg; + } + } + return FMSW_OK; +} + +/* ------------------------------------------------------------------------------------------------ */ +/* S7 rowcol z coefficients: research rowcol_coef_f32 with the 4-lane order on every ISA, int64 */ +/* strides (negative = reversed view) and the degenerate rule. */ +/* on return ra = a_i, rc = c_i, cb = b_j, cd = d_j with z_ij - offset = s_ij (a_i + b_j) - (c_i + d_j) */ +/* ------------------------------------------------------------------------------------------------ */ +#define DEGEN_SD 1e-6 +static void rowcol_coef4(const float *sim, int n, int m, int64_t ld, float offset, + float *ra, float *rc, float *cb, float *cd) +{ + /* pass 1: means (ra <- row mean, cb <- column sum); 4 rows per sweep */ + for (int j = 0; j < m; j++) cb[j] = 0.0f; + int i = 0; + for (; i + 4 <= n; i += 4) { + const float *s0 = sim + (ptrdiff_t)i * ld, *s1 = s0 + ld, *s2 = s1 + ld, *s3 = s2 + ld; + f4 a0 = {0}, a1 = {0}, a2 = {0}, a3 = {0}; + int j = 0; + for (; j + 4 <= m; j += 4) { + f4 v0 = ld4(s0 + j), v1 = ld4(s1 + j), v2 = ld4(s2 + j), v3 = ld4(s3 + j); + a0 += v0; a1 += v1; a2 += v2; a3 += v3; + st4(cb + j, ld4(cb + j) + ((v0 + v1) + (v2 + v3))); + } + float r0 = hsum4(a0), r1 = hsum4(a1), r2 = hsum4(a2), r3 = hsum4(a3); + for (; j < m; j++) { r0 += s0[j]; r1 += s1[j]; r2 += s2[j]; r3 += s3[j]; cb[j] += (s0[j] + s1[j]) + (s2[j] + s3[j]); } + ra[i] = (float)((double)r0 / m); ra[i + 1] = (float)((double)r1 / m); + ra[i + 2] = (float)((double)r2 / m); ra[i + 3] = (float)((double)r3 / m); + } + for (; i < n; i++) { + const float *s = sim + (ptrdiff_t)i * ld; + f4 acc = {0}; + int j = 0; + for (; j + 4 <= m; j += 4) { f4 v = ld4(s + j); acc += v; st4(cb + j, ld4(cb + j) + v); } + float r = hsum4(acc); + for (; j < m; j++) { r += s[j]; cb[j] += s[j]; } + ra[i] = (float)((double)r / m); + } + const float inv_n = (float)(1.0 / n); + for (int j = 0; j < m; j++) cb[j] *= inv_n; + /* pass 2: squared deviations (rc <- row sum, cd <- column sum) */ + for (int j = 0; j < m; j++) cd[j] = 0.0f; + i = 0; + for (; i + 4 <= n; i += 4) { + const float *s0 = sim + (ptrdiff_t)i * ld, *s1 = s0 + ld, *s2 = s1 + ld, *s3 = s2 + ld; + const f4 m0 = bc4(ra[i]), m1 = bc4(ra[i + 1]), m2 = bc4(ra[i + 2]), m3 = bc4(ra[i + 3]); + f4 a0 = {0}, a1 = {0}, a2 = {0}, a3 = {0}; + int j = 0; + for (; j + 4 <= m; j += 4) { + f4 v0 = ld4(s0 + j), v1 = ld4(s1 + j), v2 = ld4(s2 + j), v3 = ld4(s3 + j); + f4 d0 = v0 - m0, d1 = v1 - m1, d2 = v2 - m2, d3 = v3 - m3; + a0 += d0 * d0; a1 += d1 * d1; a2 += d2 * d2; a3 += d3 * d3; + f4 mc = ld4(cb + j); + f4 e0 = v0 - mc, e1 = v1 - mc, e2 = v2 - mc, e3 = v3 - mc; + st4(cd + j, ld4(cd + j) + ((e0 * e0 + e1 * e1) + (e2 * e2 + e3 * e3))); + } + float r0 = hsum4(a0), r1 = hsum4(a1), r2 = hsum4(a2), r3 = hsum4(a3); + for (; j < m; j++) { + float d0 = s0[j] - ra[i], d1 = s1[j] - ra[i + 1], d2 = s2[j] - ra[i + 2], d3 = s3[j] - ra[i + 3]; + r0 += d0 * d0; r1 += d1 * d1; r2 += d2 * d2; r3 += d3 * d3; + float e0 = s0[j] - cb[j], e1 = s1[j] - cb[j], e2 = s2[j] - cb[j], e3 = s3[j] - cb[j]; + cd[j] += (e0 * e0 + e1 * e1) + (e2 * e2 + e3 * e3); + } + rc[i] = r0; rc[i + 1] = r1; rc[i + 2] = r2; rc[i + 3] = r3; + } + for (; i < n; i++) { + const float *s = sim + (ptrdiff_t)i * ld; + const float mr = ra[i]; + const f4 vmr = bc4(mr); + f4 acc = {0}; + int j = 0; + for (; j + 4 <= m; j += 4) { + f4 v = ld4(s + j); + f4 dr = v - vmr, dc = v - ld4(cb + j); + acc += dr * dr; + st4(cd + j, ld4(cd + j) + dc * dc); + } + float r = hsum4(acc); + for (; j < m; j++) { float dr = s[j] - mr, dc = s[j] - cb[j]; r += dr * dr; cd[j] += dc * dc; } + rc[i] = r; + } + /* coefficients; S7: a degenerate row (column) contributes exactly 0 */ + for (int k = 0; k < n; k++) { + const double sd = sqrt((double)rc[k] / m); + if (sd < DEGEN_SD) { ra[k] = 0.0f; rc[k] = 0.0f; continue; } + const double a = 0.5 / (sd + 1e-8); + rc[k] = (float)(a * ra[k]); + ra[k] = (float)a; + } + for (int j = 0; j < m; j++) { + const double sd = sqrt((double)cd[j] / n); + if (sd < DEGEN_SD) { cb[j] = 0.0f; cd[j] = offset; continue; } + const double b = 0.5 / (sd + 1e-8); + cd[j] = (float)(b * cb[j] + offset); + cb[j] = (float)b; + } +} + +/* ------------------------------------------------------------------------------------------------ */ +/* traceback over the strip-major flag layout (research, unchanged): writes the path in FORWARD order */ +/* ------------------------------------------------------------------------------------------------ */ +static inline uint8_t flag_strip(const uint8_t *fl, int m, int i, int j) +{ + if (i <= 0 || j <= 0) return F_ST; /* row 0 / column 0: H -> V -> start */ + const int T = m + WS; + const int s = (i - 1) / WS, r = (i - 1) % WS; + return fl[((size_t)s * T + (size_t)(j + r)) * WS + r]; +} + +static int traceback(const uint8_t *fl, int m, int bi, int bj, int32_t *path) +{ + int i = bi, j = bj, k = 0; + int state = 0; /* 0 H, 1 V, 2 M, 3 X, 4 Y */ + for (;;) { + uint8_t f = flag_strip(fl, m, i, j); + if (state == 0) { + state = (f & F_HY) ? 4 : 1; + } else if (state == 1) { + if (f & F_ST) break; + state = (f & F_VX) ? 3 : 2; + } else if (state == 2) { + path[2 * k] = i - 1; path[2 * k + 1] = j - 1; k++; + i--; j--; state = 0; + } else if (state == 3) { + path[2 * k] = i - 1; path[2 * k + 1] = -1; k++; + state = (f & F_XE) ? 3 : 0; + i--; + } else { + path[2 * k] = -1; path[2 * k + 1] = j - 1; k++; + state = (f & F_YE) ? 4 : 1; + j--; + } + } + for (int a = 0, b = k - 1; a < b; a++, b--) { /* built end -> start; reversed here, once */ + int32_t t0 = path[2 * a], t1 = path[2 * a + 1]; + path[2 * a] = path[2 * b]; path[2 * a + 1] = path[2 * b + 1]; + path[2 * b] = t0; path[2 * b + 1] = t1; + } + return k; +} + +/* ------------------------------------------------------------------------------------------------ */ +/* anti-diagonal strip DP (research; row stride int64, may be negative) */ +/* strip s covers rows i0 = 1 + 16 s .. i0 + 15; lane r (vector r / VL, element r % VL) of step t */ +/* holds cell (i0 + r, t - r). up = lane r-1 of step t-1 (lane 0: row i0-1, Hrow/Xrow), diag = */ +/* lane r-1 of step t-2 (= the previous step's 'up' vector), left = lane r of step t-1. */ +/* No masks: virtual cells left of column 1 reproduce the boundary (H = V = 0); virtual cells right */ +/* of column m / below row n score BIG_NEG and can never beat the strict running best. */ +/* Workspace (floats): Hrow, Xrow [m + WS + 1] each, Sv [(m + WS + 4) WS], P [WS PADW(m)]; Sv and P */ +/* start on 64-byte boundaries (align64f). */ +/* Flags (traceback): strip-major [nstrips][m + WS][WS] bytes. */ +/* ------------------------------------------------------------------------------------------------ */ +#define BIG_NEG (-1e30f) +#define PADW(m) ((m) + 48) +#define ALIGN_SLACK 16 /* floats: up to 15 skipped to start an array on a 64-byte boundary */ + +/* the first 64-byte boundary at or after p: the vector arrays of the workspace (Sv, P, cb, cd) start there, whatever + n, m and the caller's buffer address, so their 16- and 32-byte vectors never straddle a cache line (layout only: + no value depends on where the workspace arrays are) */ +static inline float *align64f(float *p) { return (float *)(((uintptr_t)p + 63u) & ~(uintptr_t)63u); } + +static size_t ad_ws_floats(int64_t m) +{ + const size_t mm = (size_t)m; + return 2 * (mm + WS + 1) + (mm + WS + 4) * WS + (size_t)WS * (mm + 48) + 64 + 2 * ALIGN_SLACK; +} + +FMSW_API size_t fmsw_ws_floats(int n, int64_t mmax) +{ + if (n < 0) n = 0; + if (mmax < 0) mmax = 0; + return 2 * ((size_t)n + (size_t)mmax) + 3 * ALIGN_SLACK + ad_ws_floats(mmax); +} + +/* the pair drivers' workspace: ra, rc [n] at ws, then cb, cd [mmax] and the DP workspace, each from a 64-byte + boundary (fmsw_ws_floats counts the slack) */ +static void ws_layout(float *ws, int n, int64_t mmax, float **ra, float **rc, float **cb, float **cd, float **dws) +{ + *ra = ws; + *rc = ws + n; + *cb = align64f(ws + 2 * (size_t)n); + *cd = align64f(*cb + (size_t)mmax); + *dws = align64f(*cd + (size_t)mmax); +} + +FMSW_API size_t fmsw_flag_bytes(int n, int64_t mmax) +{ + if (n < 0) n = 0; + if (mmax < 0) mmax = 0; + return (size_t)((n + WS - 1) / WS) * ((size_t)mmax + WS) * WS; +} + +/* Sv[t WS + r] = score of cell (i0 + r, t - r), t = 0 .. T + 3 (T = m + WS): 0 for columns <= 0, BIG_NEG for + columns > m or rows > n. Rows are staged with the z transform into padded buffers, then 4 rows x 4 steps are moved + per 4x4 register transpose: row r's values for steps t0..t0+3 are the contiguous P[r][15 + t0 - r .. 18 + t0 - r]. */ +static void interleave_strip(const float *S, int n, int m, int64_t ld, int i0, + const float *ra, const float *rc, const float *cb, const float *cd, + float *P, float *Sv) +{ + const int PW = PADW(m); + const int T = m + WS; + const f4 z4 = {0, 0, 0, 0}; + for (int r = 0; r < WS; r++) { + float *pr = P + (size_t)r * PW; + const int i = i0 + r; + st4(pr, z4); st4(pr + 4, z4); st4(pr + 8, z4); st4(pr + 12, z4); + float *d = pr + 16; + int c = 0; + if (i <= n) { + const float *s = S + (ptrdiff_t)(i - 1) * ld; + const float a = ra[i - 1], cc = rc[i - 1]; + const f4 a4 = {a, a, a, a}, c4 = {cc, cc, cc, cc}; + for (; c + 4 <= m; c += 4) st4(d + c, ld4(s + c) * (a4 + ld4(cb + c)) - (c4 + ld4(cd + c))); + for (; c < m; c++) d[c] = s[c] * (a + cb[c]) - (cc + cd[c]); + } + for (; c < PW - 16; c++) d[c] = BIG_NEG; + } + for (int g = 0; g < WS / 4; g++) { + const float *p0 = P + (size_t)(4 * g + 0) * PW + 15 - (4 * g + 0); + const float *p1 = P + (size_t)(4 * g + 1) * PW + 15 - (4 * g + 1); + const float *p2 = P + (size_t)(4 * g + 2) * PW + 15 - (4 * g + 2); + const float *p3 = P + (size_t)(4 * g + 3) * PW + 15 - (4 * g + 3); + float *out = Sv + 4 * g; + for (int t0 = 0; t0 < T; t0 += 4) { + const f4 R0 = ld4(p0 + t0), R1 = ld4(p1 + t0), R2 = ld4(p2 + t0), R3 = ld4(p3 + t0); + const f4 a0 = __builtin_shufflevector(R0, R1, 0, 4, 1, 5), a1 = __builtin_shufflevector(R0, R1, 2, 6, 3, 7); + const f4 a2 = __builtin_shufflevector(R2, R3, 0, 4, 1, 5), a3 = __builtin_shufflevector(R2, R3, 2, 6, 3, 7); + st4(out + (size_t)(t0 + 0) * WS, __builtin_shufflevector(a0, a2, 0, 1, 4, 5)); + st4(out + (size_t)(t0 + 1) * WS, __builtin_shufflevector(a0, a2, 2, 3, 6, 7)); + st4(out + (size_t)(t0 + 2) * WS, __builtin_shufflevector(a1, a3, 0, 1, 4, 5)); + st4(out + (size_t)(t0 + 3) * WS, __builtin_shufflevector(a1, a3, 2, 3, 6, 7)); + } + } +} + +static inline __attribute__((always_inline)) +float ad_core(const float *S, int n, int m, int64_t ld, + const float *ra, const float *rc, const float *cb, const float *cd, + float go, float ge, float *ws, uint8_t *fl, const int tb, int *bi_out, int *bj_out) +{ + const float NINF = -INFINITY; + const int T = m + WS; /* steps t = 0 .. T - 1 per strip (0 unused) */ + float *Hrow = ws, *Xrow = ws + (m + WS + 1); + float *Sv = align64f(ws + 2 * ((size_t)m + WS + 1)); + float *P = align64f(Sv + ((size_t)m + WS + 4) * WS); + for (int j = 0; j <= m + WS; j++) { Hrow[j] = 0.0f; Xrow[j] = NINF; } + float best = 0.0f; + int bi = 0, bj = 0; + const vf vgo = vbc(go), vge = vbc(ge), zero = {0}, ninf = vbc(NINF); + const vi lanes = LANES; + const int nstrip = (n + WS - 1) / WS; + for (int st = 0; st < nstrip; st++) { + const int i0 = 1 + st * WS; + interleave_strip(S, n, m, ld, i0, ra, rc, cb, cd, P, Sv); + vf H1[NVEC], X1[NVEC], V1[NVEC], Y1[NVEC], HU[NVEC], lbest[NVEC]; + vi lbj[NVEC], tj[NVEC]; + for (int k = 0; k < NVEC; k++) { + H1[k] = zero; X1[k] = ninf; V1[k] = zero; Y1[k] = ninf; HU[k] = zero; + lbest[k] = vbc(best); + lbj[k] = (vi){0} - 1; + tj[k] = ((vi){0} + (1 - VL * k)) - lanes; /* column of each lane at t = 1 */ + } + uint8_t *fs = tb ? fl + (size_t)st * T * WS : NULL; + for (int t = 1; t < T; t++) { + const float *sv = Sv + (size_t)t * WS; + const vf hu0 = vbc(Hrow[t]), xu0 = vbc(Xrow[t]); + for (int k = NVEC - 1; k >= 0; k--) { + const vf hup = SHIFTIN(k ? H1[k - 1] : hu0, H1[k]); + const vf xup = SHIFTIN(k ? X1[k - 1] : xu0, X1[k]); + const vf hdg = HU[k]; + HU[k] = hup; + const vf xo = hup - vgo, xe = xup - vge; + const vf x = vmax(xo, xe); + const vf mm = hdg + vload(sv + VL * k); + const vf v0 = vmax(x, mm); + const vf v = vmax(v0, zero); + const vf yo = V1[k] - vgo, ye = Y1[k] - vge; + const vf y = vmax(yo, ye); + const vf h = vmax(v, y); + if (tb) { + vi f = ((xe > xo) & F_XE) | ((x > mm) & F_VX) | ((v0 <= zero) & F_ST) | ((ye > yo) & F_YE) + | ((y > v) & F_HY); + vb fb = __builtin_convertvector(f, vb); + memcpy(fs + (size_t)t * WS + VL * k, &fb, VL); + } + const vi gt = h > lbest[k]; + lbest[k] = vmax(lbest[k], h); + lbj[k] = vsel_i(gt, tj[k], lbj[k]); + tj[k] += 1; + H1[k] = h; X1[k] = x; V1[k] = v; Y1[k] = y; + } + /* bottom row of the strip -> row above the next strip (index 0 is a dump slot) */ + int jl = t - (WS - 1); + jl = jl > 0 ? jl : 0; + Hrow[jl] = H1[NVEC - 1][VL - 1]; + Xrow[jl] = X1[NVEC - 1][VL - 1]; + } + for (int r = 0; r < WS && i0 + r <= n; r++) { /* rows in order; a later row needs a strictly larger H */ + const int k = r / VL, l = r % VL; + if (lbj[k][l] >= 1 && lbest[k][l] > best) { best = lbest[k][l]; bi = i0 + r; bj = lbj[k][l]; } + } + } + *bi_out = bi; *bj_out = bj; + return best + 0.0f; /* -0 -> +0 (S8) */ +} + +static float ad_score(const float *S, int n, int m, int64_t ld, const float *ra, const float *rc, const float *cb, + const float *cd, float go, float ge, float *ws, int *bi, int *bj) +{ + return ad_core(S, n, m, ld, ra, rc, cb, cd, go, ge, ws, NULL, 0, bi, bj); +} + +static float ad_tb(const float *S, int n, int m, int64_t ld, const float *ra, const float *rc, const float *cb, + const float *cd, float go, float ge, float *ws, uint8_t *fl, int *bi, int *bj) +{ + return ad_core(S, n, m, ld, ra, rc, cb, cd, go, ge, ws, fl, 1, bi, bj); +} + +/* ------------------------------------------------------------------------------------------------ */ +/* argument checks shared by the pair drivers */ +/* ------------------------------------------------------------------------------------------------ */ +static int check_pairs(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, float offset, + float go, float ge, int64_t *mmax_out) +{ + if (n < 1 || n > FMSW_MAX_LEN || K < 0) return FMSW_ERR_ARGS; + if (!isfinite(offset) || !isfinite(go) || !isfinite(ge) || !(go >= ge && ge >= 0.0f)) return FMSW_ERR_ARGS; + if (K > 0 && (!sim || !pair_off || !m)) return FMSW_ERR_ARGS; + int64_t mmax = 0; + for (int k = 0; k < K; k++) { + if (m[k] < 0 || m[k] > FMSW_MAX_LEN || pair_off[k] < 0) return FMSW_ERR_ARGS; + if (m[k] > 0 && ((uintptr_t)(sim + pair_off[k]) & 63u) != 0) return FMSW_ERR_ARGS; + if (m[k] > mmax) mmax = m[k]; + } + *mmax_out = mmax; + return FMSW_OK; +} + +/* ------------------------------------------------------------------------------------------------ */ +/* S9 score-only S_f and S_r; S_r reads the row-reversed view of the same per-pair matrix */ +/* ------------------------------------------------------------------------------------------------ */ +FMSW_API int fmsw_score_fwd_rev(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, + float offset, float go, float ge, float *s_fwd, float *s_rev, float *ws, + size_t ws_len) +{ + int64_t mmax = 0; + int st = check_pairs(sim, n, pair_off, m, K, offset, go, ge, &mmax); + if (st) return st; + if (K > 0 && (!s_fwd || !s_rev || !ws)) return FMSW_ERR_ARGS; + if (K > 0 && ws_len < fmsw_ws_floats(n, mmax)) return FMSW_ERR_WORKSPACE; + float *ra, *rc, *cb, *cd, *dws; + ws_layout(ws, n, mmax, &ra, &rc, &cb, &cd, &dws); + for (int k = 0; k < K; k++) { + const int mk = m[k]; + int bi, bj; + if (mk == 0) { s_fwd[k] = s_rev[k] = 0.0f; continue; } + const float *fwd = sim + pair_off[k]; + const float *rev = fwd + (ptrdiff_t)(n - 1) * mk; /* last row; stride -m */ + rowcol_coef4(fwd, n, mk, mk, offset, ra, rc, cb, cd); + s_fwd[k] = ad_score(fwd, n, mk, mk, ra, rc, cb, cd, go, ge, dws, &bi, &bj); + rowcol_coef4(rev, n, mk, -(int64_t)mk, offset, ra, rc, cb, cd); + s_rev[k] = ad_score(rev, n, mk, -(int64_t)mk, ra, rc, cb, cd, go, ge, dws, &bi, &bj); + } + return FMSW_OK; +} + +/* ------------------------------------------------------------------------------------------------ */ +/* forward DP, optional traceback (research swz_ad_batch_f32 with per-pair matrices) */ +/* ------------------------------------------------------------------------------------------------ */ +FMSW_API int fmsw_align(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, float offset, + float go, float ge, int traceback_on, float *scores, int32_t *ends, int32_t *path_buf, + size_t path_cap, int64_t *path_off, float *ws, size_t ws_len, uint8_t *flags, + size_t flags_len) +{ + int64_t mmax = 0; + int st = check_pairs(sim, n, pair_off, m, K, offset, go, ge, &mmax); + if (st) return st; + if (K > 0 && (!scores || !ends || !ws)) return FMSW_ERR_ARGS; + if (traceback_on && (!path_off || (K > 0 && (!path_buf || !flags)))) return FMSW_ERR_ARGS; + if (K > 0 && ws_len < fmsw_ws_floats(n, mmax)) return FMSW_ERR_WORKSPACE; + if (traceback_on && K > 0 && flags_len < fmsw_flag_bytes(n, mmax)) return FMSW_ERR_WORKSPACE; + float *ra, *rc, *cb, *cd, *dws; + ws_layout(ws, n, mmax, &ra, &rc, &cb, &cd, &dws); + int64_t po = 0; + if (path_off) path_off[0] = 0; + for (int k = 0; k < K; k++) { + const int mk = m[k]; + int bi = 0, bj = 0, plen = 0; + float sc = 0.0f; + if (mk > 0) { + const float *fwd = sim + pair_off[k]; + if (traceback_on && (size_t)po + (size_t)n + (size_t)mk > path_cap) return FMSW_ERR_WORKSPACE; + rowcol_coef4(fwd, n, mk, mk, offset, ra, rc, cb, cd); + if (traceback_on) { + sc = ad_tb(fwd, n, mk, mk, ra, rc, cb, cd, go, ge, dws, flags, &bi, &bj); + plen = traceback(flags, mk, bi, bj, path_buf + 2 * po); + } else { + sc = ad_score(fwd, n, mk, mk, ra, rc, cb, cd, go, ge, dws, &bi, &bj); + } + } + scores[k] = sc; + ends[2 * k] = bi; ends[2 * k + 1] = bj; + po += plen; + if (path_off) path_off[k + 1] = po; + } + return FMSW_OK; +} + +/* ================================================================================================================ */ +/* Integer local Smith-Waterman for the BLOSUM62 Stage 2 (DP id fmswi/1): output-equivalent to biotite 1.7.1 */ +/* align_optimal(local=True, max_number=1) with affine gaps (the same score, end cell and path; its own tables, not */ +/* biotite's). Model, layout and limits: fmsw.h. From a standalone prototype (Farrar striped query profile, lazy F, */ +/* 16-bit lanes with a 32-bit redo) validated against biotite; here with a position-specific profile (composition */ +/* bias), a batch entry point, the lazy-F clamp and the two-pass traceback. */ +/* ================================================================================================================ */ +#if !defined(FMSW_GENERIC) && defined(__ARM_NEON) +#define SWI_VBYTES 16 +typedef int16_t swi_v16 __attribute__((vector_size(SWI_VBYTES))); +typedef int32_t swi_v32 __attribute__((vector_size(SWI_VBYTES))); +static inline swi_v16 swi_max16(swi_v16 a, swi_v16 b) { return (swi_v16)vmaxq_s16((int16x8_t)a, (int16x8_t)b); } +static inline swi_v32 swi_max32(swi_v32 a, swi_v32 b) { return (swi_v32)vmaxq_s32((int32x4_t)a, (int32x4_t)b); } +static inline int swi_any16(swi_v16 m) { return vmaxvq_u16((uint16x8_t)m) != 0; } +static inline int swi_any32(swi_v32 m) { return vmaxvq_u32((uint32x4_t)m) != 0; } +#elif !defined(FMSW_GENERIC) && defined(__AVX2__) +#define SWI_VBYTES 32 +typedef int16_t swi_v16 __attribute__((vector_size(SWI_VBYTES))); +typedef int32_t swi_v32 __attribute__((vector_size(SWI_VBYTES))); +static inline swi_v16 swi_max16(swi_v16 a, swi_v16 b) { return (swi_v16)_mm256_max_epi16((__m256i)a, (__m256i)b); } +static inline swi_v32 swi_max32(swi_v32 a, swi_v32 b) { return (swi_v32)_mm256_max_epi32((__m256i)a, (__m256i)b); } +static inline int swi_any16(swi_v16 m) { return _mm256_movemask_epi8((__m256i)m) != 0; } +static inline int swi_any32(swi_v32 m) { return _mm256_movemask_epi8((__m256i)m) != 0; } +#elif !defined(FMSW_GENERIC) && (defined(__SSE2__) || defined(__x86_64__)) +#define SWI_VBYTES 16 +typedef int16_t swi_v16 __attribute__((vector_size(SWI_VBYTES))); +typedef int32_t swi_v32 __attribute__((vector_size(SWI_VBYTES))); +static inline swi_v16 swi_max16(swi_v16 a, swi_v16 b) { return (swi_v16)_mm_max_epi16((__m128i)a, (__m128i)b); } +static inline swi_v32 swi_max32(swi_v32 a, swi_v32 b) { swi_v32 m = (swi_v32)(a > b); return (a & m) | (b & ~m); } +static inline int swi_any16(swi_v16 m) { return _mm_movemask_epi8((__m128i)m) != 0; } +static inline int swi_any32(swi_v32 m) { return _mm_movemask_epi8((__m128i)m) != 0; } +#else +#define SWI_VBYTES 16 +typedef int16_t swi_v16 __attribute__((vector_size(SWI_VBYTES))); +typedef int32_t swi_v32 __attribute__((vector_size(SWI_VBYTES))); +static inline swi_v16 swi_max16(swi_v16 a, swi_v16 b) { swi_v16 m = (swi_v16)(a > b); return (a & m) | (b & ~m); } +static inline swi_v32 swi_max32(swi_v32 a, swi_v32 b) { swi_v32 m = (swi_v32)(a > b); return (a & m) | (b & ~m); } +static inline int swi_any16(swi_v16 m) { for (int l = 0; l < SWI_VBYTES / 2; l++) if (m[l]) return 1; return 0; } +static inline int swi_any32(swi_v32 m) { for (int l = 0; l < SWI_VBYTES / 4; l++) if (m[l]) return 1; return 0; } +#endif +#define SWI_L16 (SWI_VBYTES / 2) +#define SWI_L32 (SWI_VBYTES / 4) +typedef uint8_t swi_b16 __attribute__((vector_size(SWI_L16))); +typedef uint8_t swi_b32 __attribute__((vector_size(SWI_L32))); + +/* shift by one lane toward higher lanes, x into lane 0 */ +#if SWI_L16 == 16 +#define SWI_SH16(v, x) __builtin_shufflevector((swi_v16){0} + (int16_t)(x), (v), 0, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30) +#define SWI_SH32(v, x) __builtin_shufflevector((swi_v32){0} + (int32_t)(x), (v), 0, 8, 9, 10, 11, 12, 13, 14) +#else +#define SWI_SH16(v, x) __builtin_shufflevector((swi_v16){0} + (int16_t)(x), (v), 0, 8, 9, 10, 11, 12, 13, 14) +#define SWI_SH32(v, x) __builtin_shufflevector((swi_v32){0} + (int32_t)(x), (v), 0, 4, 5, 6) +#endif + +/* flag bits of one cell */ +#define SWI_SRC_G1 1 /* M's predecessor (diagonal) is in state G1 */ +#define SWI_SRC_G2 2 /* M's predecessor (diagonal) is in state G2 */ +#define SWI_DEAD 4 /* M <= 0 before the clip: a path never passes through this M */ +#define SWI_G1X 8 /* G1 extends G1[i, j-1] (else opens from M[i, j-1]) */ +#define SWI_G2X 16 /* G2 extends G2[i-1, j] (else opens from M[i-1, j]) */ + +#define SWI_T int16_t +#define SWI_V swi_v16 +#define SWI_L SWI_L16 +#define SWI_VB swi_b16 +#define SWI_TMIN INT16_MIN +#define SWI_MAX(a, b) swi_max16((a), (b)) +#define SWI_ANY(m) swi_any16((SWI_V)(m)) +#define SWI_SH1(v, x) SWI_SH16((v), (x)) +#define SWI_FN(x) x##_16 +#include "fmswi_body.h" +#undef SWI_T +#undef SWI_V +#undef SWI_L +#undef SWI_VB +#undef SWI_TMIN +#undef SWI_MAX +#undef SWI_ANY +#undef SWI_SH1 +#undef SWI_FN + +#define SWI_T int32_t +#define SWI_V swi_v32 +#define SWI_L SWI_L32 +#define SWI_VB swi_b32 +#define SWI_TMIN INT32_MIN +#define SWI_MAX(a, b) swi_max32((a), (b)) +#define SWI_ANY(m) swi_any32((SWI_V)(m)) +#define SWI_SH1(v, x) SWI_SH32((v), (x)) +#define SWI_FN(x) x##_32 +#include "fmswi_body.h" +#undef SWI_T +#undef SWI_V +#undef SWI_L +#undef SWI_VB +#undef SWI_TMIN +#undef SWI_MAX +#undef SWI_ANY +#undef SWI_SH1 +#undef SWI_FN + +FMSW_API const char *fmswi_dp_id(void) { return FMSWI_DP_ID; } + +/* scalar traceback over the striped flags (biotite rules); fills the alignment fields of o and, when ops is given, + the path as 'M' / 'D' / 'I' in forward order. Returns the alignment length. */ +static int swi_traceback(const uint8_t *fl, int seg, int lanes, const uint8_t *q, const uint8_t *t, int bi, int bj, + int32_t *o, char *ops) +{ + int i = bi, j = bj, st = 0, k = 0, nid = 0, gaps = 0, prev = 0, nd = 0, ni = 0; /* st: 0 M, 1 G1, 2 G2 */ + int si = bi, sj = bj; + while (i > 0 && j > 0) { + const uint8_t f = fl[((size_t)(j - 1) * (size_t)seg + (size_t)((i - 1) % seg)) * (size_t)lanes + + (size_t)((i - 1) / seg)]; + if (st == 0) { + if (f & SWI_DEAD) break; + if (ops) ops[k] = 'M'; + nid += q[i - 1] == t[j - 1]; + si = i; sj = j; + st = f & (SWI_SRC_G1 | SWI_SRC_G2); + i--; j--; + prev = 0; + } else if (st == 1) { /* G1: target residue against a gap in the query */ + if (ops) ops[k] = 'D'; + nd++; + if (prev != 1) gaps++; + prev = 1; + st = (f & SWI_G1X) ? 1 : 0; + j--; + } else { /* G2: query residue against a gap in the target */ + if (ops) ops[k] = 'I'; + ni++; + if (prev != 2) gaps++; + prev = 2; + st = (f & SWI_G2X) ? 2 : 0; + i--; + } + k++; + } + if (ops) + for (int a = 0, b = k - 1; a < b; a++, b--) { const char x = ops[a]; ops[a] = ops[b]; ops[b] = x; } + o[FMSWI_QSTART] = si; o[FMSWI_TSTART] = sj; + o[FMSWI_ALNLEN] = k; o[FMSWI_NIDENT] = nid; o[FMSWI_GAPOPEN] = gaps; o[FMSWI_NQGAP] = nd; o[FMSWI_NTGAP] = ni; + return k; +} + +/* 64-byte aligned, freed with free(). posix_memalign rather than C11 aligned_alloc, which macOS has only from 10.15 + (an x86_64 wheel may target older releases). */ +static void *swi_alloc(size_t bytes) +{ + void *p = NULL; + return posix_memalign(&p, 64, bytes ? bytes : 1) ? NULL : p; +} + +FMSW_API int fmswi_align(const uint8_t *q, const int32_t *qprof, int n, int A, const uint8_t *t, + const int64_t *t_off, const int32_t *m, int K, int gap_open, int gap_extend, int traceback_on, + int32_t *out, char *ops, size_t ops_cap, int64_t *ops_off) +{ + if (n < 0 || n > FMSW_MAX_LEN || A < 1 || A > 255 || K < 0 || traceback_on < 0 || traceback_on > 2) + return FMSW_ERR_ARGS; + if (gap_open < 1 || gap_open > FMSWI_MAX_GAP || gap_extend < 0 || gap_extend > FMSWI_MAX_GAP) + return FMSW_ERR_ARGS; + if (K > 0 && (!out || !m || !t_off)) return FMSW_ERR_ARGS; + if (n > 0 && (!q || !qprof)) return FMSW_ERR_ARGS; + int smax = 0, smin = 0; + for (int i = 0; i < n; i++) { + if (q[i] >= A) return FMSW_ERR_ARGS; + for (int a = 0; a < A; a++) { + const int32_t s = qprof[(size_t)i * (size_t)A + (size_t)a]; + if (s > FMSWI_MAX_SCORE || s < -FMSWI_MAX_SCORE) return FMSW_ERR_ARGS; + smax = s > smax ? s : smax; + smin = s < smin ? s : smin; + } + } + int64_t mmax = 0; + for (int k = 0; k < K; k++) { + if (m[k] < 0 || m[k] > FMSW_MAX_LEN || t_off[k] < 0) return FMSW_ERR_ARGS; + if (m[k] > 0 && !t) return FMSW_ERR_ARGS; + const uint8_t *tk = t + t_off[k]; + for (int j = 0; j < m[k]; j++) + if (tk[j] >= A) return FMSW_ERR_ARGS; + mmax = m[k] > mmax ? m[k] : mmax; + } + const int use16 = smin >= -16384 && smax <= 16383 && gap_open + gap_extend <= 16384; + const int seg16 = (n + SWI_L16 - 1) / SWI_L16, seg32 = (n + SWI_L32 - 1) / SWI_L32; + const size_t lanes_rows = (size_t)seg16 * SWI_L16 > (size_t)seg32 * SWI_L32 ? (size_t)seg16 * SWI_L16 + : (size_t)seg32 * SWI_L32; + const size_t segmax = (size_t)(seg16 > seg32 ? seg16 : seg32); + swi_v16 *P16 = NULL, *Pp16 = NULL; + swi_v32 *P32 = NULL, *Pp32 = NULL; + void *ws = NULL; + uint8_t *fl = NULL; + int status = FMSW_OK; + if (n > 0 && K > 0) { + ws = swi_alloc(4 * segmax * SWI_VBYTES); + fl = traceback_on ? swi_alloc(lanes_rows * (size_t)(mmax > 0 ? mmax : 1)) : NULL; + if (use16) { + P16 = swi_alloc((size_t)A * (size_t)seg16 * sizeof(swi_v16)); + if (P16) swi_profile_16(P16, qprof, n, A, seg16); + } + if (!ws || (traceback_on && !fl) || (use16 && !P16)) { status = FMSW_ERR_WORKSPACE; goto done; } + } + int64_t po = 0; + if (ops_off) ops_off[0] = 0; + for (int k = 0; k < K; k++) { + int32_t *o = out + (size_t)k * FMSWI_NOUT; + for (int f = 0; f < FMSWI_NOUT; f++) o[f] = 0; + const int mk = m[k]; + if (n > 0 && mk > 0) { + const uint8_t *tk = t + t_off[k]; + if (traceback_on && ops && (size_t)po + (size_t)n + (size_t)mk > ops_cap) { + status = FMSW_ERR_WORKSPACE; + goto done; + } + /* traceback 1: one pass with flags; 2: a score-only pass, then flags for rows 1..bi x columns 1..bj only + (their DP values depend on no other cell, so the second pass meets the same end cell first) */ + const int one = traceback_on == 1; + int of = 1, bi = 0, bj = 0, best = 0, wide = 0; + if (use16) + best = swi_sw_16(P16, seg16, tk, mk, (int16_t)-gap_open, (int16_t)-gap_extend, one, (swi_v16 *)ws, fl, + 32767 - smax, &of, &bi, &bj); + if (of) { + if (!P32) { + P32 = swi_alloc((size_t)A * (size_t)seg32 * sizeof(swi_v32)); + if (!P32) { status = FMSW_ERR_WORKSPACE; goto done; } + swi_profile_32(P32, qprof, n, A, seg32); + } + best = swi_sw_32(P32, seg32, tk, mk, -gap_open, -gap_extend, one, (swi_v32 *)ws, fl, INT32_MAX - smax, + &of, &bi, &bj); + if (of) { status = FMSW_ERR_ARGS; goto done; } /* a score beyond int32 */ + wide = 1; + o[FMSWI_WIDE] = 1; + } + o[FMSWI_SCORE] = best; + o[FMSWI_QEND] = bi; o[FMSWI_TEND] = bj; + if (best > 0 && traceback_on) { + int seg = wide ? seg32 : seg16; + const int lanes = wide ? SWI_L32 : SWI_L16; + if (!one) { + int b2, i2 = 0, j2 = 0, of2 = 0; + if (!wide) { + const swi_v16 *P = P16; + if (bi < n) { + if (!Pp16 && !(Pp16 = swi_alloc((size_t)A * (size_t)seg16 * sizeof(swi_v16)))) { + status = FMSW_ERR_WORKSPACE; + goto done; + } + seg = (bi + SWI_L16 - 1) / SWI_L16; + swi_profile_16(Pp16, qprof, bi, A, seg); + P = Pp16; + } + b2 = swi_sw_16(P, seg, tk, bj, (int16_t)-gap_open, (int16_t)-gap_extend, 1, (swi_v16 *)ws, fl, + 32767 - smax, &of2, &i2, &j2); + } else { + const swi_v32 *P = P32; + if (bi < n) { + if (!Pp32 && !(Pp32 = swi_alloc((size_t)A * (size_t)seg32 * sizeof(swi_v32)))) { + status = FMSW_ERR_WORKSPACE; + goto done; + } + seg = (bi + SWI_L32 - 1) / SWI_L32; + swi_profile_32(Pp32, qprof, bi, A, seg); + P = Pp32; + } + b2 = swi_sw_32(P, seg, tk, bj, -gap_open, -gap_extend, 1, (swi_v32 *)ws, fl, INT32_MAX - smax, + &of2, &i2, &j2); + } + if (b2 != best || i2 != bi || j2 != bj || of2) { status = FMSWI_ERR_INTERNAL; goto done; } + } + po += swi_traceback(fl, seg, lanes, q, tk, bi, bj, o, ops ? ops + po : NULL); + } + } + if (ops_off) ops_off[k + 1] = po; + } +done: + free(P16); free(P32); free(Pp16); free(Pp32); free(ws); free(fl); + return status; +} diff --git a/src/foldmatch/_native/fmsw/fmsw.h b/src/foldmatch/_native/fmsw/fmsw.h new file mode 100644 index 0000000..e76d86b --- /dev/null +++ b/src/foldmatch/_native/fmsw/fmsw.h @@ -0,0 +1,195 @@ +/* + * fmsw.h -- FoldMatch native Smith-Waterman kernel for the ESM3 residue-code Stage-2 scoring (system esm-codes-v1). + * + * ABI 1, DP id "fmsw/1". Normative method: docs/esm-codes-method.md (S5 query prep, S6 similarity, S7 rowcol z, + * S8 Smith-Waterman setting D, S9 forward / reversed-view scores, S16 tolerance contract). + * + * ------------------------------------------------------------------------------------------------------------------ + * What the kernel computes (per pair: query of n rows x target of m residues) + * query prep (S5) qp_i = fp32( fp64(q_i) / (8 sqrt(sum_{k=0..63} fp64(q_ik)^2)) ), k ascending in float64; + * qp_i = 0 when the norm is < 1e-12; NaN / Inf in q -> status 2. + * similarity (S6) NOT computed here in production: the caller passes per-pair fp32 matrices sim = qp @ D^T + * (one BLAS GEMM per pair, D = 2 bit - 1). fmsw_sim_ref() is the fixed-order test reference + * (float32 accumulator from +0, adding bit_jk ? qp_ik : -qp_ik for k = 0..63 ascending). + * rowcol z (S7) z_ij = 0.5 ((s_ij - mu_i) / (sd_i + 1e-8) + (s_ij - nu_j) / (tau_j + 1e-8)), population std, + * applied as the affine map z_ij - offset = s_ij (a_i + b_j) - (c_i + d_j). Row and column sums + * are float32 in a FIXED 4-lane order on every ISA (lane l accumulates columns j = l mod 4 of a + * row; lanes are then added 0 + 1 + 2 + 3; a scalar tail adds the last m mod 4 columns in order; + * column sums add 4 rows at a time as ((r0 + r1) + (r2 + r3)) then single rows); two passes + * (means, then squared deviations). Degenerate rule: a row (column) whose population std is + * < 1e-6 contributes exactly 0 (a_i = c_i = 0; b_j = 0, d_j = offset). + * DP (S8) local 3-state Gotoh on the cell score z_ij - offset (setting D: offset 1.0, go 5.0, ge 0.5): + * X[i,j] = max(H[i-1,j] - go, X[i-1,j] - ge) query residue vs gap (vertical) + * M[i,j] = H[i-1,j-1] + (z_{i-1,j-1} - offset) aligned pair + * V[i,j] = max(M, X, 0) (V <= 0 before the clip: start here) + * Y[i,j] = max(V[i,j-1] - go, Y[i,j-1] - ge) gap vs target residue (horizontal) + * H[i,j] = max(V, Y) + * X <-> Y adjacency is allowed; a gap of length L costs go + (L - 1) ge. + * Best cell: the FIRST cell in row-major order with the maximal H (strict '>' update). + * Traceback tie rules: H -> Y only if Y > V; V -> X only if X > M; X extends only if + * X[i-1,j] - ge > H[i-1,j] - go; Y extends only if Y[i,j-1] - ge > V[i,j-1] - go. + * An empty alignment (no H > 0) scores +0 with no path and end cell (0, 0). + * Scores are canonicalised: a -0 result is returned as +0. + * forward / reverse S_f = SW(z(sim)); S_r = SW(z(simr)) with simr[i, :] = sim[n-1-i, :] read through a + * (S9) row-reversed VIEW of the same per-pair matrix (base = last row, row stride -m); z is recomputed + * on the view in its own row order. No reversed copy is made. + * + * Implementation: anti-diagonal strips of 16 rows (4-lane vectors on SSE2 / NEON / generic, 8-lane on AVX2; the DP is + * element-wise, so the lane width does not change any value). Given the same similarity, every build (sse2, avx2, + * neon, generic; GCC or Clang) returns bit-identical scores, end cells and paths (one DP golden hash). + * + * ------------------------------------------------------------------------------------------------------------------ + * Similarity buffer layout (fmsw_score_fwd_rev, fmsw_align) + * sim is a float32 base pointer; pair k is the C-contiguous matrix [n, m[k]] (row stride m[k]) starting at the + * float offset pair_off[k] >= 0. Every pair matrix must start on a 64-byte boundary (status 2 otherwise): the S6 + * GEMM output layout is fixed, and the kernel refuses buffers that break it. m[k] == 0 gives an empty alignment. + * + * Workspace + * ws float32, at least fmsw_ws_floats(n, mmax) floats (mmax = max_k m[k]); no alignment requirement. + * flags uint8, at least fmsw_flag_bytes(n, mmax) bytes (traceback only). + * path int32 [2 * path_cap]: path_cap counts (i, j) entries; pair k needs at most n + m[k]. + * Undersized buffers return status 1 before anything is written for that pair. + * + * Status codes: 0 ok, 1 workspace / buffer too small, 2 bad arguments. + * + * ------------------------------------------------------------------------------------------------------------------ + * Floating-point semantics (documented per ISA; S5 guarantees finite inputs) + * vmax(a, b) sse2 / avx2: _mm_max_ps / _mm256_max_ps(a, b) = (a > b) ? a : b, so the SECOND operand is returned + * when either is NaN or when a and b are zeros of either sign. + * generic: (a > b) ? a : b per lane, identical to sse2 / avx2. + * neon: FMAX: NaN if either operand is NaN; max(+0, -0) = +0. + * With finite similarities and go > 0, ge > 0 no NaN can arise, and the builds differ at most in the sign of a zero + * intermediate, which never changes a comparison, a flag, a later sum or a returned score (-0 is canonicalised). + * NaN in sim is not detected (the caller's S6 GEMM of finite inputs cannot produce one); NaN / Inf in q is (S5). + * All builds use -ffp-contract=off -fno-fast-math: no FMA contraction, no reassociation. + * + * Build (see foldmatch/_native/build.py): cc -O3 -std=c11 -fPIC -shared -ffp-contract=off -fno-fast-math + * -fvisibility=hidden [-march=x86-64 -mtune=generic on x86_64] [-mavx2 | -DFMSW_GENERIC] + * -DFMSW_SOURCE_SHA256="\"\"" -o _fmsw_. fmsw.c -lm + */ +#ifndef FMSW_H +#define FMSW_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define FMSW_API __attribute__((visibility("default"))) + +#define FMSW_ABI_VERSION 1 +#define FMSW_DP_ID "fmsw/1" + +#define FMSW_OK 0 +#define FMSW_ERR_WORKSPACE 1 +#define FMSW_ERR_ARGS 2 + +/* n, m[k] limits (int arithmetic headroom). The esm-codes Stage 2 caps n m at 2^28 cells; the BLOSUM62 Stage 2 + (fmswi_align) has no cell cap: its flag buffer is n_pad x the longest target (review finding 6, left open). */ +#define FMSW_MAX_LEN (1 << 30) + +/* provenance and dispatch */ +FMSW_API int fmsw_abi_version(void); /* FMSW_ABI_VERSION */ +FMSW_API const char *fmsw_dp_id(void); /* FMSW_DP_ID */ +FMSW_API const char *fmsw_isa(void); /* "sse2" | "avx2" | "neon" | "generic" */ +FMSW_API const char *fmsw_source_sha256(void); /* injected at build time ("unknown" if not) */ +FMSW_API const char *fmsw_compiler(void); +FMSW_API int fmsw_cpu_has_avx2(void); /* x86: CPUID AVX2 + OS YMM state; 0 elsewhere */ + +/* S5: q [n, 64] float32 (exact fp32 of fp16) -> qp [n, 64]; status 2 on NaN / Inf (rows before it are written) */ +FMSW_API int fmsw_query_prep(const float *q, int n, float *qp); + +/* S6 test reference (NOT used in production): sim [n, m] contiguous, codes uint8 [m, 8] packbits MSB-first + (bit k of residue j = codes[8 j + k / 8] >> (7 - k % 8) & 1) */ +FMSW_API int fmsw_sim_ref(const float *qp, int n, const uint8_t *codes, int m, float *sim); + +/* S9 score-only S_f (forward) and S_r (row-reversed view) for K pairs */ +FMSW_API int fmsw_score_fwd_rev(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, + float offset, float go, float ge, float *s_fwd, float *s_rev, float *ws, + size_t ws_len); + +/* forward DP for K pairs: scores [K], ends [K, 2] (1-based best cell; (0, 0) = empty); traceback != 0 also writes + the paths: path_buf [2 * path_cap] int32 (query i, target j) 0-based, -1 = gap, FORWARD order (first column first), + pair k at entries path_off[k] .. path_off[k + 1] (path_off [K + 1] int64). With traceback == 0, path_buf, path_off + and flags may be NULL (path_off, when given, is filled with zeros). */ +FMSW_API int fmsw_align(const float *sim, int n, const int64_t *pair_off, const int32_t *m, int K, float offset, + float go, float ge, int traceback, float *scores, int32_t *ends, int32_t *path_buf, + size_t path_cap, int64_t *path_off, float *ws, size_t ws_len, uint8_t *flags, + size_t flags_len); + +FMSW_API size_t fmsw_ws_floats(int n, int64_t mmax); +FMSW_API size_t fmsw_flag_bytes(int n, int64_t mmax); + +/* + * ------------------------------------------------------------------------------------------------------------------ + * Integer local Smith-Waterman for the BLOSUM62 Stage 2 (DP id "fmswi/1"; same library and ABI, separate entry point) + * + * Output-equivalent to biotite 1.7.1 align_optimal(query, target, matrix, gap_penalty=(-open, -extend), local=True, + * max_number=1): the same score, end cell and path (so every column FoldMatch derives from the path). With + * s(i, j) = qprof[i - 1][t_{j-1}] (a position-specific query profile: the BLOSUM62 row of each query residue, plus its + * composition-bias offset when that correction is on), go = open, ge = extend, the kernel computes: + * M[i,j] = max(0, s(i, j) + max(M, G1, G2)[i-1, j-1]) aligned pair; M <= 0 before the clip is "dead" + * G1[i,j] = max(M[i, j-1] - go, G1[i, j-1] - ge) target residue against a gap in the query ('D') + * G2[i,j] = max(M[i-1, j] - go, G2[i-1, j] - ge) query residue against a gap in the target ('I') + * boundary row / column: M = 0, G1 = G2 = -go. Gaps open from M only: unlike Gotoh / parasail / SSW / MMseqs2 no + * insertion run is directly followed by a deletion run. A gap of length L costs go + (L - 1) ge. + * score = max M; empty alignment (all zero outputs) when it is 0. + * end cell = the FIRST cell in row-major (query-major) order with M == score (numpy.where order). + * traceback from the end cell in state M: M -> predecessor M if M >= G1 and M >= G2, else G1 if G1 >= G2, else G2 + * (diagonal first, then 'D', then 'I'); a G state extends only if G_prev - ge > M_prev - go (ties open, so gaps sit + * as early as possible); the path stops before a dead M. Paths start and end with an aligned pair. + * These are the kernel's own tables, not biotite's literal ones: biotite starts from M = -inf on the boundary (except + * M[0,0] = 0), G1 = 0 on row 0 and G2 = 0 on column 0 (-inf elsewhere), and leaves a G cell at -inf when its value is + * <= 0. The table values differ; the score, end cell and path do not. That equivalence is checked against biotite + * (tests/test_swint.py), not derived from its code. + * + * Arguments + * q uint8 [n] query codes (< A): used for the identity count (nident compares q and t codes). + * qprof int32 [n, A] row-major position-specific scores, |s| <= FMSWI_MAX_SCORE. + * t uint8 target codes (< A), pair k at t + t_off[k], m[k] residues (m[k] == 0: empty alignment). + * gap_open >= 1 (the end-cell rule relies on it), gap_extend >= 0, both <= FMSWI_MAX_GAP. + * traceback 0: score, QEND, TEND and WIDE only (the other fields 0); 1: every field, one pass that stores a flag byte + * per cell; 2: every field, a score-only pass and then a flag pass over rows 1..QEND x columns 1..TEND only + * (exact: those cells' DP values depend on no other cell, so the second pass meets the same end cell; on + * the Fig. 2 sample that rectangle is 35% of the DP cells, cell-weighted). 1 and 2 give identical outputs. + * out int32 [K, FMSWI_NOUT]. + * ops optional (traceback only): the paths as 'M' / 'D' / 'I' in forward order, pair k at + * ops[ops_off[k] .. ops_off[k + 1]); before pair k the kernel needs ops_off[k] + n + m[k] <= ops_cap. + * ops_off optional int64 [K + 1] (filled with the running offsets; all zero without ops). + * Implementation: Farrar striped profile with lazy F in 16-bit lanes (16 on AVX2, 8 on SSE2 / NEON / generic); a pair + * whose score reaches 32767 - max(s) (or a profile / gap cost outside +-16384) is computed in 32-bit lanes (out WIDE + * = 1). Values are exact integers on every ISA, so every build returns identical results. Workspace is allocated per + * call: the profiles (A x n_pad x 2 or 4 bytes), 4 n_pad vectors and, with traceback, one flag byte per cell of the + * longest target (n_pad x mmax bytes). Status: 0 ok, 1 allocation failure or ops buffer too small, 2 bad arguments + * (or a score beyond int32), 3 internal check failed (traceback 2: the second pass did not meet the first pass's end + * cell; never expected). + */ +#define FMSWI_DP_ID "fmswi/1" +#define FMSWI_MAX_SCORE (1 << 20) +#define FMSWI_MAX_GAP (1 << 20) +#define FMSWI_ERR_INTERNAL 3 +#define FMSWI_NOUT 11 +#define FMSWI_SCORE 0 /* raw score (>= 0) */ +#define FMSWI_QSTART 1 /* 1-based first aligned query residue (0 = empty alignment or traceback off) */ +#define FMSWI_QEND 2 /* 1-based end cell (query) */ +#define FMSWI_TSTART 3 +#define FMSWI_TEND 4 +#define FMSWI_ALNLEN 5 /* path columns, gaps included */ +#define FMSWI_NIDENT 6 /* aligned pairs with equal codes */ +#define FMSWI_GAPOPEN 7 /* maximal gap runs, both sides */ +#define FMSWI_NQGAP 8 /* 'D' columns: target residue against a gap in the query */ +#define FMSWI_NTGAP 9 /* 'I' columns: query residue against a gap in the target */ +#define FMSWI_WIDE 10 /* 1 when the pair was computed in 32-bit lanes */ + +FMSW_API const char *fmswi_dp_id(void); /* FMSWI_DP_ID */ +FMSW_API int fmswi_align(const uint8_t *q, const int32_t *qprof, int n, int A, const uint8_t *t, + const int64_t *t_off, const int32_t *m, int K, int gap_open, int gap_extend, int traceback, + int32_t *out, char *ops, size_t ops_cap, int64_t *ops_off); + +#ifdef __cplusplus +} +#endif + +#endif /* FMSW_H */ diff --git a/src/foldmatch/_native/fmsw/fmswi_body.h b/src/foldmatch/_native/fmsw/fmswi_body.h new file mode 100644 index 0000000..0036193 --- /dev/null +++ b/src/foldmatch/_native/fmsw/fmswi_body.h @@ -0,0 +1,136 @@ +/* + * fmswi_body.h -- one instance of the integer striped Smith-Waterman of fmsw.c (DP id fmswi/1, API and model in + * fmsw.h). Not a public header: fmsw.c includes it twice, for 16-bit and 32-bit lanes, after defining + * SWI_T lane type (int16_t | int32_t) SWI_V vector of SWI_T + * SWI_L lanes per vector SWI_VB uint8 vector of SWI_L lanes (one flag byte per lane) + * SWI_TMIN most negative SWI_T (profile padding) SWI_MAX(a, b), SWI_ANY(mask) + * SWI_SH1(v, x) lane l <- lane l - 1, lane 0 <- x SWI_FN(name) name with the instance suffix + * + * Farrar striped layout: query row i = l * seg + k sits in lane l of segment k (seg = ceil(n / SWI_L)); rows >= n are + * padding scored SWI_TMIN, so they never become positive and never feed a real row. + */ + +/* profile [A][seg]: lane l of segment k = qprof[i][a] for query row i = l * seg + k; padding rows = SWI_TMIN */ +static void SWI_FN(swi_profile)(SWI_V *P, const int32_t *qprof, int n, int A, int seg) +{ + for (int a = 0; a < A; a++) + for (int k = 0; k < seg; k++) { + SWI_V v; + for (int l = 0; l < SWI_L; l++) { + const int i = l * seg + k; + v[l] = i < n ? (SWI_T)qprof[(size_t)i * (size_t)A + (size_t)a] : (SWI_T)SWI_TMIN; + } + P[(size_t)a * (size_t)seg + (size_t)k] = v; + } +} + +/* + * One pair. Returns the best score; *overflow is set when it reached lim (the 16-bit pass is then redone in 32 bits: + * from the next column on a lane could wrap). With tb != 0 the flags of every cell are written to fl, column-major in + * the striped layout: fl[((j - 1) * seg + k) * SWI_L + l] for query row l * seg + k (bits: SWI_SRC_* = predecessor + * state of M, SWI_DEAD = M <= 0 before the clip, SWI_G1X / SWI_G2X = G1 / G2 extends). + * + * Bounds that keep every lane in range (fmswi_align checks them before choosing 16 bits): G1, G2 >= go always (an + * open from M >= 0), so g + ge >= go + ge > SWI_TMIN; M at column j <= best(j - 1) + smax < lim + smax; the lazy-F + * carry is clamped at go before each extension (a carried value below go can never beat a G2 >= go, so the clamp + * changes no result and keeps a long carry from wrapping). + */ +static int SWI_FN(swi_sw)(const SWI_V *P, int seg, const uint8_t *t, int m, SWI_T go, SWI_T ge, int tb, SWI_V *ws, + uint8_t *fl, int lim, int *overflow, int *bi_out, int *bj_out) +{ + SWI_V *H = ws, *G1 = ws + seg, *G2 = ws + 2 * (size_t)seg, *M = ws + 3 * (size_t)seg; + const SWI_V zero = {0}, vgo = zero + go, vge = zero + ge; + for (int k = 0; k < seg; k++) { H[k] = zero; G1[k] = vgo; G2[k] = vgo; M[k] = zero; } + int best = 0, bi = 0, bj = 0; + for (int j = 1; j <= m; j++) { + const SWI_V *vP = P + (size_t)t[j - 1] * (size_t)seg; + SWI_V hd = SWI_SH1(H[seg - 1], 0); + SWI_V colmax = zero; + SWI_V vF = vgo; + SWI_V fextl = zero; /* trace: 'extends' mask of the final vF (into the next row) */ + if (!tb) { + /* score only: G1[k] holds G1 of THIS column (computed while processing the previous one) */ + for (int k = 0; k < seg; k++) { + const SWI_V hl = H[k]; + const SWI_V mc = SWI_MAX(hd + vP[k], zero); + const SWI_V g1 = G1[k]; + const SWI_V mo = mc + vgo; + G1[k] = SWI_MAX(mo, g1 + vge); + H[k] = SWI_MAX(mc, SWI_MAX(g1, vF)); + G2[k] = vF; + colmax = SWI_MAX(colmax, mc); + vF = SWI_MAX(mo, vF + vge); + hd = hl; + } + } else { + uint8_t *fc = fl + (size_t)(j - 1) * (size_t)seg * SWI_L; + SWI_V md = SWI_SH1(M[seg - 1], 0), g1d = SWI_SH1(G1[seg - 1], go), g2d = SWI_SH1(G2[seg - 1], go); + SWI_V fext = zero; /* G2 'extends' mask of this segment (from the previous one) */ + for (int k = 0; k < seg; k++) { + const SWI_V hl = H[k], mp = M[k], g1p = G1[k], g2p = G2[k]; + const SWI_V mraw = hd + vP[k]; + const SWI_V mc = SWI_MAX(mraw, zero); + const SWI_V g1o = mp + vgo, g1e = g1p + vge; + const SWI_V g1 = SWI_MAX(g1o, g1e); + /* predecessor of M, biotite order M > G1 > G2 on ties (dead G <= 0 <= M never wins) */ + const SWI_V notm = (SWI_V)((g1d > md) | (g2d > md)); + const SWI_V srcg2 = notm & (SWI_V)(g2d > g1d); + const SWI_V srcg1 = notm & ~srcg2; + const SWI_V f = (srcg1 & SWI_SRC_G1) | (srcg2 & SWI_SRC_G2) | ((SWI_V)(mraw <= zero) & SWI_DEAD) + | ((SWI_V)(g1e > g1o) & SWI_G1X) | (fext & SWI_G2X); + const SWI_VB fb = __builtin_convertvector(f, SWI_VB); + memcpy(fc + (size_t)k * SWI_L, &fb, SWI_L); + M[k] = mc; G1[k] = g1; G2[k] = vF; + H[k] = SWI_MAX(mc, SWI_MAX(g1, vF)); + colmax = SWI_MAX(colmax, mc); + const SWI_V mo = mc + vgo, fe = vF + vge; + fext = (SWI_V)(fe > mo); + vF = SWI_MAX(mo, fe); + hd = hl; md = mp; g1d = g1p; g2d = g2p; + } + fextl = fext; + } + /* lazy F: carry G2 across the segment boundary while it improves any lane. The first carried vector mixes + 'open' (M of the row above + go) and 'extend' candidates (fx: the lanes that extend); every later + improvement is an extension. */ + vF = SWI_SH1(vF, go); + SWI_V fx = tb ? SWI_SH1(fextl, 0) : zero; + for (int k = 0; SWI_ANY(vF > G2[k]);) { + if (tb) { + const SWI_V up = (SWI_V)(vF > G2[k]); + uint8_t *fk = fl + ((size_t)(j - 1) * (size_t)seg + (size_t)k) * SWI_L; + SWI_VB fb; + memcpy(&fb, fk, SWI_L); + fb |= __builtin_convertvector(up & fx & SWI_G2X, SWI_VB); + memcpy(fk, &fb, SWI_L); + fx = ~zero; + } + G2[k] = SWI_MAX(G2[k], vF); + H[k] = SWI_MAX(H[k], vF); + vF = SWI_MAX(vF, vgo) + vge; + if (++k == seg) { k = 0; vF = SWI_SH1(vF, go); } + } + /* end cell: the first cell in row-major order with M == best. With gap_open >= 1 every G is below the best + M seen so far, so H == best <=> M == best in a column that reaches the best. */ + const SWI_V vbest = zero + (SWI_T)(best > 0 ? best : 1); + if (SWI_ANY(colmax >= vbest)) { + int cm = 0; + for (int l = 0; l < SWI_L; l++) cm = colmax[l] > cm ? colmax[l] : cm; + if (cm >= best) { + const SWI_V vc = zero + (SWI_T)cm; + int imin = INT32_MAX; + for (int k = 0; k < seg; k++) { + const SWI_V eq = (SWI_V)(H[k] == vc); + if (!SWI_ANY(eq)) continue; + for (int l = 0; l < SWI_L; l++) + if (eq[l] && l * seg + k < imin) imin = l * seg + k; + } + if (cm > best || imin + 1 < bi) { best = cm; bi = imin + 1; bj = j; } + } + if (best >= lim) break; /* 16 bits could wrap from the next column on: redo in 32 bits */ + } + } + *overflow = best >= lim; + *bi_out = bi; *bj_out = bj; + return best; +} diff --git a/src/foldmatch/cli/args_utils.py b/src/foldmatch/cli/args_utils.py index d74aa52..f0f5020 100644 --- a/src/foldmatch/cli/args_utils.py +++ b/src/foldmatch/cli/args_utils.py @@ -1,6 +1,8 @@ import logging import os +import sys import warnings +from pathlib import Path from foldmatch.types.api_types import LogLevel @@ -12,10 +14,15 @@ def arg_devices(devices): class RankFilter(logging.Filter): + """Sets ``record.rank``: the rank of torch.distributed's process group when one is initialised, else the + launcher's ``RANK``, ``GLOBAL_RANK``, ``SLURM_PROCID`` or ``LOCAL_RANK``, else 0. torch.distributed is asked only + when it is already imported: without it there is no process group, and importing it here would import torch at a + command's first log record.""" + def filter(self, record): + dist = sys.modules.get("torch.distributed") try: - import torch.distributed as dist - if dist.is_available() and dist.is_initialized(): + if dist is not None and dist.is_available() and dist.is_initialized(): record.rank = str(dist.get_rank()) return True except Exception: @@ -30,26 +37,132 @@ def filter(self, record): return True -def set_log_level(level: LogLevel): +#: ``(level, has handlers)`` of the root logger when ``set_log_level(..., defer_lightning=True)`` first ran with +#: Lightning not imported: the state in which ``set_log_level`` imports Lightning when it is called again. +_ROOT_BEFORE_LIGHTNING = {} + + +def _import_lightning(): + """Import Lightning, as ``set_log_level`` does before it sets up logging. + + Lightning sets up its loggers when it is imported, from the state of the root logger: when it has no handler, + 'lightning.pytorch' and 'lightning.fabric' get handlers of their own and stop propagating, and other libraries it + imports log as the root logger then allows (matplotlib's DEBUG records, say). After a ``set_log_level(..., + defer_lightning=True)`` the import runs with the root logger's handlers and level as that call found them, so that + it sets up the same loggers as before the call; when Lightning was imported meanwhile with the handler that call + added, those two loggers get the handlers it would have given them.""" + before = _ROOT_BEFORE_LIGHTNING.pop("root", None) + if "lightning" in sys.modules: + if before is not None and not before[1]: + for name in ("lightning.fabric", "lightning.pytorch"): + logger = logging.getLogger(name) + if not logger.handlers and logger.propagate: + logger.addHandler(logging.StreamHandler()) + logger.propagate = False + return + if before is None: + import lightning # noqa: F401 + return + root = logging.getLogger() + handlers, level = list(root.handlers), root.level + try: + if not before[1]: + for handler in handlers: + root.removeHandler(handler) + root.setLevel(before[0]) + import lightning # noqa: F401 + finally: + for handler in [h for h in root.handlers if h not in handlers]: # as basicConfig(force=True) after it + root.removeHandler(handler) + handler.close() + for handler in handlers: + if handler not in root.handlers: + root.addHandler(handler) + root.setLevel(level) + + +def set_log_level(level: LogLevel, *, defer_lightning: bool = False): + """Set up logging for a command at ``level`` (``--log-level``): ``info``, ``warn`` or ``debug``, the values of + :class:`LogLevel`. Any other value is refused with ``ValueError``. + + Lightning is imported first (:func:`_import_lightning`), so that the levels set here for its loggers, and at + ``info`` the filter that ignores every warning, come after those its import sets. Its import also brings torch + and (through torchmetrics and transformers) scikit-learn, which sets ``KMP_DUPLICATE_LIB_OK``: the commands rely + on that state, as on torch being loaded before FAISS (on macOS a process that runs both, each with its own + libomp, aborts or crashes otherwise). + + ``defer_lightning=True`` sets up the same logging without importing Lightning or anything that imports torch, for + a command whose refusals should come within seconds of its launch (``build sequences-index`` and ``build + sequences-seal``; the import takes minutes from a cold file system on a compute node). Such a command calls + ``set_log_level(level)`` again before anything that imports torch, Lightning or FAISS (the seal imports none): + that call imports Lightning as this one would have, then sets up logging again, so that the process ends in the + state ``set_log_level(level)`` leaves. + Meanwhile the Hugging Face progress bars are left as they are, unless huggingface_hub is already imported (no + download comes before the second call).""" + try: + level = LogLevel(level) + except ValueError: + raise ValueError(f"unknown log level {level!r}: one of {', '.join(m.value for m in LogLevel)}") from None + if not defer_lightning: + _import_lightning() + elif "lightning" not in sys.modules: + root = logging.getLogger() + _ROOT_BEFORE_LIGHTNING.setdefault("root", (root.level, root.hasHandlers())) handler = logging.StreamHandler() handler.addFilter(RankFilter()) - - import lightning - if level == 'info': + if level is LogLevel.info: warnings.filterwarnings("ignore") logging.getLogger("lightning.pytorch").setLevel(logging.ERROR) logging.getLogger("lightning.fabric.utilities.distributed").setLevel(logging.ERROR) handler.setFormatter(logging.Formatter('[proc-%(rank)s] %(message)s')) logging.basicConfig(level=logging.INFO, handlers=[handler], force=True) - from huggingface_hub.utils import disable_progress_bars - disable_progress_bars() - elif level == 'warning': + if not defer_lightning or "huggingface_hub" in sys.modules: + from huggingface_hub.utils import disable_progress_bars + disable_progress_bars() + elif level is LogLevel.warn: handler.setFormatter(logging.Formatter( '> %(asctime)s - [proc-%(rank)s] - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' )) logging.basicConfig(level=logging.WARN, handlers=[handler], force=True) - elif level == 'debug': + else: handler.setFormatter(logging.Formatter( '> %(asctime)s - [proc-%(rank)s] - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' )) - logging.basicConfig(level=logging.DEBUG, handlers=[handler], force=True) \ No newline at end of file + logging.basicConfig(level=logging.DEBUG, handlers=[handler], force=True) + + +DOWNLOAD_MODELS_HELP = ( + "Download ESM3, the aggregator model and the esm-codes scoring bundles from Hugging Face into its cache and " + "verify their sha256, so that later runs work offline (HF_HUB_OFFLINE=1). Download only: no model is loaded and " + "no GPU is used.") + +DOWNLOAD_MODELS_SYSTEM_HELP = ( + "Fetch only this esm-codes scoring system's bundle (repeatable). Default: the bundles of every registered " + "system. ESM3 and the aggregator are always fetched.") + + +def download_models_command(systems=None): + """The ``download-models`` command of ``fm-search``, ``fm-embedding`` and ``inference``: ESM3 (the pinned + snapshot), the aggregator and the esm-codes bundles of every registered system (or of ``systems``), into the + Hugging Face cache, sha256-verified, so later runs work with ``HF_HUB_OFFLINE=1``. Download only: no model is + built, torch is not imported and no GPU is touched (a GPU login node is safe). Unknown system names are refused + before anything is downloaded.""" + import typer + from foldmatch.search import scoring_system + from foldmatch.utils import model + from foldmatch.utils.esm import loaders + known = scoring_system.known_names() + names = list(dict.fromkeys(systems)) if systems else list(known) + for name in names: + if name not in known: + raise typer.BadParameter(f"unknown esm-codes scoring system {name!r}; this FoldMatch knows: " + f"{', '.join(known) or 'none'}", param_hint="--system") + snapshot = loaders.download_esm3_snapshot() + typer.echo(f"ESM3: {loaders.ESM_REPO_ID}@{loaders.ESM_REVISION}, weights sha256 verified, in {snapshot}") + aggregator = model.download_aggregator_weights() + typer.echo(f"aggregator: {model.REPO_ID}@{model.REVISION}, sha256 verified, in {aggregator.parent}") + for name, files in model.download_esm_codes_bundles(names).items(): + s = scoring_system.get(name) + folder = Path(files[scoring_system.BUNDLE_JSON]).parent + typer.echo(f"esm-codes bundle {name}: {s.bundle_repo}@{s.bundle_revision}, {len(files)} files, sha256 " + f"verified, in {folder}") diff --git a/src/foldmatch/cli/embedding.py b/src/foldmatch/cli/embedding.py index 05eed25..42b69c5 100644 --- a/src/foldmatch/cli/embedding.py +++ b/src/foldmatch/cli/embedding.py @@ -20,7 +20,8 @@ from typing import Annotated, Optional, List from foldmatch import __version__ -from foldmatch.cli.args_utils import arg_devices, set_log_level +from foldmatch.cli.args_utils import DOWNLOAD_MODELS_HELP, DOWNLOAD_MODELS_SYSTEM_HELP, arg_devices, \ + download_models_command, set_log_level from foldmatch.types.api_types import ( Accelerator, LogLevel, @@ -77,7 +78,7 @@ def from_structures_residue( help='Output folder to store predictions.' )], output_format: Annotated[OutFormat, typer.Option( - help='Format of the output. Options: csv, pt, parquet, json.' + help='Format of the output. Options: csv, pt, parquet, json. Parquet holds one row per chain, with one list of D values per residue.' )] = OutFormat.csv, output_name: Annotated[str, typer.Option( help='File name for storing embeddings. Used when output-format is parquet or json.' @@ -324,7 +325,7 @@ def from_sequences_residue( help='Output folder to store predictions.' )], output_format: Annotated[OutFormat, typer.Option( - help='Format of the output. Options: csv, pt, parquet, json.' + help='Format of the output. Options: csv, pt, parquet, json. Parquet holds one row per sequence, with one list of D values per residue.' )] = OutFormat.csv, output_name: Annotated[str, typer.Option( help='File name for storing embeddings. Used when output-format is parquet or json.' @@ -358,7 +359,8 @@ def from_sequences_residue( set_log_level(log_level) predict( - fasta_file=fasta_file, + # Its pre-flight, which the dataset reads its records' offsets from + fasta_file=_refuse_fasta_conflicts(fasta_file, output_folder, output_format, min_res_n), min_res_n=min_res_n, batch_size=batch_size, num_workers=num_workers, @@ -427,7 +429,8 @@ def from_sequences_chain( from foldmatch.inference.chain_complete_inference import predict predict( - src_stream=fasta_file, + # Its pre-flight, which the dataset reads its records' offsets from + src_stream=_refuse_fasta_conflicts(fasta_file, output_folder, output_format, min_res_n), src_location=SrcLocation.file, src_from=SrcEsmFrom.fasta, min_res_n=min_res_n, @@ -444,18 +447,48 @@ def from_sequences_chain( ) +def _refuse_fasta_conflicts(fasta_file: Path, output_folder: Path, output_format: OutFormat, min_res_n: int): + """The FASTA checks of from-sequences, before ESM3 (P9.N), on every rank alike: a record id repeated among the + records kept (at least min_res_n residues), in every format, found by the residue pass's pre-flight + (preflight_fasta, its duplicates); with csv and pt, ids that cannot name a file . of the output folder, + or that the folder takes for one file (abc and ABC on APFS), and the FASTA file among the output files. The + pre-flight, whose offsets the dataset reads.""" + from foldmatch.errors import RefusalError + from foldmatch.utils.output_files import ITEM_FORMATS, check_outputs + from foldmatch.utils.sequence import preflight_fasta, scan_fasta + preflight = preflight_fasta(fasta_file, min_res_n) + if preflight.duplicates: + raise RefusalError( + f"{fasta_file}: {preflight.problems()[0]}. fm-embedding writes each record's embedding under its id, the " + f"first word of its header (the file .csv or .pt, or the id of its row in Parquet and JSON), so a " + f"repeated id keeps one of them: give each record its own id." + ) + if output_format.value in ITEM_FORMATS: + ids, labels = [], [] + for k, (_, name, length) in enumerate(scan_fasta(fasta_file), start=1): + if min_res_n <= 0 or length >= min_res_n: + ids.append(name) + labels.append(f"record {k}") + check_outputs(output_folder, output_format, None, names=ids, labels=labels, source=str(fasta_file), + inputs=[("the FASTA file", fasta_file)], what="FASTA id", given="those records") + return preflight + + # ===================================================================== # top-level # ===================================================================== @app.command( name="download-models", - help="Download ESM3 and aggregator models from Hugging Face." + help=DOWNLOAD_MODELS_HELP ) -def download_models(): - from foldmatch.utils.model import get_residue_model, get_aggregator_model - get_residue_model() - get_aggregator_model() +def download_models( + system: Annotated[Optional[List[str]], typer.Option( + "--system", + help=DOWNLOAD_MODELS_SYSTEM_HELP + )] = None +): + download_models_command(system) def version_callback(value: bool): diff --git a/src/foldmatch/cli/inference.py b/src/foldmatch/cli/inference.py index 9a2ae7d..6147d5b 100644 --- a/src/foldmatch/cli/inference.py +++ b/src/foldmatch/cli/inference.py @@ -1,17 +1,24 @@ +import io import os import sys from pathlib import Path import typer -from typing import Annotated, List +from typing import Annotated, List, Optional from foldmatch import __version__ -from foldmatch.cli.args_utils import arg_devices, set_log_level +from foldmatch.cli.args_utils import DOWNLOAD_MODELS_HELP, DOWNLOAD_MODELS_SYSTEM_HELP, arg_devices, \ + download_models_command, set_log_level from foldmatch.types.api_types import StructureFormat, Accelerator, SrcLocation, SrcProteinFrom, \ SrcAssemblyFrom, SrcTensorFrom, OutFormat, Strategy, LogLevel, ResEmbeddingFormat os.environ["TOKENIZERS_PARALLELISM"] = "false" +#: The help of an assembly list option. +_ASSEMBLY_LIST_HELP = ('CSV file of 4 columns, no header: Structure Name | Structure File Path or URL | Assembly Id | ' + 'Embedding Name, - when the 4th column is missing or empty. Each ' + 'row needs its own Embedding Name.') + app = typer.Typer( add_completion=False, pretty_exceptions_enable=False, @@ -21,7 +28,7 @@ @app.command( name="residue-embedding", - help="Calculate residue level embeddings of protein structures using ESM3. Predictions are stored as torch tensor files." + help="Calculate residue level embeddings of protein structures using ESM3. Predictions are stored in the format of output-format, torch tensor files by default." ) def residue_embedding( src_file: Annotated[Path, typer.Option( @@ -29,17 +36,17 @@ def residue_embedding( file_okay=True, dir_okay=False, resolve_path=True, - help='CSV file 4 columns: Structure Name | Structure File Path or URL (switch structure-location) | Chain Id (asym_i for cif files) | Output Embedding Name.' + help='CSV file of 4 columns, no header: Structure Name | Structure File Path or URL | Chain Id (label_asym_id in mmCIF and BinaryCIF files, the author chain ID in PDB files) | Embedding Name. The residue embedding of each row is stored under its Embedding Name, . when the 4th column is missing or empty.' )], output_folder: Annotated[Path, typer.Option( exists=True, file_okay=False, dir_okay=True, resolve_path=True, - help='Output path to store predictions. Embeddings are stored as torch tensor files.' + help='Output path to store predictions, in the format of output-format: one file per chain for csv and pt, one file per process for parquet and json.' )], output_format: Annotated[OutFormat, typer.Option( - help='Format of the output. Options: csv, pt, parquet, json.' + help='Format of the output. Options: csv, pt, parquet, json. Parquet holds one row per chain, with one list of D values per residue.' )] = OutFormat.pt, output_name: Annotated[str, typer.Option( help='File name for storing embeddings. Used when output-format is parquet or json.' @@ -47,9 +54,6 @@ def residue_embedding( structure_format: Annotated[StructureFormat, typer.Option( help='Structure file format.' )] = StructureFormat.mmcif, - min_res_n: Annotated[int, typer.Option( - help='When using all chains in a structure, consider only chains with at least residues.' - )] = 0, batch_size: Annotated[int, typer.Option( help='Number of samples processed together in one iteration.' )] = 1, @@ -69,7 +73,7 @@ def residue_embedding( help='Lightning strategy to control distribution of inference.' )] = Strategy.auto, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): from foldmatch.inference.esm_inference import predict @@ -80,7 +84,6 @@ def residue_embedding( src_location=SrcLocation.file, src_from=SrcProteinFrom.chain, structure_format=structure_format, - min_res_n=min_res_n, batch_size=batch_size, num_workers=num_workers, num_nodes=num_nodes, @@ -96,7 +99,7 @@ def residue_embedding( @app.command( name="structure-embedding", - help="Calculate single-chain protein embeddings from structural files. Predictions are stored in a single pandas DataFrame file." + help="Calculate single-chain protein embeddings from structural files. Predictions are stored as Parquet, one file per process: -.parquet." ) def structure_embedding( src_file: Annotated[Path, typer.Option( @@ -104,24 +107,21 @@ def structure_embedding( file_okay=True, dir_okay=False, resolve_path=True, - help='CSV file 4 columns: Structure Name | Structure File Path or URL (switch structure-location) | Chain Id (asym_i for cif files) | Output Embedding Name.' + help='CSV file of 4 columns, no header: Structure Name | Structure File Path or URL | Chain Id (label_asym_id in mmCIF and BinaryCIF files, the author chain ID in PDB files) | Embedding Name. The chain embedding of each row is stored under its Embedding Name, . when the 4th column is missing or empty.' )], output_folder: Annotated[Path, typer.Option( exists=True, file_okay=False, dir_okay=True, resolve_path=True, - help='Output path to store predictions. Embeddings are stored as a single DataFrame file (see out-df-name).' + help='Output path to store predictions: the Parquet file -.parquet of each process.' )], output_name: Annotated[str, typer.Option( - help='File name for storing embeddings as a single JSON file.' + help='File name for storing embeddings: -.parquet.' )] = 'inference', structure_format: Annotated[StructureFormat, typer.Option( help='Structure file format.' )] = StructureFormat.mmcif, - min_res_n: Annotated[int, typer.Option( - help='When using all chains in a structure, consider only chains with at least residues.' - )] = 0, batch_size: Annotated[int, typer.Option( help='Number of samples processed together in one iteration.' )] = 1, @@ -141,7 +141,7 @@ def structure_embedding( help='Lightning strategy to control distribution of inference.' )] = Strategy.auto, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): from foldmatch.inference.structure_inference import predict @@ -152,7 +152,6 @@ def structure_embedding( src_location=SrcLocation.file, src_from=SrcProteinFrom.chain, structure_format=structure_format, - min_res_n=min_res_n, batch_size=batch_size, num_workers=num_workers, num_nodes=num_nodes, @@ -167,7 +166,7 @@ def structure_embedding( @app.command( name="chain-embedding", - help="Calculate single-chain protein embeddings from residue level embeddings stored as torch tensor files. Predictions are stored as csv files." + help="Calculate single-chain protein embeddings from residue level embeddings stored as torch tensor or csv files (res-embedding-format). Predictions are stored in the format of output-format, csv files by default." ) def chain_embedding( src_file: Annotated[Path, typer.Option( @@ -175,14 +174,14 @@ def chain_embedding( file_okay=True, dir_okay=False, resolve_path=True, - help='CSV file 4 columns: Structure Name | Structure File Path or URL (switch structure-location) | Chain Id (asym_i for cif files) | Output Embedding Name.' + help='CSV file of 4 columns, no header, as for residue-embedding: Structure Name | Structure File Path or URL (not read) | Chain Id | Embedding Name, . when the 4th column is missing or empty. The residue embedding . is read from res-embedding-location, and the chain embedding is stored under the Embedding Name.' )], output_folder: Annotated[Path, typer.Option( exists=True, file_okay=False, dir_okay=True, resolve_path=True, - help='Output path to store predictions. Embeddings are stored as csv files.' + help='Output path to store predictions, in the format of output-format: one file per chain for csv and pt, one file per process for parquet and json.' )], res_embedding_location: Annotated[Path, typer.Option( exists=True, @@ -197,15 +196,9 @@ def chain_embedding( output_name: Annotated[str, typer.Option( help='File name for storing embeddings. Used when output-format is parquet or json.' )] = 'inference', - structure_format: Annotated[StructureFormat, typer.Option( - help='Structure file format.' - )] = StructureFormat.mmcif, res_embedding_format: Annotated[ResEmbeddingFormat, typer.Option( help='Format of the residue level embedding files. Options: pt (torch tensor files) or csv.' )] = ResEmbeddingFormat.pt, - min_res_n: Annotated[int, typer.Option( - help='When using all chains in a structure, consider only chains with at least residues.' - )] = 0, batch_size: Annotated[int, typer.Option( help='Number of samples processed together in one iteration.' )] = 1, @@ -225,20 +218,27 @@ def chain_embedding( help='Lightning strategy to control distribution of inference.' )] = Strategy.auto, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): + from foldmatch.dataset.utils import CHAIN_LIST_DEFAULT_NAME, read_chain_list, refuse_repeated_names, row_labels from foldmatch.inference.chain_inference import predict - from foldmatch.utils.data import adapt_csv_to_embedding_chain_stream + from foldmatch.utils.output_files import check_outputs set_log_level(log_level) + # Before any model: one name per row, one file per name, and no output over its residue embedding or the list + chains = read_chain_list(src_file) + refuse_repeated_names(chains, str(src_file), CHAIN_LIST_DEFAULT_NAME) + _refuse_chains_over_residues(output_folder, output_format, res_embedding_location, res_embedding_format) + check_outputs(output_folder, output_format, output_name, names=list(chains.item_name), labels=row_labels(chains), + source=str(src_file), inputs=[("the chain list", src_file)], stem_what="--output-name") + # The residue embedding of each row is the file of its embedding name, as residue-embedding writes it predict( - src_stream=adapt_csv_to_embedding_chain_stream(src_file, res_embedding_location, res_embedding_format=res_embedding_format), + src_stream=[(os.path.join(res_embedding_location, f"{name}.{res_embedding_format.value}"), name) + for name in chains.item_name], res_embedding_location=res_embedding_location, src_location=SrcLocation.stream, src_from=SrcTensorFrom.file, - structure_format=structure_format, - min_res_n=min_res_n, batch_size=batch_size, num_workers=num_workers, num_nodes=num_nodes, @@ -254,7 +254,7 @@ def chain_embedding( @app.command( name="assembly-embedding", - help="Calculate assembly embeddings from residue level embeddings stored as torch tensor files. Predictions are stored as csv files." + help="Calculate assembly embeddings from residue level embeddings stored as torch tensor or csv files (res-embedding-format). Predictions are stored in the format of output-format, csv files by default." ) def assembly_embedding( src_file: Annotated[Path, typer.Option( @@ -262,7 +262,7 @@ def assembly_embedding( file_okay=True, dir_okay=False, resolve_path=True, - help='CSV file 4 columns: Structure Name | Structure File Path or URL (switch structure-location) | Assembly Id | Output embedding name.' + help=_ASSEMBLY_LIST_HELP )], res_embedding_location: Annotated[Path, typer.Option( exists=True, @@ -276,7 +276,7 @@ def assembly_embedding( file_okay=False, dir_okay=True, resolve_path=True, - help='Output path to store predictions. Embeddings are stored as csv files.' + help='Output path to store predictions, in the format of output-format: one file per assembly for csv and pt, one file per process for parquet and json.' )], output_format: Annotated[OutFormat, typer.Option( help='Format of the output. Options: csv, pt, parquet, json.' @@ -315,7 +315,7 @@ def assembly_embedding( help='Lightning strategy to control distribution of inference.' )] = Strategy.auto, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): from foldmatch.inference.assembly_inferece import predict @@ -344,7 +344,7 @@ def assembly_embedding( @app.command( name="complete-embedding", - help="Calculate chain and assembly embeddings from structural files. Predictions are stored as csv files." + help="Calculate chain and assembly embeddings from structural files. Residue embeddings are stored as torch tensor files, chain and assembly embeddings in the format of output-format, csv files by default." ) def complete_embedding( src_chain_file: Annotated[Path, typer.Option( @@ -352,14 +352,13 @@ def complete_embedding( file_okay=True, dir_okay=False, resolve_path=True, - help='CSV file 4 columns: Structure Name | Structure File Path or URL (switch structure-location) | Chain Id (asym_i for cif files) | Output Embedding Name.' + help='CSV file of 4 columns, no header, as for residue-embedding: Structure Name | Structure File Path or URL | Chain Id (label_asym_id in mmCIF and BinaryCIF files, the author chain ID in PDB files) | Embedding Name, which must be . or left out: the assembly step reads the residue embeddings under that name.' )], src_assembly_file: Annotated[Path, typer.Option( exists=True, file_okay=True, dir_okay=False, - resolve_path=True, - help='CSV file 4 columns: Structure Name | Structure File Path or URL (switch structure-location) | Assembly Id | Output embedding name.' + help=_ASSEMBLY_LIST_HELP )], output_res_path: Annotated[Path, typer.Option( exists=True, @@ -373,14 +372,14 @@ def complete_embedding( file_okay=False, dir_okay=True, resolve_path=True, - help='Output path to store chain embeddings. Embeddings are stored as a single JSON file (see output_chain_name).' + help='Output path to store chain embeddings, in the format of output-format: one file per chain for csv and pt, one file per process for parquet and json (see output-chain-name).' )], output_assembly_path: Annotated[Path, typer.Option( exists=True, file_okay=False, dir_okay=True, resolve_path=True, - help='Output path to store assembly embeddings. Embeddings are stored as a single JSON file (see output_assembly_name).' + help='Output path to store assembly embeddings, in the format of output-format: one file per assembly for csv and pt, one file per process for parquet and json (see output-assembly-name).' )], output_format: Annotated[OutFormat, typer.Option( help='Format of the output. Options: csv, pt, parquet, json.' @@ -390,12 +389,12 @@ def complete_embedding( )] = 'chain-inference', output_assembly_name: Annotated[str, typer.Option( help='File name for storing assembly embeddings. Used when output-format is parquet or json.' - )] = 'chain-inference', + )] = 'assembly-inference', structure_format: Annotated[StructureFormat, typer.Option( help='Structure file format.' )] = StructureFormat.mmcif, min_res_n: Annotated[int, typer.Option( - help='When using all chains in a structure, consider only chains with at least residues.' + help='Consider only assembly chains with at least residues.' )] = 0, max_res_n: Annotated[int, typer.Option( help='Stop adding assembly chains when number of residues is greater than residues.' @@ -431,16 +430,30 @@ def complete_embedding( help='Lightning strategy to control distribution of inference.' )] = Strategy.auto, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): - + from foldmatch.dataset.utils import read_assembly_list, refuse_shared_names + # Before any model is built: the assembly step reads the residue embeddings as . + _refuse_other_embedding_names(src_chain_file) + # The assembly list, read as the assembly step reads it: a row of more than 4 fields, or a structure name given to + # two files, is refused here, before the residue step, rather than after the chain step. A list that can be read + # once (a pipe, a named pipe or <(...) on macOS; /dev/stdin or /dev/fd/N, whose open shares the descriptor's offset + # on macOS, even on a regular file; any path under /dev) is read once, here, and its text given to the assembly + # step, so that it is checked as a list in a regular file is + src_assembly_file = _resolved_list_path(src_assembly_file) + src_assembly_list = _read_once(src_assembly_file) + assemblies = read_assembly_list(src_assembly_list) + refuse_shared_names(assemblies, "stream_name", "stream") + # One name per row, one file per name, no step's file over another's or over an input, before any model + _refuse_conflicting_steps(src_chain_file, _read_once(src_assembly_list), assemblies, output_format=output_format, + folders=(output_res_path, output_chain_path, output_assembly_path), + names=(output_chain_name, output_assembly_name)) residue_embedding( src_file=src_chain_file, output_folder=output_res_path, output_format=OutFormat.pt, structure_format=structure_format, - min_res_n=min_res_n, batch_size=batch_size_res, num_workers=num_workers_res, num_nodes=num_nodes, @@ -455,8 +468,6 @@ def complete_embedding( output_format=output_format, output_name=output_chain_name, res_embedding_location=output_res_path, - structure_format=structure_format, - min_res_n=min_res_n, batch_size=batch_size_chain, num_workers=num_workers_chain, num_nodes=num_nodes, @@ -466,7 +477,7 @@ def complete_embedding( log_level=log_level ) assembly_embedding( - src_file=src_assembly_file, + src_file=_read_once(src_assembly_list), output_folder=output_assembly_path, output_format=output_format, output_name=output_assembly_name, @@ -483,14 +494,162 @@ def complete_embedding( log_level=log_level ) + +class _ListText(io.StringIO): + """The text of a list that can be read once, read into memory, under the name of the path it was read from (the + name that refusals give it, and by which it is an input of the run when it is a file).""" + + def __init__(self, text: str, name: str): + super().__init__(text) + self.name = name + + +def _resolved_list_path(src_file: Path) -> Path: + """A list path as Typer's resolve_path makes it (absolute, links resolved), but a path under /dev as given: Linux + resolves /dev/fd/N to the file the descriptor is open on (and refusals would name that file, not the path given) + and /dev/stdin from a pipe to a pipe:[N] that is no path.""" + return src_file if os.fspath(src_file).startswith("/dev/") else Path(os.path.realpath(src_file)) + + +def _read_once(src_file): + """A list path as it can be read more than once: a regular file not under /dev as it is; any other path (a pipe, + /dev/stdin, /dev/fd/N, a path under /dev) read once, into a _ListText; a _ListText, a new one of its text.""" + if isinstance(src_file, _ListText): + return _ListText(src_file.getvalue(), src_file.name) + if os.path.isfile(src_file) and not os.fspath(src_file).startswith("/dev/"): + return src_file + # As pandas reads a list file: UTF-8, line ends kept + with open(src_file, encoding="utf-8", newline="") as f: + return _ListText(f.read(), os.fspath(src_file)) + + +def _refuse_chains_over_residues(output_folder: Path, output_format: OutFormat, res_embedding_location: Path, + res_embedding_format: ResEmbeddingFormat): + """chain-embedding writes . and reads .: in + one folder and one format, each chain embedding would replace the residue embedding it is computed from, which is + refused (RefusalError), naming both options (P9.N D2).""" + from foldmatch.errors import RefusalError + from foldmatch.utils.output_files import same_folder + if output_format.value == res_embedding_format.value and same_folder(output_folder, res_embedding_location): + other = "csv" if output_format.value == "pt" else "pt" + raise RefusalError( + f"--output-folder and --res-embedding-location are one folder ({output_folder}), and --output-format is " + f"--res-embedding-format ({output_format.value}): each chain embedding ." + f"{output_format.value} would replace the residue embedding it is computed from. Give another " + f"--output-folder, or --output-format {other}, parquet or json." + ) + + +def _refuse_conflicting_steps(src_chain_file: Path, src_assembly_file, assemblies, *, output_format: OutFormat, + folders: tuple, names: tuple): + """complete-embedding's checks of its three steps' files, before its residue step (P9.N): each step's as the step + itself checks them (one name per row, one file per name, an assembly not named .<...> beside the + residue embeddings in their format), no output of any step over an input of the run (both lists and the structure + files of both), and no file written by two steps in one folder (residue /.pt, chain + /. or /-*, assembly /. or + /-*). ``src_assembly_file`` is the assembly list's path, or its text (_ListText) when it + can be read once, and ``assemblies`` its frame.""" + from foldmatch.dataset.utils import ASSEMBLY_LIST_DEFAULT_NAME, CHAIN_LIST_DEFAULT_NAME, list_source, \ + read_chain_list, refuse_repeated_names, row_labels + from foldmatch.errors import RefusalError + from foldmatch.inference.utils import list_inputs + from foldmatch.utils.output_files import check_outputs, file_id, probe_folding, refuse_assemblies_over_residues, \ + same_folder, shown + res_path, chain_path, assembly_path = folders + chain_name, assembly_name = names + foldings = {} + + def folding(folder): + key = file_id(folder) + if key not in foldings: + foldings[key] = probe_folding(folder) + return foldings[key] + + def of_list(frame, src, list_name): + # The inputs of one list, its structure files named with the list, as both lists' are checked together + return [(described if described == f"the {list_name}" else f"{described} of the {list_name}", path) + for described, path in list_inputs(frame, src, SrcLocation.file, list_name=list_name)] + + chains = read_chain_list(src_chain_file) + refuse_repeated_names(chains, str(src_chain_file), CHAIN_LIST_DEFAULT_NAME) + chain_names, chain_rows = list(chains.item_name), row_labels(chains) + assembly_source = list_source(src_assembly_file, SrcLocation.file, "assembly list") + refuse_repeated_names(assemblies, assembly_source, ASSEMBLY_LIST_DEFAULT_NAME) + assembly_names, assembly_rows = list(assemblies.item_name), row_labels(assemblies) + # Every input of the run, which no step's output may replace: a list read by one step can be written over by + # another's output before it is read (the assembly list by the residue or chain step), or after (the chain list) + inputs = of_list(chains, src_chain_file, "chain list") + of_list(assemblies, src_assembly_file, "assembly list") + # Chains that differ only as the folder does not tell apart (A and a of one structure): complete-embedding names + # them ., and its residue step writes pt files, so only another folder keeps them apart + named = "complete-embedding names each chain ." + steps = [ + ("--output-res-path", "residue", res_path, check_outputs( + res_path, OutFormat.pt, None, names=chain_names, labels=chain_rows, source=str(src_chain_file), + inputs=inputs, folding=folding(res_path), + folded_advice=f"give --output-res-path a folder of a file system that tells {{apart}} apart ({named}, " + f"and its residue step writes pt files)")), + ("--output-chain-path", "chain", chain_path, check_outputs( + chain_path, output_format, chain_name, names=chain_names, labels=chain_rows, source=str(src_chain_file), + inputs=inputs, folding=folding(chain_path), stem_what="--output-chain-name", + folded_advice=f"give --output-chain-path a folder of a file system that tells {{apart}} apart, or use " + f"--output-format parquet or json ({named})")), + ("--output-assembly-path", "assembly", assembly_path, check_outputs( + assembly_path, output_format, assembly_name, names=assembly_names, labels=assembly_rows, + source=assembly_source, folding=folding(assembly_path), stem_what="--output-assembly-name", + inputs=inputs)), + ] + refuse_assemblies_over_residues(assembly_names, assemblies.stream_name, out_folder=assembly_path, + out_format=output_format, res_folder=res_path, res_format=OutFormat.pt, + source=assembly_source, labels=assembly_rows) + for k, (option, step, folder, outputs) in enumerate(steps): + for other_option, other_step, other_folder, other_outputs in steps[k + 1:]: + if not same_folder(folder, other_folder): + continue + shared = outputs.shared_with(other_outputs) + if shared: + hint = ("an --output-format other than pt (residue embeddings are pt files)" if step == "residue" + else "other names (--output-chain-name and --output-assembly-name for parquet and json, " + "the lists' embedding names for csv and pt)") + raise RefusalError( + f"{option} and {other_option} are one folder ({folder}), in which complete-embedding's {step} and " + f"{other_step} embeddings would share {len(shared)} file(s): {shown(shared)}. Each step would " + f"replace the other's files: give the steps different folders, or {hint}." + ) + + +def _refuse_other_embedding_names(src_chain_file: Path): + """complete-embedding's assembly step reads each chain's residue embedding as ., as + assembly-embedding does: a chain list row with another embedding name is refused (RefusalError), before anything is + computed (such a list failed, or ran on the residue embeddings found under .: the row's + own, for whitespace at a line's start, an earlier run's, or another row's, as when two rows swap names).""" + from foldmatch.dataset.utils import read_chain_list + from foldmatch.errors import RefusalError + chains = read_chain_list(src_chain_file) + other = chains[chains.item_name != chains.stream_name + "." + chains.chain_id] + if len(other) > 0: + # Each field quoted, so that whitespace shows + listed = "; ".join(f"{row.stream_name!r} chain {row.chain_id!r} named {row.item_name!r}" + for row in other.head(5).itertuples()) + more = f"; and {len(other) - 5} more rows" if len(other) > 5 else "" + raise RefusalError( + f"{src_chain_file}: {len(other)} row(s) with an embedding name other than .: " + f"{listed}{more}. The assembly step of complete-embedding reads each chain's residue embedding as " + f"., as assembly-embedding does: name those rows so, or leave their 4th " + f"column out." + ) + + @app.command( name="download-models", - help="Download models from huggingface and store them in the default location." + help=DOWNLOAD_MODELS_HELP ) -def download_models(): - from foldmatch.utils.model import get_residue_model, get_aggregator_model - get_residue_model() - get_aggregator_model() +def download_models( + system: Annotated[Optional[List[str]], typer.Option( + "--system", + help=DOWNLOAD_MODELS_SYSTEM_HELP + )] = None +): + download_models_command(system) def version_callback(value: bool): if value: diff --git a/src/foldmatch/cli/search.py b/src/foldmatch/cli/search.py index 687a96a..db6935c 100644 --- a/src/foldmatch/cli/search.py +++ b/src/foldmatch/cli/search.py @@ -1,21 +1,37 @@ +import glob import os import logging +import math +import shlex +import sqlite3 +import sys +import time import typer +from contextlib import ExitStack, contextmanager, nullcontext +from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path -from typing import Annotated, Optional, List +from types import MappingProxyType +from typing import Annotated, Any, List, Mapping, Optional, Tuple from foldmatch import __version__ -from foldmatch.cli.args_utils import arg_devices, set_log_level +from foldmatch.cli.args_utils import DOWNLOAD_MODELS_HELP, DOWNLOAD_MODELS_SYSTEM_HELP, arg_devices, \ + download_models_command, set_log_level +from foldmatch.errors import RefusalError +from foldmatch.search.db_path import parse_db_path from foldmatch.search.output import ( CLUSTER_OUTPUT_FIELDS, DEFAULT_FORMAT_OUTPUT, EMBEDDING_OUTPUT_FIELDS, + ESM_CODES_OUTPUT_FIELDS, + GREEDY_CLUSTER_OUTPUT_FIELDS, SUPPORTED_OUTPUT_FIELDS, write_embedding_results, ) from foldmatch.types.api_types import ( StructureFormat, Accelerator, + ClusterMethod, Strategy, Granularity, IndexConfig, @@ -23,6 +39,8 @@ LogLevel, SignificanceMode, FormatMode, + ScoringChoice, + DEFAULT_SCORING, ) os.environ["TOKENIZERS_PARALLELISM"] = "false" @@ -62,7 +80,160 @@ _FORMAT_OUTPUT_HELP = ( "Comma-separated columns for the Stage-2 sequence-identity output, written " "as tab-separated rows with no header. " - f"Available fields: {', '.join(SUPPORTED_OUTPUT_FIELDS)}." + f"Available fields: {', '.join(SUPPORTED_OUTPUT_FIELDS)}. " + f"Only 'query sequences --scoring esm-codes' fills {', '.join(ESM_CODES_OUTPUT_FIELDS)}; " + "they are empty under BLOSUM62 scoring (and in 'query db')." +) + +# Stage 2 keeps a hit only if min(qcov, tcov) >= --min-coverage. +_MIN_COVERAGE_HELP = ( + "When --seq-identity is set, keep only hits whose query coverage and " + "subject coverage both reach this fraction (0-1); a hit is dropped when " + "either side is below it." +) + +# query sequences: one Stage-2 scoring per run (plan P6.1, D7). The filter defaults depend on it (D8), and the +# BLOSUM62-only options are refused under esm-codes (D20: exit status 2). +_SCORING_HELP = ( + "Stage-2 scoring, one per run: 'blosum62' (BLOSUM62 Smith-Waterman with BLAST-style " + "statistics) or 'esm-codes' (alignment of the ESM3 residue codes, with E-values calibrated " + "over the database's residues; needs a database built with 'build sequences " + "--residue-codes', and uses the scoring system its residue-code manifest names). Under " + "esm-codes the identity and coverage filters default to 0, --gap-open, --gap-extend, " + "--comp-bias-corr, --significance-mode and --no-seq-identity are refused, and query " + "FASTA files with duplicate ids, '<' or '>' are refused." +) + +_QUERY_MIN_SEQ_IDENTITY_HELP = ( + "When --seq-identity is set, drop hits whose alignment identity is below this " + "fraction (0-1). Default: 0.3 under --scoring blosum62, 0 under esm-codes." +) + +_QUERY_MIN_COVERAGE_HELP = ( + f"{_MIN_COVERAGE_HELP} Default: 0.8 under --scoring blosum62, 0 under esm-codes." +) + +_QUERY_MAX_EVALUE_HELP = ( + "When --seq-identity is set, drop hits whose database E-value is above this " + "(default 1e-3); pass 'inf' to disable E-value filtering. Under --scoring blosum62 " + "it forces the significance pass (and, in --significance-mode default, its 11/1 gap " + "penalties); under esm-codes the E-value is the calibrated one, over the database's " + "residue count." +) + +_BLOSUM62_ONLY = " BLOSUM62 scoring only: refused with --scoring esm-codes." + +_CLEANUP_EMBEDDINGS_HELP = ( + "Delete this build's intermediate embedding shards " + "(emb_-.parquet) from --tmp-embedding-folder once the " + "database is built. By default they are kept." +) + +# Plan section 8, P5.6 (verbatim). +_RESIDUE_CODES_HELP = ( + "Also compute 8-byte per-residue ESM3 codes in the same pass, enabling " + "`query sequences --scoring esm-codes`. Adds 8 B per residue plus up to " + "about 100 B per entry for the index, and a one-time ~26 MB download. " + "Sequence (FASTA) databases only." +) + +_FROM_SHARDS_HELP = ( + "Folder holding the embedding shards that this database's 'build sequences " + "--residue-codes' run kept (its --tmp-embedding-folder; a build keeps them " + "unless --cleanup-embeddings), or those of a 'build sequences-embed " + "--residue-codes' run (its --shards-folder)." +) + +# Plan P8b (CLI): the split build's step 1, verbatim but for --kill-on-invalid-dep=yes (without it, the job of step 2 +# stays pending for ever once step 1 fails). +_EMBED_HELP = ( + "Step 1 of a split 'build sequences' for multi-node GPU runs: the pre-flight and " + "the ESM3 pass (chain embeddings, and residue codes with --residue-codes) on every " + "rank, writing the embedding shards and, last, a run record into --shards-folder. " + "No database file is written. Then build the database with 'build sequences-index' " + "as its own one-process job on a CPU node (e.g. sbatch --dependency=afterok: --kill-on-invalid-dep=yes), so that these nodes are released." +) + +_EMBED_OUTPUT_DB_HELP = ( + "Path of the database 'build sequences-index' will build (as --output-db of 'build " + "sequences'). Refused if any of its files exist; recorded in the run record." +) + +_SHARDS_FOLDER_HELP = ( + "Folder for this step's output: the embedding shards (emb_-.parquet) " + "and, written last, the run record (emb_.run.json) that 'build " + "sequences-index --from-shards' reads. It must be on storage every node can write " + "and that outlives the job (e.g. $SCRATCH), with room for about 6 KiB per entry " + "plus 8 B per residue with --residue-codes. Use one folder per build." +) + +# Plan P8b (CLI), verbatim: the split build's step 2. +_INDEX_HELP = ( + "Step 2 of a split 'build sequences': build the database from the shards of one " + "'build sequences-embed' run, after checking against its run record that they are " + "complete and were computed from this FASTA: the FAISS index, the sequence store and, " + "when the run computed residue codes, the residue-code store, as 'build sequences' " + "builds them after its ESM3 pass. No ESM3, GPU or download. One process: run it as its " + "own job on a CPU node, not inside the GPU job; FAISS uses the node's physical cores " + "unless OMP_NUM_THREADS is set." +) + +_INDEX_FASTA_HELP = ( + "The FASTA file the run embedded. Checked against the run record (size, sha256, " + "pre-flight counts) and against the id of every shard row; it may have moved since." +) + +_INDEX_OUTPUT_DB_HELP = ( + "Path to save the FAISS database (as for 'build sequences'). Usually the run's " + "--output-db, which the record holds; another path is allowed." +) + +_INDEX_FROM_SHARDS_HELP = ( + "Folder holding the shards and the run record of a 'build sequences-embed' run (its " + "--shards-folder)." +) + +_INDEX_RUN_ID_HELP = ( + "Run id of the run to build from (its record is emb_.run.json; 'build " + "sequences-embed' logs it). Needed when the folder holds more than one run, unfinished " + "ones included." +) + +_INDEX_RESIDUE_CODES_HELP = ( + "Build the residue-code store from the run's codes (enabling `query sequences --scoring " + "esm-codes`). Default: exactly when the run computed them ('build sequences-embed " + "--residue-codes'). --residue-codes refuses a run without codes; --no-residue-codes " + "leaves the store out (add it later with 'build residue-codes --from-shards')." +) + +_INDEX_CLEANUP_EMBEDDINGS_HELP = ( + "Delete the run's record, then its shards, from --from-shards once the database is " + "built. Use it only on the last build from this run. Refused together with " + "--no-residue-codes on a run with residue codes. By default they are kept." +) + +# Plan P8b (CLI), verbatim: the split build's recovery. +_SEAL_HELP = ( + "Recovery: write the run record of embedding shards that have none, so that 'build " + "sequences-index' can build from them: a 'build sequences-embed' run whose rank 0 stopped " + "after the pass (wall time, out of memory, a full disk), or the shards a 'build sequences' " + "run kept when its index phase failed. Checks the shards against the FASTA (entry count, " + "each rank's rows, the shard schema and the id of every row) without ESM3 or a GPU, in one " + "process. The record says it was sealed: it binds the shards by these checks, not by the " + "pass." +) + +_SEAL_FASTA_HELP = "The FASTA file the run embedded." + +_SEAL_FROM_SHARDS_HELP = "Folder holding the run's shards." + +_SEAL_RUN_ID_HELP = "Run id of the shards to seal (emb_-.parquet)." + +_SEAL_MIN_RES_N_HELP = ( + "The --min-res-n the run used. Required for a run without residue codes, whose shards do " + "not record it; for a run with codes it is read from the shards, and a different value is " + "refused." ) @@ -131,12 +302,15 @@ def build_database_from_structures( help='Cells probed per query for ivf_pq (defaults to nlist // 64).' )] = None, load_batch_size: Annotated[int, typer.Option( - help='Number of vectors per batch streamed into the FAISS index. Larger ' - 'batches (e.g. 65536+) better saturate many-core nodes during ' - 'IVF-PQ build; smaller batches use less memory.' + help='Number of vectors per batch streamed into the FAISS index; larger ' + 'batches use more memory (FAISS adds IVF-PQ vectors in chunks of ' + '32768 whatever this value).' )] = 32768, + cleanup_embeddings: Annotated[bool, typer.Option( + help=_CLEANUP_EMBEDDINGS_HELP + )] = False, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): """Build an embedding database from structure files.""" @@ -159,6 +333,9 @@ def build_database_from_structures( if use_gpu_index: logging.info("GPU acceleration for FAISS index: enabled") + # Read-only, on every rank, before ESM3. + _refuse_existing_outputs(output_db, _INDEX_SUFFIXES) + from foldmatch.search.embedding_computer import EmbeddingComputer embedding_computer = EmbeddingComputer( embedding_folder=tmp_embedding_folder, @@ -190,6 +367,8 @@ def build_database_from_structures( index_type=index_type, index_config=IndexConfig(nlist=ivf_nlist, nprobe=ivf_nprobe), ) + if cleanup_embeddings: + embedding_computer.remove_shards() @build_db_app.command( @@ -202,7 +381,8 @@ def build_database_from_embeddings( file_okay=False, dir_okay=True, resolve_path=True, - help='Folder containing pre-computed embedding files (.pt, .csv, or .parquet).' + help='Folder of pre-computed chain or assembly embeddings: .pt or .csv files of one vector each, or ' + '.parquet files (columns id and embedding). Residue embeddings are refused.' )], output_db: Annotated[str, typer.Option( help='Path to save the FAISS database.' @@ -223,9 +403,9 @@ def build_database_from_embeddings( help='Cells probed per query for ivf_pq (defaults to nlist // 64).' )] = None, load_batch_size: Annotated[int, typer.Option( - help='Number of vectors per batch streamed into the FAISS index. Larger ' - 'batches (e.g. 65536+) better saturate many-core nodes during ' - 'IVF-PQ build; smaller batches use less memory.' + help='Number of vectors per batch streamed into the FAISS index; larger ' + 'batches use more memory (FAISS adds IVF-PQ vectors in chunks of ' + '32768 whatever this value).' )] = 32768, log_level: Annotated[LogLevel, typer.Option( help='Logging level.' @@ -308,10 +488,17 @@ def build_database_from_fasta( help='Cells probed per query for ivf_pq (defaults to nlist // 64).' )] = None, load_batch_size: Annotated[int, typer.Option( - help='Number of vectors per batch streamed into the FAISS index. Larger ' - 'batches (e.g. 65536+) better saturate many-core nodes during ' - 'IVF-PQ build; smaller batches use less memory.' + help='Number of vectors per batch streamed into the FAISS index; larger ' + 'batches use more memory (FAISS adds IVF-PQ vectors in chunks of ' + '32768 whatever this value).' )] = 32768, + cleanup_embeddings: Annotated[bool, typer.Option( + help=_CLEANUP_EMBEDDINGS_HELP + )] = False, + residue_codes: Annotated[bool, typer.Option( + "--residue-codes/--no-residue-codes", + help=_RESIDUE_CODES_HELP + )] = False, log_level: Annotated[LogLevel, typer.Option( help='Logging level.' )] = LogLevel.info @@ -319,44 +506,392 @@ def build_database_from_fasta( """Build an embedding database from protein sequences in a FASTA file.""" set_log_level(log_level) + # Read only, for the index phase's log lines: Lightning's DDP launcher sets OMP_NUM_THREADS during the pass when + # it is unset (to the allowed CPUs // devices per node), and FAISS starts with that value. + omp_env_at_start = os.environ.get("OMP_NUM_THREADS") + run = _sequence_build_preflight(fasta_file, min_res_n, output_db, tmp_embedding_folder, num_nodes, residue_codes) + # A --residue-codes build reports every refusal as a RefusalError (D20: + # exit 1); a plain build raises exactly what it raised before. + with run.refusals(): + _run_sequence_pass(run, fasta_file=fasta_file, min_res_n=min_res_n, batch_size=batch_size, + num_workers=num_workers, num_nodes=num_nodes, accelerator=accelerator, devices=devices, + strategy=strategy) - from foldmatch.search.embedding_computer import EmbeddingComputer - embedding_computer = EmbeddingComputer( - embedding_folder=tmp_embedding_folder, - ) - embedding_computer.compute_from_fasta( - fasta_file=fasta_file, - min_res_n=min_res_n, - batch_size=batch_size, - num_workers=num_workers, - num_nodes=num_nodes, - accelerator=accelerator, - devices=arg_devices(devices), - strategy=strategy, + if not _is_rank_zero(): + return + if num_nodes > 1: + logging.info( + f"The index phase runs in this one process: the other {num_nodes - 1} node(s) stay allocated until it " + f"ends. 'build sequences-embed' and then 'build sequences-index' (as its own job on a CPU node) " + f"release them when the ESM3 pass ends." + ) + + _build_sequence_database( + run.embedding_computer, run.preflight, fasta_file=fasta_file, output_db=output_db, min_res_n=min_res_n, + system=None if run.prepared is None else run.prepared.system, + index=_IndexOptions(use_gpu_index, index_type, ivf_nlist, ivf_nprobe, load_batch_size), + refusals=run.refusals, omp_env_at_start=omp_env_at_start, ) + if cleanup_embeddings: + run.embedding_computer.remove_shards() + + +@build_db_app.command( + name="sequences-embed", + help=_EMBED_HELP +) +def build_sequences_embed( + fasta_file: Annotated[Path, typer.Option( + exists=True, + file_okay=True, + dir_okay=False, + resolve_path=True, + help='FASTA file containing protein sequences.' + )], + output_db: Annotated[str, typer.Option( + help=_EMBED_OUTPUT_DB_HELP + )], + shards_folder: Annotated[Path, typer.Option( + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + help=_SHARDS_FOLDER_HELP + )], + min_res_n: Annotated[int, typer.Option( + min=0, + help='Consider only sequences with at least residues.' + )] = 0, + accelerator: Annotated[Accelerator, typer.Option( + help='Device used for inference.' + )] = Accelerator.auto, + num_nodes: Annotated[int, typer.Option( + help='Number of nodes to use for inference.' + )] = 1, + devices: Annotated[List[str], typer.Option( + help='The devices to use. Can be set to a positive number or "auto".' + )] = ['auto'], + strategy: Annotated[Strategy, typer.Option( + help='Lightning strategy to control distribution of inference.' + )] = Strategy.auto, + batch_size: Annotated[int, typer.Option( + help='Number of samples processed together for residue embedding inference.' + )] = 1, + num_workers: Annotated[int, typer.Option( + help='Number of subprocesses to use for data loading.' + )] = 0, + residue_codes: Annotated[bool, typer.Option( + "--residue-codes/--no-residue-codes", + help=_RESIDUE_CODES_HELP + )] = False, + log_level: Annotated[LogLevel, typer.Option( + help='Logging level.' + )] = LogLevel.info +): + """Step 1 of a split sequence build: the pre-flight and the ESM3 pass on every rank, then rank 0 checks the shards + and writes the run's record last. No database file.""" + # Every rank, read-only, before ESM3. Every refusal is a RefusalError, in both modes (D32). + set_log_level(log_level) + from foldmatch.search import split_run + before, before_error = _fasta_stat_or_error(fasta_file, "before the pre-flight") + start = time.monotonic() + with _refusals(): + run = _sequence_build_preflight(fasta_file, min_res_n, output_db, shards_folder, num_nodes, residue_codes) + preflight_seconds = time.monotonic() - start + with _refusals(): + split_run.refuse_existing_record(shards_folder, run.embedding_computer.run_id) + + # Every rank: the pass of the one-shot build, up to its barrier. + with _refusals(): + _run_sequence_pass(run, fasta_file=fasta_file, min_res_n=min_res_n, batch_size=batch_size, + num_workers=num_workers, num_nodes=num_nodes, accelerator=accelerator, devices=devices, + strategy=strategy) + after, after_error = _fasta_stat_or_error(fasta_file, "after the pass") + facts = _gather_rank_facts({ + "before": before, + "after": after, + "error": "; ".join(error for error in (before_error, after_error) if error) or None, + "size": None if before is None else before.bytes, + "preflight": split_run.preflight_counts(run.preflight), + "seconds": preflight_seconds, + }) if not _is_rank_zero(): return - from foldmatch.search.embedding_database import EmbeddingDatabase - embedding_db = EmbeddingDatabase( - db_path=output_db + options = {"accelerator": accelerator.value, "devices": list(devices), "num_nodes": num_nodes, + "strategy": strategy.value, "batch_size": batch_size, "num_workers": num_workers} + record, path = _record_embed_run(run, facts, fasta_file=fasta_file, output_db=output_db, + shards_folder=shards_folder, min_res_n=min_res_n, options=options) + codes = f", residue codes of {record['system']['name']}" if record["system"] is not None else "" + logging.info( + f"Run {record['run_id']} complete: {record['rows']} entries in {len(record['shards'])} shard(s) of " + f"{record['world_size']} rank(s){codes}; record {path}" ) - embedding_db.create_db( - embedding_batches=embedding_computer.get_embedding_batches(load_batch_size=load_batch_size), - use_gpu_index=use_gpu_index, - index_type=index_type, - index_config=IndexConfig(nlist=ivf_nlist, nprobe=ivf_nprobe), + logging.info("Build the database with: " + shlex.join([ + "fm-search", "build", "sequences-index", "--fasta-file", str(fasta_file), "--output-db", record["output_db"], + "--from-shards", str(shards_folder), "--run-id", record["run_id"], + ])) + logging.info( + f"Add --index-type to choose its FAISS index: auto picks hnsw at {_AUTO_HNSW_ENTRIES:,} entries or more; flat " + f"and hnsw hold every embedding in memory, about 12 KiB per entry at peak; ivf_pq is bounded." ) - # Persist a sidecar id->sequence store so this database can be the - # subject (or query) of a Stage-2 pairwise-alignment search. Built - # directly from the FASTA — the DB ids ARE the FASTA headers — using the - # same min_res_n filter so store and index stay in lock-step. - from foldmatch.search.sequence_store import SequenceStore - SequenceStore(embedding_db.db_folder, embedding_db.index_name).create( - fasta_file=fasta_file, - min_res_n=min_res_n, + +@build_db_app.command( + name="sequences-index", + help=_INDEX_HELP +) +def build_sequences_index( + fasta_file: Annotated[Path, typer.Option( + exists=True, + file_okay=True, + dir_okay=False, + resolve_path=True, + help=_INDEX_FASTA_HELP + )], + output_db: Annotated[str, typer.Option( + help=_INDEX_OUTPUT_DB_HELP + )], + from_shards: Annotated[Path, typer.Option( + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + help=_INDEX_FROM_SHARDS_HELP + )], + run_id: Annotated[Optional[str], typer.Option( + help=_INDEX_RUN_ID_HELP + )] = None, + residue_codes: Annotated[Optional[bool], typer.Option( + "--residue-codes/--no-residue-codes", + help=_INDEX_RESIDUE_CODES_HELP + )] = None, + cleanup_embeddings: Annotated[bool, typer.Option( + help=_INDEX_CLEANUP_EMBEDDINGS_HELP + )] = False, + use_gpu_index: Annotated[bool, typer.Option( + help='Use GPU for FAISS index (requires faiss-gpu).' + )] = False, + index_type: Annotated[IndexType, typer.Option( + help='FAISS index variant: auto (legacy heuristic), flat, hnsw, or ivf_pq (on-disk).' + )] = IndexType.auto, + ivf_nlist: Annotated[int, typer.Option( + help='IVF cell count for ivf_pq index.' + )] = 16384, + ivf_nprobe: Annotated[Optional[int], typer.Option( + help='Cells probed per query for ivf_pq (defaults to nlist // 64).' + )] = None, + load_batch_size: Annotated[int, typer.Option( + help='Number of vectors per batch streamed into the FAISS index; larger ' + 'batches use more memory (FAISS adds IVF-PQ vectors in chunks of ' + '32768 whatever this value).' + )] = 32768, + log_level: Annotated[LogLevel, typer.Option( + help='Logging level.' + )] = LogLevel.info +): + """Step 2 of a split sequence build: one process checks a run's shards against its record and the FASTA, then + builds the database from them as 'build sequences' does after its pass. No ESM3, GPU or download.""" + + # Every refusal is a RefusalError, in both modes (D32). Up to the index phase nothing is written but the build lock + # (and the database's folder when it is missing), which a refusal removes. + # The refusals up to the index phase come before torch and Lightning are imported (minutes from a cold file system + # on a compute node): the launch checks first, I2 and I3 (their WARNINGs wait for the logging setup), then + # set_log_level without its Lightning import, which the second call, before EmbeddingComputer, makes. + from foldmatch.utils.dist import is_run_id + if run_id is not None and not is_run_id(run_id): + raise typer.BadParameter( + f"{run_id!r} is not a run id: letters, digits, '.', '_' and '-' only, " + f"starting with a letter or digit.", param_hint="--run-id" + ) + launch_warnings = _refuse_multi_node_job("build sequences-index") + set_log_level(log_level, defer_lightning=True) + start = time.monotonic() + for message in launch_warnings: + logging.warning(message) + # Before torch, FAISS or pyarrow start a thread pool: the threads they start inherit this mask (D29). + from foldmatch.search.alignment import _expand_cpu_affinity + _expand_cpu_affinity(purpose="the index phase") + + from foldmatch.search import split_run + with _refusals(): + run_id, record_path = split_run.find_run(from_shards, run_id) + record = split_run.read_run_record(record_path) + codes, system = _split_code_store(record, record_path, residue_codes=residue_codes, + cleanup_embeddings=cleanup_embeddings) + db_path = parse_db_path(output_db)[2] + with _database_folder(output_db), split_run.build_lock(db_path, record, shards_folder=from_shards): + with _refusals(): + paths, preflight, fasta_stat = _split_index_checks(record, fasta_file=fasta_file, output_db=output_db, + from_shards=from_shards, codes=codes, system=system) + _log_split_build(record, db_path=db_path, from_shards=from_shards, shards=len(paths), codes=codes) + # The Lightning import set_log_level deferred, before EmbeddingComputer imports Lightning and torch, and the + # index phase FAISS; not in the build's time, as before. + deferred = time.monotonic() + set_log_level(log_level) + start += time.monotonic() - deferred + from foldmatch.search.embedding_computer import EmbeddingComputer + computer = EmbeddingComputer(from_shards, run=run_id) + with _refusals(): + computer.attach_preflight(fasta_file, record["min_res_n"], preflight) + with _whole_node_threads(): + _build_sequence_database( + computer, preflight, fasta_file=fasta_file, output_db=output_db, min_res_n=record["min_res_n"], + system=system if codes else None, + index=_IndexOptions(use_gpu_index, index_type, ivf_nlist, ivf_nprobe, load_batch_size), + refusals=_refusals, split=record, + after_sequence_store=_fasta_unchanged_guard(fasta_file, fasta_stat, output_db), + ) + if cleanup_embeddings: + # The record first: no record outlives its shards. + with _refusals(): + record_path.unlink() + computer.remove_shards() + logging.info(f"Database {db_path} built from run {run_id} in {time.monotonic() - start:.1f} s") + + +@build_db_app.command( + name="sequences-seal", + help=_SEAL_HELP +) +def build_sequences_seal( + fasta_file: Annotated[Path, typer.Option( + exists=True, + file_okay=True, + dir_okay=False, + resolve_path=True, + help=_SEAL_FASTA_HELP + )], + from_shards: Annotated[Path, typer.Option( + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + help=_SEAL_FROM_SHARDS_HELP + )], + run_id: Annotated[str, typer.Option( + help=_SEAL_RUN_ID_HELP + )], + min_res_n: Annotated[Optional[int], typer.Option( + min=0, + help=_SEAL_MIN_RES_N_HELP + )] = None, + log_level: Annotated[LogLevel, typer.Option( + help='Logging level.' + )] = LogLevel.info +): + """Recovery of a split sequence build: one process checks the shards of a run that has no run record against the + FASTA, then writes their record, sealed, so that 'build sequences-index' can build from them. No ESM3 or GPU.""" + + # Every refusal is a RefusalError, in both modes (D32). Nothing is written but the record, last. + # The launch checks come first, as in 'build sequences-index'. The seal never imports torch, Lightning or FAISS, + # so set_log_level does not import Lightning: every refusal comes before them. + from foldmatch.utils.dist import is_run_id + if not is_run_id(run_id): + raise typer.BadParameter( + f"{run_id!r} is not a run id: letters, digits, '.', '_' and '-' only, " + f"starting with a letter or digit.", param_hint="--run-id" + ) + launch_warnings = _refuse_multi_node_job("build sequences-seal") + set_log_level(log_level, defer_lightning=True) + for message in launch_warnings: + logging.warning(message) + + from foldmatch.search import split_run + with _refusals(): + split_run.refuse_existing_record(from_shards, run_id, command=split_run.SEAL_COMMAND) + record = _seal_run(fasta_file, from_shards, run_id, min_res_n) + path = split_run.write_run_record(from_shards, record) + codes = f", residue codes of {record['system']['name']}" if record["system"] is not None else "" + logging.info( + f"Run {run_id} sealed: {record['rows']} entries in {len(record['shards'])} shard(s) of " + f"{record['world_size']} rank(s){codes}; record {path}" + ) + logging.warning( + f"The record of run {run_id} is sealed: it binds the shards to {fasta_file} by their entry count, each rank's " + f"rows and the id of every row" + + (" and its seq_sha1, the sha1 of the entry's sequence" if codes else "") + + ", not by the pass that wrote them, which 'build sequences-seal' did not see." + ) + logging.info("Build the database DB with: " + shlex.join([ + "fm-search", "build", "sequences-index", "--fasta-file", str(fasta_file), "--output-db", "DB", + "--from-shards", str(from_shards), "--run-id", run_id, + ])) + + +@build_db_app.command( + name="residue-codes", + help="Build the residue-code store of a sequence database from the embedding " + "shards its 'build sequences --residue-codes' run kept, or those of the " + "'build sequences-embed --residue-codes' run it was built from, without " + "running ESM3 again, and validate it against the database. DB_FOLDER and " + "INDEX_NAME name the database (its --output-db was DB_FOLDER/INDEX_NAME)." +) +def build_residue_codes( + db_folder: Annotated[Path, typer.Argument( + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + help='Folder of the database.' + )], + index_name: Annotated[str, typer.Argument( + help='Name of the database in DB_FOLDER: its files are ' + 'INDEX_NAME.index, INDEX_NAME.sequences, ...' + )], + from_shards: Annotated[Path, typer.Option( + exists=True, + file_okay=False, + dir_okay=True, + resolve_path=True, + help=_FROM_SHARDS_HELP + )], + run_id: Annotated[Optional[str], typer.Option( + help='Run id of the build whose shards to use ' + '(emb_-.parquet). Needed only when the folder ' + 'holds the residue-code shards of more than one run.' + )] = None, + force: Annotated[bool, typer.Option( + "--force/--no-force", + help='Replace the residue-code store if the database already has ' + 'one. The new codes are written aside and replace the old ' + 'store only once they have passed validation, so a refused ' + 'rebuild leaves it as it was (the rebuild needs room for both ' + 'meanwhile).' + )] = False, + load_batch_size: Annotated[int, typer.Option( + min=1, + help='Number of entries read from the shards per batch.' + )] = 32768, + log_level: Annotated[LogLevel, typer.Option( + help='Logging level.' + )] = LogLevel.info +): + """Build a sequence database's residue-code store from kept build shards (no ESM3).""" + + set_log_level(log_level) + # Flag refusals (D20: exit 2), before anything is read. + if not index_name or index_name in (".", "..") or Path(index_name).name != index_name: + raise typer.BadParameter( + f"{index_name!r} is not a database name: INDEX_NAME is the name of the " + f"database's files in DB_FOLDER, without a folder.", param_hint="INDEX_NAME" + ) + from foldmatch.utils.dist import is_run_id + if run_id is not None and not is_run_id(run_id): + raise typer.BadParameter( + f"{run_id!r} is not a run id: letters, digits, '.', '_' and '-' only, " + f"starting with a letter or digit.", param_hint="--run-id" + ) + with _refusals(): + manifest = _residue_codes_from_shards( + db_folder, index_name, from_shards, run_id=run_id, force=force, + load_batch_size=load_batch_size, + ) + logging.info( + f"Residue-code store of {db_folder / index_name} built from the shards of run " + f"{manifest['build']['run_id']}: {manifest['entries']} entries, " + f"{manifest['total_residues']} residues." ) @@ -400,7 +935,10 @@ def query_database_from_structure( help='Minimum residue length for chains.' )] = 10, max_res: Annotated[Optional[int], typer.Option( - help='Maximum residue length for structures (None for no limit).' + help='Maximum number of residues to embed (None for no limit), counted as --min-res counts them: with ' + '"granularity=chain", those of the chain given by --chain-id, else of every protein chain of the ' + 'structure, together; with "granularity=assembly", those of every protein chain of the assembly ' + '(or asymmetric unit). Chains under --min-res are not counted. A longer query is refused.' )] = None, accelerator: Annotated[Accelerator, typer.Option( help='Device used for inference.' @@ -418,12 +956,14 @@ def query_database_from_structure( help=_FORMAT_MODE_HELP )] = FormatMode.headless_tsv, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): """Search database for similar structures.""" set_log_level(log_level) + _refuse_output_over_inputs(output_file, files=[("the query structure file", query_structure)], + databases=[("the database", db_path)]) from foldmatch.search.embedding_computer import compute_from_structure_file embedding_batches = compute_from_structure_file( @@ -468,7 +1008,8 @@ def query_database_from_embedding( file_okay=False, dir_okay=True, resolve_path=True, - help='Directory containing pre-computed embedding files (.pt, .csv, or .parquet).' + help='Directory of pre-computed chain or assembly embeddings: .pt or .csv files of one vector each, or ' + '.parquet files (columns id and embedding). Residue embeddings are refused.' )], output_file: Annotated[str, typer.Option( help=_OUTPUT_FILE_HELP @@ -501,6 +1042,12 @@ def query_database_from_embedding( """Search database using a pre-computed embedding file.""" set_log_level(log_level) + # The embedding files it reads are inputs (P9.N D6), and one query per id: the results are kept by query id (D7) + from foldmatch.utils.data import embedding_files, refuse_repeated_query_ids + _refuse_output_over_inputs(output_file, databases=[("the database", db_path)], + files=[("the query embedding file", f) + for kind in embedding_files(embedding_folder, file_extension) for f in kind]) + refuse_repeated_query_ids(embedding_folder, file_extension) logging.info("Loading database...") from foldmatch.search.embedding_database import EmbeddingDatabase @@ -513,7 +1060,8 @@ def query_database_from_embedding( results = embedding_db.search_by_embeddings( embedding_batches=stream_embeddings( path=embedding_folder, - file_extension=file_extension + file_extension=file_extension, + expected_dim=embedding_db.db.dimension, ), top_k=top_k ) @@ -588,59 +1136,69 @@ def query_database_from_fasta( 'Defaults to the value baked in at build time. ' 'Ignored for flat/hnsw indexes.' )] = None, + scoring: Annotated[ScoringChoice, typer.Option( + help=_SCORING_HELP + )] = DEFAULT_SCORING, seq_identity: Annotated[Optional[bool], typer.Option( help='Stage 2: after the embedding prefilter, pairwise-align each ' 'query against its candidate subjects and report exact sequence ' 'identity. Default: AUTO — on when the subject database has a ' 'sequence store (built from FASTA), else embedding-only. Use ' '--no-seq-identity to force off, --seq-identity to force on ' - '(errors if the store is missing).' + '(errors if the store is missing). --scoring esm-codes always ' + 'runs Stage 2.' )] = None, - min_seq_identity: Annotated[float, typer.Option( - help='When --seq-identity is set, drop hits whose alignment identity ' - 'is below this fraction (0-1).' - )] = 0.3, - min_coverage: Annotated[float, typer.Option( - help='When --seq-identity is set, drop hits whose query AND subject ' - 'coverage are below this fraction (0-1).' - )] = 0.8, - max_evalue: Annotated[float, typer.Option( - help='When --seq-identity is set, drop hits whose database E-value is ' - 'above this. Forces the significance pass (and, in ' - "--significance-mode default, its 11/1 gap penalties); pass " - "'inf' to disable E-value filtering." - )] = 1e-3, - gap_open: Annotated[int, typer.Option( - help='Gap-open penalty (positive) for Stage-2 alignment.' - )] = 11, - gap_extend: Annotated[int, typer.Option( - help='Gap-extend penalty (positive) for Stage-2 alignment.' - )] = 1, - comp_bias_corr: Annotated[bool, typer.Option( + min_seq_identity: Annotated[Optional[float], typer.Option( + help=_QUERY_MIN_SEQ_IDENTITY_HELP + )] = None, + min_coverage: Annotated[Optional[float], typer.Option( + help=_QUERY_MIN_COVERAGE_HELP + )] = None, + max_evalue: Annotated[Optional[float], typer.Option( + help=_QUERY_MAX_EVALUE_HELP + )] = None, + gap_open: Annotated[Optional[int], typer.Option( + help='Gap-open cost for Stage-2 alignment, 1 to 1048576 (the sign is ignored). Default: 11.' + + _BLOSUM62_ONLY + )] = None, + gap_extend: Annotated[Optional[int], typer.Option( + help='Gap-extend cost for Stage-2 alignment, 0 to 1048576 (the sign is ignored). Default: 1.' + + _BLOSUM62_ONLY + )] = None, + comp_bias_corr: Annotated[Optional[bool], typer.Option( help='Correct Stage-2 alignment scores for locally biased amino-acid ' 'composition. Changes the alignment itself (coordinates, ' - 'identity, coverage) as well as the raw score, bits and E-value.' - )] = False, - comp_bias_corr_scale: Annotated[float, typer.Option( + 'identity, coverage) as well as the raw score, bits and E-value. ' + 'Default: off.' + _BLOSUM62_ONLY + )] = None, + comp_bias_corr_scale: Annotated[Optional[float], typer.Option( min=0.0, max=1.0, help='Scale applied to the compositional bias correction when ' - '--comp-bias-corr is set (0-1).' - )] = 1.0, - significance_mode: Annotated[SignificanceMode, typer.Option( + '--comp-bias-corr is set (0-1). Default: 1.0.' + _BLOSUM62_ONLY + )] = None, + significance_mode: Annotated[Optional[SignificanceMode], typer.Option( help="Scale for the BitScore/Pvalue/Evalue columns. 'default' uses " "calibrated statistics, but requires the default gap penalties " "11/1; 'sampled' works at any gap penalties but its absolute " - "values are unreliable and comparable only within this tool." - )] = SignificanceMode.default, - significance_sample_size: Annotated[int, typer.Option( + "values are unreliable and comparable only within this tool. " + "Default: 'default'." + _BLOSUM62_ONLY + )] = None, + significance_sample_size: Annotated[Optional[int], typer.Option( + min=2, help="For --significance-mode sampled: number of random alignments " "sampled to estimate lambda/K for the BitScore/Pvalue/Evalue " - "columns. Larger is slower; ignored when mode is 'default'." - )] = 500, + "columns. Larger is slower; ignored when mode is 'default'. " + "Default: 500." + _BLOSUM62_ONLY + )] = None, align_workers: Annotated[Optional[int], typer.Option( - help='Process-pool size for Stage-2 alignment. Defaults to all CPUs ' - 'on the node (widening any scheduler --cpu-bind pinning); set ' - 'to 0 or 1 to run serially.' + help='Process-pool size for Stage-2 alignment; set to 0 or 1 to ' + 'run serially. Default under --scoring blosum62: all CPUs on ' + 'the node (widening any scheduler --cpu-bind pinning); under ' + 'esm-codes: up to one worker per physical core (SMT siblings ' + 'count once; the pinning widened the same way), only as many ' + 'as the work needs. An esm-codes worker needs about 5 bytes ' + 'per residue pair of the pair it aligns (up to 1.3 GiB for the ' + 'longest pairs aligned, 2^28 residue pairs).' )] = None, format_output: Annotated[str, typer.Option( help=_FORMAT_OUTPUT_HELP @@ -655,7 +1213,48 @@ def query_database_from_fasta( """Search database using protein sequences from a FASTA file.""" set_log_level(log_level) + _refuse_output_over_inputs(output_file, files=[("the FASTA file", fasta_file)], + databases=[("the database", db_path)]) + # The per-scoring defaults; under esm-codes the BLOSUM62-only options are refused (exit 2), before anything is read. + stage2 = resolve_query_stage2_options( + scoring, seq_identity=seq_identity, min_seq_identity=min_seq_identity, min_coverage=min_coverage, + max_evalue=max_evalue, gap_open=gap_open, gap_extend=gap_extend, comp_bias_corr=comp_bias_corr, + comp_bias_corr_scale=comp_bias_corr_scale, significance_mode=significance_mode, + significance_sample_size=significance_sample_size, + ) + if stage2.scoring is ScoringChoice.esm_codes: + _query_sequences_esm_codes( + db_path=db_path, fasta_file=fasta_file, tmp_embedding_folder=tmp_embedding_folder, + output_file=output_file, min_res_n=min_res_n, top_k=top_k, threshold=threshold, + use_gpu_index=use_gpu_index, accelerator=accelerator, num_nodes=num_nodes, devices=devices, + strategy=strategy, batch_size=batch_size, num_workers=num_workers, nprobe=nprobe, stage2=stage2, + align_workers=align_workers, format_output=format_output, format_mode=format_mode, + ) + return + min_seq_identity, min_coverage, max_evalue = stage2.min_seq_identity, stage2.min_coverage, stage2.max_evalue + gap_open, gap_extend, comp_bias_corr = stage2.gap_open, stage2.gap_extend, stage2.comp_bias_corr + comp_bias_corr_scale, significance_mode = stage2.comp_bias_corr_scale, stage2.significance_mode + significance_sample_size = stage2.significance_sample_size _check_comp_bias_corr_scale(comp_bias_corr_scale) + preflight = _fasta_preflight(fasta_file, min_res_n) + # Read-only, on every rank, before ESM3: whether Stage 2 runs, and whether + # the subject's sequence store can serve it. + from foldmatch.search.sequence_store import SequenceStore + subject_store = SequenceStore(*parse_db_path(db_path)[:2]) + run_stage2 = _resolve_stage2( + seq_identity, + subject_store.exists(), + f"Subject database has no sequence store ({subject_store.path}); " + f"--seq-identity requires a database built from FASTA (fm-search build sequences).", + ) + if run_stage2: + # The Stage-2 columns and gap costs, refused (exit status 2) before ESM3, as under esm-codes; then the stores + # Stage 2 reads and the kernel it aligns with. + fields = _output_fields(format_output) + _check_blosum62_gap_costs(gap_open, gap_extend, significance_mode, fields, max_evalue) + _check_significance_sample_size(significance_sample_size, significance_mode, fields, max_evalue) + _check_stage2_stores(subject_store, max_evalue) + _check_blosum62_kernel() from foldmatch.search.embedding_computer import EmbeddingComputer embedding_computer = EmbeddingComputer( @@ -670,6 +1269,7 @@ def query_database_from_fasta( accelerator=accelerator, devices=arg_devices(devices), strategy=strategy, + preflight=preflight, ) if not _is_rank_zero(): @@ -683,24 +1283,21 @@ def query_database_from_fasta( ) embedding_db.load(nprobe=nprobe) results = embedding_db.search_by_embeddings( - embedding_batches=embedding_computer.get_embedding_batches(), + embedding_batches=embedding_computer.get_embedding_batches(rows=True), top_k=top_k ) + # The shards are read rank by rank: the queries are written in input order, and a repeated id keeps its last + # record's hits, as by a 1-rank run. + results = embedding_computer.in_input_order(results) + # The query embeddings are only needed for this search. + embedding_computer.remove_shards() logging.info(f"Searched {len(results)} sequence(s)") results = _filter_results_by_threshold(results, threshold) - from foldmatch.search.sequence_store import SequenceStore - subject_store = SequenceStore(embedding_db.db_folder, embedding_db.index_name) - run_stage2 = _resolve_stage2( - seq_identity, - subject_store.exists(), - f"Subject database has no sequence store ({subject_store.path}); " - f"--seq-identity requires a database built from FASTA (fm-search build sequences).", - ) - if run_stage2: - from foldmatch.utils.fasta import iter_fasta - query_sequences = dict(iter_fasta(fasta_file)) + # Align the sequences ESM3 embedded (each query's last kept record, the one behind its Stage-1 hits), + # normalised like the stored subjects. + query_sequences = embedding_computer.input_sequences() _stage2_align_and_report( embedding_db=embedding_db, prefilter_results=results, @@ -765,8 +1362,7 @@ def query_database_from_database( 'is below this fraction (0-1).' )] = 0.3, min_coverage: Annotated[float, typer.Option( - help='When --seq-identity is set, drop hits whose query AND subject ' - 'coverage are below this fraction (0-1).' + help=_MIN_COVERAGE_HELP )] = 0.8, max_evalue: Annotated[float, typer.Option( help='When --seq-identity is set, drop hits whose database E-value is ' @@ -775,10 +1371,10 @@ def query_database_from_database( "'inf' to disable E-value filtering." )] = 1e-3, gap_open: Annotated[int, typer.Option( - help='Gap-open penalty (positive) for Stage-2 alignment.' + help='Gap-open cost for Stage-2 alignment, 1 to 1048576 (the sign is ignored).' )] = 11, gap_extend: Annotated[int, typer.Option( - help='Gap-extend penalty (positive) for Stage-2 alignment.' + help='Gap-extend cost for Stage-2 alignment, 0 to 1048576 (the sign is ignored).' )] = 1, comp_bias_corr: Annotated[bool, typer.Option( help='Correct Stage-2 alignment scores for locally biased amino-acid ' @@ -797,6 +1393,7 @@ def query_database_from_database( "values are unreliable and comparable only within this tool." )] = SignificanceMode.default, significance_sample_size: Annotated[int, typer.Option( + min=2, help="For --significance-mode sampled: number of random alignments " "sampled to estimate lambda/K for the BitScore/Pvalue/Evalue " "columns. Larger is slower; ignored when mode is 'default'." @@ -813,17 +1410,44 @@ def query_database_from_database( help=_FORMAT_MODE_HELP )] = FormatMode.headless_tsv, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): """Search subject database using all entries from query database.""" set_log_level(log_level) + _refuse_output_over_inputs(output_file, databases=[("the query database", query_db_path), + ("the subject database", subject_db_path)]) _check_comp_bias_corr_scale(comp_bias_corr_scale) if use_gpu_index: logging.info("GPU acceleration for FAISS search: enabled") + # Stage 2 needs sequences for BOTH sides: queries from the query DB's store, + # subjects from the subject DB's store. Decided, and the stores checked, + # before the search. + from foldmatch.search.sequence_store import SequenceStore + query_store = SequenceStore(*parse_db_path(query_db_path)[:2]) + subject_store = SequenceStore(*parse_db_path(subject_db_path)[:2]) + missing = [] + if not query_store.exists(): + missing.append(f"query database ({query_store.path})") + if not subject_store.exists(): + missing.append(f"subject database ({subject_store.path})") + run_stage2 = _resolve_stage2( + seq_identity, + not missing, + f"Stage-2 sequence identity requires a sequence store for: {', '.join(missing)}. " + f"Build databases with 'fm-search build sequences'.", + ) + if run_stage2: + # As in 'query sequences': the Stage-2 columns and gap costs (exit status 2), then the stores and the kernel. + fields = _output_fields(format_output) + _check_blosum62_gap_costs(gap_open, gap_extend, significance_mode, fields, max_evalue) + _check_significance_sample_size(significance_sample_size, significance_mode, fields, max_evalue) + _check_stage2_stores(subject_store, max_evalue, query_store=query_store) + _check_blosum62_kernel() + # Load subject database logging.info("\nLoading subject database...") from foldmatch.search.embedding_database import EmbeddingDatabase @@ -847,25 +1471,6 @@ def query_database_from_database( # Filter by threshold if specified results = _filter_results_by_threshold(results, threshold) - # Stage 2 needs sequences for BOTH sides: queries from the query DB's store, - # subjects from the subject DB's store. - from foldmatch.search.embedding_database import _parse_db_path - from foldmatch.search.sequence_store import SequenceStore - q_folder, q_name, _ = _parse_db_path(query_db_path) - query_store = SequenceStore(q_folder, q_name) - subject_store = SequenceStore(embedding_db.db_folder, embedding_db.index_name) - missing = [] - if not query_store.exists(): - missing.append(f"query database ({query_store.path})") - if not subject_store.exists(): - missing.append(f"subject database ({subject_store.path})") - run_stage2 = _resolve_stage2( - seq_identity, - not missing, - f"Stage-2 sequence identity requires a sequence store for: {', '.join(missing)}. " - f"Build databases with 'fm-search build sequences'.", - ) - if run_stage2: query_sequences = query_store.fetch(results.keys()) _stage2_align_and_report( @@ -899,7 +1504,7 @@ def show_statistics( help='Path to the FAISS database (directory + prefix, e.g., ./db/my_db).' )], log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): """Display database statistics.""" @@ -927,24 +1532,45 @@ def show_statistics( @app.command( name="cluster", - help="Cluster database embeddings using Leiden algorithm on similarity graph." + help="Cluster database embeddings: Leiden on the similarity graph (default), or greedy representatives in one " + "streamed pass (--method greedy)." ) def cluster_database( db_path: Annotated[str, typer.Option( help='Path to the FAISS database.' )], + method: Annotated[ClusterMethod, typer.Option( + help='Clustering method: leiden builds the similarity graph and runs Leiden (python-igraph, a fixed number ' + 'of iterations, --leiden-iterations; memory grows with the edges); ' + 'greedy picks representatives in one streamed pass over the database (memory grows with the chains, ' + 'searches only chains no representative has taken; for collections whose graph does not fit in ' + 'memory).' + )] = ClusterMethod.leiden, threshold: Annotated[float, typer.Option( - help='Similarity threshold for edge creation (0-1, where 1.0 = identical).' + help='Similarity threshold (cosine; identical chains score 1, or close to 1 on ivf_pq, where scores are PQ ' + 'estimates): with --method leiden, the edge threshold, and the kept scores must be at least 0 ' + '(Leiden\'s modularity needs non-negative edge weights); with --method greedy, a representative takes ' + 'the still-unclustered neighbours scoring at least this.' )] = 0.8, - resolution: Annotated[float, typer.Option( - help='Leiden resolution parameter (higher = more clusters).' - )] = 1.0, + resolution: Annotated[Optional[float], typer.Option( + help='Leiden resolution parameter (higher = more clusters). Default: 1.0. --method leiden only.' + )] = None, + leiden_iterations: Annotated[Optional[int], typer.Option( + help='Leiden iterations, a fixed count (at least 1). Default: 10, which on two measured 1M-chain graphs ' + 'agreed with Leiden run to convergence within seed noise; fewer are faster, with somewhat noisier ' + 'partitions. --method leiden only.' + )] = None, output_file: Annotated[str, typer.Option( help='Path to save cluster assignments, as tab-separated rows with ' - f'no header: {", ".join(CLUSTER_OUTPUT_FIELDS)}.' + f'no header: {", ".join(CLUSTER_OUTPUT_FIELDS)} (--method leiden) or ' + f'{", ".join(GREEDY_CLUSTER_OUTPUT_FIELDS)} (--method greedy, the representative\'s chain id last); ' + 'cluster_id 0 is the largest cluster.' )] = "clusters.tsv", max_neighbors: Annotated[int, typer.Option( - help='Maximum number of neighbors to consider per chain' + help='Maximum number of neighbors to consider per chain. With --method greedy, the neighbours a ' + 'representative takes from, so also a cluster\'s size cap (this + 1); when most representatives have ' + 'this many neighbours above --threshold, clusters are bounded by it rather than by --threshold (see ' + 'the statistics).' )] = 1000, min_cluster_size: Annotated[Optional[int], typer.Option( help='Minimum cluster size to include in output (filters smaller clusters).' @@ -953,41 +1579,62 @@ def cluster_database( help='Use GPU for FAISS operations (requires faiss-gpu).' )] = False, seed: Annotated[Optional[int], typer.Option( - help='Random seed for reproducibility.' + help='Random seed of the Leiden step: the same seed gives the same clusters; without it each run may ' + 'differ. At least 0. --method leiden only (greedy is deterministic).' )] = None, format_mode: Annotated[FormatMode, typer.Option( help=_FORMAT_MODE_HELP )] = FormatMode.headless_tsv, log_level: Annotated[LogLevel, typer.Option( - help='Number of nodes to use for inference of embeddings.' + help='Logging level.' )] = LogLevel.info ): - """Cluster database embeddings using Leiden algorithm.""" + """Cluster database embeddings: Leiden on the similarity graph, or greedy representatives in one streamed pass.""" set_log_level(log_level) + method = _refuse_leiden_only_options(method, resolution=resolution, leiden_iterations=leiden_iterations, seed=seed) + if leiden_iterations is not None and leiden_iterations < 1: + raise typer.BadParameter("--leiden-iterations must be at least 1.", param_hint="--leiden-iterations") + if seed is not None and seed < 0: + raise typer.BadParameter("--seed must be at least 0.", param_hint="--seed") + _refuse_max_neighbors_below_1(max_neighbors) + _refuse_output_over_inputs(output_file, databases=[("the database", db_path)]) if use_gpu_index: logging.info("GPU acceleration for FAISS operations: enabled") # Initialize clusterer logging.info("Initializing clusterer...") - from foldmatch.search.clustering import EmbeddingClusterer + from foldmatch.search.clustering import EmbeddingClusterer, _LEIDEN_ITERATIONS clusterer = EmbeddingClusterer( db_path=db_path, use_gpu=use_gpu_index ) - # Build similarity graph - clusterer.build_similarity_graph( - threshold=threshold, - max_neighbors=max_neighbors - ) - - # Perform clustering - clusterer.cluster_leiden( - resolution=resolution, - seed=seed - ) + if method is ClusterMethod.greedy: + # Greedy representatives, streamed: no graph + clusterer.cluster_greedy( + threshold=threshold, + max_neighbors=max_neighbors + ) + else: + # Build similarity graph + clusterer.build_similarity_graph( + threshold=threshold, + max_neighbors=max_neighbors + ) + # Leiden's modularity needs non-negative edge weights: refused on the kept scores, after the build + if clusterer.min_weight < 0: + raise typer.BadParameter(f"with --method leiden, Leiden's modularity needs non-negative edge weights, and " + f"this --threshold kept a score of {clusterer.min_weight:.4g}. Use a --threshold " + f"of 0 or more, or --method greedy.", param_hint="--threshold") + + # Perform clustering + clusterer.cluster_leiden( + resolution=1.0 if resolution is None else resolution, + n_iterations=_LEIDEN_ITERATIONS if leiden_iterations is None else leiden_iterations, + seed=seed + ) # Display statistics clusterer.print_statistics() @@ -1009,7 +1656,7 @@ def similarity_graph( help='Path to the FAISS database.' )], threshold: Annotated[float, typer.Option( - help='Similarity threshold for edge creation (0-1, where 1.0 = identical).' + help='Similarity threshold for edge creation (cosine; identical chains score 1, or close to 1 on ivf_pq, where scores are PQ estimates).' )] = 0.8, output_file: Annotated[str, typer.Option( help='Path to save the similarity graph (GraphML format).' @@ -1027,6 +1674,8 @@ def similarity_graph( """Build and export a similarity graph from database embeddings.""" set_log_level(log_level) + _refuse_max_neighbors_below_1(max_neighbors) + _refuse_output_over_inputs(output_file, databases=[("the database", db_path)]) if use_gpu_index: logging.info("GPU acceleration for FAISS operations: enabled") @@ -1048,6 +1697,20 @@ def similarity_graph( logging.info(f"Similarity graph saved to {output_path}") +@app.command( + name="download-models", + help=DOWNLOAD_MODELS_HELP +) +def download_models( + system: Annotated[Optional[List[str]], typer.Option( + "--system", + help=DOWNLOAD_MODELS_SYSTEM_HELP + )] = None +): + """The same command as ``fm-embedding download-models`` and ``inference download-models``.""" + download_models_command(system) + + def version_callback(value: bool): if value: typer.echo(f"{__version__}") @@ -1066,6 +1729,16 @@ def _app_callback( ): pass +def _refuse_output_over_inputs(output_file: str, *, files=(), databases=()): + """Refuse an --output-file that is an input of the command (P9.N D6), before any work: one of ``files`` + (described, path), or a file of a database of ``databases`` (described, --db-path) that the command reads + (``.index``, ``.metadata``, ``.invlists``, ``.sequences``, ``.rescodes.*``). Read-only; an output file of + an earlier command is not an input, even one named ``.hits.tsv``, and is replaced as before.""" + from foldmatch.utils.output_files import refuse_output_file_over_inputs + refuse_output_file_over_inputs(output_file, files=files, + databases=[(described, parse_db_path(db)[2]) for described, db in databases]) + + def _filter_results_by_threshold(results, threshold: float | None): """Filter search results by similarity threshold.""" if threshold is None: @@ -1091,6 +1764,1404 @@ def _check_comp_bias_corr_scale(scale: float): raise ValueError(f"--comp-bias-corr-scale must be between 0 and 1, got {scale}.") +# Files a database build writes next to ``--output-db``. Any of them already +# present refuses the build before ESM3 (create_db and SequenceStore.create +# would only refuse it after the whole pass). +_INDEX_SUFFIXES = (".index", ".metadata", ".invlists") +_SEQUENCE_DB_SUFFIXES = _INDEX_SUFFIXES + (".sequences",) + + +def _refuse_existing_outputs(output_db: str, suffixes, sidecar_globs=()) -> None: + """Raise ``FileExistsError`` if any output of this build already exists. Read-only.""" + # parse_db_path does not import FAISS: the build still loads it only after + # the ESM3 pass, as before. + db_folder, index_name, db_path = parse_db_path(output_db) + existing = [Path(f"{db_path}{suffix}") for suffix in suffixes] + existing = [path for path in existing if path.exists()] + for pattern in sidecar_globs: + existing.extend(sorted(db_folder.glob(f"{glob.escape(index_name)}{pattern}"))) + if existing: + raise FileExistsError( + f"Database already exists at {db_path} " + f"(found: {', '.join(str(path) for path in existing)}). " + f"Choose a different --output-db, or delete the existing files first." + ) + + +def _fasta_preflight(fasta_file: Path, min_res_n: int, strict: bool = False, output_db: Optional[str] = None): + """Read-only checks every rank runs before ESM3 is loaded. + + Every rank reaches the same verdict from the same files, so a refusal + stops all ranks together instead of one rank failing after the pass. + For a build (``output_db``), existing outputs are refused. The FASTA scan + (:func:`preflight_fasta`) counts the entries the pass must produce; + ``strict`` refuses duplicate ids, '<'/'>' and non-ASCII sequences, which + otherwise are logged as warnings and processed as before. + """ + if output_db is not None: + _refuse_existing_outputs(output_db, _SEQUENCE_DB_SUFFIXES, sidecar_globs=(".rescodes.*",)) + from foldmatch.utils.sequence import preflight_fasta + preflight = preflight_fasta(fasta_file, min_res_n) + preflight.check(strict=strict) + if output_db is not None: + _refuse_nothing_to_build(fasta_file, min_res_n, preflight) + return preflight + + +def _refuse_nothing_to_build(fasta_file: Path, min_res_n: int, preflight) -> None: + """Raise ``ValueError`` when the pre-flight kept no entry of the FASTA: there is nothing to build.""" + if preflight.entries_kept == 0: + raise ValueError( + f"{fasta_file} has no sequence with at least {min_res_n} residues " + f"({preflight.entries} entries in all): there is nothing to build." + ) + + +def _check_store_matches_preflight(store, preflight, output_db: str) -> None: + """The sequence store must hold exactly what the pre-flight counted. + + A mismatch means the FASTA changed during the build. The database is + removed rather than left disagreeing with itself: the store and the index + files this build wrote (the pre-flight refused any that existed before), so + the same build can be run again. + """ + meta = store.meta() + expected = { + "entry_count": preflight.unique_entries, + "total_residues": preflight.unique_residues, + "duplicates_ignored": preflight.duplicates, + } + if meta != expected: + removed = _remove_outputs(output_db, _SEQUENCE_DB_SUFFIXES) + raise ValueError( + f"Sequence store built from {preflight.path} ({meta}) does not match its " + f"pre-flight ({expected}); was the FASTA modified during the build? " + f"The database files it wrote were removed ({', '.join(path.name for path in removed)})." + ) + + +def _remove_outputs(output_db: str, suffixes) -> List[Path]: + """Delete this build's ``{output_db}{suffix}`` files; returns the ones removed.""" + _, _, db_path = parse_db_path(output_db) + removed = [] + for suffix in suffixes: + path = Path(f"{db_path}{suffix}") + if path.exists(): + path.unlink() + removed.append(path) + return removed + + +@contextmanager +def _refusals(note: Optional[str] = None): + """Report a refused check of the block as a :class:`RefusalError` (D20: exit status 1). + + The check's own exception (a ``ValueError``, an ``OSError`` such as a + missing or existing file or a full disk, or an SQLite file that is not a + database) becomes its ``__cause__``; ``note`` is appended to the message. + """ + try: + yield + except RefusalError as exc: + if note is None: + raise + raise RefusalError(f"{exc} {note}") from exc + except (ValueError, OSError, sqlite3.DatabaseError) as exc: + raise RefusalError(str(exc) if note is None else f"{exc} {note}") from exc + + +@dataclass(frozen=True) +class _IndexOptions: + """The FAISS options of a sequence build's index phase, as ``build sequences`` takes them.""" + use_gpu_index: bool + index_type: IndexType + ivf_nlist: int + ivf_nprobe: Optional[int] + load_batch_size: int + + def index_config(self) -> IndexConfig: + """A fresh ``IndexConfig`` for each build: an ivf_pq build sets its ``nprobe``.""" + return IndexConfig(nlist=self.ivf_nlist, nprobe=self.ivf_nprobe) + + +@dataclass(frozen=True) +class _SequencePass: + """What every rank of a sequence build has after its pre-flight: the FASTA pre-flight, the embedding computer + (this run's id and shards), the prepared scoring system (``--residue-codes``; else ``None``) and the mode.""" + preflight: Any + embedding_computer: Any + prepared: Any + residue_codes: bool + + @property + def residue(self) -> dict: + """The residue keywords of the pass: the codes and seq_sha1 go into each shard under ``--residue-codes``; + a plain build calls the pass exactly as before (none).""" + if not self.residue_codes: + return {} + from foldmatch.search.residue_shards import RESIDUE_CODES + return {"residue_output": RESIDUE_CODES, "scoring_system": self.prepared} + + def refusals(self): + """The refusal wrapper of this mode: :func:`_refusals` under ``--residue-codes`` (D20), else none.""" + return _refusals() if self.residue_codes else nullcontext() + + +def _sequence_build_preflight(fasta_file: Path, min_res_n: int, output_db: str, shards_folder: Path, num_nodes: int, + residue_codes: bool) -> _SequencePass: + """Every rank's read-only checks of a sequence build, before ESM3; returns the :class:`_SequencePass`. + + A plain build runs the FASTA pre-flight with the outputs-exist check and draws this run's id, raising what it + always raised; a ``--residue-codes`` build runs :func:`_residue_codes_preflight` (every refusal a + :class:`RefusalError`). + """ + if residue_codes: + preflight, embedding_computer, prepared = _residue_codes_preflight( + fasta_file, min_res_n, output_db, shards_folder, num_nodes + ) + else: + preflight = _fasta_preflight(fasta_file, min_res_n, output_db=output_db) + + from foldmatch.search.embedding_computer import EmbeddingComputer + embedding_computer = EmbeddingComputer( + embedding_folder=shards_folder, + ) + prepared = None + return _SequencePass(preflight, embedding_computer, prepared, residue_codes) + + +def _run_sequence_pass(run: _SequencePass, *, fasta_file: Path, min_res_n: int, batch_size: int, num_workers: int, + num_nodes: int, accelerator, devices, strategy) -> None: + """Every rank: the ESM3 pass of a sequence build into this run's shards, ending at the pass barrier + (``compute_from_fasta``). The caller wraps it in ``run.refusals()``.""" + run.embedding_computer.compute_from_fasta( + fasta_file=fasta_file, + min_res_n=min_res_n, + batch_size=batch_size, + num_workers=num_workers, + num_nodes=num_nodes, + accelerator=accelerator, + devices=arg_devices(devices), + strategy=strategy, + preflight=run.preflight, + **run.residue, + ) + + +#: ``omp_env_at_start`` of an index phase whose command has no pass (``build sequences-index``): nothing in the process +#: changes ``OMP_NUM_THREADS`` before it. +_OMP_ENV_NOT_RECORDED = object() + +#: The faiss-cpu version pyproject.toml pins, named by the index phase's line about a single-threaded FAISS BLAS: its +#: Linux wheels bundle an OpenMP OpenBLAS (the 1.13.2 wheels' OpenBLAS was single-threaded). +_FAISS_CPU_PIN = "1.15.1" + + +def _in_faiss_folder(path: str, package: str) -> bool: + """Whether the library at ``path`` (a real path) is one FAISS ships: inside the ``faiss`` package folder + ``package``, or in a wheel's library folder beside it (``faiss_cpu.libs``).""" + folder = os.path.dirname(path) + return path.startswith(package + os.sep) or ( + os.path.dirname(folder) == os.path.dirname(package) + and os.path.basename(folder).startswith("faiss") and folder.endswith(".libs")) + + +def _faiss_openmp_blas(runtimes) -> Optional[str]: + """The file name of FAISS's own BLAS among threadpoolctl's ``runtimes`` when it is an OpenMP build (the faiss-cpu + 1.15.1 wheel's ``libopenblaso``), else None. Such a BLAS runs the OpenMP threads FAISS runs, whatever + ``OPENBLAS_NUM_THREADS`` says.""" + import faiss + package = os.path.dirname(os.path.realpath(faiss.__file__)) + for runtime in runtimes: + path = os.path.realpath(str(runtime.get("filepath"))) + if runtime.get("user_api") == "blas" and runtime.get("threading_layer") == "openmp" \ + and _in_faiss_folder(path, package): + return os.path.basename(path) + return None + + +def _build_sequence_database(embedding_computer, preflight, *, fasta_file: Path, output_db: str, min_res_n: int, + system, index: _IndexOptions, refusals, split: Optional[Mapping] = None, + after_sequence_store=None, omp_env_at_start=_OMP_ENV_NOT_RECORDED) -> Optional[dict]: + """The index phase of a sequence build, in one process: the FAISS index from the run's shards, the sequence + store from the FASTA, and with ``system`` the residue-code store; returns the code store's manifest, else + ``None``. ``omp_env_at_start`` is ``OMP_NUM_THREADS`` as the command found it, before its pass + (:func:`_log_index_phase`). + + ``refusals`` is the mode's refusal wrapper (``_SequencePass.refusals``). ``after_sequence_store``, if given, is + called once the sequence store has passed :func:`_check_store_matches_preflight`, before the code store. + ``split`` is the run record of a split build (``build sequences-index``); ``None`` for ``build sequences``. With + it, a refusal of the FAISS index says that step 2 can be rerun with other index options (I21), and the code + store's manifest says the build was split (D26). + """ + from foldmatch.search.embedding_database import EmbeddingDatabase + embedding_db = EmbeddingDatabase( + db_path=output_db + ) + _log_index_phase(omp_env_at_start) + with refusals() if split is None else _refusals( + note=f"The shards and the record of run {split['run_id']} are kept: 'build sequences-index' can be " + f"rerun with other index options, without a new ESM3 pass."): + start = time.monotonic() + embedding_db.create_db( + embedding_batches=embedding_computer.get_embedding_batches(load_batch_size=index.load_batch_size), + use_gpu_index=index.use_gpu_index, + index_type=index.index_type, + index_config=index.index_config(), + ) + logging.info(f"FAISS index built in {time.monotonic() - start:.1f} s") + + # Persist a sidecar id->sequence store so this database can be the + # subject (or query) of a Stage-2 pairwise-alignment search. Built + # directly from the FASTA — the DB ids ARE the FASTA headers — using the + # same min_res_n filter so store and index stay in lock-step. Rank 0 only, + # after the pass, so ranks never race on the SQLite file. + from foldmatch.search.sequence_store import SequenceStore + store = SequenceStore(embedding_db.db_folder, embedding_db.index_name) + with refusals(): + start = time.monotonic() + store.create( + fasta_file=fasta_file, + min_res_n=min_res_n, + ) + logging.info(f"Sequence store built in {time.monotonic() - start:.1f} s") + _check_store_matches_preflight(store, preflight, output_db) + if after_sequence_store is not None: + after_sequence_store() + if system is None: + return None + # Last, once the database it describes is complete; its manifest is + # written last of all. A refusal leaves the database (BLOSUM62) and the + # shards (for --from-shards) in place. + start = time.monotonic() + manifest = _build_residue_code_store( + embedding_computer, embedding_db, store, preflight, system, + min_res_n=min_res_n, load_batch_size=index.load_batch_size, split=split is not None, + ) + logging.info(f"Residue-code store built in {time.monotonic() - start:.1f} s") + return manifest + + +def _log_index_phase(omp_env_at_start=_OMP_ENV_NOT_RECORDED) -> None: + """Log the threads the index phase gets (FAISS's OpenMP, the CPUs this process may use, each thread-pool runtime) + and warn, except on macOS (where ``foldmatch`` pins ``OMP_NUM_THREADS=1``), when FAISS has fewer threads than + the usable physical cores, naming the cause. Diagnostics only: nothing is changed. + + ``omp_env_at_start`` is ``OMP_NUM_THREADS`` when the command started (``None``: unset). When it was unset then and + is set now, the pass set it: Lightning's DDP launcher does so for a multi-device run (to the allowed CPUs // the + devices per node, also for ranks srun starts), and FAISS, first loaded here, starts with that value. + + One more line (a WARNING, INFO on macOS) when FAISS's BLAS runs on one thread by build: a BLAS runtime at one + thread whose threading layer is ``disabled``, or one in FAISS's own library folder (``faiss_cpu.libs`` beside the + package, or inside it) whose threading layer threadpoolctl does not report as threaded. The faiss-cpu 1.13.2 + wheel's OpenBLAS is such a library (its ivf_pq training and adds ran on one core); the OpenMP OpenBLAS of the + 1.15.1 wheel FoldMatch pins (:data:`_FAISS_CPU_PIN`) is not. + """ + import faiss + from foldmatch.search.code_alignment import _physical_cpus + omp_threads = faiss.omp_get_max_threads() + cpus = os.cpu_count() or 1 + try: + allowed = len(os.sched_getaffinity(0)) + except AttributeError: # no affinity masks (macOS) + allowed = cpus + physical = _physical_cpus() + omp_env = os.environ.get("OMP_NUM_THREADS") + set_by_pass = omp_env_at_start is None and omp_env is not None + logging.info( + f"Index phase: FAISS {faiss.__version__}, {omp_threads} OpenMP thread(s); {allowed} of {cpus} CPUs allowed " + f"({physical} physical cores); OMP_NUM_THREADS={omp_env if omp_env is not None else 'unset'}" + + (" (set during the ESM3 pass by Lightning: the allowed CPUs // devices per node; unset at the start)" + if set_by_pass else "") + ) + try: + import threadpoolctl + runtimes = threadpoolctl.threadpool_info() + except Exception as exc: # diagnostics only: never fail a build over them + logging.debug(f"threadpoolctl could not list the thread pools: {exc}") + runtimes = [] + for runtime in runtimes: + logging.info( + f"Thread pool: {runtime.get('internal_api')}, {runtime.get('num_threads')} thread(s), " + f"{os.path.basename(str(runtime.get('filepath')))}" + ) + # The k-means of an ivf_pq training and the IVF assignment of its adds are BLAS products: on a BLAS built without + # threads they run on one core whatever FAISS's OpenMP threads. A threaded BLAS that a variable limits to one + # thread (the faiss-cpu 1.15.1 wheel's OpenMP OpenBLAS at OMP_NUM_THREADS=1, for example) is not flagged. + package = os.path.dirname(os.path.realpath(faiss.__file__)) + for runtime in runtimes: + path = os.path.realpath(str(runtime.get("filepath"))) + own = _in_faiss_folder(path, package) + layer = runtime.get("threading_layer") + if runtime.get("user_api") == "blas" and runtime.get("num_threads") == 1 and ( + layer == "disabled" or (own and layer not in ("pthreads", "openmp"))): + logging.log( + logging.INFO if sys.platform == "darwin" else logging.WARNING, + f"FAISS's BLAS, {os.path.basename(path)}, is single-threaded, so the ivf_pq training and adds run on " + f"one core whatever FAISS's thread count: install the faiss-cpu {_FAISS_CPU_PIN} FoldMatch pins (pip " + f"install faiss-cpu=={_FAISS_CPU_PIN}), whose wheel bundles an OpenMP OpenBLAS, or a FAISS built on " + f"MKL (README, 'FAISS threads')." + ) + break + if sys.platform != "darwin" and omp_threads < physical: + # With OMP_NUM_THREADS unset, OpenMP starts one thread per CPU of the affinity mask, never fewer than its + # physical cores: fewer threads come from the variable, or from a limit set inside this process or by another + # OpenMP variable (not from the mask). + if set_by_pass: + cause = (f"OMP_NUM_THREADS={omp_env}, which Lightning's DDP launcher set during the ESM3 pass (it was " + f"unset at the start; Lightning sets it to the allowed CPUs // devices per node)") + remedy = (" To choose FAISS's threads, export OMP_NUM_THREADS before the run (it also sets each rank's " + "torch threads in the pass), or build in two steps ('build sequences-embed', then 'build " + "sequences-index' as its own job).") + elif omp_env is not None: + cause, remedy = f"OMP_NUM_THREADS={omp_env}", "" + else: + cause, remedy = ("an OpenMP thread limit set inside this process or by another OpenMP variable " + "(OMP_NUM_THREADS is unset)"), "" + logging.warning( + f"FAISS has {omp_threads} OpenMP thread(s) for {physical} usable physical cores, because of {cause}: " + f"the index phase may be slower than this node allows.{remedy}" + ) + + +# FaissEmbeddingDatabase.N_EMBEDDINGS_DB_THR, where --index-type auto switches from flat to hnsw (the module imports +# FAISS, which step 1 never needs). +_AUTO_HNSW_ENTRIES = 10000 + + +def _fasta_stat_or_error(fasta_file: Path, when: str): + """``(FastaStat, None)``, or ``(None, error text)`` when the FASTA cannot be stat'ed: step 1 keeps a rank's stat + error in its facts for rank 0 to refuse (E11), so that every rank still reaches the gather.""" + from foldmatch.search.split_run import fasta_stat + try: + return fasta_stat(fasta_file), None + except OSError as exc: + return None, f"stat {when}: {exc}" + + +def _gather_rank_facts(facts: dict) -> List[dict]: + """Every rank's ``facts``, in rank order: ``torch.distributed.all_gather_object`` when the pass ran in a process + group (it outlives the pass, whose barrier every rank has just left; under NCCL the objects travel on the device + Lightning set for the rank), else ``[facts]``. ``build sequences-embed`` calls it on every rank right after the + pass, with nothing before it that can raise, so no rank waits in it for a rank that refused.""" + import torch.distributed as dist + if not (dist.is_available() and dist.is_initialized()): + return [facts] + gathered = [None] * dist.get_world_size() + dist.all_gather_object(gathered, facts) + return gathered + + +def _record_embed_run(run: _SequencePass, facts: List[dict], *, fasta_file: Path, output_db: str, shards_folder: Path, + min_res_n: int, options: dict) -> Tuple[dict, Path]: + """Rank 0 of ``build sequences-embed``, after the pass: the post-pass checks, then the run record, written last; + returns ``(record, path)``. + + The world is the ranks that gathered their ``facts`` (:func:`_gather_rank_facts`). Refused, each leaving no record + and the shards in place: the embedding count and the deal of the kept entries over those ranks (E9), the shard + footers (E10), the FASTA, hashed here, against every rank's stat bracket, size and pre-flight counts (E11), and + the record's own staging, check and commit (E12). From E11 on, the refusal also names the ``build sequences-seal`` + command that records the shards once the cause is fixed. + """ + from foldmatch.search import split_run + from foldmatch.search.residue_shards import RESIDUE_CODES, pass_record + computer = run.embedding_computer + run_id = computer.run_id + world_size = len(facts) + seconds = [rank_facts["seconds"] for rank_facts in facts] + logging.info(f"Pre-flight on {world_size} rank(s): min {min(seconds):.1f} s, max {max(seconds):.1f} s") + kept = f"No run record was written; the shards are kept in {shards_folder} as run {run_id}." + seal = shlex.join(["fm-search", "build", "sequences-seal", "--fasta-file", str(fasta_file), "--from-shards", + str(shards_folder), "--run-id", run_id, "--min-res-n", str(min_res_n)]) + with _refusals(note=kept): + computer.check_embedding_count() + paths = computer.shard_files() + shards = split_run.shard_entries(paths) + _check_shards_are_the_deal(shards, run.preflight.entries_kept, world_size, shards_folder, run_id) + if run.residue_codes: + run_pass = split_run.check_codes_footers(paths, system=run.prepared.system, min_res_n=min_res_n, + world_size=world_size) + else: + from foldmatch.utils.esm.loaders import ESM_REVISION + split_run.check_plain_footers(paths) + run_pass = pass_record(min_res_n=min_res_n, esm_revision=ESM_REVISION, world_size=world_size) + with _refusals(note=f"{kept} Once the cause is fixed, 'build sequences-seal' can record them: {seal}"): + sha256, hash_seconds = split_run.fasta_sha256(fasta_file) + final = split_run.fasta_stat(fasta_file) + logging.info(f"FASTA sha256 of {fasta_file} ({final.bytes} bytes) in {hash_seconds:.1f} s") + _check_rank_fasta_facts(fasta_file, facts, final) + system = None if run.prepared is None else run.prepared.system + record = { + "kind": split_run.RUN_RECORD_KIND, + "record_version": split_run.RUN_RECORD_VERSION, + "run_id": run_id, + "command": split_run.EMBED_COMMAND, + "sealed": False, + "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "foldmatch_version": __version__, + "fasta": {"path": str(Path(fasta_file).resolve()), "bytes": final.bytes, "mtime_ns": final.mtime_ns, + "sha256": sha256}, + "min_res_n": min_res_n, + "strict": run.residue_codes, + "preflight": split_run.preflight_counts(run.preflight), + # The user's string is what step 2 gives create_db; the record holds the path it names. + "output_db": os.path.abspath(parse_db_path(output_db)[2]), + "residue_output": RESIDUE_CODES if run.residue_codes else None, + "system": None if system is None else { + "name": system.name, "digest": system.digest(), "encoding_digest": system.encoding_digest(), + }, + "world_size": world_size, + "pass": run_pass, + "options": options, + "shards": shards, + "rows": sum(shard["rows"] for shard in shards), + } + path = split_run.write_run_record(shards_folder, record) + return record, path + + +def _check_shards_are_the_deal(shards: List[dict], entries: int, world_size: int, folder: Path, run_id: str) -> None: + """Refuse (E9) unless the shards' ranks and rows are the pass's deal of ``entries`` over ``world_size`` ranks + (:func:`~foldmatch.search.split_run.dealt_rows`): a rank of the world that wrote no shard, a shard of a rank the + deal gives nothing, or a rank with other rows.""" + from foldmatch.search.split_run import dealt_rows + deal = dealt_rows(entries, world_size) + found = {shard["rank"]: shard["rows"] for shard in shards} + if found == deal: + return + missing = sorted(set(deal) - set(found)) + extra = sorted(set(found) - set(deal)) + other = [rank for rank in sorted(set(deal) & set(found)) if found[rank] != deal[rank]] + + def listed(ranks) -> str: + return ", ".join(map(str, ranks[:10])) + (", ..." if len(ranks) > 10 else "") + + problems = [] + if missing: + problems.append(f"no shard of rank(s) {listed(missing)}") + if extra: + problems.append(f"a shard of rank(s) {listed(extra)}, which the deal gives no entry") + problems.extend(f"rank {rank} has {found[rank]} row(s), not {deal[rank]}" for rank in other[:10]) + raise RefusalError( + f"The {len(shards)} shard(s) of run {run_id} in {folder} are not the deal of its {entries} entries over the " + f"{world_size} rank(s) of the pass ({'; '.join(problems)}): the shards cannot be indexed." + ) + + +def _check_rank_fasta_facts(fasta_file: Path, facts: List[dict], final) -> None: + """Refuse (E11) unless every rank read the same, unchanged FASTA: no rank's stat failed; rank 0's stat before its + pre-flight equals ``final`` (its stat after hashing), and every other rank's equals its stat after the pass; every + rank's size and pre-flight counts are rank 0's. A rank may read a node-local copy (another inode): each copy is + bracketed on its own rank.""" + from foldmatch.search.split_run import PREFLIGHT_KEYS + + errors = [f"rank {rank}: {rank_facts['error']}" for rank, rank_facts in enumerate(facts) if rank_facts["error"]] + if errors: + raise RefusalError( + f"The FASTA {fasta_file} could not be stat'ed on every rank ({'; '.join(errors[:10])}), so it cannot be " + f"shown unchanged during the pass." + ) + changed = [(0, facts[0]["before"], final)] if facts[0]["before"] != final else [] + changed.extend((rank, rank_facts["before"], rank_facts["after"]) for rank, rank_facts in enumerate(facts) + if rank > 0 and rank_facts["before"] != rank_facts["after"]) + if changed: + raise RefusalError( + f"The FASTA was modified during the pass (" + + "; ".join(f"rank {rank}: before {_stat_text(a)}, after {_stat_text(b)}" for rank, a, b in changed[:10]) + + f"): the shards may not be the embeddings of {fasta_file} as it is now." + ) + first = facts[0] + differ = [] + for rank, rank_facts in enumerate(facts[1:], 1): + if rank_facts["size"] != first["size"]: + differ.append(f"rank {rank}: {rank_facts['size']} bytes, rank 0: {first['size']}") + counts = [key for key in PREFLIGHT_KEYS if rank_facts["preflight"].get(key) != first["preflight"].get(key)] + if counts: + differ.append(f"rank {rank}: " + ", ".join(f"{key} {rank_facts['preflight'].get(key)}" for key in counts) + + ", rank 0: " + ", ".join(f"{key} {first['preflight'].get(key)}" for key in counts)) + if differ: + raise RefusalError( + f"The ranks did not read the same FASTA ({'; '.join(differ[:10])}): {fasta_file} was modified during the " + f"pass, or the ranks' copies of it differ." + ) + + +def _stat_text(stat) -> str: + """A :class:`~foldmatch.search.split_run.FastaStat` as a refusal shows it.""" + return f"{stat.bytes} bytes, mtime_ns {stat.mtime_ns}, inode {stat.ino}" + + +def _refuse_multi_node_job(command: str) -> List[str]: + """The launch checks of a command that runs as one process (``build sequences-index``, ``build sequences-seal``), + from the launch environment alone; returns the WARNINGs, for the caller to log. + + Refused: a process that is one of several ranks (I2, :func:`~foldmatch.utils.dist.multi_rank_launch`, which names + the variable), and the batch step of a SLURM job of more than one node (I3, D34): the job's other nodes would stay + allocated while one process works. A one-task srun step of such a job, and a one-node job with GPUs, get a + WARNING. The advice never makes the command's job depend on this job, which it refuses (its ``afterok`` would + never be met) or in which it keeps running: it is submitted once this job has ended, or next time from the GPU + job's script before its work, with ``$SLURM_JOB_ID`` left for that script's shell. It ends with the command line to + run there (:func:`_command_line`, after a ``cd`` to this process's folder when the line depends on it), on a line of + its own after :data:`_COMMAND_INDENT`, so that it can be selected and copied whole; next time with the paths and + run id of the GPU job's build, whose run id step 1 derives from that job unless ``FOLDMATCH_RUN_ID`` sets it, + exported before the sbatch so that the CPU job has it too. + + It runs before ``set_log_level`` and before torch or Lightning is imported, which takes minutes from a cold file + system on a compute node (P8b, N3 case E: I3 refused 118 s after the launch, after ``set_log_level``'s Lightning + import); a refusal needs no logging (``main`` prints it). The checks import only :mod:`foldmatch.utils.dist`, + which imports only the standard library. The WARNINGs are returned instead of logged, so that they go through the + logging ``set_log_level`` sets up. + """ + from foldmatch.utils.dist import _whole_number, multi_rank_launch, srun_step_id + launch = multi_rank_launch() + if launch is not None: + raise RefusalError( + f"'{command}' runs as one process ({launch}): launch it once, directly in the batch script or with " + f"srun -n 1. The command:{_COMMAND_INDENT}{_command_line(command)}" + ) + job = os.environ.get("SLURM_JOB_ID", "").strip() + if not job: + return [] + nodes = _whole_number("SLURM_JOB_NUM_NODES") + if nodes is not None and nodes > 1: + chained = ("submit it from the GPU job's script before the job's work starts: sbatch -C cpu -N 1 " + "--dependency=afterok:$SLURM_JOB_ID --kill-on-invalid-dep=yes ... SCRIPT") + build = ("with the paths and run id of that job's build (export FOLDMATCH_RUN_ID=ID before the sbatch, and pass " + "--run-id \"$FOLDMATCH_RUN_ID\")") + line = _command_line(command, in_another_job=True) + if srun_step_id() is None: + raise RefusalError( + f"SLURM job {job} holds {nodes} nodes, and '{command}' runs in one process on one of them: the others " + f"would stay allocated until it ends. Run it as its own one-node CPU job instead: once this job has " + f"ended, sbatch -C cpu -N 1 ... SCRIPT (no dependency is needed then); next time, {chained}. SCRIPT " + f"runs this command, without srun, next time {build}:{_COMMAND_INDENT}{line}" + ) + return [ + f"'{command}' runs in a one-task srun step of SLURM job {job}, which holds {nodes} nodes: the others stay " + f"allocated until it ends. Next time, run it as its own one-node CPU job: {chained}. SCRIPT runs this " + f"command, without srun, {build}:{_COMMAND_INDENT}{line}" + ] + gpus = [f"{name}={os.environ[name].strip()}" for name in ("SLURM_GPUS_ON_NODE", "SLURM_JOB_GPUS") + if os.environ.get(name, "").strip()] + if gpus: + return [ + f"SLURM job {job} has GPUs ({', '.join(gpus)}), which '{command}' does not use: run it as its own job on a " + f"CPU node (sbatch -C cpu -N 1 ...); a project with GPU hours only may run it on one GPU node, where " + f"ivf_pq fits (flat and hnsw may not)." + ] + return [] + + +#: What comes before the command line that the launch checks' advice ends with: a line of its own, indented, under the +#: ``Error:`` line of a refusal or the log line of a WARNING (:func:`_refuse_multi_node_job`). +_COMMAND_INDENT = "\n " + +#: The options of 'build sequences-index' and 'build sequences-seal' that name a path (:func:`_command_line`). +_PATH_OPTIONS = ("--fasta-file", "--output-db", "--from-shards") + + +def _command_line(command: str, *, in_another_job: bool = False) -> str: + """This process's command line, to run ``command`` (``build sequences-index``, ...) again elsewhere: the + arguments of ``sys.argv``, as given, after the program as it was invoked, shell-quoted (:func:`shlex.join`). + + Through the ``fm-search`` console script the program is ``fm-search``. Otherwise it is what ``sys.orig_argv`` + holds before those arguments: ``python -m foldmatch.cli.search`` with the interpreter and its options as typed, + ``python path/to/script.py``, or ``python -c CODE``; ``fm-search`` when ``sys.orig_argv`` does not end with them. + When ``sys.argv`` does not run ``command`` (the app called in-process), ``fm-search ...``. + + ``in_another_job``: the line is for another job, which sbatch starts in the folder it is submitted from, so it + starts with ``cd && `` when it depends on that folder: a relative ``--fasta-file``, + ``--output-db`` or ``--from-shards``, or a program other than ``fm-search`` (the interpreter, the script or + ``-m``'s module may be found from it). Not when that folder is gone (``os.getcwd`` fails). + + An argument holding a newline keeps it, inside its quotes: the line still runs, but it is more than one line. + """ + words = command.split() + argv = [str(argument) for argument in getattr(sys, "argv", ())] + arguments = argv[1:] + if arguments[:len(words)] != words: + return shlex.join(["fm-search", *words]) + " ..." + program = ["fm-search"] + if Path(argv[0]).name != "fm-search": + invoked = [str(argument) for argument in getattr(sys, "orig_argv", ())] + start = len(invoked) - len(arguments) + if start > 0 and invoked[start:] == arguments: + program = invoked[:start] + line = shlex.join(program + arguments) + if in_another_job and (program != ["fm-search"] or _relative_path_option(arguments)): + try: + line = f"cd {shlex.quote(os.getcwd())} && {line}" + except OSError: + pass + return line + + +def _relative_path_option(arguments: List[str]) -> bool: + """Whether ``arguments`` give one of :data:`_PATH_OPTIONS` a relative path (``--option PATH`` or + ``--option=PATH``).""" + for k, argument in enumerate(arguments): + option, equals, value = argument.partition("=") + if option in _PATH_OPTIONS: + if not equals: + value = arguments[k + 1] if k + 1 < len(arguments) else "" + if not os.path.isabs(value): + return True + return False + + +def _split_code_store(record: dict, record_path: Path, *, residue_codes: Optional[bool], + cleanup_embeddings: bool) -> Tuple[bool, Any]: + """``(codes, system)``: whether ``build sequences-index`` builds the residue-code store, and the registry entry of + the run's system when it does (else ``None``). + + ``--residue-codes`` unset follows the run (D24). Refused: ``--residue-codes`` on a run without codes (I8); + ``--no-residue-codes --cleanup-embeddings`` on a run with codes (I9), which would delete the only input of a later + ``build residue-codes --from-shards``; a run system this FoldMatch does not register, or registers with other + digests (I10). Only the registry entry is needed: no bundle. + """ + from foldmatch.search import scoring_system + from foldmatch.search.residue_shards import RESIDUE_CODES + run_id = record["run_id"] + run_codes = record["residue_output"] == RESIDUE_CODES + codes = run_codes if residue_codes is None else residue_codes + if codes and not run_codes: + raise RefusalError( + f"Run {run_id} computed no residue codes (its record {record_path} has no residue output), so there is no " + f"residue-code store to build from its shards: run step 1 with --residue-codes ('build sequences-embed " + f"--residue-codes'), or build without --residue-codes." + ) + if run_codes and not codes and cleanup_embeddings: + raise RefusalError( + f"--no-residue-codes with --cleanup-embeddings would delete the only copy of run {run_id}'s residue codes " + f"(its shards in {record_path.parent}), which 'build residue-codes --from-shards' needs to add the " + f"residue-code store later. Drop --cleanup-embeddings, or build the store too." + ) + if not codes: + return False, None + recorded = record["system"] + plain = "--no-residue-codes builds the database without its residue codes." + with _refusals(note=f"Run {run_id}'s residue codes are of {recorded['name']}: {plain}"): + system = scoring_system.get(recorded["name"]) + if (system.digest(), system.encoding_digest()) != (recorded["digest"], recorded["encoding_digest"]): + raise RefusalError( + f"Run {run_id}'s residue codes are of {recorded['name']} with digest {recorded['digest']} and " + f"encoding_digest {recorded['encoding_digest']}, but this FoldMatch's {system.name} has digest " + f"{system.digest()} and encoding_digest {system.encoding_digest()}: they are another FoldMatch's codes of " + f"that name. {plain}" + ) + return True, system + + +@contextmanager +def _database_folder(output_db: str): + """The folder of ``--output-db``, created with its parents when it is missing (as the builders of ``build + sequences`` create it), so that the build lock can be taken there; the folders created here are removed again, + if still empty, when the block raises.""" + folder = Path(parse_db_path(output_db)[0]) + created = [] + for parent in (folder, *folder.parents): + if parent.exists(): + break + created.append(parent) + with _refusals(): + folder.mkdir(parents=True, exist_ok=True) + try: + yield folder + except BaseException: + for path in created: # the deepest first + try: + path.rmdir() + except OSError: + break + raise + + +def _split_index_checks(record: dict, *, fasta_file: Path, output_db: str, from_shards: Path, codes: bool, + system) -> Tuple[List[Path], Any, Any]: + """The checks ``build sequences-index`` makes inside its build lock, before any database file is written; returns + ``(shard paths, pre-flight, FASTA stat)``. + + In this order: no output exists (I12, with the ``build residue-codes`` hint when only the code store is missing); + the run's shards are the recorded ones, of the recorded size and rows (I13, I14); their footers (I15): the codes + of ``system``, or under ``--no-residue-codes`` the codes of the record's system (self-consistency only, so that a + system unknown here still gives the plain database), or plain shards; the FASTA's size (I16, not hashed when it + differs), then its sha256 (I17), against the record; its pre-flight, with the record's ``min_res_n`` and + strictness (I18), and the counts the record holds (I19); the id of every shard row at its dealt position (I20). + The stat taken before the FASTA is read brackets this build's FASTA reads with :func:`_fasta_unchanged_guard`. + """ + from foldmatch.search import split_run + from foldmatch.search.residue_shards import RESIDUE_CODES + run_id = record["run_id"] + command = split_run.code_store_command(parse_db_path(output_db)[2], record, from_shards) + with _refusals(note=None if command is None else + f"The database is complete but for its residue-code store: '{command}' adds it."): + _refuse_existing_outputs(output_db, _SEQUENCE_DB_SUFFIXES, sidecar_globs=(".rescodes.*",)) + paths = split_run.check_run_shards(from_shards, record) + footing = {"min_res_n": record["min_res_n"], "world_size": record["world_size"]} + if codes: + split_run.check_codes_footers(paths, system=system, **footing) + elif record["residue_output"] == RESIDUE_CODES: + split_run.check_codes_footers(paths, expect=record["system"], **footing) + else: + split_run.check_plain_footers(paths) + + fasta = record["fasta"] + stat = split_run.fasta_stat(fasta_file) + if stat.bytes != fasta["bytes"]: + raise RefusalError( + f"{fasta_file} has {stat.bytes} bytes, but the FASTA run {run_id} embedded ({fasta['path']}) had " + f"{fasta['bytes']}: it is not the FASTA of this run." + ) + sha256, seconds = split_run.fasta_sha256(fasta_file) + logging.info(f"FASTA sha256 of {fasta_file} ({stat.bytes} bytes) in {seconds:.1f} s") + if sha256 != fasta["sha256"]: + raise RefusalError( + f"{fasta_file} (sha256 {sha256}) is not the FASTA run {run_id} embedded ({fasta['path']}, sha256 " + f"{fasta['sha256']}), though it has its size." + ) + moved = [what for what, differs in (("path", str(Path(fasta_file).resolve()) != fasta["path"]), + ("modification time", stat.mtime_ns != fasta["mtime_ns"])) if differs] + if moved: + logging.info( + f"{fasta_file} has the size and sha256 of the FASTA run {run_id} embedded, with another " + f"{' and '.join(moved)} (recorded: {fasta['path']}, mtime_ns {fasta['mtime_ns']})" + ) + preflight = _fasta_preflight(fasta_file, record["min_res_n"], strict=record["strict"], output_db=output_db) + counts = split_run.preflight_counts(preflight) + differ = [key for key in split_run.PREFLIGHT_KEYS if counts[key] != record["preflight"][key]] + if differ: + raise RefusalError( + f"The pre-flight of {fasta_file} counts " + ", ".join(f"{key} {counts[key]}" for key in differ) + + f", but run {run_id}'s record holds " + ", ".join(f"{key} {record['preflight'][key]}" for key in differ) + + f" for the same bytes: this FoldMatch's pre-flight counts differently from the one that ran step 1 " + f"(FoldMatch {record['foldmatch_version']}). Run step 1 again with this FoldMatch." + ) + seconds = split_run.check_dealt_ids(paths, record, preflight) + logging.info(f"Ids of the {record['rows']} shard rows checked against {fasta_file} in {seconds:.1f} s") + return paths, preflight, stat + + +def _log_split_build(record: dict, *, db_path: str, from_shards: Path, shards: int, codes: bool) -> None: + """The start of ``build sequences-index``'s index phase: a WARNING when the run was embedded by another FoldMatch + (D30) or its record was sealed, an INFO line when the database goes elsewhere than the recorded ``--output-db`` + (D28), then the run it is built from. + + The FoldMatch of the pass is the record's ``foldmatch_version`` for a record step 1 wrote. A sealed record holds + the seal's version there, and for a plain run also in its pass record, which the seal filled from its own process; + only residue-code shards record their pass's version (``pass.foldmatch_version``). For a sealed plain run that + version is unknown, and the sealed WARNING says so.""" + run_id = record["run_id"] + sealed = record["sealed"] + if not sealed: + pass_version = record["foldmatch_version"] + elif record["residue_output"] is not None and isinstance(record["pass"].get("foldmatch_version"), str): + pass_version = record["pass"]["foldmatch_version"] + else: + pass_version = None + if pass_version is not None and pass_version != __version__: + logging.warning( + f"Run {run_id} was embedded by FoldMatch {pass_version}, and this is FoldMatch {__version__}: its record, " + f"shard schemas and digests, pre-flight counts and ids were checked, and the database is built by this " + f"FoldMatch." + ) + if sealed: + logging.warning( + f"The record of run {run_id} was written by 'build sequences-seal' (FoldMatch " + f"{record['foldmatch_version']}): it binds the shards to the FASTA by their entry count, rows and ids " + f"(and, with residue codes, their seq_sha1), not by the pass that wrote them" + + ("." if pass_version is not None else + ", whose FoldMatch version is not recorded (shards without residue codes do not record it).") + ) + if record["output_db"] is not None and record["output_db"] != os.path.abspath(db_path): + logging.info(f"Building {db_path}, not the --output-db the run recorded ({record['output_db']})") + if codes: + built = f"residue codes of {record['system']['name']}" + elif record["system"] is not None: + built = f"its residue codes of {record['system']['name']} left out (--no-residue-codes)" + else: + built = "no residue codes" + if sealed: + when = (f"sealed {record['created_utc']} by FoldMatch {record['foldmatch_version']}; embedded by " + + (f"FoldMatch {pass_version}" if pass_version is not None else "a FoldMatch it does not record")) + else: + when = f"embedded {record['created_utc']} by FoldMatch {record['foldmatch_version']}" + logging.info( + f"Building {db_path} from run {run_id} in {from_shards}: {record['rows']} entries in {shards} shard(s), " + f"{when} on {record['world_size']} rank(s) ({record['pass']['accelerator'] or 'device not recorded'}), {built}" + + ("; sealed record" if sealed else "") + ) + + +def _fasta_unchanged_guard(fasta_file: Path, stat, output_db: str): + """The ``after_sequence_store`` hook of ``build sequences-index`` (I23): once the sequence store is built, the + FASTA must still have ``stat``, taken before it was hashed. Otherwise the database files this build wrote are + removed and the build is refused.""" + def guard() -> None: + from foldmatch.search.split_run import fasta_stat + try: + now = fasta_stat(fasta_file) + shown = _stat_text(now) + except OSError as exc: + now, shown = None, f"cannot be stat'ed ({exc})" + if now == stat: + return + removed = _remove_outputs(output_db, _SEQUENCE_DB_SUFFIXES) + raise RefusalError( + f"The FASTA changed during this build ({fasta_file}: {_stat_text(stat)} when it was checked, now " + f"{shown}), so the database may not hold the FASTA the run embedded. The database files it wrote were " + f"removed ({', '.join(path.name for path in removed)})." + ) + return guard + + +# The variables that set a BLAS runtime's threads; any of them set leaves BLAS to it (D29). +_BLAS_THREAD_VARS = ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "VECLIB_MAXIMUM_THREADS") + + +@contextmanager +def _whole_node_threads(): + """FAISS's OpenMP and the BLAS thread pools at the usable physical cores for the block (D29), restored on exit. + + FAISS gets ``faiss.omp_set_num_threads(n)`` unless ``OMP_NUM_THREADS`` is set; BLAS a ``threadpoolctl`` limit of + ``n`` unless one of :data:`_BLAS_THREAD_VARS` is set, which reaches only the BLAS libraries threadpoolctl finds + (the INFO line names them). Nothing is raised on macOS, where ``foldmatch`` pins ``OMP_NUM_THREADS=1`` against the + second libomp. ``n`` is :func:`~foldmatch.search.code_alignment._physical_cpus` of the affinity mask, so call it + after the mask was widened. Gains for ivf_pq (the k-means of its training and the encoding of its adds), not for + flat, whose adds are copies after an L2 normalisation of 1000 rows per call (below the size FAISS parallelises); + HNSW adds 100 vectors per call and scales poorly. With the faiss-cpu 1.15.1 wheel the OPQ step of an ivf_pq + training does not gain: on a 128-core node it ran fastest at 4-16 threads and 2.6x slower at 128 (P9), and a + 30M-entry training took 76 min there at 128 threads against 40 min on a 64-core node at 64. D29 still gives the + whole block ``n``. + + The limit also reaches the OpenMP OpenBLAS of the faiss-cpu 1.15.1 wheel (``libopenblaso``), which keeps no + thread count of its own (OpenBLAS 0.3.15): ``openblas_set_num_threads`` sets the calling thread's OpenMP count, + the one FAISS reads (FAISS and its OpenBLAS share an OpenMP runtime), and every BLAS call runs that count, lowered + to the build's MAX_THREADS (128 in the x86_64 wheel) when it is larger. The limit gives it the ``n`` FAISS was just + given, so FAISS keeps ``n``, at most MAX_THREADS (above it the INFO line still says ``n``, and the index phase warns + of fewer threads than cores). On exit threadpoolctl sets every pool it lists back to its count at entry, in its + listing order, which leaves that OpenMP count at OpenBLAS's (it lags FAISS's until a BLAS call); FAISS's previous + count is restored after it. ``OPENBLAS_NUM_THREADS`` does not set that OpenBLAS's threads (FAISS's OpenMP count + does), as the INFO line says when such a variable leaves BLAS alone. + """ + import faiss + from foldmatch.search.code_alignment import _physical_cpus + n = _physical_cpus() + try: + allowed = len(os.sched_getaffinity(0)) + except AttributeError: # no affinity masks (macOS) + allowed = os.cpu_count() or 1 + if sys.platform == "darwin": + logging.info("FAISS/BLAS threads left as they are on macOS (foldmatch pins OMP_NUM_THREADS)") + yield + return + raised, kept = [], [] + with ExitStack() as stack: + if "OMP_NUM_THREADS" in os.environ: + kept.append(f"FAISS threads left to OMP_NUM_THREADS={os.environ['OMP_NUM_THREADS']}") + else: + previous = faiss.omp_get_max_threads() + faiss.omp_set_num_threads(n) + stack.callback(faiss.omp_set_num_threads, previous) + raised.append("FAISS") + blas_set = [f"{name}={os.environ[name]}" for name in _BLAS_THREAD_VARS if name in os.environ] + limited = None + if blas_set: + follows = None + if raised: # FAISS was given n: an OpenMP BLAS of FAISS's runs FAISS's count, whatever the variable + try: # diagnostics only + import threadpoolctl + follows = _faiss_openmp_blas(threadpoolctl.threadpool_info()) + except Exception as exc: + logging.debug(f"threadpoolctl could not list the thread pools: {exc}") + kept.append(f"BLAS threads left to {', '.join(blas_set)}" + ( + f", except FAISS's OpenMP OpenBLAS, {follows}, which runs FAISS's OpenMP threads" if follows else "")) + else: + import threadpoolctl + stack.enter_context(threadpoolctl.threadpool_limits(limits=n, user_api="blas")) + raised.append("BLAS") + try: # diagnostics only: the libraries the limit reached + limited = sorted({os.path.basename(str(runtime.get("filepath"))) + for runtime in threadpoolctl.threadpool_info() if runtime.get("user_api") == "blas"}) + except Exception as exc: + logging.debug(f"threadpoolctl could not list the thread pools: {exc}") + set_to = [f"{'/'.join(raised)} threads set to {n} (physical cores of the {allowed} allowed CPUs)"] + if limited is not None: + set_to.append(f"BLAS libraries limited: {', '.join(limited)}" if limited else + "threadpoolctl found no BLAS library to limit") + logging.info("; ".join((set_to if raised else []) + kept)) + yield + + +def _seal_run(fasta_file: Path, from_shards: Path, run_id: str, min_res_n: Optional[int]) -> dict: + """The run record ``build sequences-seal`` writes for the shards of ``run_id``, which have none, once the checks + that bind them to the FASTA have passed. Writes nothing. + + In this order: the run has shards (S4); their footers say what the pass was (S5, S6: :func:`_seal_footers`); + then, between two stats of the FASTA, its sha256, its pre-flight with the run's ``min_res_n`` and the strictness + of its mode (S7), the shards' rows and ranks, which must be the deal of the kept entries over the pass's ranks + (S8), and the id of every shard row at its dealt position, with residue codes also its seq_sha1 against the + entry's sequence (S9); the FASTA must not change meanwhile (S10). The record is sealed: it names no database and no + pass options, and a plain run's pass record has no ESM3 revision. + """ + from foldmatch.search import split_run + from foldmatch.search.residue_shards import RESIDUE_CODES, run_shard_files + paths = run_shard_files(from_shards, run_id) + if not paths: + runs = split_run._runs(from_shards) + raise RefusalError( + f"No embedding shards of run {run_id} (emb_{run_id}-.parquet) in {from_shards}, so there is nothing " + f"to seal. It holds " + + ("; ".join(f"run {rid} ({state.describe()})" for rid, state in runs.items()) if runs else "no run") + + "." + ) + system, min_res_n, world_size, run_pass = _seal_footers(paths, folder=from_shards, run_id=run_id, + min_res_n=min_res_n) + codes = system is not None + + stat = split_run.fasta_stat(fasta_file) + sha256, seconds = split_run.fasta_sha256(fasta_file) + logging.info(f"FASTA sha256 of {fasta_file} ({stat.bytes} bytes) in {seconds:.1f} s") + preflight = _fasta_preflight(fasta_file, min_res_n, strict=codes) + _refuse_nothing_to_build(fasta_file, min_res_n, preflight) + shards = split_run.shard_entries(paths) + rows = sum(shard["rows"] for shard in shards) + with _refusals(note="The shards are incomplete: run 'build sequences-embed' again" + + ("." if codes else ", or give the --min-res-n the run used.")): + if rows != preflight.entries_kept: + raise RefusalError( + f"The {len(shards)} shard(s) of run {run_id} in {from_shards} hold {rows} row(s), but {fasta_file} has " + f"{preflight.entries_kept} entries with at least {min_res_n} residues." + ) + _check_shards_are_the_deal(shards, preflight.entries_kept, world_size, from_shards, run_id) + seconds = split_run.check_dealt_ids(paths, {"run_id": run_id, "world_size": world_size}, preflight, + sequences=codes) + logging.info(f"Ids{' and seq_sha1' if codes else ''} of the {rows} shard rows checked against {fasta_file} in " + f"{seconds:.1f} s") + try: + now = split_run.fasta_stat(fasta_file) + shown = _stat_text(now) + except OSError as exc: + now, shown = None, f"cannot be stat'ed ({exc})" + if now != stat: + raise RefusalError( + f"The FASTA changed during the seal ({fasta_file}: {_stat_text(stat)} before it was hashed, now {shown}), " + f"so the shards were not checked against one FASTA. No record was written." + ) + return { + "kind": split_run.RUN_RECORD_KIND, + "record_version": split_run.RUN_RECORD_VERSION, + "run_id": run_id, + "command": split_run.SEAL_COMMAND, + "sealed": True, + "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "foldmatch_version": __version__, + "fasta": {"path": str(Path(fasta_file).resolve()), "bytes": stat.bytes, "mtime_ns": stat.mtime_ns, + "sha256": sha256}, + "min_res_n": min_res_n, + "strict": codes, + "preflight": split_run.preflight_counts(preflight), + "output_db": None, + "residue_output": RESIDUE_CODES if codes else None, + "system": system, + "world_size": world_size, + "pass": run_pass, + "options": {}, + "shards": shards, + "rows": rows, + } + + +def _seal_footers(paths: List[Path], *, folder: Path, run_id: str, + min_res_n: Optional[int]) -> Tuple[Optional[dict], int, int, dict]: + """What the footers of a run's shards say about its pass, for ``build sequences-seal``: ``(system block or None, + min_res_n, world size, pass record)``. + + Refused (S5): shards of more than one kind (plain and residue codes, or the codes of several systems); plain + shards that are not legacy ones (:func:`~foldmatch.search.split_run.check_plain_footers`); residue shards that + are not residue codes, or whose pass record (``foldmatch.pass``) is missing or lacks its ``min_res_n`` and world + size. Refused (S6): a plain run without ``min_res_n``, which its shards do not record; for codes, shards that + disagree on their pass's ``min_res_n`` or world size, a ``min_res_n`` other than theirs, or a world smaller than + the shard count. A plain run's world is its shard count: a pass on W ranks writes min(W, N) shards of its N + entries, and their deal over that many ranks is the same. + """ + from foldmatch.search import split_run + from foldmatch.search.residue_shards import ResidueShardError, pass_record, read_shard_info + try: + infos = [read_shard_info(path) for path in paths] + except ResidueShardError as exc: + raise RefusalError(str(exc)) from exc + groups = {} + for info in infos: + groups.setdefault(info.key, []).append(info) + if len(groups) > 1: + raise RefusalError( + f"The {len(infos)} shards of run {run_id} in {folder} are mixed: " + + "; ".join(f"{len(group)} shard(s) with {group[0].describe()} (e.g. {group[0].path.name})" + for group in groups.values()) + + ". The shards of one run come from one pass, so they cannot be sealed as one run." + ) + if infos[0].residue_output is None: + split_run.check_plain_footers(paths) + if min_res_n is None: + raise RefusalError( + f"The {len(infos)} shard(s) of run {run_id} in {folder} hold no residue codes, so they do not record " + f"the --min-res-n of their pass: give the --min-res-n the run used." + ) + return None, min_res_n, len(infos), pass_record(min_res_n=min_res_n, world_size=len(infos)) + + unrecorded = [info.path.name for info in infos if info.pass_record is None] + if unrecorded: + raise RefusalError( + f"{len(unrecorded)} of the {len(infos)} shard(s) of run {run_id} in {folder} record no pass " + f"(foldmatch.pass): {', '.join(unrecorded[:10])}{', ...' if len(unrecorded) > 10 else ''}. Their " + f"--min-res-n and world size are unknown, so they cannot be sealed: run 'build sequences-embed' again." + ) + first = infos[0] + for key in ("min_res_n", "world_size"): + other = [info for info in infos if info.pass_record.get(key) != first.pass_record.get(key)] + if other: + raise RefusalError( + f"The shards of run {run_id} in {folder} disagree on their pass's {key}: " + f"{first.pass_record.get(key)!r} in {first.path.name}, {other[0].pass_record.get(key)!r} in " + f"{other[0].path.name}{f' and {len(other) - 1} more' if len(other) > 1 else ''}. The shards of one run " + f"come from one pass, so they cannot be sealed as one run." + ) + shards_min_res_n, world_size = first.pass_record.get("min_res_n"), first.pass_record.get("world_size") + if not all(isinstance(value, int) and not isinstance(value, bool) and value >= floor + for value, floor in ((shards_min_res_n, 0), (world_size, 1))): + raise RefusalError( + f"The shards of run {run_id} in {folder} record a pass without a valid --min-res-n and world size " + f"(min_res_n {shards_min_res_n!r}, world_size {world_size!r}), so they cannot be sealed: run 'build " + f"sequences-embed' again." + ) + if min_res_n is not None and min_res_n != shards_min_res_n: + raise RefusalError( + f"--min-res-n is {min_res_n}, but the shards of run {run_id} in {folder} record a pass with --min-res-n " + f"{shards_min_res_n}: leave --min-res-n out for a run with residue codes, or give the run's." + ) + if world_size < len(infos): + raise RefusalError( + f"The shards of run {run_id} in {folder} record a pass on {world_size} rank(s), but there are " + f"{len(infos)} of them: they are not the shards of one pass." + ) + system = {"name": first.system, "digest": first.system_digest, "encoding_digest": first.encoding_digest} + run_pass = split_run.check_codes_footers(paths, expect=system, min_res_n=shards_min_res_n, world_size=world_size) + if None in system.values(): + raise RefusalError( + f"The shards of run {run_id} in {folder} hold residue codes without their system's name and digests " + f"({', '.join(f'{key} {value}' for key, value in system.items())}), so they cannot be sealed." + ) + return system, shards_min_res_n, world_size, run_pass + + +def _residue_codes_preflight(fasta_file: Path, min_res_n: int, output_db: str, + tmp_embedding_folder: Path, num_nodes: int): + """Every rank's read-only, deterministic checks of a --residue-codes build, before ESM3. + + In the order of plan section 3 (BUILD): the strict FASTA pre-flight + (duplicate ids, '<' / '>' and non-ASCII sequences refused) and the + outputs-exist check, ``.rescodes.*`` included; this run's id and its + shards; then the default scoring system's bundle, which only the process + that starts the run may download (every other rank finds it in the cache: + :func:`~foldmatch.inference.chain_complete_inference.prepare_scoring_system`). + The same files give every rank the same verdict, so a refusal stops them + all together. Returns ``(preflight, embedding_computer, prepared_system)``. + """ + with _refusals(): + preflight = _fasta_preflight(fasta_file, min_res_n, strict=True, output_db=output_db) + from foldmatch.search.embedding_computer import EmbeddingComputer + embedding_computer = EmbeddingComputer(embedding_folder=tmp_embedding_folder) + embedding_computer.refuse_existing_shards() + from foldmatch.inference.chain_complete_inference import prepare_scoring_system + from foldmatch.search.residue_shards import RESIDUE_CODES + prepared = prepare_scoring_system(residue_output=RESIDUE_CODES, num_nodes=num_nodes) + return preflight, embedding_computer, prepared + + +def _code_store_build(run_id: str, shard_infos, *, from_shards: bool, split: bool = False): + """``(pass record, manifest build block)`` of a code store built from a run's shards. + + The build block describes the ESM3 pass that computed the codes, from the + shards' pass record (:func:`~foldmatch.search.residue_shards.pass_record`): + the device type and its ESM3 precision, the ranks (``devices``), the + encoder precision and the software versions of the pass. ``from_shards`` + tells whether the store was built by the pass's own command or later by + ``build residue-codes --from-shards``; ``split`` adds ``"split": true`` + for the index phase of a split build (``build sequences-index``, D26), + and the key is absent otherwise. + """ + from foldmatch.search.residue_shards import shards_pass_record + record = shards_pass_record(shard_infos) + build = { + "run_id": run_id, + "from_shards": from_shards, + "accelerator": record["accelerator"], + "devices": record["world_size"], + "esm_precision": record["esm_precision"], + "encoder_precision": record["encoder_precision"], + "foldmatch_version": record["foldmatch_version"], + "torch": record["torch"], + "esm": record["esm"], + "platform": record["platform"], + } + if split: + build["split"] = True + return record, build + + +def _build_residue_code_store(embedding_computer, embedding_db, sequence_store, preflight, system, *, + min_res_n: int, load_batch_size: int, split: bool = False): + """Rank 0, after the FAISS index and the sequence store: the residue-code store from this run's shards. + + ``create`` streams the shards' codes into the pending ``.rescodes.bin`` + and ``.rescodes.idx``; ``validate`` checks them against the sequence + store, the pre-flight and the FAISS count, then moves them into place and + writes the manifest last. The manifest's ``build`` block comes from the + shards' pass record. A refusal removes only the code store and keeps the + shards. ``split`` (``build sequences-index``): the pass ran in another + process, so the manifest records the ESM3 revision of the shards' pass + record, as ``build residue-codes --from-shards`` does; ``build + sequences`` records this installation's pin, the same value under one + installation. + """ + from foldmatch.search.residue_code_store import ResidueCodeStore + from foldmatch.search.residue_shards import RESIDUE_CODES, check_shards + code_store = ResidueCodeStore(embedding_db.db_folder, embedding_db.index_name) + kept = (f"This build's shards are kept in {embedding_computer.embedding_folder} " + f"(run id {embedding_computer.run_id}).") + with _refusals(note=kept): + infos = check_shards(embedding_computer.shard_files(), RESIDUE_CODES, system) + run_pass, build = _code_store_build(embedding_computer.run_id, infos, from_shards=False, split=split) + revision = {"esm_revision": run_pass["esm_revision"]} if split else {} + code_store.create( + embedding_computer.iter_residue_codes(system, load_batch_size=load_batch_size), + system, min_res_n=min_res_n, build=build, **revision, + ) + manifest = code_store.validate(sequence_store, len(embedding_db.db.chain_ids), preflight=preflight) + logging.info( + f"Residue-code store written to {code_store.manifest_path} ({manifest['entries']} entries, " + f"{manifest['total_residues']} residues, system {manifest['system_name']})" + ) + return manifest + + +def _residue_codes_from_shards(db_folder: Path, index_name: str, shards_folder: Path, *, run_id: Optional[str], + force: bool, load_batch_size: int): + """``build residue-codes --from-shards``: the residue-code store of a database from a run's kept shards. + + Refused before anything is written: a database without a sequence store or + FAISS files, or whose files cannot be read; a complete code store unless + ``force``; no shards, or the shards of several runs without ``run_id``; + shards without residue codes, mixed shards or another encoding than the + default system's (:func:`~foldmatch.search.residue_shards.check_shards`); + shards that do not record their pass's ``--min-res-n``; an entry count + that differs between the shards, the sequence store and the FAISS index. + Then ``create`` and ``validate`` without a pre-flight; an old store (with + ``force``) is replaced only once the new codes have passed validation. The + manifest's ``min_res_n`` and ``build`` block come from the shards' pass + record, so they describe the ESM3 pass that computed the codes. Returns + the manifest. + """ + import pyarrow.parquet as pq + from foldmatch.search import scoring_system + from foldmatch.search.faiss_database import FaissEmbeddingDatabase + from foldmatch.search.residue_code_store import ResidueCodeStore + from foldmatch.search.residue_shards import RESIDUE_CODES, check_shards, iter_residue_codes + from foldmatch.search.sequence_store import SequenceStore + + database = db_folder / index_name + sequence_store = SequenceStore(db_folder, index_name) + if not sequence_store.exists(): + raise RefusalError( + f"{database} has no sequence store ({sequence_store.path}): residue codes are " + f"only built for sequence databases ('fm-search build sequences')." + ) + try: + entry_count = sequence_store.meta()["entry_count"] + except sqlite3.DatabaseError as exc: + raise RefusalError(f"{sequence_store.path} is not a readable sequence store ({exc}).") from exc + faiss_entries = FaissEmbeddingDatabase(db_folder, index_name).saved_entry_count() + code_store = ResidueCodeStore(db_folder, index_name) + if code_store.exists() and not force: + raise RefusalError( + f"{database} already has a residue-code store ({code_store.manifest_path}). " + f"Pass --force to replace it." + ) + system = scoring_system.get(scoring_system.DEFAULT_ESM_CODES) + run_id, shards = _residue_code_shards(shards_folder, run_id) + # Every shard's footer is checked here, before any data and before anything is written. + infos = check_shards(shards, RESIDUE_CODES, system) + rows = sum(pq.read_metadata(path).num_rows for path in shards) + if not rows == entry_count == faiss_entries: + raise RefusalError( + f"The {len(shards)} shard(s) of run {run_id} in {shards_folder} hold {rows} entries, " + f"but {database} has {entry_count} in its sequence store and {faiss_entries} in its " + f"FAISS index: they are not this database's build shards, or a rank's shard is " + f"missing. Nothing was changed." + ) + record, build = _code_store_build(run_id, infos, from_shards=True) + if not isinstance(record["min_res_n"], int): + raise RefusalError( + f"The shards of run {run_id} in {shards_folder} do not record the --min-res-n of the " + f"build that wrote them (written by an earlier FoldMatch). Rebuild the database with " + f"'fm-search build sequences --residue-codes'. Nothing was changed." + ) + logging.info(f"Building the residue-code store of {database} from {len(shards)} shard(s) of run {run_id}") + code_store.create(iter_residue_codes(shards, system, load_batch_size), system, force=force, + min_res_n=record["min_res_n"], build=build, esm_revision=record["esm_revision"]) + return code_store.validate(sequence_store, faiss_entries, preflight=None) + + +def _residue_code_shards(folder: Path, run_id: Optional[str]): + """``(run id, shard paths)`` of the residue-code shards in ``folder`` (paths in the order a build reads them). + + With ``run_id``, that run's shards. Without, the only run whose shards + hold residue codes; none, or several, are refused. + """ + from foldmatch.search.embedding_computer import run_shard_files, shard_run_ids + from foldmatch.search.residue_shards import RESIDUE_CODES, read_shard_info + if run_id is not None: + shards = run_shard_files(folder, run_id) + if not shards: + runs = shard_run_ids(folder) + raise RefusalError( + f"No embedding shards of run {run_id} (emb_{run_id}-.parquet) in {folder}; " + f"it holds shards of the run(s): {', '.join(runs) if runs else 'none'}." + ) + return run_id, shards + runs = {rid: run_shard_files(folder, rid) for rid in shard_run_ids(folder)} + if not runs: + raise RefusalError( + f"No embedding shards (emb_-.parquet) in {folder}. A build keeps " + f"them in its --tmp-embedding-folder unless --cleanup-embeddings." + ) + with_codes = {} + for rid, paths in runs.items(): + try: + info = read_shard_info(paths[0]) + except ValueError as exc: + raise RefusalError( + f"{exc}. Pass --run-id to choose the run that built this database, or remove " + f"the file if it is not a shard." + ) from exc + if info.residue_output == RESIDUE_CODES: + with_codes[rid] = paths + if not with_codes: + raise RefusalError( + f"None of the embedding shards in {folder} hold residue codes (run(s) " + f"{', '.join(runs)}): they come from a build without --residue-codes. Rebuild the " + f"database with 'fm-search build sequences --residue-codes'." + ) + if len(with_codes) > 1: + listed = ", ".join(f"{rid} ({len(paths)} shard(s))" for rid, paths in with_codes.items()) + raise RefusalError( + f"{folder} holds the residue-code shards of {len(with_codes)} runs: {listed}. " + f"Pass --run-id to choose the run that built this database." + ) + return next(iter(with_codes.items())) + + +def _check_stage2_stores(subject_store, max_evalue: Optional[float], query_store=None) -> None: + """Read-only checks of the sequence stores Stage 2 will read, before ESM3 or the search. + + Reading each store's totals refuses an incomplete store, and an E-value + filter needs a non-empty subject database (:func:`check_evalue_search_space`). + Both would otherwise only fail after the embedding pass. + """ + from foldmatch.search.alignment import check_evalue_search_space + if query_store is not None: + query_store.meta() + check_evalue_search_space(max_evalue, subject_store.total_residues()) + + +def _check_blosum62_gap_costs(gap_open: int, gap_extend: int, significance_mode: SignificanceMode, + output_fields: List[str], max_evalue: Optional[float]) -> None: + """The gap costs of a BLOSUM62 Stage 2, refused with ``typer.BadParameter`` (exit status 2, D20) before ESM3 or + the search; ``align_candidates`` runs the same check (:func:`~foldmatch.search.alignment.check_gap_costs`) before + it reads a sequence. + + The costs are the absolute values of the options (the sign is ignored, as it always was), whole numbers (the + Python functions also take a float such as 11.0, as before): ``--gap-open`` must be between 1 and + :data:`~foldmatch.search.swint.MAX_GAP` and ``--gap-extend`` between 0 and that maximum, the range of the native + kernel. Under ``--significance-mode default``, a run that needs significance (an ``evalue`` or ``bits`` column in + ``--format-output``, or a finite ``--max-evalue``) must use 11/1, the only gap costs with calibrated statistics; + the message names what needs it and the three ways out. The costs and the mode are checked here first, so the + only refusal ``check_gap_costs`` has left is that 11/1 rule. + """ + from foldmatch.search import alignment, swint + from foldmatch.search.output import SIGNIFICANCE_FIELDS + costs = [] + for flag, value, low, default in (("--gap-open", gap_open, 1, 11), ("--gap-extend", gap_extend, 0, 1)): + cost = alignment.whole_number(value) + if cost is None: + raise typer.BadParameter(f"must be a whole number, got {value!r}. The default is {default}.", + param_hint=flag) + if not low <= abs(cost) <= swint.MAX_GAP: + raise typer.BadParameter( + f"must be between {low} and {swint.MAX_GAP} (the sign is ignored), the range of the Stage-2 " + f"Smith-Waterman kernel; got {value}. The default is {default}.", param_hint=flag) + costs.append(abs(cost)) + significance_mode = _significance_mode(significance_mode) + columns = [field for field in dict.fromkeys(output_fields) if field in SIGNIFICANCE_FIELDS] + try: + alignment.check_gap_costs(gap_open, gap_extend, significance_mode, bool(columns), max_evalue) + except ValueError: + needs, without = [], [] + if columns: + needs.append(f"the {' and '.join(columns)} column{'s' if len(columns) > 1 else ''} of --format-output") + without.append("a --format-output without evalue and bits") + if max_evalue is not None and math.isfinite(max_evalue): + needs.append(f"the E-value filter --max-evalue {max_evalue:g}") + without.insert(0, "--max-evalue inf") + raise typer.BadParameter( + f"--significance-mode default has calibrated statistics only for the gap costs 11/1, and this run needs " + f"them for {' and '.join(needs)}; the gap costs given are {costs[0]}/{costs[1]}. Use " + f"--gap-open 11 --gap-extend 1; or --significance-mode sampled, whose E-values and bit scores work at any " + f"gap costs but are comparable only within this tool; or {' and '.join(without)}, to run without " + f"significance.", param_hint=["--gap-open", "--gap-extend"]) from None + + +def _significance_mode(significance_mode) -> SignificanceMode: + """``--significance-mode`` as a :class:`SignificanceMode` (the Python functions may be given its value as a + string); anything else is refused with ``typer.BadParameter``.""" + try: + return SignificanceMode(significance_mode) + except ValueError: + raise typer.BadParameter(f"{significance_mode!r} is not a significance mode; expected one of " + f"{', '.join(m.value for m in SignificanceMode)}.", + param_hint="--significance-mode") from None + + +def _check_significance_sample_size(significance_sample_size: int, significance_mode: SignificanceMode, + output_fields: List[str], max_evalue: Optional[float]) -> None: + """``--significance-sample-size`` of a BLOSUM62 Stage 2 that samples its statistics (``--significance-mode + sampled``, with an ``evalue`` or ``bits`` column or a finite ``--max-evalue``) must be at least 2, else + ``typer.BadParameter`` before ESM3 or the search: from 1 or 0 alignments lambda and K come out infinite or NaN, + and a negative size failed after Stage 1. The option itself has ``min=2``; this check serves the Python + functions, which typer does not parse.""" + from foldmatch.search.output import needs_significance + filtering = max_evalue is not None and math.isfinite(max_evalue) + if _significance_mode(significance_mode) is SignificanceMode.sampled and ( + needs_significance(output_fields) or filtering) and not significance_sample_size >= 2: + raise typer.BadParameter(f"must be at least 2 with --significance-mode sampled, got " + f"{significance_sample_size!r}. The default is 500.", + param_hint="--significance-sample-size") + + +def _check_blosum62_kernel() -> None: + """The native kernel of the BLOSUM62 Stage 2 (:func:`foldmatch.search.swint.load`), loaded before ESM3 or the + search, as under esm-codes (:func:`resolve_esm_codes_database`, step 4): a library that is missing (an install + built without a C compiler), stale or of another ABI is a :class:`RefusalError` (exit status 1) whose message + gives the rebuild command. There is no other aligner.""" + from foldmatch.search import swint, swkern + try: + swint.load() + except swkern.KernelError as exc: + raise RefusalError( + f"Stage 2 (sequence identity) needs the native Smith-Waterman kernel, which cannot be used here: {exc}. " + f"--no-seq-identity runs an embedding-only search, which does not need it.") from exc + + def _resolve_stage2(seq_identity: Optional[bool], stores_available: bool, missing_msg: str) -> bool: """Decide whether to run Stage-2 sequence-identity alignment. @@ -1118,6 +3189,378 @@ def _resolve_stage2(seq_identity: Optional[bool], stores_available: bool, missin return False +# ===================================================================================================================== +# query sequences --scoring (plan P6.1-P6.3) +# ===================================================================================================================== +#: Defaults of the Stage-2 options of ``query sequences`` per scoring (P6.1). BLOSUM62's are the values the options +#: had before --scoring; esm-codes filters on the E-value only (D8). +BLOSUM62_QUERY_DEFAULTS = MappingProxyType(dict( + min_seq_identity=0.3, min_coverage=0.8, max_evalue=1e-3, gap_open=11, gap_extend=1, comp_bias_corr=False, + comp_bias_corr_scale=1.0, significance_mode=SignificanceMode.default, significance_sample_size=500, +)) +ESM_CODES_QUERY_DEFAULTS = MappingProxyType(dict(min_seq_identity=0.0, min_coverage=0.0, max_evalue=1e-3)) +#: The BLOSUM62-only options, refused under esm-codes whatever their value (D20: exit status 2). +_BLOSUM62_ONLY_OPTIONS = (("gap_open", "--gap-open"), ("gap_extend", "--gap-extend"), + ("comp_bias_corr", "--comp-bias-corr/--no-comp-bias-corr"), + ("comp_bias_corr_scale", "--comp-bias-corr-scale"), + ("significance_mode", "--significance-mode"), + ("significance_sample_size", "--significance-sample-size")) + + +@dataclass(frozen=True) +class QueryStage2Options: + """The Stage-2 options of one ``query sequences`` run, resolved for its scoring (:func:`resolve_query_stage2_options`). + The BLOSUM62-only fields are None under esm-codes.""" + scoring: ScoringChoice + min_seq_identity: float + min_coverage: float + max_evalue: Optional[float] + gap_open: Optional[int] = None + gap_extend: Optional[int] = None + comp_bias_corr: Optional[bool] = None + comp_bias_corr_scale: Optional[float] = None + significance_mode: Optional[SignificanceMode] = None + significance_sample_size: Optional[int] = None + + +def resolve_query_stage2_options(scoring=DEFAULT_SCORING, *, seq_identity: Optional[bool] = None, + min_seq_identity: Optional[float] = None, min_coverage: Optional[float] = None, + max_evalue: Optional[float] = None, gap_open: Optional[int] = None, + gap_extend: Optional[int] = None, comp_bias_corr: Optional[bool] = None, + comp_bias_corr_scale: Optional[float] = None, + significance_mode: Optional[SignificanceMode] = None, + significance_sample_size: Optional[int] = None) -> QueryStage2Options: + """``query sequences``' Stage-2 options for ``scoring`` (P6.1): an option left out (None) takes the scoring's + default (:data:`BLOSUM62_QUERY_DEFAULTS`, :data:`ESM_CODES_QUERY_DEFAULTS`), an explicit value is kept. + + Under esm-codes, every BLOSUM62-only option given (``--gap-open``, ``--gap-extend``, + ``--comp-bias-corr/--no-comp-bias-corr``, ``--comp-bias-corr-scale``, ``--significance-mode``, + ``--significance-sample-size``) and ``--no-seq-identity`` are refused, and so are filter values out of range: + ``typer.BadParameter`` (exit status 2, D20), before anything is read. Under blosum62 nothing is checked here, as + before. + """ + try: + scoring = ScoringChoice(scoring) + except ValueError: + raise typer.BadParameter(f"{scoring!r} is not a scoring; expected one of " + f"{', '.join(c.value for c in ScoringChoice)}.", param_hint="--scoring") from None + given = dict(min_seq_identity=min_seq_identity, min_coverage=min_coverage, max_evalue=max_evalue) + blosum = dict(gap_open=gap_open, gap_extend=gap_extend, comp_bias_corr=comp_bias_corr, + comp_bias_corr_scale=comp_bias_corr_scale, significance_mode=significance_mode, + significance_sample_size=significance_sample_size) + if scoring is ScoringChoice.blosum62: + values = {name: BLOSUM62_QUERY_DEFAULTS[name] if value is None else value + for name, value in {**given, **blosum}.items()} + if values["significance_mode"] is not None: + values["significance_mode"] = SignificanceMode(values["significance_mode"]) + return QueryStage2Options(scoring=scoring, **values) + refused = [flag for name, flag in _BLOSUM62_ONLY_OPTIONS if blosum[name] is not None] + problems = [] + if refused: + problems.append( + f"{', '.join(refused)} {'is a BLOSUM62 option' if len(refused) == 1 else 'are BLOSUM62 options'}: " + f"--scoring esm-codes takes its gap costs and E-value statistics from the scoring system the " + f"database's residue codes were built for") + if seq_identity is False: + problems.append("--no-seq-identity asks for an embedding-only search, which has no Stage-2 scoring: " + "drop --scoring esm-codes for it") + if problems: + raise typer.BadParameter("; ".join(problems) + ". Remove them, or use --scoring blosum62.", + param_hint="--scoring") + values = {name: ESM_CODES_QUERY_DEFAULTS[name] if value is None else value for name, value in given.items()} + for name, flag in (("min_seq_identity", "--min-seq-identity"), ("min_coverage", "--min-coverage")): + if not 0.0 <= values[name] <= 1.0: # NaN fails too + raise typer.BadParameter(f"must be a fraction between 0 and 1, got {values[name]}.", param_hint=flag) + if not values["max_evalue"] >= 0.0: + raise typer.BadParameter(f"must be >= 0 ('inf' for no E-value filter), got {values['max_evalue']}.", + param_hint="--max-evalue") + return QueryStage2Options(scoring=scoring, **values) + + +#: The ``cluster`` options of the Leiden route only, refused under ``--method greedy`` (T5: exit status 2). +_LEIDEN_ONLY_OPTIONS = (("resolution", "--resolution"), ("leiden_iterations", "--leiden-iterations"), + ("seed", "--seed")) + + +def _refuse_leiden_only_options(method=ClusterMethod.leiden, **given) -> ClusterMethod: + """``cluster``'s ``--method``: under greedy, every Leiden-only option given (not None; :data:`_LEIDEN_ONLY_OPTIONS`) + is refused in one ``typer.BadParameter`` (exit status 2), before the database is loaded. Returns the method.""" + try: + method = ClusterMethod(method) + except ValueError: + raise typer.BadParameter(f"{method!r} is not a clustering method; expected one of " + f"{', '.join(m.value for m in ClusterMethod)}.", param_hint="--method") from None + refused = [flag for name, flag in _LEIDEN_ONLY_OPTIONS if given.get(name) is not None] + if method is ClusterMethod.greedy and refused: + raise typer.BadParameter( + f"{', '.join(refused)} {'is a Leiden option' if len(refused) == 1 else 'are Leiden options'}: " + f"--method greedy has no resolution or iteration count and is deterministic. " + f"Remove {'it' if len(refused) == 1 else 'them'}, or use --method leiden.", param_hint="--method") + return method + + +def _refuse_max_neighbors_below_1(max_neighbors: int): + """``cluster`` and ``similarity-graph``: ``--max-neighbors`` below 1 is refused (exit status 2) before the database + is loaded; FAISS would refuse k = 0 with a bare AssertionError after it.""" + if max_neighbors < 1: + raise typer.BadParameter("--max-neighbors must be at least 1.", param_hint="--max-neighbors") + + +def _output_fields(format_output: str) -> List[str]: + """The ``--format-output`` fields of a ``query sequences`` or ``query db`` Stage 2, parsed before ESM3 or the + search: an empty list or an unknown field is refused with ``typer.BadParameter`` (exit status 2).""" + from foldmatch.search import output + try: + return output.parse_format_output(format_output) + except ValueError as exc: + raise typer.BadParameter(str(exc), param_hint="--format-output") from None + + +@dataclass(frozen=True) +class EsmCodesQueryDatabase: + """What :func:`resolve_esm_codes_database` found: the database's stores, its code-store manifest, its scoring + system (the registry entry the manifest names), the kernel and the prepared (sha256-verified) bundle.""" + db_path: str + sequence_store: Any + code_store: Any + manifest: Mapping[str, Any] + system: Any + kernel: Any + prepared: Any + + @property + def total_residues(self) -> int: + """N of the E-values: the database's residue count (manifest ``total_residues``).""" + return int(self.manifest["total_residues"]) + + +def resolve_esm_codes_database(db_path: str, max_evalue: Optional[float], *, num_nodes: int = 1 + ) -> EsmCodesQueryDatabase: + """The read-only resolution of ``query sequences --scoring esm-codes`` (plan P6.2), run on every rank before + ESM3, so a refusal stops every rank alike. Each step refuses with its own message, as a + :class:`~foldmatch.errors.RefusalError` (exit status 1, D20): + + 1. the database has a sequence store and a complete residue-code store: a database built without + ``--residue-codes`` and a partial build of the codes are told apart; + 2. the registry knows the system the manifest names (else the database comes from a newer FoldMatch); + 3. ``check_compatible`` of the code store against the system and the sequence store's meta; + 4. the native kernel loads (else the message gives the rebuild command); + 5. ``prepare`` of the system's bundle, pinned by its registry entry: downloaded only by the process that starts + the run, and found in the cache by every other rank + (:func:`~foldmatch.inference.chain_complete_inference.prepare_scoring_system`); + 6. the option combinations that depend on the database: a finite ``max_evalue`` needs residues to count. + + The live FAISS count is checked later, on rank 0, once the index is loaded. + """ + from foldmatch.search import scoring_system as ss + from foldmatch.search import swkern + from foldmatch.search.residue_code_store import ( + IncompatibleCodeStoreError, ResidueCodeStore, ResidueCodeStoreError, + ) + code_store = ResidueCodeStore.from_db_path(db_path) + database = code_store.base + # BLOSUM62's Stage 2 needs the kernel too (P9.S3), which step 4 has not loaded yet + blosum = "--scoring blosum62 can query this database (its Stage 2 needs the native kernel too)." + sequence_store = code_store.sequence_store() + if not sequence_store.exists(): + raise RefusalError( + f"--scoring esm-codes: {database} has no sequence store ({sequence_store.path}). It needs a sequence " + f"database with residue codes, built with 'fm-search build sequences --residue-codes'.") + # 1. The code store and its manifest. + from_shards = (f"'fm-search build residue-codes {code_store.db_folder} {code_store.index_name} " + f"--from-shards DIR'") + try: + manifest = code_store.manifest() + except FileNotFoundError as exc: + raise RefusalError( + f"--scoring esm-codes: {database} was built without --residue-codes: it has no residue codes " + f"({code_store.manifest_path} not found). Rebuild it with 'fm-search build sequences --residue-codes', " + f"or add the codes from the shards its build kept with {from_shards}. {blosum}") from exc + except IncompatibleCodeStoreError as exc: + raise RefusalError(f"--scoring esm-codes: {exc}. {blosum}") from exc + except ResidueCodeStoreError as exc: + raise RefusalError( + f"--scoring esm-codes: the residue codes of {database} are a partial build: " + f"{code_store.base}.rescodes.* has no manifest ({code_store.manifest_path}), so the build that wrote " + f"them stopped before they were validated. Rebuild them with {from_shards}, or rebuild the database " + f"with 'fm-search build sequences --residue-codes'. {blosum}") from exc + # 2. The registry. + name = manifest.get("system_name") + try: + system = ss.get(name) + except ss.UnknownScoringSystemError as exc: + raise RefusalError( + f"--scoring esm-codes: the residue codes of {database} were built for the scoring system {name!r}, " + f"which this FoldMatch ({__version__}) does not know (it knows: {', '.join(ss.known_names())}): the " + f"database was built by a newer FoldMatch; upgrade FoldMatch to query it with esm-codes. {blosum}" + ) from exc + # 3. The codes against the system and the sequence store. + with _refusals(): + manifest = code_store.check_compatible(system) + # 4. The kernel. + try: + kernel = swkern.load() + except swkern.KernelError as exc: + raise RefusalError( + f"--scoring esm-codes needs the native Smith-Waterman kernel, which cannot be used here: {exc}. " + f"--scoring blosum62 needs it too, for its Stage 2.") from exc + # 5. The bundle: one fetch, then cached-only on every other rank. + from foldmatch.inference.chain_complete_inference import prepare_scoring_system + from foldmatch.search.residue_shards import QUERY_ENCODING + with _refusals(): + prepared = prepare_scoring_system(system, residue_output=QUERY_ENCODING, num_nodes=num_nodes) + # 6. The option combinations the database decides. + from foldmatch.search.alignment import check_evalue_search_space + with _refusals(): + check_evalue_search_space(max_evalue, manifest["total_residues"]) + return EsmCodesQueryDatabase(db_path=db_path, sequence_store=sequence_store, code_store=code_store, + manifest=manifest, system=system, kernel=kernel, prepared=prepared) + + +def query_encodings_path(embedding_computer) -> Path: + """The per-run query-encoding memmap of an esm-codes query, next to the run's shards (deleted after the run).""" + return Path(embedding_computer.embedding_folder) / f"query_encodings_{embedding_computer.run_id}.f16" + + +def _release_pass_models() -> None: + """Rank 0, after the query pass: free the ESM3 model and the aggregator before Stage 2 (P6.3). + + Nothing holds them once the pass has returned but the reference cycles of its Lightning objects, so a collection + frees them; the GPU allocators' caches are emptied too.""" + import gc + gc.collect() + import torch + if torch.cuda.is_available() and torch.cuda.is_initialized(): + torch.cuda.empty_cache() + if getattr(torch.backends, "mps", None) is not None and torch.backends.mps.is_available(): + torch.mps.empty_cache() + + +@contextmanager +def _own_shard_removed_if_the_pass_fails(embedding_computer): + """Around the query pass of an esm-codes query: if it fails, this process deletes the shard it wrote (its query + embeddings and encodings, 128 B per residue), then the error goes on. Only a shard this run wrote: one that + existed before the pass (an earlier run's, refused by the pass) is left alone. Every rank cleans its own shard; + rank 0's ``finally`` removes the run's files once the pass has finished.""" + before = {path.name for path in embedding_computer.shard_files()} + try: + yield + except BaseException: + import torch.distributed as dist + from foldmatch.utils.dist import launch_rank + rank = dist.get_rank() if dist.is_available() and dist.is_initialized() else launch_rank() + own = Path(embedding_computer.embedding_folder) / f"{embedding_computer.out_name}-{rank}.parquet" + if own.name not in before: + try: + own.unlink(missing_ok=True) + except OSError as exc: + logging.warning(f"Could not delete the query shard {own} of the failed pass: {exc}") + raise + + +def _query_sequences_esm_codes(*, db_path: str, fasta_file: Path, tmp_embedding_folder: Path, output_file: str, + min_res_n: int, top_k: int, threshold: Optional[float], use_gpu_index: bool, + accelerator, num_nodes: int, devices, strategy, batch_size: int, num_workers: int, + nprobe: Optional[int], stage2: QueryStage2Options, align_workers: Optional[int], + format_output: str, format_mode: FormatMode) -> None: + """``query sequences --scoring esm-codes`` (plan section 3, QUERY; P6.1-P6.3). + + Every rank, read-only, before ESM3: the remaining option checks (exit status 2), the strict query pre-flight + (duplicate ids, '<' / '>' and non-ASCII sequences refused) and :func:`resolve_esm_codes_database` (exit status 1). + The ESM3 pass writes each query's chain embedding, S5 encoding and seq_sha1 into the run's shards. Then rank 0: + frees the models, loads the FAISS index and checks its count against the code store, builds the run's + query-encoding memmap in shard order, searches (Stage 1), and aligns with the codes + (:func:`~foldmatch.search.code_alignment.align_candidates_esm_codes`, N = the manifest's ``total_residues``). + The run's shards and the memmap are deleted at the end, whatever happens (G7); a process whose pass fails deletes + the shard it wrote. + """ + if align_workers is not None and align_workers < 0: + raise typer.BadParameter(f"must be >= 0, got {align_workers}.", param_hint="--align-workers") + output_fields = _output_fields(format_output) + with _refusals(): + preflight = _fasta_preflight(fasta_file, min_res_n, strict=True) + database = resolve_esm_codes_database(db_path, stage2.max_evalue, num_nodes=num_nodes) + + from foldmatch.search.embedding_computer import EmbeddingComputer + from foldmatch.search.residue_shards import QUERY_ENCODING + with _refusals(): + embedding_computer = EmbeddingComputer(embedding_folder=tmp_embedding_folder) + encodings_path = query_encodings_path(embedding_computer) + if encodings_path.exists(): + raise FileExistsError( + f"{encodings_path}, the query encodings of an earlier run with the same run id " + f"({embedding_computer.run_id}), is in the way: remove it, or use another --tmp-embedding-folder.") + with _own_shard_removed_if_the_pass_fails(embedding_computer): + embedding_computer.compute_from_fasta( + fasta_file=fasta_file, + min_res_n=min_res_n, + batch_size=batch_size, + num_workers=num_workers, + num_nodes=num_nodes, + accelerator=accelerator, + devices=arg_devices(devices), + strategy=strategy, + preflight=preflight, + residue_output=QUERY_ENCODING, + scoring_system=database.prepared, + ) + + if not _is_rank_zero(): + return + + query_index = None + try: + _release_pass_models() + logging.info("Loading database...") + from foldmatch.search.embedding_database import EmbeddingDatabase + embedding_db = EmbeddingDatabase(db_path=db_path, use_gpu_for_search=use_gpu_index) + embedding_db.load(nprobe=nprobe) + with _refusals(): + database.code_store.check_faiss_count(len(embedding_db.db.chain_ids)) + query_index = embedding_computer.build_query_encodings(encodings_path, database.system) + logging.info(f"Query encodings of {len(query_index)} sequence(s) ({query_index.rows} residues) in " + f"{encodings_path}") + results = embedding_db.search_by_embeddings( + embedding_batches=embedding_computer.get_embedding_batches(rows=True), + top_k=top_k + ) + # The shards are read rank by rank: the queries are aligned and written in input order, as by a 1-rank run. + results = embedding_computer.in_input_order(results) + # The query embeddings are only needed for this search. + embedding_computer.remove_shards() + logging.info(f"Searched {len(results)} sequence(s)") + results = _filter_results_by_threshold(results, threshold) + + from foldmatch.search import output + from foldmatch.search.code_alignment import CodeAlignmentPool, align_candidates_esm_codes + # The sequences ESM3 encoded (the strict pre-flight refused non-ASCII ones), normalised like the stored + # subjects; Stage 2 checks each against its encoding's length and seq_sha1. + query_sequences = {name: sequence for name, sequence in embedding_computer.input_sequences().items() + if name in results} + logging.info("Computing esm-codes alignments (Stage 2)...") + with CodeAlignmentPool(align_workers) as pool: + aligned = align_candidates_esm_codes( + query_index, + results, + database.code_store, + database.sequence_store.fetch, + database.prepared, + database.total_residues, + pool, + query_sequences=query_sequences, + min_seq_identity=stage2.min_seq_identity, + min_coverage=stage2.min_coverage, + max_evalue=stage2.max_evalue, + output_needs=output_fields, + ) + output.write_aligned_results(aligned, output_fields, output_file, format_mode) + finally: + # QueryEncodingIndex.build removes its own file when it fails. + if query_index is not None: + query_index.remove() + embedding_computer.remove_shards() + + def _stage2_align_and_report( embedding_db, prefilter_results, diff --git a/src/foldmatch/dataset/esm_assembly_from_structure.py b/src/foldmatch/dataset/esm_assembly_from_structure.py index 9145a39..cfff51f 100644 --- a/src/foldmatch/dataset/esm_assembly_from_structure.py +++ b/src/foldmatch/dataset/esm_assembly_from_structure.py @@ -2,15 +2,15 @@ import pandas as pd import torch -from biotite.structure import chain_iter from esm.sdk.api import ESMProtein from esm.utils.structure.protein_chain import ProteinChain from torch.utils.data import IterableDataset, get_worker_info -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import STRUCTURE_LIST_DEFAULT_NAME, get_structure_location, list_source, \ + read_structure_list, refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import StructureLocation, StructureFormat, SrcLocation, FileOrStreamTuple from foldmatch.utils.data import stringio_from_url -from foldmatch.utils.structure_parser import get_protein_chains, rename_atom_attr, filter_residues, get_assemblies +from foldmatch.utils.structure_parser import protein_chain_iter, rename_atom_attr, filter_residues, get_assemblies from foldmatch.utils.structure_provider import StructureProvider logger = logging.getLogger(__name__) @@ -29,29 +29,23 @@ def __init__( src_location: SrcLocation = SrcLocation.file, structure_format: StructureFormat = StructureFormat.mmcif, min_res_n: int = 0, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): super().__init__() self.min_res_n = min_res_n self.src_location = src_location self.structure_format = structure_format - self.__structure_provider = structure_provider + self.__structure_provider = structure_provider if structure_provider is not None else StructureProvider() self.data = pd.DataFrame() self.__load_stream(src_stream) def __load_stream(self, src_stream): - self.data = pd.DataFrame( - src_stream, - dtype=str, - columns=EsmAssemblyFromStructure.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - keep_default_na=False, - dtype=str, - names=EsmAssemblyFromStructure.COLUMNS - ) + # The structure list, whose columns are COLUMNS: a row of more than 3 fields is refused, a missing or empty + # embedding name is the structure name, and an embedding name given to more than one row is refused + self.data = read_structure_list(src_stream, self.src_location) + refuse_shared_names(self.data, EsmAssemblyFromStructure.STREAM_NAME_ATTR, EsmAssemblyFromStructure.STREAM_ATTR) + refuse_repeated_names(self.data, list_source(src_stream, self.src_location, "structure list"), + STRUCTURE_LIST_DEFAULT_NAME) def __iter__(self): # Handle multiple workers by splitting data across workers @@ -92,35 +86,32 @@ def __iter__(self): src_structure = row[EsmAssemblyFromStructure.STREAM_ATTR] item_name = row[EsmAssemblyFromStructure.ITEM_NAME_ATTR] - structure = stringio_from_url(src_structure) if get_structure_location(src_structure) == StructureLocation.remote else src_structure - for assembly_id in get_assemblies(structure=structure, structure_format=self.structure_format): + src = stringio_from_url(src_structure) if get_structure_location(src_structure) == StructureLocation.remote else src_structure + for assembly_id in get_assemblies(structure=src, structure_format=self.structure_format): structure = self.__structure_provider.get_structure( src_name=src_name, - src_structure=structure, + src_structure=src, structure_format=self.structure_format, assembly_id=assembly_id ) - # Get all protein chains from structure - chain_ids = get_protein_chains(structure, self.min_res_n) - protein_chain_list = [] - # Process each chain - for chain_id in chain_ids: - chain_structure = structure[structure.chain_id == chain_id] - for atom_ch in chain_iter(chain_structure): - if len(atom_ch) == 0: - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") - try: - atom_ch = filter_residues(atom_ch) - atom_ch = rename_atom_attr(atom_ch) - protein_chain = ProteinChain.from_atomarray(atom_ch) - protein_chain = ESMProtein.from_protein_chain(protein_chain) - except Exception as e: - raise IOError(f"Error while creating ESMProtein from structure chain {src_name}.{chain_id}: {e}") - - if len(protein_chain) == 0: - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") - protein_chain_list.append(protein_chain) - break # Only process first atom_ch (same as original logic) + # Each protein chain of the assembly, from its own atoms: the copies of a chain share its chain ID + for atom_ch in protein_chain_iter(structure, self.min_res_n): + chain_id = atom_ch.chain_id[0] + try: + atom_ch = filter_residues(atom_ch) + atom_ch = rename_atom_attr(atom_ch) + protein_chain = ProteinChain.from_atomarray(atom_ch) + protein_chain = ESMProtein.from_protein_chain(protein_chain) + except Exception as e: + raise IOError(f"Error while creating ESMProtein from structure chain {src_name}.{chain_id}: {e}") + + if len(protein_chain) == 0: + raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") + protein_chain_list.append(protein_chain) + if not protein_chain_list: + logger.warning(f"Assembly {item_name}-{assembly_id} has no protein chain of at least " + f"{self.min_res_n} residues: skipped") + continue yield protein_chain_list, f"{item_name}-{assembly_id}" diff --git a/src/foldmatch/dataset/esm_prot_from_chain.py b/src/foldmatch/dataset/esm_prot_from_chain.py index 0a1f9fd..7e5b614 100644 --- a/src/foldmatch/dataset/esm_prot_from_chain.py +++ b/src/foldmatch/dataset/esm_prot_from_chain.py @@ -1,7 +1,6 @@ import argparse import torch -from biotite.structure import chain_iter from esm.models.esm3 import ESM3 from esm.sdk.api import ESMProtein, SamplingConfig from esm.utils.constants.models import ESM3_OPEN_SMALL @@ -9,10 +8,11 @@ from torch.utils.data import Dataset, DataLoader import pandas as pd -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import CHAIN_LIST_DEFAULT_NAME, get_structure_location, list_source, read_chain_list, \ + refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import StructureFormat, StructureLocation, SrcLocation, FileOrStreamTuple from foldmatch.utils.data import stringio_from_url -from foldmatch.utils.structure_parser import rename_atom_attr,filter_residues +from foldmatch.utils.structure_parser import rename_atom_attr, filter_residues, protein_chain_iter from foldmatch.utils.structure_provider import StructureProvider @@ -30,29 +30,22 @@ def __init__( src_stream: FileOrStreamTuple, src_location:SrcLocation = SrcLocation.file, structure_format: StructureFormat = StructureFormat.mmcif, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): super().__init__() - self.__structure_provider = structure_provider + self.__structure_provider = structure_provider if structure_provider is not None else StructureProvider() self.src_location = src_location self.structure_format = structure_format self.data = pd.DataFrame() self.__load_stream(src_stream) def __load_stream(self, src_stream): - self.data = pd.DataFrame( - src_stream, - dtype=str, - columns=EsmProtFromChain.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - keep_default_na=False, - dtype=str, - names=EsmProtFromChain.COLUMNS - ) + # The chain list, whose columns are COLUMNS: a missing or empty embedding name is . + self.data = read_chain_list(src_stream, self.src_location) self.data = self.data.sort_values(by=self.data.columns[0]) + refuse_shared_names(self.data, EsmProtFromChain.STREAM_NAME_ATTR, EsmProtFromChain.STREAM_ATTR) + refuse_repeated_names(self.data, list_source(src_stream, self.src_location, "chain list"), + CHAIN_LIST_DEFAULT_NAME) def __len__(self): return len(self.data) @@ -65,25 +58,25 @@ def __getitem__(self, idx): structure = self.__structure_provider.get_structure( src_name=src_name, src_structure=stringio_from_url(src_structure) if get_structure_location(src_structure) == StructureLocation.remote else src_structure, - structure_format=self.structure_format, - chain_id=chain_id + structure_format=self.structure_format ) - for atom_ch in chain_iter(structure): - if len(atom_ch) == 0: - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") - try: - atom_ch = filter_residues(atom_ch) - atom_ch = rename_atom_attr(atom_ch) - protein_chain = ProteinChain.from_atomarray(atom_ch) - protein_chain = ESMProtein.from_protein_chain(protein_chain) - except Exception as e: - raise IOError(f"Error while creating ESMProtein from structure chain {src_name}.{chain_id}: {e}") - - if len(protein_chain) == 0: - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") - - return protein_chain, item_name - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") + # The protein chain with that chain ID, its pieces joined: found in the whole structure, because the atoms with + # that chain ID alone no longer tell where its pieces end + atom_ch = next(protein_chain_iter(structure, chain_id=chain_id), None) + if atom_ch is None: + raise IOError(f"No protein chain found for {src_name}.{chain_id} in structure {src_structure}.") + try: + atom_ch = filter_residues(atom_ch) + atom_ch = rename_atom_attr(atom_ch) + protein_chain = ProteinChain.from_atomarray(atom_ch) + protein_chain = ESMProtein.from_protein_chain(protein_chain) + except Exception as e: + raise IOError(f"Error while creating ESMProtein from structure chain {src_name}.{chain_id}: {e}") + + if len(protein_chain) == 0: + raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") + + return protein_chain, item_name if __name__ == '__main__': diff --git a/src/foldmatch/dataset/esm_prot_from_fasta.py b/src/foldmatch/dataset/esm_prot_from_fasta.py index 57c69e9..0172b5a 100644 --- a/src/foldmatch/dataset/esm_prot_from_fasta.py +++ b/src/foldmatch/dataset/esm_prot_from_fasta.py @@ -9,6 +9,7 @@ # parse_fasta from this module. The canonical implementations now live in the # dependency-light foldmatch.utils.fasta. from foldmatch.utils.fasta import iter_fasta, iter_fasta_offsets, parse_fasta, read_record_at +from foldmatch.utils.sequence import FastaPreflight, normalize_sequence, scan_fasta logger = logging.getLogger(__name__) @@ -23,6 +24,14 @@ class EsmProtFromFasta(Dataset): to the offset and reads that single record on demand, so peak memory is ~8 bytes per record regardless of sequence length — letting the inference pipeline handle FASTA files with hundreds of millions of sequences. + + Sequences are normalised (:func:`normalize_sequence`: ASCII whitespace + removed, ASCII letters uppercased) before ESM3 sees them, and + ``min_res_n`` applies to the normalised length, as in the sequence store. + + ``fasta_file`` may also be the :class:`FastaPreflight` of the file, whose + offsets come from the same scan; the pre-flight pass then doubles as the + index build and the file is not read twice. """ def __init__( @@ -31,17 +40,26 @@ def __init__( min_res_n=0 ): super().__init__() - self.fasta_file = str(fasta_file) - # np.fromiter streams the generator straight into the array without an - # intermediate Python list, keeping the index build itself low-memory. - self._offsets = np.fromiter( - ( - offset - for offset, _name, length in iter_fasta_offsets(self.fasta_file) - if min_res_n <= 0 or length >= min_res_n - ), - dtype=np.int64, - ) + if isinstance(fasta_file, FastaPreflight): + if fasta_file.min_res_n != min_res_n: + raise ValueError( + f"Pre-flight of {fasta_file.path} was made with min_res_n=" + f"{fasta_file.min_res_n}, not {min_res_n}" + ) + self.fasta_file = str(fasta_file.path) + self._offsets = fasta_file.offsets + else: + self.fasta_file = str(fasta_file) + # np.fromiter streams the generator straight into the array without an + # intermediate Python list, keeping the index build itself low-memory. + self._offsets = np.fromiter( + ( + offset + for offset, _name, length in scan_fasta(self.fasta_file) + if min_res_n <= 0 or length >= min_res_n + ), + dtype=np.int64, + ) # Per-process file handle, opened lazily. Never pickled (see __getstate__) # so each DataLoader worker opens its own handle and seeks independently. self._fh = None @@ -65,7 +83,8 @@ def _handle(self): def __getitem__(self, idx): offset = int(self._offsets[idx]) name, sequence = read_record_at(self._handle(), offset) - return ESMProtein(sequence=sequence), name + # Non-ASCII characters are kept one for one, as before (ESM3 reads them as ). + return ESMProtein(sequence=normalize_sequence(sequence, allow_non_ascii=True)), name def __getstate__(self): # Drop the open file handle so the dataset pickles cleanly to workers. diff --git a/src/foldmatch/dataset/esm_prot_from_structure.py b/src/foldmatch/dataset/esm_prot_from_structure.py index 9425ea8..3df3f0f 100644 --- a/src/foldmatch/dataset/esm_prot_from_structure.py +++ b/src/foldmatch/dataset/esm_prot_from_structure.py @@ -2,15 +2,15 @@ import pandas as pd import torch -from biotite.structure import chain_iter from esm.sdk.api import ESMProtein from esm.utils.structure.protein_chain import ProteinChain from torch.utils.data import IterableDataset, get_worker_info -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import STRUCTURE_LIST_DEFAULT_NAME, get_structure_location, list_source, \ + read_structure_list, refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import StructureLocation, StructureFormat, SrcLocation, FileOrStreamTuple from foldmatch.utils.data import stringio_from_url -from foldmatch.utils.structure_parser import get_protein_chains, rename_atom_attr, filter_residues +from foldmatch.utils.structure_parser import protein_chain_iter, rename_atom_attr, filter_residues from foldmatch.utils.structure_provider import StructureProvider logger = logging.getLogger(__name__) @@ -29,29 +29,23 @@ def __init__( src_location: SrcLocation = SrcLocation.file, structure_format: StructureFormat = StructureFormat.mmcif, min_res_n: int = 0, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): super().__init__() self.min_res_n = min_res_n self.src_location = src_location self.structure_format = structure_format - self.__structure_provider = structure_provider + self.__structure_provider = structure_provider if structure_provider is not None else StructureProvider() self.data = pd.DataFrame() self.__load_stream(src_stream) def __load_stream(self, src_stream): - self.data = pd.DataFrame( - src_stream, - dtype=str, - columns=EsmProtFromStructure.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - keep_default_na=False, - dtype=str, - names=EsmProtFromStructure.COLUMNS - ) + # The structure list, whose columns are COLUMNS: a row of more than 3 fields is refused, a missing or empty + # embedding name is the structure name, and an embedding name given to more than one row is refused + self.data = read_structure_list(src_stream, self.src_location) + refuse_shared_names(self.data, EsmProtFromStructure.STREAM_NAME_ATTR, EsmProtFromStructure.STREAM_ATTR) + refuse_repeated_names(self.data, list_source(src_stream, self.src_location, "structure list"), + STRUCTURE_LIST_DEFAULT_NAME) def __iter__(self): # Handle multiple workers by splitting data across workers @@ -99,26 +93,18 @@ def __iter__(self): structure_format=self.structure_format ) - # Get all protein chains from structure - chain_ids = get_protein_chains(structure, self.min_res_n) - - # Process each chain - for chain_id in chain_ids: - chain_structure = structure[structure.chain_id == chain_id] - - for atom_ch in chain_iter(chain_structure): - if len(atom_ch) == 0: - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") - try: - atom_ch = filter_residues(atom_ch) - atom_ch = rename_atom_attr(atom_ch) - protein_chain = ProteinChain.from_atomarray(atom_ch) - protein_chain = ESMProtein.from_protein_chain(protein_chain) - except Exception as e: - raise IOError(f"Error while creating ESMProtein from structure chain {src_name}.{chain_id}: {e}") - - if len(protein_chain) == 0: - raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") - - yield protein_chain, f"{item_name}.{chain_id}" - break # Only process first atom_ch (same as original logic) + # Each protein chain of the structure once, its pieces joined + for atom_ch in protein_chain_iter(structure, self.min_res_n): + chain_id = atom_ch.chain_id[0] + try: + atom_ch = filter_residues(atom_ch) + atom_ch = rename_atom_attr(atom_ch) + protein_chain = ProteinChain.from_atomarray(atom_ch) + protein_chain = ESMProtein.from_protein_chain(protein_chain) + except Exception as e: + raise IOError(f"Error while creating ESMProtein from structure chain {src_name}.{chain_id}: {e}") + + if len(protein_chain) == 0: + raise IOError(f"No atoms were found in structure chain {src_name}.{chain_id}") + + yield protein_chain, f"{item_name}.{chain_id}" diff --git a/src/foldmatch/dataset/resdiue_assembly_embedding_from_structure.py b/src/foldmatch/dataset/resdiue_assembly_embedding_from_structure.py index c89f63f..22bbb24 100644 --- a/src/foldmatch/dataset/resdiue_assembly_embedding_from_structure.py +++ b/src/foldmatch/dataset/resdiue_assembly_embedding_from_structure.py @@ -1,11 +1,11 @@ import sys from pathlib import Path -import pandas as pd from tqdm import tqdm from foldmatch.dataset.residue_assembly_embedding_from_tensor_file import ResidueAssemblyEmbeddingFromTensorFile -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import STRUCTURE_LIST_DEFAULT_NAME, get_structure_location, list_source, \ + read_structure_list, refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import SrcLocation, StructureLocation, StructureFormat, ResEmbeddingFormat, \ FileOrStreamTuple from foldmatch.utils.data import stringio_from_url @@ -30,7 +30,7 @@ def __init__( min_res_n: int = 0, max_res_n: int = sys.maxsize, res_embedding_format: ResEmbeddingFormat = ResEmbeddingFormat.pt, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): self.src_location = src_location self.structure_format = structure_format @@ -49,18 +49,14 @@ def __init__( def __get_assemblies(self, src_stream): assemblies = [] - data = pd.DataFrame( - src_stream, - dtype=str, - columns=ResidueAssemblyDatasetFromStructure.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - dtype=str, - names=ResidueAssemblyDatasetFromStructure.COLUMNS - ) + # The structure list, whose columns are COLUMNS: a row of more than 3 fields is refused, a missing or empty + # embedding name is the structure name, and an embedding name given to more than one row is refused + data = read_structure_list(src_stream, self.src_location) data = data.sort_values(by=data.columns[0]) + refuse_shared_names(data, ResidueAssemblyDatasetFromStructure.STREAM_NAME_ATTR, + ResidueAssemblyDatasetFromStructure.STREAM_ATTR) + refuse_repeated_names(data, list_source(src_stream, self.src_location, "structure list"), + STRUCTURE_LIST_DEFAULT_NAME) progress = tqdm(data.iterrows(), total=len(data), desc="Loading structures") for idx, row in progress: src_name = row[ResidueAssemblyDatasetFromStructure.STREAM_NAME_ATTR] diff --git a/src/foldmatch/dataset/residue_assembly_embedding_from_parquet.py b/src/foldmatch/dataset/residue_assembly_embedding_from_parquet.py index 10ea049..5a00f1c 100644 --- a/src/foldmatch/dataset/residue_assembly_embedding_from_parquet.py +++ b/src/foldmatch/dataset/residue_assembly_embedding_from_parquet.py @@ -4,13 +4,13 @@ import torch import pandas as pd -import pyarrow.parquet as pq from torch.utils.data import Dataset -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import ASSEMBLY_LIST_DEFAULT_NAME, get_structure_location, list_source, \ + read_assembly_list, refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import StructureLocation, StructureFormat, SrcLocation, FileOrStreamTuple -from foldmatch.utils.data import stringio_from_url +from foldmatch.utils.data import residue_embeddings_from_parquet, stringio_from_url from foldmatch.utils.structure_parser import get_protein_chains from foldmatch.utils.structure_provider import StructureProvider @@ -28,41 +28,34 @@ def __init__( self, src_stream: FileOrStreamTuple, parquet_path: Path, - embedding_dim: int, + *, src_location: SrcLocation = SrcLocation.file, structure_format: StructureFormat = StructureFormat.mmcif, min_res_n: int = 0, max_res_n: int = sys.maxsize, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): super().__init__() self.src_location = src_location self.structure_format = structure_format self.min_res_n = min_res_n self.max_res_n = max_res_n - self.embedding_dim = embedding_dim self.data = pd.DataFrame() self.__load_stream(src_stream) - self.__structure_provider = structure_provider - self.table = pq.read_table(parquet_path) - self._id_index = { - self.table.column('id')[i].as_py(): i - for i in range(len(self.table)) - } + self.__structure_provider = structure_provider if structure_provider is not None else StructureProvider() + # One row per chain, one list of D values per residue: the [L, D] tables by id, D from the column type. + ids, embeddings = residue_embeddings_from_parquet(parquet_path) + self._embeddings = dict(zip(ids, embeddings)) def __load_stream(self, src_stream): - self.data = pd.DataFrame( - src_stream, - dtype=str, - columns=ResidueAssemblyEmbeddingFromParquet.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - dtype=str, - names=ResidueAssemblyEmbeddingFromParquet.COLUMNS - ) + # The assembly list, whose columns are COLUMNS: a row of more than 4 fields is refused, a missing or empty + # embedding name is -, and an embedding name given to more than one row is refused + self.data = read_assembly_list(src_stream, self.src_location) self.data = self.data.sort_values(by=self.data.columns[0]) + refuse_shared_names(self.data, ResidueAssemblyEmbeddingFromParquet.STREAM_NAME_ATTR, + ResidueAssemblyEmbeddingFromParquet.STREAM_ATTR) + refuse_repeated_names(self.data, list_source(src_stream, self.src_location, "assembly list"), + ASSEMBLY_LIST_DEFAULT_NAME) def __len__(self): return len(self.data) @@ -84,12 +77,10 @@ def __getitem__(self, idx): total_residues = 0 for ch in chain_ids: lookup_id = f"{src_name}.{ch}" - if lookup_id not in self._id_index: + if lookup_id not in self._embeddings: logger.warning(f"Chain {lookup_id} not found in parquet file") continue - row_idx = self._id_index[lookup_id] - embedding_flat = self.table.column('embedding')[row_idx].as_py() - tensor = torch.tensor(embedding_flat, dtype=torch.float32).reshape(-1, self.embedding_dim) + tensor = torch.tensor(self._embeddings[lookup_id]) total_residues += tensor.shape[0] tensors.append(tensor) if total_residues > self.max_res_n: diff --git a/src/foldmatch/dataset/residue_assembly_embedding_from_tensor_file.py b/src/foldmatch/dataset/residue_assembly_embedding_from_tensor_file.py index 83cc324..b69fb96 100644 --- a/src/foldmatch/dataset/residue_assembly_embedding_from_tensor_file.py +++ b/src/foldmatch/dataset/residue_assembly_embedding_from_tensor_file.py @@ -5,7 +5,8 @@ import pandas as pd from torch.utils.data import Dataset, DataLoader -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import ASSEMBLY_LIST_DEFAULT_NAME, get_structure_location, list_source, \ + read_assembly_list, refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import StructureLocation, StructureFormat, SrcLocation, ResEmbeddingFormat, \ FileOrStreamTuple from foldmatch.utils.data import stringio_from_url, concatenate_tensors @@ -31,7 +32,7 @@ def __init__( min_res_n: int = 0, max_res_n: int = sys.maxsize, res_embedding_format: ResEmbeddingFormat = ResEmbeddingFormat.pt, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): super().__init__() self.res_embedding_location = res_embedding_location @@ -42,21 +43,17 @@ def __init__( self.res_embedding_format = res_embedding_format self.data = pd.DataFrame() self.__load_stream(src_stream) - self.__structure_provider = structure_provider + self.__structure_provider = structure_provider if structure_provider is not None else StructureProvider() def __load_stream(self, src_stream): - self.data = pd.DataFrame( - src_stream, - dtype=str, - columns=ResidueAssemblyEmbeddingFromTensorFile.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - dtype=str, - names=ResidueAssemblyEmbeddingFromTensorFile.COLUMNS - ) + # The assembly list, whose columns are COLUMNS: a row of more than 4 fields is refused, a missing or empty + # embedding name is -, and an embedding name given to more than one row is refused + self.data = read_assembly_list(src_stream, self.src_location) self.data = self.data.sort_values(by=self.data.columns[0]) + refuse_shared_names(self.data, ResidueAssemblyEmbeddingFromTensorFile.STREAM_NAME_ATTR, + ResidueAssemblyEmbeddingFromTensorFile.STREAM_ATTR) + refuse_repeated_names(self.data, list_source(src_stream, self.src_location, "assembly list"), + ASSEMBLY_LIST_DEFAULT_NAME) def __len__(self): return len(self.data) diff --git a/src/foldmatch/dataset/residue_embedding_from_parquet.py b/src/foldmatch/dataset/residue_embedding_from_parquet.py index 0e1e826..0936059 100644 --- a/src/foldmatch/dataset/residue_embedding_from_parquet.py +++ b/src/foldmatch/dataset/residue_embedding_from_parquet.py @@ -3,34 +3,35 @@ from typing import List import torch -import pyarrow as pa -import pyarrow.parquet as pq from torch.utils.data import Dataset +from foldmatch.utils.data import residue_embeddings_from_parquet + logger = logging.getLogger(__name__) class ResidueEmbeddingFromParquet(Dataset): def __init__( self, - parquet_path: Path | List[Path], - embedding_dim: int + parquet_path: Path | List[Path] ): super().__init__() - self.embedding_dim = embedding_dim if isinstance(parquet_path, (str, Path)): paths = [parquet_path] else: paths = parquet_path logger.info(f"Loading {len(paths)} parquet file(s): {[str(p) for p in paths]}") - self.table = pa.concat_tables([pq.read_table(p) for p in paths]) + # One row per chain, one list of D values per residue: the [L, D] tables, D from each file's column type. + self.ids = [] + self.embeddings = [] + for path in paths: + ids, embeddings = residue_embeddings_from_parquet(path) + self.ids.extend(ids) + self.embeddings.extend(embeddings) def __len__(self): - return len(self.table) + return len(self.ids) def __getitem__(self, idx): - item_id = self.table.column('id')[idx].as_py() - embedding_flat = self.table.column('embedding')[idx].as_py() - tensor = torch.tensor(embedding_flat, dtype=torch.float32).reshape(-1, self.embedding_dim) - return tensor, item_id + return torch.tensor(self.embeddings[idx]), self.ids[idx] diff --git a/src/foldmatch/dataset/residue_embedding_from_structure.py b/src/foldmatch/dataset/residue_embedding_from_structure.py index 7399817..085f785 100644 --- a/src/foldmatch/dataset/residue_embedding_from_structure.py +++ b/src/foldmatch/dataset/residue_embedding_from_structure.py @@ -1,11 +1,11 @@ import os from pathlib import Path -import pandas as pd from tqdm import tqdm from foldmatch.dataset.residue_embedding_from_tensor_file import ResidueEmbeddingFromTensorFile -from foldmatch.dataset.utils import get_structure_location +from foldmatch.dataset.utils import STRUCTURE_LIST_DEFAULT_NAME, get_structure_location, list_source, \ + read_structure_list, refuse_repeated_names, refuse_shared_names from foldmatch.types.api_types import SrcLocation, StructureLocation, StructureFormat, ResEmbeddingFormat, \ FileOrStreamTuple from foldmatch.utils.data import stringio_from_url @@ -29,7 +29,7 @@ def __init__( structure_format: StructureFormat = StructureFormat.mmcif, min_res_n: int = 0, res_embedding_format: ResEmbeddingFormat = ResEmbeddingFormat.pt, - structure_provider: StructureProvider = StructureProvider() + structure_provider: StructureProvider = None ): if not os.path.isdir(res_embedding_location): raise FileNotFoundError(f"Folder {res_embedding_location} does not exist") @@ -38,7 +38,7 @@ def __init__( self.structure_format = structure_format self.min_res_n = min_res_n self.res_embedding_format = res_embedding_format - self.__structure_provider = structure_provider + self.__structure_provider = structure_provider if structure_provider is not None else StructureProvider() super().__init__( src_stream=self.__get_chains(src_stream), src_location=SrcLocation.stream, @@ -47,18 +47,14 @@ def __init__( def __get_chains(self, src_stream): chains = [] - data = pd.DataFrame( - src_stream, - dtype=str, - columns=ResidueEmbeddingFromStructure.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - dtype=str, - names=ResidueEmbeddingFromStructure.COLUMNS - ) + # The structure list, whose columns are COLUMNS: a row of more than 3 fields is refused, a missing or empty + # embedding name is the structure name, and an embedding name given to more than one row is refused + data = read_structure_list(src_stream, self.src_location) data = data.sort_values(by=data.columns[0]) + refuse_shared_names(data, ResidueEmbeddingFromStructure.STREAM_NAME_ATTR, + ResidueEmbeddingFromStructure.STREAM_ATTR) + refuse_repeated_names(data, list_source(src_stream, self.src_location, "structure list"), + STRUCTURE_LIST_DEFAULT_NAME) progress = tqdm(data.iterrows(), total=len(data), desc="Loading structures") for idx, row in progress: src_name = row[ResidueEmbeddingFromStructure.STREAM_NAME_ATTR] diff --git a/src/foldmatch/dataset/residue_embedding_from_tensor_file.py b/src/foldmatch/dataset/residue_embedding_from_tensor_file.py index f56c4bb..5b2ec8e 100644 --- a/src/foldmatch/dataset/residue_embedding_from_tensor_file.py +++ b/src/foldmatch/dataset/residue_embedding_from_tensor_file.py @@ -1,6 +1,8 @@ import pandas as pd from torch.utils.data import Dataset +from foldmatch.dataset.utils import RESIDUE_FILE_LIST_DEFAULT_NAME, list_source, read_residue_file_list, \ + refuse_repeated_names from foldmatch.types.api_types import SrcLocation, ResEmbeddingFormat, FileOrStreamTuple from foldmatch.utils.data import load_residue_embedding @@ -25,17 +27,13 @@ def __init__( self.__load_stream(src_stream) def __load_stream(self, src_stream): - self.data = pd.DataFrame( - src_stream, - dtype=str, - columns=ResidueEmbeddingFromTensorFile.COLUMNS - ) if self.src_location == SrcLocation.stream else pd.read_csv( - src_stream, - header=None, - index_col=None, - names=ResidueEmbeddingFromTensorFile.COLUMNS - ) + # The residue file list, whose columns are COLUMNS, as strings: a row of more than 2 fields is refused, a + # missing or empty embedding name is the file's name without its extension, and an embedding name given to + # more than one row is refused + self.data = read_residue_file_list(src_stream, self.src_location) self.data = self.data.sort_values(by=self.data.columns[0]) + refuse_repeated_names(self.data, list_source(src_stream, self.src_location, "residue file list"), + RESIDUE_FILE_LIST_DEFAULT_NAME) def __len__(self): return len(self.data) @@ -43,4 +41,9 @@ def __len__(self): def __getitem__(self, idx): embedding_src = self.data.iloc[idx][ResidueEmbeddingFromTensorFile.FILE_ATTR] item_name = self.data.iloc[idx][ResidueEmbeddingFromTensorFile.ITEM_NAME_ATTR] - return load_residue_embedding(embedding_src, res_embedding_format=self.res_embedding_format), item_name + try: + embedding = load_residue_embedding(embedding_src, res_embedding_format=self.res_embedding_format) + except FileNotFoundError as e: + raise FileNotFoundError(f"No residue embedding file {embedding_src} for {item_name}: compute it first, or " + f"check the folder and the format of the residue embeddings.") from e + return embedding, item_name diff --git a/src/foldmatch/dataset/utils/__init__.py b/src/foldmatch/dataset/utils/__init__.py index 6508436..32bf6d5 100644 --- a/src/foldmatch/dataset/utils/__init__.py +++ b/src/foldmatch/dataset/utils/__init__.py @@ -1,4 +1,10 @@ -from foldmatch.dataset.utils.utils import get_structure_location +from foldmatch.dataset.utils.utils import ASSEMBLY_LIST_DEFAULT_NAME, CHAIN_LIST_DEFAULT_NAME, \ + RESIDUE_FILE_LIST_DEFAULT_NAME, STRUCTURE_LIST_DEFAULT_NAME, get_structure_location, list_file, list_source, \ + read_assembly_list, read_chain_list, read_residue_file_list, read_structure_list, refuse_repeated_names, \ + refuse_shared_names, row_labels -__all__ = ["get_structure_location"] +__all__ = ["ASSEMBLY_LIST_DEFAULT_NAME", "CHAIN_LIST_DEFAULT_NAME", "RESIDUE_FILE_LIST_DEFAULT_NAME", + "STRUCTURE_LIST_DEFAULT_NAME", "get_structure_location", "list_file", "list_source", "read_assembly_list", + "read_chain_list", "read_residue_file_list", "read_structure_list", "refuse_repeated_names", + "refuse_shared_names", "row_labels"] diff --git a/src/foldmatch/dataset/utils/utils.py b/src/foldmatch/dataset/utils/utils.py index 33457ab..46be759 100644 --- a/src/foldmatch/dataset/utils/utils.py +++ b/src/foldmatch/dataset/utils/utils.py @@ -1,6 +1,36 @@ import os +import re +from typing import Optional from urllib.parse import urlparse -from foldmatch.types.api_types import StructureLocation + +import pandas as pd + +from foldmatch.errors import RefusalError +from foldmatch.types.api_types import SrcLocation, StructureLocation +from foldmatch.utils.output_files import shown + +#: The columns of a chain list (read_chain_list): structure name | structure file or URL | chain ID | embedding name. +CHAIN_LIST_COLUMNS = ['stream_name', 'stream', 'chain_id', 'item_name'] +#: The fields of a chain list row, as read_chain_list's refusals name them. +_CHAIN_LIST_FIELDS = "structure name, structure file or URL, chain ID, embedding name" +#: The columns of an assembly list (read_assembly_list): structure name | structure file or URL | assembly ID | +#: embedding name. +ASSEMBLY_LIST_COLUMNS = ['stream_name', 'stream', 'assembly_id', 'item_name'] +#: The fields of an assembly list row, as read_assembly_list's refusals name them. +_ASSEMBLY_LIST_FIELDS = "structure name, structure file or URL, assembly ID, embedding name" +#: The columns of a structure list (read_structure_list): structure name | structure file or URL | embedding name. +STRUCTURE_LIST_COLUMNS = ['stream_name', 'stream', 'item_name'] +#: The fields of a structure list row, as read_structure_list's refusals name them. +_STRUCTURE_LIST_FIELDS = "structure name, structure file or URL, embedding name" +#: The columns of a residue file list (read_residue_file_list): residue embedding file | embedding name. +RESIDUE_FILE_LIST_COLUMNS = ['file', 'item_name'] +#: The fields of a residue file list row, as read_residue_file_list's refusals name them. +_RESIDUE_FILE_LIST_FIELDS = "residue embedding file, embedding name" +#: What a missing or empty embedding name is, per list, as refuse_repeated_names says it. +CHAIN_LIST_DEFAULT_NAME = "." +ASSEMBLY_LIST_DEFAULT_NAME = "-" +STRUCTURE_LIST_DEFAULT_NAME = "the structure name" +RESIDUE_FILE_LIST_DEFAULT_NAME = "the file's name without its extension" def get_structure_location(s: str) -> str: @@ -14,4 +44,229 @@ def get_structure_location(s: str) -> str: return StructureLocation.local # Neither URL nor existing file - raise ValueError(f"Structure file source is neither a recognized URL nor file: {s!r}") \ No newline at end of file + raise ValueError(f"Structure file source is neither a recognized URL nor file: {s!r}") + + +def read_chain_list(src_stream, src_location: SrcLocation = SrcLocation.file) -> pd.DataFrame: + """The rows of a chain list, in input order, one per chain, as strings (CHAIN_LIST_COLUMNS): the chain lists of + inference residue-embedding, structure-embedding, chain-embedding and complete-embedding, and of EsmProtFromChain. + + A file is CSV without a header, read with pandas, a missing or empty field read as '' (NA, nan or null are text); + a stream is a list of tuples, a short one read as a short line. The embedding name names what is computed for the + row; when it is missing or empty, it is ., the name under which the assembly readers look + for a chain's residue embedding. A row of more than 4 fields is refused (RefusalError, naming the file or the + stream and the row), and so is a file on which pandas raises any other ParserError (quoted): pandas reads the + leading fields of a first row of more than 4 as the index, which shifts the columns of every row, and stops at a + later one with a ParserError that names no file. A blank chain ID (biotite reads one as '') is a chain ID. The + readers refuse an embedding name given to more than one row (refuse_repeated_names). + """ + if src_location == SrcLocation.stream: + # A short row read as a short line + src_stream = [tuple(row) + ("",) * (len(CHAIN_LIST_COLUMNS) - len(row)) for row in src_stream] + data = _read_list(src_stream, src_location, CHAIN_LIST_COLUMNS, "chain list", _CHAIN_LIST_FIELDS) + unnamed = data['item_name'].isna() | (data['item_name'] == "") + data.loc[unnamed, 'item_name'] = data['stream_name'][unnamed] + "." + data['chain_id'][unnamed] + return data + + +def read_assembly_list(src_stream, src_location: SrcLocation = SrcLocation.file) -> pd.DataFrame: + """The rows of an assembly list, in input order, one per assembly (ASSEMBLY_LIST_COLUMNS): the assembly lists of + inference assembly-embedding and complete-embedding, and of ResidueAssemblyEmbeddingFromTensorFile and + ResidueAssemblyEmbeddingFromParquet. + + A file is CSV without a header, read with pandas as strings, a missing or empty field read as '' (NA, nan or null + are text, as in a chain list); a stream is a list of tuples, read by pandas as such (a missing element is None). + The embedding name names the row's output; when it is missing or empty, it is -, the + name fm-embedding gives an assembly. A row of more than 4 fields is refused as read_chain_list refuses one + (RefusalError, naming the file or the stream and the row), and so is a file on which pandas raises any other + ParserError (quoted). The readers refuse an embedding name given to more than one row (refuse_repeated_names). + """ + data = _read_list(src_stream, src_location, ASSEMBLY_LIST_COLUMNS, "assembly list", _ASSEMBLY_LIST_FIELDS) + _name_unnamed(data, data['stream_name'].fillna("") + "-" + data['assembly_id'].fillna("")) + return data + + +def read_structure_list(src_stream, src_location: SrcLocation = SrcLocation.file) -> pd.DataFrame: + """The rows of a structure list, in input order, one per structure (STRUCTURE_LIST_COLUMNS): the file or stream of + EsmProtFromStructure, EsmAssemblyFromStructure, ResidueEmbeddingFromStructure and + ResidueAssemblyDatasetFromStructure. + + A file is CSV without a header, read with pandas as strings, a missing or empty field read as '' (NA, nan or null + are text); a stream is a list of tuples, read by pandas as such (a missing element is None). The embedding name + names the outputs of the structure's chains or assemblies (., -); when it is missing or empty, it is the structure name. A row of more than 3 fields is + refused as read_chain_list refuses one (RefusalError, naming the file or the stream and the row), and so is a file + on which pandas raises any other ParserError (quoted). The readers refuse an embedding name given to more than one + row (refuse_repeated_names). + """ + data = _read_list(src_stream, src_location, STRUCTURE_LIST_COLUMNS, "structure list", _STRUCTURE_LIST_FIELDS) + _name_unnamed(data, data['stream_name']) + return data + + +def read_residue_file_list(src_stream, src_location: SrcLocation = SrcLocation.file) -> pd.DataFrame: + """The rows of a residue file list, in input order, one per residue embedding file (RESIDUE_FILE_LIST_COLUMNS): the + file or stream of ResidueEmbeddingFromTensorFile. + + A file is CSV without a header, read with pandas as strings, so that a numeric-looking name keeps its spelling + (007, 1.50), a missing or empty field read as '' (NA, nan or null are text); a stream is a list of tuples, read by + pandas as such (a missing element is None). The embedding name names the row's output; when it is missing or + empty, it is the residue embedding file's name without its extension (x for x.pt). A row of more than 2 fields is + refused as read_chain_list refuses one (RefusalError, naming the file or the stream and the row), and so is a file + on which pandas raises any other ParserError (quoted). The reader refuses an embedding name given to more than one + row (refuse_repeated_names). + """ + data = _read_list(src_stream, src_location, RESIDUE_FILE_LIST_COLUMNS, "residue file list", + _RESIDUE_FILE_LIST_FIELDS) + _name_unnamed(data, data['file'].map( + lambda f: os.path.splitext(os.path.basename(os.fspath(f)))[0] if isinstance(f, (str, os.PathLike)) else f)) + return data + + +def _name_unnamed(data: pd.DataFrame, default: pd.Series): + """Each row whose embedding name is missing or empty gets its default (D5).""" + unnamed = data['item_name'].isna() | (data['item_name'] == "") + if unnamed.any(): + data.loc[unnamed, 'item_name'] = default[unnamed] + + +def _read_list(src_stream, src_location: SrcLocation, columns: list, name: str, fields: str) -> pd.DataFrame: + """The rows of a list of these columns (its name and its fields as the refusals say them; the embedding name its + last column), as pandas reads them: a file as CSV without a header, as strings; a stream as a list of rows. A row of + more fields than columns is refused (RefusalError), and so is a file on which pandas raises any other ParserError: + pandas reads the leading fields of a first row of more fields as the index, which shifts the columns of every row, + and stops at a later one with a ParserError that names no file. A missing or empty field of a file is '', and NA, + nan, null and the other words pandas takes for NA by default are text (keep_default_na=False).""" + a_list = f"{'An' if name[0] in 'aeiou' else 'A'} {name}" + rule = f"{a_list} row holds at most {len(columns)} fields ({fields})" + next_field = "3rd" if len(columns) == 2 else f"{len(columns) + 1}th" + remove_the_extra_fields = f"remove the extra fields (a comma after the embedding name makes a {next_field})" + if src_location == SrcLocation.stream: + rows = list(src_stream) + longer = [k for k, row in enumerate(rows, start=1) if len(row) > len(columns)] + if longer: + listed = ", ".join(str(k) for k in longer[:5]) + (f" and {len(longer) - 5} more" if len(longer) > 5 else "") + raise RefusalError(f"The {name} stream: {len(longer)} row(s) of more than {len(columns)} elements (row(s) " + f"{listed}, counted from 1). {rule}: remove the extra elements.") + return pd.DataFrame(rows, dtype=str, columns=columns) + source = _list_source(src_stream, name) + try: + data = pd.read_csv( + src_stream, + header=None, + index_col=None, + keep_default_na=False, + dtype=str, + names=columns + ) + except pd.errors.ParserError as e: + message = " ".join(str(e).split()) + if re.search(r"Expected \d+ fields", message): + raise RefusalError(f"{source}: a row has more than {len(columns)} fields (pandas: {message}). {rule}: " + f"{remove_the_extra_fields}.") from e + raise RefusalError(f"{source}: pandas could not read it as CSV ({message}). {a_list} is CSV without a header, " + f"of at most {len(columns)} fields per row ({fields}): correct it where pandas' message " + f"points.") from e + # pandas reads the leading fields of a first row of more than len(columns) as the index, one level each. Read as + # strings, they are never a RangeIndex; read without dtype=str, pandas 3 makes one of an index column of evenly + # spaced integers (0, 1, 2). + if not isinstance(data.index, pd.RangeIndex): + raise RefusalError(f"{source}: its first row has {len(columns) + data.index.nlevels} fields. {rule}: " + f"{remove_the_extra_fields}.") + return data + + +def _list_source(src_stream, name: str) -> str: + """A list file as _read_list's refusals name it: its path, or the name of an open file; a text buffer is "The + " ("The chain list", say).""" + if isinstance(src_stream, (str, os.PathLike)): + return os.fspath(src_stream) + file_name = getattr(src_stream, "name", None) + return file_name if isinstance(file_name, str) else f"The {name}" + + +def list_source(src_stream, src_location: SrcLocation, name: str) -> str: + """A list as refusals name it: the path or the name of an open file, "The " for a text buffer, "The + stream" for a stream.""" + if src_location == SrcLocation.stream: + return f"The {name} stream" + return _list_source(src_stream, name) + + +def list_file(src_stream, src_location: SrcLocation) -> Optional[str]: + """The path of a list given as a file (a path, or an open file's name), or None: an input of the run (D2).""" + if src_location == SrcLocation.stream: + return None + if isinstance(src_stream, (str, os.PathLike)): + return os.fspath(src_stream) + file_name = getattr(src_stream, "name", None) + return file_name if isinstance(file_name, str) and os.path.isfile(file_name) else None + + +def row_labels(data: pd.DataFrame) -> list: + """The rows of a list's frame as refusals name them ("row 3", counted from 1 in input order): read_*_list's + frames keep the input order as their index, sorted or not.""" + return [f"row {k + 1}" for k in data.index] + + +def refuse_repeated_names(data: pd.DataFrame, source: str, default: str, name_column: str = 'item_name'): + """Refuse (RefusalError) a list that gives one embedding name to more than one row (D3), in any output format: + csv and pt write each row's embedding to the file of its name, so a repeated name keeps the last one; Parquet and + JSON store one row per name, which no reader can tell apart. Names are compared exactly here; the inference + functions compare them as their output folder does. ``default`` says what an empty name is.""" + names = data[name_column] + repeated = names[names.duplicated(keep=False) & names.notna()] + if repeated.empty: + return + groups = sorted(((name, sorted(k + 1 for k in rows.index)) for name, rows in repeated.groupby(repeated)), + key=lambda group: group[1][0]) + + def describe(group): + name, rows = group + return f"{name!r} (rows {', '.join(str(k) for k in rows[:-1])} and {rows[-1]})" + raise RefusalError( + f"{source}: {len(groups)} embedding name(s) given to more than one row: {shown(groups, describe, 'more names')}" + f". Each row's embedding is written under its name (the file .csv or .pt, or the id of its row in " + f"Parquet and JSON), so a repeated name keeps one of them: give each row its own name (a missing or empty name " + f"is {default})." + ) + + +def refuse_shared_names(data: pd.DataFrame, name_column: str, source_column: str): + """Refuse an input that gives one structure name to more than one source (file or URL). + + The name identifies the structure: StructureProvider keeps the last structure it read under the name (with the + format and the assembly), so that the rows of one entry, one per chain or assembly, read it once, from a path or + from a downloaded stream alike. A second source under the same name would get the first one's structure. One + file under two spellings is one source (:func:`_source_key`). A row without a name or a source is left to fail + where it is read, as before. + """ + sources = data[[name_column, source_column]].dropna().astype(str).drop_duplicates() + repeated = sources[sources.duplicated(name_column, keep=False)] + conflicts = [] + for name, group in repeated.groupby(name_column, sort=True): + if len({_source_key(source) for source in group[source_column]}) > 1: + conflicts.append((name, sorted(group[source_column]))) + if conflicts: + listed = "; ".join(f"'{name}' names {' and '.join(names)}" for name, names in conflicts[:5]) + more = f"; and {len(conflicts) - 5} more names" if len(conflicts) > 5 else "" + raise RefusalError( + f"A structure name must name one file or URL: {listed}{more}. The name identifies the structure, which is " + f"read once for all of its rows (one per chain or assembly): give each file its own name, or, in a " + f"folder of structures, keep one copy of each entry or choose one with the structure file extension " + f"option." + ) + + +def _source_key(source: str): + """A source as refuse_shared_names compares it: a URL with its scheme and host in lower case; an existing file by + its device and inode, so that a relative path, a symbolic or hard link and, on a case-insensitive volume, a case + variant are one file; any other path resolved.""" + parsed = urlparse(source) + if parsed.scheme.lower() in {'http', 'https', 'ftp'} and parsed.netloc: + return parsed._replace(scheme=parsed.scheme.lower(), netloc=parsed.netloc.lower()).geturl() + try: + stat = os.stat(source) + except OSError: + return os.path.realpath(source) + return stat.st_dev, stat.st_ino diff --git a/src/foldmatch/errors.py b/src/foldmatch/errors.py new file mode 100644 index 0000000..3882730 --- /dev/null +++ b/src/foldmatch/errors.py @@ -0,0 +1,29 @@ +"""Exceptions shared by the command-line tools and the library (plan D20). + +The command-line exit codes follow one rule: + +* a refused combination of options is a ``typer.BadParameter`` and exits with status 2 (click's usage error); +* a refused run, because the data, the database or the installation cannot serve it, is a :class:`RefusalError` + and exits with status 1, its message printed after ``Error: `` (``fm-search``'s ``main``). + +Dependency-free, so any module (and Stage-2 workers) can import it. +""" + + +class RefusalError(ValueError): + """A read-only check refused the run: the input, the database or the installation cannot serve the request. + + The message says what was found and what to do about it. Refusals come before any expensive or irreversible + step where possible (before ESM3, before an existing file is replaced). A ``ValueError``, so callers and the + command-line tools that already report ``ValueError`` as an ``Error: ...`` (exit status 1) handle it unchanged; + the refused check's own exception, when there is one, is its ``__cause__``. + """ + + +class InternalError(RuntimeError): + """An internal consistency check failed: a FoldMatch bug, not a problem with the input or the database. + + Raised instead of ``assert`` (which ``python -O`` removes) where a result would otherwise be wrong without notice, + e.g. when the esm-codes Stage-2 traceback score differs from the round-1 score behind the reported E-value (P6.4). + Not a ``ValueError``: the command-line tools do not report it as a one-line refusal. + """ diff --git a/src/foldmatch/foldmatch.py b/src/foldmatch/foldmatch.py index 523304c..d7a4b23 100644 --- a/src/foldmatch/foldmatch.py +++ b/src/foldmatch/foldmatch.py @@ -1,11 +1,13 @@ import torch -from biotite.structure import get_residues, chain_iter, filter_amino_acids -from esm.sdk.api import ESMProtein, SamplingConfig +from esm.sdk.api import ESMProtein from esm.utils.structure.protein_chain import ProteinChain from foldmatch.types.api_types import StreamSrc, StructureFormat +from foldmatch.utils.esm.loaders import per_residue_embedding from foldmatch.utils.model import get_aggregator_model, get_residue_model -from foldmatch.utils.structure_parser import get_structure_from_src, filter_residues, rename_atom_attr +from foldmatch.utils.sequence import normalize_sequence +from foldmatch.utils.structure_parser import get_structure_from_src, filter_residues, rename_atom_attr, \ + protein_chains_with_sizes class FoldMatch: @@ -57,22 +59,14 @@ def residue_embedding( assembly_id: str = None ): self.__check_residue_embedding() - structure = get_structure_from_src(src_structure, structure_format, chain_id, assembly_id) - - if self.max_res is not None and _protein_length(structure) > self.max_res: - raise ValueError(f"Protein too long for embedding (max {self.max_res} residues)") - - embedding_ch = [] - for atom_ch in chain_iter(structure): - atom_res = atom_ch[filter_amino_acids(atom_ch)] - if len(atom_res) == 0 or len(get_residues(atom_res)[0]) < self.min_res: - continue - protein_chain = ProteinChain.from_atomarray(atom_ch) - protein = ESMProtein.from_protein_chain(protein_chain) - protein_tensor = self.__residue_embedding.encode(protein) - embedding_ch.append(self.__residue_embedding.forward_and_sample( - protein_tensor, SamplingConfig(return_per_residue_embeddings=True) - ).per_residue_embedding) + structure = get_structure_from_src(src_structure, structure_format, assembly_id=assembly_id) + + # The protein chains of the whole structure, or those with chain_id (in an assembly, one per copy) + embedding_ch = [self.__chain_embedding(atom_ch) + for atom_ch in self.__protein_chains(structure, chain_id, assembly_id)] + if not embedding_ch: + chain = "protein chain" if chain_id is None else f"protein chain {chain_id}" + raise ValueError(f"No {chain} of at least {self.min_res} residues found in the structure") return torch.cat( embedding_ch, dim=0 @@ -87,26 +81,9 @@ def residue_embedding_by_chain( self.__check_residue_embedding() structure = get_structure_from_src(src_structure, structure_format) - if self.max_res is not None and _protein_length(structure) > self.max_res: - raise ValueError(f"Protein too long for embedding (max {self.max_res} residues)") - - embedding_by_chain = {} - for atom_ch in chain_iter(structure): - atom_ch = filter_residues(atom_ch) - if len(atom_ch) == 0 or len(get_residues(atom_ch)[0]) < self.min_res: - continue - chain_id_value = atom_ch.chain_id[0] - # Skip chains that don't match the requested chain_id (if specified) - if chain_id is not None and chain_id_value != chain_id: - continue - atom_ch = rename_atom_attr(atom_ch) - protein_chain = ProteinChain.from_atomarray(atom_ch) - protein = ESMProtein.from_protein_chain(protein_chain) - protein_tensor = self.__residue_embedding.encode(protein) - embedding_by_chain[chain_id_value] = self.__residue_embedding.forward_and_sample( - protein_tensor, SamplingConfig(return_per_residue_embeddings=True) - ).per_residue_embedding - return embedding_by_chain + # Each protein chain once, or the one with chain_id + return {atom_ch.chain_id[0]: self.__chain_embedding(atom_ch) + for atom_ch in self.__protein_chains(structure, chain_id)} def residue_embedding_by_assembly( self, @@ -118,21 +95,11 @@ def residue_embedding_by_assembly( self.__check_aggregator_embedding() structure = get_structure_from_src(src_structure, structure_format, assembly_id=assembly_id) - if self.max_res is not None and _protein_length(structure) > self.max_res: - raise ValueError(f"Protein too long for embedding (max {self.max_res} residues)") - - embedding_ch = [] - for atom_ch in chain_iter(structure): - atom_ch = filter_residues(atom_ch) - if len(atom_ch) == 0 or len(get_residues(atom_ch)[0]) < self.min_res: - continue - atom_ch = rename_atom_attr(atom_ch) - protein_chain = ProteinChain.from_atomarray(atom_ch) - protein = ESMProtein.from_protein_chain(protein_chain) - protein_tensor = self.__residue_embedding.encode(protein) - embedding_ch.append(self.__residue_embedding.forward_and_sample( - protein_tensor, SamplingConfig(return_per_residue_embeddings=True) - ).per_residue_embedding) + # Each protein chain of the assembly, one per copy; none, an empty dict (as residue_embedding_by_chain gives) + embedding_ch = [self.__chain_embedding(atom_ch) + for atom_ch in self.__protein_chains(structure, assembly_id=assembly_id)] + if not embedding_ch: + return {} assembly_id_value = assembly_id if assembly_id is not None else "0" return {assembly_id_value: torch.cat(embedding_ch, dim=0)} @@ -144,6 +111,9 @@ def sequence_embedding( if sequence.startswith(">"): sequence = "".join(line.strip() for line in sequence.splitlines() if not line.startswith(">")) + # The string the FASTA paths embed (S15): lowercase letters and whitespace + # would otherwise reach ESM3 as unknown residues. + sequence = normalize_sequence(sequence, allow_non_ascii=True) if len(sequence) < self.min_res: raise ValueError(f"Sequence too short for embedding (min {self.min_res} residues)") @@ -152,14 +122,7 @@ def sequence_embedding( raise ValueError(f"Sequence too long for embedding (max {self.max_res} residues)") protein = ESMProtein(sequence=sequence) - protein_tensor = self.__residue_embedding.encode(protein) - - result = self.__residue_embedding.forward_and_sample( - protein_tensor, - SamplingConfig(return_per_residue_embeddings=True) - ) - - return result.per_residue_embedding + return per_residue_embedding(self.__residue_embedding, protein) def aggregator_embedding( self, @@ -168,6 +131,33 @@ def aggregator_embedding( self.__check_aggregator_embedding() return self.__aggregator_embedding(residue_embedding) + def __protein_chains(self, structure, chain_id=None, assembly_id=None): + # The protein chains to embed (protein_chain_iter's), refused before any ESM3 pass when they have more than + # max_res residues together, counted as min_res counts them; none is never refused (each method then does what + # it does without max_res) + chains = list(protein_chains_with_sizes(structure, self.min_res, chain_id)) + n_res = sum(size for _, size in chains) + if self.max_res is not None and chains and n_res > self.max_res: + n = len(chains) + if chain_id is None: + where = "the structure" if assembly_id is None else f"assembly {assembly_id}" + counted = f"the {n} protein chain{'s' if n > 1 else ''} of {where}" + elif assembly_id is None: + counted = f"protein chain {chain_id}" + else: + counted = f"the {n} cop{'ies' if n > 1 else 'y'} of protein chain {chain_id} in assembly {assembly_id}" + raise ValueError(f"Protein too long for embedding: {counted} {'have' if n > 1 else 'has'} {n_res} " + f"residues (max {self.max_res} residues)") + return [atom_ch for atom_ch, _ in chains] + + def __chain_embedding(self, atom_ch): + # One protein chain, read as the build reads it (EsmProtFromStructure, EsmAssemblyFromStructure) + atom_ch = filter_residues(atom_ch) + atom_ch = rename_atom_attr(atom_ch) + protein_chain = ProteinChain.from_atomarray(atom_ch) + protein = ESMProtein.from_protein_chain(protein_chain) + return per_residue_embedding(self.__residue_embedding, protein) + def __check_residue_embedding(self): if self.__residue_embedding is None: self.load_residue_embedding() @@ -190,8 +180,3 @@ def _load_res_model(device=None): if not device: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") return get_residue_model(device) - - -def _protein_length(structure): - protein_atoms = structure[filter_amino_acids(structure)] - return len(get_residues(protein_atoms)[0]) diff --git a/src/foldmatch/inference/assembly_complete_inference.py b/src/foldmatch/inference/assembly_complete_inference.py index 9c4eacc..0cf8aac 100644 --- a/src/foldmatch/inference/assembly_complete_inference.py +++ b/src/foldmatch/inference/assembly_complete_inference.py @@ -8,12 +8,14 @@ from lightning import Trainer from foldmatch.dataset.esm_assembly_from_structure import EsmAssemblyFromStructure -from foldmatch.inference.utils import is_distributed +from foldmatch.dataset.utils import list_source, row_labels +from foldmatch.inference.utils import is_distributed, list_inputs from foldmatch.modules.assembly_complete_module import AssemblyCompleteModule from foldmatch.types.api_types import StructureFormat, Accelerator, Devices, Strategy, \ FileOrStreamTuple, SrcLocation, OutFormat from foldmatch.utils.data import identity_collate from foldmatch.utils.model import get_residue_model, get_aggregator_model +from foldmatch.utils.output_files import check_outputs from foldmatch.writer.batch_writer import ParquetBatchWriter, JsonStorage, TensorBatchWriter, CsvBatchWriter @@ -43,6 +45,12 @@ def predict( min_res_n=min_res_n ) + # Before the models: each structure's outputs, -, name files of their own in + # out_folder, and none an input (P9.N) + data = inference_set.data.sort_index() # in input order + check_outputs(out_folder, out_format, out_name, prefixes=list(data.item_name), separator="-", + labels=row_labels(data), source=list_source(src_stream, src_location, "structure list"), + inputs=list_inputs(data, src_stream, src_location, list_name="structure list")) logger.info("assembly-inference set running as iterator") inference_dataloader = DataLoader( diff --git a/src/foldmatch/inference/assembly_inferece.py b/src/foldmatch/inference/assembly_inferece.py index 2d891f5..b0aa41a 100644 --- a/src/foldmatch/inference/assembly_inferece.py +++ b/src/foldmatch/inference/assembly_inferece.py @@ -7,6 +7,10 @@ from foldmatch.types.api_types import FileOrStreamTuple, SrcLocation, Accelerator, Devices, Strategy, \ StructureFormat, SrcAssemblyFrom, OutFormat, ResEmbeddingFormat from foldmatch.inference.chain_inference import predict as chain_predict +from foldmatch.dataset.utils import list_source, row_labels +from foldmatch.inference.utils import list_inputs +from foldmatch.utils.output_files import check_outputs, refuse_assemblies_over_residues, \ + refuse_outputs_over_inputs, residue_files def predict( @@ -48,6 +52,23 @@ def predict( max_res_n=max_res_n, res_embedding_format=res_embedding_format ) + # Before the model: each assembly names a file of its own in out_folder, and none an input, its residue embeddings + # included (P9.N). From a structure list, the reader's rows are its own, one per assembly + data, listed = inference_set.data.sort_index(), src_from == SrcAssemblyFrom.assembly # in input order + list_name = "assembly list" if listed else "structure list" + source = list_source(src_stream, src_location, list_name) + labels = row_labels(data) if listed else None + outputs = check_outputs(out_folder, out_format, out_name, names=list(data.item_name), labels=labels, + source=source, inputs=list_inputs(data, src_stream, src_location, rows=listed, + list_name=list_name)) + refuse_assemblies_over_residues(list(data.item_name), data.stream_name, out_folder=out_folder, + out_format=out_format, res_folder=res_embedding_location, + res_format=res_embedding_format, source=source, labels=labels) + if outputs is not None and outputs.existing(): + # A residue embedding that is an output's file through a link from another folder: only a file that exists + # can be (none in a new output folder, where the residue folder is not listed) + refuse_outputs_over_inputs(outputs, residue_files(res_embedding_location, data.stream_name, + res_embedding_format)) logger.info(f"assembly-inference set contains {len(inference_set)} samples") logger.info(f"Delegating assembly-inference to chain-inference method") diff --git a/src/foldmatch/inference/chain_complete_inference.py b/src/foldmatch/inference/chain_complete_inference.py index 8834902..31b58f2 100644 --- a/src/foldmatch/inference/chain_complete_inference.py +++ b/src/foldmatch/inference/chain_complete_inference.py @@ -1,6 +1,8 @@ import logging +import os from datetime import timedelta from pathlib import Path +from typing import Optional import torch from lightning.pytorch.strategies import DDPStrategy @@ -9,12 +11,18 @@ from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure -from foldmatch.inference.utils import is_distributed +from foldmatch.dataset.utils import list_source, row_labels +from foldmatch.inference.utils import is_distributed, list_inputs from foldmatch.modules.chain_complete_module import ChainCompleteModule +from foldmatch.search import scoring_system as _ss +from foldmatch.search.residue_shards import QUERY_ENCODING, check_residue_output from foldmatch.types.api_types import StructureFormat, Accelerator, Devices, Strategy, \ SrcEsmFrom, FileOrStreamTuple, SrcLocation, OutFormat from foldmatch.utils.data import identity_collate +from foldmatch.utils.dist import launch_rank, srun_step_id from foldmatch.utils.model import get_residue_model, get_aggregator_model +from foldmatch.utils.output_files import check_outputs +from foldmatch.utils.sequence import FastaPreflight, preflight_fasta from foldmatch.writer.batch_writer import ParquetBatchWriter, JsonStorage, TensorBatchWriter, CsvBatchWriter @@ -34,9 +42,57 @@ def predict( out_name: str = 'inference', out_folder: Path = None, return_predictions: bool = True, + residue_output: Optional[str] = None, + scoring_system=None, ): + """Chain embeddings of structures or FASTA sequences (ESM3 + the residue aggregator). + + ``residue_output`` (plan P5.2) also computes a per-residue payload in the same ESM3 pass: + ``"residue_codes"`` (builds) or ``"query_encoding"`` (queries), with the encoders of ``scoring_system``. That is + the system's name, its :class:`~foldmatch.search.scoring_system.EsmCodesSystem` (exactly the registry entry) or + a :class:`~foldmatch.search.scoring_system.PreparedSystem` from :func:`prepare_scoring_system`; a build defaults + to ``DEFAULT_ESM_CODES``, a query must name the database's system. Each prediction is then + ``(chain_embeddings, ids, payloads, seq_sha1s)`` and a Parquet output gets the residue columns + (:class:`ParquetBatchWriter`). Refused before ESM3 loads, on every rank alike: structure input (the encoders + were trained on sequence-only ESM3), an output other than Parquet, a system other than the registry entry of its + name or a bundle that does not match it, and FASTA input the method cannot take (duplicate ids, ``<``/``>``, + non-ASCII; :meth:`FastaPreflight.problems`). Any accelerator works, the Apple GPU (MPS) included (D18). + + Without ``residue_output`` nothing changes: same module, same writer, same outputs byte for byte. + """ logger = logging.getLogger(__name__) + residue_output = check_residue_output(residue_output) + residue_head = None + prepared = None + if src_from == SrcEsmFrom.fasta: + # Before any model: the output name names files of the output folder, and none of them is the FASTA file + # (P9.N; the ids, and so the files of csv and pt output, are fm-embedding's to check) + fasta = src_stream.path if isinstance(src_stream, FastaPreflight) else src_stream + check_outputs(out_folder, out_format, out_name, source=str(fasta), + inputs=[("the FASTA file", fasta)] if isinstance(fasta, (str, os.PathLike)) else ()) + if residue_output is not None: + prepared = check_residue_pass(residue_output, scoring_system, src_from=src_from, out_folder=out_folder, + out_format=out_format, num_nodes=num_nodes) + src_stream = strict_fasta_preflight(src_stream, min_res_n) + logger.info(f"{residue_output} with {prepared.system.name}: loading the residue-code head") + residue_head = prepared.load_head(device=torch.device("cpu")) + elif scoring_system is not None: + raise ValueError("scoring_system is only used with a residue_output") + + if out_folder is not None: + if out_format == OutFormat.parquet: + inference_writer = ParquetBatchWriter(out_folder, out_name) if residue_output is None else \ + residue_shard_writer(out_folder, out_name, residue_output, prepared.system, min_res_n=min_res_n) + elif out_format == OutFormat.json: + inference_writer = JsonStorage(out_folder, out_name) + elif out_format == OutFormat.pt: + inference_writer = TensorBatchWriter(out_folder) + else: + inference_writer = CsvBatchWriter(out_folder) + else: + inference_writer = None + inference_set = EsmProtFromStructure( src_stream=src_stream, src_location=src_location, @@ -47,6 +103,13 @@ def predict( min_res_n=min_res_n ) + if src_from == SrcEsmFrom.structure: + # Before the models: each structure's outputs, ., name files of their own in + # out_folder, and none an input (P9.N) + data = inference_set.data.sort_index() # in input order + check_outputs(out_folder, out_format, out_name, prefixes=list(data.item_name), separator=".", + labels=row_labels(data), source=list_source(src_stream, src_location, "structure list"), + inputs=list_inputs(data, src_stream, src_location, list_name="structure list")) logger.info(f"{src_from.name}-inference set running as iterator") inference_dataloader = DataLoader( @@ -66,20 +129,14 @@ def predict( module = ChainCompleteModule( res_model=res_model, aggregator_model=aggregator_model + ) if residue_output is None else ChainCompleteModule( + res_model=res_model, + aggregator_model=aggregator_model, + residue_head=residue_head, + residue_output=residue_output, ) logger.info(f"rcsb-esm + rcsb-aggregator module ready") - if out_folder is not None: - if out_format == OutFormat.parquet: - inference_writer = ParquetBatchWriter(out_folder, out_name) - elif out_format == OutFormat.json: - inference_writer = JsonStorage(out_folder, out_name) - elif out_format == OutFormat.pt: - inference_writer = TensorBatchWriter(out_folder) - else: - inference_writer = CsvBatchWriter(out_folder) - else: - inference_writer = None if strategy == Strategy.auto and is_distributed(devices, num_nodes): strategy = DDPStrategy(timeout=timedelta(days=1)) trainer = Trainer( @@ -99,3 +156,97 @@ def predict( ) return prediction + + +def residue_shard_writer(out_folder, out_name: str, residue_output: str, system, *, + min_res_n: int) -> ParquetBatchWriter: + """The Parquet shard writer of a residue pass (:class:`ParquetBatchWriter` with the residue columns). Its pass + record (schema metadata ``foldmatch.pass``) holds ``min_res_n`` and the ESM3 revision the pass loads, the pinned + one (``load_head`` has checked it is the system's ESM3); the writer adds the device and the ranks.""" + from foldmatch.utils.esm.loaders import ESM_REVISION + return ParquetBatchWriter(out_folder, out_name, residue_output=residue_output, system=system, + min_res_n=min_res_n, esm_revision=ESM_REVISION) + + +# ===================================================================================================================== +# residue pass pre-flight: read-only and deterministic, so every rank refuses together, before ESM3 +# ===================================================================================================================== +def _env_int(name: str) -> int: + value = os.environ.get(name, "").strip() + return int(value) if value.isdigit() else 0 + + +def bundle_fetch_allowed(num_nodes: int = 1) -> bool: + """Whether this process may download the esm-codes bundle, or must find it cached. + + Only the process that starts a run fetches it: rank 0 of a single-node run whose other ranks are started later + by Lightning (they inherit the cache rank 0 filled, and check it cached-only). When the ranks are started + independently (the tasks of a multi-task ``srun`` step, a ``torchrun`` world, or the nodes of a multi-node run), + no rank fetches, so a bundle missing from the cache is refused by every rank alike instead of by all but one; + run ``fm-embedding download-models`` first there. + """ + if launch_rank() > 0 or num_nodes > 1: + return False + if srun_step_id() is not None and (_env_int("SLURM_STEP_NUM_TASKS") or _env_int("SLURM_NTASKS")) > 1: + return False + return _env_int("WORLD_SIZE") <= 1 + + +def prepare_scoring_system(scoring_system=None, *, residue_output: str = "residue_codes", + local_files_only: Optional[bool] = None, num_nodes: int = 1) -> "_ss.PreparedSystem": + """The checked bundle of the scoring system of a residue pass (:func:`scoring_system.prepare`). + + ``scoring_system`` is a name, an ``EsmCodesSystem`` (refused unless it is exactly the registry entry of its + name) or an already ``PreparedSystem`` (re-checked against the registry, not fetched again). ``None`` is the + default system for a build and refused for a query, which must use the database's system. The bundle is + fetched only where :func:`bundle_fetch_allowed` says so (``local_files_only=None``); everywhere else it must be + cached already, and the check reads and writes nothing else. + """ + residue_output = check_residue_output(residue_output) + if isinstance(scoring_system, _ss.PreparedSystem): + _ss.registered(scoring_system.system) + return scoring_system + if scoring_system is None: + if residue_output == QUERY_ENCODING: + raise ValueError("a query-encoding pass needs the scoring system of the database it searches (the one " + "its residue-code manifest names); there is no default") + system = _ss.get(_ss.DEFAULT_ESM_CODES) + elif isinstance(scoring_system, str): + system = _ss.get(scoring_system) + else: + system = _ss.registered(scoring_system) + if local_files_only is None: + local_files_only = not bundle_fetch_allowed(num_nodes) + return _ss.prepare(system, local_files_only=local_files_only) + + +def check_residue_pass(residue_output, scoring_system=None, *, src_from=SrcEsmFrom.fasta, out_folder=None, + out_format=OutFormat.parquet, num_nodes: int = 1) -> "_ss.PreparedSystem": + """The refusals of a residue pass that do not need the input (see :func:`predict`), then the prepared system.""" + residue_output = check_residue_output(residue_output) + if residue_output is None: + raise ValueError("check_residue_pass needs a residue output") + if src_from != SrcEsmFrom.fasta: + raise ValueError(f"{residue_output} needs sequence (FASTA) input: the residue-code encoders were trained on " + "sequence-only ESM3 embeddings, and structure input gives ESM3 coordinates") + if out_folder is not None and out_format != OutFormat.parquet: + raise ValueError(f"{residue_output} is written to Parquet shards only, not {OutFormat(out_format).value}") + return prepare_scoring_system(scoring_system, residue_output=residue_output, num_nodes=num_nodes) + + +def strict_fasta_preflight(src_stream, min_res_n: int) -> FastaPreflight: + """The FASTA pre-flight of a residue pass: ``src_stream`` (a path, or its :class:`FastaPreflight`) refused if it + has duplicate ids, ``<``/``>`` or non-ASCII sequences, whose residues would not line up with stored codes.""" + if isinstance(src_stream, FastaPreflight): + if src_stream.min_res_n != min_res_n: + raise ValueError(f"Pre-flight of {src_stream.path} was made with min_res_n={src_stream.min_res_n}, " + f"not {min_res_n}") + problems = src_stream.problems() + if problems: + raise ValueError(f"{src_stream.path}: " + "; ".join(problems) + ".") + return src_stream + if not isinstance(src_stream, (str, os.PathLike)): + raise ValueError(f"a residue pass reads a FASTA file, not {type(src_stream).__name__}") + preflight = preflight_fasta(Path(src_stream), min_res_n) + preflight.check(strict=True) + return preflight diff --git a/src/foldmatch/inference/chain_inference.py b/src/foldmatch/inference/chain_inference.py index 8f84a98..2fce5f2 100644 --- a/src/foldmatch/inference/chain_inference.py +++ b/src/foldmatch/inference/chain_inference.py @@ -1,4 +1,5 @@ import logging +import os from pathlib import Path import torch @@ -8,11 +9,15 @@ from foldmatch.dataset.residue_embedding_from_parquet import ResidueEmbeddingFromParquet from foldmatch.dataset.residue_embedding_from_structure import ResidueEmbeddingFromStructure from foldmatch.dataset.residue_embedding_from_tensor_file import ResidueEmbeddingFromTensorFile +from foldmatch.dataset.utils import list_source, row_labels +from foldmatch.errors import RefusalError +from foldmatch.inference.utils import list_inputs, refuse_min_res_n from foldmatch.modules.chain_module import ChainModule from foldmatch.types.api_types import Accelerator, Devices, Strategy, FileOrStreamTuple, SrcLocation, \ SrcTensorFrom, StructureFormat, OutFormat, ResEmbeddingFormat from foldmatch.utils.data import collate_embeddings from foldmatch.utils.model import get_aggregator_model +from foldmatch.utils.output_files import check_outputs, format_value, shown from foldmatch.writer.batch_writer import CsvBatchWriter, TensorBatchWriter, ParquetBatchWriter, JsonStorage @@ -34,23 +39,32 @@ def predict( out_folder: Path = None, inference_set=None, res_embedding_format: ResEmbeddingFormat = ResEmbeddingFormat.pt, - embedding_dim: int = 1536, + *, return_predictions: bool = True, ): logger = logging.getLogger(__name__) + if inference_set is not None or src_from != SrcTensorFrom.structure: + refuse_min_res_n(min_res_n, "the chains of an inference_set or of residue embedding files or Parquet " + "(src_from file or parquet)") if inference_set is None: if src_from == SrcTensorFrom.parquet: inference_set = ResidueEmbeddingFromParquet( - parquet_path=src_stream, - embedding_dim=embedding_dim + parquet_path=src_stream ) + _check_parquet_outputs(inference_set, src_stream, out_folder, out_format, out_name) elif src_from == SrcTensorFrom.file: inference_set = ResidueEmbeddingFromTensorFile( src_stream=src_stream, src_location=src_location, res_embedding_format=res_embedding_format ) + listed = inference_set.data.sort_index() # in input order + check_outputs(out_folder, out_format, out_name, names=list(listed.item_name), labels=row_labels(listed), + source=list_source(src_stream, src_location, "residue file list"), + inputs=list_inputs(listed, src_stream, src_location, + list_name="residue file list", column="file", + what="residue embedding file")) else: inference_set = ResidueEmbeddingFromStructure( src_stream=src_stream, @@ -60,6 +74,11 @@ def predict( min_res_n=min_res_n, res_embedding_format=res_embedding_format ) + # The reader's own list of residue files, ., named . + check_outputs(out_folder, out_format, out_name, names=list(inference_set.data.item_name), + source=list_source(src_stream, src_location, "structure list"), + inputs=list_inputs(inference_set.data, src_stream, src_location, list_name="structure list", + column="file", what="residue embedding file", rows=False)) logger.info(f"chain-inference set contains {len(inference_set)} samples") inference_dataloader = DataLoader( @@ -106,3 +125,27 @@ def predict( ) return prediction + + +def _check_parquet_outputs(inference_set: ResidueEmbeddingFromParquet, src_stream, out_folder, out_format, out_name): + """The checks of a run on residue embeddings in Parquet (P9.N), before the model: an id held by more than one row + of the input, in any output format (each row's output is written under its id); then those of check_outputs, the + input files among the inputs.""" + paths = [src_stream] if isinstance(src_stream, (str, os.PathLike)) else list(src_stream) + seen, repeated = set(), [] + for chain_id in inference_set.ids: + if chain_id in seen and chain_id not in repeated: + repeated.append(chain_id) + seen.add(chain_id) + if repeated: + raise RefusalError( + f"The residue embeddings in Parquet ({', '.join(str(p) for p in paths)}): {len(repeated)} id(s) held by " + f"more than one row: {shown([repr(i) for i in repeated])}. Each row's chain embedding is written under its " + f"id, so a repeated id keeps one of them: give each chain its own id." + ) + if out_folder is not None and format_value(out_format) in ("csv", "pt"): + names = list(inference_set.ids) + else: + names = None + check_outputs(out_folder, out_format, out_name, names=names, source="The residue embeddings in Parquet", + inputs=[("the residue embedding Parquet file", p) for p in paths]) diff --git a/src/foldmatch/inference/esm_inference.py b/src/foldmatch/inference/esm_inference.py index da93cb1..80db918 100644 --- a/src/foldmatch/inference/esm_inference.py +++ b/src/foldmatch/inference/esm_inference.py @@ -7,11 +7,14 @@ from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure from foldmatch.dataset.esm_prot_from_chain import EsmProtFromChain +from foldmatch.dataset.utils import list_source, row_labels +from foldmatch.inference.utils import list_inputs, refuse_min_res_n from foldmatch.modules.esm_module import EsmModule from foldmatch.types.api_types import StructureFormat, Accelerator, Devices, Strategy, \ SrcProteinFrom, FileOrStreamTuple, SrcLocation, OutFormat from foldmatch.utils.data import identity_collate from foldmatch.utils.model import get_residue_model +from foldmatch.utils.output_files import check_outputs from foldmatch.writer.batch_writer import TensorBatchWriter, CsvBatchWriter, ParquetBatchWriter, JsonStorage @@ -34,6 +37,8 @@ def predict( ): logger = logging.getLogger(__name__) + if src_from == SrcProteinFrom.chain: + refuse_min_res_n(min_res_n, "the chains of a chain list (src_from chain)") inference_set = EsmProtFromChain( src_stream=src_stream, src_location=src_location, @@ -44,6 +49,13 @@ def predict( structure_format=structure_format, min_res_n=min_res_n ) + # Before the model: each output names a file of its own in out_folder, and none an input (P9.N) + data = inference_set.data.sort_index() # in input order + list_name = "chain list" if src_from == SrcProteinFrom.chain else "structure list" + check_outputs(out_folder, out_format, out_name, source=list_source(src_stream, src_location, list_name), + labels=row_labels(data), inputs=list_inputs(data, src_stream, src_location, list_name=list_name), + **({"names": list(data.item_name)} if src_from == SrcProteinFrom.chain else + {"prefixes": list(data.item_name), "separator": "."})) if hasattr(inference_set, '__len__'): logger.info(f"esm-inference set contains {len(inference_set)} samples") else: @@ -67,7 +79,7 @@ def predict( if out_folder is not None: if out_format == OutFormat.parquet: - inference_writer = ParquetBatchWriter(out_folder, out_name) + inference_writer = ParquetBatchWriter(out_folder, out_name, residue_embeddings=True) elif out_format == OutFormat.json: inference_writer = JsonStorage(out_folder, out_name) elif out_format == OutFormat.pt: diff --git a/src/foldmatch/inference/sequence_inference.py b/src/foldmatch/inference/sequence_inference.py index e05dfb1..d62c77b 100644 --- a/src/foldmatch/inference/sequence_inference.py +++ b/src/foldmatch/inference/sequence_inference.py @@ -1,4 +1,5 @@ import logging +import os from pathlib import Path import torch @@ -10,6 +11,8 @@ from foldmatch.types.api_types import Accelerator, Devices, Strategy, OutFormat from foldmatch.utils.data import identity_collate from foldmatch.utils.model import get_residue_model +from foldmatch.utils.output_files import check_outputs +from foldmatch.utils.sequence import FastaPreflight from foldmatch.writer.batch_writer import TensorBatchWriter, CsvBatchWriter, ParquetBatchWriter, JsonStorage @@ -29,6 +32,11 @@ def predict( ): logger = logging.getLogger(__name__) + # Before the model: the output name names files of the output folder, and none of them is the FASTA file (P9.N; + # the ids, and so the files of csv and pt output, are fm-embedding's to check) + fasta = fasta_file.path if isinstance(fasta_file, FastaPreflight) else fasta_file + check_outputs(out_folder, out_format, out_name, source=str(fasta), + inputs=[("the FASTA file", fasta)] if isinstance(fasta, (str, os.PathLike)) else ()) inference_set = EsmProtFromFasta( fasta_file=fasta_file, min_res_n=min_res_n @@ -53,7 +61,7 @@ def predict( if out_folder is not None: if out_format == OutFormat.parquet: - inference_writer = ParquetBatchWriter(out_folder, out_name) + inference_writer = ParquetBatchWriter(out_folder, out_name, residue_embeddings=True) elif out_format == OutFormat.json: inference_writer = JsonStorage(out_folder, out_name) elif out_format == OutFormat.pt: diff --git a/src/foldmatch/inference/structure_inference.py b/src/foldmatch/inference/structure_inference.py index 24196ee..01b216a 100644 --- a/src/foldmatch/inference/structure_inference.py +++ b/src/foldmatch/inference/structure_inference.py @@ -7,11 +7,14 @@ from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure from foldmatch.dataset.esm_prot_from_chain import EsmProtFromChain +from foldmatch.dataset.utils import list_source +from foldmatch.inference.utils import list_inputs, refuse_min_res_n from foldmatch.modules.chain_complete_module import ChainCompleteModule from foldmatch.types.api_types import StructureFormat, Accelerator, Devices, Strategy, \ SrcProteinFrom, FileOrStreamTuple, SrcLocation from foldmatch.utils.data import identity_collate from foldmatch.utils.model import get_residue_model, get_aggregator_model +from foldmatch.utils.output_files import check_outputs from foldmatch.writer.batch_writer import ParquetBatchWriter @@ -33,6 +36,8 @@ def predict( ): logger = logging.getLogger(__name__) + if src_from == SrcProteinFrom.chain: + refuse_min_res_n(min_res_n, "the chains of a chain list (src_from chain)") inference_set = EsmProtFromChain( src_stream=src_stream, src_location=src_location, @@ -43,6 +48,11 @@ def predict( structure_format=structure_format, min_res_n=min_res_n ) + # Before the models: the output name names files of the output folder, and none of them is an input (P9.N) + list_name = "chain list" if src_from == SrcProteinFrom.chain else "structure list" + if out_name is not None: + check_outputs(out_folder, "parquet", out_name, source=list_source(src_stream, src_location, list_name), + inputs=list_inputs(inference_set.data, src_stream, src_location, list_name=list_name)) logger.info(f"structure-inference set running as iterator") inference_dataloader = DataLoader( diff --git a/src/foldmatch/inference/utils.py b/src/foldmatch/inference/utils.py index de2eeaf..bf52864 100644 --- a/src/foldmatch/inference/utils.py +++ b/src/foldmatch/inference/utils.py @@ -1,3 +1,8 @@ +import os +from typing import Optional + +from foldmatch.errors import RefusalError + def is_distributed(devices, num_nodes) -> bool: """Multi-process (DDP) run → the post-inference barrier needs more than NCCL's 30-min default.""" @@ -7,4 +12,31 @@ def is_distributed(devices, num_nodes) -> bool: return devices != 1 # >1, or -1 (= all visible) if isinstance(devices, (list, tuple)): return len(devices) > 1 - return True \ No newline at end of file + return True + + +def refuse_min_res_n(min_res_n: int, listed: str): + """min_res_n selects the protein chains of structures read whole (src_from structure): an input that lists its + chains (``listed``) would ignore it, so a value other than 0 is refused, before anything is read or loaded.""" + if min_res_n != 0: + raise RefusalError( + f"min_res_n={min_res_n} selects the protein chains of structures read whole (src_from structure), but " + f"{listed} are embedded as listed: leave min_res_n at 0, or list only the chains to embed." + ) + + +def list_inputs(data, src_stream=None, src_location=None, *, list_name: str = "list", column: Optional[str] = "stream", + what: str = "structure file", rows: bool = True) -> list: + """The inputs of a run from a list (D2), as (described, path): the list file (when the list is given as a file), + and the local files of ``column`` of its frame (structure files, residue embedding files), with their rows when + ``rows`` (the frame is the list given, not one a reader made of it).""" + from foldmatch.dataset.utils import list_file + inputs = [] + path = None if src_stream is None else list_file(src_stream, src_location) + if path is not None: + inputs.append((f"the {list_name}", path)) + if column is not None: + for k, f in zip(data.index, data[column]): + if isinstance(f, (str, os.PathLike)): + inputs.append((f"the {what} of row {k + 1}" if rows else f"the {what}", f)) + return inputs diff --git a/src/foldmatch/model/residue_code_encoder.py b/src/foldmatch/model/residue_code_encoder.py new file mode 100644 index 0000000..51c0626 --- /dev/null +++ b/src/foldmatch/model/residue_code_encoder.py @@ -0,0 +1,383 @@ +"""ESM3 residue-code encoders of the ``esm-codes-v1`` scoring system (method spec S2-S5). + +A residue's ESM3 embedding (``per_residue_embedding`` rows 1..L, ``[L, 1536]``) becomes + +* a **target code**, 64 sign bits packed into 8 bytes (``uint8[L, 8]``), stored in the database, and +* a **query encoding**, 64 fp16 numbers (``float16[L, 64]``), used at search time. + +The arithmetic is part of the method, so it is reproduced cast for cast from the research encoders +(``rcsb-embedding-search`` ``sw_mlp/sw_common.py``): + +* S2, zcw50_clean transform: ``x16 = fp16(x)``; ``xs = fp32(x16[:, dims]) - offset``; ``xw = fp32(fp16(xs @ M))``, + with ``dims`` = 0..1535 without 1361 and 1489, ``offset`` fp32[1534] and ``M`` fp32[1534, 1534]. The whitener is + **not** folded into the encoders' first layer: folding changes the rounding, and with it the codes. +* S3, encoders: ``f(x) = lin(x) + fc2(GELU_exact(fc1(x)))`` (1534 -> 1024 -> 64), float32 with no TF32, no autocast + and no dropout. ``f_q`` encodes queries, ``f_t`` targets. +* S4, target code: ``bit_k = (f_t(xw)_k >= 0)``, packed MSB-first as ``numpy.packbits(bits, axis=1)``. +* S5, query encoding: ``q = fp16(f_q(xw))``. + +The weights come from the bundle file ``esm_codes_v1_encoders.pt`` (HF repo ``rcsb/foldmatch-esm-codes``), read with +``torch.load(weights_only=True)`` by :func:`load_esm_codes_encoders`. Which revision and sha256 to load is the +scoring-system registry's business; this module only refuses files that are not a well-formed bundle of a known +architecture, and checks a sha256 when given one. +""" +from __future__ import annotations + +import contextlib +import hashlib +import os +from typing import Optional + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from foldmatch.utils import tensor_digest as _digest +from foldmatch.utils.tensor_digest import DIGEST_METHOD # noqa: F401 (public: bundle.json content_digests.method) + +ENCODERS_FORMAT = "foldmatch-esm-codes-encoders/1" +CODE_BITS = 64 +CODE_BYTES = CODE_BITS // 8 +STATE_KEYS = ("lin.weight", "lin.bias", "fc1.weight", "fc1.bias", "fc2.weight", "fc2.bias") + +#: Every encoder architecture a bundle may name, exactly as the bundle stores it (lists, not tuples: that is what a +#: saved dict holds). A bundle's ``architecture`` must equal one of these entries; the shapes are then checked tensor +#: by tensor with ``strict=True``. +ARCHITECTURES = { + "esm-codes-mlp-v1": { + "name": "esm-codes-mlp-v1", + "d_embed": 1536, + "dims_dropped": [1361, 1489], + "d_in": 1534, + "hidden": 1024, + "d_out": 64, + "activation": "gelu_exact", + "skip": "linear", + "bias": True, + "dropout": 0.0, + "encoders": ["f_q", "f_t"], + }, +} + +#: The parts of a bundle's ``conventions`` that this implementation hard-codes. A bundle that documents anything else +#: describes a different method and is refused rather than silently run with this module's arithmetic. +IMPLEMENTED_CONVENTIONS = { + ("transform", "formula"): "x16 = fp16(x); xs = fp32(x16[:, dims]) - offset; xw = fp32(fp16(xs @ M))", + ("transform", "input_cast"): "float16", + ("transform", "output_cast"): "float16", + ("transform", "folded_into_encoders"): False, + ("encoder", "formula"): "f(x) = lin(x) + fc2(GELU_exact(fc1(x)))", + ("encoder", "dtype"): "float32", + ("code", "encoder"): "f_t", + ("code", "bits"): CODE_BITS, + ("code", "bytes"): CODE_BYTES, + ("code", "sign_rule"): "bit_k = (h_k >= 0)", + ("code", "bitorder"): "big", + ("query", "encoder"): "f_q", + ("query", "formula"): "q = fp16(f_q(xw))", + ("query", "dtype"): "float16", +} + + +class EsmCodesBundleError(ValueError): + """The file is not a usable esm-codes encoder bundle (tampered, wrong format, wrong architecture...).""" + + +# ===================================================================================================================== +# Precision context: fp32 matmuls, no TF32 / bf16, no autocast +# ===================================================================================================================== +_PRECISION_PARENTS = ("", "cudnn", "mkldnn") # saved and restored, never set ("" = torch.backends itself) +_PRECISION_CHILDREN = ("cuda.matmul", "cudnn.conv", "cudnn.rnn", "mkldnn.matmul", "mkldnn.conv", "mkldnn.rnn") + + +def _backend(path: str): + obj = torch.backends + for part in filter(None, path.split(".")): + obj = getattr(obj, part) + return obj + + +def _fp32_precision_handles(paths): + """Backends exposing the per-backend ``fp32_precision`` setting (torch >= 2.9); empty on older torch.""" + handles = [] + for path in paths: + try: + obj = _backend(path) + obj.fp32_precision + except (AttributeError, RuntimeError): + continue + handles.append(obj) + return handles + + +def _read(getter): + """(True, value), or (False, None) when torch refuses the read (legacy and new TF32 APIs mixed by the caller).""" + try: + return True, getter() + except RuntimeError: + return False, None + + +@contextlib.contextmanager +def strict_fp32(*device_types: str): + """Run float32 matmuls in full IEEE fp32 and disable autocast, then restore the caller's settings exactly. + + Inside: ``torch.set_float32_matmul_precision("highest")``, cuBLAS and cuDNN TF32 off, and on torch >= 2.9 every + per-backend ``fp32_precision`` (cuda.matmul, cudnn.conv/rnn, mkldnn.matmul/conv/rnn) set to ``"ieee"``, so + neither TF32 nor oneDNN's bf16 path can be used; autocast is disabled for ``"cpu"`` and every ``device_types``. + + torch >= 2.9 keeps the legacy flags (``allow_tf32``, the matmul precision) and the new per-backend settings side + by side and raises when they disagree. Both are therefore saved and restored, each through its own API; a legacy + value that torch refuses to read (the caller mixed the APIs) is left untouched. Not thread-safe: these are global + torch settings, as for any caller of ``set_float32_matmul_precision``. + """ + parents = _fp32_precision_handles(_PRECISION_PARENTS) + children = _fp32_precision_handles(_PRECISION_CHILDREN) + saved_new = [(h, h.fp32_precision) for h in parents + children] + ok_prec, prec = _read(torch.get_float32_matmul_precision) + ok_cublas, cublas = (False, None) if children else _read(lambda: torch.backends.cuda.matmul.allow_tf32) + ok_cudnn, cudnn = _read(lambda: torch.backends.cudnn.allow_tf32) + try: + if ok_prec: + torch.set_float32_matmul_precision("highest") + if ok_cublas: + torch.backends.cuda.matmul.allow_tf32 = False + if ok_cudnn: + torch.backends.cudnn.allow_tf32 = False + for h in children: + h.fp32_precision = "ieee" + with contextlib.ExitStack() as stack: + for device_type in dict.fromkeys(("cpu",) + tuple(device_types)): + try: + stack.enter_context(torch.autocast(device_type=device_type, enabled=False)) + except RuntimeError: # a device type without autocast support has nothing to disable + pass + yield + finally: + if ok_prec: + torch.set_float32_matmul_precision(prec) + if ok_cublas: + torch.backends.cuda.matmul.allow_tf32 = cublas + if ok_cudnn: + torch.backends.cudnn.allow_tf32 = cudnn + for h, value in reversed(saved_new): + h.fp32_precision = value + + +# ===================================================================================================================== +# Modules +# ===================================================================================================================== +_BIT_WEIGHTS = (128, 64, 32, 16, 8, 4, 2, 1) + + +def packbits_rows(bits: torch.Tensor) -> torch.Tensor: + """``numpy.packbits(bits, axis=1)`` on the tensor's own device: bool ``[L, 8k]`` -> uint8 ``[L, k]``, the first + bit of each byte in its most significant position.""" + if bits.dim() != 2 or bits.shape[1] % 8: + raise ValueError(f"expected a [L, 8k] bit matrix, got shape {tuple(bits.shape)}") + rows, width = bits.shape + weights = torch.tensor(_BIT_WEIGHTS, dtype=torch.int32, device=bits.device) + return (bits.reshape(rows, width // 8, 8).to(torch.int32) * weights).sum(dim=2).to(torch.uint8) + + +class Zcw50Transform(nn.Module): + """S2 with every cast explicit: ``xw = fp32(fp16(fp32(fp16(x)[:, dims]) - offset) @ M))``.""" + + def __init__(self, dims: torch.Tensor, offset: torch.Tensor, M: torch.Tensor, d_embed: int): + super().__init__() + self.d_embed = int(d_embed) + self.register_buffer("dims", dims) + self.register_buffer("offset", offset) + self.register_buffer("M", M) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if x.dim() != 2 or x.shape[1] != self.d_embed: + raise ValueError(f"expected per-residue embeddings [L, {self.d_embed}], got shape {tuple(x.shape)}") + x16 = x.to(torch.float16) # x16 = fp16(x) + xs = x16.index_select(1, self.dims).to(torch.float32) - self.offset # fp32(x16[:, dims]) - offset + return (xs @ self.M).to(torch.float16).to(torch.float32) # fp32(fp16(xs @ M)) + + +class CodeEncoder(nn.Module): + """S3: ``f(x) = lin(x) + fc2(GELU_exact(fc1(x)))`` (the research ``sw_common.Encoder``, mlp with skip).""" + + def __init__(self, d_in: int, hidden: int, d_out: int): + super().__init__() + self.lin = nn.Linear(d_in, d_out) + self.fc1 = nn.Linear(d_in, hidden) + self.fc2 = nn.Linear(hidden, d_out) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = self.lin(x) + return y + self.fc2(F.gelu(self.fc1(x))) # GELU: approximate="none" + + +class ResidueCodeHead(nn.Module): + """The transform plus both encoders. Build it with :func:`load_esm_codes_encoders`. + + Both ``encode_*`` methods take one protein's embeddings ``[L, 1536]`` (any float dtype; S2 casts to fp16 first), + move them to the head's device, and run under :func:`strict_fp32` and ``torch.no_grad()``. Encode one protein at a + time: that is how the reference values were made, and a BLAS call's rounding may depend on the batch shape. + """ + + def __init__(self, transform: Zcw50Transform, f_q: CodeEncoder, f_t: CodeEncoder, meta: Optional[dict] = None): + super().__init__() + self.transform = transform + self.f_q = f_q + self.f_t = f_t + self.meta = dict(meta or {}) + + @property + def device(self) -> torch.device: + return self.transform.M.device + + def _check_dtypes(self): + bad = [name for name, t in list(self.named_parameters()) + [("transform.offset", self.transform.offset), + ("transform.M", self.transform.M)] + if t.dtype != torch.float32] + if bad or self.transform.dims.dtype != torch.int64: + raise RuntimeError(f"residue-code head tensors are no longer float32 / int64 ({bad or ['dims']}); " + "the method requires the exact fp32 weights (do not call .half() / .to(dtype))") + + def _xw(self, x: torch.Tensor) -> torch.Tensor: + self._check_dtypes() + return self.transform(x.to(self.device)) + + def encode_target(self, x: torch.Tensor) -> torch.Tensor: + """S4: uint8 ``[L, 8]`` packed sign bits of ``f_t``, on the head's device.""" + with strict_fp32(self.device.type), torch.no_grad(): + return packbits_rows(self.f_t(self._xw(x)) >= 0) + + def encode_query(self, x: torch.Tensor) -> torch.Tensor: + """S5: float16 ``[L, 64]`` query encoding ``fp16(f_q(xw))``, on the head's device.""" + with strict_fp32(self.device.type), torch.no_grad(): + return self.f_q(self._xw(x)).to(torch.float16) + + def content_digests(self) -> dict: + """Content hashes of the transform and both encoders (``foldmatch-tensor-digest/1``), as bundle.json lists them + under ``content_digests``; independent of the file container.""" + t = self.transform + return { + "transform_sha256": tensor_digest({"dims": t.dims, "offset": t.offset, "M": t.M}), + "query_encoder_sha256": tensor_digest(self.f_q.state_dict()), + "target_encoder_sha256": tensor_digest(self.f_t.state_dict()), + } + + +# ===================================================================================================================== +# Loading +# ===================================================================================================================== +def tensor_digest(named) -> str: + """``foldmatch-tensor-digest/1`` (:func:`foldmatch.utils.tensor_digest.tensor_digest`) of torch tensors and / or + numpy arrays: sha256 over the ASCII tag and a newline, then, for every name in sorted order, an 8-byte + little-endian length and the compact sorted-key JSON ``{"dtype", "name", "shape"}`` (numpy dtype string, + little-endian), and an 8-byte little-endian length and the raw C-order little-endian bytes.""" + return _digest.tensor_digest({name: value.detach().cpu().numpy() if isinstance(value, torch.Tensor) else value + for name, value in named.items()}) + + +def _sha256_file(path) -> str: + h = hashlib.sha256() + with open(path, "rb") as handle: + for block in iter(lambda: handle.read(1 << 24), b""): + h.update(block) + return h.hexdigest() + + +def _tensor(obj, where: str, dtype: torch.dtype, shape: tuple) -> torch.Tensor: + if not isinstance(obj, torch.Tensor): + raise EsmCodesBundleError(f"{where}: expected a tensor, got {type(obj).__name__}") + if obj.dtype != dtype or tuple(obj.shape) != tuple(shape): + raise EsmCodesBundleError(f"{where}: expected {dtype} {list(shape)}, got {obj.dtype} {list(obj.shape)}") + if obj.is_floating_point() and not bool(torch.isfinite(obj).all()): + raise EsmCodesBundleError(f"{where}: non-finite values") + return obj + + +def _encoder(state, where: str, arch: dict, device) -> CodeEncoder: + if not isinstance(state, dict) or not all(isinstance(v, torch.Tensor) for v in state.values()): + raise EsmCodesBundleError(f"{where}: expected a state_dict (a dict of tensors)") + with torch.device("meta"): # no random init (and no global RNG draw) for weights that are replaced anyway + module = CodeEncoder(arch["d_in"], arch["hidden"], arch["d_out"]) + try: + module.load_state_dict(state, strict=True, assign=True) + except RuntimeError as e: + raise EsmCodesBundleError(f"{where}: does not match architecture {arch['name']!r}: {e}") from e + for name, p in module.named_parameters(): + _tensor(p.data, f"{where}.{name}", torch.float32, tuple(p.shape)) + return module.to(device).requires_grad_(False).eval() + + +def head_from_bundle(bundle, source: str = "", device="cpu") -> ResidueCodeHead: + """Build the head from an already ``weights_only``-loaded bundle dict (see :func:`load_esm_codes_encoders`).""" + if not isinstance(bundle, dict): + raise EsmCodesBundleError(f"{source}: not an esm-codes encoder bundle (a {type(bundle).__name__})") + if "format" not in bundle: + kind = ("a bare state_dict" if bundle and all(isinstance(v, torch.Tensor) for v in bundle.values()) + else "a dict without a format tag") + raise EsmCodesBundleError(f"{source}: {kind}, not an esm-codes encoder bundle ({ENCODERS_FORMAT})") + if bundle["format"] != ENCODERS_FORMAT: + raise EsmCodesBundleError(f"{source}: unknown format {bundle['format']!r}; this FoldMatch reads " + f"{ENCODERS_FORMAT!r} (a newer bundle needs a newer FoldMatch)") + arch = bundle.get("architecture") + name = arch.get("name") if isinstance(arch, dict) else None + if name not in ARCHITECTURES: + raise EsmCodesBundleError(f"{source}: unknown encoder architecture {name!r}; known: {sorted(ARCHITECTURES)}") + if arch != ARCHITECTURES[name]: + diff = sorted(k for k in set(arch) | set(ARCHITECTURES[name]) if arch.get(k) != ARCHITECTURES[name].get(k)) + raise EsmCodesBundleError(f"{source}: architecture {name!r} differs from the known one in {diff}") + conventions = bundle.get("conventions") + if not isinstance(conventions, dict): + raise EsmCodesBundleError(f"{source}: no conventions") + for (section, key), want in IMPLEMENTED_CONVENTIONS.items(): + got = conventions.get(section, {}).get(key) if isinstance(conventions.get(section), dict) else None + if got != want or type(got) is not type(want): + raise EsmCodesBundleError(f"{source}: convention {section}.{key} = {got!r}; this implementation does " + f"{want!r}") + missing = [k for k in ("transform", "f_q", "f_t") if k not in bundle] + if missing: + raise EsmCodesBundleError(f"{source}: missing {missing}") + tr = bundle["transform"] + if not isinstance(tr, dict) or set(tr) != {"dims", "offset", "M"}: + raise EsmCodesBundleError(f"{source}: transform must hold exactly dims, offset and M") + d_in = arch["d_in"] + dims = _tensor(tr["dims"], f"{source}: transform.dims", torch.int64, (d_in,)) + keep = np.setdiff1d(np.arange(arch["d_embed"]), np.asarray(arch["dims_dropped"])) + if len(keep) != d_in or not np.array_equal(dims.cpu().numpy(), keep): + raise EsmCodesBundleError(f"{source}: transform.dims is not 0..{arch['d_embed'] - 1} without " + f"{arch['dims_dropped']}") + offset = _tensor(tr["offset"], f"{source}: transform.offset", torch.float32, (d_in,)) + M = _tensor(tr["M"], f"{source}: transform.M", torch.float32, (d_in, d_in)) + transform = Zcw50Transform(dims.clone(), offset.clone(), M.clone(), arch["d_embed"]) + f_q = _encoder(bundle["f_q"], f"{source}: f_q", arch, device) + f_t = _encoder(bundle["f_t"], f"{source}: f_t", arch, device) + meta = {k: bundle.get(k) for k in ("format", "system", "architecture", "conventions", "provenance")} + meta["source"] = source + head = ResidueCodeHead(transform, f_q, f_t, meta).to(device) + for module in head.modules(): # the architecture has none; inference never drops anything + if isinstance(module, nn.Dropout): + module.p = 0.0 + return head.eval() + + +def load_esm_codes_encoders(path, *, sha256: Optional[str] = None, device="cpu") -> ResidueCodeHead: + """Load ``esm_codes_v1_encoders.pt`` with ``torch.load(weights_only=True)`` and build the head. + + Refused (:class:`EsmCodesBundleError`): a sha256 other than ``sha256`` (when given), anything ``weights_only`` + cannot read, a bare state_dict, an unknown format tag, an unknown or altered architecture, conventions this module + does not implement, and tensors of the wrong dtype, shape or with non-finite values. Parameters are loaded with + ``strict=True``; the head comes back in eval mode on ``device`` with gradients off. + """ + path = os.fspath(path) + if sha256 is not None: + got = _sha256_file(path) + if got != sha256.lower(): + raise EsmCodesBundleError(f"{path}: sha256 {got} does not match the expected {sha256.lower()} " + "(tampered, truncated or a different file)") + try: + bundle = torch.load(path, map_location="cpu", weights_only=True) + except Exception as e: # noqa: BLE001 (pickle / zip / unsafe-global errors all mean "not a usable bundle") + raise EsmCodesBundleError(f"{path}: not readable with torch.load(weights_only=True): " + f"{type(e).__name__}: {e}") from e + return head_from_bundle(bundle, source=path, device=device) diff --git a/src/foldmatch/model/residue_embedding_aggregator.py b/src/foldmatch/model/residue_embedding_aggregator.py index 6f18488..0711ac0 100644 --- a/src/foldmatch/model/residue_embedding_aggregator.py +++ b/src/foldmatch/model/residue_embedding_aggregator.py @@ -46,8 +46,20 @@ def __init__( self.embedding = nn.Sequential(res_block) def forward(self, x, x_mask=None): - if x.dim() == 2: - return self.embedding(self.transformer(x, src_key_padding_mask=x_mask).sum(dim=0)) - if x.dim() == 3: - return self.embedding(self.transformer(x, src_key_padding_mask=x_mask).sum(dim=1)) - raise RuntimeError("Tensor dimension error. Allowed shapes (batch, sequence, residue_embeddings) or (sequence, residue_embeddings)") + if x.dim() not in (2, 3): + raise RuntimeError("Tensor dimension error. Allowed shapes (batch, sequence, residue_embeddings) or (sequence, residue_embeddings)") + # PyTorch's nested-tensor fast path exists only on CPU and CUDA; elsewhere (Apple GPU) + # its mask check calls aten::_nested_tensor_from_mask_left_aligned, which is not + # implemented there, before the device is even checked. Skip it for the call. + use_nested_tensor = self.transformer.use_nested_tensor + if x.device.type not in ("cpu", "cuda"): + self.transformer.use_nested_tensor = False + try: + x = self.transformer(x, src_key_padding_mask=x_mask) + finally: + self.transformer.use_nested_tensor = use_nested_tensor + # The fast path returns padded rows as zeros; any other path leaves values there. + # Zero them so every path sums the same rows (a no-op after the fast path). + if x_mask is not None: + x = x.masked_fill(x_mask.unsqueeze(-1), 0.0) + return self.embedding(x.sum(dim=x.dim() - 2)) diff --git a/src/foldmatch/modules/assembly_complete_module.py b/src/foldmatch/modules/assembly_complete_module.py index 9232b3d..22a1091 100644 --- a/src/foldmatch/modules/assembly_complete_module.py +++ b/src/foldmatch/modules/assembly_complete_module.py @@ -1,10 +1,11 @@ import logging -from esm.sdk.api import SamplingConfig from lightning import LightningModule from torch import cat from foldmatch.utils.data import collate_seq_embeddings +from foldmatch.utils.esm.loaders import per_residue_embedding +from foldmatch.utils.model import release_mps_cache logger = logging.getLogger(__name__) @@ -31,9 +32,7 @@ def predict_step(self, prot_batch, batch_idx): prot_embeddings = [] assembly_n_res = 0 for esm_prot in esm_chains: - embeddings = self.esm3.forward_and_sample( - self.esm3.encode(esm_prot), SamplingConfig(return_per_residue_embeddings=True) - ).per_residue_embedding + embeddings = per_residue_embedding(self.esm3, esm_prot) prot_embeddings.append(embeddings) assembly_n_res += embeddings.shape[0] if 0 < self.max_res_n < assembly_n_res: @@ -41,5 +40,6 @@ def predict_step(self, prot_batch, batch_idx): assembly_embeddings.append(cat(prot_embeddings, dim=0)) assembly_names.append(name) res_batch_embedding, res_batch_mask = collate_seq_embeddings(assembly_embeddings) - - return self.aggregator(res_batch_embedding, res_batch_mask), tuple(assembly_names) \ No newline at end of file + assembly_batch_embedding = self.aggregator(res_batch_embedding, res_batch_mask) + release_mps_cache(self.device) + return assembly_batch_embedding, tuple(assembly_names) \ No newline at end of file diff --git a/src/foldmatch/modules/chain_complete_module.py b/src/foldmatch/modules/chain_complete_module.py index 955b8dc..43c6035 100644 --- a/src/foldmatch/modules/chain_complete_module.py +++ b/src/foldmatch/modules/chain_complete_module.py @@ -1,34 +1,111 @@ +import hashlib import logging -from esm.sdk.api import SamplingConfig +import numpy as np from lightning import LightningModule +from foldmatch.search.residue_shards import QUERY_ENCODING_DTYPE, RESIDUE_CODES, check_residue_output from foldmatch.utils.data import collate_seq_embeddings +from foldmatch.utils.esm.loaders import per_residue_embedding +from foldmatch.utils.model import release_mps_cache +from foldmatch.utils.sequence import normalize_sequence, seq_sha1 logger = logging.getLogger(__name__) +# Rows whose residue payload failed are logged one by one up to this many per rank, then only counted. +_LOGGED_FAILURES = 10 + + class ChainCompleteModule(LightningModule): + """ESM3 per-residue embeddings -> chain embeddings, optionally with a per-residue payload (plan P5.1). + + The aggregator always gets ESM3's per-residue embeddings unchanged, BOS and EOS included. With + ``residue_output`` (``"residue_codes"`` or ``"query_encoding"``) and the ``residue_head`` of the scoring system + (:class:`foldmatch.model.residue_code_encoder.ResidueCodeHead`), the rows ``1..L`` of each protein (S1) are also + encoded, one protein at a time: S4 target codes (``L * 8`` bytes) or S5 query encodings (``float16[L, 64]`` + little-endian, ``L * 128`` bytes). The step then returns ``(chain_embeddings, names, payloads, seq_sha1s)`` + instead of ``(chain_embeddings, names)``, ``seq_sha1`` being the SHA-1 of the normalised sequence (S15). + + ``predict_step`` never raises for a residue payload: under DDP a rank that raised would leave the others waiting + for the timeout. A protein whose ESM3 rows are not ``L + 2`` (the tokenizer did not keep one token per residue), + whose sequence is not ASCII, or whose encoding failed gets an empty payload instead, which the code-store + validation refuses on rank 0 after the pass. + """ def __init__( self, res_model, - aggregator_model + aggregator_model, + residue_head=None, + residue_output=None, ): super().__init__() self.esm3 = res_model self.aggregator = aggregator_model + residue_output = check_residue_output(residue_output) + if (residue_head is None) != (residue_output is None): + raise ValueError("residue_head and residue_output go together: pass both (a residue pass) or neither") + self.residue_head = residue_head + self.residue_output = residue_output + self.residue_failures = 0 def on_predict_start(self): logger.info(f"ESM + Aggregator device: {self.device} | {self.trainer.global_rank + 1}/{self.trainer.world_size}") + def on_predict_end(self): + if self.residue_failures: + logger.error( + f"{self.residue_failures} protein(s) on this rank got an empty {self.residue_output} payload; " + f"the residue store validation will refuse them" + ) + def predict_step(self, prot_batch, batch_idx): prot_embeddings = [] prot_names = [] for esm_prot, name in prot_batch: - embeddings = self.esm3.forward_and_sample( - self.esm3.encode(esm_prot), SamplingConfig(return_per_residue_embeddings=True) - ).per_residue_embedding + embeddings = per_residue_embedding(self.esm3, esm_prot) prot_embeddings.append(embeddings) prot_names.append(name) res_batch_embedding, res_batch_mask = collate_seq_embeddings(prot_embeddings) - return self.aggregator(res_batch_embedding, res_batch_mask), tuple(prot_names) \ No newline at end of file + chain_embeddings = self.aggregator(res_batch_embedding, res_batch_mask) + if self.residue_output is None: + release_mps_cache(self.device) + return chain_embeddings, tuple(prot_names) + payloads = [] + digests = [] + for (esm_prot, name), embeddings in zip(prot_batch, prot_embeddings): + payload, digest = self._residue_payload(esm_prot.sequence, name, embeddings) + payloads.append(payload) + digests.append(digest) + release_mps_cache(self.device) + return chain_embeddings, tuple(prot_names), tuple(payloads), tuple(digests) + + def _residue_payload(self, sequence, name, embeddings): + """``(payload bytes, 20-byte seq_sha1)`` of one protein; an empty payload instead of any exception.""" + sequence = sequence or "" + if not sequence.isascii(): + # S15 refuses non-ASCII input before the pass; the row still gets a 20-byte seq_sha1 (of the UTF-8). + digest = hashlib.sha1(normalize_sequence(sequence, allow_non_ascii=True).encode("utf-8", "surrogatepass")) + return self._failed(name, "the sequence is not ASCII"), digest.digest() + digest = seq_sha1(sequence) + if embeddings.dim() != 2 or embeddings.shape[0] != len(sequence) + 2: + return self._failed(name, f"ESM3 gave {tuple(embeddings.shape)} rows for {len(sequence)} residues " + f"(expected {len(sequence) + 2} with BOS and EOS)"), digest + try: + rows = embeddings[1:-1] # S1: residues 1..L, BOS and EOS removed (a view; the aggregator input is untouched) + if self.residue_output == RESIDUE_CODES: + payload = self.residue_head.encode_target(rows).cpu().numpy() + else: + payload = np.ascontiguousarray(self.residue_head.encode_query(rows).cpu().numpy(), + dtype=QUERY_ENCODING_DTYPE) + return payload.tobytes(), digest + except Exception as e: # noqa: BLE001 (never raise inside the distributed pass; validated after it) + return self._failed(name, f"{type(e).__name__}: {e}"), digest + + def _failed(self, name, reason): + self.residue_failures += 1 + if self.residue_failures <= _LOGGED_FAILURES: + logger.error(f"{name}: no {self.residue_output} payload ({reason}); the row is written empty") + elif self.residue_failures == _LOGGED_FAILURES + 1: + logger.error(f"more proteins without a {self.residue_output} payload; only counting them from now on") + return b"" diff --git a/src/foldmatch/modules/esm_module.py b/src/foldmatch/modules/esm_module.py index fd09dcc..f2dffab 100644 --- a/src/foldmatch/modules/esm_module.py +++ b/src/foldmatch/modules/esm_module.py @@ -1,8 +1,10 @@ import logging -from esm.sdk.api import SamplingConfig from lightning import LightningModule +from foldmatch.utils.esm.loaders import per_residue_embedding +from foldmatch.utils.model import release_mps_cache + logger = logging.getLogger(__name__) class EsmModule(LightningModule): @@ -18,9 +20,9 @@ def on_predict_start(self): logger.info(f"ESM device: {self.device} | {self.trainer.global_rank + 1}/{self.trainer.world_size}") def predict_step(self, prot_batch, batch_idx): - return tuple([self.__compute_embeddings(esm_prot) for esm_prot, name in prot_batch]), tuple([name for esm_prot, name in prot_batch]) + embeddings = tuple([self.__compute_embeddings(esm_prot) for esm_prot, name in prot_batch]) + release_mps_cache(self.device) + return embeddings, tuple([name for esm_prot, name in prot_batch]) def __compute_embeddings(self, esm_prot): - return self.esm3.forward_and_sample( - self.esm3.encode(esm_prot), SamplingConfig(return_per_residue_embeddings=True) - ).per_residue_embedding \ No newline at end of file + return per_residue_embedding(self.esm3, esm_prot) \ No newline at end of file diff --git a/src/foldmatch/search/alignment.py b/src/foldmatch/search/alignment.py index 2f19967..07de47c 100644 --- a/src/foldmatch/search/alignment.py +++ b/src/foldmatch/search/alignment.py @@ -1,19 +1,29 @@ import logging import math import os +import sys import time from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, Union import numpy as np from foldmatch.types.api_types import SignificanceMode -from . import karlin_altschul +from . import karlin_altschul, swint from .output import DEFAULT_OUTPUT_FIELDS logger = logging.getLogger(__name__) +# Stage 2 of a BLOSUM62 search aligns every Stage-1 candidate with a local +# Smith-Waterman (affine gaps, a gap of length L costs open + (L-1)*extend) on +# the native integer kernel fmswi_align (:mod:`swint`; model in fmsw.h). Its +# output is that of biotite 1.7.1's align_optimal(local=True, max_number=1): +# the same score, end cell and path, hence the same columns. There is no other +# engine: a missing or stale library raises swkern.KernelError with the +# rebuild command, as for esm-codes. biotite still supplies the BLOSUM62 +# matrix and the 'sampled' significance estimate, and is the tests' oracle. + # Significance (p-/E-values, bit scores) comes in two flavors, selected by # ``significance_mode``: # @@ -85,13 +95,25 @@ def _available_cpus() -> int: return os.cpu_count() or 1 +def _pool_start_method() -> str: + """Start method of the Stage-2 worker pool: the platform default up to Python 3.13. + + That is ``'fork'`` on Linux (and other POSIX systems) and ``'spawn'`` on + macOS and Windows. Pinned through ``multiprocessing.get_context`` instead of + inherited, so Python 3.14's switch of the POSIX default to ``'forkserver'`` + (or a ``set_start_method`` call by an embedding application) cannot silently + change how the workers start. + """ + return "spawn" if sys.platform in ("darwin", "win32") else "fork" + + def _resolve_num_workers(requested: Optional[int], n_tasks: int) -> int: """Resolve the worker count: ``None`` -> all CPUs; never more than tasks.""" avail = _available_cpus() if requested is None else requested return max(1, min(avail, n_tasks)) -def _expand_cpu_affinity() -> int: +def _expand_cpu_affinity(purpose: str = "alignment") -> int: """Widen this process's CPU affinity mask to all CPUs, returning the count. A process launched under a binding scheduler (e.g. NERSC ``srun`` pins each @@ -103,7 +125,8 @@ def _expand_cpu_affinity() -> int: This is safe even on shared/allocated machines: the kernel intersects the requested set with the process's cpuset cgroup, so ``sched_setaffinity`` cannot escape a real allocation — it just un-does soft ``--cpu-bind`` masks. - No-op where ``sched_setaffinity`` is unavailable (non-Linux). + No-op where ``sched_setaffinity`` is unavailable (non-Linux). ``purpose`` + names what the CPUs are widened for in the log line. """ n = os.cpu_count() or 1 if not hasattr(os, "sched_setaffinity"): @@ -116,7 +139,7 @@ def _expand_cpu_affinity() -> int: return before after = len(os.sched_getaffinity(0)) if after != before: - logger.info(f"Widened CPU affinity for alignment: {before} -> {after} CPUs") + logger.info(f"Widened CPU affinity for {purpose}: {before} -> {after} CPUs") return after # The biotite ProteinSequence alphabet (20 canonical + ambiguity codes B/Z/X and @@ -147,9 +170,13 @@ def _expand_cpu_affinity() -> int: _COMP_BIAS_WINDOW = 40 # Per-process globals populated by the multiprocessing initializer so the -# (immutable, picklable-but-large) substitution matrix is built once per worker. -_MATRIX = None -_GAP: Tuple[int, int] = (-11, -1) +# kernel and the substitution matrix are loaded once per worker. +# The integer kernel (swint.IntKernel) and biotite's BLOSUM62 as an int32 +# 24x24 table in swint.ALPHABET order (the rows of every query profile). +_INT_KERNEL: Optional["swint.IntKernel"] = None +_INT_MATRIX: Optional[np.ndarray] = None +# Positive gap costs (open, extend), as the kernel takes them. +_GAP_COSTS: Tuple[int, int] = (11, 1) # Karlin-Altschul E-value estimator (biotite) and the subject-DB residue total, # also per-process. _ESTIMATOR is None unless significance_mode='sampled'. _ESTIMATOR = None @@ -167,6 +194,15 @@ def _expand_cpu_affinity() -> int: # Compositional bias correction scale, or None when the correction is off; set # per-process by the pool initializer. _COMP_BIAS_SCALE: Optional[float] = None +# A pool worker whose initializer failed keeps the error here and every task it +# gets raises it: an initializer that raised would make multiprocessing.Pool +# start a new worker for ever, and Stage 2 would hang instead of failing. +_INIT_ERROR: Optional[BaseException] = None +# Cell-based dispatch (as the esm-codes Stage 2): tasks of contiguous candidates +# of at most max(floor, total / (TASKS_PER_WORKER * workers)) DP cells, queued +# longest first. +_TASK_CELLS_FLOOR = 1 << 20 +_TASKS_PER_WORKER = 4 # MMseqs2's (integer BLOSUM62, background) pair for the correction, built lazily. _MMSEQS_SCORING: Optional[Tuple[np.ndarray, np.ndarray]] = None @@ -179,13 +215,23 @@ class AlignmentMetrics: scalar fields are always computed (cheap); the heavy strings (``cigar``, ``q_aln``, ``t_aln``, ``q_seq``, ``t_seq``) are ``None`` unless the active output format requested them (see :class:`_FieldNeeds`). + + The same record carries both Stage-2 scorings. Under BLOSUM62 ``score`` is + the integer Smith-Waterman score and the esm-codes fields at the end stay + ``None``. Under esm-codes (:mod:`foldmatch.search.code_alignment`, method + S9-S13) ``score`` is the float ``S_f`` (an exact float32 value) and + ``score_rev`` the float ``S_r``, so ``S = score - score_rev`` in float64; + ``q90``, ``score_adj`` (``S' = S - Q90``), ``log_evalue`` (ln E), + ``bit_score`` and ``evalue`` are ``None`` when the panel pass was skipped + (D11), and ``ecal`` tells whether the hit is inside the calibrated domain + (S13). """ identity_aln: float # identical positions / alignment length == fident identity_shorter: float # identical positions / length of the shorter sequence query_coverage: float # aligned query residues / full query length == qcov subject_coverage: float # aligned subject residues / full subject length == tcov aln_len: int # alignment length (columns, gaps included) == alnlen - score: int # raw Smith-Waterman score (comp-bias corrected when on) == raw + score: Union[int, float] # raw score == raw: the int SW score (comp-bias corrected when on); esm-codes S_f # Karlin-Altschul significance. Scale depends on significance_mode: # calibrated ('default') or relative-only ('sampled'). # Never BLAST-comparable — see module header. @@ -211,6 +257,12 @@ class AlignmentMetrics: t_aln: Optional[str] = None q_seq: Optional[str] = None t_seq: Optional[str] = None + # esm-codes only (None under BLOSUM62). + score_rev: Optional[float] = None # S_r, the score-only run on the row-reversed similarity == rawrev + q90: Optional[float] = None # the query's panel quantile Q90 (S10) == q90 + score_adj: Optional[float] = None # S' = fp64(S_f) - fp64(S_r) - Q90 (S11) == rawadj + log_evalue: Optional[float] = None # ln E (S11); evalue is rendered from it, so it never underflows + ecal: Optional[bool] = None # inside the calibrated domain (S13) == ecal @dataclass @@ -226,19 +278,14 @@ def sanitize_sequence(sequence: str) -> str: return "".join(c if c in _BIOTITE_ALPHABET else "X" for c in sequence.strip().upper()) -def _to_protein(sequence: str): - """Build a biotite ProteinSequence from a sequence string. +def _sanitized_bytes(sequence: str) -> bytes: + """``sanitize_sequence(sequence)`` as ASCII bytes (a byte-table fast path for ASCII text). - Returns ``(protein, clean_str, length)``; ``(None, "", 0)`` for an empty - sequence. The sanitized ``clean_str`` is what is actually aligned, so the - ``qseq``/``tseq``/``qaln``/``taln`` columns and the position indices are all - derived from it (keeping ``qaln`` a gapped substring of ``qseq``). + The sanitized text is what is actually aligned, so the ``qseq``/``tseq``/``qaln``/``taln`` columns and the + position indices are all derived from it (keeping ``qaln`` a gapped substring of ``qseq``). """ - import biotite.sequence as seq - clean = sanitize_sequence(sequence) - if not clean: - return None, "", 0 - return seq.ProteinSequence(clean), clean, len(clean) + clean = swint.sanitize_bytes(sequence) + return clean if clean is not None else sanitize_sequence(sequence).encode("ascii") def _mmseqs_blosum62() -> Tuple[np.ndarray, np.ndarray]: @@ -310,100 +357,6 @@ def local_composition_bias(sequence: str, scale: float = 1.0) -> np.ndarray: return np.trunc(np.where(corrected < 0.0, corrected - 0.5, corrected + 0.5)).astype(np.int64) -def _positional_query(query_protein, bias: np.ndarray): - """Score a query position-specifically: BLOSUM62 row of each residue plus its bias. - - biotite has no per-position scoring hook, so the query becomes a sequence - over its own positions with an ``L x |protein alphabet|`` substitution - matrix. With zero bias this reproduces the plain BLOSUM62 alignment exactly - (score and trace). - """ - import biotite.sequence as seq - import biotite.sequence.align as align - n = len(query_protein) - alphabet = seq.Alphabet(list(range(n))) - positional = seq.GeneralSequence(alphabet, list(range(n))) - scores = _MATRIX.score_matrix()[query_protein.code] + bias[:, None] - matrix = align.SubstitutionMatrix(alphabet, _MATRIX.get_alphabet2(), scores.astype(np.int32)) - return positional, matrix - - -def _count_runs(mask: np.ndarray) -> int: - """Number of maximal runs of ``True`` in a 1-D boolean mask (gap-open count).""" - if mask.size == 0: - return 0 - return int(mask[0]) + int(np.count_nonzero(mask[1:] & ~mask[:-1])) - - -def _run_length_encode(ops: str) -> str: - """Collapse a per-column op string (e.g. 'MMMDDMM') to CIGAR ('3M2D2M').""" - if not ops: - return "" - out: List[str] = [] - prev = ops[0] - count = 1 - for ch in ops[1:]: - if ch == prev: - count += 1 - else: - out.append(f"{count}{prev}") - prev = ch - count = 1 - out.append(f"{count}{prev}") - return "".join(out) - - -def _derive_alignment_extras(trace, query_str: str, subject_str: str, needs: "_FieldNeeds"): - """Compute the mmseqs column values that need the alignment trace. - - Returns ``(n_ident, mismatch, gap_open, q_start, q_end, t_start, t_end, - cigar, q_aln, t_aln)``. The trace must be non-empty; column ``(-1, -1)`` never - occurs (biotite emits no double-gap columns), so the three masks partition - the columns exactly. - """ - q_idx = trace[:, 0] - s_idx = trace[:, 1] - q_gap = q_idx == -1 - s_gap = s_idx == -1 - both = ~q_gap & ~s_gap - - q_arr = np.frombuffer(query_str.encode("ascii"), dtype=np.uint8) - s_arr = np.frombuffer(subject_str.encode("ascii"), dtype=np.uint8) - - # 1-indexed bounds over each side's non-gap columns (min/max, so a terminal - # gap column — should one ever occur — cannot skew the reported extent). - q_res = q_idx[~q_gap] - s_res = s_idx[~s_gap] - q_start = int(q_res.min()) + 1 - q_end = int(q_res.max()) + 1 - t_start = int(s_res.min()) + 1 - t_end = int(s_res.max()) + 1 - - # Identity/mismatch only over columns aligned on both sides. - matches = q_arr[q_idx[both]] == s_arr[s_idx[both]] - n_ident = int(np.count_nonzero(matches)) - aligned_pairs = int(both.sum()) - mismatch = aligned_pairs - n_ident - - # A gap-open event is one maximal run of gaps; count query- and target-side - # runs independently (no column is a gap on both sides). - gap_open = _count_runs(q_gap) + _count_runs(s_gap) - - cigar = q_aln = t_aln = None - if needs.cigar: - ops = np.empty(trace.shape[0], dtype="U1") - ops[both] = "M" - ops[q_gap] = "D" # deletion: gap in query - ops[s_gap] = "I" # insertion: gap in target - cigar = _run_length_encode("".join(ops.tolist())) - if needs.qaln: - q_aln = "".join(query_str[i] if i != -1 else "-" for i in q_idx) - if needs.taln: - t_aln = "".join(subject_str[i] if i != -1 else "-" for i in s_idx) - - return n_ident, mismatch, gap_open, q_start, q_end, t_start, t_end, cigar, q_aln, t_aln - - def _robinson_frequency_vector(alphabet) -> np.ndarray: """Robinson background frequencies as a vector aligned to ``alphabet`` order.""" freq = np.array( @@ -472,15 +425,25 @@ def _bit_score(score: int, lam: float, k: float) -> float: def _worker_init(gap_open: int, gap_extend: int, lam: Optional[float], k: Optional[float], db_residues: Optional[int], alp_params=None, needs: Optional["_FieldNeeds"] = None, - comp_bias_scale: Optional[float] = None): - global _MATRIX, _GAP, _ESTIMATOR, _DB_RESIDUES, _LAMBDA, _K, _ALP, _OUTPUT_NEEDS, _COMP_BIAS_SCALE + comp_bias_scale: Optional[float] = None, isa: Optional[str] = None): + """Set what every task needs, once per process: the kernel of ``isa`` (the + one the parent loaded; ``None`` dispatches), the matrix, the gap costs, the + significance parameters and the output needs. The serial path calls it + in-process, where an error is raised at once; see :func:`_pool_worker_init`.""" + global _INT_KERNEL, _INT_MATRIX, _GAP_COSTS, _ESTIMATOR, _DB_RESIDUES, _LAMBDA, _K, _ALP, _OUTPUT_NEEDS + global _COMP_BIAS_SCALE, _INIT_ERROR import biotite.sequence.align as align + _INIT_ERROR = None _OUTPUT_NEEDS = needs if needs is not None else _FieldNeeds(False, False, False, False, False) - _MATRIX = align.SubstitutionMatrix.std_protein_matrix() # BLOSUM62 - # Gap penalties are passed through unmodified: biotite charges a length-L gap - # as open + (L-1)*extend, which is exactly MMseqs2's convention, so 11/1 here - # is the scoring system its hardcoded lambda/K were shipped with. - _GAP = (-abs(gap_open), -abs(gap_extend)) + _INT_KERNEL = swint.load(isa) + matrix = align.SubstitutionMatrix.std_protein_matrix() # BLOSUM62 + if "".join(matrix.get_alphabet1().get_symbols()) != swint.ALPHABET: + raise RuntimeError("biotite's protein alphabet differs from the kernel's code order (swint.ALPHABET)") + _INT_MATRIX = np.ascontiguousarray(matrix.score_matrix(), dtype=np.int32) + # The kernel charges a length-L gap as open + (L-1)*extend, biotite's and + # MMseqs2's convention, so 11/1 here is the scoring system the hardcoded + # lambda/K were shipped with. The sign of a cost is ignored, as it always was. + _GAP_COSTS = (abs(gap_open), abs(gap_extend)) _ALP = alp_params if alp_params is not None: _ESTIMATOR = None @@ -492,157 +455,311 @@ def _worker_init(gap_open: int, gap_extend: int, lam: Optional[float], k: Option _COMP_BIAS_SCALE = comp_bias_scale -def _align(query_protein, query_str: str, query_len: int, - subject_protein, subject_str: str, subject_len: int, - scoring=None) -> AlignmentMetrics: - """Local alignment of one pair. ``scoring`` is ``(positional_query, matrix)`` - from :func:`_positional_query` when the compositional bias correction is on.""" - import biotite.sequence.align as align - if scoring is None: - aln = align.align_optimal( - query_protein, subject_protein, _MATRIX, - gap_penalty=_GAP, local=True, max_number=1, - )[0] - else: - positional, matrix = scoring - aln = align.align_optimal( - positional, subject_protein, matrix, - gap_penalty=_GAP, local=True, max_number=1, - )[0] - # Identity is measured on residues, not positions: re-attach the real query. - aln = align.Alignment([query_protein, subject_protein], aln.trace, aln.score) - score = int(aln.score) - # Compute the bit score here so both return paths carry it. None when - # significance is off; otherwise on whatever scale the active mode implies. - bit_score = _bit_score(score, _LAMBDA, _K) if _LAMBDA is not None else None - needs = _OUTPUT_NEEDS - q_seq = query_str if needs.qseq else None - t_seq = subject_str if needs.tseq else None - trace = aln.trace - aln_len = int(trace.shape[0]) - if aln_len == 0: - return AlignmentMetrics( - 0.0, 0.0, 0.0, 0.0, 0, score, bit_score=bit_score, - query_len=query_len, subject_len=subject_len, - cigar=("" if needs.cigar else None), - q_aln=("" if needs.qaln else None), - t_aln=("" if needs.taln else None), - q_seq=q_seq, t_seq=t_seq, - ) - identity_aln = float(align.get_sequence_identity(aln, mode="all")) - identity_shorter = float(align.get_sequence_identity(aln, mode="shortest")) - q_aligned = int((trace[:, 0] != -1).sum()) - s_aligned = int((trace[:, 1] != -1).sum()) - (n_ident, mismatch, gap_open, q_start, q_end, t_start, t_end, - cigar, q_aln, t_aln) = _derive_alignment_extras(trace, query_str, subject_str, needs) - - pvalue = evalue = None - if _ALP is not None: - # Calibrated: ALP constants + ALP's finite-size-corrected area. - # Pairwise p-value: search space = the two sequences (n = subject length). - # MMseqs2 itself emits no p-value; this is the ALP p-value of the - # pairwise E, kept for continuity with the 'sampled' mode. - pvalue = karlin_altschul.pvalue_from_evalue( - karlin_altschul.evalue(score, query_len, subject_len, _ALP) - ) - # Database E-value: this is the MMseqs2-comparable quantity — search - # space = the whole target DB as one concatenated sequence, exactly as - # MMseqs2 does (no per-sequence term, no multiply by sequence count). - if _DB_RESIDUES: - evalue = karlin_altschul.evalue(score, query_len, _DB_RESIDUES, _ALP) - elif _ESTIMATOR is not None: - # Sampled/relative-only: biotite's raw K*m*n*exp(-lambda*S), no edge correction. - pvalue = _pvalue_from_evalue( - _evalue_from_log(_ESTIMATOR.log_evalue(score, query_len, subject_len)) - ) - if _DB_RESIDUES: - evalue = _evalue_from_log( - _ESTIMATOR.log_evalue(score, query_len, _DB_RESIDUES) - ) - - return AlignmentMetrics( - identity_aln=identity_aln, - identity_shorter=identity_shorter, - query_coverage=q_aligned / query_len if query_len else 0.0, - subject_coverage=s_aligned / subject_len if subject_len else 0.0, - aln_len=aln_len, - score=score, - bit_score=bit_score, - pvalue=pvalue, - evalue=evalue, - query_len=query_len, - subject_len=subject_len, - n_ident=n_ident, - mismatch=mismatch, - gap_open=gap_open, - q_start=q_start, - q_end=q_end, - t_start=t_start, - t_end=t_end, - cigar=cigar, - q_aln=q_aln, - t_aln=t_aln, - q_seq=q_seq, - t_seq=t_seq, - ) +def _pool_worker_init(*init_args): + """The pool's initializer: :func:`_worker_init` with its error kept for the + tasks, which raise it (as the esm-codes Stage 2 does), so a worker that + cannot load the kernel fails Stage 2 instead of hanging it.""" + global _INIT_ERROR + try: + _worker_init(*init_args) + except BaseException as e: # noqa: BLE001 (re-raised by every task of this worker) + _INIT_ERROR = e def _align_query(task): """Align one query against a chunk of its candidate subjects. Runs in a worker. The chunk may be all of a query's candidates or a slice of them (see - :func:`_chunk_candidate_tasks`); either way the query protein is built once - for the chunk and a partial hit list is returned, keyed by ``query_id`` so - the caller can regroup a query that was split across several chunks. + :func:`_chunk_candidate_tasks`); either way the query profile is built once + for the chunk, ONE ``fmswi_align`` call aligns all its candidates, and a + partial hit list is returned, keyed by ``query_id`` so the caller can + regroup a query that was split across several chunks. """ + if _INIT_ERROR is not None: + raise _INIT_ERROR query_id, query_seq, candidates = task - query_protein, query_clean, query_len = _to_protein(query_seq) - if query_protein is None: + qbytes = _sanitized_bytes(query_seq) + if not qbytes: return query_id, [] - scoring = None + query_clean = qbytes.decode("ascii") + query_len = len(qbytes) + qcodes = swint.codes_of(qbytes) + bias = None if _COMP_BIAS_SCALE is not None: # Built from the raw sequence so MMseqs2's letter folding (e.g. J->L) # applies; sanitize_sequence maps characters 1:1, so positions line up. - scoring = _positional_query(query_protein, local_composition_bias(query_seq, _COMP_BIAS_SCALE)) - hits: List[Hit] = [] + bias = local_composition_bias(query_seq, _COMP_BIAS_SCALE) + qprof = swint.query_profile(qcodes, _INT_MATRIX, bias) + ids: List[str] = [] + embs: List[float] = [] + cleans: List[bytes] = [] for subject_id, subject_seq, emb_score in candidates: - subject_protein, subject_clean, subject_len = _to_protein(subject_seq) - if subject_protein is None: + clean = _sanitized_bytes(subject_seq) + if not clean: continue - metrics = _align( - query_protein, query_clean, query_len, - subject_protein, subject_clean, subject_len, - scoring=scoring, + ids.append(subject_id) + embs.append(emb_score) + cleans.append(clean) + if not cleans: + return query_id, [] + lens = np.fromiter((len(c) for c in cleans), dtype=np.int32, count=len(cleans)) + offs = np.zeros(len(cleans), dtype=np.int64) + if len(cleans) > 1: + np.cumsum(lens[:-1], out=offs[1:]) + needs = _OUTPUT_NEEDS + want_ops = needs.cigar or needs.qaln or needs.taln + res = swint.align_batch(qcodes, qprof, swint.codes_of(b"".join(cleans)), offs, lens, + _GAP_COSTS[0], _GAP_COSTS[1], ops=want_ops, kernel=_INT_KERNEL) + rows = res.out.tolist() + q_seq = query_clean if needs.qseq else None + lam, kk, alp, est, db_res = _LAMBDA, _K, _ALP, _ESTIMATOR, _DB_RESIDUES + # Bit scores and database E-values depend only on the score within a task + # (one query), so each is computed once per distinct score. + bits_of: Dict[int, Optional[float]] = {} + evalue_of: Dict[int, Optional[float]] = {} + hits: List[Hit] = [] + for idx, (subject_id, emb_score, clean, row) in enumerate(zip(ids, embs, cleans, rows)): + (score, q_start, q_end, t_start, t_end, aln_len, n_ident, gap_open, n_qgap, n_tgap, _wide) = row + subject_len = len(clean) + subject_str = clean.decode("ascii") if (needs.tseq or needs.taln or needs.qaln) else None + t_seq = subject_str if needs.tseq else None + # The bit score is computed here so both return paths carry it. None when + # significance is off; otherwise on whatever scale the active mode implies. + if score in bits_of: + bit_score = bits_of[score] + else: + bit_score = bits_of[score] = _bit_score(score, lam, kk) if lam is not None else None + if aln_len == 0: + metrics = AlignmentMetrics( + 0.0, 0.0, 0.0, 0.0, 0, score, bit_score=bit_score, + query_len=query_len, subject_len=subject_len, + cigar=("" if needs.cigar else None), + q_aln=("" if needs.qaln else None), + t_aln=("" if needs.taln else None), + q_seq=q_seq, t_seq=t_seq, + ) + hits.append(Hit(subject_id=subject_id, emb_score=emb_score, metrics=metrics)) + continue + cigar = q_aln = t_aln = None + if want_ops: + ops = res.ops[res.ops_off[idx]:res.ops_off[idx + 1]] + if needs.cigar: + cigar = swint.run_length_encode(ops) + if needs.qaln or needs.taln: + qa, ta = swint.gapped_strings(ops, query_clean, subject_str, q_start, t_start) + q_aln = qa if needs.qaln else None + t_aln = ta if needs.taln else None + pvalue = evalue = None + if alp is not None: + # Calibrated: ALP constants + ALP's finite-size-corrected area. + # Pairwise p-value: search space = the two sequences (n = subject length). + # MMseqs2 itself emits no p-value; this is the ALP p-value of the + # pairwise E, kept for continuity with the 'sampled' mode. + pvalue = karlin_altschul.pvalue_from_evalue( + karlin_altschul.evalue(score, query_len, subject_len, alp) + ) + # Database E-value: this is the MMseqs2-comparable quantity — search + # space = the whole target DB as one concatenated sequence, exactly as + # MMseqs2 does (no per-sequence term, no multiply by sequence count). + if db_res: + if score in evalue_of: + evalue = evalue_of[score] + else: + evalue = evalue_of[score] = karlin_altschul.evalue(score, query_len, db_res, alp) + elif est is not None: + # Sampled/relative-only: biotite's raw K*m*n*exp(-lambda*S), no edge correction. + pvalue = _pvalue_from_evalue(_evalue_from_log(est.log_evalue(score, query_len, subject_len))) + if db_res: + evalue = _evalue_from_log(est.log_evalue(score, query_len, db_res)) + # Identity is n_ident (aligned pairs with equal residues) over the + # alignment length ('all') or the shorter full sequence ('shortest'), as + # biotite's get_sequence_identity counts it; both divide Python ints. + metrics = AlignmentMetrics( + identity_aln=n_ident / aln_len, + identity_shorter=n_ident / min(query_len, subject_len), + query_coverage=(aln_len - n_qgap) / query_len, + subject_coverage=(aln_len - n_tgap) / subject_len, + aln_len=aln_len, + score=score, + bit_score=bit_score, + pvalue=pvalue, + evalue=evalue, + query_len=query_len, + subject_len=subject_len, + n_ident=n_ident, + mismatch=aln_len - n_qgap - n_tgap - n_ident, + gap_open=gap_open, + q_start=q_start, + q_end=q_end, + t_start=t_start, + t_end=t_end, + cigar=cigar, + q_aln=q_aln, + t_aln=t_aln, + q_seq=q_seq, + t_seq=t_seq, ) hits.append(Hit(subject_id=subject_id, emb_score=emb_score, metrics=metrics)) return query_id, hits -def _chunk_candidate_tasks(tasks, worker_budget: int, tasks_per_worker: int = 4): - """Split per-query tasks into per-(query, candidate-chunk) tasks for the pool. +def _align_indexed(item): + """Pool entry point: ``(index, task) -> (index, _align_query(task))``.""" + index, task = item + return index, _align_query(task) + + +def _chunk_candidate_tasks(tasks, worker_budget: int, tasks_per_worker: int = _TASKS_PER_WORKER, + floor_cells: Optional[int] = None): + """Split per-query tasks into contiguous candidate slices of about equal DP cells. Parallelizing over whole queries pins a single query with many candidates to one worker (every other core idle); splitting a query's candidate list into - chunks lets the pool spread those alignments across all workers. Chunk size - is chosen so there are roughly ``tasks_per_worker * worker_budget`` chunks in - total — enough to keep every worker fed and to smooth out the load imbalance - from variable-length (quadratic-cost) alignments — while still batching - several candidates per chunk to amortize the per-chunk query rebuild and the - task's pickling overhead. + slices lets the pool spread those alignments across all workers. The slice + budget is ``max(floor_cells, ceil(total_cells / (tasks_per_worker * + worker_budget)))`` (``floor_cells`` defaults to :data:`_TASK_CELLS_FLOOR`, + enough cells to amortize a task's pickling and its kernel call), with + ``cells = len(query) * len(subject)`` on the unsanitised strings (an + estimate for scheduling only); a candidate that alone exceeds the budget is + its own slice. Returns ``(chunks, cells)`` in query-then-candidate order, so + the caller can queue them longest first and still regroup the results in + this order. Queries with no candidates are dropped here (they emit no work); the caller re-seeds them so they still report an empty hit list. """ - total_pairs = sum(len(candidates) for _, _, candidates in tasks) - if total_pairs == 0: - return list(tasks) - target_chunks = tasks_per_worker * max(1, worker_budget) - chunk_size = max(1, math.ceil(total_pairs / target_chunks)) - chunked = [] + if floor_cells is None: + floor_cells = _TASK_CELLS_FLOOR + per_task = [] + total = 0 for query_id, query_seq, candidates in tasks: - for start in range(0, len(candidates), chunk_size): - chunked.append((query_id, query_seq, candidates[start:start + chunk_size])) - return chunked + n = len(query_seq) + cells = [n * len(seq) for _, seq, _ in candidates] + per_task.append(cells) + total += sum(cells) + budget = max(int(floor_cells), -(-total // (max(1, tasks_per_worker) * max(1, worker_budget)))) + chunks, chunk_cells = [], [] + for (query_id, query_seq, candidates), cells in zip(tasks, per_task): + start, acc = 0, 0 + for k, c in enumerate(cells): + if k > start and acc + c > budget: + chunks.append((query_id, query_seq, candidates[start:k])) + chunk_cells.append(acc) + start, acc = k, 0 + acc += c + if start < len(candidates): + chunks.append((query_id, query_seq, candidates[start:])) + chunk_cells.append(acc) + return chunks, chunk_cells + + +def check_evalue_search_space(max_evalue: Optional[float], subject_db_size: Optional[int]) -> Optional[float]: + """The E-value threshold :func:`align_candidates` filters on, or ``None`` for no filter. + + ``None`` and non-finite values ('inf', the documented way to switch the + filter off) mean no filter. A finite threshold needs a positive + ``subject_db_size`` (the database search space); otherwise this raises a + ``ValueError`` that names the cause and the ways out. Cheap, so callers + can run it before any expensive work (the CLI does, before ESM3). + """ + if max_evalue is None or not math.isfinite(max_evalue): + return None + if subject_db_size is None: + raise ValueError( + "max_evalue filtering requires subject_db_size (the database " + "search space); pass it or set max_evalue=None." + ) + if subject_db_size <= 0: + # An empty store reaches here as 0, not None: SequenceStore.total_residues() + # is COALESCE(SUM(length), 0). Without this check no database E-value is + # computed and the filter compares None with a float, a TypeError from + # deep inside the result loop, long after the cause. + raise ValueError( + f"max_evalue filtering needs a non-empty database, but subject_db_size " + f"is {subject_db_size}: the search space is empty, so no E-value is " + f"defined. This usually means the sequence store is empty or was never " + f"built for this database. Delete the database's files and rebuild it " + f"with 'fm-search build sequences' (a build does not overwrite an " + f"existing database), or pass max_evalue=None (--max-evalue inf) to " + f"rank hits without E-value filtering." + ) + return max_evalue + + +def whole_number(value) -> Optional[int]: + """``value`` as an int when it is a whole number: an int or a numpy integer + (a bool is not one), or a float with a whole value (11.0, which biotite's + aligner took as 11); else ``None``.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, np.integer)): + return int(value) + if isinstance(value, (float, np.floating)) and float(value).is_integer(): + return int(value) + return None + + +def check_gap_costs( + gap_open: int, + gap_extend: int, + significance_mode: SignificanceMode = SignificanceMode.default, + significance: bool = False, + max_evalue: Optional[float] = None, +) -> Tuple[int, int]: + """The gap costs of a BLOSUM62 Stage 2 as ``(open, extend)``, checked. + + The costs are the absolute values of the arguments (the sign is ignored, as + it always was), which must be whole numbers (:func:`whole_number`: 11.0 is + 11, as biotite took it), else a ``TypeError``. Raises a ``ValueError`` + that names the cause unless + + * ``open`` is between 1 and :data:`swint.MAX_GAP` and ``extend`` between 0 + and :data:`swint.MAX_GAP`: what the Smith-Waterman kernel accepts (its + end-cell rule needs a gap to cost at least 1 to open; ``MAX_GAP`` is its + ``FMSWI_MAX_GAP``); + * with ``significance_mode='default'``, when significance is computed + (``significance``: a significance column is requested; or a finite + ``max_evalue``, which forces it), the costs are 11/1, the only ones with + calibrated statistics (:func:`karlin_altschul.params_for`). + + ``None`` and non-finite ``max_evalue`` mean no E-value filter, as in + :func:`check_evalue_search_space`. Cheap, so callers can run it before any + expensive work (the CLI does, before ESM3); :func:`align_candidates` runs + it before it reads a sequence. + """ + costs = [] + for name, value, low in (("gap_open", gap_open, 1), ("gap_extend", gap_extend, 0)): + whole = whole_number(value) + if whole is None: + raise TypeError(f"{name} must be a whole number, got {value!r}.") + if abs(whole) < low: + raise ValueError( + f"{name} must be between {low} and {swint.MAX_GAP} (its sign is ignored), got {value}: a gap of " + f"length L costs gap_open + (L - 1) * gap_extend, and the kernel's end-cell rule needs a gap to " + f"cost at least 1 to open." + ) + if abs(whole) > swint.MAX_GAP: + raise ValueError( + f"{name} must be between {low} and {swint.MAX_GAP} (its sign is ignored), got {value}: " + f"{swint.MAX_GAP} is the most the Smith-Waterman kernel takes (FMSWI_MAX_GAP)." + ) + costs.append(abs(whole)) + costs = tuple(costs) + filtering = max_evalue is not None and math.isfinite(max_evalue) + if SignificanceMode(significance_mode) is SignificanceMode.default and (significance or filtering): + try: + karlin_altschul.params_for(*costs) + except ValueError as exc: + # Without an E-value threshold, these gap penalties would have + # been fine whenever no significance column was requested — say + # how to get that behaviour back. + if not filtering: + raise + raise ValueError( + f"{exc} E-value filtering (max_evalue={max_evalue:g}) requires " + f"significance; disable it to use these gap penalties." + ) from None + return costs def align_candidates( @@ -680,7 +797,8 @@ def align_candidates( :meth:`SequenceStore.fetch`), used once to gather all candidate subject sequences across queries. min_seq_identity: drop hits whose ``identity_aln`` is below this. - min_coverage: drop hits whose query *and* subject coverage are below this. + min_coverage: drop hits whose query or subject coverage is below this + (a hit is kept only if ``min(qcov, tcov) >= min_coverage``). max_evalue: drop hits whose database E-value is above this. ``None`` or a non-finite value disables the filter. Because the E-value is a significance quantity, setting it forces ``compute_significance=True`` @@ -688,7 +806,11 @@ def align_candidates( (or one deliberately skipping the significance pass) must pass ``max_evalue=None``. These three thresholds default to the same values as the ``fm-search query`` options that expose them. - gap_open / gap_extend: positive BLOSUM62 gap penalties (negated internally). + gap_open / gap_extend: BLOSUM62 gap costs, whole numbers (11.0 is + 11); a gap of length L costs ``gap_open + (L - 1) * gap_extend``. + The sign is ignored; ``gap_open`` must be at least 1 and + ``gap_extend`` at least 0, both at most :data:`swint.MAX_GAP` + (:func:`check_gap_costs`, run before any sequence is read). comp_bias_corr: correct the Smith-Waterman scores for locally biased amino-acid composition, as MMseqs2's ``--comp-bias-corr`` does (see :func:`local_composition_bias`). Off by default (plain BLOSUM62); @@ -704,8 +826,10 @@ def align_candidates( N workers (also widening affinity so they can spread). Capped at the number of queries. subject_db_size: total residue count of the subject database, used as the - search space for the database E-value. If ``None``, no E-value is - reported (the pairwise p-value is still computed). + search space for the database E-value. If ``None`` (or ``0``, which is + what an empty store reports), no E-value is reported (the pairwise + p-value is still computed), and combining that with a ``max_evalue`` + raises a ``ValueError`` up front. compute_significance: when ``True``, annotate each hit with a bit score, a pairwise p-value and (if ``subject_db_size`` is given) a database E-value. @@ -714,7 +838,7 @@ def align_candidates( ``'default'`` uses calibrated Karlin-Altschul constants with a finite-size-corrected search space, giving exact, reproducible values; it supports only ``gap_open=11, gap_extend=1`` - and raises otherwise. ``'sampled'`` restores the previous behavior — + and raises otherwise (up front, when significance is computed). ``'sampled'`` restores the previous behavior — lambda/K estimated by sampling, whose absolute values are unreliable (they do not converge with sample size) though the ranking is sound; use it for gap penalties ``'default'`` cannot serve. Neither mode is @@ -735,20 +859,17 @@ def align_candidates( embedding score desc; with ``compute_significance=False`` the raw alignment score is used directly. Queries with no surviving hit map to ``[]``. + + The alignments run on the native kernel (:mod:`swint`), loaded before any + sequence is read: :class:`~foldmatch.search.swkern.KernelError` (with the + rebuild command) when it cannot be used. There is no fallback. """ start_time = time.perf_counter() # An E-value threshold only means something once the significance pass has # run against a known search space, so turn it on rather than silently - # dropping every hit (evalue stays None when it is off). A non-finite - # threshold ('inf') is the documented way to switch the filter off. - if max_evalue is not None and not math.isfinite(max_evalue): - max_evalue = None + # dropping every hit (evalue stays None when it is off). + max_evalue = check_evalue_search_space(max_evalue, subject_db_size) if max_evalue is not None: - if subject_db_size is None: - raise ValueError( - "max_evalue filtering requires subject_db_size (the database " - "search space); pass it or set max_evalue=None." - ) compute_significance = True if not 0.0 <= comp_bias_corr_scale <= 1.0: raise ValueError( @@ -768,6 +889,12 @@ def align_candidates( f"Unknown significance_mode {significance_mode!r}; expected one of: " f"{', '.join(m.value for m in SignificanceMode)}." ) from None + # The gap costs the kernel takes (and, under 'default' significance, 11/1), + # checked before any sequence is read or any worker starts. + gap_open, gap_extend = check_gap_costs(gap_open, gap_extend, significance_mode, compute_significance, max_evalue) + # The kernel, as for esm-codes: a missing or stale library raises KernelError + # here, with the rebuild command; the workers load the same ISA. + kernel = swint.load() # Gather every candidate subject sequence in a single random-access read. all_subject_ids = { @@ -798,18 +925,8 @@ def align_candidates( lam = k = alp_params = None if compute_significance and tasks: if significance_mode is SignificanceMode.default: - try: - alp_params = karlin_altschul.params_for(gap_open, gap_extend) - except ValueError as exc: - # Without an E-value threshold, these gap penalties would have - # been fine whenever no significance column was requested — say - # how to get that behaviour back. - if max_evalue is None: - raise - raise ValueError( - f"{exc} E-value filtering (max_evalue={max_evalue:g}) requires " - f"significance; disable it to use these gap penalties." - ) from None + # 11/1, checked by check_gap_costs above. + alp_params = karlin_altschul.params_for(gap_open, gap_extend) logger.info( f"Using calibrated significance (BLOSUM62 {gap_open}/{gap_extend}: " f"lambda={alp_params.lam:.8f}, K={alp_params.k:.9f})" @@ -831,14 +948,15 @@ def align_candidates( if comp_bias_scale is not None and tasks: logger.info(f"Applying compositional bias correction (scale {comp_bias_scale:g})") - init_args = (gap_open, gap_extend, lam, k, subject_db_size, alp_params, needs, comp_bias_scale) + init_args = (gap_open, gap_extend, lam, k, subject_db_size, alp_params, needs, comp_bias_scale, kernel.isa) if not serial and tasks: # Parallelize over (query, subject) *pairs* rather than whole queries by - # chunking each query's candidate list, so even a single query with many - # candidates uses the whole node instead of one core. Size the pool by - # the number of chunks (capped, as before, at the worker budget). + # cutting each query's candidate list into slices of about equal DP + # cells, so even a single query with many candidates uses the whole node + # instead of one core. Size the pool by the number of slices (capped, as + # before, at the worker budget). worker_budget = _available_cpus() if num_workers is None else num_workers - pair_tasks = _chunk_candidate_tasks(tasks, worker_budget) + pair_tasks, chunk_cells = _chunk_candidate_tasks(tasks, worker_budget) n_workers = _resolve_num_workers(num_workers, len(pair_tasks)) else: pair_tasks = tasks @@ -848,18 +966,25 @@ def align_candidates( total_pairs = sum(len(candidates) for _, _, candidates in tasks) logger.info( f"Aligning {total_pairs} (query, subject) pairs from {len(tasks)} " - f"queries in {len(pair_tasks)} chunks across {n_workers} CPU workers" + f"queries in {len(pair_tasks)} chunks across {n_workers} CPU workers " + f"(kernel {kernel.isa})" ) import multiprocessing as mp - with mp.Pool( + with mp.get_context(_pool_start_method()).Pool( processes=n_workers, - initializer=_worker_init, + initializer=_pool_worker_init, initargs=init_args, ) as pool: - # chunksize=1: each candidate-chunk is its own dispatch unit (we've - # already sized the chunks) so variable-length alignments stay - # balanced; pool.map preserves task order. - raw = pool.map(_align_query, pair_tasks, chunksize=1) + # chunksize=1: each slice is its own dispatch unit (we've already + # sized them), queued longest first so the largest ones do not end + # the run alone. The results are put back by task index, so the + # per-query hit lists below are built in input order and the stable + # sort sees the same input whatever the completion order. + order = sorted(range(len(pair_tasks)), key=lambda i: -chunk_cells[i]) + raw = [None] * len(pair_tasks) + for index, result in pool.imap_unordered( + _align_indexed, [(i, pair_tasks[i]) for i in order], chunksize=1): + raw[index] = result else: _worker_init(*init_args) raw = [_align_query(task) for task in pair_tasks] diff --git a/src/foldmatch/search/clustering.py b/src/foldmatch/search/clustering.py index b94e210..8e72b25 100644 --- a/src/foldmatch/search/clustering.py +++ b/src/foldmatch/search/clustering.py @@ -1,21 +1,107 @@ +import ctypes import logging +import operator +import random +import sys +import time +import weakref import numpy as np import faiss import igraph as ig -import leidenalg -from typing import Dict, List, Optional +from typing import Dict, Optional from pathlib import Path from tqdm import tqdm from foldmatch.search import FaissEmbeddingDatabase from foldmatch.search.embedding_database import _parse_db_path -from foldmatch.search.output import write_cluster_results -from foldmatch.types.api_types import FormatMode +from foldmatch.search.output import CLUSTER_OUTPUT_FIELDS, GREEDY_CLUSTER_OUTPUT_FIELDS, write_cluster_results +from foldmatch.types.api_types import FormatMode, IndexType logger = logging.getLogger(__name__) +# Byte budget of the search results of one block of chains (k int64 ids and k float32 scores a chain): 16,384 chains +# a FAISS call at k = 1000. The block's embeddings are read with one reconstruct_n call of the same rows. +_GRAPH_RESULT_BYTES = 16_384 * 1000 * 12 +# Row cap of a block, which bounds its embeddings at small k (16,384 x d x 4 bytes, 96 MiB at d = 1536). +_GRAPH_MAX_ROWS = 16_384 +# Kept node ids are int32 (8 bytes an edge instead of 16 while the search runs) when the largest id fits, else int64. +_GRAPH_INT32_MAX_ID = np.iinfo(np.int32).max +# Edges (and weights) turned into Python objects at a time when handed to igraph. +_GRAPH_EDGE_CHUNK = 1 << 20 +# An igraph attribute is a list of Python objects, one float an edge for the weights: equal weights share one float +# when the kept weights' float32 bit patterns span at most this many values, in a table of 128 MiB of pointers and 16 +# MiB of flags at most (scores from 0.8 to 1.03 span about 3.6M); a wider span (scores of both signs, 0.0 with -0.0 +# included, or, with scores up to about 1.03, scores below about 0.26) gives one float an edge, as ndarray.tolist(). +_GRAPH_WEIGHT_TABLE_MAX = 1 << 24 +# Chains whose output rows are made at a time by export_clusters. +_EXPORT_CHUNK = 1 << 20 +# Leiden iterations of cluster_leiden: a fixed count, since run-to-convergence is open-ended (221 iterations on a +# 1M-chain graph). +_LEIDEN_ITERATIONS = 10 + + +def _graph_id_dtype(n_chains: int): + """The dtype of the kept node ids of a graph of ``n_chains`` chains: int32 when the largest id fits, else int64.""" + return np.int32 if n_chains - 1 <= _GRAPH_INT32_MAX_ID else np.int64 + + +def _edge_tuples(blocks: list): + """Yield the edges of the (m, 2) blocks as tuples of two ints, block by block, in chunks of _GRAPH_EDGE_CHUNK. + The list is emptied as it goes, so each block is released once consumed.""" + blocks.reverse() + while blocks: + block = blocks.pop() + for s in range(0, len(block), _GRAPH_EDGE_CHUNK): + yield from map(tuple, block[s:s + _GRAPH_EDGE_CHUNK].tolist()) + del block + + +def _weight_list(blocks: list) -> list: + """The float32 weight blocks as one list of Python floats, in order, emptying the list as it goes. Each weight is + the float of its own float32 value, as ndarray.tolist() gives; equal bit patterns share one float object when the + patterns span at most _GRAPH_WEIGHT_TABLE_MAX values (0.0 and -0.0 stay apart).""" + weights = [] + nonempty = [b for b in blocks if len(b)] + if not nonempty: + blocks.clear() + return weights + lo = min(int(b.view(np.uint32).min()) for b in nonempty) + span = max(int(b.view(np.uint32).max()) for b in nonempty) - lo + 1 + del nonempty + table = seen = None + if span <= _GRAPH_WEIGHT_TABLE_MAX: + table = np.empty(span, dtype=object) + seen = np.zeros(span, dtype=bool) + blocks.reverse() + while blocks: + block = blocks.pop() + for s in range(0, len(block), _GRAPH_EDGE_CHUNK): + if table is None: + weights.extend(block[s:s + _GRAPH_EDGE_CHUNK].tolist()) + continue + idx = block[s:s + _GRAPH_EDGE_CHUNK].view(np.uint32).astype(np.int64) - lo + new = np.unique(idx[~seen[idx]]) + if len(new): + table[new] = (new + lo).astype(np.uint32).view(np.float32).tolist() + seen[new] = True + weights.extend(table[idx].tolist()) + del block + return weights + + +def _release_freed_memory(): + """Hand freed heap memory back to the OS with glibc's malloc_trim(0); a no-op elsewhere or without the symbol.""" + if not sys.platform.startswith("linux"): + return + try: + ctypes.CDLL(None).malloc_trim(0) + except (OSError, AttributeError): + pass + + class EmbeddingClusterer: - """Cluster protein structure embeddings using Leiden algorithm on similarity graphs.""" + """Cluster protein structure embeddings: Leiden on the similarity graph (:meth:`cluster_leiden`), or greedy + representatives in one streamed pass over the database (:meth:`cluster_greedy`).""" def __init__(self, db_path: str, use_gpu: bool = False): """ @@ -33,6 +119,13 @@ def __init__(self, db_path: str, use_gpu: bool = False): self.graph = None self.clusters = None + # The smallest kept edge weight of the last build_similarity_graph, and a weak reference to the graph it was + # recorded for (cluster_leiden refuses a negative one for that graph only) + self.min_weight = None + self._min_weight_graph = None + # The method of the last clustering ("leiden" or "greedy"), and under greedy each cluster's representative + self.method = None + self.representatives = None def build_similarity_graph( self, @@ -43,12 +136,15 @@ def build_similarity_graph( Build a similarity graph using FAISS k-NN search with threshold filtering. Args: - threshold: Minimum similarity score to create an edge (0-1, where 1.0 = identical) + threshold: Similarity threshold for edge creation (cosine; identical chains score 1, or close to 1 on + ivf_pq, where scores are PQ estimates) max_neighbors: Maximum number of neighbors to consider per node Returns: igraph Graph object with edges weighted by similarity """ + self.min_weight = None + self._min_weight_graph = None if self.db.index is None: raise ValueError("Database not loaded. Call load_database() first.") @@ -58,6 +154,9 @@ def build_similarity_graph( # Clamp max_neighbors to actual database size k = min(max_neighbors, n_chains) + # Chains per block, from the result budget and the row cap (k = 0 is left to FAISS to refuse, as before, after + # one chain is read) + rows = 1 if k < 1 else max(1, min(n_chains, _GRAPH_MAX_ROWS, _GRAPH_RESULT_BYTES // (k * 12))) # Get CPU index for reconstruction if using GPU index_to_use = self.db.index @@ -65,75 +164,281 @@ def build_similarity_graph( logging.info("Moving GPU index to CPU for reconstruction...") index_to_use = faiss.index_gpu_to_cpu(self.db.index) - edges = [] - weights = [] - - # Search incrementally to avoid segfault with large batch searches - # Both IndexFlatIP and IndexHNSWFlat support reconstruction through their storage + # ivf_pq decodes its PQ codes, which are not unit vectors: the query is normalised, as in query db, so a score + # is the inner product of the chain's unit decoded embedding with each stored code. flat and hnsw store the + # normalised embeddings, searched as they are. + normalize = self.db.index_type == IndexType.ivf_pq + + id_dtype = _graph_id_dtype(n_chains) + pair_blocks = [] + weight_blocks = [] + n_edges = 0 + start_time = time.perf_counter() + + # Blocks of chains in ascending order, read with reconstruct_n (no direct map needed on ivf_pq) and searched + # with one FAISS call each. An edge (i, j) is kept from the search of the lower chain i only, which also drops + # self-hits and the -1 ids FAISS fills in; edges come in chain order, then in FAISS rank order. A block's kept + # pairs are stored with int32 ids when the database's largest id fits, and a block with no kept edge is not + # stored. logging.info("Performing k-NN search ...") - for i in tqdm(range(n_chains), desc="Processing chains", unit="chain"): - # Reconstruct single embedding - query_embedding = index_to_use.reconstruct(i) - - # Ensure contiguous C-order array with correct shape - query_embedding = np.ascontiguousarray(query_embedding, dtype=np.float32).reshape(1, -1) - - # Search for k nearest neighbors - scores_i, indices_i = self.db.index.search(query_embedding, k) - - # Build edges from this query - for j, score in zip(indices_i[0], scores_i[0]): - # Skip self-loops and edges below threshold - if i < j and score >= threshold: # i < j to avoid duplicate edges - edges.append((i, j)) - weights.append(float(score)) - - logging.info(f"Created graph with {n_chains} nodes and {len(edges)} edges") - - # Create igraph from edge list - self.graph = ig.Graph(n=n_chains, edges=edges, directed=False) - self.graph.es['weight'] = weights + with tqdm(total=n_chains, desc="Processing chains", unit="chain") as progress: + for start in range(0, n_chains, rows): + end = min(start + rows, n_chains) + embeddings = np.ascontiguousarray(index_to_use.reconstruct_n(start, end - start), dtype=np.float32) + if normalize: + faiss.normalize_L2(embeddings) + scores, indices = self.db.index.search(embeddings, k) + del embeddings + + chains = np.arange(start, end, dtype=np.int64)[:, None] + block_rows, block_cols = np.nonzero((chains < indices) & (scores >= threshold)) + if len(block_rows): + pair_blocks.append(np.stack([block_rows + start, indices[block_rows, block_cols]], + axis=1).astype(id_dtype, copy=False)) + weight_blocks.append(scores[block_rows, block_cols]) + n_edges += len(block_rows) + del scores, indices, chains, block_rows, block_cols + + progress.set_postfix(edges=n_edges, refresh=False) + progress.update(end - start) + + logging.info(f"Created graph with {n_chains} nodes and {n_edges} edges") + + # Create igraph from the blocks, with no concatenation, through a generator of int tuples: python-igraph 1.0.0 + # converts an edge ndarray through Python objects as well, at about 170 bytes an edge at its peak against 50 to + # 57. Each block is released once consumed, and each weight block once its floats are in the list. An error + # raised while igraph pulls the edges (MemoryError, KeyboardInterrupt) surfaces from python-igraph 1.0.0 as a + # SystemError, with the original as its __cause__. + self.graph = ig.Graph(n=n_chains, edges=_edge_tuples(pair_blocks), directed=False) + # glibc keeps freed blocks in the heap: give them back before the weights and before Leiden. + _release_freed_memory() + # The smallest kept weight (+inf with no edge), before the weight blocks are consumed: Leiden refuses negative + # weights + min_weight = min((float(b.min()) for b in weight_blocks), default=float("inf")) + self.graph.es['weight'] = _weight_list(weight_blocks) + _release_freed_memory() # Add chain IDs as vertex attribute self.graph.vs['name'] = self.chain_ids + self.min_weight = min_weight + self._min_weight_graph = weakref.ref(self.graph) + + logging.info(f"Similarity graph built in {time.perf_counter() - start_time:.2f}s") return self.graph def cluster_leiden( self, resolution: float = 1.0, - n_iterations: int = -1, + n_iterations: int = _LEIDEN_ITERATIONS, seed: Optional[int] = None - ) -> List[int]: + ) -> np.ndarray: """ Apply Leiden clustering algorithm to the similarity graph. + The call is python-igraph's ``Graph.community_leiden(objective_function="modularity", weights="weight", + resolution=resolution, beta=0.01, n_iterations=n_iterations)``: modularity at ``resolution`` with the edge + weights. It runs a fixed number of iterations (``n_iterations``, the CLI's --leiden-iterations), 10 by default, + since run-to-convergence is open-ended on sparse graphs (221 iterations on a 1M-chain graph). + + Seeding: python-igraph draws from a ``random.Random(seed)`` of its own during the call + (``ig.set_random_number_generator``), so Python's global ``random`` state is neither used nor changed; the same + seed gives the same partition, and ``seed=None`` (seeded from ``os.urandom``) a different one each call. + Caveats: afterwards python-igraph's generator is set back to the ``random`` module, its default (python-igraph + has no getter), so a caller's own igraph generator is replaced; and the generator is process-global, so igraph + calls of other threads during the Leiden step draw from it. + + Cluster ids are numbered by decreasing cluster size, ties by the smallest chain index in the cluster (0 is the + largest cluster), as under greedy. + Args: resolution: Resolution parameter (higher = more clusters, default: 1.0) - n_iterations: Number of iterations (-1 for until convergence) - seed: Random seed for reproducibility + n_iterations: Number of Leiden iterations, at least 1 (default: 10) + seed: Random seed for reproducibility, an integer of at least 0 (``operator.index``: NumPy integers are + accepted, floats refused with TypeError) Returns: - List of cluster IDs (one per chain) + Array of cluster IDs (one per chain), int32 below 2**31 chains + + Raises: + ValueError: ``n_iterations`` below 1, ``seed`` below 0, no graph, or a graph built by + build_similarity_graph with a negative kept weight (modularity needs non-negative edge weights) """ + if n_iterations < 1: + raise ValueError("n_iterations must be at least 1 (Leiden runs a fixed number of iterations).") + if seed is not None: + seed = operator.index(seed) + if seed < 0: + raise ValueError("--seed must be at least 0.") if self.graph is None: raise ValueError("Graph not built. Call build_similarity_graph() first.") + if self._min_weight_graph is not None and self._min_weight_graph() is self.graph and self.min_weight < 0: + raise ValueError(f"Leiden needs non-negative edge weights: the graph's smallest weight is " + f"{self.min_weight:.4g} (edges keep scores at or above the threshold of " + f"build_similarity_graph).") logging.info(f"Running Leiden clustering (resolution={resolution})...") + start_time = time.perf_counter() + + # Higher edge weights (higher similarity) encourage nodes to be in same cluster. The objective must be exactly + # "modularity": python-igraph normalises the resolution only for that spelling. A private generator: Python's + # global random state is left alone. + rng = random.Random(seed) + ig.set_random_number_generator(rng) + try: + partition = self.graph.community_leiden( + objective_function="modularity", + weights="weight", + resolution=resolution, + beta=0.01, + n_iterations=n_iterations + ) + finally: + # python-igraph 1.0.0's default; it has no getter + ig.set_random_number_generator(random) + + # Cluster ids by decreasing size, ties by smallest chain index (igraph numbers by first appearance) + n = self.graph.vcount() + dtype = _graph_id_dtype(n) + membership = np.fromiter(partition.membership, dtype=dtype, count=n) + del partition + sizes = np.bincount(membership) + # Every cluster has a member, so minimum.at overwrites the fill + first = np.full(len(sizes), n - 1, dtype=dtype) + np.minimum.at(first, membership, np.arange(n, dtype=dtype)) + order = np.lexsort((first, -sizes)) + new_id = np.empty(len(sizes), dtype=dtype) + new_id[order] = np.arange(len(sizes), dtype=dtype) + self.clusters = new_id[membership] + del membership, first, order, new_id + + logging.info(f"Found {len(sizes)} clusters") + logging.info(f"Leiden clustering completed in {time.perf_counter() - start_time:.2f}s") + self.method = "leiden" + self.representatives = None - # Use RBConfigurationVertexPartition with weights - # Higher edge weights (higher similarity) encourage nodes to be in same cluster - partition = leidenalg.find_partition( - self.graph, - leidenalg.RBConfigurationVertexPartition, - weights='weight', - resolution_parameter=resolution, - n_iterations=n_iterations, - seed=seed - ) + return self.clusters - self.clusters = partition.membership - logging.info(f"Found {len(set(self.clusters))} clusters") + def cluster_greedy( + self, + threshold: float = 0.8, + max_neighbors: int = 1000 + ) -> np.ndarray: + """ + Greedy representatives in database order, in one streamed pass over the database (no graph). + + The partition: chains are taken in database order; a chain already in a cluster is passed over; any other + chain becomes a representative, and its cluster takes every chain of its k-NN result row (k = + min(max_neighbors, number of chains), searched as build_similarity_graph searches it) that scores at least + ``threshold`` and is in no cluster yet. A member joins on its representative's score for it. A cluster has at + most k + 1 chains, and a representative whose k-th neighbour scores at least ``threshold`` may lose neighbours + above the threshold beyond rank k (its cluster may be cut): such a chain can become a representative, or a + singleton, that scores at least ``threshold`` with an earlier representative. + + How it is computed: build_similarity_graph's blocks, in ascending order; a block searches, in one FAISS call, + only its chains in no cluster at the block's start (a block with none is not read), then takes them in order, + passing over those an earlier chain of the block has taken. Each block's results are dropped before the next, + so memory grows with the number of chains, not with the edges. Where a chain's search result does not depend + on the other chains of its FAISS call (hnsw, and flat while a block's rows x dimension stay below FAISS's BLAS + switch, faiss.cvar.distance_compute_blas_threshold), this is the partition above bit for bit, whatever the + block size; otherwise a score can move by float rounding with the call's rows (up to about 2e-6 on flat at + 1536 dimensions, a few 1e-7 on ivf_pq), so a pair at ``threshold`` or tied at the k-th rank can change. There + is no randomness. + + Cluster ids are numbered by decreasing cluster size, ties by representative index (0 is the largest cluster); + ``self.representatives[i]`` is the chain index of cluster i's representative. + + Args: + threshold: A representative takes the chains in no cluster yet scoring at least this (cosine; identical + chains score 1, or close to 1 on ivf_pq, where scores are PQ estimates) + max_neighbors: Neighbours searched per representative, so also the cluster-size cap (this + 1) + + Returns: + Array of cluster IDs (one per chain) + """ + if self.db.index is None: + raise ValueError("Database not loaded. Call load_database() first.") + if max_neighbors < 1: + raise ValueError("--max-neighbors must be at least 1.") + + n_chains = len(self.chain_ids) + # Clamp max_neighbors to actual database size + k = min(max_neighbors, n_chains) + logging.info(f"\nGreedy clustering with threshold >= {threshold}") + logging.info(f"Searching up to {k} neighbors per representative, only chains in no cluster yet") + # Chains per block, as build_similarity_graph's + rows = 1 if k < 1 else max(1, min(n_chains, _GRAPH_MAX_ROWS, _GRAPH_RESULT_BYTES // (k * 12))) + + # Get CPU index for reconstruction if using GPU + index_to_use = self.db.index + if self.db.is_gpu_index: + logging.info("Moving GPU index to CPU for reconstruction...") + index_to_use = faiss.index_gpu_to_cpu(self.db.index) + + # ivf_pq queries are normalised, as in build_similarity_graph + normalize = self.db.index_type == IndexType.ivf_pq + + # Each chain's representative (-1: in no cluster yet), int32 when the largest id fits + labels = np.full(n_chains, -1, dtype=_graph_id_dtype(n_chains)) + n_searched = 0 + n_saturated = 0 + n_representatives = 0 + start_time = time.perf_counter() + + with tqdm(total=n_chains, desc="Clustering chains", unit="chain") as progress: + for start in range(0, n_chains, rows): + end = min(start + rows, n_chains) + # The block's chains in no cluster at its start: the only ones searched + todo = start + np.flatnonzero(labels[start:end] < 0) + if len(todo): + embeddings = np.ascontiguousarray(index_to_use.reconstruct_n(start, end - start), dtype=np.float32) + if normalize: + faiss.normalize_L2(embeddings) + queries = np.ascontiguousarray(embeddings[todo - start], dtype=np.float32) + del embeddings + scores, ids = self.db.index.search(queries, k) + del queries + n_searched += len(todo) + + # Kept cells: at least the threshold, and not the -1 ids FAISS fills in + ok = (scores >= threshold) & (ids >= 0) + for r, row in enumerate(todo.tolist()): + if labels[row] >= 0: + continue + labels[row] = row + n_representatives += 1 + neighbors = ids[r][ok[r]] + labels[neighbors[labels[neighbors] < 0]] = row + # The k-th neighbour is kept: the cluster may be cut (never when every chain is searched) + if k < n_chains and ok[r, -1]: + n_saturated += 1 + del scores, ids, ok + + progress.set_postfix(clusters=n_representatives, searched=n_searched, refresh=False) + progress.update(end - start) + + # Cluster ids by decreasing size, ties by representative index + is_rep = labels == np.arange(n_chains, dtype=labels.dtype) + rank = (np.cumsum(is_rep, dtype=labels.dtype) - 1)[labels] + sizes = np.bincount(rank) + order = np.argsort(-sizes, kind="stable") + new_id = np.empty_like(order, dtype=labels.dtype) + new_id[order] = np.arange(len(order), dtype=labels.dtype) + self.clusters = new_id[rank] + self.representatives = np.flatnonzero(is_rep)[order] + del labels, is_rep, rank, order, new_id + + self.method = "greedy" + self.k = k + self.max_neighbors = max_neighbors + self.n_searched = n_searched + self.n_saturated = n_saturated + + n_singletons = int(np.count_nonzero(sizes == 1)) + share = n_searched / n_chains if n_chains else 0.0 + logging.info(f"Searched {n_searched} of {n_chains} chains ({share:.2%})") + logging.info(f"Found {n_representatives} clusters, {n_singletons} singletons") + logging.info(f"Greedy clustering completed in {time.perf_counter() - start_time:.2f}s") return self.clusters @@ -145,19 +450,16 @@ def get_cluster_statistics(self) -> Dict: Dictionary with clustering statistics """ if self.clusters is None: - raise ValueError("Clustering not performed. Call cluster_leiden() first.") - - cluster_sizes = {} - for cluster_id in self.clusters: - cluster_sizes[cluster_id] = cluster_sizes.get(cluster_id, 0) + 1 + raise ValueError("Clustering not performed. Call cluster_leiden() or cluster_greedy() first.") - sizes = list(cluster_sizes.values()) - n_clusters = len(cluster_sizes) + # Memberships are 0..C-1 (Leiden and greedy) + sizes = np.bincount(self.clusters).tolist() + n_clusters = len(sizes) n_singletons = sum(1 for s in sizes if s == 1) - # Calculate modularity if graph exists + # Calculate modularity if graph exists and the partition is not greedy's, which is not computed on a graph modularity = None - if self.graph is not None: + if self.graph is not None and self.method != "greedy": modularity = self.graph.modularity(self.clusters, weights='weight') stats = { @@ -170,6 +472,13 @@ def get_cluster_statistics(self) -> Dict: 'median_cluster_size': np.median(sizes), 'modularity': modularity } + if self.method == "greedy": + stats.update({ + 'method': self.method, + 'n_searched': self.n_searched, + 'n_saturated': self.n_saturated, + 'n_at_cap': sum(1 for s in sizes if s >= self.max_neighbors), + }) return stats @@ -180,11 +489,13 @@ def export_clusters( format_mode: FormatMode = FormatMode.headless_tsv, ): """ - Export cluster assignments as tab-separated rows with no header. + Export cluster assignments as tab-separated rows with no header, in chain order. Columns are :data:`~foldmatch.search.output.CLUSTER_OUTPUT_FIELDS` (``chain_id, cluster_id, cluster_size``), matching the file convention - used by the search commands. + used by the search commands; after :meth:`cluster_greedy`, + :data:`~foldmatch.search.output.GREEDY_CLUSTER_OUTPUT_FIELDS`, which add + the representative's chain id. Args: output_file: Path to output file @@ -192,22 +503,30 @@ def export_clusters( format_mode: File layout (header or headless); see :class:`FormatMode` """ if self.clusters is None: - raise ValueError("Clustering not performed. Call cluster_leiden() first.") + raise ValueError("Clustering not performed. Call cluster_leiden() or cluster_greedy() first.") output_path = Path(output_file) output_path.parent.mkdir(parents=True, exist_ok=True) - # Calculate cluster sizes - cluster_sizes = {} - for cluster_id in self.clusters: - cluster_sizes[cluster_id] = cluster_sizes.get(cluster_id, 0) + 1 - - assignments = [ - (chain_id, cluster_id, cluster_sizes[cluster_id]) - for chain_id, cluster_id in zip(self.chain_ids, self.clusters) - if min_cluster_size is None or cluster_sizes[cluster_id] >= min_cluster_size - ] - exported = write_cluster_results(assignments, str(output_path), format_mode) + # Calculate cluster sizes (memberships are 0..C-1) + cluster_sizes = np.bincount(self.clusters) + greedy = self.method == "greedy" + + def _assignments(): + # Rows made _EXPORT_CHUNK chains at a time, not as one list of every chain + for s in range(0, len(self.clusters), _EXPORT_CHUNK): + cluster_ids = np.asarray(self.clusters[s:s + _EXPORT_CHUNK]) + sizes = cluster_sizes[cluster_ids] + columns = [self.chain_ids[s:s + _EXPORT_CHUNK], cluster_ids.tolist(), sizes.tolist()] + if greedy: + columns.append([self.chain_ids[r] for r in self.representatives[cluster_ids].tolist()]) + chunk = zip(*columns) + if min_cluster_size is not None: + chunk = (row for row, keep in zip(chunk, (sizes >= min_cluster_size).tolist()) if keep) + yield from chunk + + exported = write_cluster_results(_assignments(), str(output_path), format_mode, + GREEDY_CLUSTER_OUTPUT_FIELDS if greedy else CLUSTER_OUTPUT_FIELDS) logging.info(f"Cluster assignments exported to {output_file}") if min_cluster_size: @@ -227,3 +546,16 @@ def print_statistics(self): logging.info(f"Cluster size (med): {stats['median_cluster_size']:.1f}") if stats['modularity'] is not None: logging.info(f"Modularity: {stats['modularity']:.4f}") + if stats.get('method') == "greedy": + n_chains, n_clusters = stats['n_chains'], stats['n_clusters'] + saturated_share = stats['n_saturated'] / n_clusters + logging.info(f"Method: {stats['method']}") + logging.info(f"Chains searched: {stats['n_searched']} of {n_chains} " + f"({stats['n_searched'] / n_chains:.2%})") + logging.info(f"Representatives with all --max-neighbors neighbours at or above --threshold: " + f"{stats['n_saturated']} of {n_clusters} ({saturated_share:.2%}); their clusters may be cut, " + f"and a larger --max-neighbors can change them.") + logging.info(f"Clusters of at least --max-neighbors chains: {stats['n_at_cap']}") + if saturated_share > 0.5: + logging.warning("Most clusters are bounded by --max-neighbors, not --threshold: a representative or " + "singleton may score at or above --threshold with an earlier representative.") diff --git a/src/foldmatch/search/code_alignment.py b/src/foldmatch/search/code_alignment.py new file mode 100644 index 0000000..cb1d55d --- /dev/null +++ b/src/foldmatch/search/code_alignment.py @@ -0,0 +1,1765 @@ +"""esm-codes Stage 2: align Stage-1 candidates with ESM3 residue codes (plan section 8 P6.4; method S5-S14, S16). + +For each query (its S5 encoding in the per-run query memmap, :class:`~foldmatch.search.query_encodings. +QueryEncodingIndex`) and its Stage-1 candidates (their S4 codes in the database's +:class:`~foldmatch.search.residue_code_store.ResidueCodeStore`, looked up **by id**: the ``.bin`` order depends on the +number of build ranks), :func:`align_candidates_esm_codes` runs:: + + parent drop fully degenerate queries (S13b: n = 1 or all q rows bitwise identical; no Stage-2 work, no hits) + round 1 score only, per pair: S_f and S_r (S9; one similarity GEMM per pair, S_r on its row-reversed view) + over the panel members (D11: only when E-values are needed) and the candidates, in chunked tasks + parent S = fp64(S_f) - fp64(S_r); Q90 = quantile(panel S, 0.9) (S10); S' = S - Q90, ln E, bits (S11); + survivors: ln E <= ln(max_evalue), or every candidate when there is no E filter + round 2 survivors only: forward traceback of S_f (S12); its score must equal round 1's S_f bit for bit + (:class:`~foldmatch.errors.InternalError` otherwise); in the worker too, the paths' vectorised S12 + column tallies and the requested CIGAR / alignment strings, the targets' non-standard counts and + whether they have enough distinct codes (S13) + parent identity and coverage, ecal (S13), filters (alnlen > 0, identity, min(qcov, tcov)), sort (S14) + +**Tasks.** A round-1 task is ``(task id, source, query memmap offset, query length, [(offset, length), ...])``: the +rows of its targets in the panel codes or in the code store; a round-2 task also carries the query and target +sequences and which strings the output needs. Workers hold everything else (:func:`_init_worker`: the kernel, +the scoring npz through :meth:`~foldmatch.search.scoring_system.WorkerSpec.load_panel`, the code-store memmap, the +query memmap and a :class:`~foldmatch.search.swkern.Workspace`) and return numpy arrays. The task size adapts to the +batch: at most ``task_cells`` similarity cells, and small enough for at least 4 tasks per worker (but not below +2^20 cells), so that one interactive query still uses every core. Workers import neither torch nor biotite. + +**Dataflow** (:class:`_Dataflow`). A batch's round-1 tasks are queued longest first (LPT, by ``n x sum(m)``); each +query moves on as soon as its own tasks are back: statistics and survivors when its round 1 is complete (its round-2 +tasks are queued ahead of round 1), hits when its round 2 is complete (its paths are then released). The next batch is +looked up and queued while the workers still have this one's tasks (at most two batches open). Tasks are submitted one +at a time, at most twice the pool's workers in flight (:class:`_Session`), and their results consumed in completion +order: the parent holds the round-2 payload of the queries in flight only, and builds hits while the workers compute. + +**Reproducibility (S16).** Every pair's similarity is its own fp32 GEMM into a 64-byte-aligned buffer, so a pair's +bits depend only on the pair, the BLAS build and the BLAS thread count (multi-threaded OpenBLAS changes last bits, +within S16): with BLAS single-threaded the output is bit-identical for any ``task_cells``, ``block_floats``, +``query_batch``, worker count (in-process or pool) and candidate order. The pool's workers are started with +``OMP/OPENBLAS/MKL/VECLIB_*_NUM_THREADS=1`` (and limited with threadpoolctl when it is installed); in-process tasks run +under a threadpoolctl limit of 1 thread, so without threadpoolctl a pool that was asked for workers never runs tasks +in-process, and an explicitly serial run (0 or 1 workers) needs ``*_NUM_THREADS=1`` in the environment to give the +pool's bits. + +**Pool.** :class:`CodeAlignmentPool` is the persistent ``spawn`` pool of one run (a +:class:`concurrent.futures.ProcessPoolExecutor`), started lazily at the first batch that needs it and reused for both +rounds and every batch; without a worker count it starts only as many workers as the batch's work justifies (one per +:data:`MIN_WORKER_CELLS` similarity cells, up to one per physical core); ``None``, ``0`` or ``1`` as the aligner's ``pool`` runs +the same task functions in-process. A worker that ends without returning its task (killed, e.g. by the out-of-memory killer, or unable to start) stops the run +with :class:`WorkerPoolError` instead of leaving it waiting. Spawn workers re-import the main module, so a script +that runs this Stage 2 with a pool must guard its entry point with ``if __name__ == "__main__":`` (the ``fm-search`` +command does); without the guard the workers cannot start and the run fails with :class:`WorkerPoolError`. +""" +from __future__ import annotations + +import logging +import math +import multiprocessing as mp +import os +import queue +import time +from collections import deque +from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures.process import BrokenProcessPool +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any, Dict, Iterable, Iterator, List, Mapping, Optional, Sequence, Tuple + +import numpy as np + +from foldmatch.errors import InternalError +from foldmatch.search import code_statistics as cs +from foldmatch.search import stage2_profile +from foldmatch.search import swkern +from foldmatch.search.alignment import ( + _BIOTITE_ALPHABET, AlignmentMetrics, Hit, _expand_cpu_affinity, +) +from foldmatch.utils.sequence import STANDARD_AMINO_ACIDS, normalize_sequence, seq_sha1 + +logger = logging.getLogger(__name__) + +PANEL_POLICIES = ("auto", "always", "never") +#: Output fields whose values need the panel pass (Q90): with none of them requested and no finite ``max_evalue``, +#: ``panel_policy="auto"`` skips the panel (D11). ``q90`` and ``rawadj`` are esm-codes fields (P6.5). +PANEL_FIELDS = frozenset({"evalue", "bits", "q90", "rawadj"}) +DEFAULT_TASK_CELLS = 1 << 24 +TASK_CELLS_FLOOR = 1 << 20 +TASKS_PER_WORKER = 4 +#: A pool sized by default (``num_workers=None``) starts at most one worker per this many similarity cells of a batch +#: (about 40 ms of one EPYC core, against about 0.3 s to start a spawn worker), so a small query does not start a +#: worker on every CPU; a later batch with more work grows it (by restart, at least doubling). 2^23 gives a 300-aa +#: query with 100 candidates and the panel 14 workers. +MIN_WORKER_CELLS = 1 << 23 +DEFAULT_QUERY_BATCH = 256 +#: A worker keeps its kernel buffers (:class:`~foldmatch.search.swkern.Workspace`) between tasks up to this size: +#: after a task whose pairs needed more (a long pair: up to ``max_pair_cells`` float32 cells of similarity, about +#: 1 GiB, plus its traceback flags) they are released, so only the workers on such a pair at the time hold that +#: memory. 128 MiB covers the pairs of the calibrated domain (966 x 3,000 aa: 11 MiB of similarity) many times over. +WORKSPACE_KEEP_BYTES = 128 << 20 +_BLAS_THREAD_VARS = ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "VECLIB_MAXIMUM_THREADS", + "BLIS_NUM_THREADS") +_PANEL, _CODES = 0, 1 +_EXAMPLES = 10 + + +class WorkerPoolError(RuntimeError): + """A Stage-2 worker process ended without returning its task's result: it was killed (for example by the + out-of-memory killer) or could not start (a script without an ``if __name__ == "__main__":`` guard). The pool + is stopped; a persistent :class:`CodeAlignmentPool` starts a new one at its next batch.""" + + +#: swkern's once-per-process warning about pairs above ``max_pair_cells``. Stage 2 drops it (in a pool worker it +#: would bypass the parent's logging setup, once per worker) and the parent logs the run's count once instead. +_SWKERN_SKIP_WARNING = "esm-codes Stage 2: skipping pairs whose similarity exceeds max_pair_cells" + + +class _DropSwkernSkipWarning(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + return not str(record.msg).startswith(_SWKERN_SKIP_WARNING) + + +_DROP_SKIP_WARNING = _DropSwkernSkipWarning() + + +# ===================================================================================================================== +# output needs +# ===================================================================================================================== +@dataclass(frozen=True) +class OutputNeeds: + """What the requested output columns need beyond the always-computed scalars: the heavy strings, and whether any + column needs the panel pass (``significance``: evalue, bits, q90 or rawadj).""" + cigar: bool = False + qaln: bool = False + taln: bool = False + qseq: bool = False + tseq: bool = False + significance: bool = False + + @classmethod + def from_fields(cls, fields: Iterable[str]) -> "OutputNeeds": + fs = set(fields) + return cls(cigar="cigar" in fs, qaln="qaln" in fs, taln="taln" in fs, qseq="qseq" in fs, + tseq="tseq" in fs, significance=bool(fs & PANEL_FIELDS)) + + @classmethod + def coerce(cls, value) -> "OutputNeeds": + """An :class:`OutputNeeds`, or the ``--format-output`` field names to derive it from.""" + if isinstance(value, cls): + return value + if isinstance(value, (str, bytes)) or not hasattr(value, "__iter__"): + raise TypeError(f"output_needs must be an OutputNeeds or a sequence of output field names, got {value!r}") + return cls.from_fields(value) + + +# ===================================================================================================================== +# task planning +# ===================================================================================================================== +def target_task_cells(total_cells: int, workers: int, task_cells: int) -> int: + """Similarity cells per task for a batch of ``total_cells``: at most ``task_cells``, and small enough to give at + least ``TASKS_PER_WORKER`` tasks per worker, but not below ``TASK_CELLS_FLOOR`` (2^20) unless ``task_cells`` is.""" + adaptive = max(TASK_CELLS_FLOOR, -(-int(total_cells) // (TASKS_PER_WORKER * max(1, int(workers))))) + return max(1, min(int(task_cells), adaptive)) + + +def plan_chunks(n: int, lengths: Sequence[int], target_cells: int) -> List[Tuple[int, int]]: + """Ranges ``[k0, k1)`` of consecutive targets whose ``n x m`` cells add up to at most ``target_cells`` (greedy, + in order); a target that alone exceeds it is its own chunk. Chunking never changes a pair's bits.""" + out: List[Tuple[int, int]] = [] + k0, acc = 0, 0 + for k, m in enumerate(lengths): + cells = n * int(m) + if k > k0 and acc + cells > target_cells: + out.append((k0, k)) + k0, acc = k, 0 + acc += cells + if k0 < len(lengths): + out.append((k0, len(lengths))) + return out + + +def _lpt(tasks: list, costs: list) -> list: + """Longest-processing-time-first order (stable: equal costs keep their task id order).""" + order = sorted(range(len(tasks)), key=lambda i: -costs[i]) + return [tasks[i] for i in order] + + +# ===================================================================================================================== +# worker state and task functions (also run in-process by the serial path) +# ===================================================================================================================== +@dataclass(frozen=True) +class _WorkerInit: + """Picklable worker setup: the kernel ISA, the scoring system's worker view, the code store and query memmaps as + ``(path, rows, width)``, and the kernel blocking limits.""" + isa: str + spec: Any # scoring_system.WorkerSpec + codes: Tuple[str, int, int] # code store .bin, N_res, code_bytes + queries: Tuple[str, int, int] # query encodings, rows, width + block_floats: int + max_pair_cells: int + profile: bool = False # profiling mode (stage2_profile): tasks also return their timing record + + +class _WorkerState: + __slots__ = ("init", "kernel", "params", "panel_codes", "codes", "queries", "workspace", "blas_limit", "error", + "init_record") + + def __init__(self, init: _WorkerInit, error: Optional[BaseException] = None): + self.init = init + self.error = error + self.kernel = self.params = self.panel_codes = self.codes = self.queries = self.workspace = None + self.blas_limit = None + self.init_record = None # profiling: this process's start-up, sent with its first task + + +_STATE: Optional[_WorkerState] = None + + +def _open_rows(path: str, rows: int, width: int, dtype) -> np.ndarray: + """A raw ``[rows, width]`` file mapped read-only (an empty array for 0 rows, which np.memmap cannot map).""" + if rows == 0: + return np.empty((0, width), dtype=dtype) + return np.memmap(path, dtype=dtype, mode="r", shape=(int(rows), int(width))) + + +def _blas_limit_available() -> bool: + """Whether this process can limit its BLAS to one thread at run time (threadpoolctl is installed).""" + try: + import threadpoolctl # noqa: F401 + except ImportError: + return False + return True + + +def _limit_blas_threads(): + """threadpoolctl's limiter of the BLAS pools to 1 thread, or None when threadpoolctl is not installed.""" + try: + import threadpoolctl + except ImportError: + return None + return threadpoolctl.threadpool_limits(limits=1, user_api="blas") + + +_BLAS_CONTROL_NOTED = False + + +def _blas_controllable() -> bool: + """Whether threadpoolctl finds a BLAS it can limit in this process. It cannot see every BLAS: Apple Accelerate + (numpy's BLAS on macOS) is not one it supports, so a limit there has no effect.""" + try: + import threadpoolctl + except ImportError: + return False + return any(info.get("user_api") == "blas" for info in threadpoolctl.threadpool_info()) + + +def _note_uncontrolled_blas() -> None: + """Once per process, when in-process tasks start: say so if threadpoolctl cannot limit this BLAS (they then run + with its own thread count, like a run without threadpoolctl).""" + global _BLAS_CONTROL_NOTED + if _BLAS_CONTROL_NOTED: + return + _BLAS_CONTROL_NOTED = True + if not _blas_controllable(): + logger.info("esm-codes Stage 2 runs in-process and threadpoolctl cannot limit this BLAS (for example Apple " + "Accelerate): it keeps its own thread count (set OMP/OPENBLAS/MKL/VECLIB_*_NUM_THREADS=1 for " + "the bits of a worker pool; S16 holds either way)") + + +def _init_worker(init: _WorkerInit, raise_errors: bool = False) -> None: + """Load what every task needs, once per process. In a pool worker an error is kept and raised by every task + instead (a failing initializer would only report a broken pool).""" + global _STATE + _STATE = None + if init.profile: + t_unix, t0 = time.time(), time.perf_counter() + try: + st = _WorkerState(init) + st.kernel = swkern.load(init.isa) + st.params = init.spec.dp_params() + st.panel_codes, _ = init.spec.load_panel() + st.codes = _open_rows(*init.codes, dtype=np.uint8) + st.queries = _open_rows(*init.queries, dtype=np.float16) + st.workspace = swkern.Workspace() + if not raise_errors: + st.blas_limit = _limit_blas_threads() + if init.profile: + st.init_record = stage2_profile.init_record(t_unix, time.perf_counter() - t0) + _STATE = st + except BaseException as e: # noqa: BLE001 (re-raised by every task of this worker) + if raise_errors: + raise + _STATE = _WorkerState(init, error=e) + + +def _init_pool_worker(init: _WorkerInit) -> None: + """The pool's initializer: swkern's oversize-pair warning dropped for the life of the worker process (the parent + reports the run's count), then :func:`_init_worker` with its errors kept for the tasks.""" + logging.getLogger(swkern.__name__).addFilter(_DROP_SKIP_WARNING) + _init_worker(init) + + +def _reset_worker_state() -> None: + global _STATE + _STATE = None + + +def _state() -> _WorkerState: + st = _STATE + if st is None: + raise InternalError("esm-codes Stage-2 task run in a process whose worker state was not initialised") + if st.error is not None: + raise st.error + return st + + +def _query_prep(st: _WorkerState, q_off: int, n: int) -> np.ndarray: + return swkern.query_prep(np.asarray(st.queries[q_off:q_off + n]), st.kernel) + + +def _task_timer(st: _WorkerState) -> Optional[stage2_profile.TaskTimer]: + return stage2_profile.TaskTimer() if st.init.profile else None + + +def _task_record(st: _WorkerState, timer: stage2_profile.TaskTimer, kind: str, n: int, pairs, result: tuple) -> dict: + init, st.init_record = st.init_record, None + return timer.record(kind, n, pairs, result, init) + + +def _round1_task(task): + """Round 1: ``(task id, S_f float32 [k], S_r float32 [k], skipped bool [k])`` of one query against a chunk of + panel members or candidates (profiling mode: plus the task's timing record).""" + tid, source, q_off, n, pairs = task + st = _state() + timer = _task_timer(st) + qp = _query_prep(st, q_off, n) + if timer is not None: + timer.lap("prep") + codes = st.panel_codes if source == _PANEL else st.codes + r = swkern.score_fwd_rev(qp, codes, pairs[:, 0], st.params, lengths=pairs[:, 1], + block_floats=st.init.block_floats, max_pair_cells=st.init.max_pair_cells, + kernel=st.kernel, workspace=st.workspace, timings=None if timer is None else timer.t) + st.workspace.release(WORKSPACE_KEEP_BYTES) + if timer is None: + return tid, r.s_fwd, r.s_rev, r.skipped + timer.lap("swkern") + out = (tid, r.s_fwd, r.s_rev, r.skipped) + return out + (_task_record(st, timer, "panel" if source == _PANEL else "candidates", n, pairs, out),) + + +def _distinct_codes(codes: np.ndarray, cap: int) -> int: + """``min(distinct codes, cap)``: S13 only asks whether a target has at least ``cap`` + (``target_min_distinct_codes``) distinct codes, so the count stops there. For ``cap <= 2`` (esm-codes-v1: 2) + that is one comparison of every code with the first instead of a sort.""" + if len(codes) == 0: + return 0 + v = np.ascontiguousarray(codes).view(np.uint64).ravel() + if cap <= 1: + return 1 + if cap == 2: + return 2 if bool((v != v[0]).any()) else 1 + return min(int(np.unique(v).size), int(cap)) + + +#: the S12 column tallies a round-2 task returns per survivor (``path_columns`` keys), in this order +_COLUMNS = ("aln_len", "n_ident", "mismatch", "gap_open", "q_aligned", "t_aligned", "q_start", "q_end", "t_start", + "t_end") + + +def _round2_task(task): + """Round 2: the forward traceback of survivors, and what the output needs from their paths, so the parent only + builds the hits: ``(task id, S_f float32 [k], ends int32 [k, 2], columns int64 [k, len(_COLUMNS)] (S12 tallies), + end_ok bool [k] (a nonempty path ends at the best cell), non-standard residues int64 [k], distinct codes int64 [k] + (counted up to ``target_min_distinct_codes``), skipped bool [k], strings)``; ``strings`` holds the CIGAR, qaln, + taln and tseq strings asked for by ``want`` (else None each). The task is ``(task id, query memmap offset, + query length, [(offset, length), ...], query sequence, [target sequences], want (cigar, qaln, taln, tseq))`` + (profiling mode: plus the task's timing record).""" + tid, q_off, n, pairs, qseq, t_seqs, want = task + st = _state() + timer = _task_timer(st) + qp = _query_prep(st, q_off, n) + if timer is not None: + timer.lap("prep") + a = swkern.align(qp, st.codes, pairs[:, 0], st.params, lengths=pairs[:, 1], traceback=True, + block_floats=st.init.block_floats, max_pair_cells=st.init.max_pair_cells, kernel=st.kernel, + workspace=st.workspace, timings=None if timer is None else timer.t) + st.workspace.release(WORKSPACE_KEEP_BYTES) + if timer is not None: + timer.lap("swkern") + cap = st.init.spec.calibrated_domain.target_min_distinct_codes + distinct = np.array([_distinct_codes(st.codes[o:o + m], cap) for o, m in pairs.tolist()], np.int64) + if timer is not None: + timer.lap("distinct") + columns, end_ok, nonstd, strings = _path_outputs(a, qseq, t_seqs, pairs[:, 1], want) + out = (tid, a.scores, a.ends, columns, end_ok, nonstd, distinct, a.skipped, strings) + if timer is None: + return out + timer.lap("metrics") + return out + (_task_record(st, timer, "round2", n, pairs, out),) + + +def _path_outputs(a: "swkern.AlignResult", qseq: str, t_seqs: Sequence[str], lengths: np.ndarray, want): + """Round 2's per-survivor outputs of :func:`_round2_task` from the kernel's paths: the S12 column tallies + (:func:`path_columns` over the sanitised sequences), whether each nonempty path ends at its best cell, the + targets' non-standard counts (S13) and the requested strings (CIGAR, gapped query and target, sanitised + target), each exactly as the parent built them per hit before.""" + want_cigar, want_qaln, want_taln, want_tseq = want + k = len(t_seqs) + q_bytes = np.frombuffer(sanitized_bytes(qseq), np.uint8) + t_clean = [sanitized_bytes(x) for x in t_seqs] + nonstd = np.array([nonstandard_count(x) for x in t_seqs], np.int64) + t_starts = np.zeros(k, np.int64) + if k > 1: + np.cumsum(np.asarray(lengths, np.int64)[:-1], out=t_starts[1:]) + t_bytes = np.frombuffer(b"".join(t_clean), np.uint8) + path_off = np.asarray(a.path_off, np.int64) + cols = path_columns(a.path, path_off, q_bytes, t_bytes, t_starts) + columns = np.stack([cols[c] for c in _COLUMNS], axis=1) if k else np.zeros((0, len(_COLUMNS)), np.int64) + end_ok = np.ones(k, bool) + nonempty = np.flatnonzero(np.diff(path_off) > 0) + if nonempty.size: + path = np.asarray(a.path).reshape(-1, 2) + end_ok[nonempty] = np.all(np.asarray(a.ends)[nonempty] == path[path_off[nonempty + 1] - 1] + 1, axis=1) + cigars = q_alns = t_alns = t_strs = None + if want_cigar or want_qaln or want_taln: + ps = _PathStrings(a.path, path_off, q_bytes, t_bytes, t_starts, cigar=want_cigar, qaln=want_qaln, + taln=want_taln) + bounds = list(zip(path_off[:-1].tolist(), path_off[1:].tolist())) + if want_cigar: + cigars = [ps.cigar(lo, hi) for lo, hi in bounds] + if want_qaln: + q_alns = [ps.qaln[lo:hi].decode("ascii") for lo, hi in bounds] + if want_taln: + t_alns = [ps.taln[lo:hi].decode("ascii") for lo, hi in bounds] + if want_tseq: + t_strs = [x.decode("ascii") for x in t_clean] + return columns, end_ok, nonstd, (cigars, q_alns, t_alns, t_strs) + + +# ===================================================================================================================== +# the pool +# ===================================================================================================================== +@contextmanager +def _single_threaded_blas_env(): + """``*_NUM_THREADS=1`` in this process's environment while the pool's workers are spawned (they inherit it).""" + saved = {k: os.environ.get(k) for k in _BLAS_THREAD_VARS} + os.environ.update(dict.fromkeys(_BLAS_THREAD_VARS, "1")) + try: + yield + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +def _openmp_thread_counts() -> list: + """``(controller, threads)`` for every OpenMP runtime threadpoolctl finds in this process; empty without + threadpoolctl, or when it cannot list them.""" + try: + import threadpoolctl + return [(lib, lib.num_threads) + for lib in threadpoolctl.ThreadpoolController().select(user_api="openmp").lib_controllers] + except Exception: # noqa: BLE001 (nothing to set back then: threadpoolctl's own restore only) + return [] + + +@contextmanager +def _blas_one_thread_here(): + """BLAS at one thread for the block, as in a pool worker (threadpoolctl); every thread count as before after it. + + An OpenMP BLAS keeps no thread count of its own (OpenBLAS 0.3.15's OpenMP build, the faiss-cpu wheel's BLAS on + Linux): the limit sets the OpenMP count of its runtime, the one FAISS runs (and torch, which shares it when torch + is loaded first), and threadpoolctl's restore sets that count to the one the BLAS reported at entry, which lags + behind FAISS's until a BLAS call (P9: FAISS set to 7 came back at 128). So once threadpoolctl has restored its + pools, each OpenMP runtime gets its count at entry again. + """ + openmp = _openmp_thread_counts() + limiter = _limit_blas_threads() + if limiter is not None: + _note_uncontrolled_blas() + try: + yield + finally: + if limiter is not None: + limiter.restore_original_limits() + for lib, threads in openmp: + if lib.num_threads != threads: + lib.set_num_threads(threads) + + +@contextmanager +def _swkern_skip_warning_dropped(): + """In-process tasks: swkern's oversize-pair warning dropped, as in a pool worker (the parent logs the count).""" + log = logging.getLogger(swkern.__name__) + added = _DROP_SKIP_WARNING not in log.filters + log.addFilter(_DROP_SKIP_WARNING) + try: + yield + finally: + if added: + log.removeFilter(_DROP_SKIP_WARNING) + + +def _stop_workers(executor: ProcessPoolExecutor) -> None: + """Terminate an executor's worker processes now, without waiting for their tasks (``terminate_workers`` from + Python 3.14; before it, through the executor's process table), then release the executor.""" + terminate = getattr(executor, "terminate_workers", None) + if callable(terminate): + try: + terminate() + except Exception: # noqa: BLE001 (an executor that is already broken or shut down) + pass + else: + for proc in list((getattr(executor, "_processes", None) or {}).values()): + if proc.is_alive(): + proc.terminate() + executor.shutdown(wait=True, cancel_futures=True) + + +_LOST_WORKER = ("esm-codes Stage 2: a worker process ended without returning its task. It was killed (for example " + "by the out-of-memory killer: a worker holds the similarity matrix of its largest pair, up to " + "max_pair_cells float32 cells, so fewer --align-workers need less memory) or it could not start (a " + "script that runs this Stage 2 with a worker pool must guard its entry point with 'if __name__ == " + "\"__main__\":', because spawn workers re-import the main module). The worker pool was stopped.") + + +class CodeAlignmentPool: + """The persistent Stage-2 worker pool of one run (plan P7 design): ``spawn`` workers of a + :class:`concurrent.futures.ProcessPoolExecutor`, created lazily at the first batch with more than one task (after + ESM3 and the aggregator are freed) and started as its tasks are submitted, reused for both rounds and every + batch, restarted only when the worker setup (database, query memmap, limits) changes. + + ``num_workers``: ``None`` uses up to one worker per physical core (the CPU affinity mask is widened first, as for + BLOSUM62; SMT siblings count once), but starts only as many workers as the work justifies (one per ``min_worker_cells`` similarity cells of the batch, + :data:`MIN_WORKER_CELLS`; a later, larger batch restarts the pool with more); ``0`` or ``1`` runs the task + functions in-process; ``N > 1`` uses N workers, started as tasks are submitted (a small first batch starts fewer, + and later batches start the rest). While no pool is running, a batch with a single task, or (``None``) with less + work than one worker's share, runs in-process, but only when BLAS can be limited to one thread here + (threadpoolctl); otherwise a pool of one worker is started, so the bits never depend on the batch. + + A worker that ends without returning its task (killed, or unable to start) raises :class:`WorkerPoolError` in + the parent instead of a wait that never ends; the pool is stopped and the next batch starts a new one. Spawn + workers re-import the main module: a script that uses a pool needs an ``if __name__ == "__main__":`` guard. + Use it as a context manager, or call :meth:`close`. + """ + + def __init__(self, num_workers: Optional[int] = None, *, start_method: str = "spawn", + min_worker_cells: int = MIN_WORKER_CELLS): + if num_workers is not None and (isinstance(num_workers, bool) or not isinstance(num_workers, int) + or num_workers < 0): + raise ValueError(f"num_workers must be None or an int >= 0, got {num_workers!r}") + if start_method not in mp.get_all_start_methods(): + raise ValueError(f"start method {start_method!r} is not available here ({mp.get_all_start_methods()})") + self.num_workers = num_workers + self.start_method = start_method + self.min_worker_cells = _positive_int("min_worker_cells", min_worker_cells) + self._pool: Optional[ProcessPoolExecutor] = None + self._init: Optional[_WorkerInit] = None + self._requested: Optional[int] = None + self._in_process: Optional[_WorkerInit] = None + self.workers = 0 # worker processes the running pool may start (0 = in-process or not started) + self.starts = 0 # process pools started so far + + def __enter__(self) -> "CodeAlignmentPool": + return self + + def __exit__(self, exc_type, exc, tb): + self.close(terminate=exc_type is not None) + + def __repr__(self): + return (f"CodeAlignmentPool(num_workers={self.num_workers!r}, start_method={self.start_method!r}, " + f"workers={self.workers})") + + @property + def serial(self) -> bool: + return self.num_workers is not None and self.num_workers <= 1 + + def requested_workers(self) -> int: + """The worker budget: 1 when serial, else ``num_workers`` or one per physical core of the usable CPUs (after + widening affinity; SMT siblings count once: two workers on one core run slower than one, P7.2).""" + if self.serial: + return 1 + if self._requested is None: + _expand_cpu_affinity() + self._requested = max(1, _physical_cpus() if self.num_workers is None else int(self.num_workers)) + return self._requested + + def plan_workers(self, cells: int) -> int: + """The workers a batch of ``cells`` similarity cells is planned for (task sizes) and runs on: 1 when serial, + ``num_workers`` when given, else (``None``) one per ``min_worker_cells`` cells, at least the running pool's + workers and at most the budget (:meth:`requested_workers`).""" + budget = self.requested_workers() + if self.serial or self.num_workers is not None: + return budget + work = max(1, -(-max(int(cells), 0) // self.min_worker_cells)) + return min(budget, max(work, self.workers)) + + @property + def processes(self) -> int: + """Worker processes the running pool has started so far (the executor starts them as tasks are + submitted).""" + return len(getattr(self._pool, "_processes", None) or {}) if self._pool is not None else 0 + + def run(self, fn, tasks: list, init: _WorkerInit, profile: Optional[stage2_profile.Stage2Profile] = None, + workers: Optional[int] = None) -> Iterator: + """The results of ``fn`` over ``tasks``, in completion order, on ``workers`` workers (default: the budget; + the aligner passes :meth:`plan_workers`). ``profile`` (profiling mode) receives the executor start-up and + each task's submit time.""" + if not tasks: + return + if self._pool is not None and self._init != init: + logger.info("esm-codes Stage 2: the worker setup changed; restarting the worker pool") + self.close() + want = self.requested_workers() if workers is None else max(1, min(int(workers), self.requested_workers())) + if self._pool is None: + if self.serial or (min(want, len(tasks)) <= 1 and _blas_limit_available()): + yield from self._run_here(fn, tasks, init) + return + self._start(init, want, profile) + elif want > self.workers: + logger.info(f"esm-codes Stage 2: this batch needs {want} workers; restarting the pool of {self.workers}") + self.close() + self._start(init, want, profile) + yield from self._run_in_pool(fn, tasks, profile) + + def _run_here(self, fn, tasks, init): + if _STATE is None or _STATE.init != init or _STATE.error is not None: + _init_worker(init, raise_errors=True) + if self._in_process is None and not _blas_limit_available(): + logger.info("esm-codes Stage 2 runs in-process without threadpoolctl: BLAS keeps this process's thread " + "count (set OMP/OPENBLAS/MKL/VECLIB_*_NUM_THREADS=1 for the bits of a worker pool; S16 holds " + "either way)") + self._in_process = init + with _blas_one_thread_here(), _swkern_skip_warning_dropped(): + for task in tasks: + yield fn(task) + + def _run_in_pool(self, fn, tasks, profile=None): + """Submit every task (longest first: the caller's order), then yield the results as they complete. Tasks not + started yet are cancelled when the caller stops early (an error); a lost worker raises + :class:`WorkerPoolError`.""" + futures = [] + try: + try: + # The executor spawns its workers as tasks are submitted: they inherit *_NUM_THREADS=1 from here. + with _single_threaded_blas_env(): + for task in tasks: + if profile is not None: + profile.submitted(task) + futures.append(self._pool.submit(fn, task)) + for future in as_completed(futures): + yield future.result() + except BrokenProcessPool as exc: + self.close(terminate=True) + raise WorkerPoolError(_LOST_WORKER) from exc + finally: + for future in futures: + future.cancel() + + def session(self, init: _WorkerInit, cells: int, tasks: int, + profile: Optional[stage2_profile.Stage2Profile] = None) -> "_Session": + """A :class:`_Session` for one aligner call whose first batch has ``cells`` similarity cells in ``tasks`` + round-1 tasks: in-process (serial, or no pool running and the batch qualifies, as in :meth:`run`), else on + this pool, started or grown to :meth:`plan_workers` (a pool with another worker setup is restarted).""" + if self._pool is not None and self._init != init: + logger.info("esm-codes Stage 2: the worker setup changed; restarting the worker pool") + self.close() + want = self.plan_workers(cells) + if self._pool is None: + if self.serial or (min(want, tasks) <= 1 and _blas_limit_available()): + return _Session(self, init, True, profile) + self._start(init, want, profile) + elif want > self.workers: + logger.info(f"esm-codes Stage 2: this batch needs {want} workers; restarting the pool of {self.workers}") + self.close() + self._start(init, want, profile) + return _Session(self, init, False, profile) + + def _start(self, init: _WorkerInit, workers: int, profile=None): + t0 = time.perf_counter() + ctx = mp.get_context(self.start_method) + # max_workers is the pool's size for the rest of the run: the executor starts the workers as tasks are + # submitted, so a small first batch starts fewer and a larger batch later starts the rest + self._pool = ProcessPoolExecutor(max_workers=workers, mp_context=ctx, initializer=_init_pool_worker, + initargs=(init,)) + self._init, self.workers = init, workers + self.starts += 1 + if profile is not None: + profile.executor_created(time.perf_counter() - t0, workers) + logger.info(f"esm-codes Stage 2: a pool of up to {workers} {self.start_method} workers") + + def close(self, terminate: bool = False) -> None: + """Stop the workers (``terminate``: at once, without waiting for queued or running tasks) and release the + in-process state.""" + pool, self._pool = self._pool, None + if pool is not None: + self._init, self.workers = None, 0 + if terminate: + _stop_workers(pool) + else: + pool.shutdown(wait=True) + if self._in_process is not None and _STATE is not None and _STATE.init == self._in_process: + _reset_worker_state() + self._in_process = None + + +class _Session: + """The tasks of one aligner call: submitted one at a time and their results taken in completion order, on the + pool's executor (at most :attr:`window` in flight: twice the pool's workers, so every worker has its next task + queued while the parent bounds what it holds) or in-process (each task runs when it is submitted, one in flight). + ``with`` it: leaving cancels the tasks not started yet. A worker that ends without returning its task raises + :class:`WorkerPoolError` (the pool is stopped).""" + + def __init__(self, pool: CodeAlignmentPool, init: _WorkerInit, in_process: bool, + profile: Optional[stage2_profile.Stage2Profile]): + self.pool, self.init, self.in_process, self.profile = pool, init, in_process, profile + self.done: "queue.SimpleQueue" = queue.SimpleQueue() + self.pending: set = set() + self.local: deque = deque() + self.inflight = 0 + self._exit = None + if in_process: + if _STATE is None or _STATE.init != init or _STATE.error is not None: + _init_worker(init, raise_errors=True) + if pool._in_process is None and not _blas_limit_available(): + logger.info("esm-codes Stage 2 runs in-process without threadpoolctl: BLAS keeps this process's " + "thread count (set OMP/OPENBLAS/MKL/VECLIB_*_NUM_THREADS=1 for the bits of a worker pool; " + "S16 holds either way)") + pool._in_process = init + self.workers, self.window = 1, 1 + else: + self.workers = pool.workers + self.window = max(2 * pool.workers, 2) + + def __enter__(self) -> "_Session": + if self.in_process: + blas, warn = _blas_one_thread_here(), _swkern_skip_warning_dropped() + blas.__enter__() + warn.__enter__() + self._exit = (warn, blas) + return self + + def __exit__(self, exc_type, exc, tb): + for future in self.pending: + future.cancel() + self.pending.clear() + if self._exit is not None: + for cm in self._exit: + cm.__exit__(None, None, None) + self._exit = None + + @property + def full(self) -> bool: + return (len(self.local) if self.in_process else self.inflight) >= self.window + + def submit(self, fn, task, tag, rnd: Optional[int] = None) -> None: + """Run ``fn(task)``; its result comes back from :meth:`next` with ``tag`` (``rnd``: the profile's round).""" + if self.in_process: + self.local.append((tag, fn(task))) + return + prof, executor = self.profile, self.pool._pool + if prof is not None: + prof.submitted(task, rnd) + try: + if self.pool.processes < self.pool.workers: + # the executor may spawn a worker for this task: it inherits *_NUM_THREADS=1 from here + with _single_threaded_blas_env(): + future = executor.submit(fn, task) + else: + future = executor.submit(fn, task) + except BrokenProcessPool as exc: + self.pool.close(terminate=True) + raise WorkerPoolError(_LOST_WORKER) from exc + self.pending.add(future) + self.inflight += 1 + future.add_done_callback(lambda f, tag=tag: self.done.put((tag, f))) + + def next(self): + """``(tag, result)`` of a finished task (the oldest in-process one, else the next to complete).""" + if self.local: + return self.local.popleft() + tag, future = self.done.get() + self.pending.discard(future) + self.inflight -= 1 + try: + return tag, future.result() + except BrokenProcessPool as exc: + self.pool.close(terminate=True) + raise WorkerPoolError(_LOST_WORKER) from exc + + @property + def idle(self) -> bool: + return not self.local and self.inflight == 0 + + +def _physical_cpus(cpus: Optional[Iterable[int]] = None, siblings=None) -> int: + """The physical cores among the usable CPUs (``cpus``, default: this process's affinity mask): on Linux, CPUs that + share a core (SMT siblings, ``/sys/devices/system/cpu/cpuN/topology/thread_siblings_list``) count once; where the + topology cannot be read (or off Linux) every CPU counts. ``siblings(cpu) -> str`` replaces the sysfs read + (tests).""" + if cpus is None: + try: + cpus = os.sched_getaffinity(0) + except AttributeError: + return os.cpu_count() or 1 + cpus = sorted(set(int(c) for c in cpus)) + if not cpus: + return 1 + if siblings is None: + def siblings(cpu): + with open(f"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list") as f: + return f.read() + cores = set() + try: + for cpu in cpus: + group = [] + for part in siblings(cpu).strip().split(","): + lo, _, hi = part.partition("-") + group.extend(range(int(lo), int(hi or lo) + 1)) + cores.add(frozenset(group) if cpu in group else frozenset((cpu,))) + except (OSError, ValueError): + return len(cpus) + return max(1, len(cores)) + + +def _as_pool(pool) -> Tuple[CodeAlignmentPool, bool]: + """``(pool, owned)``: a :class:`CodeAlignmentPool` as given, or a new one (closed by the caller) for ``None`` + (in-process) or a worker count.""" + if isinstance(pool, CodeAlignmentPool): + return pool, False + if pool is None: + return CodeAlignmentPool(0), True + if isinstance(pool, int) and not isinstance(pool, bool): + return CodeAlignmentPool(pool), True + raise TypeError(f"pool must be a CodeAlignmentPool, a worker count or None, got {type(pool).__name__}") + + +# ===================================================================================================================== +# sequences and path metrics (numpy only) +# ===================================================================================================================== +def _sanitize_table() -> bytes: + table = bytearray(b"X" * 256) + for b in range(128): + ch = chr(b).upper() + if ch in _BIOTITE_ALPHABET: + table[b] = ord(ch) + return bytes(table) + + +_SANITIZE = _sanitize_table() +_STANDARD_BYTES = STANDARD_AMINO_ACIDS.encode("ascii") + + +def sanitized_bytes(sequence: str) -> bytes: + """``alignment.sanitize_sequence(sequence)`` as ASCII bytes, for the normalised sequences Stage 2 sees (no + surrounding whitespace): every residue upper-cased and anything outside the biotite protein alphabet made ``X``, + one byte per residue (a non-ASCII character becomes ``X``).""" + return sequence.encode("ascii", "replace").translate(_SANITIZE) + + +def nonstandard_count(sequence: str) -> int: + """Residues outside the 20 standard amino acids (S13, S15).""" + return len(sequence.encode("ascii", "replace").translate(None, _STANDARD_BYTES)) + + +def path_columns(path: np.ndarray, path_off: np.ndarray, q_bytes: np.ndarray, t_bytes: np.ndarray, + t_starts: np.ndarray) -> Dict[str, np.ndarray]: + """S12 column tallies of K forward paths at once (vectorised over every column of every path). + + ``path`` int [P, 2] (query i, target j; 0-based, -1 = gap) with path k at ``path[path_off[k]:path_off[k + 1]]``; + ``q_bytes`` the sanitised query as uint8; ``t_bytes`` the sanitised targets concatenated, target k starting at + ``t_starts[k]``. Returns int64 [K] arrays ``aln_len``, ``n_ident``, ``n_aligned`` (pairs), ``mismatch``, + ``gap_open`` (maximal gap runs, each side counted separately), ``q_aligned`` / ``t_aligned`` (columns with a + query / target residue) and the 1-based bounds ``q_start``, ``q_end``, ``t_start``, ``t_end`` (0 for an empty + path). Equal, path by path, to :func:`foldmatch.search.swkern.path_metrics`.""" + path = np.asarray(path, np.int64).reshape(-1, 2) + path_off = np.asarray(path_off, np.int64) + k = len(path_off) - 1 + lens = np.diff(path_off) + seg = np.repeat(np.arange(k), lens) + qi, tj = path[:, 0], path[:, 1] + q_gap, t_gap = qi < 0, tj < 0 + both = ~q_gap & ~t_gap + ident = np.zeros(len(path), bool) + if both.any(): + ident[both] = q_bytes[qi[both]] == t_bytes[np.asarray(t_starts, np.int64)[seg[both]] + tj[both]] + + def count(mask): + return np.bincount(seg, weights=mask, minlength=k).astype(np.int64) + + nonempty = lens > 0 + head = np.zeros(len(path), bool) + head[path_off[:-1][nonempty]] = True + q_open = q_gap & (head | ~np.r_[False, q_gap[:-1]]) + t_open = t_gap & (head | ~np.r_[False, t_gap[:-1]]) + out = dict(aln_len=lens.astype(np.int64), n_ident=count(ident), n_aligned=count(both), + gap_open=count(q_open) + count(t_open), q_aligned=count(~q_gap), t_aligned=count(~t_gap)) + out["mismatch"] = out["n_aligned"] - out["n_ident"] + big = np.int64(1) << 62 + starts = path_off[:-1][nonempty] + for name, idx, gap, aligned in (("q", qi, q_gap, out["q_aligned"]), ("t", tj, t_gap, out["t_aligned"])): + lo, hi = np.zeros(k, np.int64), np.zeros(k, np.int64) + if starts.size: + lo[nonempty] = np.minimum.reduceat(np.where(gap, big, idx), starts) + hi[nonempty] = np.maximum.reduceat(np.where(gap, -1, idx), starts) + has = aligned > 0 + out[f"{name}_start"] = np.where(has, lo + 1, 0) + out[f"{name}_end"] = np.where(has, hi + 1, 0) + return out + + +def _gapped(idx: np.ndarray, seq: np.ndarray) -> str: + out = np.full(len(idx), ord("-"), np.uint8) + res = idx >= 0 + out[res] = seq[idx[res]] + return out.tobytes().decode("ascii") + + +class _PathStrings: + """The CIGAR and gapped-string columns of many paths at once (``path`` int [P, 2] with path k at + ``path[path_off[k]:path_off[k + 1]]``): per path the same strings as :func:`swkern.cigar_from_path` and + :func:`_gapped` of its segment, from one vectorised pass over all the columns. ``q_bytes`` the sanitised query, + ``t_bytes`` the sanitised targets concatenated (target k from ``t_starts[k]``).""" + + def __init__(self, path, path_off, q_bytes, t_bytes, t_starts, *, cigar: bool, qaln: bool, taln: bool): + path = np.asarray(path).reshape(-1, 2) + self.path_off = np.asarray(path_off, np.int64) + self.cigar_tokens = self.run_starts = self.qaln = self.taln = None + if cigar: + ops = swkern.path_ops(path) + boundary = np.ones(len(ops), bool) + if len(ops) > 1: + boundary[1:] = ops[1:] != ops[:-1] + heads = self.path_off[:-1][np.diff(self.path_off) > 0] + boundary[heads] = True # every path starts a run + starts = np.flatnonzero(boundary) + runs = np.diff(np.append(starts, len(ops))) + self.run_starts = starts + self.cigar_tokens = [f"{r}{chr(o)}" for r, o in zip(runs.tolist(), ops[starts].tolist())] + if qaln or taln: + qi, tj = path[:, 0], path[:, 1] + if qaln: + g = np.full(len(path), ord("-"), np.uint8) + res = qi >= 0 + g[res] = q_bytes[qi[res]] + self.qaln = g.tobytes() + if taln: + g = np.full(len(path), ord("-"), np.uint8) + res = tj >= 0 + seg = np.repeat(np.arange(len(self.path_off) - 1), np.diff(self.path_off)) + g[res] = t_bytes[np.asarray(t_starts, np.int64)[seg[res]] + tj[res]] + self.taln = g.tobytes() + + def cigar(self, lo: int, hi: int) -> str: + if hi <= lo: + return "" + a, b = np.searchsorted(self.run_starts, (lo, hi)).tolist() + return "".join(self.cigar_tokens[a:b]) + + +# ===================================================================================================================== +# the aligner +# ===================================================================================================================== +class _Query: + """One query of a batch: its encoding rows, candidates and the results of both rounds.""" + __slots__ = ("qid", "off", "n", "seq", "cand_ids", "cand_emb", "cand_off", "cand_len", "s_f", "s_r", "skipped", + "panel_f", "panel_r", "q90", "s", "s_prime", "lne", "bits", "surv", "parts", "inside", "r1_left", + "r2_left") + + def __init__(self, qid, off, n, seq, cand_ids, cand_emb, inside: bool): + self.qid, self.off, self.n, self.seq = qid, off, n, seq + self.cand_ids, self.cand_emb = cand_ids, cand_emb + self.inside = inside # the query side of S13 + self.q90 = None + self.parts: list = [] + self.r1_left = self.r2_left = 0 + + +def _log_threshold(max_evalue) -> Optional[float]: + """``ln(max_evalue)``, ``-inf`` for 0, or None for no E filter (None or +inf).""" + if max_evalue is None: + return None + try: + value = float(max_evalue) + except (TypeError, ValueError): + raise ValueError(f"max_evalue must be a number or None, got {max_evalue!r}") from None + if math.isnan(value) or value < 0: + raise ValueError(f"max_evalue must be >= 0 (or None / inf for no E-value filter), got {max_evalue!r}") + if value == math.inf: + return None + return -math.inf if value == 0 else math.log(value) + + +def _check_fraction(name: str, value) -> float: + try: + v = float(value) + except (TypeError, ValueError): + raise ValueError(f"{name} must be a number in [0, 1], got {value!r}") from None + if not 0.0 <= v <= 1.0: + raise ValueError(f"{name} must be in [0, 1], got {value!r}") + return v + + +def _positive_int(name: str, value) -> int: + if isinstance(value, bool) or not isinstance(value, (int, np.integer)) or value < 1: + raise ValueError(f"{name} must be an int >= 1, got {value!r}") + return int(value) + + +def align_candidates_esm_codes( + query_index, + prefilter_results: Mapping[str, Tuple[Sequence[str], Sequence[float]]], + code_store, + fetch_subject_sequences, + prepared, + db_residues: int, + pool, + *, + query_sequences: Mapping[str, str], + min_seq_identity: float = 0.0, + min_coverage: float = 0.0, + max_evalue: Optional[float] = 1e-3, + output_needs, + block_floats: int = swkern.DEFAULT_BLOCK_FLOATS, + task_cells: int = DEFAULT_TASK_CELLS, + max_pair_cells: int = swkern.DEFAULT_MAX_PAIR_CELLS, + query_batch: int = DEFAULT_QUERY_BATCH, + panel_policy: str = "auto", + stats: Optional[Dict[str, Any]] = None, +) -> Dict[str, List[Hit]]: + """esm-codes Stage 2 (module docstring): ``{query_id: [Hit, ...]}`` in ``prefilter_results`` order, each list + sorted by S descending, then the embedding score descending, then the target id (S14). + + Args: + query_index: the run's :class:`~foldmatch.search.query_encodings.QueryEncodingIndex` (``entries``, + ``memmap()``, ``memmap_args()``). Every query in ``prefilter_results`` must have an encoding whose length + and ``seq_sha1`` match ``query_sequences``; queries are processed in memmap order. + prefilter_results: Stage-1 output ``{query_id: ([target_ids], [embedding scores])}``. + code_store: the database's :class:`~foldmatch.search.residue_code_store.ResidueCodeStore` (complete; + ``lookup_rows()`` by id, its ``.bin`` mapped by the workers). + fetch_subject_sequences: ``ids -> {id: sequence}`` (``SequenceStore.fetch``), called once per query with + survivors, for its survivors only; every survivor's sequence must be there, with as many residues as its + codes. + prepared: the :class:`~foldmatch.search.scoring_system.PreparedSystem` of the database's system. + db_residues: N, the database's residue count (manifest ``total_residues``); E-values need N > 0. + pool: a :class:`CodeAlignmentPool` (reused, not closed), a worker count (a pool for this call only) or + ``None`` (in-process). Its spawn workers re-import the main module: a script that calls this function + with a pool must guard its entry point with ``if __name__ == "__main__":``. A worker that ends without + returning its task (killed, or unable to start) raises :class:`WorkerPoolError`. + query_sequences: ``{query_id: sequence}`` (normalised here, S15), for identity, the strings and S13. + min_seq_identity / min_coverage: keep a hit only if ``fident >= min_seq_identity`` and + ``min(qcov, tcov) >= min_coverage`` (D8: both 0 under esm-codes). + max_evalue: survivors are ``ln E <= ln(max_evalue)``; ``None`` or ``inf`` traces every candidate. + output_needs: an :class:`OutputNeeds` or the ``--format-output`` field names: the strings are built only for + the requested columns, and the panel policy looks at the E-value columns. + block_floats / max_pair_cells: kernel blocking (``swkern``); a pair above ``max_pair_cells`` cells is skipped + (no score, no hit) and counted. + task_cells / query_batch: task-size cap and queries per batch (scheduling only; the output does not depend + on them). + panel_policy: ``"auto"`` (D11) skips the panel pass when there is no finite ``max_evalue`` and no + evalue / bits / q90 / rawadj column: hits are still ranked by S, and those columns render empty; + ``"always"`` runs it; ``"never"`` skips it and refuses a finite ``max_evalue``. + stats: a dict to receive the run's counters (also logged once at INFO): pairs, survivors, tasks, workers, + the per-query Q90, the fully degenerate queries (S13b) and the calibrated-domain counts (S13); in the + profiling mode (``FOLDMATCH_STAGE2_PROFILE=1``, :mod:`~foldmatch.search.stage2_profile`) also + ``stats["profile"]``, the time breakdown of the parent and the workers (the hits do not change). + """ + t_start = time.perf_counter() + prof = stage2_profile.Stage2Profile() if stage2_profile.profiling_enabled() else None + needs = OutputNeeds.coerce(output_needs) + min_seq_identity = _check_fraction("min_seq_identity", min_seq_identity) + min_coverage = _check_fraction("min_coverage", min_coverage) + ln_max = _log_threshold(max_evalue) + block_floats = _positive_int("block_floats", block_floats) + task_cells = _positive_int("task_cells", task_cells) + max_pair_cells = _positive_int("max_pair_cells", max_pair_cells) + query_batch = _positive_int("query_batch", query_batch) + if isinstance(db_residues, bool) or not isinstance(db_residues, (int, np.integer)) or db_residues < 0: + raise ValueError(f"db_residues must be an int >= 0 (the database's total residues), got {db_residues!r}") + db_residues = int(db_residues) + if panel_policy not in PANEL_POLICIES: + raise ValueError(f"panel_policy must be one of {PANEL_POLICIES}, got {panel_policy!r}") + if panel_policy == "never" and ln_max is not None: + raise ValueError("panel_policy='never' cannot filter by E-value: pass max_evalue=None (or inf)") + need_panel = panel_policy == "always" or (panel_policy == "auto" and (ln_max is not None or needs.significance)) + + system = prepared.system + spec = prepared.for_worker() + kernel = swkern.load() + run = _Run(spec, needs, need_panel, ln_max, min_seq_identity, min_coverage, db_residues, task_cells) + run.prof = prof + results: Dict[str, List[Hit]] = {qid: [] for qid in prefilter_results} + if prof is not None: + t_phase = time.perf_counter() + prof.add("setup", t_phase - t_start) + queries = _collect_queries(query_index, prefilter_results, query_sequences, run) + if prof is not None: + prof.add("collect", time.perf_counter() - t_phase) + if queries and need_panel and db_residues <= 0: + raise ValueError("E-values need a non-empty database (db_residues > 0), but the database has " + f"{db_residues} residues; pass max_evalue=None and panel_policy='never' to rank by S only") + logger.info( + f"esm-codes Stage 2: system {system.name} (digest {system.digest()}, calibration {system.calibration_version}, " + f"bundle {system.bundle_repo}@{system.bundle_revision}), kernel {kernel.info()}, N = {db_residues} database " + f"residues, {len(prefilter_results)} queries ({len(queries)} to align), panel " + f"{'on' if need_panel else 'skipped'} (policy {panel_policy})") + pool, owned = _as_pool(pool) + try: + if queries: + _check_code_store(code_store) + manifest = code_store.manifest() + init = _WorkerInit(isa=kernel.isa, spec=spec, + codes=(str(code_store.bin_path), int(manifest["total_residues"]), + int(manifest["code_bytes"])), + queries=tuple(query_index.memmap_args()), block_floats=block_floats, + max_pair_cells=max_pair_cells, profile=prof is not None) + if need_panel: + _codes, panel_offsets = spec.load_panel() + run.set_panel(panel_offsets) + _Dataflow(queries, run, pool, init, code_store, fetch_subject_sequences, results, query_batch).execute() + except BaseException: + if owned: + pool.close(terminate=True) + raise + if owned: + if prof is not None: + t_phase = time.perf_counter() + pool.close() + if prof is not None: + prof.add("pool_close", time.perf_counter() - t_phase) + if prof is not None: + t_phase = time.perf_counter() + run.finish(results, time.perf_counter() - t_start, max_pair_cells, stats) + if prof is not None: + prof.add("finish", time.perf_counter() - t_phase) + summary = prof.summary(run.workers) + logger.info(stage2_profile.Stage2Profile.log_line(summary)) + if stats is not None: + stats["profile"] = summary + return results + + +def _check_code_store(code_store) -> None: + """The workers map the ``.bin`` with the manifest's shape; :meth:`open_memmap` checks its size first.""" + codes = code_store.open_memmap() + del codes + + +class _Run: + """Counters and settings of one :func:`align_candidates_esm_codes` call.""" + + def __init__(self, spec, needs, need_panel, ln_max, min_id, min_cov, db_residues, task_cells): + self.spec, self.needs, self.need_panel, self.ln_max = spec, needs, need_panel, ln_max + self.min_id, self.min_cov, self.db_residues, self.task_cells = min_id, min_cov, db_residues, task_cells + self.domain = spec.calibrated_domain + self.panel_pairs = None + self.panel_lengths: List[int] = [] + self.panel_residues = 0 + self.queries = self.no_candidates = 0 + self.degenerate: List[str] = [] + self.batches = self.round1_tasks = self.round2_tasks = self.workers = 0 + self.candidate_pairs = self.panel_pair_count = self.skipped_pairs = self.survivors = 0 + self.largest_skipped = (0, 0) # (query length, target length) of the largest skipped pair + self.q90: Dict[str, float] = {} + self.query_domain = dict(aligned=0, short=0, long=0, nonstandard=0) + self.hit_domain = dict(hits=0, ecal0=0, query_outside=0, target_long=0, target_nonstandard=0, + target_low_distinct=0) + self.prof: Optional[stage2_profile.Stage2Profile] = None # profiling mode only + + def query_inside(self, seq: str) -> bool: + """The query side of S13 (length, non-standard fraction), counted for the once-per-run warning.""" + d, qd, n = self.domain, self.query_domain, len(seq) + short, long = n < d.query_min_length, n > d.query_max_length + nonstandard = nonstandard_count(seq) / n > d.max_nonstandard_fraction + qd["aligned"] += 1 + qd["short"] += int(short) + qd["long"] += int(long) + qd["nonstandard"] += int(nonstandard) + return not (short or long or nonstandard) + + def set_panel(self, offsets: np.ndarray): + offsets = np.asarray(offsets, np.int64) + lengths = np.diff(offsets) + self.panel_pairs = np.column_stack((offsets[:-1], lengths)).astype(np.int64) + self.panel_lengths = lengths.tolist() + self.panel_residues = int(lengths.sum()) + + def finish(self, results, seconds: float, max_pair_cells: int, stats: Optional[dict]): + qd, hd = self.query_domain, self.hit_domain + logger.info( + f"esm-codes Stage 2 done in {seconds:.2f}s: {self.candidate_pairs} candidate pairs" + f"{f' and {self.panel_pair_count} panel pairs' if self.need_panel else ''} scored in " + f"{self.round1_tasks} round-1 tasks, {self.survivors} survivors traced back in {self.round2_tasks} " + f"round-2 tasks ({self.batches} batches, {self.workers or 'no'} worker processes); {hd['hits']} hits, " + f"{hd['hits'] - hd['ecal0']} inside the calibrated domain (S13); queries: {qd['aligned']} aligned, " + f"{self.no_candidates} without candidates, {len(self.degenerate)} fully degenerate (S13b, no hits); " + f"{self.skipped_pairs} pairs skipped above max_pair_cells={max_pair_cells}") + if self.skipped_pairs: + n, m = self.largest_skipped + logger.warning( + f"esm-codes Stage 2: {self.skipped_pairs} candidate pair(s) skipped because their similarity matrix " + f"(query length x target length) exceeds max_pair_cells={max_pair_cells} cells (the largest: {n} x " + f"{m}); they get no score and no hit") + outside = qd["short"] + qd["long"] + qd["nonstandard"] + if outside or hd["ecal0"] or self.degenerate: + d = self.domain + logger.warning( + "esm-codes: E-values are calibrated only inside the calibrated domain (S13: queries of " + f"{d.query_min_length}-{d.query_max_length} aa, targets up to {d.target_max_length} aa, at most " + f"{d.max_nonstandard_fraction:.0%} non-standard residues, targets with at least " + f"{d.target_min_distinct_codes} distinct codes); outside it hits are reported with ecal=0. This run: " + f"{qd['short']} queries shorter than {d.query_min_length} aa, {qd['long']} longer than " + f"{d.query_max_length} aa, {qd['nonstandard']} with too many non-standard residues; {hd['ecal0']} of " + f"{hd['hits']} hits with ecal=0 ({hd['query_outside']} for the query, {hd['target_long']} target too " + f"long, {hd['target_nonstandard']} target non-standard, {hd['target_low_distinct']} target with too " + f"few distinct codes); {len(self.degenerate)} fully degenerate queries (one residue, or all encoding " + f"rows identical) report no hits (S13b)" + + (f": {', '.join(self.degenerate[:_EXAMPLES])}" if self.degenerate else "")) + if stats is not None: + stats.update( + queries=self.queries, queries_without_candidates=self.no_candidates, + degenerate_queries=list(self.degenerate), panel=self.need_panel, batches=self.batches, + round1_tasks=self.round1_tasks, round2_tasks=self.round2_tasks, workers=self.workers, + candidate_pairs=self.candidate_pairs, panel_pairs=self.panel_pair_count, + skipped_pairs=self.skipped_pairs, survivors=self.survivors, q90=dict(self.q90), + query_domain=dict(qd), hit_domain=dict(hd), hits=sum(len(h) for h in results.values())) + + +def _is_degenerate(q16: np.ndarray) -> bool: + """S13b: one residue, or every q fp16 row bitwise identical (then S_f = S_r for every target).""" + if len(q16) < 2: + return True + rows = np.ascontiguousarray(q16).view(np.uint16) + return bool(np.all(rows == rows[0])) + + +def _collect_queries(query_index, prefilter_results, query_sequences, run: _Run) -> List[_Query]: + """The queries to align, in memmap order: checked against their encodings; those without candidates and the + fully degenerate ones (S13b) left out (and counted).""" + entries = query_index.entries + memmap = None + out: List[_Query] = [] + for qid, value in prefilter_results.items(): + run.queries += 1 + subject_ids, emb_scores = value + subject_ids, emb_scores = list(subject_ids), [float(s) for s in emb_scores] + if len(subject_ids) != len(emb_scores): + raise ValueError(f"query {qid!r}: {len(subject_ids)} candidate ids but {len(emb_scores)} scores") + entry = entries.get(qid) + if entry is None: + raise ValueError(f"query {qid!r} has no query encoding in {query_index.path} (the ESM3 pass did not " + "encode it)") + seq = query_sequences.get(qid) + if seq is None: + raise ValueError(f"query {qid!r} has no sequence") + try: + seq = normalize_sequence(seq) # S15 (a no-op for the run's normalised sequences) + except ValueError as e: + raise ValueError(f"query {qid!r}: {e} (S15)") from e + if len(seq) != entry.length: + raise ValueError(f"query {qid!r}: its sequence has {len(seq)} residues, its encoding {entry.length} rows") + if seq_sha1(seq) != bytes(entry.seq_sha1): + raise ValueError(f"query {qid!r}: its sequence is not the one its encoding was computed from (seq_sha1 " + "differs)") + if not subject_ids: + run.no_candidates += 1 + continue + if memmap is None: + memmap = query_index.memmap() + q16 = memmap[entry.offset:entry.offset + entry.length] + if not np.all(np.isfinite(q16)): + raise ValueError(f"query {qid!r}: its encoding contains NaN or Inf (S5)") + if _is_degenerate(q16): + run.degenerate.append(qid) + continue + out.append(_Query(qid, int(entry.offset), int(entry.length), seq, subject_ids, emb_scores, + run.query_inside(seq))) + out.sort(key=lambda q: q.off) + return out + + +class _Batch: + """One batch of queries in the dataflow: its round-1 task size, the queries not finished yet, its profile + rounds.""" + __slots__ = ("index", "queries", "target", "open", "r1_round", "r2_round", "r1_left", "r2_left", "sequences") + + def __init__(self, index: int, queries: List[_Query]): + self.index, self.queries = index, queries + self.target = 0 + self.open = len(queries) + self.r1_round = self.r2_round = None + self.r1_left = self.r2_left = 0 + self.sequences: Dict[str, str] = {} # the batch's survivor sequences fetched so far (as one batch fetch) + + +class _Dataflow: + """Stage 2 of one call as a dataflow on a :class:`_Session` instead of two barriers per batch: + + * a batch's round-1 tasks (panel and candidate chunks of every query, longest first) are queued when the batch + is opened (code lookup, task plan); the next batch is opened, at most two at a time, while the workers still + have this one's tasks, so its lookup and plan overlap their work; + * a query whose round-1 tasks are all back gets its statistics (S, Q90, ln E, survivors) at once, its survivors' + sequences, and its round-2 tasks, queued ahead of any round-1 task; + * a query whose round-2 tasks are all back gets its hits at once (``_hits``), and its paths are released: the + parent holds the round-2 payload of the queries in flight only (at most the session's window of tasks), not of + a whole batch, and builds hits while the workers compute; + * a batch that needs more workers than the session has (the pool sized by default: more work than any batch + before; or an in-process session and a batch with more than one worker's share) is opened once the workers are + idle, on a session of the grown pool (at least twice the workers, up to the budget; :meth:`_grow`), so a + small first batch does not keep the whole call in-process or on a few workers. + + Scheduling only: every pair is the same task function on the same worker setup, so the output is the same, bit for + bit, as with any other order, task size or worker count (S16).""" + + def __init__(self, queries: List[_Query], run: _Run, pool: CodeAlignmentPool, init: _WorkerInit, code_store, + fetch_subject_sequences, results: Dict[str, List[Hit]], query_batch: int): + self.run, self.pool, self.init = run, pool, init + self.code_store, self.fetch, self.results = code_store, fetch_subject_sequences, results + self.pending = [queries[b0:b0 + query_batch] for b0 in range(0, len(queries), query_batch)] + self.pending.reverse() # pop() gives the next batch + self.round1: deque = deque() # (fn, task, tag, rnd) to submit, low priority + self.round2: deque = deque() # high priority + self.open_batches: List[_Batch] = [] + self.tid = 0 + self.session: Optional[_Session] = None + + def _cells(self, batch_queries) -> int: + run = self.run + return sum(q.n * ((run.panel_residues if run.need_panel else 0) + int(q.cand_len.sum())) + for q in batch_queries) + + def execute(self): + run, prof = self.run, self.run.prof + first = self._lookup(self.pending.pop()) + cells = self._cells(first) + self._enter_session(cells, self._round1_tasks_estimate(first, cells)) + staged = None # (batch, cells) of a batch waiting for the workers to be idle, to grow the pool + try: + self._open(first, cells) + while True: + session = self.session + if (staged is None and self.pending and len(self.open_batches) < 2 + and len(self.round1) < 4 * session.window): + q = self._lookup(self.pending.pop()) + staged = self._open_or_stage(q, self._cells(q)) + while not session.full and (self.round2 or self.round1): + fn, task, tag, rnd = (self.round2 or self.round1).popleft() + if prof is not None: + prof.round_started(rnd) + session.submit(fn, task, tag, rnd) + if session.idle: + if self.open_batches: # every open query has a task in flight or queued + raise InternalError(f"esm-codes Stage 2: {sum(b.open for b in self.open_batches)} queries " + "left unfinished with no task in flight") + if staged is None: + if not self.pending: + break + q = self._lookup(self.pending.pop()) + staged = self._open_or_stage(q, self._cells(q)) + if staged is not None: + self._grow(*staged) + self._open(*staged) + staged = None + continue + tag, res = session.next() + if prof is not None: + res = prof.received(res, tag[1].r1_round if tag[0] == 1 else tag[1].r2_round) + if tag[0] == 1: + self._round1_result(tag, res) + else: + self._round2_result(tag, res) + except BaseException as exc: + self._exit_session(exc) + raise + self._exit_session(None) + + # ---------------------------------------------------------------------------------------------------- sessions + def _enter_session(self, cells: int, tasks: int) -> None: + session = self.pool.session(self.init, cells, tasks, self.run.prof) + session.__enter__() + self.session = session + + def _exit_session(self, exc: Optional[BaseException]) -> None: + session, self.session = self.session, None + if session is not None: + self.run.workers = max(self.run.workers, self.pool.processes) + session.__exit__(None if exc is None else type(exc), exc, None) + + def _open_or_stage(self, batch: List[_Query], cells: int): + """Open ``batch`` on the current session, or return it as ``(batch, cells)`` when it needs a larger pool + (:meth:`_needs_growth`): it is then opened by :meth:`_grow` once the session's tasks are done.""" + if self._needs_growth(batch, cells): + return batch, cells + self._open(batch, cells) + return None + + def _needs_growth(self, batch: List[_Query], cells: int) -> bool: + """Whether ``batch`` needs more workers than the session has: an in-process session whose pool would start + workers for this batch (as :meth:`CodeAlignmentPool.session` decides at the first batch), or a pool sized by + default that plans more workers for this batch than it has.""" + pool, session = self.pool, self.session + if pool.serial: + return False + want = pool.plan_workers(cells) + if session.in_process: + return min(want, self._round1_tasks_estimate(batch, cells)) > 1 + return want > session.workers + + def _grow(self, batch: List[_Query], cells: int) -> None: + """A new session for ``batch`` once the current one is idle, on a pool grown to at least twice the + session's workers (at most the budget; geometric, so a call restarts its pool only a few times).""" + workers = self.session.workers + self._exit_session(None) + self._enter_session(max(int(cells), 2 * workers * self.pool.min_worker_cells), + max(2, self._round1_tasks_estimate(batch, cells))) + + # ---------------------------------------------------------------------------------------------------- batches + def _lookup(self, batch: List[_Query]) -> List[_Query]: + """The candidates' code rows by id (the .bin order depends on the build's ranks).""" + prof = self.run.prof + if prof is not None: + t0 = time.perf_counter() + offsets, lengths = self.code_store.lookup_rows([tid for q in batch for tid in q.cand_ids]) + start = 0 + for q in batch: + end = start + len(q.cand_ids) + q.cand_off, q.cand_len = offsets[start:end], lengths[start:end] + start = end + if prof is not None: + prof.add("lookup", time.perf_counter() - t0) + return batch + + def _round1_tasks_estimate(self, batch: List[_Query], cells: int) -> int: + """How many round-1 tasks the first batch will have (for the in-process decision of a one-task batch).""" + if len(batch) > 1 or self.run.need_panel: + return 2 + target = target_task_cells(cells, self.pool.plan_workers(cells), self.run.task_cells) + return len(plan_chunks(batch[0].n, batch[0].cand_len.tolist(), target)) + + def _open(self, batch_queries: List[_Query], cells: int): + run, prof, session = self.run, self.run.prof, self.session + if prof is not None: + t0 = time.perf_counter() + b = _Batch(run.batches, batch_queries) + run.batches += 1 + workers = session.workers if session.in_process else min(self.pool.plan_workers(cells), session.workers) + b.target = target = target_task_cells(cells, workers, run.task_cells) + tasks, costs = [], [] + panel_chunks: Dict[int, List[Tuple[int, int]]] = {} + for qi, q in enumerate(batch_queries): + k = len(q.cand_ids) + q.s_f = np.full(k, np.nan, np.float32) + q.s_r = np.full(k, np.nan, np.float32) + q.skipped = np.zeros(k, bool) + q.r1_left = 0 + if run.need_panel: + p = len(run.panel_lengths) + q.panel_f = np.full(p, np.nan, np.float32) + q.panel_r = np.full(p, np.nan, np.float32) + chunks = panel_chunks.get(q.n) + if chunks is None: + chunks = panel_chunks[q.n] = plan_chunks(q.n, run.panel_lengths, target) + for k0, k1 in chunks: + pairs = run.panel_pairs[k0:k1] + tasks.append(((self._next_tid(), _PANEL, q.off, q.n, pairs), (1, b, qi, _PANEL, k0, k1))) + costs.append(q.n * int(pairs[:, 1].sum())) + q.r1_left += 1 + run.panel_pair_count += p + for k0, k1 in plan_chunks(q.n, q.cand_len.tolist(), target): + pairs = np.column_stack((q.cand_off[k0:k1], q.cand_len[k0:k1])) + tasks.append(((self._next_tid(), _CODES, q.off, q.n, pairs), (1, b, qi, _CODES, k0, k1))) + costs.append(q.n * int(pairs[:, 1].sum())) + q.r1_left += 1 + run.candidate_pairs += k + run.round1_tasks += len(tasks) + b.r1_left = len(tasks) + if prof is not None: + prof.add("plan_r1", time.perf_counter() - t0) + b.r1_round = prof.begin_round("round1", min(workers, len(tasks))) + for task, tag in _lpt(tasks, costs): + self.round1.append((_round1_task, task, tag, b.r1_round)) + self.open_batches.append(b) + for qi, q in enumerate(batch_queries): # a query without round-1 tasks (no candidates: never here) + if q.r1_left == 0: + self._statistics_and_round2(b, qi) + + def _next_tid(self) -> int: + self.tid += 1 + return self.tid - 1 + + def _close_query(self, b: _Batch): + b.open -= 1 + if b.open == 0: + self.open_batches.remove(b) + prof = self.run.prof + if prof is not None and b.r2_round is not None: + prof.add("round2", prof.end_round(None, self._round_workers(), b.r2_round)) + + def _round_workers(self) -> int: + return 1 if self.session.in_process else self.session.workers + + # ---------------------------------------------------------------------------------------------------- round 1 + def _round1_result(self, tag, res): + prof = self.run.prof + if prof is not None: + tc = time.perf_counter() + _kind, b, qi, source, k0, k1 = tag + tid, s_f, s_r, skipped = res + q = b.queries[qi] + if source == _PANEL: + q.panel_f[k0:k1], q.panel_r[k0:k1] = s_f, s_r + if np.any(skipped): + raise ValueError(f"query {q.qid!r} ({q.n} residues) is too long for the panel pass: its similarity " + f"with a panel member exceeds max_pair_cells={self.init.max_pair_cells} cells") + else: + q.s_f[k0:k1], q.s_r[k0:k1], q.skipped[k0:k1] = s_f, s_r, skipped + q.r1_left -= 1 + b.r1_left -= 1 + if prof is not None: + prof.add("consume_r1", time.perf_counter() - tc) + if b.r1_left == 0: + prof.add("round1", prof.end_round(None, self._round_workers(), b.r1_round)) + if q.r1_left == 0: + self._statistics_and_round2(b, qi) + + def _statistics_and_round2(self, b: _Batch, qi: int): + run, prof = self.run, self.run.prof + q = b.queries[qi] + if prof is not None: + t0 = time.perf_counter() + _statistics(q, run) + q.panel_f = q.panel_r = None + if prof is not None: + prof.add("statistics", time.perf_counter() - t0) + if not q.surv.size: + self._close_query(b) + return + if prof is not None: + t0 = time.perf_counter() + # the sequences of this query's survivors not fetched for the batch yet (queries of a batch share targets) + known = b.sequences + surv_ids = [q.cand_ids[k] for k in q.surv.tolist()] + wanted = {t for t in surv_ids if t not in known} + if wanted: + fetched = self.fetch(wanted) + missing = sorted(t for t in wanted if t not in fetched) + if missing: + raise ValueError(f"{len(missing)} target(s) have residue codes but no sequence in the sequence store " + f"(first: {', '.join(missing[:_EXAMPLES])}): the code store and the database " + "disagree") + known.update((t, fetched[t]) for t in wanted) + t_seqs = [known[t] for t in surv_ids] + ln_l = q.cand_len[q.surv].tolist() + bad = [t for t, x, m in zip(surv_ids, t_seqs, ln_l) if len(x) != m] + if bad: + raise ValueError(f"{len(bad)} target(s) have residue codes of another length than their stored sequence " + f"(first: {', '.join(bad[:_EXAMPLES])}): the code store and the database disagree") + if prof is not None: + t1 = time.perf_counter() + prof.add("fetch", t1 - t0) + # round-2 tasks: the survivors in chunks of the batch's task size, longest first, ahead of round 1; each + # carries its sequences, so the worker also returns the path metrics and strings the output needs + q.parts = [] + needs = run.needs + want = (needs.cigar, needs.qaln, needs.taln, needs.tseq) + off, ln = q.cand_off[q.surv], q.cand_len[q.surv] + tasks, costs = [], [] + for k0, k1 in plan_chunks(q.n, ln_l, b.target): + pairs = np.column_stack((off[k0:k1], ln[k0:k1])) + tasks.append(((self._next_tid(), q.off, q.n, pairs, q.seq, t_seqs[k0:k1], want), (2, b, qi, k0, k1))) + costs.append(q.n * int(pairs[:, 1].sum())) + q.r2_left = len(tasks) + b.r2_left += len(tasks) + run.round2_tasks += len(tasks) + if prof is not None: + prof.add("plan_r2", time.perf_counter() - t1) + if b.r2_round is None: + b.r2_round = prof.begin_round("round2", self._round_workers()) + for task, tag in _lpt(tasks, costs): + self.round2.append((_round2_task, task, tag, b.r2_round)) + + # ---------------------------------------------------------------------------------------------------- round 2 + def _round2_result(self, tag, res): + run, prof = self.run, self.run.prof + if prof is not None: + tc = time.perf_counter() + _kind, b, qi, k0, k1 = tag + tid, scores, ends, columns, end_ok, nonstd, distinct, skipped, strings = res + q = b.queries[qi] + want = q.s_f[q.surv[k0:k1]] + if np.any(skipped) or not np.array_equal(np.asarray(scores, np.float32).view(np.uint32), + want.view(np.uint32)): + bad = np.flatnonzero(np.asarray(scores, np.float32).view(np.uint32) != want.view(np.uint32)) + raise InternalError( + f"esm-codes Stage 2: the round-2 traceback score of query {q.qid!r} differs from its round-1 S_f " + f"(targets {[q.cand_ids[int(q.surv[k0 + i])] for i in bad[:_EXAMPLES].tolist()]}: " + f"{np.asarray(scores)[bad[:_EXAMPLES]].tolist()} vs {want[bad[:_EXAMPLES]].tolist()}); the reported " + "raw score would not be the one behind the E-value") + part = (k0, np.asarray(ends), np.asarray(columns, np.int64), np.asarray(end_ok, bool), + np.asarray(nonstd, np.int64), np.asarray(distinct, np.int64), strings) + q.parts.append(part) + q.r2_left -= 1 + b.r2_left -= 1 + if prof is not None: + prof.round2_payload(_part_bytes(part), len(part[5])) + prof.add("consume_r2", time.perf_counter() - tc) + if q.r2_left == 0: + if prof is not None: + held = sum(_part_bytes(p) for p in q.parts) + self.results[q.qid] = _hits(q, run) + if prof is not None: + prof.round2_payload(-held, -int(q.surv.size)) + self._close_query(b) + + +def _statistics(q: _Query, run: _Run): + """S9-S11 in the parent: S, Q90, S', ln E, bits; then the survivors.""" + spec = run.spec + valid = ~q.skipped + if q.skipped.any(): + run.skipped_pairs += int(q.skipped.sum()) + m = int(q.cand_len[q.skipped].max()) + if q.n * m > run.largest_skipped[0] * run.largest_skipped[1]: + run.largest_skipped = (q.n, m) + q.s = q.s_f.astype(np.float64) - q.s_r.astype(np.float64) + q.s_prime = q.lne = q.bits = None + if run.need_panel: + q.q90 = cs.q90(q.panel_f.astype(np.float64) - q.panel_r.astype(np.float64), spec.panel_quantile, + spec.quantile_method) + run.q90[q.qid] = q.q90 + q.s_prime = q.s - q.q90 + q.lne = np.full(len(q.s), np.nan) + q.bits = np.full(len(q.s), np.nan) + if valid.any(): + q.lne[valid] = cs.log_evalue(q.s_prime[valid], q.n, run.db_residues, spec) + q.bits[valid] = cs.bit_score(q.s_prime[valid], q.n, spec) + if run.ln_max is None: + surv = valid + else: + surv = valid & (np.nan_to_num(q.lne, nan=np.inf) <= run.ln_max) + q.surv = np.flatnonzero(surv) + run.survivors += int(q.surv.size) + + +def _part_bytes(part) -> int: + """The round-2 payload of one task result the parent holds (arrays and strings).""" + n = sum(int(x.nbytes) for x in part[1:6]) + for lst in part[6]: + if lst is not None: + n += sum(len(x) for x in lst) + return n + + +def _assemble(q: _Query): + """The round-2 parts of a query in survivor order: ends [S, 2], columns [S, len(_COLUMNS)], end_ok [S], + non-standard counts [S], distinct [S], and the four string lists (or None).""" + parts = sorted(q.parts, key=lambda p: p[0]) + s = int(q.surv.size) + if sum(len(p[5]) for p in parts) != s: + raise InternalError(f"esm-codes Stage 2: round 2 returned {sum(len(p[5]) for p in parts)} of {s} survivors " + f"of query {q.qid!r}") + ends = np.concatenate([p[1] for p in parts]).reshape(-1, 2) + columns = np.concatenate([p[2] for p in parts]).reshape(-1, len(_COLUMNS)) + end_ok = np.concatenate([p[3] for p in parts]) + nonstd = np.concatenate([p[4] for p in parts]) + distinct = np.concatenate([p[5] for p in parts]) + strings = [] + for j in range(4): + lists = [p[6][j] for p in parts] + strings.append(None if any(x is None for x in lists) else [x for lst in lists for x in lst]) + q.parts = [] + return ends, columns, end_ok, nonstd, distinct, strings + + +def _hits(q: _Query, run: _Run) -> List[Hit]: + """Identity and coverage, filters, ecal (S13) and the S14 order of one query's survivors, from the column + tallies, string and count lists the round-2 workers returned (:func:`_path_outputs`). The per-hit values come + from the arrays as Python lists (``tolist`` gives the same ints and floats as ``int()`` / ``float()`` of each + element), so the loop only builds the objects.""" + needs, d = run.needs, run.domain + prof = run.prof + if prof is not None: + t0 = time.perf_counter() + ends, columns, end_ok, nonstd, distinct, (cig_l, qaln_l, taln_l, tseq_l) = _assemble(q) + if prof is not None: + t1 = time.perf_counter() + prof.add("hits_assemble", t1 - t0) + surv = q.surv + cols = {c: columns[:, j] for j, c in enumerate(_COLUMNS)} + t_len = q.cand_len[surv] + aln_len = cols["aln_len"] + with np.errstate(divide="ignore", invalid="ignore"): + identity = np.where(aln_len > 0, cols["n_ident"] / np.maximum(aln_len, 1), 0.0) + qcov = cols["q_aligned"] / q.n + tcov = np.where(t_len > 0, cols["t_aligned"] / np.maximum(t_len, 1), 0.0) + kept = (aln_len > 0) & (identity >= run.min_id) & (np.minimum(qcov, tcov) >= run.min_cov) + kidx = np.flatnonzero(kept) + hits: List[Hit] = [] + if kidx.size: + # every kept path (aln_len > 0) must end at the kernel's best cell + off_end = ~end_ok[kidx] + if off_end.any(): + i = int(kidx[np.flatnonzero(off_end)[0]]) + raise InternalError(f"esm-codes Stage 2: the path of query {q.qid!r} against " + f"{q.cand_ids[int(surv[i])]!r} does not end at the kernel's best cell " + f"{ends[i].tolist()}") + ks = surv[kidx] + kidx_l, ks_l, m_l = kidx.tolist(), ks.tolist(), t_len[kidx].tolist() + ident_l, qcov_l, tcov_l = identity[kidx].tolist(), qcov[kidx].tolist(), tcov[kidx].tolist() + aln_l, nid_l, mis_l, gap_l = (cols[c][kidx].tolist() for c in ("aln_len", "n_ident", "mismatch", "gap_open")) + qs_l, qe_l, ts_l, te_l = (cols[c][kidx].tolist() for c in ("q_start", "q_end", "t_start", "t_end")) + sf_l, sr_l, dist_l = q.s_f[ks].tolist(), q.s_r[ks].tolist(), distinct[kidx].tolist() + ns_l = nonstd[kidx].tolist() + if run.need_panel: + lne_l, bits_l, sp_l = q.lne[ks].tolist(), q.bits[ks].tolist(), q.s_prime[ks].tolist() + if prof is not None: + t2 = time.perf_counter() + prof.add("hits_metrics", t2 - t1) + + q_inside, n = q.inside, q.n + hd = run.hit_domain + q_str = sanitized_bytes(q.seq).decode("ascii") if needs.qseq else None + t_max, ns_max, cap = d.target_max_length, d.max_nonstandard_fraction, d.target_min_distinct_codes + cand_ids, cand_emb, q90 = q.cand_ids, q.cand_emb, q.q90 + lne = evalue = bit_score = s_prime = None + with np.errstate(over="ignore", under="ignore"): + for j, i in enumerate(kidx_l if kidx.size else ()): + k, m = ks_l[j], m_l[j] + t_nonstd = ns_l[j] / m if m else 1.0 + t_long, t_nonstandard = m > t_max, t_nonstd > ns_max + t_low = dist_l[j] < cap + ecal = q_inside and not (t_long or t_nonstandard or t_low) + hd["hits"] += 1 + if not ecal: + hd["ecal0"] += 1 + hd["query_outside"] += int(not q_inside) + hd["target_long"] += int(t_long) + hd["target_nonstandard"] += int(t_nonstandard) + hd["target_low_distinct"] += int(t_low) + cigar = cig_l[i] if needs.cigar else None + q_aln = qaln_l[i] if needs.qaln else None + t_aln = taln_l[i] if needs.taln else None + if run.need_panel: + lne = lne_l[j] + evalue = float(np.exp(lne)) + bit_score, s_prime = bits_l[j], sp_l[j] + n_ident = nid_l[j] + metrics = AlignmentMetrics( + identity_aln=ident_l[j], identity_shorter=n_ident / min(n, m), + query_coverage=qcov_l[j], subject_coverage=tcov_l[j], aln_len=aln_l[j], + score=sf_l[j], bit_score=bit_score, pvalue=None, evalue=evalue, query_len=n, subject_len=m, + n_ident=n_ident, mismatch=mis_l[j], gap_open=gap_l[j], + q_start=qs_l[j], q_end=qe_l[j], t_start=ts_l[j], t_end=te_l[j], cigar=cigar, q_aln=q_aln, + t_aln=t_aln, q_seq=q_str, t_seq=tseq_l[i] if needs.tseq else None, + score_rev=sr_l[j], q90=q90 if run.need_panel else None, score_adj=s_prime, log_evalue=lne, + ecal=bool(ecal)) + hits.append(Hit(subject_id=cand_ids[k], emb_score=cand_emb[k], metrics=metrics)) + if prof is not None: + t3 = time.perf_counter() + prof.add("hits_objects", t3 - t2) + # S14: S = fp64(S_f) - fp64(S_r) descending (score and score_rev hold the exact float32 values), then the + # embedding score descending, then the target id ascending + hits.sort(key=lambda h: (-(h.metrics.score - h.metrics.score_rev), -h.emb_score, h.subject_id)) + if prof is not None: + prof.add("hits_sort", time.perf_counter() - t3) + return hits diff --git a/src/foldmatch/search/code_statistics.py b/src/foldmatch/search/code_statistics.py new file mode 100644 index 0000000..e2a1585 --- /dev/null +++ b/src/foldmatch/search/code_statistics.py @@ -0,0 +1,143 @@ +"""E-value statistics of esm-codes systems (method spec S10, S11; plan section 8 P4). numpy only, no torch. + +For a query of length ``m`` against a database of ``N`` residues, with ``S = fp64(S_f) - fp64(S_r)`` and +``S' = S - Q90(query)``:: + + lambda(m), lnK(m) = linear in ln m between the six bin centres; clamped below the first and above the last + lnE = lnK(m) + ln m + ln N - lambda(m) S' (float64, left to right) + bits = (lambda(m) S' - lnK(m)) / ln 2 (so E = m N 2^-bits), printed as floor(bits + 0.5) + +The table and its rules come from the scoring system: every function takes a ``table``, any object with +``bin_centres``, ``lam``, ``lnK``, ``low_m_rule`` and ``high_m_rule`` (an +:class:`~foldmatch.search.scoring_system.EsmCodesSystem`, or the picklable worker view of a prepared one). Only the +``"clamp"`` rule is implemented (S11, D15); a table naming any other rule is refused, never approximated. + +The interpolation is ``numpy.interp(ln m, ln centres, values)``, the research code's function, so the values at the +centres are the table's own numbers bit for bit. +""" +from __future__ import annotations + +import math +import sys +from typing import Tuple + +import numpy as np + +IMPLEMENTED_M_RULES = ("clamp",) +LN2 = math.log(2.0) +LN10 = math.log(10.0) +_LOG_NORMAL_MIN = math.log(sys.float_info.min) # below this exp() is subnormal (or 0) and loses digits +_LOG_MAX = math.log(sys.float_info.max) +#: |bits| above this is refused by render_bits: beyond 2^53 a float64 no longer holds every integer, and the array +#: path's int64 would saturate at 2^63. Real bit scores are a few thousand at most. +MAX_RENDERED_BITS = float(2 ** 53) + + +def _table(table) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + for rule in ("low_m_rule", "high_m_rule"): + value = getattr(table, rule) + if value not in IMPLEMENTED_M_RULES: + raise ValueError(f"{rule}={value!r} is not implemented by this FoldMatch (implemented: " + f"{', '.join(IMPLEMENTED_M_RULES)})") + c = np.asarray(table.bin_centres, np.float64) + lam = np.asarray(table.lam, np.float64) + lnk = np.asarray(table.lnK, np.float64) + if c.ndim != 1 or c.size < 2 or lam.shape != c.shape or lnk.shape != c.shape: + raise ValueError("the E-value table needs matching 1-D bin_centres, lam and lnK of at least 2 entries") + if not (np.all(c > 0) and np.all(np.diff(c) > 0)): + raise ValueError("bin_centres must be positive and strictly increasing") + return np.log(c), lam, lnk + + +def _positive(name: str, value) -> np.ndarray: + a = np.asarray(value, np.float64) + if not np.all(np.isfinite(a)) or not np.all(a > 0): + raise ValueError(f"{name} must be finite and > 0, got {value!r}") + return a + + +def _out(a): + return float(a) if np.ndim(a) == 0 else a + + +def lam_lnk(m, table): + """``(lambda(m), lnK(m))`` for query length(s) ``m`` > 0 (S11): linear in ln m between the centres, the first + centre's values below it and the last centre's above it (both rules ``"clamp"``).""" + lc, lam, lnk = _table(table) + x = np.log(_positive("m (query length)", m)) + return (_out(np.interp(x, lc, lam, left=lam[0], right=lam[-1])), + _out(np.interp(x, lc, lnk, left=lnk[0], right=lnk[-1]))) + + +def log_evalue(s_prime, m, n_residues, table): + """``lnE = lnK(m) + ln m + ln N - lambda(m) S'`` in float64 (S11). ``n_residues`` is N, the database residue + count (manifest ``total_residues``); it must be > 0, since an empty database has no search space.""" + la, lk = lam_lnk(m, table) + m = np.asarray(m, np.float64) + n = _positive("n_residues (database residues)", n_residues) + sp = np.asarray(s_prime, np.float64) + if not np.all(np.isfinite(sp)): + raise ValueError("S' must be finite") + return _out((lk + np.log(m) + np.log(n)) - la * sp) + + +def bit_score(s_prime, m, table): + """``bits = (lambda(m) S' - lnK(m)) / ln 2`` (S11), so that ``E = m N 2^-bits``.""" + la, lk = lam_lnk(m, table) + sp = np.asarray(s_prime, np.float64) + if not np.all(np.isfinite(sp)): + raise ValueError("S' must be finite") + return _out((la * sp - lk) / LN2) + + +def render_bits(bits): + """The printed bit score, ``floor(bits + 0.5)`` (D9): -2.5 -> -2, -0.4 -> 0, -1.7 -> -2 (the BLOSUM path's + ``int(bits + 0.5)`` truncates towards zero and would print -1). An int, or an int64 array. A bit score that is + not finite or exceeds ``MAX_RENDERED_BITS`` (2^53) in magnitude is refused, on both paths alike.""" + a = np.asarray(bits, np.float64) + if not np.all(np.isfinite(a)): + raise ValueError(f"bit score must be finite, got {bits!r}") + if np.any(np.abs(a) > MAX_RENDERED_BITS): + raise ValueError(f"bit score beyond +/-2^53 cannot be rendered exactly, got {bits!r}") + r = np.floor(a + 0.5) + return int(r) if r.ndim == 0 else r.astype(np.int64) + + +def format_evalue_from_log(log_e: float) -> str: + """E rendered as ``%.3E`` from ln E, without under- or overflow: ``-1000 -> '5.076E-435'``. + + Inside the normal float range the text is exactly ``f"{math.exp(log_e):.3E}"``, the format of the BLOSUM62 path. + Outside it the mantissa and exponent come from ``log10 E = ln E / ln 10``. ``-inf`` gives ``0.000E+00``.""" + x = float(log_e) + if math.isnan(x): + raise ValueError("ln E is NaN") + if x == -math.inf: + return "0.000E+00" + if x == math.inf: + return "INF" + if _LOG_NORMAL_MIN <= x <= _LOG_MAX: + return f"{math.exp(x):.3E}" + return _format_from_log10(x / LN10) + + +def _format_from_log10(t: float) -> str: + """``%.3E`` text of ``10 ** t`` for any finite ``t`` (no float of that size needed).""" + exponent = math.floor(t) + mantissa = f"{10.0 ** (t - exponent):.3f}" + if mantissa == "10.000": # 9.9995... rounds up into the next decade + mantissa, exponent = "1.000", exponent + 1 + return f"{mantissa}E{'-' if exponent < 0 else '+'}{abs(exponent):02d}" + + +def q90(panel_scores, quantile: float = 0.9, method: str = "linear") -> float: + """S10: ``numpy.quantile(float64 S(query, panel members), quantile)`` with linear interpolation.""" + if method != "linear": + raise ValueError(f"quantile method {method!r} is not implemented (implemented: linear)") + if not 0.0 < float(quantile) < 1.0: + raise ValueError(f"quantile must be in (0, 1), got {quantile!r}") + s = np.asarray(panel_scores, np.float64) + if s.ndim != 1 or s.size == 0: + raise ValueError("panel scores must be a non-empty 1-D array") + if not np.all(np.isfinite(s)): + raise ValueError("panel scores must be finite (a skipped or failed panel pair has no score)") + return float(np.quantile(s, float(quantile), method="linear")) diff --git a/src/foldmatch/search/db_path.py b/src/foldmatch/search/db_path.py new file mode 100644 index 0000000..8f7bb34 --- /dev/null +++ b/src/foldmatch/search/db_path.py @@ -0,0 +1,21 @@ +"""Where a database's files live, from the ``--db-path`` / ``--output-db`` string. + +Dependency-free (no FAISS), so the CLI can check a database's files on every +rank before ESM3 without loading the vector library. +""" +from pathlib import Path + + +def parse_db_path(db_path: str) -> tuple[Path, str, str]: + """Split a database path into (directory, index name, resolved path). + + The files are ``{resolved path}{suffix}``: ``.index``, ``.metadata``, + ``.invlists`` and ``.sequences``. A bare name lives in the current + directory, and an empty name is ``embeddings``. + """ + db_path = Path(db_path) + db_dir = db_path.parent + index_name = db_path.name or "embeddings" + if db_dir == Path('.'): + db_dir = Path.cwd() + return db_dir, index_name, str(db_dir / index_name) diff --git a/src/foldmatch/search/embedding_computer.py b/src/foldmatch/search/embedding_computer.py index 9c84142..14c08d9 100644 --- a/src/foldmatch/search/embedding_computer.py +++ b/src/foldmatch/search/embedding_computer.py @@ -1,8 +1,6 @@ import logging -import os from pathlib import Path -from secrets import token_hex -from typing import Iterator, Optional +from typing import Dict, Iterator, List, Mapping, Optional, TypeVar import numpy as np import pyarrow.parquet as pq @@ -16,24 +14,56 @@ Accelerator, Granularity, SrcEsmFrom, Devices, Strategy, ) +from foldmatch.search.query_encodings import QueryEncodingIndex +# run_shard_files and shard_run_ids live with the shard schema (torch-free); they are imported from here too. +from foldmatch.search.residue_shards import ( + _SHARD_PREFIX, ResidueCodeBatch, check_residue_output, iter_residue_codes, run_shard_files, shard_run_ids, +) +from foldmatch.utils.dist import RUN_ID_ENV, exported_run_id, is_run_id, run_id +from foldmatch.utils.fasta import read_id_at, read_record_at +from foldmatch.utils.sequence import FastaPreflight, normalize_sequence, preflight_fasta logger = logging.getLogger(__name__) +_V = TypeVar("_V") + class EmbeddingComputer: """Compute chain or assembly embeddings via a residue->chain inference pipeline. - Owns the per-run scratch directories and the multi-GPU bookkeeping: each - rank's __init__ creates its own per-rank scratch dir, then after residue - inference all ranks consolidate onto rank 0's directory so chain/assembly - inference and tensor loading see a single shared location. + Owns the per-run shard naming and the multi-GPU bookkeeping. Every rank + writes its embeddings to ``{embedding_folder}/emb_{run_id}-{rank}.parquet`` + under one run id shared by all ranks (:mod:`foldmatch.utils.dist`), and + rank 0 reads back exactly that run's shards. Before a pass every rank + refuses to start if shards with this run id already exist, and after a + FASTA pass rank 0 checks that the shards hold one embedding per input + entry, so a lost rank or a shard of an earlier run with the same id fails + loudly instead of being overwritten or merged. """ def __init__( self, embedding_folder: Path, + *, + run: Optional[str] = None, ): + """``run`` binds the computer to the shards of that run id (``build sequences-index`` reads a finished run's + shards): :func:`~foldmatch.utils.dist.run_id` is then never called, and a malformed id raises + ``ValueError``. Without it, this run's id is drawn as every pass draws it.""" self.embedding_folder = embedding_folder - self.out_name = f"emb_{_consolidate_id()}" + if run is None: + # Raises on a rank > 0 that was given no id to share, before any model loads. + self.run_id = run_id() + elif is_run_id(run): + self.run_id = run + else: + raise ValueError( + f"{run!r} is not a run id: letters, digits, '.', '_' and '-' only, starting with a letter or digit." + ) + self.out_name = f"{_SHARD_PREFIX}{self.run_id}" + # Number of embeddings rank 0 must find in the shards; known for FASTA input. + self.expected_count: Optional[int] = None + # The pre-flight of the FASTA pass (its kept records, in input order); None for structure input. + self.preflight: Optional[FastaPreflight] = None def compute_from_structures( self, @@ -56,6 +86,7 @@ def compute_from_structures( structure_folder = Path(structure_folder) if not structure_folder.exists(): raise ValueError(f"Structure directory does not exist: {structure_folder}") + self.refuse_existing_shards() logging.info(f"Listing structure files from: {structure_folder}") structure_files = tuple(structure_folder.glob(f"*{file_extension}")) @@ -64,46 +95,48 @@ def compute_from_structures( f"No structure files found with extension {file_extension} in {structure_folder}" ) - if granularity == 'chain': - chain_predict( - src_stream=[ - (str_file.stem, str_file, str_file.stem) - for str_file in structure_files - ], - src_location=SrcLocation.stream, - structure_format=structure_format, - min_res_n=min_res, - out_folder=self.embedding_folder, - out_name=self.out_name, - accelerator=accelerator, - batch_size=batch_size, - num_workers=num_workers, - num_nodes=num_nodes, - devices=devices, - strategy=strategy, - out_format=OutFormat.parquet, - return_predictions=False, - ) - else: - assembly_predict( - src_stream=[ - (str_file.stem, str_file, str_file.stem) - for str_file in structure_files - ], - src_location=SrcLocation.stream, - structure_format=structure_format, - min_res_n=min_res, - out_folder=self.embedding_folder, - out_name=self.out_name, - accelerator=accelerator, - batch_size=batch_size, - num_workers=num_workers, - num_nodes=num_nodes, - devices=devices, - strategy=strategy, - out_format=OutFormat.parquet, - return_predictions=False, - ) + # The other ranks Lightning launches inherit the exported run id. + with exported_run_id(self.run_id): + if granularity == 'chain': + chain_predict( + src_stream=[ + (str_file.stem, str_file, str_file.stem) + for str_file in structure_files + ], + src_location=SrcLocation.stream, + structure_format=structure_format, + min_res_n=min_res, + out_folder=self.embedding_folder, + out_name=self.out_name, + accelerator=accelerator, + batch_size=batch_size, + num_workers=num_workers, + num_nodes=num_nodes, + devices=devices, + strategy=strategy, + out_format=OutFormat.parquet, + return_predictions=False, + ) + else: + assembly_predict( + src_stream=[ + (str_file.stem, str_file, str_file.stem) + for str_file in structure_files + ], + src_location=SrcLocation.stream, + structure_format=structure_format, + min_res_n=min_res, + out_folder=self.embedding_folder, + out_name=self.out_name, + accelerator=accelerator, + batch_size=batch_size, + num_workers=num_workers, + num_nodes=num_nodes, + devices=devices, + strategy=strategy, + out_format=OutFormat.parquet, + return_predictions=False, + ) def compute_from_fasta( self, @@ -115,30 +148,145 @@ def compute_from_fasta( devices: Devices = 'auto', strategy: Strategy = Strategy.auto, accelerator: Accelerator = Accelerator.auto, + preflight: Optional[FastaPreflight] = None, + residue_output: Optional[str] = None, + scoring_system=None, ): - """Compute chain embeddings from protein sequences in a FASTA file.""" - - chain_predict( - src_stream=fasta_file, - src_from=SrcEsmFrom.fasta, - min_res_n=min_res_n, - batch_size=batch_size, - num_workers=num_workers, - num_nodes=num_nodes, - accelerator=accelerator, - devices=devices, - out_format=OutFormat.parquet, - out_folder=self.embedding_folder, - out_name=self.out_name, - strategy=strategy, - return_predictions=False, - ) + """Compute chain embeddings from protein sequences in a FASTA file. + + ``preflight`` is the :func:`preflight_fasta` result for this file and + ``min_res_n``, as the CLI computes it on every rank before the pass; + its offsets feed the dataset, so the file is scanned once. Without it + the pre-flight runs here (problems are only logged, or refused with a + residue output). - def get_embedding_batches(self, load_batch_size: int = 32768) -> Iterator[tuple[list[str], np.ndarray]]: - """Stream (ids, [B, D] float32) batches from all per-rank Parquet shards. Rank-0 only.""" + ``residue_output`` (``"residue_codes"`` for a build, ``"query_encoding"`` + for a query) and ``scoring_system`` go to the pass unchanged (see + :func:`foldmatch.inference.chain_complete_inference.predict`); the + shards then also hold the per-residue payloads, read back with + :meth:`iter_residue_codes` or :meth:`build_query_encodings`. Without + them the pass is called exactly as before. + """ - parquet_files = sorted(Path(self.embedding_folder).glob(f"{self.out_name}-*.parquet")) + residue_output = check_residue_output(residue_output) + preflight = self.attach_preflight(fasta_file, min_res_n, preflight, strict=residue_output is not None) + self.refuse_existing_shards() + # Only a residue pass gets the residue keywords: a plain pass is called exactly as before. + residue = {} if residue_output is None else { + "residue_output": residue_output, "scoring_system": scoring_system, + } + # The other ranks Lightning launches inherit the exported run id. + with exported_run_id(self.run_id): + chain_predict( + # EsmProtFromFasta takes the pre-flight for the path and reuses its offsets. + src_stream=preflight, + src_from=SrcEsmFrom.fasta, + min_res_n=min_res_n, + batch_size=batch_size, + num_workers=num_workers, + num_nodes=num_nodes, + accelerator=accelerator, + devices=devices, + out_format=OutFormat.parquet, + out_folder=self.embedding_folder, + out_name=self.out_name, + strategy=strategy, + return_predictions=False, + **residue, + ) + # A rank's shard is complete only once that rank has left the pass: wait for every rank, so rank 0 may read + # the shards (get_embedding_batches, iter_residue_codes, build_query_encodings) as soon as this returns. + _wait_for_every_rank() + + def attach_preflight(self, fasta_file: Path, min_res_n: int, preflight: Optional[FastaPreflight] = None, *, + strict: bool = False) -> FastaPreflight: + """Bind the FASTA pre-flight of this run's input to the computer and return it. + + ``preflight`` must be :func:`preflight_fasta`'s result for ``fasta_file`` and ``min_res_n`` (a mismatch + raises ``ValueError``); without it the pre-flight runs here, its problems refused (``strict``) or logged. + Sets :attr:`expected_count` (one embedding per kept entry, which rank 0 checks the shards against) and + :attr:`preflight` (the input order), as a FASTA pass does before it starts. + """ + if preflight is None: + preflight = preflight_fasta(fasta_file, min_res_n) + preflight.check(strict=strict) + elif Path(preflight.path).resolve() != Path(fasta_file).resolve() or preflight.min_res_n != min_res_n: + raise ValueError( + f"Pre-flight of {preflight.path} (min_res_n={preflight.min_res_n}) does not " + f"match {fasta_file} (min_res_n={min_res_n})" + ) + # One embedding per kept entry; rank 0 checks the shards against it after the pass. + self.expected_count = preflight.entries_kept + self.preflight = preflight + return preflight + + def shard_files(self) -> list[Path]: + """This run's per-rank Parquet shards, ``emb_{run_id}-{rank}.parquet`` (:func:`run_shard_files`).""" + return run_shard_files(self.embedding_folder, self.run_id) + + def refuse_existing_shards(self) -> None: + """Raise ``FileExistsError`` if shards with this run id already exist. Read-only. + + Every rank runs it before the pass, so before ESM3 loads and before any + rank writes (the ranks only write after they have all joined the process + group). Shards with this id can only come from an earlier run with the + same id, e.g. the same command run twice in one srun step, or a reused + ``FOLDMATCH_RUN_ID``: this run would overwrite them, or rank 0 would + mix them into its own. + """ + existing = self.shard_files() + if existing: + raise FileExistsError( + f"{len(existing)} embedding shard(s) of an earlier run with the same run id " + f"({self.run_id}) are in {self.embedding_folder} ({existing[0].name}, ...). " + f"This run would overwrite them or mix them into its own: remove them, use " + f"another --tmp-embedding-folder, or set a different {RUN_ID_ENV}." + ) + + def count_embeddings(self) -> int: + """Rows over this run's shards, from the Parquet footers (no data read).""" + return sum(pq.ParquetFile(path).metadata.num_rows for path in self.shard_files()) + + def check_embedding_count(self) -> None: + """Raise unless the shards hold exactly :attr:`expected_count` embeddings. Rank-0 only. + + A shortfall means a rank's shard is missing; an excess means shards of + an earlier run with the same id (e.g. the same SLURM step) are mixed in. + A no-op when the expected count is unknown (structure input). + """ + if self.expected_count is None: + return + found = self.count_embeddings() + if found != self.expected_count: + raise ValueError( + f"Found {found} embeddings in {len(self.shard_files())} shard(s) " + f"{self.embedding_folder}/{self.out_name}-*.parquet, but the input has " + f"{self.expected_count} entries. Either a rank's shard is missing, or " + f"shards of an earlier run with the same run id ({self.run_id}) are mixed " + f"in: remove them or use another --tmp-embedding-folder." + ) + + def remove_shards(self) -> None: + """Delete this run's shards. Rank-0 only, once every rank has finished writing.""" + for path in self.shard_files(): + path.unlink(missing_ok=True) + + def get_embedding_batches(self, load_batch_size: int = 32768, *, + rows: bool = False) -> Iterator[tuple[list, np.ndarray]]: + """Stream (ids, [B, D] float32) batches from all per-rank Parquet shards. Rank-0 only. + + Checks the embedding count first (:meth:`check_embedding_count`), so a + mismatch fails before anything is built from the batches. + + With ``rows`` each batch is keyed by the rows' numbers in this read order + (0, 1, ...) instead of their ids, so that a repeated id is still two rows: + what :meth:`in_input_order` takes. + """ + + self.check_embedding_count() + parquet_files = self.shard_files() logging.info(f"Streaming embeddings from {len(parquet_files)} Parquet shard(s) in {self.embedding_folder}/{self.out_name}-*.parquet") + row = 0 for parquet_file in parquet_files: logging.info(f" Parquet file {parquet_file}") pf = pq.ParquetFile(parquet_file) @@ -148,7 +296,137 @@ def get_embedding_batches(self, load_batch_size: int = 32768) -> Iterator[tuple[ arr = np.ascontiguousarray( flat.reshape(len(record), -1).astype(np.float32, copy=False) ) - yield ids, arr + if rows: + yield list(range(row, row + len(ids))), arr + row += len(ids) + else: + yield ids, arr + + def input_ids(self) -> List[str]: + """The ids of the kept records of the FASTA pass, in input order (duplicates included): the order of a + 1-rank run's shard. Read from the FASTA at the pre-flight's offsets, as the pass's dataset reads them.""" + preflight = self._fasta_preflight() + with open(preflight.path, "r") as fh: + return [read_id_at(fh, offset) for offset in preflight.offsets.tolist()] + + def input_sequences(self) -> Dict[str, str]: + """``{id: sequence}`` of the kept records of the FASTA pass, in input order: each sequence normalised exactly + as the pass's dataset gives it to ESM3, and a repeated id's the one of its last kept record, the record whose + embedding a search keeps (:meth:`in_input_order`). Records the pass did not embed (below ``min_res_n``) are + not in it, so Stage 2 aligns the sequence behind each query's Stage-1 hits.""" + preflight = self._fasta_preflight() + out: Dict[str, str] = {} + with open(preflight.path, "r") as fh: + for offset in preflight.offsets.tolist(): + name, sequence = read_record_at(fh, offset) + out[name] = normalize_sequence(sequence, allow_non_ascii=True) + return out + + def _fasta_preflight(self) -> FastaPreflight: + if self.preflight is None: + raise ValueError("the input order is known only after a FASTA pass (compute_from_fasta)") + return self.preflight + + def _row_positions(self, names: List[str]) -> np.ndarray: + """The input position (index in ``names``, :meth:`input_ids`) of every shard row, in the order + :meth:`get_embedding_batches` reads the rows. + + A rank's shard holds its entries in the order the distributed sampler dealt them, round robin: row ``i`` of + rank ``r``'s shard (``emb_{run id}-{r}``) is entry ``r + i * world_size``, and a 1-rank shard is in input + order. Checked against the ids: when every row's id is the input's at that position, the positions are + exact, a repeated id included. Otherwise (shards not dealt that way: never by Lightning's predict sampler) + an id is placed by its own position, which is exact for an id that occurs once; a repeated id's rows then + take its positions in read order, with a warning. An id that is not in the input raises ``ValueError`` + (shards of another run mixed in).""" + shards = self.shard_files() + prefix = f"{self.out_name}-" + ranks = [int(path.name[len(prefix):-len(".parquet")]) for path in shards] + row_ids = [pq.ParquetFile(path).read(columns=["id"]).column("id").to_pylist() for path in shards] + first: Dict[str, int] = {} + for k, name in enumerate(names): + first.setdefault(name, k) + unknown = [qid for ids in row_ids for qid in ids if qid not in first] + if unknown: + raise ValueError( + f"{len(unknown)} query id(s) from the embedding shards of run {self.run_id} are not in " + f"{self.preflight.path} (e.g. {', '.join(map(repr, unknown[:3]))}): shards of another run are " + f"mixed in. Remove them or use another --tmp-embedding-folder.") + total, world = len(names), len(shards) + if sorted(ranks) == list(range(world)): + dealt = [np.arange(rank, total, world, dtype=np.int64) for rank in ranks] + if all(len(pos) == len(ids) and all(names[p] == qid for p, qid in zip(pos.tolist(), ids)) + for pos, ids in zip(dealt, row_ids)): + return np.concatenate(dealt) if dealt else np.zeros(0, np.int64) + repeated = len(first) < total + if repeated: + logger.warning( + f"The embedding shards of run {self.run_id} are not in the order the distributed sampler deals the " + f"entries; a query id that occurs more than once in {self.preflight.path} may be reported with the " + f"embedding of another of its records.") + later: Dict[str, List[int]] = {} + for k, name in enumerate(names): + later.setdefault(name, []).append(k) + for positions in later.values(): + positions.reverse() # pop() gives an id's positions in input order + out = [] + for ids in row_ids: + for qid in ids: + positions = later[qid] + out.append(positions.pop() if positions else first[qid]) + return np.asarray(out, dtype=np.int64) + + def in_input_order(self, results: Mapping[int, _V]) -> Dict[str, _V]: + """``results`` of a search keyed by shard row (as built from :meth:`get_embedding_batches` with ``rows``), + keyed by query id in the input FASTA's order, with the values a 1-rank run gives. Rank-0 only, after a + FASTA pass. + + Rank 0 reads the shards rank by rank, and the distributed sampler deals the entries round-robin, so in a + multi-rank run the rows come in rank order (rank 0's first). Put back in input order + (:meth:`_row_positions`), a multi-rank run writes the queries of a 1-rank run, whose shard is already in + input order: each id at its first kept record, and a repeated id (duplicate ids, which only an embedding + search and BLOSUM62 accept) with the value of its last kept record, as a 1-rank run's search (a dict, the + later record replacing the earlier) gives it. Values are never changed. A row that is not in ``results`` + (a filtered subset) is left out. + """ + names = self.input_ids() + positions = self._row_positions(names) + by_position: Dict[int, _V] = {} + for row, value in results.items(): + if not 0 <= row < len(positions): + raise ValueError(f"row {row!r} is not a row of the embedding shards of run {self.run_id} " + f"({len(positions)} rows)") + by_position[int(positions[row])] = value + last: Dict[str, int] = {} + for position in sorted(by_position): + last[names[position]] = position # dict order: each id at its first position + return {name: by_position[position] for name, position in last.items()} + + def iter_residue_codes(self, system, load_batch_size: int = 32768) -> Iterator[ResidueCodeBatch]: + """Stream this run's residue codes, one shard at a time in shard order. Rank-0 only. + + Yields :class:`~foldmatch.search.residue_shards.ResidueCodeBatch` ``(ids, codes, seq_sha1s)`` (one uint8 + ``[L, 8]`` array per entry; what ``ResidueCodeStore.create`` takes), ``load_batch_size`` entries at a time, + reading only the ``id``, ``residue_codes`` and ``seq_sha1`` columns. + Before anything is yielded: the embedding count (:meth:`check_embedding_count`) and every shard's schema + (:func:`~foldmatch.search.residue_shards.check_shards`): shards without residue codes, mixed shards and + shards of another encoding than ``system``'s are refused (``ResidueShardError``). + """ + self.check_embedding_count() + return iter_residue_codes(self.shard_files(), system, load_batch_size) + + def build_query_encodings(self, dst, system) -> QueryEncodingIndex: + """This run's query encodings as a per-run fp16 memmap at ``dst`` (:meth:`QueryEncodingIndex.build`, one + pass in shard order; the same shard checks, plus one encoding per input entry). Rank-0 only.""" + self.check_embedding_count() + return QueryEncodingIndex.build(self.shard_files(), dst, system=system, expected_count=self.expected_count) + +def _wait_for_every_rank() -> None: + """``torch.distributed.barrier()`` when the pass ran in a process group (every rank calls the pass, so every rank + reaches it); nothing otherwise.""" + import torch.distributed as dist + if dist.is_available() and dist.is_initialized(): + dist.barrier() + def compute_from_structure_file( query_structure: Path, @@ -229,6 +507,3 @@ def compute_from_structure_file( return arr = np.ascontiguousarray(np.stack(embeddings).astype(np.float32, copy=False)) yield ids, arr - -def _consolidate_id(): - return os.environ.get('SLURM_JOB_ID', token_hex(16)) \ No newline at end of file diff --git a/src/foldmatch/search/embedding_database.py b/src/foldmatch/search/embedding_database.py index 4a266bc..bf65ac2 100644 --- a/src/foldmatch/search/embedding_database.py +++ b/src/foldmatch/search/embedding_database.py @@ -5,6 +5,7 @@ import numpy as np import time +from foldmatch.search.db_path import parse_db_path as _parse_db_path from foldmatch.search.faiss_database import FaissEmbeddingDatabase logger = logging.getLogger(__name__) @@ -164,14 +165,3 @@ def search_by_database( def get_db_statistics(self) -> Dict: """Get database statistics.""" return self.db.get_statistics() - - -def _parse_db_path(output_db: str) -> tuple[Path, str, str]: - """Split a database path into (directory, index name, resolved path).""" - output_db_path = Path(output_db) - db_dir = output_db_path.parent - index_name = output_db_path.name or "embeddings" - if db_dir == Path('.'): - db_dir = Path.cwd() - return db_dir, index_name, str(db_dir / index_name) - diff --git a/src/foldmatch/search/faiss_database.py b/src/foldmatch/search/faiss_database.py index 85486c2..3159329 100644 --- a/src/foldmatch/search/faiss_database.py +++ b/src/foldmatch/search/faiss_database.py @@ -40,6 +40,23 @@ def _index_add_batched(index, embedding_array: np.ndarray, batch_size: int = 100 index.add(np.ascontiguousarray(embedding_array[start:end])) +def _quantize_outside_parallel_regions(index) -> None: + """Make ``index.search`` of an IVF index run its coarse quantizer outside OpenMP parallel regions (FAISS parallel_mode 3). + + FAISS 1.14.2 to 1.15.1 lose half or more of the coarse centroids otherwise, with no error. In parallel_mode 0 (the + default) ``IndexIVF::search`` calls the quantizer inside a parallel loop over the queries. With T threads, nlist >= + max(10000, 1024 * T), and fewer than T queries in a thread's slice, the flat quantizer splits its centroids into T + segments, one per thread, but the nested region has only one thread: only the first nlist // T centroids are + searched (facebookresearch/faiss#5824). parallel_mode 3 calls the quantizer once for the whole batch, outside the + region, and then splits the same per-query scan over the threads. It is not stored in the index file. + + The defect is likely solved in the next FAISS release: the upstream fix (facebookresearch/faiss#5824) was merged on + 2026-10-31, but it is not in 1.15.1, the latest release and FoldMatch's pin. Re-examine this workaround when the + faiss-cpu pin moves to a release that has it. + """ + faiss.extract_index_ivf(index).parallel_mode = 3 + + class FaissEmbeddingDatabase: """FAISS-based database for structure chain embeddings.""" @@ -228,6 +245,7 @@ def _create_ivf_pq_ondisk( if config.nprobe is None: config.nprobe = max(1, config.nlist // 64) ivf_index.nprobe = config.nprobe + _quantize_outside_parallel_regions(self.index) # Phase E: persist the populated index. This is the canonical FAISS # save order — write_index serializes both the trained header AND the @@ -282,6 +300,7 @@ def load_database(self, use_gpu: bool = False): if self.index_config is not None and self.index_config.nprobe is not None: ivf_index = faiss.extract_index_ivf(self.index) ivf_index.nprobe = self.index_config.nprobe + _quantize_outside_parallel_regions(self.index) else: self.index = faiss.read_index(str(index_file)) @@ -292,6 +311,24 @@ def load_database(self, use_gpu: bool = False): if use_gpu: self.move_to_gpu() + def saved_entry_count(self) -> int: + """Number of embeddings in the database on disk, from its metadata file; the index itself is not read. + + Raises ``ValueError`` if the database files do not exist, as :meth:`load_database` does, or if the + metadata file is not readable FAISS metadata (truncated, or another file), naming it. + """ + index_file = self.db_folder / f"{self.index_name}.index" + metadata_file = self.db_folder / f"{self.index_name}.metadata" + if not index_file.exists() or not metadata_file.exists(): + raise ValueError(f"Database files not found in: {self.db_folder}") + try: + with open(metadata_file, 'rb') as f: + return len(pickle.load(f)['embedding_ids']) + except Exception as e: # unpickling a damaged file can raise almost anything + raise ValueError( + f"{metadata_file} is not readable FAISS database metadata ({type(e).__name__}: {e})" + ) from e + def set_nprobe(self, nprobe: int) -> None: """Override the search-time ``nprobe`` for IVF-family indexes. diff --git a/src/foldmatch/search/karlin_altschul.py b/src/foldmatch/search/karlin_altschul.py index 28f1b02..c4256ba 100644 --- a/src/foldmatch/search/karlin_altschul.py +++ b/src/foldmatch/search/karlin_altschul.py @@ -48,8 +48,8 @@ * **Gap convention.** MMseqs2's Smith-Waterman charges a length-L gap as ``open + (L-1)*extend`` (a length-1 gap costs 11 at nominal 11/1), which is - exactly biotite's convention — so biotite's native ``(-11, -1)`` feeds these - constants directly with no adjustment. ALP itself *documents* its penalty as + exactly the convention of biotite and of the Stage-2 kernel that reproduces + it — so their nominal 11/1 feeds these constants directly with no adjustment. ALP itself *documents* its penalty as ``open + L*extend`` and MMseqs2 passes 11/1 through unconverted, so the hardcoded lambda/K are calibrated to a gap model one extension unit away from what the aligner scores. Reproducing MMseqs2 means inheriting that offset. diff --git a/src/foldmatch/search/output.py b/src/foldmatch/search/output.py index d83cd68..817bde2 100644 --- a/src/foldmatch/search/output.py +++ b/src/foldmatch/search/output.py @@ -28,18 +28,35 @@ * ``fident`` is nident/alnlen (identical columns over the whole alignment length, gaps included) — the same fraction MMseqs2 reports; ``pident`` is that fraction as a percentage (100*fident), per the field's name. - * ``bits`` is emitted as an integer via MMseqs2's ``int(bitScore + 0.5)``. + * ``bits`` is emitted as an integer. BLOSUM62 keeps MMseqs2's + ``int(bitScore + 0.5)``, so its output is unchanged; it truncates towards + zero, so a negative bit score (``--significance-mode sampled`` with low + gap costs gives some) prints one higher: -7.6 prints -7. esm-codes bit + scores are rounded half up, ``floor(bitScore + 0.5)`` (method S11): -7.6 + prints -8. + * ``evalue`` is ``%.3E``; under esm-codes it is rendered from ln E, so it + never underflows (``5.076E-435``). * ``gapopen`` counts maximal gap RUNS (query- and target-side independently), not gap characters. * CIGAR ops follow MMseqs2: ``M`` aligned pair, ``D`` gap in query (deletion), ``I`` gap in target (insertion). + +esm-codes columns (``query sequences --scoring esm-codes``, method S9-S13): +``raw`` is ``S_f`` as ``%.3f`` (an integer under BLOSUM62, as before); the +optional ``rawrev`` (``S_r``), ``q90`` (the query's panel quantile) and +``rawadj`` (``S' = S_f - S_r - Q90``) are ``%.3f``, and ``ecal`` is ``1`` for +a hit inside the calibrated domain (S13), else ``0``. These four render empty +under BLOSUM62 (and ``query db``); ``evalue``, ``bits``, ``q90`` and +``rawadj`` render empty when the panel pass was skipped (D11). --------------------------------------------------------------------------- # """ import logging +import numbers from dataclasses import dataclass from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple +from foldmatch.search.code_statistics import format_evalue_from_log, render_bits from foldmatch.types.api_types import FormatMode if TYPE_CHECKING: # avoids a runtime import cycle; alignment imports this module @@ -158,28 +175,37 @@ def _rows(): # rather than a query/target pair, so it has its own column set — but the same # tab-separated, no-header file convention. CLUSTER_OUTPUT_FIELDS: Tuple[str, ...] = ("chain_id", "cluster_id", "cluster_size") +# Columns of ``fm-search cluster --method greedy``: the representative's chain +# id follows (equal to ``chain_id`` on the representative's own row). +GREEDY_CLUSTER_OUTPUT_FIELDS: Tuple[str, ...] = CLUSTER_OUTPUT_FIELDS + ("representative",) def write_cluster_results( - assignments: Iterable[Tuple[str, int, int]], + assignments: Iterable[Tuple], output_file: str, format_mode: FormatMode = FormatMode.headless_tsv, + fields: Tuple[str, ...] = CLUSTER_OUTPUT_FIELDS, ) -> int: - """Write cluster assignments in :data:`CLUSTER_OUTPUT_FIELDS` order. - - ``assignments`` is an iterable of ``(chain_id, cluster_id, cluster_size)``; - the caller owns which rows to include (e.g. a ``min_cluster_size`` filter). - Emitting the columns here keeps their order next to the declared field list, - which the CLI help text is generated from. Returns the number of data rows. + """Write cluster assignments in ``fields`` order (:data:`CLUSTER_OUTPUT_FIELDS` + or :data:`GREEDY_CLUSTER_OUTPUT_FIELDS`). + + ``assignments`` is an iterable of rows of one value per field, e.g. + ``(chain_id, cluster_id, cluster_size)``; the caller owns which rows to + include (e.g. a ``min_cluster_size`` filter). Emitting the columns here keeps + their order next to the declared field list, which the CLI help text is + generated from. Returns the number of data rows. """ delimiter = format_spec(format_mode).delimiter - return _emit( - CLUSTER_OUTPUT_FIELDS, - (delimiter.join((str(chain_id), str(cluster_id), str(cluster_size))) - for chain_id, cluster_id, cluster_size in assignments), - output_file, - format_mode, - ) + n_fields = len(fields) + + def _rows(): + for row in assignments: + if len(row) != n_fields: + raise ValueError(f"cluster row {row!r} has {len(row)} values for the {n_fields} fields " + f"{', '.join(fields)}") + yield delimiter.join(map(str, row)) + + return _emit(fields, _rows(), output_file, format_mode) # --------------------------------------------------------------------------- # @@ -204,7 +230,7 @@ def write_cluster_results( "tend": "1-indexed alignment end position in target sequence", "tlen": "Target sequence length", "alnlen": "Alignment length (number of aligned columns)", - "raw": "Raw alignment score", + "raw": "Raw alignment score (blosum62: the integer Smith-Waterman score; esm-codes: S_f)", "bits": "Bit score", "cigar": "Alignment CIGAR string (M match, D gap in query, I gap in target)", "qseq": "Query sequence", @@ -214,10 +240,17 @@ def write_cluster_results( "mismatch": "Number of mismatches", "qcov": "Fraction of query sequence covered by alignment", "tcov": "Fraction of target sequence covered by alignment", + "rawrev": "esm-codes: score of the row-reversed query, S_r (empty under blosum62)", + "q90": "esm-codes: the query's 0.9 quantile of S over the score panel, Q90 (empty under blosum62)", + "rawadj": "esm-codes: adjusted score S_f - S_r - Q90, the score behind evalue and bits (empty under blosum62)", + "ecal": "esm-codes: 1 if the hit is inside the calibrated domain of the E-values, else 0 (empty under blosum62)", } SUPPORTED_OUTPUT_FIELDS: Tuple[str, ...] = tuple(OUTPUT_FIELD_DESCRIPTIONS) +# Fields only esm-codes Stage 2 computes; they render empty under BLOSUM62. +ESM_CODES_OUTPUT_FIELDS: Tuple[str, ...] = ("rawrev", "q90", "rawadj", "ecal") + # MMseqs2's own default column set, plus the Stage-1 embedding score right # after the identifiers (this tool's prefilter has no MMseqs2 equivalent, and # keeping it in the default output makes the two stages' files line up). @@ -262,14 +295,44 @@ def needs_significance(fields: List[str]) -> bool: return any(f in SIGNIFICANCE_FIELDS for f in fields) +def _format_3f(value: Optional[float]) -> str: + """``%.3f``, empty for None; a value that rounds to zero prints ``0.000``, never ``-0.000``.""" + if value is None: + return "" + text = f"{value:.3f}" + return "0.000" if text == "-0.000" else text + + +def _render_raw(m) -> str: + """The integer BLOSUM62 score as before; the float esm-codes S_f as ``%.3f``.""" + return str(m.score) if isinstance(m.score, numbers.Integral) else _format_3f(m.score) + + +def _render_bits(m) -> str: + """BLOSUM62 (an integer score): MMseqs2's ``int(bits + 0.5)``, unchanged; esm-codes: ``floor(bits + 0.5)``.""" + if m.bit_score is None: + return "" + if isinstance(m.score, numbers.Integral): + return str(int(m.bit_score + 0.5)) + return str(render_bits(m.bit_score)) + + +def _render_evalue(m) -> str: + """From ln E when the aligner gives it (esm-codes: no underflow), else the E-value as ``%.3E``.""" + if m.log_evalue is not None: + return format_evalue_from_log(m.log_evalue) + return f"{m.evalue:.3E}" if m.evalue is not None else "" + + # Per-field renderers: ``(query_id, hit) -> str``. Significance-derived fields # render to "" when significance was not computed; heavy strings render to "" -# when they were not requested (and so were never materialized). +# when they were not requested (and so were never materialized); the esm-codes +# fields render to "" under BLOSUM62. _FIELD_RENDERERS: Dict[str, Callable[[str, "Hit"], str]] = { "query": lambda qid, h: qid, "target": lambda qid, h: h.subject_id, "embscore": lambda qid, h: format_embedding_score(h.emb_score), - "evalue": lambda qid, h: f"{h.metrics.evalue:.3E}" if h.metrics.evalue is not None else "", + "evalue": lambda qid, h: _render_evalue(h.metrics), "gapopen": lambda qid, h: str(h.metrics.gap_open), "pident": lambda qid, h: f"{100.0 * h.metrics.identity_aln:.3f}", "fident": lambda qid, h: f"{h.metrics.identity_aln:.3f}", @@ -281,8 +344,8 @@ def needs_significance(fields: List[str]) -> bool: "tend": lambda qid, h: str(h.metrics.t_end), "tlen": lambda qid, h: str(h.metrics.subject_len), "alnlen": lambda qid, h: str(h.metrics.aln_len), - "raw": lambda qid, h: str(h.metrics.score), - "bits": lambda qid, h: str(int(h.metrics.bit_score + 0.5)) if h.metrics.bit_score is not None else "", + "raw": lambda qid, h: _render_raw(h.metrics), + "bits": lambda qid, h: _render_bits(h.metrics), "cigar": lambda qid, h: h.metrics.cigar or "", "qseq": lambda qid, h: h.metrics.q_seq or "", "tseq": lambda qid, h: h.metrics.t_seq or "", @@ -291,6 +354,10 @@ def needs_significance(fields: List[str]) -> bool: "mismatch": lambda qid, h: str(h.metrics.mismatch), "qcov": lambda qid, h: f"{h.metrics.query_coverage:.3f}", "tcov": lambda qid, h: f"{h.metrics.subject_coverage:.3f}", + "rawrev": lambda qid, h: _format_3f(h.metrics.score_rev), + "q90": lambda qid, h: _format_3f(h.metrics.q90), + "rawadj": lambda qid, h: _format_3f(h.metrics.score_adj), + "ecal": lambda qid, h: "" if h.metrics.ecal is None else ("1" if h.metrics.ecal else "0"), } diff --git a/src/foldmatch/search/query_encodings.py b/src/foldmatch/search/query_encodings.py new file mode 100644 index 0000000..268daf3 --- /dev/null +++ b/src/foldmatch/search/query_encodings.py @@ -0,0 +1,138 @@ +"""Per-run query encodings for Stage 2 (plan P5.4): one fp16 memmap plus an id -> (offset, length, seq_sha1) map. + +The query pass writes each query's S5 encoding (``float16[L, 64]``) into its Parquet shards +(:mod:`foldmatch.search.residue_shards`). :meth:`QueryEncodingIndex.build` reads them once, in shard order, and +appends every encoding to one raw little-endian fp16 file, ``[total rows, 64]``, that Stage-2 workers map read-only +(:func:`open_memmap`) instead of receiving arrays through pickles. The map gives each query's first row, its row +count and the SHA-1 of its normalised sequence (S15). + +No torch: workers import this module. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Dict, Iterable, NamedTuple, Optional + +import numpy as np + +from foldmatch.search.residue_shards import ( + QUERY_ENCODING, QUERY_ENCODING_DTYPE, ResidueShardError, check_shards, iter_payload_batches, payload_row_bytes, + residue_width, +) + +_EXAMPLES = 10 + + +class QueryEncodingEntry(NamedTuple): + offset: int # first row in the memmap + length: int # rows (= residues) + seq_sha1: bytes # 20 bytes + + +def open_memmap(path, rows: int, width: int) -> np.ndarray: + """The encodings file as a read-only ``float16 [rows, width]`` array (an empty array for 0 rows, which + ``np.memmap`` cannot map).""" + if rows == 0: + return np.empty((0, width), dtype=QUERY_ENCODING_DTYPE) + return np.memmap(path, dtype=QUERY_ENCODING_DTYPE, mode="r", shape=(rows, width)) + + +class QueryEncodingIndex: + """The query encodings of one run: ``path`` (raw fp16), ``rows``, ``width`` and ``entries``.""" + + def __init__(self, path, rows: int, width: int, entries: Dict[str, QueryEncodingEntry], system_name: str = ""): + self.path = Path(path) + self.rows = int(rows) + self.width = int(width) + self.entries = entries + self.system_name = system_name + self._memmap = None + + @classmethod + def build(cls, shards: Iterable, dst, *, system, expected_count: Optional[int] = None, + load_batch_size: int = 8192) -> "QueryEncodingIndex": + """One sequential pass over ``shards``, in the given order, writing ``dst`` (which must not exist). + + Refused (:class:`ResidueShardError`), with ``dst`` removed again: shards that are not query encodings of + ``system`` or are mixed (:func:`check_shards`, footers only, before ``dst`` is created); a duplicate id; a + query with an empty payload (its ESM3 rows did not line up with its sequence) or one whose size is not a + whole number of rows; and, with ``expected_count``, a different number of queries. + """ + shards = [Path(p) for p in shards] + check_shards(shards, QUERY_ENCODING, system) + row_bytes = payload_row_bytes(QUERY_ENCODING, system) + width = residue_width(QUERY_ENCODING, system) + dst = Path(dst) + out = open(dst, "xb") # never another run's file: FileExistsError before anything is written + try: + with out: + entries, rows = _append_encodings(out, shards, row_bytes, load_batch_size, expected_count) + if os.path.getsize(dst) != rows * row_bytes: + raise ResidueShardError(f"{dst}: wrote {os.path.getsize(dst)} bytes, expected {rows * row_bytes}") + except BaseException: + dst.unlink(missing_ok=True) + raise + return cls(dst, rows, width, entries, system.name) + + def __len__(self) -> int: + return len(self.entries) + + def __contains__(self, name) -> bool: + return name in self.entries + + def memmap(self) -> np.ndarray: + """The encodings, read-only ``float16 [rows, width]``.""" + if self._memmap is None: + self._memmap = open_memmap(self.path, self.rows, self.width) + return self._memmap + + def get(self, name: str) -> np.ndarray: + """Query ``name``'s encoding, a ``float16 [length, width]`` view of the memmap (``KeyError`` if unknown).""" + entry = self.entries[name] + return self.memmap()[entry.offset:entry.offset + entry.length] + + def memmap_args(self) -> tuple: + """``(path, rows, width)`` for :func:`open_memmap` in a worker (picklable).""" + return str(self.path), self.rows, self.width + + def remove(self) -> None: + """Delete the encodings file (after the run).""" + self._memmap = None + self.path.unlink(missing_ok=True) + + +def _append_encodings(out, shards, row_bytes: int, load_batch_size: int, expected_count: Optional[int]): + entries: Dict[str, QueryEncodingEntry] = {} + rows = 0 + empty, ragged, duplicates = [], [], [] + for shard in shards: + for batch in iter_payload_batches(shard, QUERY_ENCODING, load_batch_size): + sizes = np.diff(batch.offsets) + for k, name in enumerate(batch.ids): + size = int(sizes[k]) + if size == 0: + empty.append(name) + elif size % row_bytes: + ragged.append(name) + if name in entries: + duplicates.append(name) + continue + entries[name] = QueryEncodingEntry(rows + int(batch.offsets[k]) // row_bytes, size // row_bytes, + batch.seq_sha1s[k]) + out.write(memoryview(batch.data)) # a refused batch is written too; build() removes the file + rows += batch.data.size // row_bytes + problems = [] + if duplicates: + problems.append(f"{len(duplicates)} duplicate query id(s) (e.g. {duplicates[:_EXAMPLES]})") + if empty: + problems.append(f"{len(empty)} query(ies) without an encoding: their ESM3 rows did not match their sequence " + f"length (e.g. {empty[:_EXAMPLES]})") + if ragged: + problems.append(f"encoding(s) that are not a whole number of {row_bytes}-byte rows (e.g. {ragged[:_EXAMPLES]})") + if expected_count is not None and not problems and len(entries) != expected_count: + problems.append(f"{len(entries)} query encodings in {len(shards)} shard(s), but the input has " + f"{expected_count} queries") + if problems: + raise ResidueShardError("query encodings refused: " + "; ".join(problems)) + return entries, rows diff --git a/src/foldmatch/search/residue_code_store.py b/src/foldmatch/search/residue_code_store.py new file mode 100644 index 0000000..1b7ef79 --- /dev/null +++ b/src/foldmatch/search/residue_code_store.py @@ -0,0 +1,1000 @@ +"""Residue-code store: the per-residue ESM3 codes of a sequence database (plan section 8 P5.5, Appendix B; D10). + +Three files sit next to a FAISS database ``P`` = ``{db_folder}/{index_name}`` (the path split of +:func:`foldmatch.search.db_path.parse_db_path`, as for ``P.sequences``): + +* ``P.rescodes.bin``: raw ``uint8[N_res, code_bytes]`` (8 bytes per residue for esm-codes-v1, method spec S4), the + entries' codes back to back in the order :meth:`ResidueCodeStore.create` received them. Read it with + :meth:`ResidueCodeStore.open_memmap`. +* ``P.rescodes.idx``: SQLite, ``entries(id TEXT PRIMARY KEY, offset INTEGER, length INTEGER, seq_sha1 BLOB)`` + (``WITHOUT ROWID``). ``offset`` and ``length`` count residues, i.e. rows of the ``.bin``, so the codes of entry + ``id`` are ``codes[offset:offset + length]``. ``seq_sha1`` is :func:`foldmatch.utils.sequence.seq_sha1` of the + sequence the codes were computed from (20 bytes). A ``meta`` table holds the build's totals and digests; create() + writes it last, so an idx without it is the remains of an interrupted build. +* ``P.rescodes.json``: the manifest (Appendix B). It is the commit point. :meth:`ResidueCodeStore.validate` writes it + last, only after every check passed, through a temporary file, fsync and ``os.replace``. Without it the store is + partial (:meth:`ResidueCodeStore.is_partial`) and nothing reads it. + +Lifecycle (plan section 3, BUILD; rank 0, after the ESM3 pass and the FAISS build):: + + store.create(batches, system) # P.rescodes.{bin,idx}.tmp, fsynced: the pending build + store.validate(sequence_store, faiss_ids_count, preflight=...) # checks (a)-(e), then the swap, manifest last + +:meth:`ResidueCodeStore.create` writes the new codes aside, as ``P.rescodes.bin.tmp`` and ``P.rescodes.idx.tmp``, so a +complete store it replaces (``force``) stays whole and usable until the new codes have passed validation. Only then +does :meth:`ResidueCodeStore.validate` swap them in: the new manifest is written to ``P.rescodes.json.tmp`` and +fsynced, the old manifest is removed, the two files are renamed into place, and the new manifest is renamed last. A +failed validation, or a failed or interrupted create(), discards the pending files only: the store it was to replace +(if any), the sequence store and FAISS are untouched, so the database still serves BLOSUM62, and its codes too when it +had them. A query calls :meth:`ResidueCodeStore.check_compatible` on every rank before ESM3 (read-only and cheap), then +:meth:`ResidueCodeStore.check_faiss_count` on rank 0 once the index is loaded. + +Digests (recorded in the manifest): + +* ``bin_sha256``: sha256 of the ``.bin`` bytes, computed while they are written. +* ``entries_digest``: sha256 over ``ENTRIES_DIGEST_TAG`` followed, for every entry in ascending id order (the order of + the ids' UTF-8 bytes, which is SQLite's ``BINARY`` order), by the uint32 little-endian byte length of the UTF-8 id, + the id, and its 20-byte ``seq_sha1`` (:func:`entries_digest`). It does not depend on the order of the entries. + +No torch: Stage-2 workers open the memmap. Only the ``_INSTALLED`` defaults that name this installation's ESM3 pin +(``create(esm_revision=...)``, ``check_compatible(esm_weights_sha256=...)``) import ``foldmatch.utils.esm.loaders``. +""" +from __future__ import annotations + +import dataclasses +import hashlib +import json +import logging +import os +import platform +import sqlite3 +import struct +from contextlib import contextmanager +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, Mapping, NamedTuple, Optional, Sequence, Tuple + +import numpy as np + +from foldmatch.search.db_path import parse_db_path +from foldmatch.search.scoring_system import ENCODING_FIELDS, EsmCodesSystem +from foldmatch.search.scoring_system import get as get_system +from foldmatch.search.sequence_store import _INSERT_BATCH, _SQLITE_VAR_CHUNK, SequenceStore +from foldmatch.utils.sequence import STANDARD_AMINO_ACIDS + +logger = logging.getLogger(__name__) + +RESCODES_BIN_SUFFIX = ".rescodes.bin" +RESCODES_IDX_SUFFIX = ".rescodes.idx" +RESCODES_MANIFEST_SUFFIX = ".rescodes.json" +RESCODES_SUFFIXES = (RESCODES_BIN_SUFFIX, RESCODES_IDX_SUFFIX, RESCODES_MANIFEST_SUFFIX) +_TMP_SUFFIX = ".tmp" +_SQLITE_SIDECARS = ("-journal", "-wal", "-shm") + +MANIFEST_VERSION = 1 +MANIFEST_KIND = "foldmatch-residue-codes" +IDX_FORMAT = "foldmatch-residue-codes-idx/1" +ENTRIES_DIGEST_TAG = b"foldmatch-residue-codes-entries/1\n" +SEQ_SHA1_BYTES = 20 + +#: Column names of a pyarrow batch given to :meth:`ResidueCodeStore.create` (the build shards' columns, P5.3). +ID_COLUMN = "id" +CODES_COLUMN = "residue_codes" +SEQ_SHA1_COLUMN = "seq_sha1" + +# Ids quoted per failed check (validate) or per refused lookup. +_EXAMPLE_IDS = 10 + +_ESM_KEYS = ("repo", "revision", "weights_sha256") +_BUNDLE_KEYS = ("repo", "revision", "files") +_CROSS_CHECK_KEYS = ("preflight_entries", "preflight_residues", "sequence_store_entries", "sequence_store_residues", + "faiss_entries", "seq_sha1_mismatches", "length_mismatches") +_BUILD_KEYS = ("foldmatch_version", "created_utc", "run_id", "from_shards", "accelerator", "devices", "esm_precision", + "encoder_precision", "torch", "esm", "platform") +#: Every key of a version-1 manifest (Appendix B). check_compatible() refuses a manifest that lacks any of them, +#: at the top level or inside the blocks of ``_NESTED_KEYS``. +MANIFEST_KEYS = ("manifest_version", "kind", "system_name", "system_digest", "encoding_digest", "system", + "code_bytes", "bitorder", "esm", "bundle", "entries", "total_residues", "min_res_n", "bin_sha256", + "entries_digest", "nonstandard_entries", "input_normalisation", "cross_checks", "build") +_NESTED_KEYS = { + "system": tuple(f.name for f in dataclasses.fields(EsmCodesSystem)), + "esm": _ESM_KEYS, + "bundle": _BUNDLE_KEYS, + "cross_checks": _CROSS_CHECK_KEYS, + "build": _BUILD_KEYS, +} +# What create() records in the idx meta table, for validate() to put in the manifest. Written in one transaction +# after the last entry, so either all are present or the idx is incomplete. +_PENDING_KEYS = ("store_format", "system_json", "esm_revision", "min_res_n", "build_json", "entries", + "total_residues", "bin_sha256", "entries_digest") + +_STANDARD_BYTES = STANDARD_AMINO_ACIDS.encode("ascii") +_REBUILD_HINT = ("Rebuild the codes from the kept build shards with " + "'fm-search build residue-codes DB_FOLDER INDEX_NAME --from-shards DIR', " + "or rebuild the database with 'fm-search build sequences --residue-codes'") +# For entries the ESM3 pass could not encode (empty payloads): the same shards would fail again. +_FRESH_BUILD_HINT = ("The empty payloads were written by the ESM3 pass itself, so rebuilding from the same shards " + "would fail the same way: rebuild the database with 'fm-search build sequences " + "--residue-codes' (and check those entries' sequences)") + + +class ResidueCodeStoreError(ValueError): + """The residue-code store refused: malformed input to create(), a duplicate or unknown id, a partial store. + + A ``ValueError``, so the command-line tools report it as a one-line error.""" + + +class CodeStoreValidationError(ResidueCodeStoreError): + """:meth:`ResidueCodeStore.validate` found faults (and removed the code store). + + ``failures`` maps each failed check, ``"a"`` to ``"e"``, to ``(count, examples)``: for (a)-(d) the number of + entries and the first ids (at most 10, ascending); for (e) the number of disagreeing totals and their + descriptions.""" + + def __init__(self, message: str, failures: Mapping[str, Tuple[int, Tuple[str, ...]]]): + super().__init__(message) + self.failures = dict(failures) + + +class IncompatibleCodeStoreError(ResidueCodeStoreError): + """The residue-code store cannot be used with this system, sequence store or installation (check_compatible).""" + + +class _Installed: + """Type of :data:`_INSTALLED`.""" + __slots__ = () + + def __repr__(self) -> str: + return "_INSTALLED" + + +#: Default of every check_compatible() argument: compare with what this installation uses. An omitted argument is +#: therefore never a skipped check, and ``None`` is refused (TypeError) instead of meaning "don't check". +_INSTALLED = _Installed() + + +class CodeEntry(NamedTuple): + """One entry of the index: its codes are ``codes[offset:offset + length]`` of :meth:`open_memmap`.""" + offset: int + length: int + seq_sha1: bytes + + +def entries_digest(pairs: Iterable[Tuple[str, bytes]]) -> str: + """``entries_digest`` (module docstring) of ``(id, seq_sha1)`` pairs given in any order.""" + return _entries_digest_sorted(sorted(pairs, key=lambda pair: pair[0].encode("utf-8"))) + + +def _entries_digest_sorted(rows: Iterable[Tuple[str, bytes]]) -> str: + h = hashlib.sha256(ENTRIES_DIGEST_TAG) + for entry_id, sha1 in rows: + data = entry_id.encode("utf-8") + h.update(struct.pack(" "ResidueCodeStore": + """The store of the database named by a ``--db-path`` / ``--output-db`` string.""" + db_folder, index_name, _ = parse_db_path(db_path) + return cls(db_folder, index_name) + + def sequence_store(self) -> SequenceStore: + """The database's sequence store (same folder and index name).""" + return SequenceStore(self.db_folder, self.index_name) + + # ----------------------------------------------------------------------------------------------------------------- + # state + # ----------------------------------------------------------------------------------------------------------------- + def _pending_paths(self) -> Tuple[Path, ...]: + """The files of a pending build (create() not yet validated, or validate() interrupted).""" + return (self.tmp_manifest_path, self.tmp_bin_path, self.tmp_idx_path, + *(Path(f"{self.tmp_idx_path}{s}") for s in _SQLITE_SIDECARS)) + + def _other_paths(self) -> Tuple[Path, ...]: + """Every file of the store except the manifest.""" + return (*self._pending_paths(), self.bin_path, self.idx_path, + *(Path(f"{self.idx_path}{s}") for s in _SQLITE_SIDECARS)) + + def exists(self) -> bool: + """A complete store: its manifest, written last, is there.""" + return self.manifest_path.exists() + + def is_partial(self) -> bool: + """Store files without a manifest: a build that stopped (or has not reached validate()) before its end.""" + return not self.exists() and any(path.exists() for path in self._other_paths()) + + def has_pending_build(self) -> bool: + """Files of a build that create() wrote and validate() has not swapped in (next to a complete store when it + is a replacement). Readers never see them.""" + return any(path.exists() for path in self._pending_paths()) + + def remove(self) -> List[Path]: + """Delete every file of the store, the manifest first (so no manifest ever names missing data), and any + pending build. Returns the files removed.""" + return self._unlink((self.manifest_path, *self._other_paths())) + + def _discard_pending(self) -> List[Path]: + """Delete the pending build only; a complete store stays as it is.""" + return self._unlink(self._pending_paths()) + + def _unlink(self, paths) -> List[Path]: + removed = [] + for path in paths: + try: + os.unlink(path) + except FileNotFoundError: + continue + removed.append(path) + if removed: + _fsync_dir(self.db_folder) + return removed + + # ----------------------------------------------------------------------------------------------------------------- + # build + # ----------------------------------------------------------------------------------------------------------------- + def create(self, batches, system: EsmCodesSystem, force: bool = False, *, min_res_n: int = 0, + build: Optional[Mapping[str, Any]] = None, esm_revision=_INSTALLED) -> Dict[str, Any]: + """Write the pending ``.bin`` and ``.idx`` from ``batches`` (streamed); :meth:`validate` then checks them, + swaps them in and writes the manifest. + + ``batches`` is an iterable of batches, each either ``(ids, codes, seq_sha1s)`` (equal-length sequences: + ``str`` ids; codes as ``bytes`` or ``uint8 [L, code_bytes]`` arrays, an empty payload allowed; 20-byte + ``seq_sha1``\\ s) or a pyarrow ``RecordBatch``/``Table`` with the columns ``id``, ``residue_codes`` and + ``seq_sha1``. Entries are stored in the order given. An empty payload (the build channel's mark for an entry + it could not encode, P5.1) is stored as such; validate() check (c) refuses it. + + The codes are written aside (``P.rescodes.{bin,idx}.tmp``). A complete store (manifest present) is refused + unless ``force``; with ``force`` it stays whole and readable until :meth:`validate` has accepted the new codes + and swaps them in, so a failed rebuild leaves it as it was. A partial store is cleared, and so are the pending + files of an earlier create() that was never validated. The ``.bin`` sha256 is computed while writing; index + rows go in in transactions of 50,000; a duplicate id raises (pre-flight refuses duplicates, so it is a bug); + both files are fsynced; then ``entries_digest`` is computed and the totals recorded. Any failure removes what + was written (the pending files only). ``build`` adds to (or overrides) the manifest's ``build`` block + (``run_id``, ``from_shards``, ``accelerator``, ``devices``, ``esm_precision``, ``encoder_precision``, ...); + ``esm_revision`` is the ESM3 repo revision recorded in the manifest (by default this installation's pin, when + it is the system's ESM3). ``system`` is used as given (the build passes the registry entry). + + Returns ``{"entries", "total_residues", "bin_sha256", "entries_digest"}``. + """ + if not isinstance(system, EsmCodesSystem): + raise TypeError(f"create needs an EsmCodesSystem, got {type(system).__name__}") + if isinstance(min_res_n, bool) or not isinstance(min_res_n, int) or min_res_n < 0: + raise TypeError(f"min_res_n must be an int >= 0, got {min_res_n!r}") + build_json = json.dumps(_build_info(build), sort_keys=True, allow_nan=False) + esm_revision = _installed_esm_revision(system) if esm_revision is _INSTALLED else esm_revision + if esm_revision is not None and not isinstance(esm_revision, str): + raise TypeError(f"esm_revision must be a str or None, got {esm_revision!r}") + + if self.exists(): + if not force: + raise FileExistsError( + f"Residue-code store already exists at {self.manifest_path}. Pass force (--force) to replace " + f"it, or choose a different database path.") + logger.info(f"Building a replacement for the residue-code store {self.base}.rescodes.*: the current " + f"store stays in place until the new codes have passed validation") + elif self.is_partial(): + logger.warning(f"Clearing a partial residue-code store at {self.base} (no manifest: an earlier build " + f"stopped before its end)") + self.remove() # the manifest-less remains, pending files included + if self._discard_pending(): + logger.warning(f"Cleared the unvalidated residue codes an earlier build left at {self.tmp_bin_path}") + self.db_folder.mkdir(parents=True, exist_ok=True) + + code_bytes = system.code_bytes + bin_hash = hashlib.sha256() + entries = total = 0 + conn = None + try: + with open(self.tmp_bin_path, "xb") as out: + conn = sqlite3.connect(str(self.tmp_idx_path)) + # A derived artefact (rebuildable from the shards) that nothing reads before its manifest exists: + # durability comes from the fsync before the manifest, not from the journal. + conn.execute("PRAGMA journal_mode = MEMORY") + conn.execute("PRAGMA synchronous = OFF") + conn.execute("CREATE TABLE entries (id TEXT PRIMARY KEY NOT NULL, offset INTEGER NOT NULL, " + "length INTEGER NOT NULL, seq_sha1 BLOB NOT NULL) WITHOUT ROWID") + conn.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value)") + conn.commit() + rows: list = [] + for batch in batches: + ids, payloads, sha1s = _columns(batch) + chunks = [] + for entry_id, payload, sha1 in zip(ids, payloads, sha1s): + if not isinstance(entry_id, str): + raise ResidueCodeStoreError(f"entry ids must be str, got {entry_id!r}") + data = _payload_bytes(payload, code_bytes, entry_id) + length = len(data) // code_bytes + rows.append((entry_id, total, length, _seq_sha1_bytes(sha1, entry_id))) + chunks.append(data) + total += length + if len(rows) >= _INSERT_BATCH: + _insert(conn, rows) + entries += len(rows) + rows = [] + data = b"".join(chunks) + out.write(data) + bin_hash.update(data) + if rows: + _insert(conn, rows) + entries += len(rows) + out.flush() + os.fsync(out.fileno()) + + digest = _entries_digest_sorted(conn.execute("SELECT id, seq_sha1 FROM entries ORDER BY id")) + pending = { + "store_format": IDX_FORMAT, "system_json": system.to_json(), "esm_revision": esm_revision, + "min_res_n": min_res_n, "build_json": build_json, "entries": entries, "total_residues": total, + "bin_sha256": bin_hash.hexdigest(), "entries_digest": digest, + } + conn.executemany("INSERT INTO meta (key, value) VALUES (?, ?)", [(k, pending[k]) for k in _PENDING_KEYS]) + conn.commit() + conn.close() + conn = None + _fsync_file(self.tmp_idx_path) + _fsync_dir(self.db_folder) + except BaseException: + if conn is not None: + conn.close() + self._discard_pending() + raise + logger.info(f"Residue codes written to {self.tmp_bin_path} ({entries} entries, {total} residues); " + f"validation pending") + return {"entries": entries, "total_residues": total, "bin_sha256": pending["bin_sha256"], + "entries_digest": digest} + + def _read_pending(self) -> Dict[str, Any]: + """The idx meta create() recorded; ResidueCodeStoreError if create() did not finish.""" + if not self.tmp_idx_path.exists(): + raise ResidueCodeStoreError( + f"No pending residue-code build at {self.tmp_idx_path}: create() has not run (validate() checks the " + f"codes create() wrote, then swaps them in)") + with _sqlite_errors(self.tmp_idx_path): + conn = _connect_ro(self.tmp_idx_path) + try: + has_meta = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'").fetchone() is not None + pending = dict(conn.execute("SELECT key, value FROM meta").fetchall()) if has_meta else {} + finally: + conn.close() + missing = [k for k in _PENDING_KEYS if k not in pending] + if missing or pending.get("store_format") != IDX_FORMAT: + raise ResidueCodeStoreError( + f"Residue-code index {self.tmp_idx_path} is incomplete (its build stopped before it recorded " + f"{missing or 'its format'}). {_REBUILD_HINT}.") + return pending + + def validate(self, sequence_store: SequenceStore, faiss_ids_count: int, *, preflight) -> Dict[str, Any]: + """Check the pending build (:meth:`create`) against the database, then swap it in and write the manifest + (the commit point). Returns the manifest. + + The checks, run via ``ATTACH`` of the sequence store (both read-only): + + * (a) no sequence-store entry without codes; + * (b) no codes entry without a sequence-store entry; + * (c) no length mismatch, empty payloads included; + * (d) ``seq_sha1 == sha1(stored sequence)`` (both sides went through ``normalize_sequence``, so no + ``upper()``); + * (e) the entry count and ``N_res`` equal the sequence store's meta, the pre-flight's counts and the FAISS + count, and the store's own records (its index rows, its ``.bin`` size). + + ``preflight`` is the build's :class:`foldmatch.utils.sequence.FastaPreflight` (its ``unique_entries`` and + ``unique_residues``), or ``None`` when there was none (a rebuild from shards); it has no default, so it is + never left out by accident. Any failure discards the pending build and raises + (:class:`CodeStoreValidationError` naming each failed check with its first 10 ids); a complete store it was to + replace, the sequence store and FAISS are untouched, so the database still serves BLOSUM62. + + The swap, once every check passed: the manifest goes to ``P.rescodes.json.tmp`` and is fsynced (the only + step that needs space, taken while an old store is still whole), the old manifest is removed, the pending + ``.bin`` and ``.idx`` are renamed into place, and the new manifest is renamed last, each step made durable + with a directory fsync. A failure during the renames leaves no manifest and removes the store's files. + """ + if not (hasattr(sequence_store, "meta") and hasattr(sequence_store, "path")): + raise TypeError(f"validate needs a SequenceStore, got {type(sequence_store).__name__}") + if isinstance(faiss_ids_count, bool) or not isinstance(faiss_ids_count, (int, np.integer)) \ + or faiss_ids_count < 0: + raise TypeError(f"faiss_ids_count must be an int >= 0, got {faiss_ids_count!r}") + if preflight is not None and not (hasattr(preflight, "unique_entries") + and hasattr(preflight, "unique_residues")): + raise TypeError(f"preflight must be a FastaPreflight or None, got {type(preflight).__name__}") + replacing = self.exists() + try: + pending = self._read_pending() + system = EsmCodesSystem.from_json(pending["system_json"]) + seq_meta = _read_sequence_meta(sequence_store, ResidueCodeStoreError) + failures, scan = self._scan(sequence_store) + problems = self._totals_problems(pending, system, scan, seq_meta, preflight, int(faiss_ids_count)) + if problems: + failures["e"] = (len(problems), tuple(problems)) + if failures: + raise CodeStoreValidationError( + self._failure_message(failures, scan["empty_payloads"], replacing), failures) + manifest = _manifest(system, pending, scan, seq_meta, preflight, int(faiss_ids_count)) + # Written and fsynced (not yet renamed) while a store being replaced is still whole: the only step of + # the swap that needs space. + _write_synced(self.tmp_manifest_path, + json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n") + except BaseException: + self._discard_pending() + raise + self._swap_in() + logger.info(f"Residue-code store {self.manifest_path} validated ({manifest['entries']} entries, " + f"{manifest['total_residues']} residues)" + ("; it replaces the previous store" if replacing + else "")) + return manifest + + def _swap_in(self) -> None: + """Replace the store by the validated pending build: old manifest out, ``.bin`` and ``.idx`` renamed into + place, the new manifest renamed last (each step made durable). On failure no manifest is left, and the store's + files are removed (it is partial).""" + try: + if self.manifest_path.exists(): + os.unlink(self.manifest_path) # from here on the old store is partial, never mixed with the new one + _fsync_dir(self.db_folder) + for sidecar in _SQLITE_SIDECARS: + Path(f"{self.idx_path}{sidecar}").unlink(missing_ok=True) + os.replace(self.tmp_bin_path, self.bin_path) + os.replace(self.tmp_idx_path, self.idx_path) + _fsync_dir(self.db_folder) + os.replace(self.tmp_manifest_path, self.manifest_path) + _fsync_dir(self.db_folder) + except BaseException: + self.remove() + raise + + def _scan(self, sequence_store: SequenceStore) -> Tuple[Dict[str, Tuple[int, Tuple[str, ...]]], Dict[str, int]]: + """Checks (a)-(d) and the totals, in SQL over the pending idx with the sequence store attached.""" + seq_path = Path(sequence_store.path) + if not seq_path.exists(): + raise FileNotFoundError(f"Sequence store not found at {seq_path}") + with _sqlite_errors(f"{self.tmp_idx_path} with the sequence store {seq_path}"): + return self._scan_sql(seq_path) + + def _scan_sql(self, seq_path: Path): + failures: Dict[str, Tuple[int, Tuple[str, ...]]] = {} + conn = _connect_ro(self.tmp_idx_path) + try: + conn.execute("ATTACH DATABASE ? AS seq", (_ro_uri(seq_path),)) + conn.create_function("fm_seq_sha1", 1, _sql_seq_sha1, deterministic=True) + conn.create_function("fm_nonstandard", 1, _sql_nonstandard, deterministic=True) + rows, residues = conn.execute( + "SELECT COUNT(*), COALESCE(SUM(length), 0) FROM main.entries").fetchone() + + def count_and_ids(where: str, id_expr: str) -> Tuple[int, Tuple[str, ...]]: + n = conn.execute(f"SELECT COUNT(*) {where}").fetchone()[0] + ids = conn.execute(f"SELECT {id_expr} {where} ORDER BY {id_expr} LIMIT {_EXAMPLE_IDS}").fetchall() \ + if n else [] + return n, tuple(r[0] for r in ids) + + missing = count_and_ids( + "FROM seq.sequences s WHERE NOT EXISTS (SELECT 1 FROM main.entries c WHERE c.id = s.id)", "s.id") + extra = count_and_ids( + "FROM main.entries c WHERE NOT EXISTS (SELECT 1 FROM seq.sequences s WHERE s.id = c.id)", "c.id") + join = "FROM main.entries c JOIN seq.sequences s ON s.id = c.id" + _, length_mm, empty, sha1_mm, nonstandard = conn.execute( + f"SELECT COUNT(*), COALESCE(SUM(c.length != s.length), 0), " + f"COALESCE(SUM(c.length = 0 AND s.length > 0), 0), " + f"COALESCE(SUM(c.seq_sha1 IS NOT fm_seq_sha1(s.sequence)), 0), " + f"COALESCE(SUM(fm_nonstandard(s.sequence)), 0) {join}").fetchone() + if missing[0]: + failures["a"] = missing + if extra[0]: + failures["b"] = extra + if length_mm: + failures["c"] = count_and_ids(f"{join} WHERE c.length != s.length", "c.id") + if sha1_mm: + failures["d"] = count_and_ids(f"{join} WHERE c.seq_sha1 IS NOT fm_seq_sha1(s.sequence)", "c.id") + finally: + conn.close() + bin_size = self.tmp_bin_path.stat().st_size if self.tmp_bin_path.exists() else None + return failures, {"rows": rows, "residues": residues, "empty_payloads": empty, "length_mismatches": length_mm, + "seq_sha1_mismatches": sha1_mm, "nonstandard_entries": nonstandard, "bin_size": bin_size} + + def _totals_problems(self, pending, system, scan, seq_meta, preflight, faiss_ids_count) -> List[str]: + """Check (e): every count of entries and of residues agrees.""" + entries, residues = int(pending["entries"]), int(pending["total_residues"]) + problems = [] + + def compare(what: str, got_entries, got_residues=None): + if got_entries != entries: + problems.append(f"{what} has {got_entries} entries, the code store {entries}") + if got_residues is not None and got_residues != residues: + problems.append(f"{what} has {got_residues} residues, the code store {residues}") + + compare("the code index", scan["rows"], scan["residues"]) + want_size = residues * system.code_bytes + if scan["bin_size"] != want_size: + problems.append(f"{self.tmp_bin_path.name} is {scan['bin_size']} bytes, expected {want_size} " + f"({residues} residues x {system.code_bytes})") + compare("the sequence store meta", seq_meta["entry_count"], seq_meta["total_residues"]) + if preflight is not None: + compare("the pre-flight", preflight.unique_entries, preflight.unique_residues) + compare("the FAISS index", faiss_ids_count) + return problems + + def _failure_message(self, failures, empty_payloads: int, replacing: bool) -> str: + what = { + "a": "sequence-store entr(ies) have no residue codes", + "b": "residue-code entr(ies) are not in the sequence store", + "c": "entr(ies) have codes of another length than their sequence", + "d": "entr(ies) have a seq_sha1 that is not the sha1 of their stored sequence", + } + lines = [] + for check in ("a", "b", "c", "d"): + if check in failures: + n, ids = failures[check] + empty = (f", {empty_payloads} of them with an empty payload (the build could not encode them)" + if check == "c" and empty_payloads else "") + lines.append(f"({check}) {n} {what[check]}{empty} (first: {', '.join(ids)})") + if "e" in failures: + lines.append(f"(e) the totals disagree: {'; '.join(failures['e'][1])}") + hint = _FRESH_BUILD_HINT if "c" in failures and empty_payloads else _REBUILD_HINT + if replacing: + return (f"The new residue codes for {self.base} failed validation and were discarded: " + "; ".join(lines) + + f". The existing residue-code store {self.manifest_path} and the database are unchanged. " + f"{hint}.") + return (f"Residue-code store {self.base}.rescodes.* failed validation and was removed: " + "; ".join(lines) + + f". The database itself is unchanged and still serves --scoring blosum62. {hint}.") + + # ----------------------------------------------------------------------------------------------------------------- + # read + # ----------------------------------------------------------------------------------------------------------------- + def manifest(self) -> Dict[str, Any]: + """The parsed manifest (read-only).""" + if not self.manifest_path.exists(): + if self.is_partial(): + raise ResidueCodeStoreError( + f"Residue-code store {self.base}.rescodes.* is incomplete (no manifest): its build stopped " + f"before it was validated. {_REBUILD_HINT}.") + raise FileNotFoundError( + f"No residue-code store for {self.base} ({self.manifest_path} not found): the database was built " + f"without --residue-codes. {_REBUILD_HINT}.") + try: + data = json.loads(self.manifest_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + raise IncompatibleCodeStoreError(f"{self.manifest_path} is not readable JSON: {e}") from e + if not isinstance(data, dict): + raise IncompatibleCodeStoreError(f"{self.manifest_path} is not a JSON object") + return data + + def check_compatible(self, system=_INSTALLED, *, sequence_store=_INSTALLED, + esm_weights_sha256=_INSTALLED) -> Dict[str, Any]: + """Refuse a store this query cannot use; return its manifest. Read-only and cheap (no hashing, no scan), so + it runs on every rank before ESM3. + + Refused (:class:`IncompatibleCodeStoreError`, all problems named): an unknown ``manifest_version`` or + ``kind``; any missing key (``MANIFEST_KEYS`` and the nested blocks); an ``encoding_digest`` other than the + system's; ESM3 weights, target encoder or transform content hashes (sha256, not commit strings) other than the + system's, or ESM3 weights other than the ones this FoldMatch loads; any other encoding field, the code width + or bit order other than the system's; ``entries`` or ``total_residues`` other than the sequence store meta; a + missing index or a ``.bin`` of the wrong size. + + Every argument defaults to :data:`_INSTALLED`, what this installation uses: ``system`` the registry entry the + manifest names (an unknown name raises + :class:`~foldmatch.search.scoring_system.UnknownScoringSystemError`), ``sequence_store`` the database's own + sequence store (or pass a ``SequenceStore`` or its ``meta()`` mapping), ``esm_weights_sha256`` the ESM3 + weights this FoldMatch pins. So leaving an argument out never skips its check, and ``None`` is a TypeError. + A statistics-only system (another name, the same encoding) is compatible: the codes depend only on the + encoding fields. + """ + for name, value in (("system", system), ("sequence_store", sequence_store), + ("esm_weights_sha256", esm_weights_sha256)): + if value is None: + raise TypeError(f"check_compatible: {name}=None would skip a check; leave it out to compare with " + f"this installation") + manifest = self.manifest() + where = str(self.manifest_path) + version = manifest.get("manifest_version") + if isinstance(version, bool) or version != MANIFEST_VERSION: + raise IncompatibleCodeStoreError( + f"{where}: unknown manifest_version {version!r}; this FoldMatch reads version {MANIFEST_VERSION} (a " + f"store written by a newer FoldMatch needs a FoldMatch upgrade)") + if manifest.get("kind") != MANIFEST_KIND: + raise IncompatibleCodeStoreError(f"{where}: kind {manifest.get('kind')!r}, expected {MANIFEST_KIND!r}") + missing = _missing_keys(manifest) + if missing: + raise IncompatibleCodeStoreError(f"{where}: the manifest lacks {', '.join(missing)}. {_REBUILD_HINT}.") + if system is _INSTALLED: + system = get_system(manifest["system_name"]) + elif not isinstance(system, EsmCodesSystem): + raise TypeError(f"check_compatible: system must be an EsmCodesSystem, got {type(system).__name__}") + if esm_weights_sha256 is _INSTALLED: + from foldmatch.utils.esm.loaders import ESM_WEIGHTS_SHA256 as esm_weights_sha256 + if sequence_store is _INSTALLED: + sequence_store = self.sequence_store() + seq_meta = _sequence_meta(sequence_store) + + built = manifest["system"] + want = system.to_dict() + problems = [] + if manifest["encoding_digest"] != system.encoding_digest(): + problems.append(f"encoding_digest {manifest['encoding_digest']!r}, {system.name} has " + f"{system.encoding_digest()!r}") + if built["name"] != manifest["system_name"]: + problems.append(f"system_name {manifest['system_name']!r} but the recorded system is {built['name']!r}") + for label, field in (("ESM3 weights", "esm_weights_sha256"), ("target encoder", "target_encoder_sha256"), + ("transform", "transform_sha256")): + if built[field] != want[field]: + problems.append(f"built with {label} sha256 {built[field]}, {system.name} has {want[field]}") + if manifest["esm"]["weights_sha256"] != want["esm_weights_sha256"]: + problems.append(f"built with ESM3 weights {manifest['esm']['weights_sha256']}, {system.name} has " + f"{want['esm_weights_sha256']}") + if manifest["esm"]["weights_sha256"] != esm_weights_sha256: + problems.append(f"built with ESM3 weights {manifest['esm']['weights_sha256']}, this FoldMatch loads " + f"{esm_weights_sha256}") + for field in ENCODING_FIELDS: + if field not in ("esm_weights_sha256", "target_encoder_sha256", "transform_sha256") \ + and built[field] != want[field]: + problems.append(f"built with {field} {built[field]!r}, {system.name} has {want[field]!r}") + if manifest["code_bytes"] != system.code_bytes or manifest["bitorder"] != system.bitorder: + problems.append(f"codes of {manifest['code_bytes']!r} bytes, bit order {manifest['bitorder']!r}; " + f"{system.name} has {system.code_bytes} bytes, {system.bitorder!r}") + if manifest["entries"] != seq_meta["entry_count"]: + problems.append(f"{manifest['entries']!r} entries, the sequence store {seq_meta['entry_count']}") + if manifest["total_residues"] != seq_meta["total_residues"]: + problems.append(f"{manifest['total_residues']!r} residues, the sequence store " + f"{seq_meta['total_residues']}") + if not self.idx_path.exists(): + problems.append(f"{self.idx_path} is missing") + if not self.bin_path.exists(): + problems.append(f"{self.bin_path} is missing") + elif isinstance(manifest["total_residues"], int) and isinstance(manifest["code_bytes"], int): + want_size = manifest["total_residues"] * manifest["code_bytes"] + size = self.bin_path.stat().st_size + if size != want_size: + problems.append(f"{self.bin_path.name} is {size} bytes, the manifest needs {want_size}") + if problems: + raise IncompatibleCodeStoreError( + f"Residue-code store {self.base}.rescodes.* cannot be used with {system.name}: " + "; ".join(problems) + + f". --scoring blosum62 can query this database (its Stage 2 needs the native kernel too). " + f"{_REBUILD_HINT}.") + return manifest + + def check_faiss_count(self, faiss_ids_count: int) -> None: + """Refuse a store whose entry count is not the live FAISS index's (rank 0, once the index is loaded).""" + entries = self.manifest().get("entries") + if entries != faiss_ids_count: + raise IncompatibleCodeStoreError( + f"Residue-code store {self.manifest_path} has {entries!r} entries, the FAISS index " + f"{faiss_ids_count}. --scoring blosum62 still works. {_REBUILD_HINT}.") + + def _require_complete(self) -> None: + if not self.exists(): + self.manifest() # raises the partial / not-found error + + def lookup(self, ids: Iterable[str]) -> Dict[str, CodeEntry]: + """``{id: CodeEntry(offset, length, seq_sha1)}`` for every id (repeats collapse); an id the store lacks + raises :class:`ResidueCodeStoreError` naming the first 10. Queried in ``IN`` chunks of 900.""" + self._require_complete() + unique = list(dict.fromkeys(ids)) + out: Dict[str, CodeEntry] = {} + if unique: + for entry_id, offset, length, sha1 in self._select(unique, "offset, length, seq_sha1"): + out[entry_id] = CodeEntry(offset, length, sha1) + self._refuse_missing(unique, out) + return out + + def lookup_rows(self, ids: Sequence[str]) -> Tuple[np.ndarray, np.ndarray]: + """``(offsets, lengths)``, int64 arrays aligned with ``ids`` (repeats allowed): the code rows of each id, as + :meth:`lookup` gives them, without building an entry per id (Stage 2 looks up a whole batch of candidates + at once). Missing ids raise as in :meth:`lookup`.""" + self._require_complete() + ids = ids if isinstance(ids, list) else list(ids) + unique = list(dict.fromkeys(ids)) + rows: Dict[str, Tuple[int, int]] = {} + if unique: + for entry_id, offset, length in self._select(unique, "offset, length"): + rows[entry_id] = (offset, length) + self._refuse_missing(unique, rows) + if not ids: + return np.zeros(0, np.int64), np.zeros(0, np.int64) + table = np.array([rows[i] for i in unique], np.int64).reshape(-1, 2) + position = dict(zip(unique, range(len(unique)))) + picks = np.fromiter((position[i] for i in ids), np.int64, len(ids)) + found = table[picks] + return np.ascontiguousarray(found[:, 0]), np.ascontiguousarray(found[:, 1]) + + def _select(self, unique: List[str], columns: str) -> Iterator[tuple]: + """``(id, *columns)`` rows of the ids, queried in ``IN`` chunks of 900 on one read-only connection.""" + with _sqlite_errors(self.idx_path): + conn = _connect_ro(self.idx_path) + try: + for start in range(0, len(unique), _SQLITE_VAR_CHUNK): + chunk = unique[start:start + _SQLITE_VAR_CHUNK] + placeholders = ",".join("?" * len(chunk)) + yield from conn.execute(f"SELECT id, {columns} FROM entries WHERE id IN ({placeholders})", chunk) + finally: + conn.close() + + def _refuse_missing(self, unique: List[str], found: Mapping[str, object]) -> None: + missing = [i for i in unique if i not in found] + if missing: + raise ResidueCodeStoreError( + f"{len(missing)} id(s) have no residue codes in {self.idx_path} (first: " + f"{', '.join(map(str, missing[:_EXAMPLE_IDS]))}): the code store and the database disagree. " + f"{_REBUILD_HINT}.") + + def open_memmap(self) -> np.ndarray: + """The codes, ``uint8 [N_res, code_bytes]``, memory-mapped read-only. An empty store gives an empty array + (``np.memmap`` cannot map 0 bytes).""" + manifest = self.manifest() + code_bytes, total = manifest.get("code_bytes"), manifest.get("total_residues") + if not all(isinstance(v, int) and not isinstance(v, bool) and v >= 0 for v in (code_bytes, total)) \ + or code_bytes == 0: + raise IncompatibleCodeStoreError(f"{self.manifest_path}: unusable code_bytes {code_bytes!r} / " + f"total_residues {total!r}") + size = self.bin_path.stat().st_size + if size != total * code_bytes: + raise IncompatibleCodeStoreError(f"{self.bin_path} is {size} bytes, the manifest needs " + f"{total * code_bytes}. {_REBUILD_HINT}.") + if total == 0: + return np.empty((0, code_bytes), dtype=np.uint8) + return np.memmap(self.bin_path, dtype=np.uint8, mode="r", shape=(total, code_bytes)) + + +# ===================================================================================================================== +# helpers +# ===================================================================================================================== +def _columns(batch) -> Tuple[list, list, list]: + if hasattr(batch, "schema") and hasattr(batch, "column"): + names = set(batch.schema.names) + lacking = [c for c in (ID_COLUMN, CODES_COLUMN, SEQ_SHA1_COLUMN) if c not in names] + if lacking: + raise ResidueCodeStoreError(f"a residue-code batch lacks the column(s) {lacking} (it has {sorted(names)})") + columns = [batch.column(c).to_pylist() for c in (ID_COLUMN, CODES_COLUMN, SEQ_SHA1_COLUMN)] + else: + try: + columns = list(batch) + except TypeError: + columns = [] + if len(columns) != 3: + raise ResidueCodeStoreError("a residue-code batch must be (ids, codes, seq_sha1s) or a pyarrow batch with " + f"the columns {ID_COLUMN}, {CODES_COLUMN}, {SEQ_SHA1_COLUMN}") + lengths = {len(c) for c in columns} + if len(lengths) != 1: + raise ResidueCodeStoreError(f"a residue-code batch has columns of different lengths {sorted(lengths)}") + return columns[0], columns[1], columns[2] + + +def _payload_bytes(value, code_bytes: int, entry_id: str) -> bytes: + if isinstance(value, np.ndarray): + if value.dtype != np.uint8 or value.ndim not in (1, 2) or (value.ndim == 2 and value.shape[1] != code_bytes): + raise ResidueCodeStoreError(f"{entry_id!r}: codes must be uint8 [L, {code_bytes}], got {value.dtype} " + f"{value.shape}") + data = np.ascontiguousarray(value).tobytes() + elif isinstance(value, (bytes, bytearray)): + data = bytes(value) + elif isinstance(value, memoryview) and value.itemsize == 1: + data = value.tobytes() + else: + raise ResidueCodeStoreError(f"{entry_id!r}: codes must be bytes or a uint8 array, got " + f"{type(value).__name__}") + if len(data) % code_bytes: + raise ResidueCodeStoreError(f"{entry_id!r}: {len(data)} code bytes is not a whole number of " + f"{code_bytes}-byte residues") + return data + + +def _seq_sha1_bytes(value, entry_id: str) -> bytes: + if isinstance(value, (bytes, bytearray, memoryview)): + data = bytes(value) + if len(data) == SEQ_SHA1_BYTES: + return data + raise ResidueCodeStoreError(f"{entry_id!r}: seq_sha1 must be {SEQ_SHA1_BYTES} bytes, got {value!r:.80}") + + +def _insert(conn: sqlite3.Connection, rows: list) -> None: + """One transaction of index rows; a duplicate id (in ``rows`` or already stored) raises naming it.""" + try: + conn.executemany("INSERT INTO entries (id, offset, length, seq_sha1) VALUES (?, ?, ?, ?)", rows) + conn.commit() + except sqlite3.IntegrityError as e: + conn.rollback() + ids = [r[0] for r in rows] + seen, duplicates = set(), [] + for entry_id in ids: + if entry_id in seen: + duplicates.append(entry_id) + seen.add(entry_id) + unique = list(dict.fromkeys(ids)) + for start in range(0, len(unique), _SQLITE_VAR_CHUNK): + chunk = unique[start:start + _SQLITE_VAR_CHUNK] + duplicates.extend(r[0] for r in conn.execute( + f"SELECT id FROM entries WHERE id IN ({','.join('?' * len(chunk))})", chunk)) + duplicates = sorted(set(duplicates)) + raise ResidueCodeStoreError( + f"duplicate entry id(s) in the residue codes: {', '.join(duplicates[:_EXAMPLE_IDS]) or e}. The " + f"pre-flight refuses duplicate ids before ESM3, so this is a bug.") from e + + +def _sql_seq_sha1(sequence) -> Optional[bytes]: + """SQL ``fm_seq_sha1``: sha1 of the ASCII bytes of an already normalised sequence (NULL if not ASCII, which then + mismatches every stored seq_sha1).""" + if not isinstance(sequence, str) or not sequence.isascii(): + return None + return hashlib.sha1(sequence.encode("ascii")).digest() + + +def _sql_nonstandard(sequence) -> int: + """SQL ``fm_nonstandard``: 1 if the sequence has a residue outside the 20 standard amino acids.""" + if not isinstance(sequence, str): + return 1 + if not sequence.isascii(): + return 1 + return int(bool(sequence.encode("ascii").translate(None, _STANDARD_BYTES))) + + +def _ro_uri(path) -> str: + return Path(path).resolve().as_uri() + "?mode=ro" + + +def _connect_ro(path) -> sqlite3.Connection: + if not Path(path).exists(): + raise FileNotFoundError(f"{path} not found") + return sqlite3.connect(_ro_uri(path), uri=True) + + +def _fsync_file(path) -> None: + fd = os.open(path, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _fsync_dir(path) -> None: + """Make renames and unlinks in ``path`` durable; a no-op where directories cannot be opened or fsynced.""" + try: + fd = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(fd) + except OSError: + pass + finally: + os.close(fd) + + +def _write_synced(path: Path, text: str) -> None: + """Write ``path`` and fsync it (the caller renames it into place).""" + with open(path, "w", encoding="utf-8") as f: + f.write(text) + f.flush() + os.fsync(f.fileno()) + + +@contextmanager +def _sqlite_errors(what): + """A file that is not the SQLite database it should be: a :class:`ResidueCodeStoreError` naming it (SQLite's own + message, e.g. 'file is not a database', names no file).""" + try: + yield + except sqlite3.DatabaseError as e: + if isinstance(e, sqlite3.IntegrityError): + raise + raise ResidueCodeStoreError(f"{what}: not a readable residue-code index or sequence store ({e})") from e + + +def _read_sequence_meta(sequence_store, error: type) -> Mapping[str, Any]: + """``sequence_store.meta()``, a corrupt file reported as ``error`` naming it.""" + try: + return sequence_store.meta() + except sqlite3.DatabaseError as e: + raise error(f"{sequence_store.path} is not a readable sequence store ({e})") from e + + +def _dist_version(name: str) -> Optional[str]: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + +def _build_info(build: Optional[Mapping[str, Any]]) -> Dict[str, Any]: + from foldmatch import __version__ + info: Dict[str, Any] = { + "foldmatch_version": __version__, + "created_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "run_id": None, "from_shards": None, "accelerator": None, "devices": None, "esm_precision": None, + "encoder_precision": None, + "torch": _dist_version("torch"), "esm": _dist_version("esm"), + "platform": f"{platform.system()}-{platform.machine()}", + } + if build is not None: + if not isinstance(build, Mapping) or not all(isinstance(k, str) for k in build): + raise TypeError(f"build must be a mapping with str keys, got {build!r}") + info.update(build) + return info + + +def _installed_esm_revision(system: EsmCodesSystem) -> Optional[str]: + """This installation's ESM3 revision pin, when it serves the system's ESM3 weights.""" + from foldmatch.utils.esm import loaders + if (loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_SHA256) == (system.esm_repo, system.esm_weights_sha256): + return loaders.ESM_REVISION + return None + + +def _sequence_meta(sequence_store) -> Mapping[str, Any]: + if hasattr(sequence_store, "meta"): + meta = _read_sequence_meta(sequence_store, IncompatibleCodeStoreError) + elif isinstance(sequence_store, Mapping): + meta = sequence_store + else: + raise TypeError(f"sequence_store must be a SequenceStore or its meta() mapping, got " + f"{type(sequence_store).__name__}") + for key in ("entry_count", "total_residues"): + if key not in meta: + raise TypeError(f"sequence_store meta lacks {key!r}") + return meta + + +def _missing_keys(manifest: Mapping[str, Any]) -> List[str]: + missing = [k for k in MANIFEST_KEYS if k not in manifest] + for block, keys in _NESTED_KEYS.items(): + if block not in manifest: + continue + value = manifest[block] + if not isinstance(value, Mapping): + missing.append(f"{block} (not an object)") + continue + missing.extend(f"{block}.{k}" for k in keys if k not in value) + return missing + + +def _manifest(system: EsmCodesSystem, pending, scan, seq_meta, preflight, faiss_ids_count: int) -> Dict[str, Any]: + """The Appendix B manifest of a store that passed validation.""" + return { + "manifest_version": MANIFEST_VERSION, + "kind": MANIFEST_KIND, + "system_name": system.name, + "system_digest": system.digest(), + "encoding_digest": system.encoding_digest(), + "system": system.to_dict(), + "code_bytes": system.code_bytes, + "bitorder": system.bitorder, + "esm": {"repo": system.esm_repo, "revision": pending["esm_revision"], + "weights_sha256": system.esm_weights_sha256}, + "bundle": {"repo": system.bundle_repo, "revision": system.bundle_revision, + "files": dict(system.bundle_files)}, + "entries": int(pending["entries"]), + "total_residues": int(pending["total_residues"]), + "min_res_n": int(pending["min_res_n"]), + "bin_sha256": pending["bin_sha256"], + "entries_digest": pending["entries_digest"], + "nonstandard_entries": int(scan["nonstandard_entries"]), + "input_normalisation": system.input_normalisation, + "cross_checks": { + "preflight_entries": None if preflight is None else int(preflight.unique_entries), + "preflight_residues": None if preflight is None else int(preflight.unique_residues), + "sequence_store_entries": int(seq_meta["entry_count"]), + "sequence_store_residues": int(seq_meta["total_residues"]), + "faiss_entries": faiss_ids_count, + "seq_sha1_mismatches": int(scan["seq_sha1_mismatches"]), + "length_mismatches": int(scan["length_mismatches"]), + }, + "build": json.loads(pending["build_json"]), + } diff --git a/src/foldmatch/search/residue_shards.py b/src/foldmatch/search/residue_shards.py new file mode 100644 index 0000000..395458f --- /dev/null +++ b/src/foldmatch/search/residue_shards.py @@ -0,0 +1,436 @@ +"""Parquet embedding shards with a residue column: the schema contract between the ESM3 pass and its readers. + +A pass with a residue output (plan P5) writes, next to each chain embedding, one per-residue payload and the SHA-1 +of the sequence it was computed from: + +* builds, ``residue_codes``: the S4 target codes, ``uint8[L, 8]`` as ``L * 8`` bytes; +* queries, ``query_encoding``: the S5 query encodings, ``float16[L, 64]`` little-endian as ``L * 128`` bytes. + +Columns ``id: string | embedding: list | : large_binary | seq_sha1: fixed_size_binary(20)``. +``large_binary`` has 64-bit offsets, so a row group may hold more than 2 GiB of payloads. ``seq_sha1`` is +``sha1(normalize_sequence(sequence))`` (S15). A row whose payload could not be computed (the ESM3 rows did not match +the sequence length) has an empty payload; the readers pass it on as length 0 and the code-store validation refuses +it after the pass (the ESM3 pass itself never raises, P5.1). + +Schema metadata, fixed when the writer is constructed: + +* ``foldmatch.residue_output``: ``residue_codes`` or ``query_encoding``; +* ``foldmatch.system``: the scoring system's name; +* ``foldmatch.encoding_digest``: :meth:`EsmCodesSystem.encoding_digest`, the fields the target codes depend on; +* ``foldmatch.system_digest``: :meth:`EsmCodesSystem.digest`. The query encodings also depend on the query encoder, + which only the full digest covers; +* ``foldmatch.pass``: how the pass computed the payloads (:func:`pass_record`, JSON): its ``min_res_n``, the device + type and precision ESM3 ran on, the number of ranks, the ESM3 revision and the software versions. The writer + completes it at the start of the pass, once the device is known, before any row is written. The code store's + manifest takes its ``build`` block from it, also when the store is rebuilt from the shards later. + +The readers (:func:`check_shards`, :func:`iter_residue_codes`, ``foldmatch.search.query_encodings``) read every +shard's footer before any data and refuse shards without the column, mixed shards (different kinds, systems or +digests) and stale shards (another system's encoding): residue codes need the system's ``encoding_digest`` (a system +that differs only in its statistics may reuse them), query encodings its ``digest`` too. + +A plain pass (no residue output) keeps the legacy schema ``id | embedding`` with no metadata, byte for byte. + +A run's shards are ``emb_-.parquet`` (:func:`run_shard_files`, :func:`shard_run_ids`; also imported +from ``foldmatch.search.embedding_computer``). + +pyarrow and numpy only: no torch, so Stage-2 code (and ``foldmatch.search.split_run``) can import it. +""" +from __future__ import annotations + +import glob +import json +import logging +import platform +import re +from dataclasses import dataclass, field +from importlib import metadata +from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, NamedTuple, Optional, Sequence + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from foldmatch.utils.dist import is_run_id + +RESIDUE_CODES = "residue_codes" +QUERY_ENCODING = "query_encoding" +RESIDUE_OUTPUTS = (RESIDUE_CODES, QUERY_ENCODING) + +ID = "id" +EMBEDDING = "embedding" +SEQ_SHA1 = "seq_sha1" +SEQ_SHA1_BYTES = 20 + +META_RESIDUE_OUTPUT = b"foldmatch.residue_output" +META_SYSTEM = b"foldmatch.system" +META_ENCODING_DIGEST = b"foldmatch.encoding_digest" +META_SYSTEM_DIGEST = b"foldmatch.system_digest" +META_KEYS = (META_RESIDUE_OUTPUT, META_SYSTEM, META_ENCODING_DIGEST, META_SYSTEM_DIGEST) +META_PASS = b"foldmatch.pass" + +#: The fields of a ``foldmatch.pass`` record (:func:`pass_record`). +PASS_KEYS = ("min_res_n", "accelerator", "world_size", "esm_precision", "encoder_precision", "esm_revision", + "foldmatch_version", "torch", "esm", "platform") +#: The residue-code encoders always run in fp32, without TF32 or autocast (method spec S3). +ENCODER_PRECISION = "fp32-no-tf32" + +logger = logging.getLogger(__name__) + +#: The legacy shard schema (``id | embedding``), unchanged for plain passes. +BASE_FIELDS = ((ID, pa.string()), (EMBEDDING, pa.list_(pa.float32()))) +PAYLOAD_TYPE = pa.large_binary() +SEQ_SHA1_TYPE = pa.binary(SEQ_SHA1_BYTES) + +#: Payload byte order: little-endian, the native order of every supported platform. +QUERY_ENCODING_DTYPE = np.dtype(" Optional[str]: + """``None``, ``"residue_codes"`` or ``"query_encoding"``; anything else raises ``ValueError``.""" + if value is None: + return None + value = getattr(value, "value", value) + if value not in RESIDUE_OUTPUTS: + raise ValueError(f"unknown residue output {value!r}; expected None or one of {list(RESIDUE_OUTPUTS)}") + return value + + +def residue_width(residue_output: str, system) -> int: + """Payload values per residue: the code bytes (8) or the query encoding's dimensions (64; both encoders map + to ``code_bits`` outputs).""" + return system.code_bytes if check_residue_output(residue_output) == RESIDUE_CODES else system.code_bits + + +def payload_row_bytes(residue_output: str, system) -> int: + """Payload bytes per residue: 8 for residue codes, 128 for query encodings (64 x float16).""" + if check_residue_output(residue_output) == RESIDUE_CODES: + return system.code_bytes + if np.dtype(system.query_dtype) != np.dtype(np.float16): + raise ResidueShardError(f"{system.name}: query_dtype {system.query_dtype!r} is not implemented (float16)") + return system.code_bits * QUERY_ENCODING_DTYPE.itemsize + + +def shard_metadata(residue_output: str, system) -> dict: + return { + META_RESIDUE_OUTPUT: check_residue_output(residue_output).encode("ascii"), + META_SYSTEM: system.name.encode("ascii"), + META_ENCODING_DIGEST: system.encoding_digest().encode("ascii"), + META_SYSTEM_DIGEST: system.digest().encode("ascii"), + } + + +def esm_precision(device_type: Optional[str]) -> Optional[str]: + """How ESM3 computes on ``device_type``: ``"bf16-autocast"`` on CUDA (esm 3.2.3 runs ``ESM3.logits`` under bf16 + autocast there, and only there), ``"fp32"`` elsewhere (CPU, Apple MPS); ``None`` for an unknown device.""" + if device_type is None: + return None + return "bf16-autocast" if device_type == "cuda" else "fp32" + + +def _dist_version(name: str) -> Optional[str]: + try: + return metadata.version(name) + except metadata.PackageNotFoundError: + return None + + +def pass_record(*, min_res_n: Optional[int] = None, esm_revision: Optional[str] = None, + accelerator: Optional[str] = None, world_size: Optional[int] = None) -> Dict[str, Any]: + """The ``foldmatch.pass`` record of a residue pass (``PASS_KEYS``): ``min_res_n``, ``accelerator`` (the device + type the module ran on: ``cpu``, ``cuda``, ``mps``), ``world_size`` (the pass's ranks), the ESM3 and encoder + precision, the ESM3 revision, and the versions of FoldMatch, torch and esm and the platform of this process. + ``None`` where unknown.""" + from foldmatch import __version__ + return { + "min_res_n": min_res_n, + "accelerator": accelerator, + "world_size": world_size, + "esm_precision": esm_precision(accelerator), + "encoder_precision": ENCODER_PRECISION, + "esm_revision": esm_revision, + "foldmatch_version": __version__, + "torch": _dist_version("torch"), + "esm": _dist_version("esm"), + "platform": f"{platform.system()}-{platform.machine()}", + } + + +def pass_metadata(record: Dict[str, Any]) -> dict: + """The schema metadata entry that holds a :func:`pass_record` (compact, sorted JSON).""" + return {META_PASS: json.dumps(record, sort_keys=True, separators=(",", ":"), allow_nan=False).encode("ascii")} + + +def shard_schema(residue_output: Optional[str] = None, system=None, record: Optional[Dict[str, Any]] = None + ) -> pa.Schema: + """The shard schema: the legacy ``id | embedding`` (no metadata) without a residue output, else with the + residue column, ``seq_sha1`` and the metadata of ``system`` (required then) and of the pass ``record`` + (:func:`pass_record`; one with every field unknown by default).""" + residue_output = check_residue_output(residue_output) + if residue_output is None: + if record is not None: + raise ValueError("a pass record is only kept in a shard with a residue output") + return pa.schema(list(BASE_FIELDS)) + if system is None: + raise ValueError(f"a {residue_output} shard schema needs the scoring system") + metadata = shard_metadata(residue_output, system) + metadata.update(pass_metadata(pass_record() if record is None else record)) + return pa.schema(list(BASE_FIELDS) + [(residue_output, PAYLOAD_TYPE), (SEQ_SHA1, SEQ_SHA1_TYPE)], + metadata=metadata) + + +def _concat_payloads(payloads: Sequence[bytes], total: int): + """One contiguous buffer holding ``payloads`` back to back (``total`` bytes).""" + return b"".join(payloads) + + +def large_binary_array(payloads: Sequence[bytes]) -> pa.LargeBinaryArray: + """A ``large_binary`` array of ``payloads`` with explicit 64-bit offsets (no 2 GiB limit per chunk).""" + n = len(payloads) + offsets = np.zeros(n + 1, dtype=np.int64) + np.cumsum(np.fromiter((len(p) for p in payloads), dtype=np.int64, count=n), out=offsets[1:]) + data = _concat_payloads(payloads, int(offsets[-1])) + return pa.LargeBinaryArray.from_buffers(PAYLOAD_TYPE, n, [None, pa.py_buffer(offsets), pa.py_buffer(data)]) + + +# ===================================================================================================================== +# shard files +# ===================================================================================================================== +# A run's shards are {folder}/emb_{run id}-{rank}.parquet. Run ids may contain '-' (utils.dist), so the rank is the +# last '-'-separated field. +_SHARD_PREFIX = "emb_" +_ANY_SHARD = re.compile(rf"{_SHARD_PREFIX}(?P.+)-\d+\.parquet") + + +def run_shard_files(folder: Path, run_id: str) -> list[Path]: + """The shards of run ``run_id`` in ``folder``, in the order rank 0 reads them (sorted by path). + + The glob is narrowed by an exact match, so a run whose id merely starts + with this one (``abc`` vs ``abc-1``) is never picked up. The id is + escaped in the glob: run ids hold no glob metacharacters + (:func:`foldmatch.utils.dist.run_id`), but a name found in a folder may. + """ + out_name = f"{_SHARD_PREFIX}{run_id}" + shard_name = re.compile(rf"{re.escape(out_name)}-\d+\.parquet") + return sorted( + path for path in Path(folder).glob(f"{glob.escape(out_name)}-*.parquet") + if shard_name.fullmatch(path.name) + ) + + +def shard_run_ids(folder: Path) -> list[str]: + """The run ids of every shard in ``folder``, sorted. Read-only, file names only. + + Only well-formed run ids (:func:`foldmatch.utils.dist.is_run_id`) count: + a file such as ``emb_copy[1]-0.parquet`` is not a shard of any run. + """ + names = (path.name for path in Path(folder).glob(f"{_SHARD_PREFIX}*.parquet")) + return sorted({match["run_id"] for match in map(_ANY_SHARD.fullmatch, names) + if match and is_run_id(match["run_id"])}) + + +# ===================================================================================================================== +# readers +# ===================================================================================================================== +@dataclass(frozen=True) +class ShardInfo: + """What a shard's footer says (read with ``pq.read_schema``: no data). ``pass_record`` is its parsed + ``foldmatch.pass`` metadata (:func:`pass_record`), ``None`` if it has none; it describes the pass, so it is not + part of :attr:`key` (the ranks of one run may be told apart by it in principle; :func:`shards_pass_record`).""" + path: Path + residue_output: Optional[str] + system: Optional[str] + encoding_digest: Optional[str] + system_digest: Optional[str] + column_types: tuple + pass_record: Optional[Dict[str, Any]] = field(default=None, compare=False) + + @property + def key(self) -> tuple: + return self.residue_output, self.system, self.encoding_digest, self.system_digest, self.column_types + + def describe(self) -> str: + if self.residue_output is None: + return "no residue column (a plain pass)" + return (f"{self.residue_output} of {self.system} (encoding_digest {self.encoding_digest}, system_digest " + f"{self.system_digest})") + + +def _type_name(t: pa.DataType) -> str: + # Parquet names a list's item field "element" where Arrow says "item": compare the value types only. + if pa.types.is_list(t) or pa.types.is_large_list(t): + return f"{'large_' if pa.types.is_large_list(t) else ''}list<{_type_name(t.value_type)}>" + return str(t) + + +def _column_types(schema: pa.Schema) -> tuple: + return tuple((f.name, _type_name(f.type)) for f in schema) + + +def read_shard_info(path) -> ShardInfo: + """The footer of the shard at ``path``; a file that is not a readable Parquet shard (empty, truncated, missing) + is a :class:`ResidueShardError` naming it.""" + try: + schema = pq.read_schema(path) + except (pa.ArrowException, OSError) as e: + raise ResidueShardError(f"{path}: not a readable Parquet embedding shard ({e})") from e + meta = schema.metadata or {} + + def text(key): + value = meta.get(key) + return value.decode("utf-8", "replace") if value is not None else None + + record = None + if META_PASS in meta: + try: + record = json.loads(meta[META_PASS].decode("utf-8")) + except ValueError: + record = None + if not isinstance(record, dict): + raise ResidueShardError(f"{path}: its {META_PASS.decode()} metadata is not a JSON object") + return ShardInfo(path=Path(path), residue_output=text(META_RESIDUE_OUTPUT), system=text(META_SYSTEM), + encoding_digest=text(META_ENCODING_DIGEST), system_digest=text(META_SYSTEM_DIGEST), + column_types=_column_types(schema), pass_record=record) + + +def shards_pass_record(infos: Sequence[ShardInfo]) -> Dict[str, Any]: + """The pass record of one run's shards (``PASS_KEYS``, every field ``None`` when they hold none): the first + shard's, which is rank 0's in path order. The ranks of a run share their settings, so a field on which the + shards disagree, or a shard without a record, is only logged.""" + out: Dict[str, Any] = dict.fromkeys(PASS_KEYS) + records = [info.pass_record for info in infos if info.pass_record is not None] + if not records: + return out + first = records[0] + out.update({key: first.get(key) for key in PASS_KEYS}) + differing = sorted({key for record in records[1:] for key in PASS_KEYS if record.get(key) != first.get(key)}) + if differing or len(records) != len(infos): + logger.warning(f"{len(infos)} shard(s) ({infos[0].path.name}, ...): {len(records)} record their pass" + + (f", and they differ in {', '.join(differing)}" if differing else "") + + f"; using {infos[0].path.name}'s") + return out + + +def check_shards(paths: Iterable, residue_output: str, system) -> List[ShardInfo]: + """Refuse (:class:`ResidueShardError`) unless every shard in ``paths`` holds ``residue_output`` computed with + ``system``'s encoding. Reads the footers only, all of them before any data. + + Mixed shards (different kinds, systems, digests or column types between them) are refused naming each group; + then the common metadata must be ``system``'s: the ``encoding_digest`` for residue codes, and also the + ``system_digest`` for query encodings. The columns must be exactly :func:`shard_schema`'s. + """ + residue_output = check_residue_output(residue_output) + if residue_output is None: + raise ValueError("check_shards needs a residue output") + infos = [read_shard_info(p) for p in paths] + if not infos: + return infos + groups: dict = {} + for info in infos: + groups.setdefault(info.key, []).append(info) + if len(groups) > 1: + shown = "; ".join(f"{len(g)} shard(s) with {g[0].describe()} (e.g. {g[0].path.name})" + for g in groups.values()) + raise ResidueShardError(f"mixed embedding shards: {shown}. Shards of one run must come from one pass; remove " + "the shards of other runs or use another folder.") + info = infos[0] + where = f"{len(infos)} shard(s) ({info.path.name}, ...)" + if info.residue_output != residue_output: + raise ResidueShardError(f"{where} hold {info.describe()}, not {residue_output}: they were written by a pass " + f"without --residue-codes or for another purpose") + want_types = _column_types(shard_schema(residue_output, system)) + if info.column_types != want_types: + raise ResidueShardError(f"{where} have columns {list(info.column_types)}, expected {list(want_types)}") + stale = [] + if info.encoding_digest != system.encoding_digest(): + stale.append(f"encoding_digest {info.encoding_digest}, {system.name} has {system.encoding_digest()}") + if residue_output == QUERY_ENCODING and info.system_digest != system.digest(): + stale.append(f"system_digest {info.system_digest}, {system.name} has {system.digest()}") + if stale: + raise ResidueShardError(f"stale embedding shards: {where} hold {info.describe()}, which is not the encoding " + f"of {system.name} ({'; '.join(stale)}). Rebuild them with this FoldMatch.") + return infos + + +class PayloadBatch(NamedTuple): + """One record batch of a residue shard: ``offsets`` (int64 ``[n + 1]``, in bytes, starting at 0) delimit each + row's payload in ``data`` (uint8).""" + ids: List[str] + offsets: np.ndarray + data: np.ndarray + seq_sha1s: List[bytes] + + +def _payload_buffers(column: pa.Array, where: str): + if column.null_count: + raise ResidueShardError(f"{where}: {column.null_count} null payload(s)") + if not isinstance(column, pa.LargeBinaryArray): + raise ResidueShardError(f"{where}: payload column is {column.type}, expected large_binary") + _, offsets_buf, data_buf = column.buffers() + n = len(column) + offsets = np.frombuffer(offsets_buf, dtype=np.int64, count=column.offset + n + 1)[column.offset:] + start, stop = int(offsets[0]), int(offsets[-1]) + data = (np.frombuffer(data_buf, dtype=np.uint8, count=stop)[start:stop] if stop + else np.empty(0, dtype=np.uint8)) + return offsets - start, data + + +def iter_payload_batches(path, residue_output: str, batch_size: int = 32768) -> Iterator[PayloadBatch]: + """Stream one shard's ``id``, payload and ``seq_sha1`` columns (column projection: the embeddings are not + read). Opens the shard when iteration starts and closes it when the shard is done.""" + residue_output = check_residue_output(residue_output) + where = f"{Path(path).name}" + try: + with pq.ParquetFile(path) as pf: + for record in pf.iter_batches(batch_size=batch_size, columns=[ID, residue_output, SEQ_SHA1]): + offsets, data = _payload_buffers(record.column(residue_output), where) + sha1 = record.column(SEQ_SHA1) + if sha1.null_count: + raise ResidueShardError(f"{where}: {sha1.null_count} null seq_sha1 value(s)") + yield PayloadBatch(record.column(ID).to_pylist(), offsets, data, sha1.to_pylist()) + except (pa.ArrowException, OSError) as e: + # A shard damaged after its footer was checked (truncated data pages, a vanished file). + raise ResidueShardError(f"{path}: the shard could not be read ({e})") from e + + +class ResidueCodeBatch(NamedTuple): + """Residue codes of consecutive entries, in shard order, as the code store's ``create()`` takes them: a + 3-tuple ``(ids, codes, seq_sha1s)`` with one uint8 ``[L, 8]`` array per entry in ``codes`` (``[0, 8]`` for an + empty payload). The arrays are read-only, C-contiguous views of the batch's Arrow buffer, consecutive in it, so + ``codes[k].tobytes()`` is entry ``k``'s byte layout in the store.""" + ids: List[str] + codes: List[np.ndarray] + seq_sha1s: List[bytes] + + @property + def lengths(self) -> np.ndarray: + """Residues per entry (int64).""" + return np.fromiter((len(c) for c in self.codes), dtype=np.int64, count=len(self.codes)) + + +def iter_residue_codes(paths: Sequence, system, load_batch_size: int = 32768) -> Iterator[ResidueCodeBatch]: + """:func:`check_shards` on every shard (before anything is yielded), then the residue codes one shard at a + time, in the given order, ``load_batch_size`` entries per batch.""" + paths = [Path(p) for p in paths] + check_shards(paths, RESIDUE_CODES, system) + return _residue_code_batches(paths, system.code_bytes, load_batch_size) + + +def _residue_code_batches(paths, code_bytes: int, load_batch_size: int) -> Iterator[ResidueCodeBatch]: + for path in paths: + for batch in iter_payload_batches(path, RESIDUE_CODES, load_batch_size): + sizes = np.diff(batch.offsets) + bad = np.flatnonzero(sizes % code_bytes) + if bad.size: + raise ResidueShardError(f"{path.name}: {bad.size} residue-code payload(s) are not a multiple of " + f"{code_bytes} bytes (e.g. {batch.ids[int(bad[0])]!r}, " + f"{int(sizes[bad[0]])} bytes)") + codes = batch.data.reshape(-1, code_bytes) + bounds = batch.offsets[1:-1] // code_bytes + yield ResidueCodeBatch(batch.ids, np.split(codes, bounds) if len(batch.ids) else [], batch.seq_sha1s) diff --git a/src/foldmatch/search/scoring_system.py b/src/foldmatch/search/scoring_system.py new file mode 100644 index 0000000..83757c9 --- /dev/null +++ b/src/foldmatch/search/scoring_system.py @@ -0,0 +1,916 @@ +"""Versioned esm-codes scoring systems (plan section 8 P4; method spec S1-S16, ``docs/esm-codes-method.md``). + +An :class:`EsmCodesSystem` names every number and convention that shapes esm-codes output: the ESM3 weights and +embedding tap, the encoders and transform (by content digest), the code layout, the Smith-Waterman setting, the panel, +the E-value table and its rules, the calibrated domain, and the Hugging Face bundle (repo, revision and file sha256s) +that holds the weights, the panel and the table. It is pure metadata: no torch, no numpy and no I/O. + +:func:`prepare` turns a system into a :class:`PreparedSystem`: it fetches the bundle from the registry pin +(:func:`foldmatch.utils.model.get_esm_codes_bundle`, sha256-verified) and refuses it unless ``bundle.json`` and the +scoring npz agree with the system exactly. :meth:`PreparedSystem.for_worker` gives Stage-2 workers a picklable +:class:`WorkerSpec` (paths and scalars). Importing this module loads neither torch nor numpy. + +Principles (plan section 1): + +* **Versioned.** Registry entries are immutable. Any change, even a statistics-only one, is a new system name (D17); + golden digests in ``tests/test_scoring_system.py`` fail CI on any edit of a frozen entry. +* **One source of truth.** The bundle identity (repo, revision, file sha256s) exists only in the registry entry; + nothing else in FoldMatch names a bundle revision. +* **Refuse, don't degrade.** ``__post_init__`` refuses inconsistent values; ``from_json`` refuses unknown or missing + fields; :func:`get` refuses unknown names and lists the known ones. + +Digests: :meth:`EsmCodesSystem.digest` covers every field; :meth:`EsmCodesSystem.encoding_digest` covers only the +fields that determine the stored residue codes (``ENCODING_FIELDS``), so a statistics-only system can share a +database's codes but never its digest. Both are the first 16 hex digits of a sha256 over a tag line and the canonical +JSON (sorted keys, no whitespace, shortest round-trip floats), so they are identical on every platform. +""" +from __future__ import annotations + +import dataclasses +import hashlib +import json +import math +import re +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any, Dict, Mapping, Tuple + +SYSTEM_DIGEST_TAG = "foldmatch-esm-codes-system/1" +ENCODING_DIGEST_TAG = "foldmatch-esm-codes-encoding/1" +DIGEST_HEX_DIGITS = 16 + +#: The system every ``--residue-codes`` build uses (v1 has no user choice). Queries always use the system named in the +#: database manifest, never this default. +DEFAULT_ESM_CODES = "esm-codes-v1" + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_COMMIT_HEX = re.compile(r"[0-9a-f]{40}") +_NAME = re.compile(r"[a-z0-9][a-z0-9._-]*") +_REPO = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*") +_FILE = re.compile(r"[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*") +TABLE_SIZE = 6 + + +class UnknownScoringSystemError(ValueError): + """No registry entry has this name (for a database: it was built by a newer FoldMatch).""" + + +class BundleUnavailableError(ValueError): + """A bundle file is neither in the Hugging Face cache nor downloadable (offline mode, Hub unreachable).""" + + +class BundleMismatchError(ValueError): + """The bundle does not match its registry entry: a file sha256, a ``bundle.json`` / ``meta_json`` field, the + panel or the table differs, or a file is missing at the pinned revision. Never worked around.""" + + +def is_commit(revision) -> bool: + """A full 40-hex lowercase commit hash (what every bundle revision must be).""" + return isinstance(revision, str) and _COMMIT_HEX.fullmatch(revision) is not None + + +# ===================================================================================================================== +# value checks +# ===================================================================================================================== +def _str(owner: str, name: str, value) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{owner}.{name} must be a non-empty string, got {value!r}") + return value + + +def _hex(owner: str, name: str, value, pattern: re.Pattern, what: str) -> str: + if not isinstance(value, str) or not pattern.fullmatch(value): + raise ValueError(f"{owner}.{name} must be {what}, got {value!r}") + return value + + +def _int(owner: str, name: str, value) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{owner}.{name} must be an integer, got {value!r}") + return int(value) + + +def _float(owner: str, name: str, value) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{owner}.{name} must be a number, got {value!r}") + value = float(value) + if not math.isfinite(value): + raise ValueError(f"{owner}.{name} must be finite, got {value!r}") + return value + + +def _floats(owner: str, name: str, value, size: int) -> Tuple[float, ...]: + if isinstance(value, (str, bytes)) or not hasattr(value, "__iter__"): + raise ValueError(f"{owner}.{name} must be a sequence of {size} numbers, got {value!r}") + out = tuple(_float(owner, f"{name}[{i}]", v) for i, v in enumerate(value)) + if len(out) != size: + raise ValueError(f"{owner}.{name} must hold {size} numbers, got {len(out)}") + return out + + +def _nested(cls, owner: str, name: str, value): + if isinstance(value, cls): + return value + if isinstance(value, Mapping): + return cls.from_dict(value, where=f"{owner}.{name}") + raise ValueError(f"{owner}.{name} must be a {cls.__name__} or a mapping, got {value!r}") + + +def _set(obj, name: str, value): + object.__setattr__(obj, name, value) + + +def _fields_from(cls, data, where: str) -> dict: + """The mapping's fields for ``cls``, refusing unknown and missing ones.""" + if not isinstance(data, Mapping): + raise ValueError(f"{where}: expected a JSON object, got {type(data).__name__}") + names = [f.name for f in dataclasses.fields(cls)] + unknown = sorted(set(data) - set(names)) + missing = [n for n in names if n not in data] + if unknown: + raise ValueError(f"{where}: unknown field(s) {unknown}; {cls.__name__} has {names}") + if missing: + raise ValueError(f"{where}: missing field(s) {missing}") + return {n: data[n] for n in names} + + +def _plain(value): + """JSON form of a field value: nested dataclasses become objects, tuples become lists.""" + if dataclasses.is_dataclass(value): + return {f.name: _plain(getattr(value, f.name)) for f in dataclasses.fields(value)} + if isinstance(value, tuple): + return [_plain(v) for v in value] + return value + + +def canonical_json(obj) -> str: + """Sorted keys, no whitespace, ASCII, no NaN / Inf; Python's float repr is the shortest round-trip form, so the + text (and every digest over it) is the same on every platform.""" + return json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def _digest(tag: str, obj) -> str: + return hashlib.sha256(tag.encode("ascii") + b"\n" + canonical_json(obj).encode("ascii")).hexdigest()[ + :DIGEST_HEX_DIGITS] + + +# ===================================================================================================================== +# nested parts +# ===================================================================================================================== +@dataclass(frozen=True) +class Normalisation: + """S7 rowcol z: ``z_ij = 0.5 ((s_ij - mu_i) / (sigma_i + eps) + (s_ij - nu_j) / (tau_j + eps))`` with ``std`` + standard deviations; a row or column whose std is below ``degenerate_std_lt`` contributes exactly 0.""" + name: str + std: str + eps: float + degenerate_std_lt: float + + def __post_init__(self): + o = "Normalisation" + _set(self, "name", _str(o, "name", self.name)) + _set(self, "std", _str(o, "std", self.std)) + for n in ("eps", "degenerate_std_lt"): + v = _float(o, n, getattr(self, n)) + if v <= 0: + raise ValueError(f"{o}.{n} must be > 0, got {v!r}") + _set(self, n, v) + + @classmethod + def from_dict(cls, data, where: str = "Normalisation") -> "Normalisation": + return cls(**_fields_from(cls, data, where)) + + +@dataclass(frozen=True) +class CalibratedDomain: + """S13 / D14 / D15: a hit is inside the calibrated domain (``ecal = 1``) only when the query length is in + ``[query_min_length, query_max_length]``, the target length is at most ``target_max_length``, both sequences have + at most ``max_nonstandard_fraction`` residues outside the 20 standard amino acids, and the target has at least + ``target_min_distinct_codes`` distinct residue codes.""" + query_min_length: int + query_max_length: int + target_max_length: int + max_nonstandard_fraction: float + target_min_distinct_codes: int + + def __post_init__(self): + o = "CalibratedDomain" + for n in ("query_min_length", "query_max_length", "target_max_length", "target_min_distinct_codes"): + v = _int(o, n, getattr(self, n)) + if v < 1: + raise ValueError(f"{o}.{n} must be >= 1, got {v!r}") + _set(self, n, v) + if self.query_min_length > self.query_max_length: + raise ValueError(f"{o}: query_min_length {self.query_min_length} > query_max_length " + f"{self.query_max_length}") + f = _float(o, "max_nonstandard_fraction", self.max_nonstandard_fraction) + if not 0.0 <= f <= 1.0: + raise ValueError(f"{o}.max_nonstandard_fraction must be in [0, 1], got {f!r}") + _set(self, "max_nonstandard_fraction", f) + + @classmethod + def from_dict(cls, data, where: str = "CalibratedDomain") -> "CalibratedDomain": + return cls(**_fields_from(cls, data, where)) + + +# ===================================================================================================================== +# the system +# ===================================================================================================================== +IDENTITY_FIELDS = ("name", "calibration_version") +#: The fields that determine the stored residue codes' bytes (S1-S4); :meth:`EsmCodesSystem.encoding_digest`. The +#: encoder architecture and precision are among them: with the same weights, GELU-tanh instead of exact GELU, or a +#: different precision, changes the codes (P2 NotRecoverableFromWeightsTest), and the content digests cover only the +#: weight tensors. +ENCODING_FIELDS = ( + "esm_repo", "esm_weights_sha256", "embedding_tap", "input_normalisation", + "input_cast", "transform_sha256", "transform_output_cast", + "target_encoder_sha256", "encoder_architecture", "encoder_dtype", "code_bits", "code_bytes", "bitorder", + "sign_rule", +) +SCORING_FIELDS = ( + "query_encoder_sha256", "query_dtype", "query_prep", "query_prep_eps", "similarity", "determinism", + "normalisation", "offset", "gap_open", "gap_extend", "dp", "score_form", "score_diff", +) +STATISTICS_FIELDS = ( + "panel_sha256", "panel_size", "panel_residues", "panel_quantile", "quantile_method", + "bin_centres", "lam", "lnK", "low_m_rule", "high_m_rule", "calibrated_domain", "degenerate_query_rule", +) +BUNDLE_FIELDS = ("bundle_repo", "bundle_revision", "bundle_files") + +_STR_FIELDS = ("calibration_version", "esm_repo", "embedding_tap", "input_normalisation", "input_cast", + "transform_output_cast", "encoder_architecture", "encoder_dtype", "bitorder", "sign_rule", + "query_dtype", "query_prep", "similarity", "determinism", "dp", "score_form", "score_diff", + "quantile_method", "low_m_rule", "high_m_rule", "degenerate_query_rule") +_SHA_FIELDS = ("esm_weights_sha256", "transform_sha256", "target_encoder_sha256", "query_encoder_sha256", + "panel_sha256") + + +@dataclass(frozen=True) +class EsmCodesSystem: + """One esm-codes scoring system (method spec S1-S16). Every field is required; see the module docstring. + + Construction normalises (sequences become tuples, integral numbers of float fields become floats, ``bundle_files`` + becomes ``((name, sha256), ...)`` sorted by name, nested mappings become :class:`Normalisation` / + :class:`CalibratedDomain`) and refuses inconsistent values with :class:`ValueError`. + """ + # identity + name: str + calibration_version: str + # encoding (S1-S4): what the stored residue codes depend on + esm_repo: str + esm_weights_sha256: str + embedding_tap: str + input_normalisation: str + input_cast: str + transform_sha256: str + transform_output_cast: str + target_encoder_sha256: str + encoder_architecture: str # S3: the encoder module (bundle.json encoding.architecture.name) + encoder_dtype: str # S3: "float32" = IEEE fp32, no TF32, no autocast (encoding.encoder.dtype) + code_bits: int + code_bytes: int + bitorder: str + sign_rule: str + # scoring (S5-S9) + query_encoder_sha256: str + query_dtype: str + query_prep: str + query_prep_eps: float + similarity: str + determinism: str + normalisation: Normalisation + offset: float + gap_open: float + gap_extend: float + dp: str + score_form: str + score_diff: str + # panel and statistics (S10-S13b) + panel_sha256: str + panel_size: int + panel_residues: int + panel_quantile: float + quantile_method: str + bin_centres: Tuple[float, ...] + lam: Tuple[float, ...] + lnK: Tuple[float, ...] + low_m_rule: str + high_m_rule: str + calibrated_domain: CalibratedDomain + degenerate_query_rule: str + # bundle: the only place a bundle repo, revision or file sha256 is named + bundle_repo: str + bundle_revision: str + bundle_files: Tuple[Tuple[str, str], ...] + + def __post_init__(self): + o = "EsmCodesSystem" + name = self.name + if not isinstance(name, str) or not _NAME.fullmatch(name): + raise ValueError(f"{o}.name must match {_NAME.pattern}, got {name!r}") + for n in _STR_FIELDS: + _set(self, n, _str(o, n, getattr(self, n))) + for n in _SHA_FIELDS: + _set(self, n, _hex(o, n, getattr(self, n), _SHA256, "a lowercase 64-hex sha256")) + for n in ("code_bits", "code_bytes", "panel_size", "panel_residues"): + v = _int(o, n, getattr(self, n)) + if v < 1: + raise ValueError(f"{o}.{n} must be >= 1, got {v!r}") + _set(self, n, v) + if self.code_bits != 8 * self.code_bytes: + raise ValueError(f"{o}: code_bits {self.code_bits} != 8 * code_bytes {self.code_bytes}") + if self.panel_residues < self.panel_size: + raise ValueError(f"{o}: panel_residues {self.panel_residues} < panel_size {self.panel_size}") + for n in ("query_prep_eps", "offset", "gap_open", "gap_extend", "panel_quantile"): + _set(self, n, _float(o, n, getattr(self, n))) + if self.query_prep_eps <= 0: + raise ValueError(f"{o}.query_prep_eps must be > 0, got {self.query_prep_eps!r}") + if not self.gap_open >= self.gap_extend >= 0.0: + raise ValueError(f"{o}: needs gap_open >= gap_extend >= 0, got gap_open={self.gap_open!r}, " + f"gap_extend={self.gap_extend!r}") + if not 0.0 < self.panel_quantile < 1.0: + raise ValueError(f"{o}.panel_quantile must be in (0, 1), got {self.panel_quantile!r}") + for n in ("bin_centres", "lam", "lnK"): + _set(self, n, _floats(o, n, getattr(self, n), TABLE_SIZE)) + c = self.bin_centres + if c[0] <= 0 or any(b <= a for a, b in zip(c, c[1:])): + raise ValueError(f"{o}.bin_centres must be positive and strictly increasing, got {c!r}") + if any(v <= 0 for v in self.lam): + raise ValueError(f"{o}.lam must be > 0, got {self.lam!r}") + _set(self, "normalisation", _nested(Normalisation, o, "normalisation", self.normalisation)) + _set(self, "calibrated_domain", _nested(CalibratedDomain, o, "calibrated_domain", self.calibrated_domain)) + if not isinstance(self.bundle_repo, str) or not _REPO.fullmatch(self.bundle_repo): + raise ValueError(f"{o}.bundle_repo must be a Hugging Face repo id 'owner/name', got {self.bundle_repo!r}") + _set(self, "bundle_revision", _hex(o, "bundle_revision", self.bundle_revision, _COMMIT_HEX, + "a lowercase 40-hex commit (never a branch, a tag or None)")) + _set(self, "bundle_files", _bundle_files(o, self.bundle_files)) + + # --------------------------------------------------------------------------------------------------------------- + def to_dict(self) -> Dict[str, Any]: + """JSON-ready dict: nested parts as objects, tuples as lists, ``bundle_files`` as ``[[name, sha256], ...]``.""" + return {f.name: _plain(getattr(self, f.name)) for f in dataclasses.fields(self)} + + def to_json(self) -> str: + """Canonical JSON text (:func:`canonical_json`); ``from_json(s.to_json()) == s``.""" + return canonical_json(self.to_dict()) + + @classmethod + def from_dict(cls, data, where: str = "EsmCodesSystem") -> "EsmCodesSystem": + return cls(**_fields_from(cls, data, where)) + + @classmethod + def from_json(cls, text) -> "EsmCodesSystem": + """From JSON text or an already-parsed mapping; unknown and missing fields (at any level) are refused.""" + data = json.loads(text) if isinstance(text, (str, bytes, bytearray)) else text + return cls.from_dict(data) + + def digest(self) -> str: + """16 hex digits over every field.""" + return _digest(SYSTEM_DIGEST_TAG, self.to_dict()) + + def encoding_digest(self) -> str: + """16 hex digits over ``ENCODING_FIELDS`` only (what the stored residue codes depend on).""" + d = self.to_dict() + return _digest(ENCODING_DIGEST_TAG, {k: d[k] for k in ENCODING_FIELDS}) + + def bundle_sha256(self, filename: str) -> str: + """The pinned sha256 of one bundle file.""" + for name, sha in self.bundle_files: + if name == filename: + return sha + raise KeyError(f"{self.name}: {filename!r} is not a bundle file ({[n for n, _ in self.bundle_files]})") + + +BUNDLE_JSON = "bundle.json" + + +def _bundle_files(o: str, value) -> Tuple[Tuple[str, str], ...]: + if isinstance(value, Mapping): + items = list(value.items()) + elif isinstance(value, (str, bytes)) or not hasattr(value, "__iter__"): + raise ValueError(f"{o}.bundle_files must be (name, sha256) pairs or a mapping, got {value!r}") + else: + items = [] + for pair in value: + if isinstance(pair, (str, bytes)) or not hasattr(pair, "__iter__") or len(tuple(pair)) != 2: + raise ValueError(f"{o}.bundle_files entries must be (name, sha256) pairs, got {pair!r}") + items.append(tuple(pair)) + out = [] + for name, sha in items: + if not isinstance(name, str) or not _FILE.fullmatch(name) or ".." in name.split("/"): + raise ValueError(f"{o}.bundle_files: {name!r} is not a relative file name in the repo") + out.append((name, _hex(o, f"bundle_files[{name!r}]", sha, _SHA256, "a lowercase 64-hex sha256"))) + names = [n for n, _ in out] + if len(set(names)) != len(names): + raise ValueError(f"{o}.bundle_files names a file twice: {sorted(names)}") + if BUNDLE_JSON not in names: + raise ValueError(f"{o}.bundle_files must include {BUNDLE_JSON}, got {sorted(names)}") + return tuple(sorted(out)) + + +_PARTITION = IDENTITY_FIELDS + ENCODING_FIELDS + SCORING_FIELDS + STATISTICS_FIELDS + BUNDLE_FIELDS +if sorted(_PARTITION) != sorted(f.name for f in dataclasses.fields(EsmCodesSystem)) or \ + len(set(_PARTITION)) != len(_PARTITION): + raise AssertionError("the field groups must partition EsmCodesSystem's fields") + + +# ===================================================================================================================== +# registry +# ===================================================================================================================== +def _registry(*systems: EsmCodesSystem) -> Mapping[str, EsmCodesSystem]: + out: Dict[str, EsmCodesSystem] = {} + for s in systems: + if s.name in out: + raise AssertionError(f"registry: {s.name} registered twice") + out[s.name] = s + return MappingProxyType(out) + + +#: Every system this FoldMatch can build and query, by name. Entries are immutable once frozen (golden digests in +#: tests/test_scoring_system.py); a change of any value is a new system name, never an edit (D17). +REGISTRY: Mapping[str, EsmCodesSystem] = _registry( + # esm-codes-v1: model M8_s0 (L2m_B8_s0), setting D, S = S_f - S_r, UniRef50 1,000-member panel, M3t999 table. + # FROZEN 2026-10-14 (after P0.6, P0.7 and the upload of the bundle to rcsb/foldmatch-esm-codes). Every value + # below equals the bundle's bundle.json / meta_json (prepare() checks it); the content digests are + # foldmatch-tensor-digest/1; the table is panel/fit2/results/ship_final.json (md5 c51735f9...). + EsmCodesSystem( + name="esm-codes-v1", + calibration_version="foldmatch-evalue-M8_s0-D_fmr-pnlU50s1000-M3t999-v1", + esm_repo="rcsb/rcsb-esm", + esm_weights_sha256="5ead5a135c658068db6a4f1b933e72d6110992c4668822e1c0e2dcc53e38acd9", + embedding_tap="per_residue_embedding", + input_normalisation="ascii_upper_nows_v1", + input_cast="float16", + transform_sha256="e8989b1cefc839784555ff973dabb9a2f7ef4bdf81ac78ad47767714227c3336", + transform_output_cast="float16", + target_encoder_sha256="58da263e005a79fd1689b0f5605adbb85b51532bca99e4ef7330916a50df6686", + encoder_architecture="esm-codes-mlp-v1", + encoder_dtype="float32", + code_bits=64, + code_bytes=8, + bitorder="big", + sign_rule="bit_k = (h_k >= 0)", + query_encoder_sha256="d8bed195068be23a60586632fa471a08f3a767fe8cf34b06aa6e733d90cd3bf8", + query_dtype="float16", + query_prep="fp64_seq_k_asc", + query_prep_eps=1e-12, + similarity="pm1_fp32_blas_gemm_per_pair", + determinism="tolerance_s16_v1", + normalisation=Normalisation(name="rowcol_z", std="population", eps=1e-08, degenerate_std_lt=1e-06), + offset=1.0, + gap_open=5.0, + gap_extend=0.5, + dp="fmsw/1", + score_form="fwd_minus_rev_view", + score_diff="float64", + panel_sha256="7aea2b00a24ee41c5a9f9adb91a5723ee02dff12dca23998767ab87da4b9dcf7", + panel_size=1000, + panel_residues=355626, + panel_quantile=0.9, + quantile_method="linear", + bin_centres=(32.58437573116228, 50.34161310783617, 80.8700900525453, 149.85910754690784, + 276.49344608131264, 594.9175724688861), + lam=(0.18775162858061797, 0.1227859873586807, 0.10488523441851651, 0.08222582327223689, + 0.04784353519669357, 0.041619283794126286), + lnK=(-13.42382627313042, -13.969792295008805, -13.73906582316063, -13.454607332784638, + -14.484498724905178, -15.878915750325046), + low_m_rule="clamp", + high_m_rule="clamp", + calibrated_domain=CalibratedDomain(query_min_length=21, query_max_length=966, target_max_length=3000, + max_nonstandard_fraction=0.01, target_min_distinct_codes=2), + degenerate_query_rule="no_hits", + bundle_repo="rcsb/foldmatch-esm-codes", + bundle_revision="3a14f46b72f5971f47203a57ba418e87054142bd", + bundle_files=( + ("LICENSE", "49f7a86fb623fa33dcb951fd53f840795989eadb9f03dda5c0587a2f451258d1"), + ("README.md", "70bd8e84c48b2a511ec3b37cededa7eaa1f46e7b3da6ea86f5881126bc02416c"), + ("bundle.json", "dcd29bddbff0bb7dc93209bbe292d422d7bff21dd48c80f6e1f6b74f23789fe8"), + ("esm_codes_v1_encoders.pt", "1696ab91089ece8bce787d38b9d4d68bbc8ca4abff06f836f6e9363a23a1c02b"), + ("esm_codes_v1_scoring.npz", "a5bebd626ec874626fb9372a63d10c32a001ed09c358717448bc9e1fd0ddedd6"), + ), + ), +) + + +def known_names() -> Tuple[str, ...]: + return tuple(sorted(REGISTRY)) + + +def get(name: str) -> EsmCodesSystem: + """The registered system ``name``; :class:`UnknownScoringSystemError` (listing the known names) otherwise.""" + system = REGISTRY.get(name) if isinstance(name, str) else None + if system is None: + known = ", ".join(known_names()) or "none" + raise UnknownScoringSystemError( + f"unknown esm-codes scoring system {name!r}; this FoldMatch knows: {known}. A database built by a newer " + "FoldMatch needs a FoldMatch upgrade.") + return system + + +def registered(system: EsmCodesSystem) -> EsmCodesSystem: + """``system``, refused unless it is exactly this FoldMatch's registry entry of its name. + + A system that comes from elsewhere (parsed from a database manifest, say) and differs from the entry of its name + in any value, the bundle repo, revision and file sha256s included, is not that system: nothing is fetched or run + for it. :class:`UnknownScoringSystemError` names the differing fields (or lists the known names).""" + if not isinstance(system, EsmCodesSystem): + raise TypeError(f"expected an EsmCodesSystem, got {type(system).__name__}") + entry = get(system.name) + if entry != system: + diff = [f.name for f in dataclasses.fields(EsmCodesSystem) if getattr(entry, f.name) != getattr(system, f.name)] + raise UnknownScoringSystemError( + f"this {system.name} differs from this FoldMatch's registry entry {system.name} in {diff}: only the " + "registered system is used, exactly as registered (any change is a new system name)") + return system + + +# ===================================================================================================================== +# what this FoldMatch implements +# ===================================================================================================================== +class UnsupportedSystemError(ValueError): + """The system names a method this FoldMatch does not implement (a newer DP, rule or encoding).""" + + +BUNDLE_FORMAT = "foldmatch-esm-codes-bundle/1" +ENCODERS_FORMAT = "foldmatch-esm-codes-encoders/1" # = foldmatch.model.residue_code_encoder.ENCODERS_FORMAT +SCORING_FORMAT = "foldmatch-esm-codes-scoring/1" + +#: Method values hard-coded by this FoldMatch's encoder module, kernel (fmsw/1) and statistics. A system naming +#: anything else is refused by :func:`check_implemented`, never run with this code's arithmetic. +IMPLEMENTED = MappingProxyType({ + "embedding_tap": "per_residue_embedding", + "input_normalisation": "ascii_upper_nows_v1", + "input_cast": "float16", + "transform_output_cast": "float16", + "code_bits": 64, + "code_bytes": 8, + "bitorder": "big", + "sign_rule": "bit_k = (h_k >= 0)", + "encoder_architecture": "esm-codes-mlp-v1", # = a residue_code_encoder.ARCHITECTURES key + "encoder_dtype": "float32", # = residue_code_encoder.IMPLEMENTED_CONVENTIONS encoder.dtype + "query_dtype": "float16", + "query_prep": "fp64_seq_k_asc", + "query_prep_eps": 1e-12, + "similarity": "pm1_fp32_blas_gemm_per_pair", + "determinism": "tolerance_s16_v1", + "normalisation": Normalisation(name="rowcol_z", std="population", eps=1e-08, degenerate_std_lt=1e-06), + "score_form": "fwd_minus_rev_view", + "score_diff": "float64", + "quantile_method": "linear", + "degenerate_query_rule": "no_hits", +}) + + +def check_implemented(system: EsmCodesSystem) -> None: + """Refuse (:class:`UnsupportedSystemError`) a system this FoldMatch cannot run exactly: its DP must be the + kernel's (``swkern.DP_ID``, ``fmsw/1``) with setting D (``swkern.SETTING_D``), its m rules ones that + :mod:`code_statistics` implements, and every value in ``IMPLEMENTED`` must match.""" + from foldmatch.search import code_statistics, swkern # numpy + ctypes only; loads no library + bad = [] + if system.dp != swkern.DP_ID: + bad.append(f"dp={system.dp!r} (this kernel is {swkern.DP_ID!r})") + setting = (system.offset, system.gap_open, system.gap_extend) + kernel_setting = (swkern.SETTING_D.offset, swkern.SETTING_D.gap_open, swkern.SETTING_D.gap_extend) + if setting != kernel_setting: + bad.append(f"(offset, gap_open, gap_extend)={setting} (fmsw/1 setting D is {kernel_setting})") + for rule in ("low_m_rule", "high_m_rule"): + if getattr(system, rule) not in code_statistics.IMPLEMENTED_M_RULES: + bad.append(f"{rule}={getattr(system, rule)!r} (implemented: {code_statistics.IMPLEMENTED_M_RULES})") + if (system.code_bits, system.code_bytes) != (swkern.DIM, swkern.CODE_BYTES): + bad.append(f"code layout {system.code_bits} bits / {system.code_bytes} bytes (the kernel reads " + f"{swkern.DIM} / {swkern.CODE_BYTES})") + for name, value in IMPLEMENTED.items(): + got = getattr(system, name) + if got != value or type(got) is not type(value): + bad.append(f"{name}={got!r} (implemented: {value!r})") + if bad: + raise UnsupportedSystemError(f"{system.name}: this FoldMatch does not implement " + "; ".join(bad)) + + +# ===================================================================================================================== +# prepare(): the verified bundle of a system +# ===================================================================================================================== +def _both(field_path, doc_path): + return field_path, doc_path, doc_path + + +#: (system field path, bundle.json path, scoring npz meta_json path or None): every system value that the bundle +#: also states. prepare() requires each pair to be equal exactly (numbers as float64, strings as text). +DOCUMENT_FIELDS = ( + _both(("name",), ("system",)), + _both(("calibration_version",), ("calibration_version",)), + _both(("esm_repo",), ("esm3", "repo")), + _both(("esm_weights_sha256",), ("esm3", "weights_sha256")), + _both(("embedding_tap",), ("esm3", "embedding_tap")), + _both(("input_normalisation",), ("esm3", "input_normalisation")), + (("input_cast",), ("encoding", "input_cast"), None), + (("transform_sha256",), ("content_digests", "transform_sha256"), None), + (("transform_output_cast",), ("encoding", "transform_output_cast"), None), + (("target_encoder_sha256",), ("content_digests", "target_encoder_sha256"), None), + (("encoder_architecture",), ("encoding", "architecture", "name"), None), + (("encoder_dtype",), ("encoding", "encoder", "dtype"), None), + (("code_bits",), ("encoding", "code_bits"), ("code", "bits")), + (("code_bytes",), ("encoding", "code_bytes"), ("code", "bytes")), + (("bitorder",), ("encoding", "bitorder"), ("code", "bitorder")), + (("sign_rule",), ("encoding", "sign_rule"), ("code", "sign_rule")), + (("query_encoder_sha256",), ("content_digests", "query_encoder_sha256"), None), + (("query_dtype",), ("encoding", "query_dtype"), None), + _both(("query_prep",), ("scoring", "query_prep")), + _both(("query_prep_eps",), ("scoring", "query_prep_eps")), + _both(("similarity",), ("scoring", "similarity")), + _both(("determinism",), ("scoring", "determinism")), + *(_both(("normalisation", k), ("scoring", "normalisation", k)) for k in ("name", "std", "eps", + "degenerate_std_lt")), + *(_both((k,), ("scoring", k)) for k in ("offset", "gap_open", "gap_extend", "dp", "score_form", "score_diff")), + _both(("panel_sha256",), ("panel", "panel_sha256")), + (("panel_sha256",), ("content_digests", "panel_sha256"), None), + _both(("panel_size",), ("panel", "size")), + _both(("panel_residues",), ("panel", "residues")), + _both(("panel_quantile",), ("panel", "quantile")), + _both(("quantile_method",), ("panel", "quantile_method")), + *(_both((k,), ("statistics", k)) for k in ("bin_centres", "lam", "lnK", "low_m_rule", "high_m_rule")), + *(_both(("calibrated_domain", k), ("calibrated_domain", k)) for k in ( + "query_min_length", "query_max_length", "target_max_length", "max_nonstandard_fraction", + "target_min_distinct_codes")), + _both(("degenerate_query_rule",), ("degenerate_query_rule",)), + (("bundle_repo",), ("hf_repo",), None), +) +_MISSING = object() + + +def _lookup(obj, path): + for key in path: + if dataclasses.is_dataclass(obj): + obj = getattr(obj, key) + elif isinstance(obj, Mapping) and key in obj: + obj = obj[key] + else: + return _MISSING + return obj + + +def _same(want, got) -> bool: + """Exact equality: strings as text, integers as integers, floats as float64 (an integral JSON number equals the + same float), sequences element by element. bool never equals a number.""" + if isinstance(want, bool) or isinstance(got, bool): + return type(want) is type(got) and want == got + if isinstance(want, str): + return isinstance(got, str) and got == want + if isinstance(want, int): + return isinstance(got, int) and got == want + if isinstance(want, float): + return isinstance(got, (int, float)) and float(got) == want + if isinstance(want, tuple): + return isinstance(got, (list, tuple)) and len(got) == len(want) and all(map(_same, want, got)) + return want == got + + +def _compare(system: EsmCodesSystem, doc, column: int, what: str, problems: list): + for entry in DOCUMENT_FIELDS: + doc_path = entry[column] + if doc_path is None: + continue + want, got = _lookup(system, entry[0]), _lookup(doc, doc_path) + if got is _MISSING: + problems.append(f"{what} has no {'.'.join(doc_path)}") + elif not _same(want, got): + problems.append(f"{what} {'.'.join(doc_path)} = {got!r}, the system has {'.'.join(entry[0])} = " + f"{want!r}") + + +def _table_digest(system: EsmCodesSystem) -> str: + import numpy as np + from foldmatch.utils.tensor_digest import tensor_digest + return tensor_digest({k: np.array(getattr(system, k), np.float64) for k in ("bin_centres", "lam", "lnK")}) + + +def _panel_digest(codes, offsets) -> str: + from foldmatch.utils.tensor_digest import tensor_digest + return tensor_digest({"codes": codes, "offsets": offsets}) + + +def _bundle_roles(system: EsmCodesSystem, bundle: Mapping, problems: list) -> Dict[str, str]: + """bundle.json ``files``: each listed file pinned with the same sha256 in the system; the encoders and scoring + files identified by their format tags.""" + files = bundle.get("files") + roles: Dict[str, str] = {} + if not isinstance(files, Mapping) or not files: + problems.append("bundle.json has no files") + return roles + pinned = dict(system.bundle_files) + for name, info in files.items(): + sha = info.get("sha256") if isinstance(info, Mapping) else None + if name not in pinned: + problems.append(f"bundle.json lists {name}, which the system does not pin") + elif sha != pinned[name]: + problems.append(f"bundle.json gives {name} sha256 {sha}, the system pins {pinned[name]}") + fmt = info.get("format") if isinstance(info, Mapping) else None + if fmt in (ENCODERS_FORMAT, SCORING_FORMAT): + if fmt in roles: + problems.append(f"bundle.json lists two {fmt} files") + roles[fmt] = name + for fmt in (ENCODERS_FORMAT, SCORING_FORMAT): + if fmt not in roles: + problems.append(f"bundle.json lists no {fmt} file") + return roles + + +def _load_scoring_npz(path) -> Dict[str, Any]: + import numpy as np + with np.load(path, allow_pickle=False) as z: + return {k: z[k] for k in z.files} + + +def _check_scoring(system: EsmCodesSystem, arrays: Mapping[str, Any], problems: list): + import numpy as np + meta_raw = arrays.get("meta_json") + try: + meta = json.loads(str(meta_raw)) if meta_raw is not None else None + except ValueError: + meta = None + if not isinstance(meta, Mapping): + problems.append("the scoring npz has no readable meta_json") + else: + if meta.get("format") != SCORING_FORMAT: + problems.append(f"the scoring npz format is {meta.get('format')!r}, expected {SCORING_FORMAT!r}") + _compare(system, meta, 2, "meta_json", problems) + for k in ("bin_centres", "lam", "lnK"): + a = arrays.get(k) + want = np.array(getattr(system, k), np.float64) + if a is None or a.dtype != np.float64 or a.shape != want.shape: + problems.append(f"the scoring npz {k} is not float64 {list(want.shape)}") + elif not np.array_equal(a.view(np.uint64), want.view(np.uint64)): + first = int(np.flatnonzero(a.view(np.uint64) != want.view(np.uint64))[0]) + problems.append(f"the scoring npz {k} = {a.tolist()} differs from the system's {want.tolist()} at index " + f"{first} (exact float64 comparison)") + codes, offsets, lengths = (arrays.get(k) for k in ("panel_codes", "panel_offsets", "panel_lengths")) + shape_ok = (codes is not None and codes.dtype == np.uint8 and codes.shape == (system.panel_residues, + system.code_bytes) + and offsets is not None and offsets.dtype == np.int64 and offsets.shape == (system.panel_size + 1,)) + if not shape_ok: + problems.append(f"the scoring npz panel is not uint8 [{system.panel_residues}, {system.code_bytes}] codes " + f"with int64 [{system.panel_size + 1}] offsets") + return + if offsets[0] != 0 or offsets[-1] != system.panel_residues or np.any(np.diff(offsets) < 1): + problems.append("the scoring npz panel offsets are not 0 .. panel_residues with every member non-empty") + if lengths is None or lengths.shape != (system.panel_size,) or not np.array_equal(lengths, np.diff(offsets)): + problems.append("the scoring npz panel_lengths do not equal diff(panel_offsets)") + got = _panel_digest(codes, offsets) + if got != system.panel_sha256: + problems.append(f"the panel content digest is {got}, the system has panel_sha256 {system.panel_sha256}") + + +def _raise_problems(system: EsmCodesSystem, problems: list): + if problems: + shown = problems[:10] + ([f"... and {len(problems) - 10} more"] if len(problems) > 10 else []) + raise BundleMismatchError(f"{system.name}: the bundle {system.bundle_repo}@{system.bundle_revision} does " + "not match the registry entry: " + "; ".join(shown)) + + +@dataclass(frozen=True) +class WorkerSpec: + """What a Stage-2 worker needs, as picklable paths and scalars (no arrays, no torch): the scoring npz (sha256 + re-checked by :meth:`load_panel`), the DP setting, the panel quantile and the E-value table with its rules (so a + WorkerSpec is also a ``table`` for :mod:`code_statistics`).""" + system_name: str + system_digest: str + scoring_npz: str + scoring_sha256: str + dp: str + offset: float + gap_open: float + gap_extend: float + panel_size: int + panel_residues: int + panel_quantile: float + quantile_method: str + bin_centres: Tuple[float, ...] + lam: Tuple[float, ...] + lnK: Tuple[float, ...] + low_m_rule: str + high_m_rule: str + calibrated_domain: CalibratedDomain + degenerate_query_rule: str + + def dp_params(self): + from foldmatch.search import swkern + return swkern.DPParams(offset=self.offset, gap_open=self.gap_open, gap_extend=self.gap_extend) + + def load_panel(self): + """``(codes uint8 [panel_residues, 8], offsets int64 [panel_size + 1])`` from the scoring npz, after checking + its sha256 again (the file may have changed since prepare()).""" + from foldmatch.utils.model import _file_sha256 + got = _file_sha256(self.scoring_npz) + if got != self.scoring_sha256: + raise BundleMismatchError(f"{self.scoring_npz} has sha256 {got}, expected {self.scoring_sha256} " + f"({self.system_name})") + a = _load_scoring_npz(self.scoring_npz) + codes, offsets = a["panel_codes"], a["panel_offsets"] + if codes.shape[0] != self.panel_residues or len(offsets) != self.panel_size + 1: + raise BundleMismatchError(f"{self.scoring_npz}: panel shape differs from {self.system_name}") + return codes, offsets + + +@dataclass(frozen=True, eq=False) +class PreparedSystem: + """A system whose bundle has been fetched and checked against it (:func:`prepare`).""" + system: EsmCodesSystem + files: Mapping[str, Any] # file name -> local path (sha256-verified) + bundle: Mapping[str, Any] # parsed bundle.json + encoders_file: str + scoring_file: str + + @property + def encoders_path(self): + return self.files[self.encoders_file] + + @property + def scoring_path(self): + return self.files[self.scoring_file] + + def for_worker(self) -> WorkerSpec: + s = self.system + return WorkerSpec( + system_name=s.name, system_digest=s.digest(), scoring_npz=str(self.scoring_path), + scoring_sha256=s.bundle_sha256(self.scoring_file), dp=s.dp, offset=s.offset, gap_open=s.gap_open, + gap_extend=s.gap_extend, panel_size=s.panel_size, panel_residues=s.panel_residues, + panel_quantile=s.panel_quantile, quantile_method=s.quantile_method, bin_centres=s.bin_centres, + lam=s.lam, lnK=s.lnK, low_m_rule=s.low_m_rule, high_m_rule=s.high_m_rule, + calibrated_domain=s.calibrated_domain, degenerate_query_rule=s.degenerate_query_rule) + + def load_head(self, device="cpu"): + """The residue-code head (torch) from the encoders file, refused unless its content digests, architecture, + encoder dtype and system name equal the system's and the system's ESM3 weights are the ones this FoldMatch + loads.""" + from foldmatch.model.residue_code_encoder import load_esm_codes_encoders + from foldmatch.utils.esm import loaders + s = self.system + if (s.esm_repo, s.esm_weights_sha256) != (loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_SHA256): + raise UnsupportedSystemError( + f"{s.name} needs ESM3 weights {s.esm_weights_sha256} from {s.esm_repo}; this FoldMatch loads " + f"{loaders.ESM_WEIGHTS_SHA256} from {loaders.ESM_REPO_ID}") + head = load_esm_codes_encoders(self.encoders_path, sha256=s.bundle_sha256(self.encoders_file), device=device) + got = head.content_digests() + want = {k: getattr(s, k) for k in ("transform_sha256", "query_encoder_sha256", "target_encoder_sha256")} + problems = [f"{k} = {got.get(k)}, the system has {v}" for k, v in want.items() if got.get(k) != v] + arch = head.meta.get("architecture") if isinstance(head.meta.get("architecture"), Mapping) else {} + if arch.get("name") != s.encoder_architecture: + problems.append(f"the encoders file architecture is {arch.get('name')!r}, the system has " + f"encoder_architecture {s.encoder_architecture!r}") + conventions = head.meta.get("conventions") if isinstance(head.meta.get("conventions"), Mapping) else {} + encoder = conventions.get("encoder") if isinstance(conventions.get("encoder"), Mapping) else {} + if encoder.get("dtype") != s.encoder_dtype: + problems.append(f"the encoders file encoder dtype is {encoder.get('dtype')!r}, the system has " + f"encoder_dtype {s.encoder_dtype!r}") + if head.meta.get("system") != s.name: + problems.append(f"the encoders file is for system {head.meta.get('system')!r}") + _raise_problems(s, problems) + return head + + +def prepare(system: EsmCodesSystem, *, cache_dir=None, local_files_only: bool = False) -> PreparedSystem: + """Fetch the bundle of ``system`` (a registry entry, see :func:`registered`) from its registry pin + (:func:`foldmatch.utils.model.get_esm_codes_bundle`, sha256 verified) and refuse it unless ``bundle.json`` and + the scoring npz agree with the system exactly. + + Checked, deterministic and, once the bundle is cached, read-only (so it can run on every rank before ESM3): + :func:`check_implemented`; that ``system`` is the registry entry of its name; ``bundle.json`` format, every + ``DOCUMENT_FIELDS`` value (numbers as float64), that every file it lists is pinned with the same sha256, and its + table digest; the scoring npz ``meta_json`` likewise, its table arrays bit for bit, and the panel's shape, offsets + and content digest. Any difference raises :class:`BundleMismatchError` naming the fields. torch is not needed + (only :meth:`PreparedSystem.load_head` imports it). + + ``local_files_only=True`` never downloads: a bundle file that is not cached raises + :class:`BundleUnavailableError`. Use it for the every-rank pre-flight after one process (``download-models``, or + rank 0) has fetched the bundle, so that no rank writes to the cache and a network failure cannot split the ranks. + """ + if not isinstance(system, EsmCodesSystem): + raise TypeError(f"prepare needs an EsmCodesSystem, got {type(system).__name__}") + check_implemented(system) + from foldmatch.utils.model import get_esm_codes_bundle + from foldmatch.utils.tensor_digest import DIGEST_METHOD + files = get_esm_codes_bundle(system, cache_dir=cache_dir, local_files_only=local_files_only) + problems: list = [] + try: + bundle = json.loads(Path(files[BUNDLE_JSON]).read_text(encoding="utf-8")) + except (OSError, ValueError) as e: + raise BundleMismatchError(f"{system.name}: {files[BUNDLE_JSON]} is not readable JSON: {e}") from e + if not isinstance(bundle, Mapping) or bundle.get("bundle_format") != BUNDLE_FORMAT: + got = bundle.get("bundle_format") if isinstance(bundle, Mapping) else None + raise BundleMismatchError(f"{system.name}: bundle.json format {got!r}, this FoldMatch reads " + f"{BUNDLE_FORMAT!r}") + _compare(system, bundle, 1, "bundle.json", problems) + digests = bundle.get("content_digests") if isinstance(bundle.get("content_digests"), Mapping) else {} + if digests.get("method") != DIGEST_METHOD: + problems.append(f"bundle.json content_digests.method = {digests.get('method')!r}, expected {DIGEST_METHOD!r}") + table = _table_digest(system) + if digests.get("table_sha256") != table: + problems.append(f"bundle.json content_digests.table_sha256 = {digests.get('table_sha256')!r}, the system's " + f"table digests to {table}") + roles = _bundle_roles(system, bundle, problems) + if SCORING_FORMAT in roles and roles[SCORING_FORMAT] in files: + _check_scoring(system, _load_scoring_npz(files[roles[SCORING_FORMAT]]), problems) + _raise_problems(system, problems) + return PreparedSystem(system=system, files=MappingProxyType(dict(files)), bundle=bundle, + encoders_file=roles[ENCODERS_FORMAT], scoring_file=roles[SCORING_FORMAT]) diff --git a/src/foldmatch/search/sequence_store.py b/src/foldmatch/search/sequence_store.py index 927a67f..27eb142 100644 --- a/src/foldmatch/search/sequence_store.py +++ b/src/foldmatch/search/sequence_store.py @@ -1,9 +1,10 @@ import logging import sqlite3 from pathlib import Path -from typing import Dict, Iterable +from typing import Dict, Iterable, Optional from foldmatch.utils.fasta import iter_fasta +from foldmatch.utils.sequence import normalize_sequence logger = logging.getLogger(__name__) @@ -20,6 +21,11 @@ # hundreds of millions of sequences without materializing the FASTA in memory. _INSERT_BATCH = 50_000 +# Keys of the ``meta`` table, written once when the store is built so readers +# get the corpus totals (e.g. the E-value search space) without scanning every +# row. Stores built before the table existed have none of them. +META_KEYS = ("entry_count", "total_residues", "duplicates_ignored") + class SequenceStore: """SQLite-backed ``id -> sequence`` store sitting next to a FAISS database. @@ -49,11 +55,15 @@ def create(self, fasta_file: Path, min_res_n: int = 0, force: bool = False) -> i The FASTA is read one record at a time and inserted in batches, so peak RAM is independent of corpus size (scales to hundreds of millions of - sequences). Applies the same ``min_res_n`` filter used when building the - index so store and index stay in lock-step. Duplicate headers keep the + sequences). Sequences are normalised (:func:`normalize_sequence`) and the + same ``min_res_n`` filter used when building the index is applied to the + normalised length, so store and index stay in lock-step. Duplicate headers keep the first occurrence — deduplication is delegated to the table's primary key via ``INSERT OR IGNORE`` (no in-RAM ``seen`` set). + The totals (``entry_count``, ``total_residues``, ``duplicates_ignored``) + are recorded in the ``meta`` table; see :meth:`meta`. + Returns the number of sequences stored. """ if self.path.exists(): @@ -77,12 +87,15 @@ def create(self, fasta_file: Path, min_res_n: int = 0, force: bool = False) -> i "CREATE TABLE sequences (" "id TEXT PRIMARY KEY, sequence TEXT NOT NULL, length INTEGER NOT NULL)" ) + conn.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value)") insert_sql = "INSERT OR IGNORE INTO sequences (id, sequence, length) VALUES (?, ?, ?)" seen_input = 0 batch: list = [] for name, sequence in iter_fasta(fasta_file): - sequence = sequence.strip().upper() + # The same string ESM3 embedded (EsmProtFromFasta normalises too), + # so stored residues line up with the embedding's. + sequence = normalize_sequence(sequence, allow_non_ascii=True) if min_res_n > 0 and len(sequence) < min_res_n: continue batch.append((name, sequence, len(sequence))) @@ -95,11 +108,20 @@ def create(self, fasta_file: Path, min_res_n: int = 0, force: bool = False) -> i conn.executemany(insert_sql, batch) conn.commit() - stored = conn.execute("SELECT COUNT(*) FROM sequences").fetchone()[0] + # Totals over what was actually stored (first occurrence of each id), + # computed once here so readers never need to scan the table. + stored, total_residues = conn.execute( + "SELECT COUNT(*), COALESCE(SUM(length), 0) FROM sequences" + ).fetchone() + duplicates = seen_input - stored + conn.executemany( + "INSERT INTO meta (key, value) VALUES (?, ?)", + zip(META_KEYS, (stored, total_residues, duplicates)), + ) + conn.commit() finally: conn.close() - duplicates = seen_input - stored if duplicates > 0: logger.warning( f"{duplicates} duplicate FASTA id(s) in sequence store; kept first occurrence of each" @@ -136,22 +158,49 @@ def fetch(self, ids: Iterable[str]) -> Dict[str, str]: finally: conn.close() - def count(self) -> int: - """Number of sequences in the store.""" + def meta(self) -> Dict[str, Optional[int]]: + """Corpus totals: ``entry_count``, ``total_residues``, ``duplicates_ignored``. + + Read from the ``meta`` table written at build time. A store built + before that table existed falls back to ``COUNT(*)`` and + ``SUM(length)`` over the rows; its ``duplicates_ignored`` cannot be + recovered and is ``None``. + + :meth:`create` fills the ``meta`` table last, so a store that has the + table but not its totals was interrupted while being built: it holds + only part of the FASTA, and this raises ``ValueError`` instead of + reporting that part as the database. + """ if not self.path.exists(): raise FileNotFoundError(f"Sequence store not found at {self.path}") conn = sqlite3.connect(str(self.path)) try: - return conn.execute("SELECT COUNT(*) FROM sequences").fetchone()[0] + has_meta = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'meta'" + ).fetchone() is not None + meta: Dict[str, Optional[int]] = dict.fromkeys(META_KEYS) + if has_meta: + meta.update(conn.execute("SELECT key, value FROM meta").fetchall()) + if meta["entry_count"] is None: + raise ValueError( + f"Sequence store {self.path} is incomplete: its build stopped before " + f"it recorded its totals. Delete the database's files and rebuild it " + f"with 'fm-search build sequences'." + ) + if meta["entry_count"] is None: + meta["entry_count"] = conn.execute("SELECT COUNT(*) FROM sequences").fetchone()[0] + if meta["total_residues"] is None: + meta["total_residues"] = conn.execute( + "SELECT COALESCE(SUM(length), 0) FROM sequences" + ).fetchone()[0] + return {key: meta[key] for key in META_KEYS} finally: conn.close() + def count(self) -> int: + """Number of sequences in the store.""" + return self.meta()["entry_count"] + def total_residues(self) -> int: """Sum of all sequence lengths — the search space for database E-values.""" - if not self.path.exists(): - raise FileNotFoundError(f"Sequence store not found at {self.path}") - conn = sqlite3.connect(str(self.path)) - try: - return conn.execute("SELECT COALESCE(SUM(length), 0) FROM sequences").fetchone()[0] - finally: - conn.close() + return self.meta()["total_residues"] diff --git a/src/foldmatch/search/split_run.py b/src/foldmatch/search/split_run.py new file mode 100644 index 0000000..fd83411 --- /dev/null +++ b/src/foldmatch/search/split_run.py @@ -0,0 +1,853 @@ +"""The run record of a split sequence build (plan P8b): what ``build sequences-embed`` leaves for ``build +sequences-index``. + +Step 1, ``build sequences-embed``, runs the pre-flight and the ESM3 pass of ``build sequences`` on every rank and +keeps the shards (``emb_-.parquet``); rank 0 then checks them and writes, last, this run's record +``emb_.run.json`` beside them. Step 2, ``build sequences-index``, one process on a CPU node, picks one run by +its record (:func:`find_run`), checks the shards against it (:func:`check_run_shards` and the footers) and the FASTA +(size, sha256 and pre-flight counts, then the id of every shard row at its dealt position, :func:`check_dealt_ids`), +and runs the index phase of ``build sequences`` on them. ``build sequences-seal`` writes the record of complete +shards that have none (a step 1 whose rank 0 stopped after the pass, or the shards a ``build sequences`` kept). + +The record: + +* is staged as ``emb_.run.json.tmp`` (created exclusively), verified as step 2 reads it, and committed with a + hard link that fails if a record exists (:func:`write_run_record`). It is the last file its command writes and is + never rewritten; +* names its shards by file name, so a copied folder still works. It is no shard (shard names end in + ``-.parquet``) and no embedding file of ``build embeddings`` (``.pt``, ``.csv``, ``.parquet``); +* binds the shards to the FASTA by the pass's deal: Lightning's predict sampler gives rank ``r`` of ``W`` the entries + ``r, r + W, r + 2W, ...`` (``indices[rank::num_replicas]``), and a rank given no entry writes no shard + (:func:`dealt_rows`). + +pyarrow, numpy and the standard library only: no torch. Every refusal is a :class:`~foldmatch.errors.RefusalError`. +""" +from __future__ import annotations + +import errno +import glob +import hashlib +import json +import logging +import os +import re +import socket +import sqlite3 +import time +from contextlib import contextmanager +from datetime import datetime, timezone +from pathlib import Path +from secrets import token_hex +from typing import Any, Dict, Iterator, List, NamedTuple, Optional, Sequence, Tuple + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from foldmatch.errors import RefusalError +from foldmatch.search.db_path import parse_db_path +from foldmatch.search.residue_code_store import _fsync_dir +from foldmatch.search.residue_shards import ( + _SHARD_PREFIX, BASE_FIELDS, ID, PASS_KEYS, PAYLOAD_TYPE, RESIDUE_CODES, SEQ_SHA1, SEQ_SHA1_TYPE, + ResidueShardError, _column_types, check_shards, read_shard_info, run_shard_files, shard_run_ids, shard_schema, shards_pass_record, +) +from foldmatch.search.sequence_store import SequenceStore +from foldmatch.utils.dist import RUN_ID_ENV, is_run_id +from foldmatch.utils.fasta import read_id_at, read_record_at +from foldmatch.utils.sequence import seq_sha1 + +logger = logging.getLogger(__name__) + +RUN_RECORD_KIND = "foldmatch-sequence-run" +RUN_RECORD_VERSION = 1 +RUN_RECORD_SUFFIX = ".run.json" +#: The commands that write a record (its ``command``); only the seal's is ``sealed``. +EMBED_COMMAND = "build sequences-embed" +SEAL_COMMAND = "build sequences-seal" +#: The pre-flight counts a record holds (:class:`~foldmatch.utils.sequence.FastaPreflight`), which step 2's own +#: pre-flight must reproduce. +PREFLIGHT_KEYS = ("entries", "entries_kept", "residues", "duplicates", "unique_entries", "unique_residues", + "nonstandard_entries", "nonstandard_residues", "non_ascii_entries", "angle_bracket_entries", + "max_length") +#: The keys of a record's ``fasta`` and ``system`` blocks. +FASTA_KEYS = ("path", "bytes", "mtime_ns", "sha256") +SYSTEM_KEYS = ("name", "digest", "encoding_digest") +BUILD_LOCK_SUFFIX = ".build.lock" + +# A record is named as its run's shards are, emb_, with its own suffix. +_RECORD_PREFIX = _SHARD_PREFIX +_TMP_SUFFIX = ".tmp" +_RECORD_NAME = re.compile(rf"{_RECORD_PREFIX}(?P.+){re.escape(RUN_RECORD_SUFFIX)}") +_TMP_RECORD_NAME = re.compile(rf"{_RECORD_PREFIX}(?P.+){re.escape(RUN_RECORD_SUFFIX + _TMP_SUFFIX)}") +_SHARD_NAME = re.compile(rf"{_RECORD_PREFIX}(?P.+)-(?P\d+)\.parquet") +_CREATED_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +#: ``hashlib.file_digest`` takes no chunk size: the FASTA is hashed in reads of this many bytes. +HASH_CHUNK = 16 << 20 +#: Where a hard link cannot be made, the record is committed with ``os.replace`` after an existence check. +_LINK_UNSUPPORTED = frozenset({errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP}) +#: Ids shown in a refusal. +_EXAMPLES = 10 +#: Offsets turned into Python ints at a time by :func:`check_dealt_ids`. +_OFFSET_CHUNK = 1 << 16 +#: A database's files (``cli.search._SEQUENCE_DB_SUFFIXES``) and its residue-code store's (``{index name}.rescodes.*``). +_DATABASE_SUFFIXES = (".index", ".metadata", ".invlists", ".sequences") +_CODE_STORE_GLOB = ".rescodes.*" + +#: The type(s) of every key of a record; ``None`` stands for JSON null. +_RECORD_TYPES = { + "kind": (str,), "record_version": (int,), "run_id": (str,), "command": (str,), "sealed": (bool,), + "created_utc": (str,), "foldmatch_version": (str,), "fasta": (dict,), "min_res_n": (int,), "strict": (bool,), + "preflight": (dict,), "output_db": (str, None), "residue_output": (str, None), "system": (dict, None), + "world_size": (int,), "pass": (dict,), "options": (dict,), "shards": (list,), "rows": (int,), +} +_FASTA_TYPES = dict(zip(FASTA_KEYS, ((str,), (int,), (int,), (str,)))) +_PASS_TYPES = {"min_res_n": (int, None), "world_size": (int, None)} +_SHARD_TYPES = {"name": (str,), "rank": (int,), "rows": (int,), "bytes": (int,)} + + +def _is(value, kind) -> bool: + """JSON typing: ``None`` is null, an ``int`` is never a ``bool``.""" + if kind is None: + return value is None + if kind is int: + return isinstance(value, int) and not isinstance(value, bool) + return isinstance(value, kind) + + +def _type_names(kinds) -> str: + return " or ".join("null" if kind is None else {dict: "object", list: "array", str: "string", int: "integer", + bool: "boolean"}[kind] for kind in kinds) + + +# ===================================================================================================================== +# paths and discovery +# ===================================================================================================================== +def run_record_path(folder, run_id: str) -> Path: + """``{folder}/emb_{run_id}.run.json``.""" + if not is_run_id(run_id): + raise RefusalError(f"{run_id!r} is not a run id: letters, digits, '.', '_' and '-' only, starting with a " + f"letter or digit.") + return Path(folder) / f"{_RECORD_PREFIX}{run_id}{RUN_RECORD_SUFFIX}" + + +def run_record_tmp_path(folder, run_id: str) -> Path: + """``{folder}/emb_{run_id}.run.json.tmp``, where the record is staged.""" + path = run_record_path(folder, run_id) + return path.with_name(path.name + _TMP_SUFFIX) + + +def _ids_matching(folder, pattern: re.Pattern) -> List[str]: + names = (path.name for path in Path(folder).glob(f"{_RECORD_PREFIX}*")) + return sorted({match["run_id"] for match in map(pattern.fullmatch, names) + if match and is_run_id(match["run_id"])}) + + +def record_run_ids(folder) -> List[str]: + """The run ids of the run records in ``folder``, sorted. File names only; ``emb_copy[1].run.json`` names no + run (only well-formed ids count, :func:`~foldmatch.utils.dist.is_run_id`).""" + return _ids_matching(folder, _RECORD_NAME) + + +def tmp_record_run_ids(folder) -> List[str]: + """The run ids of the staged (``.tmp``) run records in ``folder``, sorted: records whose command stopped before + committing them.""" + return _ids_matching(folder, _TMP_RECORD_NAME) + + +def _record_name_id(path: Path) -> Optional[str]: + """The run id in a record's file name (staged or not), ``None`` if it is not a record's name.""" + match = _RECORD_NAME.fullmatch(path.name) or _TMP_RECORD_NAME.fullmatch(path.name) + return match["run_id"] if match else None + + +def _shard_rank(path) -> int: + match = _SHARD_NAME.fullmatch(Path(path).name) + if match is None: + raise RefusalError(f"{path} is not an embedding shard (emb_-.parquet).") + return int(match["rank"]) + + +# ===================================================================================================================== +# FASTA facts +# ===================================================================================================================== +class FastaStat(NamedTuple): + """What a stat of the FASTA says about its content: size, modification time and inode. ``st_dev`` is left out: + the nodes that mount one Lustre file system may report different devices for the same file.""" + bytes: int + mtime_ns: int + ino: int + + +def fasta_stat(path) -> FastaStat: + st = os.stat(path) + return FastaStat(st.st_size, st.st_mtime_ns, st.st_ino) + + +def fasta_sha256(path) -> Tuple[str, float]: + """``(sha256 hex digest, seconds)`` of the file at ``path``, read in :data:`HASH_CHUNK` pieces.""" + start = time.monotonic() + digest = hashlib.sha256() + with open(path, "rb") as f: + while True: + chunk = f.read(HASH_CHUNK) + if not chunk: + break + digest.update(chunk) + return digest.hexdigest(), time.monotonic() - start + + +def preflight_counts(preflight) -> Dict[str, int]: + """The :data:`PREFLIGHT_KEYS` counts of a :class:`~foldmatch.utils.sequence.FastaPreflight`.""" + return {key: int(getattr(preflight, key)) for key in PREFLIGHT_KEYS} + + +# ===================================================================================================================== +# shard facts +# ===================================================================================================================== +def dealt_rows(n: int, world_size: int) -> Dict[int, int]: + """``{rank: rows}`` of the pass's deal of ``n`` entries over ``world_size`` ranks: rank ``r`` gets entries ``r, r + + W, ...``, so the ranks that write a shard are exactly ``0 ... min(W, n) - 1``.""" + return {rank: len(range(rank, n, world_size)) for rank in range(min(world_size, n))} + + +def shard_entries(paths: Sequence) -> List[Dict[str, Any]]: + """A record's ``shards`` entries, in the order of ``paths``: ``name``, ``rank`` (from the name), ``rows`` (from + the Parquet footer) and ``bytes``.""" + out = [] + for path in map(Path, paths): + rank = _shard_rank(path) + try: + rows = pq.read_metadata(path).num_rows + size = path.stat().st_size + except (pa.ArrowException, OSError) as exc: + raise RefusalError(f"{path}: not a readable Parquet embedding shard ({exc}).") from exc + out.append({"name": path.name, "rank": rank, "rows": int(rows), "bytes": int(size)}) + return out + + +def _read_schema(path) -> pa.Schema: + try: + return pq.read_schema(path) + except (pa.ArrowException, OSError) as exc: + raise RefusalError(f"{path}: not a readable Parquet embedding shard ({exc}).") from exc + + +def _listed(paths, limit: int = _EXAMPLES) -> str: + names = [Path(p).name for p in paths] + return ", ".join(names[:limit]) + (", ..." if len(names) > limit else "") + + +def check_plain_footers(paths: Sequence) -> None: + """Refuse unless every shard is a plain one: the legacy schema ``id | embedding`` (``shard_schema(None)``) with no + ``foldmatch.*`` metadata, as a pass without ``--residue-codes`` writes it.""" + want = _column_types(shard_schema(None)) + bad = [] + for path in paths: + schema = _read_schema(path) + foldmatch_keys = [key for key in (schema.metadata or {}) if key.startswith(b"foldmatch.")] + if _column_types(schema) != want or foldmatch_keys: + bad.append((Path(path), schema, foldmatch_keys)) + if bad: + path, schema, keys = bad[0] + shown = (f"{path.name} has columns {[name for name, _ in _column_types(schema)]}" + + (f" and the metadata {', '.join(k.decode('utf-8', 'replace') for k in keys)}" if keys else "")) + raise RefusalError( + f"{len(bad)} of the {len(paths)} shard(s) are not plain embedding shards (columns id | embedding, no " + f"residue codes): {_listed(p for p, _, _ in bad)}. E.g. {shown}. A run without residue codes has only " + f"plain shards: these were written by another pass, or changed since." + ) + + +_CODES_COLUMNS = _column_types(pa.schema(list(BASE_FIELDS) + [(RESIDUE_CODES, PAYLOAD_TYPE), + (SEQ_SHA1, SEQ_SHA1_TYPE)])) + + +def _pass_value(record, key): + return None if not isinstance(record, dict) else record.get(key) + + +def check_codes_footers(paths: Sequence, *, system=None, expect: Optional[Dict[str, Any]] = None, min_res_n: int, + world_size: int) -> Dict[str, Any]: + """Refuse unless every shard holds the residue codes of one pass with ``min_res_n`` on ``world_size`` ranks; + returns the shards' pass record (:func:`~foldmatch.search.residue_shards.shards_pass_record`). + + With ``system`` (an ``EsmCodesSystem``) the shards must hold its encoding + (:func:`~foldmatch.search.residue_shards.check_shards`: mixed and stale shards are refused). With ``expect``, a + record's ``system`` block, they must form one group of residue-code shards whose system name, digest and + encoding digest are the block's: a check of the run's self-consistency that needs no registry entry. Exactly one + of the two is given. Then every shard must record its pass (``foldmatch.pass``) with this ``min_res_n`` and + ``world_size``. + """ + paths = [Path(p) for p in paths] + if (system is None) == (expect is None): + raise ValueError("check_codes_footers takes exactly one of system and expect") + if not paths: + raise RefusalError("There are no shards to check.") + try: + infos = check_shards(paths, RESIDUE_CODES, system) if system is not None else list(map(read_shard_info, paths)) + except ResidueShardError as exc: + raise RefusalError(str(exc)) from exc + if expect is not None: + groups: Dict[tuple, list] = {} + for info in infos: + groups.setdefault(info.key, []).append(info) + if len(groups) > 1: + shown = "; ".join(f"{len(g)} shard(s) with {g[0].describe()} (e.g. {g[0].path.name})" + for g in groups.values()) + raise RefusalError(f"Mixed embedding shards: {shown}. The shards of one run come from one pass.") + info = infos[0] + found = {"name": info.system, "digest": info.system_digest, "encoding_digest": info.encoding_digest} + want = {key: (expect or {}).get(key) for key in SYSTEM_KEYS} + where = f"{len(infos)} shard(s) ({_listed(paths, 3)})" + if info.residue_output != RESIDUE_CODES or info.column_types != _CODES_COLUMNS: + raise RefusalError(f"{where} hold {info.describe()} with columns {[n for n, _ in info.column_types]}, " + f"not the residue codes of {want['name']}.") + if found != want: + raise RefusalError(f"{where} hold the residue codes of {found['name']} (digest {found['digest']}, " + f"encoding_digest {found['encoding_digest']}), not those of {want['name']} (digest " + f"{want['digest']}, encoding_digest {want['encoding_digest']}).") + bad = [info for info in infos + if not (_is(_pass_value(info.pass_record, "min_res_n"), int) + and _pass_value(info.pass_record, "min_res_n") == min_res_n + and _is(_pass_value(info.pass_record, "world_size"), int) + and _pass_value(info.pass_record, "world_size") == world_size)] + if bad: + shown = ", ".join( + f"{info.path.name} (" + ("no pass record" if info.pass_record is None else + f"min_res_n {_pass_value(info.pass_record, 'min_res_n')}, world_size " + f"{_pass_value(info.pass_record, 'world_size')}") + ")" + for info in bad[:_EXAMPLES]) + raise RefusalError( + f"{len(bad)} of the {len(infos)} shard(s) do not record a pass with --min-res-n {min_res_n} on " + f"{world_size} rank(s): {shown}{', ...' if len(bad) > _EXAMPLES else ''}." + ) + return shards_pass_record(infos) + + +# ===================================================================================================================== +# record I/O +# ===================================================================================================================== +def refuse_existing_record(folder, run_id: str, *, command: str = EMBED_COMMAND) -> None: + """Refuse if ``folder`` holds a run record of ``run_id``, or its staged ``.tmp``. Read-only. + + The advice is ``command``'s: step 1 (:data:`EMBED_COMMAND`) found an earlier run with its id; the seal + (:data:`SEAL_COMMAND`) found the run it would seal recorded already (use the record), or a record whose writer + stopped (delete the ``.tmp``, then seal again).""" + path = run_record_path(folder, run_id) + sealing = command == SEAL_COMMAND + if path.exists(): + if sealing: + raise RefusalError( + f"{path} exists: run {run_id} in {folder} has its run record already, so there is nothing to seal. " + f"Use it: 'build sequences-index --from-shards {folder} --run-id {run_id}' builds from it." + ) + raise RefusalError( + f"{path} exists: a run record of an earlier run with the same run id ({run_id}) is in {folder}. Remove " + f"it and its shards, use another --shards-folder, or set a different {RUN_ID_ENV}." + ) + tmp = run_record_tmp_path(folder, run_id) + if tmp.exists() and sealing: + raise RefusalError( + f"{tmp} exists: the run record of run {run_id} was being written when its command stopped. Delete the " + f".tmp, then run 'build sequences-seal' again." + ) + if tmp.exists(): + raise RefusalError( + f"{tmp} exists: an earlier run with the same run id ({run_id}) stopped while it wrote its run record. " + f"Delete the .tmp (and that run's shards, unless you seal them), use another --shards-folder, or set a " + f"different {RUN_ID_ENV}." + ) + + +def _write_all(fd: int, data: bytes) -> None: + view = memoryview(data) + while view: + view = view[os.write(fd, view):] + + +def write_run_record(folder, record: Dict[str, Any]) -> Path: + """Write ``record`` as ``{folder}/emb_{run id}.run.json`` and return its path. + + The record is staged in ``emb_{run id}.run.json.tmp``, created exclusively (an existing ``.tmp`` is refused and + left alone), as sorted, indented JSON, and fsynced; the staged file is verified as step 2 reads it + (:func:`read_run_record` and :func:`check_run_shards`), then committed with ``os.link``, which fails if a record + exists; the ``.tmp`` is unlinked and the folder fsynced. Where hard links are not supported, an existing record + is refused and the ``.tmp`` is renamed into place (``os.replace``). Any failure removes the ``.tmp`` and raises a + :class:`RefusalError` naming the file. + """ + folder = Path(folder) + run_id = record.get("run_id") if isinstance(record, dict) else None + final = run_record_path(folder, run_id) + tmp = run_record_tmp_path(folder, run_id) + try: + text = json.dumps(record, indent=2, sort_keys=True, allow_nan=False) + "\n" + except (TypeError, ValueError) as exc: + raise RefusalError(f"The run record {final} cannot be written as JSON ({exc}).") from exc + try: + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except OSError as exc: + raise RefusalError(f"The run record cannot be staged in {tmp} ({exc}).") from exc + try: + try: + _write_all(fd, text.encode("utf-8")) + os.fsync(fd) + finally: + os.close(fd) + try: + check_run_shards(folder, read_run_record(tmp)) + except RefusalError as exc: + raise RefusalError(f"The staged run record {tmp} failed its check: {exc}") from exc + try: + os.link(tmp, final) + except OSError as exc: + if exc.errno not in _LINK_UNSUPPORTED: + raise + if final.exists(): + raise FileExistsError(errno.EEXIST, "a run record exists", str(final)) from exc + logger.info(f"{folder} does not support hard links ({exc.strerror}): the run record is renamed into " + f"place instead") + os.replace(tmp, final) + except BaseException as exc: + tmp.unlink(missing_ok=True) + if isinstance(exc, RefusalError): + raise + if isinstance(exc, OSError): + raise RefusalError(f"The run record {final} cannot be written ({exc}); {tmp} was removed.") from exc + raise + try: + tmp.unlink(missing_ok=True) + except OSError as exc: # the record is committed: a leftover .tmp is only reported + logger.warning(f"Could not remove {tmp} after committing {final} ({exc}); delete it.") + _fsync_dir(folder) + return final + + +def _bad(path, problem: str) -> RefusalError: + return RefusalError(f"{path} is not a valid run record: {problem}.") + + +def _check_fields(path, block: Dict[str, Any], types: Dict[str, tuple], where: str) -> None: + for key, kinds in types.items(): + if key not in block: + raise _bad(path, f"the key {where}{key} is missing") + if not any(_is(block[key], kind) for kind in kinds): + raise _bad(path, f"{where}{key} is {block[key]!r}, expected {_type_names(kinds)}") + + +def read_run_record(path) -> Dict[str, Any]: + """The run record at ``path``, checked; refused (naming the file and the key) if it cannot be read, is not a + JSON object, has another ``kind`` or ``record_version`` (a larger one: written by a newer FoldMatch), lacks a key + or has one of a wrong type, a negative count, or fields that disagree: the run id and the file name, the strict + mode and the residue output, the command and ``sealed``, the pass's ``min_res_n`` and ``world_size`` and the + record's, the shard rows and ``rows`` and the pre-flight's kept entries, and the shards and the deal of those + entries over ``world_size`` ranks (:func:`dealt_rows`). Unknown keys are ignored.""" + path = Path(path) + try: + text = path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise RefusalError(f"The run record {path} cannot be read ({exc}).") from exc + try: + record = json.loads(text) + except ValueError as exc: + raise _bad(path, f"it is not JSON ({exc})") from exc + if not isinstance(record, dict): + raise _bad(path, "it is not a JSON object") + if record.get("kind") != RUN_RECORD_KIND: + raise _bad(path, f"kind is {record.get('kind')!r}, not {RUN_RECORD_KIND!r}") + version = record.get("record_version") + if _is(version, int) and version > RUN_RECORD_VERSION: + raise RefusalError(f"The run record {path} was written by a newer FoldMatch (record_version {version}; this " + f"FoldMatch reads version {RUN_RECORD_VERSION}): upgrade FoldMatch.") + if not _is(version, int) or version != RUN_RECORD_VERSION: + raise _bad(path, f"record_version is {version!r}, not {RUN_RECORD_VERSION}") + _check_fields(path, record, _RECORD_TYPES, "") + _check_fields(path, record["fasta"], _FASTA_TYPES, "fasta.") + _check_fields(path, record["preflight"], dict.fromkeys(PREFLIGHT_KEYS, (int,)), "preflight.") + _check_fields(path, record["pass"], {key: _PASS_TYPES.get(key, (str, None)) for key in PASS_KEYS}, "pass.") + if record["system"] is not None: + _check_fields(path, record["system"], dict.fromkeys(SYSTEM_KEYS, (str,)), "system.") + for k, shard in enumerate(record["shards"]): + if not isinstance(shard, dict): + raise _bad(path, f"shards[{k}] is {shard!r}, expected object") + _check_fields(path, shard, _SHARD_TYPES, f"shards[{k}].") + + run_id = record["run_id"] + if not is_run_id(run_id): + raise _bad(path, f"run_id {run_id!r} is not a run id") + name_id = _record_name_id(path) + if name_id != run_id: + raise _bad(path, f"run_id is {run_id!r}, but the file is named for " + + (f"run {name_id!r}" if name_id else "no run (emb_.run.json)")) + if record["command"] not in (EMBED_COMMAND, SEAL_COMMAND): + raise _bad(path, f"command is {record['command']!r}, not {EMBED_COMMAND!r} or {SEAL_COMMAND!r}") + if record["sealed"] != (record["command"] == SEAL_COMMAND): + raise _bad(path, f"sealed is {record['sealed']} for the command {record['command']!r}") + try: + datetime.strptime(record["created_utc"], _CREATED_FORMAT) + except ValueError as exc: + raise _bad(path, f"created_utc {record['created_utc']!r} is not {_CREATED_FORMAT}") from exc + if not re.fullmatch(r"[0-9a-f]{64}", record["fasta"]["sha256"]): + raise _bad(path, f"fasta.sha256 {record['fasta']['sha256']!r} is not a sha256 hex digest") + counts = {"min_res_n": record["min_res_n"], "rows": record["rows"], "fasta.bytes": record["fasta"]["bytes"], + **{f"preflight.{key}": value for key, value in record["preflight"].items() if key in PREFLIGHT_KEYS}, + **{f"shards[{k}].{key}": shard[key] for k, shard in enumerate(record["shards"]) + for key in ("rank", "rows", "bytes")}} + negative = [key for key, value in counts.items() if value < 0] + if negative: + raise _bad(path, f"{negative[0]} is negative ({counts[negative[0]]})") + if record["world_size"] < 1: + raise _bad(path, f"world_size is {record['world_size']}, not at least 1") + if record["residue_output"] not in (None, RESIDUE_CODES): + raise _bad(path, f"residue_output is {record['residue_output']!r}, not {RESIDUE_CODES!r} or null") + codes = record["residue_output"] is not None + if record["strict"] != codes: + raise _bad(path, f"strict is {record['strict']}, but residue_output is {record['residue_output']!r}") + if (record["system"] is not None) != codes: + raise _bad(path, f"system is {'set' if record['system'] is not None else 'null'}, but residue_output is " + f"{record['residue_output']!r}") + for key in ("min_res_n", "world_size"): + if record["pass"][key] != record[key]: + raise _bad(path, f"pass.{key} is {record['pass'][key]!r}, but {key} is {record[key]}") + preflight = record["preflight"] + if preflight["unique_entries"] != preflight["entries_kept"] - preflight["duplicates"]: + raise _bad(path, f"preflight.unique_entries is {preflight['unique_entries']}, not entries_kept - " + f"duplicates ({preflight['entries_kept'] - preflight['duplicates']})") + if preflight["entries_kept"] > preflight["entries"]: + raise _bad(path, f"preflight.entries_kept ({preflight['entries_kept']}) exceeds preflight.entries " + f"({preflight['entries']})") + shards = record["shards"] + if sum(shard["rows"] for shard in shards) != record["rows"]: + raise _bad(path, f"rows is {record['rows']}, but the shards hold {sum(s['rows'] for s in shards)}") + if record["rows"] != preflight["entries_kept"]: + raise _bad(path, f"rows is {record['rows']}, but preflight.entries_kept is {preflight['entries_kept']}") + names = [shard["name"] for shard in shards] + for k, shard in enumerate(shards): + want = f"{_RECORD_PREFIX}{run_id}-{shard['rank']}.parquet" + if shard["name"] != want: + raise _bad(path, f"shards[{k}] is named {shard['name']!r}, not {want!r} (run {run_id}, rank " + f"{shard['rank']})") + if len(set(names)) != len(names) or names != sorted(names): + raise _bad(path, "the shards are not listed once each, in file-name order") + deal = dealt_rows(record["rows"], record["world_size"]) + found = {shard["rank"]: shard["rows"] for shard in shards} + if found != deal: + raise _bad(path, f"the shards ({_deal_text(found)}) are not the deal of {record['rows']} entries over " + f"{record['world_size']} rank(s) ({_deal_text(deal)})") + return record + + +def _deal_text(rows: Dict[int, int]) -> str: + if not rows: + return "no shard" + return ", ".join(f"rank {rank}: {n} row(s)" for rank, n in sorted(rows.items())[:_EXAMPLES]) + ( + ", ..." if len(rows) > _EXAMPLES else "") + + +# ===================================================================================================================== +# selection and checks +# ===================================================================================================================== +class _RunState(NamedTuple): + shards: int + record: bool + tmp: bool + + def describe(self) -> str: + if self.record: + return f"run record, {self.shards} shard(s)" + if self.tmp: + return f"{self.shards} shard(s), an unfinished run record (.tmp)" + return f"{self.shards} shard(s), no run record" + + +def _runs(folder: Path) -> Dict[str, _RunState]: + """Every run in ``folder``: a well-formed run id with shards, a record or a staged record.""" + records, tmps = set(record_run_ids(folder)), set(tmp_record_run_ids(folder)) + ids = set(shard_run_ids(folder)) | records | tmps + return {rid: _RunState(len(run_shard_files(folder, rid)), rid in records, rid in tmps) for rid in sorted(ids)} + + +def _seal_command(folder: Path, run_id: str) -> str: + return (f"'fm-search build sequences-seal --fasta-file FASTA --from-shards {folder} --run-id {run_id}' (with " + f"--min-res-n for a run without residue codes)") + + +def find_run(folder, run_id: Optional[str] = None) -> Tuple[str, Path]: + """``(run id, record path)`` of the run step 2 builds from (plan D33). + + With ``run_id``, that run, which must have a record. Without, the folder must hold exactly one run of any kind + (shards, a record or a staged ``.tmp``), and it must have a record: a run without one did not finish (or its + shards were kept by ``build sequences``), and building silently from an older run beside it is refused. + """ + folder = Path(folder) + runs = _runs(folder) + if run_id is not None: + path = run_record_path(folder, run_id) + if path.is_file(): + return run_id, path + state = runs.get(run_id) + with_records = [rid for rid, s in runs.items() if s.record] + if state is None: + what = "neither a run record nor embedding shards" + else: + what = state.describe() + advice = (f" If its shards are complete, {_seal_command(folder, run_id)} records them." + if state is not None and state.shards else "") + if state is not None and state.tmp: + advice = f" Delete {run_record_tmp_path(folder, run_id)} first.{advice}" + raise RefusalError( + f"Run {run_id} has no run record ({path.name}) in {folder}: {what}.{advice} Runs with a record here: " + f"{', '.join(with_records) if with_records else 'none'}." + ) + if not runs: + raise RefusalError( + f"No run in {folder}: no run record (emb_.run.json) and no embedding shards. Pass the " + f"--shards-folder of a 'build sequences-embed' run." + ) + if len(runs) > 1: + listed = "; ".join(f"{rid} ({state.describe()})" for rid, state in runs.items()) + raise RefusalError( + f"{folder} holds {len(runs)} runs: {listed}. Pass --run-id to choose one ('build sequences-embed' logs " + f"its run id; its record is emb_.run.json)." + ) + (rid, state), = runs.items() + if state.record: + return rid, run_record_path(folder, rid) + first = f"delete {run_record_tmp_path(folder, rid)} first, then " if state.tmp else "" + raise RefusalError( + f"Run {rid} in {folder} has no run record ({state.describe()}): it did not finish, or its shards were kept " + f"by 'build sequences'. If they are complete, {first}run {_seal_command(folder, rid)}; otherwise run 'build " + f"sequences-embed' again." + ) + + +def check_run_shards(folder, record: Dict[str, Any]) -> List[Path]: + """The shards of the record's run in ``folder``, in the order step 2 reads them (:func:`run_shard_files`), after + checking that they are exactly the recorded ones, with the recorded size and rows (from the footers).""" + folder = Path(folder) + run_id = record["run_id"] + paths = run_shard_files(folder, run_id) + recorded = {shard["name"]: shard for shard in record["shards"]} + found = {path.name for path in paths} + missing, extra = sorted(set(recorded) - found), sorted(found - set(recorded)) + if missing or extra: + raise RefusalError( + f"The shards of run {run_id} in {folder} are not the {len(recorded)} its record lists: " + + "; ".join(part for part in (f"missing {_listed(missing)}" if missing else "", + f"not in the record {_listed(extra)}" if extra else "") if part) + + ". They changed since the run (a purge, a partial copy, or shards of another run with the same id)." + ) + changed = [] + for path in paths: + shard = recorded[path.name] + try: + size = path.stat().st_size + rows = pq.read_metadata(path).num_rows + except (pa.ArrowException, OSError) as exc: + raise RefusalError(f"{path}: the footer of a shard of run {run_id} cannot be read ({exc}); it changed " + f"since the run.") from exc + if size != shard["bytes"] or rows != shard["rows"]: + changed.append(f"{path.name} ({size} bytes, {rows} rows; recorded {shard['bytes']} bytes, " + f"{shard['rows']} rows)") + if changed: + raise RefusalError(f"{len(changed)} shard(s) of run {run_id} in {folder} changed since the run: " + f"{', '.join(changed[:_EXAMPLES])}{', ...' if len(changed) > _EXAMPLES else ''}.") + return paths + + +def _sha1_rows(column: pa.ChunkedArray, path: Path) -> np.ndarray: + """A shard's ``seq_sha1`` column as a ``uint8`` array of ``[rows, 20]`` (20 B per row, no Python objects).""" + array = column.combine_chunks() + if array.type != SEQ_SHA1_TYPE or array.null_count: + raise RefusalError(f"{path}: its seq_sha1 column is not {SEQ_SHA1_TYPE} without nulls ({array.type}, " + f"{array.null_count} null(s)).") + width = SEQ_SHA1_TYPE.byte_width + if not len(array): + return np.zeros((0, width), np.uint8) + data = np.frombuffer(array.buffers()[1], dtype=np.uint8) + return data[array.offset * width:(array.offset + len(array)) * width].reshape(-1, width) + + +def check_dealt_ids(paths: Sequence, record: Dict[str, Any], preflight, *, sequences: bool = False) -> float: + """Refuse unless every shard row holds the id of the FASTA entry the pass's deal gave it: kept entry ``k`` (in + the order of ``preflight.offsets``) is row ``k // W`` of rank ``k % W``'s shard, ``W`` the record's + ``world_size``. One sequential pass over the kept records, each shard's id column loaded once (about 100 B per + entry). Returns the seconds it took. + + With ``sequences`` (residue-code shards), every row's ``seq_sha1`` must also be the + :func:`~foldmatch.utils.sequence.seq_sha1` of its entry's sequence, read in the same pass (20 B more per row): + shards computed from another FASTA with the same ids, order and lengths are refused too. Their pre-flight was + strict, so every sequence is ASCII.""" + start = time.monotonic() + world = record["world_size"] + total = preflight.entries_kept + by_rank: Dict[int, list] = {} + sha1s: Dict[int, Any] = {} + names: Dict[int, str] = {} + for path in map(Path, paths): + rank = _shard_rank(path) + try: + table = pq.read_table(path, columns=[ID, SEQ_SHA1] if sequences else [ID]) + by_rank[rank] = table.column(ID).to_pylist() + if sequences: + sha1s[rank] = _sha1_rows(table.column(SEQ_SHA1), path) + except (pa.ArrowException, OSError, KeyError) as exc: + what = "ids and seq_sha1" if sequences else "ids" + raise RefusalError(f"{path}: the {what} of a shard of run {record['run_id']} cannot be read ({exc}).") \ + from exc + names[rank] = path.name + found = {rank: len(ids) for rank, ids in by_rank.items()} + deal = dealt_rows(total, world) + if found != deal: + raise RefusalError( + f"The shards of run {record['run_id']} ({_deal_text(found)}) are not the deal of the {total} entries of " + f"{preflight.path} over {world} rank(s) ({_deal_text(deal)})." + ) + mismatches, shown = 0, [] + sha1_mismatches, sha1_shown = 0, [] + offsets = preflight.offsets + with open(preflight.path, "r") as fh: + for begin in range(0, total, _OFFSET_CHUNK): + for k, offset in enumerate(offsets[begin:begin + _OFFSET_CHUNK].tolist(), begin): + rank, row = k % world, k // world + got = by_rank[rank][row] + if sequences: + want, sequence = read_record_at(fh, offset) + else: + want = read_id_at(fh, offset) + if got != want: + mismatches += 1 + if len(shown) < _EXAMPLES: + shown.append(f"{names[rank]} row {row} holds {got!r}, entry {k} is {want!r}") + elif sequences and sha1s[rank][row].tobytes() != seq_sha1(sequence): + sha1_mismatches += 1 + if len(sha1_shown) < _EXAMPLES: + sha1_shown.append(f"{names[rank]} row {row} ({got!r}, entry {k})") + if mismatches: + raise RefusalError( + f"{mismatches} shard row(s) of run {record['run_id']} do not hold the id of the entry of " + f"{preflight.path} the pass dealt them: {'; '.join(shown)}{'; ...' if mismatches > len(shown) else ''}. " + f"These shards were not computed from this FASTA in this order." + ) + if sha1_mismatches: + raise RefusalError( + f"{sha1_mismatches} shard row(s) of run {record['run_id']} hold a seq_sha1 that is not the sha1 of the " + f"sequence of the entry of {preflight.path} the pass dealt them, though they hold its id: " + f"{'; '.join(sha1_shown)}{'; ...' if sha1_mismatches > len(sha1_shown) else ''}. These shards were not " + f"computed from this FASTA's sequences." + ) + return time.monotonic() - start + + +# ===================================================================================================================== +# the build lock +# ===================================================================================================================== +def _database_files(db_path: str) -> List[Path]: + db_folder, index_name, _ = parse_db_path(db_path) + present = [Path(f"{db_path}{suffix}") for suffix in _DATABASE_SUFFIXES] + present = [path for path in present if path.exists()] + present.extend(sorted(Path(db_folder).glob(f"{glob.escape(index_name)}{_CODE_STORE_GLOB}"))) + return present + + +def _only_code_store_pending(db_path: str) -> bool: + """The database's ``.index``, ``.metadata`` and complete ``.sequences`` exist, and no residue-code manifest.""" + db_folder, index_name, _ = parse_db_path(db_path) + if not all(Path(f"{db_path}{suffix}").exists() for suffix in (".index", ".metadata")): + return False + if Path(f"{db_path}.rescodes.json").exists(): + return False + store = SequenceStore(Path(db_folder), index_name) + if not store.exists(): + return False + try: + store.meta() + except (ValueError, OSError, sqlite3.DatabaseError): + return False + return True + + +@contextmanager +def build_lock(db_path, record: Dict[str, Any], *, shards_folder=None) -> Iterator[Path]: + """Hold ``{db_path}.build.lock`` (``db_path``: :func:`~foldmatch.search.db_path.parse_db_path`'s resolved path) + for a step-2 build of the record's run. + + The lock is created exclusively and holds the run id, the host, the pid, ``SLURM_JOB_ID``, the start time and a + random ``lock_id``. It is removed when the block ends, also on an exception (which then propagates), so it stays + only when the process is killed (wall time, out of memory); a lock that is no longer this one (deleted, and maybe + created again by another build) is left alone. It is told apart by its ``lock_id``, not by its inode, which Linux + can give a file created right after this one was deleted. An existing lock is refused with its content, the + database files present and the recovery; when only the residue-code store is missing from an otherwise complete + database, the refusal also names the ``build residue-codes`` command that adds it (``shards_folder`` is the run's + folder). + """ + db_path = str(db_path) + lock = Path(f"{db_path}{BUILD_LOCK_SUFFIX}") + content = { + "run_id": record["run_id"], + "host": socket.gethostname(), + "pid": os.getpid(), + "slurm_job_id": os.environ.get("SLURM_JOB_ID") or None, + "started_utc": datetime.now(timezone.utc).strftime(_CREATED_FORMAT), + "lock_id": token_hex(16), + } + try: + fd = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError: + raise RefusalError(_lock_refusal(lock, db_path, record, shards_folder)) from None + except OSError as exc: + raise RefusalError(f"The build lock {lock} cannot be created ({exc}).") from exc + try: + try: + _write_all(fd, (json.dumps(content, indent=2, sort_keys=True) + "\n").encode("utf-8")) + os.fsync(fd) + finally: + os.close(fd) + except OSError as exc: + lock.unlink(missing_ok=True) + raise RefusalError(f"The build lock {lock} cannot be written ({exc}).") from exc + try: + yield lock + finally: + if _lock_id(lock) == content["lock_id"]: + lock.unlink(missing_ok=True) + + +def _lock_id(lock: Path) -> Optional[str]: + """The ``lock_id`` of the build lock at ``lock``; None when it is gone or holds none (not a lock of this + version, or not JSON).""" + try: + held = json.loads(lock.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, ValueError): + return None + return held.get("lock_id") if isinstance(held, dict) else None + + +def code_store_command(db_path, record: Dict[str, Any], shards_folder=None) -> Optional[str]: + """The ``fm-search build residue-codes`` command that adds the residue-code store to the database at ``db_path`` + from the record's run (``shards_folder`` holds its shards), when the run computed residue codes and the database + is complete but for that store (a step-2 build stopped after its sequence store); else ``None``.""" + db_path = str(db_path) + if record.get("residue_output") != RESIDUE_CODES or not _only_code_store_pending(db_path): + return None + db_folder, index_name, _ = parse_db_path(db_path) + return (f"fm-search build residue-codes {db_folder} {index_name} --from-shards " + f"{shards_folder if shards_folder is not None else 'SHARDS'} --run-id {record['run_id']}") + + +def _lock_refusal(lock: Path, db_path: str, record: Dict[str, Any], shards_folder) -> str: + try: + held = " ".join(lock.read_text(encoding="utf-8").split()) + except (OSError, UnicodeDecodeError) as exc: + held = f"unreadable: {exc}" + files = _database_files(db_path) + message = ( + f"{lock} exists ({held}): a 'build sequences-index' of {db_path} is running, or was killed. Database files " + f"present: {', '.join(path.name for path in files) if files else 'none'}. If no such job runs, delete the " + f"lock and these files, then rerun." + ) + command = code_store_command(db_path, record, shards_folder) + if command is not None: + message += ( + f" The database is complete but for its residue-code store: '{command}' adds it (delete the lock first)." + ) + return message diff --git a/src/foldmatch/search/stage2_profile.py b/src/foldmatch/search/stage2_profile.py new file mode 100644 index 0000000..59987dd --- /dev/null +++ b/src/foldmatch/search/stage2_profile.py @@ -0,0 +1,337 @@ +"""Profiling mode of esm-codes Stage 2 (plan section 8 P7, profiling gate): where the time of +:func:`~foldmatch.search.code_alignment.align_candidates_esm_codes` goes, in the parent and in the workers. + +Off by default. ``FOLDMATCH_STAGE2_PROFILE=1`` (anything but empty, ``0``, ``false``, ``no`` or ``off``) turns it on +for the next Stage-2 call; the pool's workers learn it from the worker setup. Profiling only reads clocks, sizes +and the peak RSS: the hits are the same, bit for bit, with it on or off; when it is off the aligner and the kernel +wrapper only test ``is not None`` once per batch, query, task or block. + +When it is on, the call logs one ``esm-codes Stage 2 profile`` INFO line and, given a ``stats`` dict, stores the +breakdown in ``stats["profile"]`` (:meth:`Stage2Profile.summary`): + +``parent`` seconds per parent phase: ``setup``, ``collect`` (query checks), per batch ``lookup`` (SQLite lookup of + the candidates' codes), ``plan_r1`` / ``plan_r2`` (task lists), ``round1`` / ``round2`` (wall time from the + first submit to the last result of each batch's round, including the pool start-up of the first batch; the + rounds of a batch, and of consecutive batches, overlap: the aligner runs Stage 2 as a dataflow, so the + phases add up to more than ``total``), ``consume_r1`` / + ``consume_r2`` (the parent's work on each result), ``statistics`` (S, Q90, ln E, bits, survivors), + ``fetch`` (survivor sequences), ``hits_assemble`` / ``hits_metrics`` (vectorised S12 metrics) / + ``hits_objects`` (one ``Hit`` per kept survivor, CIGAR and strings) / ``hits_sort``, ``finish``, + ``pool_close`` (a pool owned by the call), ``total``; ``profile_overhead`` is the time profiling itself + spent in the parent (pickled task sizes), included in the phases above. +``workers`` per task kind (``panel`` and ``candidates`` in round 1, ``round2``): tasks, pairs, similarity cells, + worker seconds and their split: ``prep`` (S5 query prep), ``sim`` (decode + per-pair GEMMs), ``kernel`` + (one native call per block: rowcol z + DP, and the traceback in round 2), ``paths`` (round 2: gathering + the paths), ``distinct`` (round 2: distinct-code counts), ``metrics`` (round 2: the S12 column tallies, + non-standard counts and requested strings), ``driver`` (the rest of the kernel wrapper: + block planning, buffers, ctypes) and ``other`` (the rest of the task); ns per cell for ``sim`` and + ``kernel``. +``ipc`` pickled bytes of the tasks (parent) and results (worker) with the time a ``pickle.dumps`` of them took + (estimates of the pickling cost the executor pays once more), the wait from submit to task start + (queueing included), the latency from task end to the parent's receipt, the idle gaps of each worker + between tasks, and per round the worker utilisation ``busy / (workers x wall)``. +``pool`` executor creation, workers, and per worker the delay from the first submit to its module import, its + initializer seconds and when it was ready. +``memory`` the largest worker and parent peak RSS (:func:`max_rss_bytes`) and the peak round-2 path payload held by + the parent at any time (bytes, the survivors it belonged to, and bytes per survivor). +``tasks`` the per-task records (kind, pid, n, pairs, cells, start / end, components), for the benchmark. +""" +from __future__ import annotations + +import os +import pickle +import sys +import time +from typing import Any, Dict, List, Optional + +import numpy as np + +PROFILE_ENV = "FOLDMATCH_STAGE2_PROFILE" +#: When this module was first imported in this process (in a spawn worker: right after interpreter start-up and the +#: re-import of the main module, before the initializer runs). +IMPORT_UNIX = time.time() + +KINDS = ("panel", "candidates", "round2") +_COMPONENTS = ("prep", "sim", "kernel", "paths", "distinct", "metrics", "driver", "other") + + +def profiling_enabled() -> bool: + """Whether ``FOLDMATCH_STAGE2_PROFILE`` asks for the profiling mode.""" + return os.environ.get(PROFILE_ENV, "").strip().lower() not in ("", "0", "false", "no", "off") + + +def max_rss_bytes() -> int: + """This process's peak resident set size in bytes: Linux ``VmHWM`` (of the current address space, so a spawn + worker does not inherit its parent's peak, as ``ru_maxrss`` does across fork + exec), else ``ru_maxrss`` + (bytes on macOS).""" + try: + with open("/proc/self/status") as f: + for line in f: + if line.startswith("VmHWM:"): + return int(line.split()[1]) * 1024 + except OSError: + pass + try: + import resource + except ImportError: # pragma: no cover (Windows) + return 0 + rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return int(rss if sys.platform == "darwin" else rss * 1024) + + +def process_start_unix() -> Optional[float]: + """This process's start time (Linux ``/proc``), seconds since the epoch; None elsewhere.""" + try: + with open("/proc/self/stat") as f: + fields = f.read().rsplit(")", 1)[1].split() + with open("/proc/stat") as f: + btime = next(int(line.split()[1]) for line in f if line.startswith("btime")) + return btime + int(fields[19]) / os.sysconf("SC_CLK_TCK") + except Exception: # noqa: BLE001 (informational only) + return None + + +def init_record(start_unix: float, seconds: float) -> Dict[str, Any]: + """A worker's start-up record, sent back with its first task.""" + return dict(pid=os.getpid(), process_start_unix=process_start_unix(), import_unix=IMPORT_UNIX, + init_start_unix=start_unix, init_seconds=seconds) + + +class TaskTimer: + """The phase timers of one task in a worker (or in-process): :meth:`lap` adds the time since the previous lap to + a component; the kernel wrapper adds ``sim`` / ``kernel`` / ``paths`` / ``blocks`` to :attr:`t` itself.""" + __slots__ = ("t", "start", "last", "start_unix") + + def __init__(self): + self.start_unix = time.time() + self.start = self.last = time.perf_counter() + self.t: Dict[str, float] = {} + + def lap(self, name: str) -> None: + now = time.perf_counter() + self.t[name] = self.t.get(name, 0.0) + (now - self.last) + self.last = now + + def record(self, kind: str, n: int, pairs: np.ndarray, result: tuple, init: Optional[dict]) -> Dict[str, Any]: + """The task's record (sent back as the last element of its result tuple).""" + end = time.perf_counter() + rec = dict(kind=kind, pid=os.getpid(), n=int(n), pairs=int(len(pairs)), + cells=int(n) * int(np.asarray(pairs)[:, 1].sum()) if len(pairs) else 0, + start_unix=self.start_unix, end_unix=time.time(), seconds=end - self.start, t=dict(self.t)) + p0 = time.perf_counter() + rec["result_bytes"] = len(pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL)) + rec["result_pickle_s"] = time.perf_counter() - p0 + rec["rss_max_bytes"] = max_rss_bytes() + if init is not None: + rec["init"] = init + return rec + + +def _stats(values) -> Dict[str, float]: + v = np.asarray(list(values), np.float64) + if v.size == 0: + return dict(n=0, sum=0.0, p50=0.0, p95=0.0, max=0.0) + return dict(n=int(v.size), sum=float(v.sum()), p50=float(np.percentile(v, 50)), p95=float(np.percentile(v, 95)), + max=float(v.max())) + + +class Stage2Profile: + """The parent side of one profiled Stage-2 call (see the module docstring).""" + + def __init__(self): + self.t0 = time.perf_counter() + self.parent: Dict[str, float] = {} + self.tasks: List[Dict[str, Any]] = [] + self.inits: List[Dict[str, Any]] = [] + self.rounds: List[Dict[str, Any]] = [] + self._submit: Dict[int, float] = {} + self._round: Optional[Dict[str, Any]] = None + self.task_bytes = 0 + self.task_pickle_s = 0.0 + self.executor_create_s = 0.0 + self.executors = 0 + self.workers = 0 + self.r2_parts_peak_bytes = 0 + self.r2_parts_peak_survivors = 0 + self._r2_bytes = self._r2_surv = 0 + + # ---------------------------------------------------------------------------------------------------- parent + def add(self, phase: str, seconds: float) -> None: + self.parent[phase] = self.parent.get(phase, 0.0) + seconds + + def begin_round(self, name: str, workers: int) -> int: + """A new round (``round1`` / ``round2`` of one batch); returns its index. Rounds of different batches may + overlap in time (the aligner runs Stage 2 as a dataflow).""" + self._round = dict(name=name, workers=int(workers), start_unix=time.time(), first_submit_unix=None, + tasks=0, t_start=None) + self.rounds.append(self._round) + return len(self.rounds) - 1 + + def round_started(self, rnd: int) -> None: + """The first task of round ``rnd`` is being submitted (or run in-process): its wall time starts here.""" + rd = self.rounds[rnd] + if rd["t_start"] is None: + rd["t_start"] = time.perf_counter() + rd["start_unix"] = time.time() + + def end_round(self, wall: Optional[float], workers: int, rnd: Optional[int] = None) -> float: + """Round ``rnd`` (default: the latest) is complete; ``wall`` None measures it from :meth:`round_started`. + Returns the wall time.""" + rd = self.rounds[-1 if rnd is None else rnd] if self.rounds else None + if rd is None: + return 0.0 + if wall is None: + wall = time.perf_counter() - rd["t_start"] if rd["t_start"] is not None else 0.0 + rd.update(wall=wall, end_unix=time.time(), workers=max(int(workers), rd["workers"])) + if rd is self._round: + self._round = None + return wall + + def submitted(self, task, rnd: Optional[int] = None) -> None: + """Called for each task submitted to the pool: its time, and its pickled size (an estimate of the pickling + the executor does; the time this takes is counted as profiling overhead). ``rnd``: its round (default: the + latest).""" + now = time.time() + p0 = time.perf_counter() + self.task_bytes += len(pickle.dumps(task, protocol=pickle.HIGHEST_PROTOCOL)) + dt = time.perf_counter() - p0 + self.task_pickle_s += dt + self.add("profile_overhead", dt) + rd = self.rounds[-1 if rnd is None else rnd] if self.rounds else None + self._submit[int(task[0])] = now + if rd is not None: + rd["tasks"] += 1 + if rd["first_submit_unix"] is None: + rd["first_submit_unix"] = now + + def received(self, result: tuple, rnd: Optional[int] = None) -> tuple: + """A task result with its trailing record: the record is kept (with the submit and receipt times and its + round ``rnd``, default the latest) and the plain result returned.""" + rec = result[-1] + rec["receive_unix"] = time.time() + rec["submit_unix"] = self._submit.pop(int(result[0]), None) + rec["round"] = len(self.rounds) - 1 if rnd is None else rnd + init = rec.pop("init", None) + if init is not None: + first = [r["first_submit_unix"] for r in self.rounds if r["first_submit_unix"] is not None] + init["first_submit_unix"] = min(first) if first else None + self.inits.append(init) + self.tasks.append(rec) + return result[:-1] + + def executor_created(self, seconds: float, workers: int) -> None: + self.executor_create_s += seconds + self.executors += 1 + self.workers = max(self.workers, int(workers)) + + def round2_payload(self, nbytes: int, survivors: int) -> None: + """The parent received (positive) or released (negative, once the query's hits are built) round-2 payload + (ends, paths, offsets, distinct counts) of ``survivors`` survivors; the peak of what it holds is kept.""" + self._r2_bytes += int(nbytes) + self._r2_surv += int(survivors) + if self._r2_bytes > self.r2_parts_peak_bytes: + self.r2_parts_peak_bytes = self._r2_bytes + self.r2_parts_peak_survivors = self._r2_surv + + # ---------------------------------------------------------------------------------------------------- summary + def summary(self, workers: int) -> Dict[str, Any]: + self.parent["total"] = time.perf_counter() - self.t0 + out: Dict[str, Any] = dict(parent={k: float(v) for k, v in self.parent.items()}) + kinds: Dict[str, Dict[str, float]] = {} + for rec in self.tasks: + k = kinds.setdefault(rec["kind"], dict(tasks=0, pairs=0, cells=0, seconds=0.0, + **dict.fromkeys(_COMPONENTS, 0.0))) + t = rec["t"] + k["tasks"] += 1 + k["pairs"] += rec["pairs"] + k["cells"] += rec["cells"] + k["seconds"] += rec["seconds"] + swk = t.get("swkern", 0.0) + for c in ("prep", "sim", "kernel", "paths", "distinct", "metrics"): + k[c] += t.get(c, 0.0) + k["driver"] += swk - t.get("sim", 0.0) - t.get("kernel", 0.0) - t.get("paths", 0.0) + k["other"] += rec["seconds"] - swk - t.get("prep", 0.0) - t.get("distinct", 0.0) - t.get("metrics", 0.0) + for k in kinds.values(): + cells = max(k["cells"], 1) + k["sim_ns_per_cell"] = 1e9 * k["sim"] / cells + k["kernel_ns_per_cell"] = 1e9 * k["kernel"] / cells + out["workers"] = kinds + # IPC: queueing, return latency, per-worker idle gaps, utilisation per round + waits = [r["start_unix"] - r["submit_unix"] for r in self.tasks if r.get("submit_unix") is not None] + returns = [r["receive_unix"] - r["end_unix"] for r in self.tasks] + rounds = [] + gaps_all: List[float] = [] + for i, rd in enumerate(self.rounds): + recs = [r for r in self.tasks if r["round"] == i] + busy = sum(r["seconds"] for r in recs) + wall = rd.get("wall", 0.0) or 0.0 + w = max(rd.get("workers", 1), 1) + by_pid: Dict[int, List[Dict[str, Any]]] = {} + for r in recs: + by_pid.setdefault(r["pid"], []).append(r) + gaps, first_start, tail = [], [], [] + for lst in by_pid.values(): + lst.sort(key=lambda r: r["start_unix"]) + gaps += [b["start_unix"] - a["end_unix"] for a, b in zip(lst, lst[1:])] + if rd.get("first_submit_unix") is not None: + first_start.append(lst[0]["start_unix"] - rd["first_submit_unix"]) + if rd.get("end_unix") is not None: + tail.append(rd["end_unix"] - lst[-1]["end_unix"]) + gaps_all += gaps + rounds.append(dict(name=rd["name"], tasks=len(recs), workers=w, processes=len(by_pid), wall=wall, + busy=busy, utilisation=busy / (w * wall) if wall > 0 else None, + first_task_start=_stats(first_start), idle_between_tasks=_stats(gaps), + tail_idle=_stats(tail))) + out["ipc"] = dict(task_bytes=int(self.task_bytes), task_pickle_s=float(self.task_pickle_s), + result_bytes=int(sum(r["result_bytes"] for r in self.tasks)), + result_pickle_s=float(sum(r["result_pickle_s"] for r in self.tasks)), + submit_to_start=_stats(waits), end_to_receipt=_stats(returns), + idle_between_tasks=_stats(gaps_all), rounds=rounds) + inits = [] + for rec in self.inits: + fs = rec.get("first_submit_unix") + inits.append(dict(pid=rec["pid"], init_seconds=rec["init_seconds"], + import_after_submit=(rec["import_unix"] - fs) if fs is not None else None, + ready_after_submit=(rec["init_start_unix"] + rec["init_seconds"] - fs) + if fs is not None else None, + process_start_after_submit=(rec["process_start_unix"] - fs) + if fs is not None and rec.get("process_start_unix") is not None else None)) + out["pool"] = dict(executors=self.executors, executor_create_s=float(self.executor_create_s), + workers=max(self.workers, int(workers)), worker_starts=inits, + init_seconds=_stats(i["init_seconds"] for i in inits), + ready_after_submit=_stats(i["ready_after_submit"] for i in inits + if i["ready_after_submit"] is not None)) + out["memory"] = dict(worker_rss_max_bytes=max((r["rss_max_bytes"] for r in self.tasks + if r["pid"] != os.getpid()), default=0), + in_process_rss_max_bytes=max((r["rss_max_bytes"] for r in self.tasks + if r["pid"] == os.getpid()), default=0), + parent_rss_max_bytes=max_rss_bytes(), + round2_parts_peak_bytes=int(self.r2_parts_peak_bytes), + round2_parts_peak_survivors=int(self.r2_parts_peak_survivors), + round2_parts_bytes_per_survivor=(self.r2_parts_peak_bytes + / self.r2_parts_peak_survivors + if self.r2_parts_peak_survivors else None)) + out["tasks"] = [dict(kind=r["kind"], pid=r["pid"], round=r["round"], n=r["n"], pairs=r["pairs"], + cells=r["cells"], start_unix=r["start_unix"], end_unix=r["end_unix"], + submit_unix=r.get("submit_unix"), receive_unix=r["receive_unix"], seconds=r["seconds"], + result_bytes=r["result_bytes"], **{k: float(v) for k, v in r["t"].items()}) + for r in self.tasks] + return out + + @staticmethod + def log_line(summary: Dict[str, Any]) -> str: + p, w, ipc, mem = summary["parent"], summary["workers"], summary["ipc"], summary["memory"] + parts = [f"{k} {v:.3f}" for k, v in p.items()] + kinds = [] + for name in KINDS: + k = w.get(name) + if k: + kinds.append(f"{name}: {k['tasks']} tasks {k['seconds']:.3f} worker-s (sim {k['sim']:.3f}, kernel " + f"{k['kernel']:.3f}, prep {k['prep']:.3f}, driver {k['driver']:.3f}, other " + f"{k['other']:.3f})") + util = ", ".join(f"{r['name']} {r['utilisation']:.2f}" for r in ipc["rounds"] + if r["utilisation"] is not None) + return (f"esm-codes Stage 2 profile: parent s: {', '.join(parts)}; workers: {'; '.join(kinds)}; " + f"IPC {ipc['task_bytes']} B out, {ipc['result_bytes']} B back; utilisation {util or 'n/a'}; " + f"worker RSS max {mem['worker_rss_max_bytes'] / 2**20:.1f} MiB, round-2 parts peak " + f"{mem['round2_parts_peak_bytes'] / 2**20:.2f} MiB") diff --git a/src/foldmatch/search/swint.py b/src/foldmatch/search/swint.py new file mode 100644 index 0000000..f0a80c1 --- /dev/null +++ b/src/foldmatch/search/swint.py @@ -0,0 +1,422 @@ +"""ctypes wrapper of ``fmswi_align``, the integer Smith-Waterman entry point of the native fmsw library (DP id +``fmswi/1``): the alignment engine of the BLOSUM62 Stage 2 (:mod:`foldmatch.search.alignment`). + +The kernel is output-equivalent to biotite 1.7.1 ``align_optimal(..., local=True, max_number=1)`` with affine gaps: +the same raw score, end cell and path, hence the same alignment columns (model and tie rules in ``fmsw.h``; checked +against biotite by tests/test_swint.py). It aligns one query against a batch of targets per call, so a Stage-2 task +makes one native call for all its candidates. + +numpy + ctypes only (never torch, never biotite). The library is the one :func:`foldmatch.search.swkern.load` selects +(ISA dispatch, ``FOLDMATCH_SW_ISA``, ABI and stale-source checks): a missing, stale or wrong-ABI library raises +:class:`~foldmatch.search.swkern.KernelError` with the rebuild command, as for esm-codes. +""" +from __future__ import annotations + +import ctypes as C +import hashlib +import logging +import struct +import time +from dataclasses import dataclass +from typing import Dict, List, Optional, Sequence, Tuple + +import numpy as np + +from foldmatch import _native + +from . import swkern + +logger = logging.getLogger(__name__) + +DP_ID = "fmswi/1" +NOUT = 11 +(SCORE, QSTART, QEND, TSTART, TEND, ALNLEN, NIDENT, GAPOPEN, NQGAP, NTGAP, WIDE) = range(NOUT) +MAX_SCORE = 1 << 20 # FMSWI_MAX_SCORE: |profile score| bound +MAX_GAP = 1 << 20 # FMSWI_MAX_GAP: gap_open and gap_extend bound +STATUS_INTERNAL = 3 # FMSWI_ERR_INTERNAL: the prefix traceback missed the end cell (never expected) + +#: biotite's ProteinSequence alphabet in code order (the rows / columns of its BLOSUM62). +ALPHABET = "ACDEFGHIKLMNPQRSTVWYBZX*" + +_SANITIZE = bytes(c if chr(c) in ALPHABET else ord("X") for c in range(256)) +_TO_CODE = bytes(ALPHABET.index(chr(c)) if chr(c) in ALPHABET else 255 for c in range(256)) + +_vp = C.c_void_p + +#: Traceback strategies of the kernel, with identical outputs: 'full' = one pass storing a flag byte per DP cell; +#: 'prefix' = a score-only pass, then the flag pass over rows 1..qend x columns 1..tend only. Stage 2 uses +#: :data:`DEFAULT_TRACE`: on the Stage 2 of the paper's Fig. 2 search (17.7M pairs, one 128-thread AVX2 node) both gave +#: byte-identical tables in the same wall time, prefix with less worker CPU (4,181 s against 6,103-6,700 s) and a +#: lower peak memory (35.9 against 42.4 GB: fewer flag pages touched). 'full' is kept for the tests. +TRACE_MODES = {"full": 1, "prefix": 2} +DEFAULT_TRACE = "prefix" + + +def trace_mode(name: str = DEFAULT_TRACE) -> int: + """The kernel's traceback argument for a strategy of :data:`TRACE_MODES`.""" + try: + return TRACE_MODES[name] + except KeyError: + raise ValueError(f"{name!r} is not a traceback strategy; expected one of {', '.join(TRACE_MODES)}") from None + + +class IntKernel: + """``fmswi_*`` of a loaded fmsw library.""" + + def __init__(self, kernel: swkern.Kernel): + lib = kernel.lib + try: + lib.fmswi_dp_id.restype = C.c_char_p + lib.fmswi_dp_id.argtypes = [] + lib.fmswi_align.restype = C.c_int + lib.fmswi_align.argtypes = [_vp, _vp, C.c_int, C.c_int, _vp, _vp, _vp, C.c_int, C.c_int, C.c_int, C.c_int, + _vp, _vp, C.c_size_t, _vp] + except AttributeError as e: + raise swkern.KernelError(f"{kernel.path} has no integer Smith-Waterman entry point ({e}); " + f"{_native.rebuild_hint()}") from e + dp = lib.fmswi_dp_id().decode() + if dp != DP_ID: + raise swkern.KernelError(f"{kernel.path} implements integer DP {dp!r}, expected {DP_ID!r}") + self.kernel = kernel + self.lib = lib + self.isa = kernel.isa + + def __repr__(self): + return f"IntKernel(isa={self.isa!r}, path={str(self.kernel.path)!r})" + + +_INT_KERNELS = {} + + +def load(isa: Optional[str] = None, **kw) -> IntKernel: + """The integer kernel of :func:`swkern.load` (cached; same arguments). Raises + :class:`~foldmatch.search.swkern.KernelError` (with the rebuild command) when no usable library is found.""" + k = swkern.load(isa, **kw) + ik = _INT_KERNELS.get(id(k)) + if ik is None or ik.kernel is not k: + ik = _INT_KERNELS[id(k)] = IntKernel(k) + return ik + + +# ===================================================================================================================== +# sequences +# ===================================================================================================================== +def sanitize_bytes(sequence: str) -> Optional[bytes]: + """``alignment.sanitize_sequence(sequence)`` as ASCII bytes, or None when the stripped, uppercased text is not ASCII + (the caller then uses the generic function: Python's ``upper`` and ``strip`` act on non-ASCII text).""" + text = sequence.strip().upper() + if not text.isascii(): + return None + return text.encode("ascii").translate(_SANITIZE) + + +def codes_of(clean: bytes) -> np.ndarray: + """uint8 codes (biotite order) of sanitized ASCII bytes.""" + return np.frombuffer(clean.translate(_TO_CODE), dtype=np.uint8) + + +def query_profile(qcodes: np.ndarray, matrix: np.ndarray, bias: Optional[np.ndarray] = None) -> np.ndarray: + """int32 [n, A] position-specific scores: row ``qcodes[i]`` of ``matrix`` plus ``bias[i]`` (composition bias).""" + prof = np.asarray(matrix, dtype=np.int32)[np.asarray(qcodes, dtype=np.intp)] + if bias is not None: + prof = prof + np.asarray(bias, dtype=np.int32)[:, None] + return np.ascontiguousarray(prof, dtype=np.int32) + + +# ===================================================================================================================== +# the batch call +# ===================================================================================================================== +@dataclass +class BatchResult: + """One :func:`align_batch` call: ``out`` int32 [K, NOUT] (fields SCORE .. WIDE); with ``ops`` the paths as + ``b'M' / b'D' / b'I'`` (forward order), pair k at ``ops[ops_off[k]:ops_off[k + 1]]``.""" + out: np.ndarray + ops: Optional[bytes] = None + ops_off: Optional[np.ndarray] = None + + def path(self, k: int) -> str: + return self.ops[self.ops_off[k]:self.ops_off[k + 1]].decode("ascii") + + +def align_batch(qcodes: np.ndarray, qprof: np.ndarray, tcodes: np.ndarray, t_off: Sequence[int], + t_len: Sequence[int], gap_open: int, gap_extend: int, *, traceback: bool = True, ops: bool = False, + trace: str = DEFAULT_TRACE, kernel: Optional[IntKernel] = None) -> BatchResult: + """Align one query (codes ``qcodes`` [n], profile ``qprof`` int32 [n, A]) against K targets, target k being + ``tcodes[t_off[k]:t_off[k] + t_len[k]]``. ``gap_open`` >= 1 and ``gap_extend`` >= 0 are positive costs, both at + most :data:`MAX_GAP` (a gap of length L costs open + (L - 1) extend). ``ops`` also returns the paths (traceback + only). ``trace`` picks the traceback strategy (:data:`TRACE_MODES`; the outputs do not depend on it).""" + k = kernel or load() + q = np.ascontiguousarray(qcodes, dtype=np.uint8) + prof = np.ascontiguousarray(qprof, dtype=np.int32) + n = int(q.shape[0]) + if prof.ndim != 2 or prof.shape[0] != n: + raise ValueError(f"qprof must be int32 [n={n}, A], got {prof.shape}") + A = int(prof.shape[1]) + t = np.ascontiguousarray(tcodes, dtype=np.uint8) + off = np.ascontiguousarray(t_off, dtype=np.int64) + lens = np.ascontiguousarray(t_len, dtype=np.int32) + K = int(lens.shape[0]) + if off.shape[0] != K: + raise ValueError("t_off and t_len must have the same length") + if K and (off.min() < 0 or int((off + lens).max()) > t.shape[0]): + raise ValueError("a target range lies outside tcodes") + out = np.zeros((K, NOUT), dtype=np.int32) + want_ops = bool(ops and traceback) + cap = int(K * n + int(lens.sum())) if want_ops else 0 + ops_buf = C.create_string_buffer(max(cap, 1)) if want_ops else None + ops_off = np.zeros(K + 1, dtype=np.int64) if want_ops else None + st = k.lib.fmswi_align( + q.ctypes.data if n else None, prof.ctypes.data if n else None, n, A, + t.ctypes.data if t.size else None, off.ctypes.data if K else None, lens.ctypes.data if K else None, K, + int(gap_open), int(gap_extend), trace_mode(trace) if traceback else 0, out.ctypes.data if K else None, + ops_buf, cap, ops_off.ctypes.data if want_ops else None) + if st == swkern.STATUS_ARGS: + raise ValueError("fmswi_align refused its arguments (code outside the alphabet, score or gap cost out of " + "range, gap_open < 1, or a score beyond int32)") + if st == STATUS_INTERNAL: + raise swkern.KernelError("fmswi_align: the prefix traceback pass did not meet the score pass's end cell") + if st != swkern.STATUS_OK: + raise MemoryError(f"fmswi_align could not allocate its workspace (status {st})") + if want_ops: + return BatchResult(out, ops_buf.raw[:int(ops_off[-1])], ops_off) + return BatchResult(out) + + +def align_strings(query: str, targets: List[str], matrix: np.ndarray, gap_open: int, gap_extend: int, *, + bias: Optional[np.ndarray] = None, ops: bool = False, trace: str = DEFAULT_TRACE, + kernel: Optional[IntKernel] = None) -> BatchResult: + """Convenience for tests: sanitized ASCII strings in, :func:`align_batch` out.""" + qb = query.encode("ascii").translate(_SANITIZE) + qc = codes_of(qb) + tb = [s.encode("ascii").translate(_SANITIZE) for s in targets] + lens = np.fromiter((len(b) for b in tb), dtype=np.int32, count=len(tb)) + off = np.zeros(len(tb), dtype=np.int64) + if len(tb) > 1: + np.cumsum(lens[:-1], out=off[1:]) + return align_batch(qc, query_profile(qc, matrix, bias), codes_of(b"".join(tb)), off, lens, gap_open, gap_extend, + ops=ops, trace=trace, kernel=kernel) + + +def run_length_encode(ops: bytes) -> str: + """CIGAR of a path (``b'MMMDDM'`` -> ``'3M2D1M'``).""" + out: List[str] = [] + k, n = 0, len(ops) + while k < n: + c = ops[k] + e = k + 1 + while e < n and ops[e] == c: + e += 1 + out.append(f"{e - k}{chr(c)}") + k = e + return "".join(out) + + +def gapped_strings(ops: bytes, query: str, target: str, q_start: int, t_start: int) -> Tuple[str, str]: + """The aligned query and target strings (``'-'`` for a gap) of a path starting at the 1-based (q_start, t_start).""" + qa: List[str] = [] + ta: List[str] = [] + i, j = q_start - 1, t_start - 1 + for c in ops: + if c == 77: # 'M' + qa.append(query[i]); ta.append(target[j]); i += 1; j += 1 + elif c == 68: # 'D': gap in the query + qa.append("-"); ta.append(target[j]); j += 1 + else: # 'I': gap in the target + qa.append(query[i]); ta.append("-"); i += 1 + return "".join(qa), "".join(ta) + + +# ===================================================================================================================== +# golden digest (no biotite: run by swkern.selftest and tests/test_swkern.py on every build and ISA) +# ===================================================================================================================== +GOLDEN_SEED = 20261031 +_GOLDEN_GAPS = ((11, 1), (10, 1), (5, 2), (3, 3), (1, 1), (20, 5), (2, 0), (1, 4)) +_GOLDEN_A = len(ALPHABET) + + +@dataclass(frozen=True) +class GoldenCase: + """One :func:`align_batch` call of the golden set: a query (codes and profile) against its targets.""" + kind: str + qcodes: np.ndarray # uint8 [n] + qprof: np.ndarray # int32 [n, A] + targets: Tuple[np.ndarray, ...] # uint8 codes each + gap_open: int + gap_extend: int + + +def _golden_mutant(g, seg: np.ndarray) -> np.ndarray: + """A related target: ``seg`` with a random share (0-59%) of its codes substituted, ~5% deleted, an insertion of + 0..14 codes and random flanks of 0..39 codes (never empty).""" + rate = g.int(0, 60) + sub = g.ints(0, 100, seg.shape) < rate + seg = np.where(sub, g.ints(0, _GOLDEN_A, seg.shape), seg) + seg = seg[g.ints(0, 20, seg.shape) != 0] + pos = g.int(0, len(seg) + 1) + t = np.concatenate([g.ints(0, _GOLDEN_A, (g.int(0, 40),)), seg[:pos], g.ints(0, _GOLDEN_A, (g.int(0, 15),)), + seg[pos:], g.ints(0, _GOLDEN_A, (g.int(0, 40),))]) + return (t if len(t) else g.ints(0, _GOLDEN_A, (1,))).astype(np.uint8) + + +def golden_cases(seed: int = GOLDEN_SEED) -> List[GoldenCase]: + """The deterministic inputs of the golden digest, integer only (swkern's splitmix64 stream: the same bytes on every + platform and numpy version). The profiles come from a synthetic symmetric 24 x 24 score table (diagonal 4..11, + elsewhere -4..2), not from biotite's BLOSUM62, so the digest needs only numpy: + + * ``random``: 48 queries of 1..300 codes against 4 random targets each, the gap settings in turn; + * ``related``: 48 queries against 4 mutated copies of one of their segments (substitutions, indels, flanks); + * ``bias``: 16 related cases with per-position offsets -3..3 (the shape of the composition-bias correction); + * ``longgap``: a 60-code insertion in the target, then in the query, at 11/1, 20/5 and 3/3; + * ``wide``: scores beyond the 16-bit limit (a 1,200-code query with +25 offsets against itself and a mutated + copy, at 11/1 and 1/4), a profile beyond +-16,384 and one with a single code below -32,768 (32-bit lanes from + the start), scores just below and just above the limit, 32767 - max(s), and a column best that reaches it to + the unit; + * ``carry``: a lazy-F loop long enough to wrap 16 bits without the clamp of F at -open, at 1000/1000 (n = 1024 + for 8 lanes, 2048 for 16); + * ``edge``: n = 1, m = 1, an empty target, and a profile with no positive score (empty alignments). + """ + g = swkern._SplitMix64(seed) + A = _GOLDEN_A + upper = np.triu(g.ints(-4, 3, (A, A)), 1) + table = (upper + upper.T + np.diag(g.ints(4, 12, (A,)))).astype(np.int32) + + def codes(n): + return g.ints(0, A, (n,)).astype(np.uint8) + + def case(kind, q, targets, gap, offsets=None): + prof = table[q.astype(np.intp)] + if offsets is not None: + prof = prof + np.asarray(offsets, dtype=np.int32)[:, None] + return GoldenCase(kind, q, np.ascontiguousarray(prof, dtype=np.int32), tuple(targets), gap[0], gap[1]) + + out: List[GoldenCase] = [] + for c in range(48): + q = codes(g.int(1, 301)) + out.append(case("random", q, [codes(g.int(1, 301)) for _ in range(4)], _GOLDEN_GAPS[c % len(_GOLDEN_GAPS)])) + for c in range(64): + q = codes(g.int(20, 301)) + targets = [] + for _ in range(4): + a, b = sorted((g.int(0, len(q)), g.int(0, len(q)))) + targets.append(_golden_mutant(g, q[a:b + 1])) + if c < 48: + out.append(case("related", q, targets, _GOLDEN_GAPS[c % len(_GOLDEN_GAPS)])) + else: + out.append(case("bias", q, targets, (11, 1), g.ints(-3, 4, (len(q),)))) + core, insert = codes(150), codes(60) + gapped = np.concatenate([core[:75], insert, core[75:]]) + for gap in ((11, 1), (20, 5), (3, 3)): + out.append(case("longgap", core, [gapped, np.concatenate([gapped[:120], gapped[150:]])], gap)) + out.append(case("longgap", gapped, [core], gap)) + q = codes(1200) + mutant = _golden_mutant(g, q) + for gap in ((11, 1), (1, 4)): + out.append(case("wide", q, [q, mutant, q[:40]], gap, np.full(len(q), 25))) + big = codes(200) + out.append(GoldenCase("wide", big, np.ascontiguousarray(table[big.astype(np.intp)] * 2000, dtype=np.int32), + (big, _golden_mutant(g, big)), 11, 1)) + # one code scores -40,000 on every row and no score is above 11: 32-bit lanes from the start, although the score + # (750, the query against itself, which holds that code 4 times) is far below the 16-bit limit; in 16-bit lanes + # the -40,000 would wrap to +25,536 + floor = table[q[:120].astype(np.intp)] + floor[:, q[60]] = -40000 + out.append(GoldenCase("wide", q[:120], floor, (q[:120],), 11, 1)) + # every cell scores 11, so every score is a multiple of 11 and none lands on 32767: 2,977 columns end at + # 32,747, below the 16-bit limit 32767 - 11, and 2,978 at 32,758, above it + flat = codes(3000) + out.append(GoldenCase("wide", flat, np.full((3000, A), 11, dtype=np.int32), (flat, flat[:2978], flat[:2977]), + 11, 1)) + # row 1 scores 10 and every other row 11, and the target starts with code 1, which scores only on row 1: column + # j's best is 11 j - 1, so column 2,978's 32,757 reaches the limit 32767 - 11 one column before 32,768 would wrap + lim = np.full((2979, A), 11, dtype=np.int32) + lim[:, 1] = -100 + lim[0] = 10 + lim_t = np.zeros(2979, dtype=np.uint8) + lim_t[0] = 1 + out.append(GoldenCase("wide", np.zeros(2979, np.uint8), lim, (lim_t,), 11, 1)) + # a +200 diagonal of 162 cells (32,400, codes 1..22 in turn) ends on row 384, the last row of lane 2 when a lane + # holds 128 rows (8 lanes at n = 1024, 16 at n = 2048), and every row below it scores -16,000: at 1000/1000 the + # lazy-F loop of that column runs for over 32 steps (the carry into lane 3 keeps improving G2). With F clamped at + # -open no lane goes below -2,000; unclamped, lane 0's F falls by 1,000 a step from -1,000 and wraps to +32,536, + # a false best in the last column (code 23, scoring 0) + for n in (1024, 2048): + carry = np.full((n, A), -200, dtype=np.int32) + carry[384:] = -16000 + carry[:, 23] = 0 + carry_t = np.zeros(202, dtype=np.uint8) + for d in range(162): + carry_t[39 + d] = 1 + d % 22 + carry[222 + d, 1 + d % 22] = 200 + carry_t[201] = 23 + out.append(GoldenCase("carry", np.zeros(n, np.uint8), carry, (carry_t,), 1000, 1000)) + out.append(case("edge", codes(1), [codes(1), codes(50), np.zeros(0, np.uint8)], (11, 1))) + out.append(case("edge", codes(50), [codes(1)], (5, 2))) + neg = codes(40) + out.append(GoldenCase("edge", neg, np.full((40, A), -1, dtype=np.int32), (codes(40),), 11, 1)) + return out + + +def golden_digest_update(h, case: GoldenCase, mode: int, out: np.ndarray, ops: Optional[bytes], + ops_off: Optional[np.ndarray]): + """One call's record: the inputs ( BatchResult: + """One golden case with traceback ``mode`` (0 score only, 1 'full', 2 'prefix'); paths with a traceback.""" + lens = np.fromiter((len(t) for t in case.targets), dtype=np.int32, count=len(case.targets)) + off = np.zeros(len(lens), dtype=np.int64) + if len(lens) > 1: + np.cumsum(lens[:-1], out=off[1:]) + trace = {1: "full", 2: "prefix"}.get(mode, DEFAULT_TRACE) + return align_batch(case.qcodes, case.qprof, np.concatenate(case.targets), off, lens, case.gap_open, + case.gap_extend, traceback=mode != 0, ops=mode != 0, trace=trace, kernel=kernel) + + +# sha256 of the golden records (golden_digest_update) of golden_cases() in traceback modes 0, 1 and 2, case by case. +# Computed 2026-10-31 on an Apple M1 Max: Apple clang 21 neon and generic and GCC 14.2 neon and generic (arm64), Apple +# clang sse2, avx2 and generic (x86_64 under Rosetta) all give this value; every conforming build reproduces it bit +# for bit (the kernel's values are exact integers). +GOLDEN_SHA256 = "77a7ccf5eac920023fd947758b44abff3feada5be33cb6e91dcf09f335540cab" + + +def selftest(kernel: Optional[swkern.Kernel] = None) -> Dict[str, object]: + """The integer DP's part of :func:`swkern.selftest`: the golden digest of ``kernel``'s ``fmswi_align`` (default: + the dispatched library), and the agreement of its traceback modes (1 and 2 identical; 0 the same score, end cell + and WIDE, every other field 0). Raises :class:`~foldmatch.search.swkern.KernelError` on failure.""" + t0 = time.perf_counter() + ik = load() if kernel is None else IntKernel(kernel) + head = [SCORE, QEND, TEND, WIDE] + h = golden_digest_new() + cases = golden_cases() + for case in cases: + runs = [run_golden(case, mode, ik) for mode in (0, 1, 2)] + for mode, r in enumerate(runs): + golden_digest_update(h, case, mode, r.out, r.ops, r.ops_off) + score_only, full, prefix = runs + if not (np.array_equal(full.out, prefix.out) and full.ops == prefix.ops + and np.array_equal(score_only.out[:, head], full.out[:, head]) + and not np.delete(score_only.out, head, axis=1).any()): + raise swkern.KernelError(f"fmswi selftest: the traceback modes disagree on a {case.kind!r} case " + f"(n={len(case.qcodes)}, gaps {case.gap_open}/{case.gap_extend}; {ik})") + digest = h.hexdigest() + if digest != GOLDEN_SHA256: + raise swkern.KernelError(f"fmswi selftest: golden digest {digest} != {GOLDEN_SHA256} ({ik})") + report = dict(golden_sha256=digest, cases=len(cases), pairs=sum(len(c.targets) for c in cases), + seconds=round(time.perf_counter() - t0, 3)) + logger.info("fmswi selftest passed: %s", report) + return report diff --git a/src/foldmatch/search/swkern.py b/src/foldmatch/search/swkern.py new file mode 100644 index 0000000..30f74b0 --- /dev/null +++ b/src/foldmatch/search/swkern.py @@ -0,0 +1,1159 @@ +"""ctypes wrapper of the native Smith-Waterman kernel ``fmsw`` (ABI 1, DP id ``fmsw/1``) for esm-codes Stage 2. + +numpy + ctypes only: this module never imports torch (S6: the similarity is an fp32 numpy GEMM on the CPU, never +torch, never a GPU, never TF32 / autocast). Method: ``docs/esm-codes-method.md`` (S5-S9, S16). + +Pipeline for one query against K candidates (codes: uint8 [rows, 8], packbits MSB-first):: + + qp = query_prep(q16) # S5, fixed order in C (float64 norm) + res = score_fwd_rev(qp, codes, offsets, params) # round 1: S_f, S_r (score only) + aln = align(qp, codes, offsets, params) # round 2: forward traceback (survivors) + +Each candidate's similarity is its own GEMM ``numpy.matmul(qp, D_t.T, out=buf)`` (S6) into a 64-byte-aligned, +C-contiguous ``[n, m_t]`` slice of a block buffer, so its bits depend only on the pair and the BLAS build, never on +the blocking, the candidate order or the round. ``S_r`` reads the same matrix through a row-reversed view (S9). +With the fixed-order reference similarity (``reference_sim=True``, ``fmsw_sim_ref``) the whole chain from q fp16 is +bit-identical on every build: that is the DP golden hash checked by :func:`selftest` and the tests. + +Dispatch (:func:`load`): x86_64 loads ``_fmsw_sse2`` and switches to ``_fmsw_avx2`` when ``fmsw_cpu_has_avx2()``; +arm64 loads ``_fmsw_neon``; anything else ``_fmsw_generic``. ``FOLDMATCH_SW_ISA`` overrides the choice and is +validated (``avx2`` on a CPU without AVX2 is refused rather than left to SIGILL). The library must report ABI 1 and +the source sha256 of the installed kernel source (a stale developer build is refused). + +BLAS threads: multi-core speed comes from the Stage-2 worker pool; workers keep BLAS single-threaded +(``OMP/OPENBLAS/MKL/VECLIB_*_NUM_THREADS=1``), which keeps reruns bit-identical (S16). Any thread count stays +within S16. +""" +from __future__ import annotations + +import ctypes as C +import hashlib +import logging +import os +import re +import struct +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, Iterator, List, Optional, Sequence, Tuple + +import numpy as np + +from foldmatch import _native + +logger = logging.getLogger(__name__) + +ABI_VERSION = 1 +DP_ID = "fmsw/1" +ISA_ENV = "FOLDMATCH_SW_ISA" +DEFAULT_BLOCK_FLOATS = 1 << 17 # 512 KB of similarity per block (P0.4 sweep; tuned in P7) +DEFAULT_MAX_PAIR_CELLS = 1 << 28 # about 1 GiB of similarity: larger pairs are skipped with a warning (D15) +ALIGN_BYTES = 64 +_ALIGN_FLOATS = ALIGN_BYTES // 4 +DIM = 64 # code bits / query encoding width +CODE_BYTES = 8 + +STATUS_OK, STATUS_WORKSPACE, STATUS_ARGS = 0, 1, 2 +_STATUS_TEXT = {1: "workspace or buffer too small", 2: "bad arguments"} + + +class KernelError(RuntimeError): + """The native kernel cannot be used (missing, stale, wrong ABI, refused ISA) or a call failed.""" + + +# ===================================================================================================================== +# parameters +# ===================================================================================================================== +@dataclass(frozen=True) +class DPParams: + """Smith-Waterman parameters: cell score ``z - offset``, a gap of length L costs ``gap_open + (L-1) gap_extend``. + + The kernel computes in float32, so every value must be exactly representable in float32 (no silent rounding).""" + offset: float + gap_open: float + gap_extend: float + + def __post_init__(self): + for name in ("offset", "gap_open", "gap_extend"): + v = float(getattr(self, name)) + if not np.isfinite(v) or float(np.float32(v)) != v: + raise ValueError(f"DPParams.{name}={v!r} must be finite and exactly representable in float32") + object.__setattr__(self, name, v) + if not self.gap_open >= self.gap_extend >= 0.0: + raise ValueError("DPParams needs gap_open >= gap_extend >= 0") + + +SETTING_D = DPParams(offset=1.0, gap_open=5.0, gap_extend=0.5) # esm-codes-v1 (S8); also the golden-hash setting + + +# ===================================================================================================================== +# library loading and dispatch +# ===================================================================================================================== +_vp = C.c_void_p + + +class Kernel: + """One loaded ISA library. Use :func:`load` (dispatch, caching) rather than constructing it directly.""" + + def __init__(self, path: Path, isa: str): + self.path = Path(path).resolve() + try: + lib = C.CDLL(str(self.path)) + except OSError as e: + raise KernelError(f"cannot load the native kernel {self.path}: {e}; {_native.rebuild_hint()}") from e + try: + lib.fmsw_abi_version.restype = C.c_int + lib.fmsw_abi_version.argtypes = [] + abi = int(lib.fmsw_abi_version()) + except AttributeError as e: + raise KernelError(f"{self.path} is not an fmsw kernel library ({e}); {_native.rebuild_hint()}") from e + if abi != ABI_VERSION: + raise KernelError(f"native kernel {self.path} has ABI {abi}, this FoldMatch needs ABI {ABI_VERSION}; " + f"{_native.rebuild_hint()}") + for name in ("fmsw_isa", "fmsw_dp_id", "fmsw_source_sha256", "fmsw_compiler"): + getattr(lib, name).restype = C.c_char_p + getattr(lib, name).argtypes = [] + lib.fmsw_cpu_has_avx2.restype = C.c_int + lib.fmsw_cpu_has_avx2.argtypes = [] + lib.fmsw_query_prep.restype = C.c_int + lib.fmsw_query_prep.argtypes = [_vp, C.c_int, _vp] + lib.fmsw_sim_ref.restype = C.c_int + lib.fmsw_sim_ref.argtypes = [_vp, C.c_int, _vp, C.c_int, _vp] + lib.fmsw_score_fwd_rev.restype = C.c_int + lib.fmsw_score_fwd_rev.argtypes = [_vp, C.c_int, _vp, _vp, C.c_int, C.c_float, C.c_float, C.c_float, + _vp, _vp, _vp, C.c_size_t] + lib.fmsw_align.restype = C.c_int + lib.fmsw_align.argtypes = [_vp, C.c_int, _vp, _vp, C.c_int, C.c_float, C.c_float, C.c_float, C.c_int, + _vp, _vp, _vp, C.c_size_t, _vp, _vp, C.c_size_t, _vp, C.c_size_t] + lib.fmsw_ws_floats.restype = C.c_size_t + lib.fmsw_ws_floats.argtypes = [C.c_int, C.c_int64] + lib.fmsw_flag_bytes.restype = C.c_size_t + lib.fmsw_flag_bytes.argtypes = [C.c_int, C.c_int64] + self.lib = lib + self.abi = abi + self.isa = lib.fmsw_isa().decode() + if self.isa != isa: + raise KernelError(f"{self.path} reports ISA {self.isa!r}, expected {isa!r}; {_native.rebuild_hint()}") + self.dp_id = lib.fmsw_dp_id().decode() + if self.dp_id != DP_ID: + raise KernelError(f"{self.path} implements DP {self.dp_id!r}, expected {DP_ID!r}") + self.source_sha256 = lib.fmsw_source_sha256().decode() + self.compiler = lib.fmsw_compiler().decode() + + def info(self) -> Dict[str, object]: + return dict(isa=self.isa, abi=self.abi, source_sha256=self.source_sha256, path=str(self.path), + dp_id=self.dp_id, compiler=self.compiler) + + def cpu_has_avx2(self) -> bool: + return bool(self.lib.fmsw_cpu_has_avx2()) + + def ws_floats(self, n: int, mmax: int) -> int: + return int(self.lib.fmsw_ws_floats(int(n), int(mmax))) + + def flag_bytes(self, n: int, mmax: int) -> int: + return int(self.lib.fmsw_flag_bytes(int(n), int(mmax))) + + def __repr__(self): + return f"Kernel(isa={self.isa!r}, path={str(self.path)!r})" + + +_KERNELS: Dict[tuple, Kernel] = {} +_LOAD_LOCK = threading.Lock() +_LOGGED_PATHS: set = set() + + +def _open(isa: str, search_dirs: Optional[Tuple[Path, ...]], check_source: bool) -> Kernel: + path = _native.find_library(isa, search_dirs) + if path is None: + where = ", ".join(str(d) for d in (search_dirs or (_native.NATIVE_DIR,))) + raise KernelError(f"native kernel library {_native.library_filename(isa)} not found in {where}; " + f"{_native.rebuild_hint(missing=True)}") + kernel = Kernel(path, isa) + if check_source: + tree = _native.source_sha256() + if tree is not None and kernel.source_sha256 != tree: + raise KernelError(f"stale native kernel {path}: built from source sha256 {kernel.source_sha256}, the " + f"installed source is {tree}; {_native.rebuild_hint()}") + return kernel + + +def _select(requested: Optional[str], machine: str, search_dirs, check_source: bool) -> Kernel: + machine = _native.normalized_machine(machine) + allowed = _native.machine_isas(machine) + if requested is not None: + isa = requested.strip().lower() + if isa not in _native.ISAS: + raise KernelError(f"{ISA_ENV}={requested!r} is not a kernel ISA; expected one of " + f"{', '.join(_native.ISAS)}") + if isa not in allowed: + raise KernelError(f"{ISA_ENV}={isa} cannot run on this {machine or 'unknown'} machine; available: " + f"{', '.join(allowed)}") + if isa == "avx2": + probe = _open("sse2", search_dirs, check_source) # the probe runs in the baseline library + if not probe.cpu_has_avx2(): + raise KernelError(f"{ISA_ENV}=avx2 refused: this CPU or OS does not support AVX2 (the avx2 library " + "would stop with an illegal instruction); unset it to use sse2") + return _open(isa, search_dirs, check_source) + if machine == "x86_64": + base = _open("sse2", search_dirs, check_source) + return _open("avx2", search_dirs, check_source) if base.cpu_has_avx2() else base + if machine == "arm64": + return _open("neon", search_dirs, check_source) + return _open("generic", search_dirs, check_source) + + +def load(isa: Optional[str] = None, *, search_dirs: Optional[Iterable[Path]] = None, machine: Optional[str] = None, + check_source: bool = True) -> Kernel: + """The kernel for this machine (cached). ``isa`` (else ``$FOLDMATCH_SW_ISA``) forces an ISA after validation. + ``search_dirs`` (default: the installed package directory) and ``machine`` exist for tests. + Raises :class:`KernelError` with the rebuild command when no usable library is found.""" + requested = isa if isa is not None else (os.environ.get(ISA_ENV) or "").strip() or None + m = _native.normalized_machine(machine) + dirs = tuple(Path(d) for d in search_dirs) if search_dirs is not None else None + key = (requested, m, dirs, check_source) + with _LOAD_LOCK: + kernel = _KERNELS.get(key) + if kernel is None: + kernel = _select(requested, m, dirs, check_source) + _KERNELS[key] = kernel + if str(kernel.path) not in _LOGGED_PATHS: + _LOGGED_PATHS.add(str(kernel.path)) + logger.info("fmsw kernel: %s", kernel.info()) + return kernel + + +def _check_status(status: int, what: str): + if status != STATUS_OK: + raise KernelError(f"{what} returned status {status} ({_STATUS_TEXT.get(status, 'unknown')})") + + +# ===================================================================================================================== +# buffers +# ===================================================================================================================== +def aligned_empty(n_floats: int) -> np.ndarray: + """Uninitialised float32 [n_floats] whose data starts on a 64-byte boundary, also when ``n_floats`` is 0 (an + empty numpy slice keeps its base's address, so the empty buffer is cut from an aligned one-float view).""" + n_floats = int(n_floats) + raw = np.empty(n_floats + _ALIGN_FLOATS, np.float32) + off = (-raw.ctypes.data % ALIGN_BYTES) // 4 + return raw[off:off + max(n_floats, 1)][:n_floats] + + +def _pad(cells: int) -> int: + return -(-int(cells) // _ALIGN_FLOATS) * _ALIGN_FLOATS + + +class Workspace: + """Reusable buffers of one process (one per thread via :func:`default_workspace`); grown on demand, and shrunk + only by :meth:`release` (a Stage-2 worker calls it after a task whose buffers grew past a bound, so one long pair + does not keep its buffers for the rest of the run). Not thread-safe: never share one between threads.""" + + def __init__(self): + self._ws = np.empty(0, np.float32) + self._flags = np.empty(0, np.uint8) + self._sim = aligned_empty(0) + self._dec = np.empty(0, np.float32) + self._path = np.empty(0, np.int32) + self._poff = np.empty(0, np.int64) + + def kernel_floats(self, need: int) -> np.ndarray: + if self._ws.size < need: + self._ws = np.empty(int(need), np.float32) + return self._ws + + def flags(self, need: int) -> np.ndarray: + need = max(int(need), 1) + if self._flags.size < need: + self._flags = np.empty(need, np.uint8) + return self._flags + + def sim(self, n_floats: int) -> np.ndarray: + """64-byte-aligned float32 buffer of at least ``n_floats`` floats.""" + if self._sim.size < n_floats: + self._sim = aligned_empty(max(int(n_floats), _ALIGN_FLOATS)) + return self._sim + + def decoded(self, rows: int) -> np.ndarray: + """C-contiguous float32 [rows, 64] view of the decode buffer.""" + if self._dec.size < rows * DIM: + self._dec = np.empty(int(rows) * DIM, np.float32) + return self._dec[:rows * DIM].reshape(rows, DIM) + + def path_entries(self, entries: int) -> np.ndarray: + """int32 buffer of at least ``2 x entries`` values (the traceback's (i, j) path entries).""" + need = 2 * max(int(entries), 1) + if self._path.size < need: + self._path = np.empty(need, np.int32) + return self._path + + def path_offsets(self, count: int) -> np.ndarray: + """int64 buffer of at least ``count`` values (a kernel call's path offsets).""" + if self._poff.size < count: + self._poff = np.empty(int(count), np.int64) + return self._poff + + def nbytes(self) -> int: + return int(self._ws.nbytes + self._flags.nbytes + self._sim.nbytes + self._dec.nbytes + self._path.nbytes + + self._poff.nbytes) + + def release(self, keep_bytes: int = 0) -> bool: + """Drop every buffer when they hold more than ``keep_bytes`` together (the next call allocates what it needs + again; the buffers' contents never carry from one call to the next, so the results are the same). Returns + whether they were dropped.""" + if self.nbytes() <= int(keep_bytes): + return False + self.__init__() + return True + + +_TLS = threading.local() + + +def default_workspace() -> Workspace: + ws = getattr(_TLS, "workspace", None) + if ws is None: + ws = _TLS.workspace = Workspace() + return ws + + +# ===================================================================================================================== +# S5 query prep, codes, S6 similarity +# ===================================================================================================================== +def query_prep(q16: np.ndarray, kernel: Optional[Kernel] = None) -> np.ndarray: + """S5: q fp16 [n, 64] -> qp float32 [n, 64] (C, float64 norm in ascending k, zero rows stay zero). + NaN or Inf raises ValueError.""" + q16 = np.asarray(q16) + if q16.dtype != np.float16 or q16.ndim != 2 or q16.shape[1] != DIM: + raise TypeError(f"query encodings must be float16 [n, {DIM}], got {q16.dtype} {q16.shape}") + k = kernel or load() + q = np.ascontiguousarray(q16, dtype=np.float32) # exact: every fp16 is an fp32 + qp = np.empty_like(q) + st = k.lib.fmsw_query_prep(q.ctypes.data, len(q), qp.ctypes.data) + if st == STATUS_ARGS: + raise ValueError("query encoding contains NaN or Inf (S5)") + _check_status(st, "fmsw_query_prep") + return qp + + +def _require_qp(qp: np.ndarray) -> np.ndarray: + if not isinstance(qp, np.ndarray) or qp.dtype != np.float32 or qp.ndim != 2 or qp.shape[1] != DIM: + raise TypeError(f"qp must be a float32 [n, {DIM}] numpy array (query_prep output)") + if not qp.flags.c_contiguous: + raise ValueError("qp must be C-contiguous (S6 operand layout)") + if qp.shape[0] < 1: + raise ValueError("empty query") + return qp + + +def _as_codes(codes) -> np.ndarray: + codes = np.asarray(codes) + if codes.dtype != np.uint8 or codes.ndim != 2 or codes.shape[1] != CODE_BYTES: + raise TypeError(f"codes must be uint8 [rows, {CODE_BYTES}], got {codes.dtype} {codes.shape}") + return codes + + +# +-1 table: _LUT[byte] = 2 * unpackbits(byte) - 1 (MSB first), so _LUT[codes] is decode(codes) exactly +_LUT = np.unpackbits(np.arange(256, dtype=np.uint8)[:, None], axis=1).astype(np.float32) * 2 - 1 + + +def decode_codes(codes: np.ndarray, out: Optional[np.ndarray] = None) -> np.ndarray: + """codes uint8 [m, 8] -> D float32 [m, 64] with D = 2 bit - 1 (bit k = packbits MSB-first), C-contiguous.""" + codes = _as_codes(codes) + m = len(codes) + if out is None: + out = np.empty((m, DIM), np.float32) + elif out.dtype != np.float32 or out.shape != (m, DIM) or not out.flags.c_contiguous: + raise ValueError(f"decode buffer must be C-contiguous float32 [{m}, {DIM}]") + np.take(_LUT, codes, axis=0, out=out.reshape(m, CODE_BYTES, 8), mode="clip") + return out + + +def _require_sim_out(out: np.ndarray, n: int, m: int): + if not isinstance(out, np.ndarray) or out.dtype != np.float32: + raise TypeError("similarity output must be a float32 numpy array") + if out.shape != (n, m) or not out.flags.c_contiguous: + raise ValueError(f"similarity output must be C-contiguous [{n}, {m}] (ld = m), got {out.shape}") + if out.ctypes.data % ALIGN_BYTES: + raise ValueError("similarity output must start on a 64-byte boundary (S6)") + + +def similarity(qp: np.ndarray, codes_t: np.ndarray, out: np.ndarray, *, + decoded: Optional[np.ndarray] = None) -> np.ndarray: + """S6 for one pair: ``out = numpy.matmul(qp, D_t.T)`` as one fp32 GEMM (``D_t.T`` a transposed view, never + materialised) into a 64-byte-aligned, C-contiguous [n, m] buffer. Never torch, never a GPU.""" + _require_qp(qp) + D = decode_codes(codes_t, out=decoded) + _require_sim_out(out, qp.shape[0], D.shape[0]) + if D.shape[0]: + np.matmul(qp, D.T, out=out) + return out + + +def sim_ref(qp: np.ndarray, codes_t: np.ndarray, out: Optional[np.ndarray] = None, + kernel: Optional[Kernel] = None) -> np.ndarray: + """The fixed-order test reference similarity ``fmsw_sim_ref`` (golden hash only; not the production S6).""" + _require_qp(qp) + codes_t = np.ascontiguousarray(_as_codes(codes_t)) + n, m = qp.shape[0], len(codes_t) + if out is None: + out = aligned_empty(n * m).reshape(n, m) + _require_sim_out(out, n, m) + k = kernel or load() + _check_status(k.lib.fmsw_sim_ref(qp.ctypes.data, n, codes_t.ctypes.data, m, out.ctypes.data), "fmsw_sim_ref") + return out + + +# ===================================================================================================================== +# candidate lists, blocks and the score / align drivers +# ===================================================================================================================== +_SKIPPED = {"pairs": 0, "warned": False} +_SKIP_LOCK = threading.Lock() + + +def skipped_pair_count() -> int: + """Pairs skipped by ``max_pair_cells`` in this process so far.""" + return _SKIPPED["pairs"] + + +def reset_skipped_pairs(): + with _SKIP_LOCK: + _SKIPPED.update(pairs=0, warned=False) + + +def _note_skipped(count: int, n: int, max_pair_cells: int): + if not count: + return + with _SKIP_LOCK: + _SKIPPED["pairs"] += int(count) + warn = not _SKIPPED["warned"] + _SKIPPED["warned"] = True + if warn: + logger.warning("esm-codes Stage 2: skipping pairs whose similarity exceeds max_pair_cells=%d cells " + "(first: query length %d); they get no score. Further skips are counted, not logged.", + max_pair_cells, n) + + +def _candidates(codes, offsets, lengths) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """(codes, starts int64 [K], lengths int64 [K]). ``offsets`` is either K + 1 boundaries (candidate k = + codes[offsets[k]:offsets[k+1]]) or, with ``lengths``, K start rows.""" + codes = _as_codes(codes) + off = np.asarray(offsets, dtype=np.int64).ravel() + if lengths is None: + if off.size < 1: + raise ValueError("offsets needs K + 1 >= 1 entries") + starts, lens = off[:-1], np.diff(off) + else: + starts, lens = off, np.asarray(lengths, dtype=np.int64).ravel() + if starts.shape != lens.shape: + raise ValueError("starts and lengths differ in size") + if lens.size and (lens.min() < 0 or starts.min() < 0 or int((starts + lens).max()) > len(codes)): + raise ValueError("candidate rows out of range of codes") + return codes, starts, lens + + +def plan_blocks(n: int, lengths: Sequence[int], skipped: Sequence[bool], block_floats: int) -> List[Tuple[int, int]]: + """Candidate ranges [k0, k1) whose 64-byte-padded per-pair matrices fit in ``block_floats`` floats (greedy, in + order). A candidate that alone exceeds the budget is its own block; a skipped candidate ends the current block. + Every pair gets the same layout ([n, m] at a 64-byte boundary) in any block, so blocking never changes bits.""" + if block_floats < 1: + raise ValueError("block_floats must be >= 1") + blocks, k0, acc = [], None, 0 + for k, (m, skip) in enumerate(zip(lengths, skipped)): + if skip: + if k0 is not None: + blocks.append((k0, k)) + k0 = None + continue + p = _pad(n * int(m)) + if k0 is not None and acc + p > block_floats: + blocks.append((k0, k)) + k0 = None + if k0 is None: + k0, acc = k, 0 + acc += p + if k0 is not None: + blocks.append((k0, len(lengths))) + return blocks + + +class _Layout: + """Every block of one :func:`score_fwd_rev` / :func:`align` call, laid out once per call (vectorised) instead of + block by block: the blocks of :func:`plan_blocks`; per pair its float offset from its block's start + (``pair_off`` int64 [K]: the 64-byte-padded [n, m] matrices of a block one after another, exactly the layout of + :class:`_Block`) and its length as int32 (``m32``); per block its buffer size (``floats``), its decoded rows + (``rows``) and whether its targets are consecutive rows of the codes (``contiguous``); ``mmax`` the longest + scored pair. Python lists of the per-pair values save numpy scalar access in the per-pair loops.""" + __slots__ = ("blocks", "pair_off", "m32", "floats", "rows", "contiguous", "mmax", "starts_l", "lens_l", "off_l") + + def __init__(self, n: int, starts: np.ndarray, lens: np.ndarray, skipped: np.ndarray, block_floats: int): + K = len(lens) + self.blocks = plan_blocks(n, lens.tolist(), skipped.tolist(), block_floats) + self.m32 = lens.astype(np.int32) + self.starts_l, self.lens_l = starts.tolist(), lens.tolist() + if not self.blocks: + self.pair_off = np.zeros(K, np.int64) + self.floats = self.rows = self.contiguous = [] + self.off_l, self.mmax = self.pair_off.tolist(), 0 + return + padded = -(-(n * lens) // _ALIGN_FLOATS) * _ALIGN_FLOATS + csum = np.zeros(K + 1, np.int64) + np.cumsum(padded, out=csum[1:]) + rsum = np.zeros(K + 1, np.int64) + np.cumsum(lens, out=rsum[1:]) + b0 = np.fromiter((b[0] for b in self.blocks), np.int64, len(self.blocks)) + b1 = np.fromiter((b[1] for b in self.blocks), np.int64, len(self.blocks)) + first = np.zeros(K, np.int64) # each pair's block start (blocks are increasing ranges) + first[b0] = b0 + np.maximum.accumulate(first, out=first) + self.pair_off = csum[:-1] - csum[first] # (skipped pairs belong to no block; their value is unused) + self.off_l = self.pair_off.tolist() + self.floats = (csum[b1] - csum[b0]).tolist() + self.rows = (rsum[b1] - rsum[b0]).tolist() + breaks = np.zeros(K, np.int64) + if K > 1: + np.cumsum(starts[1:] != starts[:-1] + lens[:-1], out=breaks[1:]) + self.contiguous = (breaks[b1 - 1] == breaks[b0]).tolist() + self.mmax = int(lens[~skipped].max()) + + +def _fill_block(layout: _Layout, b: int, qp: np.ndarray, codes: np.ndarray, D: np.ndarray, D3: np.ndarray, + sim: np.ndarray) -> None: + """Block ``b`` of ``layout``: its targets decoded into ``D`` (``D3`` the [rows, 8, 8] view of the same buffer) + and each pair's similarity as its own fp32 GEMM (S6) into ``sim`` at the pair's offset -- the same decode, the + same GEMM operands and the same output layout as :class:`_Block`.""" + k0, k1 = layout.blocks[b] + n = qp.shape[0] + starts_l, lens_l, off_l = layout.starts_l, layout.lens_l, layout.off_l + r = 0 + if layout.contiguous[b]: + rows = layout.rows[b] + if rows: + s0 = starts_l[k0] + np.take(_LUT, codes[s0:s0 + rows], axis=0, out=D3[:rows], mode="clip") + for k in range(k0, k1): + m = lens_l[k] + if m: + o = off_l[k] + np.matmul(qp, D[r:r + m].T, out=sim[o:o + n * m].reshape(n, m)) # S6: one fp32 GEMM per pair + r += m + return + for k in range(k0, k1): + m = lens_l[k] + if m: + s, o = starts_l[k], off_l[k] + np.take(_LUT, codes[s:s + m], axis=0, out=D3[r:r + m], mode="clip") + np.matmul(qp, D[r:r + m].T, out=sim[o:o + n * m].reshape(n, m)) # S6: one fp32 GEMM per pair + r += m + + +class _Block: + """One block's similarity matrices in the workspace buffer, filled by per-pair GEMMs (or ``fmsw_sim_ref``). + Production calls lay out all their blocks at once (:class:`_Layout`, :func:`_fill_block`: the same matrices at + the same offsets); this class remains for ``reference_sim`` and for the benchmark's per-stage timings.""" + __slots__ = ("sim", "pair_off", "m", "K") + + def __init__(self, kernel: Kernel, ws: Workspace, qp: np.ndarray, codes: np.ndarray, starts: np.ndarray, + lens: np.ndarray, k0: int, k1: int, reference_sim: bool): + n = qp.shape[0] + ms = lens[k0:k1] + padded = -(-(n * ms) // _ALIGN_FLOATS) * _ALIGN_FLOATS + pair_off = np.zeros(k1 - k0, np.int64) + if k1 - k0 > 1: + np.cumsum(padded[:-1], out=pair_off[1:]) + total = int(padded.sum()) + sim = ws.sim(total) + if sim.ctypes.data % ALIGN_BYTES: # by construction; S6 layout guard + raise KernelError("similarity block buffer is not 64-byte aligned") + self.sim, self.pair_off, self.m, self.K = sim, pair_off, ms.astype(np.int32), k1 - k0 + st, ln = starts[k0:k1], ms + if reference_sim: + qptr = qp.ctypes.data + for s, m, o in zip(st.tolist(), ln.tolist(), pair_off.tolist()): + if m: + c = np.ascontiguousarray(codes[s:s + m]) + _check_status(kernel.lib.fmsw_sim_ref(qptr, n, c.ctypes.data, m, sim[o:].ctypes.data), + "fmsw_sim_ref") + return + rows = int(ln.sum()) + D = ws.decoded(rows) + contiguous = bool(np.all(st[1:] == st[:-1] + ln[:-1])) if len(st) > 1 else True + if contiguous and rows: + s0 = int(st[0]) + np.take(_LUT, codes[s0:s0 + rows], axis=0, out=D.reshape(rows, CODE_BYTES, 8), mode="clip") + r = 0 + for s, m, o in zip(st.tolist(), ln.tolist(), pair_off.tolist()): + if not m: + continue + Dk = D[r:r + m] + if not contiguous: + np.take(_LUT, codes[s:s + m], axis=0, out=Dk.reshape(m, CODE_BYTES, 8), mode="clip") + np.matmul(qp, Dk.T, out=sim[o:o + n * m].reshape(n, m)) # S6: one fp32 GEMM per pair + r += m + + +@dataclass +class ScoreResult: + """Round-1 scores: ``s_fwd`` (S_f) and ``s_rev`` (S_r) float32 [K]; skipped pairs are NaN in both.""" + s_fwd: np.ndarray + s_rev: np.ndarray + skipped: np.ndarray + + +@dataclass +class AlignResult: + """Forward DP: ``scores`` float32 [K] (S_f), ``ends`` int32 [K, 2] (1-based best cell, (0, 0) = empty), + ``path`` int32 [P, 2] (query i, target j; 0-based, -1 = gap; forward order as the kernel wrote it), candidate k + at ``path[path_off[k]:path_off[k+1]]``; ``skipped`` pairs have a NaN score and no path.""" + scores: np.ndarray + ends: np.ndarray + path: np.ndarray + path_off: np.ndarray + skipped: np.ndarray + + def path_of(self, k: int) -> np.ndarray: + return self.path[self.path_off[k]:self.path_off[k + 1]] + + +def _prepare(qp, codes, offsets, lengths, max_pair_cells): + qp = _require_qp(qp) + codes, starts, lens = _candidates(codes, offsets, lengths) + n = qp.shape[0] + skipped = (lens * n) > int(max_pair_cells) + _note_skipped(int(skipped.sum()), n, max_pair_cells) + return qp, codes, starts, lens, skipped + + +def _add_time(timings: Dict[str, float], name: str, seconds: float) -> None: + timings[name] = timings.get(name, 0.0) + seconds + + +def score_fwd_rev(qp: np.ndarray, codes: np.ndarray, offsets, params: DPParams, *, lengths=None, + block_floats: int = DEFAULT_BLOCK_FLOATS, max_pair_cells: int = DEFAULT_MAX_PAIR_CELLS, + reference_sim: bool = False, kernel: Optional[Kernel] = None, + workspace: Optional[Workspace] = None, timings: Optional[Dict[str, float]] = None) -> ScoreResult: + """Round 1 (S9): S_f and S_r of one query against K candidates, score only. One similarity per pair (its own + GEMM); S_r runs on the row-reversed view of it. See :func:`plan_blocks` for ``block_floats``. + + ``timings`` (profiling only): a dict that receives the seconds spent in the similarity (``sim``: decode + + per-pair GEMMs) and in the kernel call (``kernel``: rowcol z + DP of S_f and S_r), and the ``blocks`` count; the + results are the same with or without it.""" + k = kernel or load() + ws = workspace or default_workspace() + qp, codes, starts, lens, skipped = _prepare(qp, codes, offsets, lengths, max_pair_cells) + n, K = qp.shape[0], len(lens) + s_f = np.full(K, np.nan, np.float32) + s_r = np.full(K, np.nan, np.float32) + if reference_sim: + for k0, k1 in plan_blocks(n, lens.tolist(), skipped.tolist(), block_floats): + b = _Block(k, ws, qp, codes, starts, lens, k0, k1, True) + kw = ws.kernel_floats(k.ws_floats(n, int(b.m.max()))) + _check_status(k.lib.fmsw_score_fwd_rev(b.sim.ctypes.data, n, b.pair_off.ctypes.data, b.m.ctypes.data, + b.K, params.offset, params.gap_open, params.gap_extend, + s_f[k0:].ctypes.data, s_r[k0:].ctypes.data, kw.ctypes.data, + kw.size), "fmsw_score_fwd_rev") + return ScoreResult(s_f, s_r, skipped) + lay = _Layout(n, starts, lens, skipped, block_floats) + if not lay.blocks: + return ScoreResult(s_f, s_r, skipped) + # one set of buffers and pointers for the call (block k's arguments are offsets into them) + sim, kw = ws.sim(max(lay.floats)), ws.kernel_floats(k.ws_floats(n, lay.mmax)) + D = ws.decoded(max(lay.rows)) + D3 = D.reshape(-1, CODE_BYTES, 8) + sim_p, po_p, m_p = sim.ctypes.data, lay.pair_off.ctypes.data, lay.m32.ctypes.data + sf_p, sr_p, kw_p, kw_n = s_f.ctypes.data, s_r.ctypes.data, kw.ctypes.data, kw.size + if sim_p % ALIGN_BYTES: # by construction; S6 layout guard + raise KernelError("similarity block buffer is not 64-byte aligned") + fn, off, go, ge = k.lib.fmsw_score_fwd_rev, params.offset, params.gap_open, params.gap_extend + for b, (k0, k1) in enumerate(lay.blocks): + if timings is not None: + t0 = time.perf_counter() + _fill_block(lay, b, qp, codes, D, D3, sim) + if timings is not None: + t1 = time.perf_counter() + _add_time(timings, "sim", t1 - t0) + st = fn(sim_p, n, po_p + 8 * k0, m_p + 4 * k0, k1 - k0, off, go, ge, sf_p + 4 * k0, sr_p + 4 * k0, kw_p, + kw_n) + if timings is not None: + _add_time(timings, "kernel", time.perf_counter() - t1) + _add_time(timings, "blocks", 1) + if st: + _check_status(st, "fmsw_score_fwd_rev") + return ScoreResult(s_f, s_r, skipped) + + +def align(qp: np.ndarray, codes: np.ndarray, offsets, params: DPParams, *, lengths=None, traceback: bool = True, + block_floats: int = DEFAULT_BLOCK_FLOATS, max_pair_cells: int = DEFAULT_MAX_PAIR_CELLS, + reference_sim: bool = False, kernel: Optional[Kernel] = None, + workspace: Optional[Workspace] = None, timings: Optional[Dict[str, float]] = None) -> AlignResult: + """Round 2: forward DP of one query against K candidates; with ``traceback`` also the paths (forward order, + exactly as the kernel wrote them; never reversed here). The scores equal :func:`score_fwd_rev`'s S_f bit for + bit (same per-pair GEMM, same DP). + + ``timings`` (profiling only): as for :func:`score_fwd_rev` (``kernel`` is rowcol z + DP + traceback here), plus + ``paths``, the seconds spent gathering the paths of the blocks; the results are the same with or without it.""" + k = kernel or load() + ws = workspace or default_workspace() + qp, codes, starts, lens, skipped = _prepare(qp, codes, offsets, lengths, max_pair_cells) + n, K = qp.shape[0], len(lens) + scores = np.full(K, np.nan, np.float32) + ends = np.zeros((K, 2), np.int32) + if reference_sim: + return _align_blockwise(k, ws, qp, codes, starts, lens, skipped, params, traceback, block_floats, scores, + ends, timings) + counts = np.zeros(K, np.int64) + lay = _Layout(n, starts, lens, skipped, block_floats) + path = np.zeros((0, 2), np.int32) + if lay.blocks: + # one set of buffers and pointers for the call; the blocks' paths go one after another into one buffer + sim, kw = ws.sim(max(lay.floats)), ws.kernel_floats(k.ws_floats(n, lay.mmax)) + D = ws.decoded(max(lay.rows)) + D3 = D.reshape(-1, CODE_BYTES, 8) + sim_p, po_p, m_p = sim.ctypes.data, lay.pair_off.ctypes.data, lay.m32.ctypes.data + sc_p, en_p, kw_p, kw_n = scores.ctypes.data, ends.ctypes.data, kw.ctypes.data, kw.size + if sim_p % ALIGN_BYTES: # by construction; S6 layout guard + raise KernelError("similarity block buffer is not 64-byte aligned") + if traceback: + fl = ws.flags(k.flag_bytes(n, lay.mmax)) + cap = int(n * int((~skipped).sum()) + int(lens[~skipped].sum())) # each pair needs <= n + m entries + pbuf = ws.path_entries(cap) + poff = ws.path_offsets(max(k1 - k0 for k0, k1 in lay.blocks) + 1) + fl_p, fl_n, pb_p, pf_p = fl.ctypes.data, fl.size, pbuf.ctypes.data, poff.ctypes.data + else: + cap, fl_p, fl_n, pb_p, pf_p = 0, None, 0, None, None + fn, off, go, ge, tb = k.lib.fmsw_align, params.offset, params.gap_open, params.gap_extend, int(bool(traceback)) + used = 0 # path entries written so far + for b, (k0, k1) in enumerate(lay.blocks): + if timings is not None: + t0 = time.perf_counter() + _fill_block(lay, b, qp, codes, D, D3, sim) + if timings is not None: + t2 = time.perf_counter() + _add_time(timings, "sim", t2 - t0) + st = fn(sim_p, n, po_p + 8 * k0, m_p + 4 * k0, k1 - k0, off, go, ge, tb, sc_p + 4 * k0, en_p + 8 * k0, + None if pb_p is None else pb_p + 8 * used, cap - used, pf_p, kw_p, kw_n, fl_p, fl_n) + if timings is not None: + _add_time(timings, "kernel", time.perf_counter() - t2) + _add_time(timings, "blocks", 1) + if st: + _check_status(st, "fmsw_align") + if traceback: + pb = poff[:k1 - k0 + 1] + counts[k0:k1] = np.diff(pb) + used += int(pb[-1]) + if traceback: + if timings is not None: + t3 = time.perf_counter() + path = pbuf[:2 * used].reshape(-1, 2).copy() # compact, and the workspace buffer is reused + if timings is not None: + _add_time(timings, "paths", time.perf_counter() - t3) + if timings is not None: + timings.setdefault("paths", 0.0) + path_off = np.zeros(K + 1, np.int64) + np.cumsum(counts, out=path_off[1:]) + return AlignResult(scores, ends, path, path_off, skipped) + + +def _align_blockwise(k: Kernel, ws: Workspace, qp, codes, starts, lens, skipped, params: DPParams, traceback: bool, + block_floats: int, scores, ends, timings) -> AlignResult: + """:func:`align` block by block through :class:`_Block` (the ``reference_sim`` path).""" + n, K = qp.shape[0], len(lens) + counts = np.zeros(K, np.int64) + parts: List[Tuple[int, np.ndarray, np.ndarray]] = [] + reference_sim = True + for k0, k1 in plan_blocks(n, lens.tolist(), skipped.tolist(), block_floats): + if timings is not None: + t0 = time.perf_counter() + b = _Block(k, ws, qp, codes, starts, lens, k0, k1, reference_sim) + if timings is not None: + t1 = time.perf_counter() + _add_time(timings, "sim", t1 - t0) + mmax = int(b.m.max()) + kw = ws.kernel_floats(k.ws_floats(n, mmax)) + if traceback: + fl = ws.flags(k.flag_bytes(n, mmax)) + cap = int(b.K * n + int(b.m.sum())) + pbuf = np.empty(2 * max(cap, 1), np.int32) + poff = np.empty(b.K + 1, np.int64) + fl_ptr, fl_len, p_ptr, po_ptr = fl.ctypes.data, fl.size, pbuf.ctypes.data, poff.ctypes.data + else: + cap, fl_ptr, fl_len, p_ptr, po_ptr = 0, None, 0, None, None + if timings is not None: + t2 = time.perf_counter() + st = k.lib.fmsw_align(b.sim.ctypes.data, n, b.pair_off.ctypes.data, b.m.ctypes.data, b.K, + params.offset, params.gap_open, params.gap_extend, int(bool(traceback)), + scores[k0:].ctypes.data, ends[k0:].ctypes.data, p_ptr, cap, po_ptr, + kw.ctypes.data, kw.size, fl_ptr, fl_len) + if timings is not None: + _add_time(timings, "kernel", time.perf_counter() - t2) + _add_time(timings, "blocks", 1) + _check_status(st, "fmsw_align") + if traceback: + counts[k0:k1] = np.diff(poff) + parts.append((k0, pbuf[:2 * int(poff[-1])].reshape(-1, 2), poff)) + if timings is not None: + t3 = time.perf_counter() + path_off = np.zeros(K + 1, np.int64) + np.cumsum(counts, out=path_off[1:]) + if parts: + path = np.concatenate([p for _, p, _ in parts]) if len(parts) > 1 else parts[0][1].copy() + else: + path = np.zeros((0, 2), np.int32) + if timings is not None: + _add_time(timings, "paths", time.perf_counter() - t3) + return AlignResult(scores, ends, path, path_off, skipped) + + +# ===================================================================================================================== +# paths: CIGAR and metrics (numpy only; no biotite) +# ===================================================================================================================== +_OP_M, _OP_I, _OP_D = ord("M"), ord("I"), ord("D") + + +def path_ops(path: np.ndarray) -> np.ndarray: + """Column types of a path as uint8 'M' (both residues), 'I' (query residue vs target gap, j == -1) and 'D' + (gap in the query, i == -1) -- the BLOSUM path's CIGAR convention.""" + p = np.asarray(path).reshape(-1, 2) + ops = np.full(len(p), _OP_M, np.uint8) + ops[p[:, 1] < 0] = _OP_I + ops[p[:, 0] < 0] = _OP_D + return ops + + +def cigar_from_path(path: np.ndarray) -> str: + ops = path_ops(path) + if not len(ops): + return "" + starts = np.flatnonzero(np.r_[True, ops[1:] != ops[:-1]]) + runs = np.diff(np.r_[starts, len(ops)]) + return "".join(f"{r}{chr(o)}" for r, o in zip(runs.tolist(), ops[starts].tolist())) + + +_CIGAR_RE = re.compile(r"(\d+)([MID])") + + +def expand_cigar(cigar: str) -> str: + """'3M1D2M' -> 'MMMDMM' (refuses anything but M/I/D runs).""" + parts = _CIGAR_RE.findall(cigar) + if "".join(f"{c}{o}" for c, o in parts) != cigar: + raise ValueError(f"not an M/I/D CIGAR: {cigar!r}") + return "".join(o * int(c) for c, o in parts) + + +@dataclass(frozen=True) +class PathMetrics: + """MMseqs2-style column tallies of one path. Bounds are 1-based (0 for an empty path); ``q_aligned`` / + ``t_aligned`` count the columns with a query / target residue (coverage = aligned / length).""" + aln_len: int + n_ident: int + mismatch: int + gap_open: int + q_aligned: int + t_aligned: int + q_start: int + q_end: int + t_start: int + t_end: int + cigar: Optional[str] = None + q_aln: Optional[str] = None + t_aln: Optional[str] = None + + +def _runs(mask: np.ndarray) -> int: + return int(mask[0]) + int(np.count_nonzero(mask[1:] & ~mask[:-1])) if mask.size else 0 + + +def path_metrics(path: np.ndarray, query_seq: str, target_seq: str, *, strings: bool = False) -> PathMetrics: + """Identity, mismatches, gap opens, bounds (and with ``strings`` the CIGAR and gapped strings) of a path over + the given sequences (already sanitised and upper-cased by the caller; compared byte for byte).""" + p = np.asarray(path, dtype=np.int64).reshape(-1, 2) + qi, tj = p[:, 0], p[:, 1] + q_gap, t_gap = qi < 0, tj < 0 + both = ~q_gap & ~t_gap + q_arr = np.frombuffer(query_seq.encode("ascii"), np.uint8) + t_arr = np.frombuffer(target_seq.encode("ascii"), np.uint8) + n_ident = int(np.count_nonzero(q_arr[qi[both]] == t_arr[tj[both]])) + q_res, t_res = qi[~q_gap], tj[~t_gap] + cigar = q_aln = t_aln = None + if strings: + cigar = cigar_from_path(p) + qa = np.full(len(p), ord("-"), np.uint8) + ta = qa.copy() + qa[~q_gap] = q_arr[q_res] + ta[~t_gap] = t_arr[t_res] + q_aln, t_aln = qa.tobytes().decode("ascii"), ta.tobytes().decode("ascii") + return PathMetrics( + aln_len=len(p), n_ident=n_ident, mismatch=int(both.sum()) - n_ident, gap_open=_runs(q_gap) + _runs(t_gap), + q_aligned=int(q_res.size), t_aligned=int(t_res.size), + q_start=int(q_res.min()) + 1 if q_res.size else 0, q_end=int(q_res.max()) + 1 if q_res.size else 0, + t_start=int(t_res.min()) + 1 if t_res.size else 0, t_end=int(t_res.max()) + 1 if t_res.size else 0, + cigar=cigar, q_aln=q_aln, t_aln=t_aln) + + +# ===================================================================================================================== +# float64 diagnostics (S16 path-tie adjudication; tests) +# ===================================================================================================================== +def query_prep64(q16: np.ndarray) -> np.ndarray: + """float64 S5: q / (8 |q|) per row, zero rows stay zero.""" + q = np.asarray(q16, dtype=np.float64) + nrm = np.sqrt(np.sum(q * q, axis=1, keepdims=True)) + zero = nrm < 1e-12 + return np.where(zero, 0.0, q / np.where(zero, 1.0, 8.0 * nrm)) + + +def rowcol_z64(sim: np.ndarray) -> np.ndarray: + """float64 S7 rowcol z with the degenerate rule (a row / column with population std < 1e-6 contributes 0).""" + s = np.asarray(sim, dtype=np.float64) + sd, tau = s.std(axis=1, keepdims=True), s.std(axis=0, keepdims=True) + zr = np.where(sd < 1e-6, 0.0, (s - s.mean(axis=1, keepdims=True)) / (sd + 1e-8)) + zc = np.where(tau < 1e-6, 0.0, (s - s.mean(axis=0, keepdims=True)) / (tau + 1e-8)) + return 0.5 * (zr + zc) + + +def z64_pair(q16: np.ndarray, codes_t: np.ndarray) -> np.ndarray: + """float64 z of a pair (S5 -> S6 -> S7 in float64).""" + return rowcol_z64(query_prep64(q16) @ decode_codes(codes_t).astype(np.float64).T) + + +def path_score(z: np.ndarray, path: np.ndarray, params: DPParams) -> float: + """Score of a path on a z matrix: sum of (z_ij - offset) over aligned pairs minus go + (L - 1) ge per maximal + gap run (X and Y runs counted separately, as in the DP).""" + p = np.asarray(path, dtype=np.int64).reshape(-1, 2) + if not len(p): + return 0.0 + qi, tj = p[:, 0], p[:, 1] + both = (qi >= 0) & (tj >= 0) + s = float(np.sum(np.asarray(z, np.float64)[qi[both], tj[both]] - params.offset)) + for gap in (tj < 0, qi < 0): + runs, total = _runs(gap), int(gap.sum()) + s -= runs * params.gap_open + (total - runs) * params.gap_extend + return s + + +def s16_score_bound(ref) -> np.ndarray: + """S16 bound on |S - S_ref| for S_f / S_r: 5e-5 max(|ref|, 1).""" + return 5e-5 * np.maximum(np.abs(np.asarray(ref, np.float64)), 1.0) + + +def s16_path_margin(s_f) -> float: + """S16 near-tie margin for paths: 1e-4 + 2e-6 |S_f|.""" + return 1e-4 + 2e-6 * abs(float(s_f)) + + +# ===================================================================================================================== +# DP golden hash and selftest +# ===================================================================================================================== +_SM_GAMMA = np.uint64(0x9E3779B97F4A7C15) +_SM_M1, _SM_M2 = np.uint64(0xBF58476D1CE4E5B9), np.uint64(0x94D049BB133111EB) + + +class _SplitMix64: + """Portable integer stream (splitmix64 of a counter): the golden inputs never depend on numpy's RNG streams or + on libm, so they are the same bytes on every platform and numpy version.""" + + def __init__(self, seed: int): + self._seed = np.uint64(seed) + self._pos = 0 + + def u64(self, size: int) -> np.ndarray: + i = np.arange(self._pos + 1, self._pos + size + 1, dtype=np.uint64) + self._pos += size + z = self._seed + i * _SM_GAMMA + z = (z ^ (z >> np.uint64(30))) * _SM_M1 + z = (z ^ (z >> np.uint64(27))) * _SM_M2 + return z ^ (z >> np.uint64(31)) + + def ints(self, lo: int, hi: int, shape=()) -> np.ndarray: + """Integers in [lo, hi) (modulo reduction; the bias is < 2^-50 for these ranges).""" + size = int(np.prod(shape)) if shape != () else 1 + v = (self.u64(size) % np.uint64(hi - lo)).astype(np.int64) + lo + return v.reshape(shape) if shape != () else v[0] + + def int(self, lo: int, hi: int) -> int: + return int(self.ints(lo, hi)) + + def qrows(self, rows: int) -> np.ndarray: + """int64 [rows, 64] in [-2048, 2048): q = value / 256 is exact in fp16.""" + return (self.u64(rows * DIM) >> np.uint64(52)).astype(np.int64).reshape(rows, DIM) - 2048 + + def code_rows(self, rows: int) -> np.ndarray: + return self.u64(rows).astype(" np.ndarray: + return (qint.astype(np.float64) / 256.0).astype(np.float16) + + +GOLDEN_SEED = 20261013 + + +def golden_pairs(count: int = 1000, seed: int = GOLDEN_SEED) -> List[Tuple[str, np.ndarray, np.ndarray]]: + """Deterministic synthetic pairs (kind, q fp16 [n, 64], codes uint8 [m, 8]): half random (n, m in [1, 400]), + 40% related (target codes = sign bits of a noisy, gapped copy of a query segment with flanks), 10% edge cases + (n = 1, m = 1, identical codes, identical query rows, zero query rows, 1 x 1). Integer arithmetic only.""" + g = _SplitMix64(seed) + n_random, n_related = count // 2, (2 * count) // 5 + out = [] + for p in range(count): + if p < n_random: + n, m = g.int(1, 401), g.int(1, 401) + out.append(("random", _q16(g.qrows(n)), g.code_rows(m))) + elif p < n_random + n_related: + n = g.int(20, 401) + q = g.qrows(n) + a, b = sorted((g.int(0, n), g.int(0, n))) + seg = q[a:b + 1] + seg = seg[g.ints(0, 10, (len(seg),)) != 0] # ~10% deletions + ins = g.qrows(g.int(0, 10)) + pos = g.int(0, len(seg) + 1) + t = np.concatenate([g.qrows(g.int(0, 60)), seg[:pos], ins, seg[pos:], g.qrows(g.int(0, 60))]) + if not len(t): + t = g.qrows(1) + amp = g.int(300, 1500) + t = t + g.ints(-amp, amp + 1, t.shape) + out.append(("related", _q16(q), np.packbits(t >= 0, axis=1))) + else: + e = (p - n_random - n_related) % 6 + n, m = g.int(1, 200), g.int(1, 200) + if e == 0: + q, c = g.qrows(1), g.code_rows(m) + elif e == 1: + q, c = g.qrows(n), g.code_rows(1) + elif e == 2: + q, c = g.qrows(n), np.repeat(g.code_rows(1), m, axis=0) + elif e == 3: + q, c = np.repeat(g.qrows(1), n, axis=0), g.code_rows(m) + elif e == 4: + q = g.qrows(n) + q[g.ints(0, 10, (n,)) < 3] = 0 + c = g.code_rows(m) + else: + q, c = g.qrows(1), g.code_rows(1) + out.append((f"edge{e}", _q16(q), c)) + return out + + +def dp_digest_update(h, n: int, m: int, s_f: float, s_r: float, path: np.ndarray): + """One pair's record: PairRun: + """S5 -> S6 (BLAS, or the reference) -> S_f, S_r (score_fwd_rev) and the forward path (align) for one pair.""" + k = kernel or load() + qp = query_prep(q16, k) + off = np.array([0, len(codes)], np.int64) + r1 = score_fwd_rev(qp, codes, off, params, reference_sim=reference_sim, kernel=k) + r2 = align(qp, codes, off, params, reference_sim=reference_sim, kernel=k) + return PairRun(float(r1.s_fwd[0]), float(r1.s_rev[0]), float(r2.scores[0]), tuple(int(x) for x in r2.ends[0]), + r2.path_of(0).copy()) + + +# sha256 of the DP golden records (dp_digest_update) of golden_pairs(1000) through fmsw_sim_ref, setting D. +# Computed 2026-10-13 on an Apple M1 Max: Apple clang 21 neon and generic (arm64), sse2 and generic (x86_64 under +# Rosetta) all give this value; every conforming build reproduces it bit for bit. tests/test_swkern.py also holds +# the F1 + synthetic constant (F1 is not shipped in wheels, so selftest() checks the synthetic part). +DP_GOLDEN_SYNTH_SHA256 = "6b7974aae7081dc8de65a8c240c24094ab0c09ae1ba276a062684f26e647017e" + + +def blas_info() -> Dict[str, object]: + """numpy's BLAS (np.show_config) and, when threadpoolctl is installed, the BLAS thread pools.""" + info: Dict[str, object] = {"numpy": np.__version__} + try: + blas = np.show_config(mode="dicts")["Build Dependencies"]["blas"] + info["blas"] = {k: blas.get(k) for k in ("name", "version", "openblas configuration") if k in blas} + except Exception as e: # pragma: no cover - numpy < 1.26 + info["blas"] = f"unavailable ({e})" + try: + import threadpoolctl + info["threadpools"] = [{k: d.get(k) for k in ("internal_api", "version", "num_threads", "architecture")} + for d in threadpoolctl.threadpool_info() if d.get("user_api") == "blas"] + except ImportError: + pass + return info + + +def selftest(kernel: Optional[Kernel] = None, *, count: int = 1000) -> Dict[str, object]: + """Check this install: the DP golden hash through ``fmsw_sim_ref`` (bit-identical on every conforming build), + then the production BLAS path against the golden values within S16 (scores; paths identical unless the float64 + path scores tie within the S16 margin), then the library's integer DP (``fmswi_align``, the BLOSUM62 Stage 2): + its golden digest (:func:`foldmatch.search.swint.selftest`, bit-identical on every conforming build). Logs the + kernel and numpy's BLAS. Raises KernelError on failure.""" + t0 = time.perf_counter() + k = kernel or load() + pairs = golden_pairs(count) + h = dp_golden_new() + ref: List[PairRun] = [] + for _kind, q16, codes in pairs: + r = run_pair(q16, codes, reference_sim=True, kernel=k) + if not (np.isfinite(r.s_f) and np.isfinite(r.s_r)) or r.s_tb != r.s_f: + raise KernelError(f"selftest: non-finite or inconsistent golden scores ({r.s_f}, {r.s_r}, {r.s_tb})") + dp_digest_update(h, len(q16), len(codes), r.s_f, r.s_r, r.path) + ref.append(r) + digest = h.hexdigest() + report: Dict[str, object] = dict(kernel=k.info(), blas=blas_info(), pairs=count, dp_golden_sha256=digest) + if count == 1000 and digest != DP_GOLDEN_SYNTH_SHA256: + raise KernelError(f"selftest: DP golden hash {digest} != {DP_GOLDEN_SYNTH_SHA256} ({k.info()})") + worst, path_diff, path_ties = 0.0, 0, 0 + for (_kind, q16, codes), g in zip(pairs, ref): + b = run_pair(q16, codes, reference_sim=False, kernel=k) + for got, want in ((b.s_f, g.s_f), (b.s_r, g.s_r)): + ratio = abs(got - want) / float(s16_score_bound(want)) + worst = max(worst, ratio) + if not np.array_equal(b.path, g.path): + path_diff += 1 + z = z64_pair(q16, codes) + if abs(path_score(z, b.path, SETTING_D) - path_score(z, g.path, SETTING_D)) >= s16_path_margin(g.s_f): + raise KernelError(f"selftest: BLAS path differs from the golden path beyond the S16 tie margin " + f"(n={len(q16)}, m={len(codes)})") + path_ties += 1 + if worst > 1.0: + raise KernelError(f"selftest: BLAS scores outside S16 (max |delta| / bound = {worst:.3g})") + from foldmatch.search import swint # imports this module: loaded here, not at the top + report.update(s16_max_score_ratio=worst, blas_path_differences=path_diff, blas_path_near_ties=path_ties, + fmswi=swint.selftest(k), seconds=round(time.perf_counter() - t0, 3)) + logger.info("fmsw selftest passed: %s", report) + return report + + +def main(argv: Optional[Sequence[str]] = None) -> int: + """``python -m foldmatch.search.swkern``: load the kernel (dispatch, or ``$FOLDMATCH_SW_ISA``), run + :func:`selftest` and print its report as JSON. Exit status 1 when the kernel is unusable or the selftest fails. + Used after installs (Docker, CI, wheels) to check that the packaged libraries work on this machine.""" + import argparse + import json + import sys + ap = argparse.ArgumentParser(prog="python -m foldmatch.search.swkern", + description="Run the native kernel selftest and print its report as JSON.") + ap.parse_args(argv) + try: + report = selftest() + except KernelError as e: + print(f"fmsw selftest failed: {e}", file=sys.stderr) + return 1 + print(json.dumps(report, indent=1, default=str)) + return 0 + + +if __name__ == "__main__": + # Run the package module's main: swint imports foldmatch.search.swkern, a second copy of this module under -m, and + # its KernelError is the one this copy's main would not catch. + from foldmatch.search.swkern import main as _main + raise SystemExit(_main()) diff --git a/src/foldmatch/types/api_types.py b/src/foldmatch/types/api_types.py index 6bc8b22..a332fe6 100644 --- a/src/foldmatch/types/api_types.py +++ b/src/foldmatch/types/api_types.py @@ -83,6 +83,12 @@ class IndexType(str, Enum): ivf_pq = "ivf_pq" # OPQ + IVF-PQ with OnDiskInvertedLists — approximate, on-disk +class ClusterMethod(str, Enum): + """Clustering method of ``fm-search cluster``.""" + leiden = "leiden" # Leiden on the similarity graph: memory grows with the edges (the default) + greedy = "greedy" # greedy representatives in one streamed pass: memory grows with the chains + + class FormatMode(str, Enum): """How a result file is laid out. @@ -95,6 +101,16 @@ class FormatMode(str, Enum): tsv = "tsv" # tab-separated with a header row +class ScoringChoice(str, Enum): + """Stage-2 scoring of ``fm-search query sequences``: one scoring per query run, never both (D7, user + 2026-10-13). ``blosum62`` is the default for now; which one becomes the default is decided before the release.""" + blosum62 = "blosum62" # BLOSUM62 Smith-Waterman (native kernel), BLAST-style statistics (the existing path) + esm_codes = "esm-codes" # ESM3 residue codes, the system named in the database manifest (S1-S16) + + +DEFAULT_SCORING = ScoringChoice.blosum62 + + class SignificanceMode(str, Enum): """Scale used for Stage-2 bit scores, p-values and E-values.""" default = "default" # calibrated constants + finite-size correction; needs gaps 11/1 diff --git a/src/foldmatch/utils/data.py b/src/foldmatch/utils/data.py index a641281..18b65a0 100644 --- a/src/foldmatch/utils/data.py +++ b/src/foldmatch/utils/data.py @@ -4,14 +4,19 @@ import pandas as pd import torch import logging +import pickle +import re import typer from requests import RequestException, ConnectTimeout from io import StringIO, BytesIO import numpy as np +import pyarrow as pa +import pyarrow.compute as pc import pyarrow.parquet as pq +from foldmatch.errors import RefusalError from foldmatch.types.api_types import ResEmbeddingFormat, StructureFormat from typing import Optional, Iterator, List, Tuple from pathlib import Path @@ -25,7 +30,62 @@ def load_residue_embedding( if res_embedding_format == ResEmbeddingFormat.csv: values = pd.read_csv(file_path, header=None, index_col=None).values return torch.from_numpy(values).float() - return torch.load(file_path, map_location=torch.device('cpu')) + return torch.load(file_path, map_location=torch.device('cpu'), weights_only=True) + + +def residue_dimension(kind: pa.DataType) -> Optional[int]: + """D when ``kind`` is the Parquet type of residue embeddings, ``list>`` (one row per + chain, one list of D values per residue, as ParquetBatchWriter(residue_embeddings=True) writes them), else None.""" + if ((pa.types.is_list(kind) or pa.types.is_large_list(kind)) and pa.types.is_fixed_size_list(kind.value_type) + and pa.types.is_floating(kind.value_type.value_type)): + return kind.value_type.list_size + return None + + +def residue_embeddings_from_parquet(parquet_path: Path) -> Tuple[List[str], List[np.ndarray]]: + """The ids and the residue embeddings ([L, D] float32 arrays) of a Parquet file whose 'embedding' column holds one + list of D values per residue (:func:`residue_dimension`): D comes from the column type. + + The column is combined once, and each table is a view of its values (Arrow flatten and a numpy reshape, no Python + lists). A :class:`RefusalError` names the file when it cannot be read as Parquet (a folder too) or lacks the column + id or embedding, and when the column is flat (a list of numbers per row: chain or assembly embeddings, or residue + embeddings as earlier versions wrote them, each table flattened), of another type, or holds a null row, residue + or value (flatten skips them, which would shift the tables after them). + """ + pf = _parquet_file(parquet_path) + kind = pf.schema_arrow.field('embedding').type + dim = residue_dimension(kind) + if dim is None: + if ((pa.types.is_list(kind) or pa.types.is_large_list(kind) or pa.types.is_fixed_size_list(kind)) + and pa.types.is_floating(kind.value_type)): + raise RefusalError( + f"{parquet_path} holds flat 'embedding' rows (one list of numbers per row): chain or assembly " + f"embeddings, or residue embeddings as earlier versions wrote them, each table flattened. Residue " + f"embeddings in Parquet now hold one list of D values per residue: recompute them, or convert an " + f"earlier file as the CHANGELOG shows." + ) + raise RefusalError( + f"{parquet_path}: its 'embedding' column is of type {kind}, not residue embeddings, which are of type " + f"list> (D in the type), as ParquetBatchWriter(residue_embeddings=True) writes " + f"them. A column of lists of D values per residue converts with .cast(pa.list_(pa.list_(pa.float32(), D)))." + ) + try: + table = pf.read(columns=['id', 'embedding']) + except (OSError, pa.ArrowException) as exc: # a corrupted page, a null residue that pyarrow cannot read back + raise RefusalError( + f"{parquet_path} could not be read as Parquet ({_cause(exc)}). Write it again: an interrupted run or a " + f"damaged copy leaves an unreadable file." + ) from exc + column = table.column('embedding').combine_chunks() + residues = column.flatten() + values = residues.flatten() + if column.null_count or residues.null_count or values.null_count: + raise RefusalError(f"{parquet_path}: its 'embedding' column holds a null row, residue or value.") + values = values.to_numpy(zero_copy_only=False).astype(np.float32, copy=False).reshape(-1, dim) + offsets = column.offsets.to_numpy() + offsets = offsets - offsets[0] + return table.column('id').to_pylist(), [values[start:end] for start, end in zip(offsets[:-1], offsets[1:])] + def collate_seq_embeddings( batch_list: List[torch.Tensor] @@ -103,6 +163,7 @@ def __stringio_from_url(url): def concatenate_tensors(file_list, max_residues, dim=0, res_embedding_format=ResEmbeddingFormat.pt): """ Concatenates a list of tensors stored in individual files along a specified dimension. + A file that cannot be loaded is skipped with a warning, by design; a ValueError is raised only when none loads. Args: file_list (list of str): List of file paths to tensor files. @@ -121,6 +182,8 @@ def concatenate_tensors(file_list, max_residues, dim=0, res_embedding_format=Res total_residues += tensor.shape[0] tensors.append(tensor) except Exception as e: + # A warning and not an error, by design: an assembly whose chains include some for which ESM3 could not be + # computed is still embedded from the chains that were. Only when no file loads is it an error (below). import logging logging.getLogger(__name__).warning(f"Error loading tensor from {file}: {e}") continue @@ -132,12 +195,6 @@ def concatenate_tensors(file_list, max_residues, dim=0, res_embedding_format=Res else: raise ValueError(f"No valid tensors were loaded to concatenate. {', '.join(file_list)}") -def adapt_csv_to_embedding_chain_stream(src_file, res_embedding_location, res_embedding_format=ResEmbeddingFormat.pt): - def __parse_row(row): - r = row.split(",") - return os.path.join(res_embedding_location, f"{r[0]}.{r[2]}.{res_embedding_format.value}"), f"{r[0]}.{r[2]}" - return tuple([__parse_row(r.strip()) for r in open(src_file) if len(r.split(",")) > 2]) - # Generated by ChatGPT 5.1 import time @@ -203,18 +260,287 @@ def run_with_retries( _BATCH_EXTS = ('.parquet',) _SUPPORTED_EXTS = _POINT_EXTS + _BATCH_EXTS + +def _cause(exc: BaseException, limit: int = 200) -> str: + """The first line of an exception's message, without ANSI codes (torch's bold, which would end the red of an + 'Error:' line) and cut to ``limit`` characters (a CSV parse error quotes the whole row). The exception's type comes + first when the line holds no letter (a KeyError's key), and alone when there is no message (torch.load of an empty + file).""" + lines = re.sub(r"\x1b\[[0-9;]*m", "", str(exc)).strip().splitlines() + text = lines[0].strip() if lines else type(exc).__name__ + if not any(c.isalpha() for c in text): + text = f"{type(exc).__name__}: {text}" + return text if len(text) <= limit else f"{text[:limit]}..." + + +def _file_vector(f: Path) -> np.ndarray: + """The embedding vector of a .pt or .csv file, as a float32 [D] array. + + The file holds one vector: a dense tensor, or a CSV of numbers, with at most one axis longer than 1 ([D], [1, D], + [D, 1]; a .csv of one column, as CsvBatchWriter writes a chain or assembly embedding, or of one line). Anything + else is refused with a :class:`RefusalError` that names the file: a file that cannot be read, no values, NaN or + infinite values (such an embedding is never found, and as a query it finds nothing) and a table, such as residue + embeddings ([L, D], one row per residue), whose mean is not a chain embedding. Non-finite values are reported + before the shape in a CSV, where a line cut short reads as NaN (as does the empty field above the index column of + pandas' to_csv layout), and after it in a .pt. + """ + ext = f.suffix[1:] + if ext == 'pt': + try: + emb = torch.load(f, map_location='cpu', weights_only=True) + except Exception as exc: # an empty or truncated file, not a torch file, a pickled object + # torch's message on a pickle that weights_only refuses advises weights_only=False, which the CLI cannot do + cause = ("it holds pickled objects other than tensors, such as a numpy array, or is not a torch file" + if isinstance(exc, pickle.UnpicklingError) else _cause(exc)) + raise RefusalError( + f"{f} could not be read as a tensor ({cause}). A .pt embedding holds one vector: a tensor of D values " + f"saved with torch.save." + ) from exc + if not isinstance(emb, torch.Tensor): + raise RefusalError( + f"{f} holds a {type(emb).__name__}, not a tensor. A .pt embedding holds one vector: a tensor of D " + f"values saved with torch.save." + ) + try: + values = emb.detach().float().numpy() + except Exception as exc: # a sparse, quantized, nested or meta tensor + raise RefusalError( + f"{f} holds a tensor that is not D dense values ({_cause(exc)}). A .pt embedding holds one vector: a " + f"tensor of D values saved with torch.save." + ) from exc + else: # .csv + try: + values = pd.read_csv(f, header=None).to_numpy(dtype=np.float32) + except (ValueError, OverflowError) as exc: # a parse, decode or empty-file error, a non-numeric or huge cell + raise RefusalError( + f"{f} is not a CSV of numbers ({_cause(exc)}). A .csv embedding holds one vector: numbers separated by " + f"commas, one per line or all on one line, with no header line or index column." + ) from exc + except OSError as exc: # a file that cannot be opened, or a folder named *.csv + raise RefusalError(f"{f} could not be read ({_cause(exc)}).") from exc + if values.size == 0: + raise RefusalError(f"{f} holds no values.") + axes = [n for n in values.shape if n > 1] # the length-1 axes dropped + bad = int(np.count_nonzero(~np.isfinite(values))) + if bad and (ext == 'csv' or len(axes) <= 1): + if (ext == 'csv' and bad == 1 and len(axes) > 1 and np.isnan(values[0, 0]) + and np.array_equal(values[0, 1:], np.arange(values.shape[1] - 1))): # ',0,1,...' over 'i,...' lines + hint = (" Its first line is a column header over an index column, as pandas' to_csv writes them unless " + "header=False and index=False: the empty field above the index reads as NaN.") + elif ext == 'csv': + hint = (" In a CSV an empty field reads as NaN: a line cut short by an interrupted write, or a comma at " + "the end of each line.") + else: + hint = "" + raise RefusalError( + f"{f} holds {bad} NaN or infinite value(s) among its {values.size}: such an embedding is never found, and " + f"as a query it finds nothing.{hint}" + ) + if len(axes) > 1: + raise RefusalError(_table_message(f, values, axes)) + return values.reshape(-1) + + +def _table_message(f: Path, values: np.ndarray, axes: List[int]) -> str: + """Why a .pt or .csv embedding file with two or more axes longer than 1 is refused, and what to do instead.""" + ext = f.suffix[1:] + text = ( + f"{f} holds a table of {' x '.join(map(str, axes))} values, not one embedding vector. 'fm-search build " + f"embeddings' and 'fm-search query embedding' read one embedding vector per .pt or .csv file (a .csv holds it " + f"as one value per line or all on one line). Residue embeddings (one row per residue, as 'fm-embedding ... " + f"residue' and 'inference residue-embedding' write them) are not chain embeddings: compute chain embeddings " + f"with 'fm-embedding from-structures chain' or 'fm-embedding from-sequences chain', or from residue files " + f"NAME.CHAIN.{ext} with 'inference chain-embedding --src-file LIST --res-embedding-location DIR " + f"--res-embedding-format {ext} --output-folder OUT', LIST holding one line NAME,,CHAIN per file." + ) + if ext == 'csv': # pandas' layouts: a header line 0,1,...,C-1 (unless header=False), an index column + rows, columns = values.shape + if np.array_equal(values[0], np.arange(columns)): + first = ",".join(map(str, range(columns))) if columns <= 3 else f"0,1,...,{columns - 1}" + text += f" Its first line is {first}: a column header such as pandas writes unless header=False; remove it." + if 2 in (rows, columns): + text += " A header line, an index column or a decimal comma makes a vector a table." + return text + + +class _OneDimension: + """The one dimension of a stream's embeddings: the database's when given (a query), else the first item's.""" + + def __init__(self, dimension: Optional[int]): + self.dimension = dimension + self.first = None # the item that set the dimension; None when it is the database's + + def check(self, item: str, n: int) -> None: + """Refuse ``item`` (a file, or a Parquet file and row) when its ``n`` values are not the stream's dimension.""" + if self.dimension is None: + self.dimension, self.first = n, item + elif n != self.dimension and self.first is None: + raise RefusalError( + f"{item} has {n} values, but the database has dimension {self.dimension}: query it with embeddings " + f"of the kind it was built from." + ) + elif n != self.dimension: + raise RefusalError( + f"{item} has {n} values, but {self.first} has {self.dimension}: all embeddings of a database have one " + f"dimension." + ) + + +def _parquet_file(f: Path) -> pq.ParquetFile: + """The Parquet file ``f``, which must hold the columns id and embedding (``iter_batches`` and ``read`` would drop a + missing one without a word). A :class:`RefusalError` names a file that cannot be opened, with the system's reason + (a missing path or broken link, no read permission: pyarrow's message puts it after the whole path, past + :func:`_cause`'s cut), a folder, a file that cannot be read as Parquet (empty, or without the footer that an + interrupted run leaves out) and a file without one of the two columns.""" + try: + pf = pq.ParquetFile(f) + except (OSError, pa.ArrowException) as exc: + if isinstance(exc, OSError) and exc.errno: # not opened: a missing path or broken link, no read permission + raise RefusalError(f"{f} could not be opened ({os.strerror(exc.errno)}).") from exc + if isinstance(f, (str, Path)) and Path(f).is_dir(): # a dataset folder named *.parquet, as Spark writes one + raise RefusalError(f"{f} is a folder, not a Parquet file.") from exc + raise RefusalError( + f"{f} could not be read as Parquet ({_cause(exc)}). Write it again: an interrupted run or a damaged copy " + f"leaves an unreadable file." + ) from exc + missing = [f"'{name}'" for name in ('id', 'embedding') if name not in pf.schema_arrow.names] + if missing: + raise RefusalError( + f"{f} has no {' or '.join(missing)} column: a Parquet embedding file holds the columns id and embedding." + ) + return pf + + +def _read_batches(f: Path, batches: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]: + """The ``batches`` of the Parquet file ``f``: a read error (a corrupted page) is a :class:`RefusalError` that names + the file. An error in the caller's loop body is raised in the caller, not in this generator, so it is not caught.""" + try: + yield from batches + except (OSError, pa.ArrowException) as exc: + raise RefusalError( + f"{f} could not be read as Parquet ({_cause(exc)}). Write it again: an interrupted run or a damaged copy " + f"leaves an unreadable file." + ) from exc + + +def _parquet_vectors(f: Path, batch_size: int, dimension: _OneDimension) -> Iterator[tuple[list, np.ndarray]]: + """The (ids, [B, D] float32) batches of a Parquet file's columns id and embedding, one vector per row. + + A list of floats stores no shape, so every row of the file must hold as many values as its first row, across all + its batches: a file of residue embeddings flattened one table per row, whose rows differ in length, is refused + (its values were cut into equal slices, one per row). So are residue embeddings as FoldMatch writes them (one + list of values per residue, :func:`residue_dimension`), any other 'embedding' column that is not a list of + floats, a row with no values (null or empty) and NaN or infinite values. The file's first row is checked against + the stream's dimension. + """ + pf = _parquet_file(f) + kind = pf.schema_arrow.field('embedding').type + if residue_dimension(kind) is not None: + raise RefusalError( + f"{f} holds residue embeddings (one list of values per residue), not one embedding vector per row: compute " + f"chain embeddings with 'fm-embedding from-structures chain' or 'fm-embedding from-sequences chain', or in " + f"Python with foldmatch.inference.chain_inference.predict(src_stream=FILE, src_from=SrcTensorFrom.parquet, " + f"out_folder=OUT), which writes one chain embedding per .csv file to the existing folder OUT." + ) + if not ((pa.types.is_list(kind) or pa.types.is_large_list(kind) or pa.types.is_fixed_size_list(kind)) + and pa.types.is_floating(kind.value_type)): + raise RefusalError( + f"{f}: its 'embedding' column is of type {kind}, not a list of floating-point numbers: a row holds one " + f"embedding vector." + ) + first = None # the file's first row (id, number of values) + for record in _read_batches(f, pf.iter_batches(batch_size=batch_size, columns=['id', 'embedding'])): + ids = record.column('id').to_pylist() + column = record.column('embedding') + lengths = pc.list_value_length(column).fill_null(0).to_numpy(zero_copy_only=False) + empty = np.flatnonzero(lengths == 0) + if empty.size: + raise RefusalError(f"{f}: row {ids[int(empty[0])]} has no embedding.") + first_batch = first is None + if first_batch: + first = (ids[0], int(lengths[0])) + other = np.flatnonzero(lengths != first[1]) + if other.size: + i = int(other[0]) + raise RefusalError( + f"{f}: row {first[0]} has {first[1]} values, but row {ids[i]} has {int(lengths[i])}. A row holds one " + f"embedding vector; rows of different lengths are usually residue embeddings flattened by an earlier " + f"FoldMatch: compute chain embeddings with 'fm-embedding from-structures chain' or 'fm-embedding " + f"from-sequences chain'." + ) + if first_batch: # after the row lengths, whose refusal says more about residue embeddings + dimension.check(f"{f} (row {first[0]})", first[1]) + # flatten() honours a slice's offset (.values does not); the lengths are checked, so the reshape is exact. + flat = column.flatten().to_numpy(zero_copy_only=False) + arr = np.ascontiguousarray(flat.reshape(len(ids), first[1]).astype(np.float32, copy=False)) + finite = np.isfinite(arr) + if not finite.all(): + i = int(np.flatnonzero(~finite.all(axis=1))[0]) + raise RefusalError( + f"{f}: row {ids[i]} holds {first[1] - int(finite[i].sum())} NaN or infinite value(s) among its " + f"{first[1]}: such an embedding is never found, and as a query it finds nothing." + ) + yield ids, arr + + def stream_embeddings( path: Path, file_extension: Optional[str] = None, batch_size: int = 32768, + expected_dim: Optional[int] = None, ) -> Iterator[tuple[list[str], np.ndarray]]: """Yield (ids, [B, D] float32) batches from a file or directory of .pt / .csv / .parquet. - Parquet shards are streamed via ``ParquetFile.iter_batches`` (many chains per file). - .pt and .csv hold one chain per file (ID = filename stem) and are chunked into - batches of ``batch_size``. A single file is handled as a degenerate directory-of-one; - ``file_extension`` is ignored in that case. + Parquet shards are streamed via ``ParquetFile.iter_batches`` (many chains per file): each + row of the 'embedding' column, a list of floats, holds one vector, every row of a file as + many values as its first row (:func:`_parquet_vectors`; residue embeddings, null + rows and NaN are refused). .pt and .csv hold one chain or assembly vector per file (ID = + filename stem) and are chunked into batches of ``batch_size``: a tensor or CSV with at + most one axis longer than 1 ([D], [1, D], [D, 1]; a .csv of one column, as CsvBatchWriter + writes a vector, or of one line). A table, such as residue embeddings ([L, D], one row per + residue), is refused with a :class:`RefusalError` naming the file, not averaged: its mean + is not a chain embedding. So are files that cannot be read, of no values or with NaN or + infinite values (:func:`_file_vector`), and a .pt and a .csv of one stem (one id twice). + Every embedding has one dimension: ``expected_dim`` (a query's database's) when given, + else the first one's (Parquet files are read first, then .pt, then .csv). A single file is + handled as a degenerate directory-of-one; ``file_extension`` is ignored in that case. """ + parquet_files, point_files = embedding_files(path, file_extension) + + by_id = {} + for f in point_files: + by_id.setdefault(f.stem, []).append(f) + pairs = [same for same in by_id.values() if len(same) > 1] # a .pt and a .csv of one stem + if pairs: + a, b = pairs[0][:2] + more = f", as do {len(pairs) - 1} more pair(s) of .pt and .csv files" if len(pairs) > 1 else "" + raise RefusalError( + f"{a} and {b} give the same id, {a.stem}{more}: the id of a .pt or .csv embedding is its file name " + f"without the extension. Read one kind of file with --file-extension .pt or --file-extension .csv (runs " + f"that write residue and chain embeddings to one folder leave such pairs)." + ) + + dimension = _OneDimension(expected_dim) + for parquet_file in parquet_files: + yield from _parquet_vectors(parquet_file, batch_size, dimension) + + ids_buf: list[str] = [] + emb_buf: list[np.ndarray] = [] + for f in point_files: + vector = _file_vector(f) + dimension.check(str(f), vector.shape[0]) + ids_buf.append(f.stem) + emb_buf.append(vector) + if len(ids_buf) >= batch_size: + yield ids_buf, np.ascontiguousarray(np.stack(emb_buf)) + ids_buf, emb_buf = [], [] + if ids_buf: + yield ids_buf, np.ascontiguousarray(np.stack(emb_buf)) + + +def embedding_files(path: Path, file_extension: Optional[str] = None) -> Tuple[List[Path], List[Path]]: + """The (Parquet files, .pt and .csv files) that stream_embeddings reads from a file or a folder, each list sorted + by extension then name; a ValueError for a missing path, an unsupported extension or a folder without them.""" p = Path(path) if not p.exists(): raise ValueError(f"Embeddings path does not exist: {path}") @@ -244,36 +570,39 @@ def stream_embeddings( parquet_files = [f for f in files if f.suffix in _BATCH_EXTS] point_files = [f for f in files if f.suffix in _POINT_EXTS] + return parquet_files, point_files - for parquet_file in parquet_files: - pf = pq.ParquetFile(parquet_file) - for record in pf.iter_batches(batch_size=batch_size, columns=['id', 'embedding']): - ids = record.column('id').to_pylist() - flat = record.column('embedding').values.to_numpy(zero_copy_only=False) - arr = np.ascontiguousarray( - flat.reshape(len(record), -1).astype(np.float32, copy=False) - ) - yield ids, arr - ids_buf: list[str] = [] - emb_buf: list[np.ndarray] = [] +def refuse_repeated_query_ids(path: Path, file_extension: Optional[str] = None): + """Refuse (RefusalError) query embeddings in which two vectors share an id (P9.N D7), read before any search as + stream_embeddings would read them: the ids of the Parquet files (their id column only) and the stems of the .pt and + .csv files. A search's results are kept by query id, so a repeated id kept one vector's hits and dropped the + others' without notice. A Parquet file whose ids cannot be read is left to stream_embeddings, which refuses it.""" + parquet_files, point_files = embedding_files(path, file_extension) + where = {} + for f in parquet_files: + try: + ids = pq.read_table(f, columns=["id"]).column("id").to_pylist() + except Exception: + continue + for embedding_id in ids: + where.setdefault(embedding_id, []).append(f.name) for f in point_files: - if f.suffix == '.pt': - emb = torch.load(f, map_location='cpu', weights_only=True) - if isinstance(emb, torch.Tensor): - emb = emb.detach().cpu().numpy() - else: # .csv - emb = pd.read_csv(f, header=None).values - emb = np.asarray(emb, dtype=np.float32) - if emb.ndim > 1: - emb = np.mean(emb, axis=0) - ids_buf.append(f.stem) - emb_buf.append(emb.reshape(-1)) - if len(ids_buf) >= batch_size: - yield ids_buf, np.ascontiguousarray(np.stack(emb_buf)) - ids_buf, emb_buf = [], [] - if ids_buf: - yield ids_buf, np.ascontiguousarray(np.stack(emb_buf)) + where.setdefault(f.stem, []).append(f.name) + repeated = [(embedding_id, files) for embedding_id, files in where.items() if len(files) > 1] + if repeated: + def describe(item): + embedding_id, files = item + counts = {name: files.count(name) for name in dict.fromkeys(files)} + return f"{embedding_id!r} in " + " and ".join( + f"{n} rows of {name}" if n > 1 else name for name, n in counts.items()) + listed = "; ".join(describe(item) for item in repeated[:5]) + more = f"; and {len(repeated) - 5} more ids" if len(repeated) > 5 else "" + raise RefusalError( + f"{path}: {len(repeated)} id(s) held by more than one query embedding: {listed}{more}. The results are " + f"written by query id, so a repeated id would keep the hits of one of its embeddings and drop the others': " + f"give each embedding its own id, or query its files apart (--file-extension, or one folder per kind)." + ) STRUCTURE_FORMAT_EXTENSIONS = { StructureFormat.pdb: ('.pdb', '.pdb.gz'), diff --git a/src/foldmatch/utils/dist.py b/src/foldmatch/utils/dist.py new file mode 100644 index 0000000..6d0b27f --- /dev/null +++ b/src/foldmatch/utils/dist.py @@ -0,0 +1,175 @@ +"""Run identity shared by every rank of a distributed inference pass. + +Each rank writes its own embedding shard, ``emb_{run_id}-{rank}.parquet``, and +rank 0 reads all of them back once the pass is over. That only works if every +rank derives the same ``run_id`` *before* the process group exists, from what +it was launched with: + +* ``FOLDMATCH_RUN_ID`` when set (by the user, or inherited from the parent); +* in a task of an ``srun`` step, whose ranks SLURM starts itself, + ``{SLURM_JOB_ID}_{SLURM_STEP_ID}_{SLURM_RESTART_COUNT}_{digest}``: the + same on every task of the step and different for each step and requeue, + while ``digest``, of the command's arguments, tells apart the commands that + one step runs one after another (a build, then a query); +* otherwise a fresh random id, which only the process that starts the run may + draw. It exports the id (:func:`exported_run_id`) around ``trainer.predict`` + so the child processes Lightning launches inherit it. A rank > 0 without an + id refuses to start, rather than writing a shard rank 0 never reads. + +The batch step of an ``sbatch`` script is not an srun step: it is one task, +so only Lightning can start more ranks there, and they inherit the id. Each +command it runs therefore draws its own id, instead of all of them sharing +the job's. + +Dependency-free on purpose: it runs before torch or any model is loaded. +""" +import hashlib +import os +import re +import sys +from contextlib import contextmanager +from secrets import token_hex +from typing import Iterator, Optional + +RUN_ID_ENV = "FOLDMATCH_RUN_ID" + +# SLURM numbers ordinary steps below this; the values above it name the batch, +# extern, interactive and pending steps (SLURM_MAX_NORMAL_STEP_ID in slurm.h). +_SLURM_MAX_NORMAL_STEP_ID = 0xFFFFFFF0 + +# The id becomes part of file names and of a glob pattern, so it is restricted +# to characters that are neither path separators nor glob metacharacters. +_RUN_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") + + +def is_run_id(value) -> bool: + """Whether ``value`` is a well-formed run id (what :func:`run_id` accepts in ``FOLDMATCH_RUN_ID``).""" + return isinstance(value, str) and _RUN_ID_RE.fullmatch(value) is not None + + +def _env_int(name: str) -> Optional[int]: + value = os.environ.get(name, "").strip() + return int(value) if value else None + + +def launch_rank() -> int: + """This process's rank as given by its launch environment. + + Read from environment variables only, so it can be called before the + process group is initialised. ``RANK`` (torchrun) is the global rank. If + it is absent, the result is the largest of ``SLURM_PROCID``, ``NODE_RANK`` + and ``LOCAL_RANK`` (Lightning's launcher): exact on a single node or under + SLURM, and in every case 0 exactly for the process that starts the run. + """ + rank = _env_int("RANK") + if rank is not None: + return rank + return max(_env_int("SLURM_PROCID") or 0, _env_int("NODE_RANK") or 0, _env_int("LOCAL_RANK") or 0) + + +def srun_step_id() -> Optional[str]: + """``SLURM_STEP_ID`` when this process is a task of an ``srun`` step, else ``None``. + + The batch step of an sbatch script has no ``SLURM_STEP_ID``, or one of + SLURM's special step values, and is not an srun step. + """ + step = os.environ.get("SLURM_STEP_ID", "").strip() + if not os.environ.get("SLURM_JOB_ID", "").strip(): + return None + if not step.isdigit() or int(step) >= _SLURM_MAX_NORMAL_STEP_ID: + return None + return step + + +def _whole_number(name: str) -> Optional[int]: + """``name``'s value as a whole number; ``None`` if it is unset or not one.""" + value = os.environ.get(name, "").strip() + return int(value) if value.isdigit() else None + + +def multi_rank_launch() -> Optional[str]: + """``NAME=value`` of the launch variable that shows this process is one of several ranks, else ``None``. + + A rank above 0 (``RANK``, ``LOCAL_RANK``, ``NODE_RANK`` or ``SLURM_PROCID``), ``WORLD_SIZE`` above 1, or a + task of an srun step (:func:`srun_step_id`) with more than one task (``SLURM_STEP_NUM_TASKS``, else + ``SLURM_NTASKS``). This is the rule of + :func:`~foldmatch.inference.chain_complete_inference.bundle_fetch_allowed`: the batch step of an sbatch script + is one process, whatever ``SLURM_NTASKS`` its job has. Values that are not whole numbers are ignored. For the + commands that must run as one process (``build sequences-index``, ``build sequences-seal``). + """ + for name in ("RANK", "LOCAL_RANK", "NODE_RANK", "SLURM_PROCID"): + rank = _whole_number(name) + if rank is not None and rank > 0: + return f"{name}={rank}" + world_size = _whole_number("WORLD_SIZE") + if world_size is not None and world_size > 1: + return f"WORLD_SIZE={world_size}" + if srun_step_id() is not None: + name = "SLURM_STEP_NUM_TASKS" if _whole_number("SLURM_STEP_NUM_TASKS") else "SLURM_NTASKS" + tasks = _whole_number(name) + if tasks is not None and tasks > 1: + return f"{name}={tasks}" + return None + + +def command_digest() -> str: + """Eight hex digits of SHA-1 over this command's arguments (``sys.argv[1:]``). + + Every task of an srun step runs the same command line, so they agree; + two different commands in one step do not. The program path is left out, + so ``fm-search ...`` and ``python -m foldmatch.cli.search ...`` agree too. + """ + arguments = b"\0".join(os.fsencode(argument) for argument in sys.argv[1:]) + return hashlib.sha1(arguments).hexdigest()[:8] + + +def run_id() -> str: + """The id shared by every rank of this run (see the module docstring). + + Raises ``ValueError`` for a malformed ``FOLDMATCH_RUN_ID``, and on a rank + > 0 that has no id to share: drawing its own would write a shard under a + name rank 0 never reads, silently dropping that rank's embeddings. + """ + explicit = os.environ.get(RUN_ID_ENV) + if explicit: + if not _RUN_ID_RE.fullmatch(explicit): + raise ValueError( + f"{RUN_ID_ENV}={explicit!r} is not a valid run id: use letters, digits, " + f"'.', '_' and '-' only, starting with a letter or digit." + ) + return explicit + step_id = srun_step_id() + if step_id is not None: + job_id = os.environ["SLURM_JOB_ID"].strip() + restarts = os.environ.get("SLURM_RESTART_COUNT", "").strip() or "0" + return f"{job_id}_{step_id}_{restarts}_{command_digest()}" + rank = launch_rank() + if rank > 0: + raise ValueError( + f"This process is rank {rank} of a distributed run but has no run id to share " + f"with rank 0, so rank 0 would never read its embeddings. Set {RUN_ID_ENV} to " + f"the same value on every rank whenever neither srun nor Lightning starts the " + f"ranks (torchrun, or one command per node with NODE_RANK set)." + ) + return token_hex(8) + + +@contextmanager +def exported_run_id(rid: str) -> Iterator[None]: + """Export ``rid`` as ``FOLDMATCH_RUN_ID`` for the duration of the block. + + Wrap ``trainer.predict`` in it: Lightning launches the other ranks as child + processes that re-run the command and inherit the environment, so they + compute the same :func:`run_id`. The previous value is restored afterwards: + an id the user set stays set, and one exported here is cleared, so a later + run in the same process draws a fresh id. + """ + previous = os.environ.get(RUN_ID_ENV) + os.environ[RUN_ID_ENV] = rid + try: + yield + finally: + if previous is None: + os.environ.pop(RUN_ID_ENV, None) + else: + os.environ[RUN_ID_ENV] = previous diff --git a/src/foldmatch/utils/esm/loaders.py b/src/foldmatch/utils/esm/loaders.py index 2b35019..a98e6ca 100644 --- a/src/foldmatch/utils/esm/loaders.py +++ b/src/foldmatch/utils/esm/loaders.py @@ -1,20 +1,52 @@ +"""ESM3: the pinned snapshot, its weights check, the model builder and the per-residue embedding. + +torch and esm are imported only by the functions that build or run the model, so the pins, the weights check and +:func:`download_esm3_snapshot` (``download-models``) work in processes without torch, and importing this module +never touches a GPU. +""" +from __future__ import annotations + +import hashlib +import os +import re +import threading +import weakref +from contextlib import contextmanager from pathlib import Path -import torch -from esm.models.esm3 import ESM3 -from esm.models.vqvae import StructureTokenEncoder -from esm.tokenization import TokenizerCollection, EsmSequenceTokenizer, StructureTokenizer, SecondaryStructureTokenizer, \ - SASADiscretizingTokenizer, InterProQuantizedTokenizer, ResidueAnnotationsTokenizer +from huggingface_hub import hf_hub_download, snapshot_download, try_to_load_from_cache +from huggingface_hub.errors import LocalEntryNotFoundError + +ESM_REPO_ID = "rcsb/rcsb-esm" +# Pinned: the aggregator and everything calibrated downstream read these weights as frozen +# features, so a new upload to the repo must not change them silently. +ESM_REVISION = "636acb420daf674df37ab0220f56061a5de2f169" +ESM_WEIGHTS_FILE = "data/weights/esm3_sm_open_v1.pth" +# The same blob is served by rcsb/rcsb-esm@636acb42 and the EvolutionaryScale/esm3-sm-open-v1 snapshots. +ESM_WEIGHTS_SHA256 = "5ead5a135c658068db6a4f1b933e72d6110992c4668822e1c0e2dcc53e38acd9" + +_SHA256_HEX = re.compile(r"[0-9a-f]{64}") +# Files fully hashed by this process, keyed by identity, so one process never hashes the +# same file twice. Nothing is written to disk: the Hugging Face cache may be shared or read-only. +_hashed = {} + + +class EsmWeightsMismatchError(ValueError): + """The ESM3 weights file is not the pinned release (``ESM_WEIGHTS_SHA256``). + + A ``ValueError``, so the command-line tools report it as a one-line + ``Error: ...`` with its remedy instead of a traceback. + """ -from huggingface_hub import snapshot_download class DataRoot: _instance = None def __new__(cls): if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance.path = Path(snapshot_download(repo_id="rcsb/rcsb-esm")) + instance = super().__new__(cls) + instance.path = _esm_snapshot() + cls._instance = instance return cls._instance def get(self): @@ -25,13 +57,109 @@ def data_root(): return DataRoot().get() +def _esm_snapshot(): + # A file that was not in the cache before this call is hashed once after its download; + # a cached one is checked by its blob name only (see verify_esm_weights). + cached = isinstance( + try_to_load_from_cache(ESM_REPO_ID, ESM_WEIGHTS_FILE, revision=ESM_REVISION), str + ) + path = Path(snapshot_download(repo_id=ESM_REPO_ID, revision=ESM_REVISION)) + verify_esm_weights(path / ESM_WEIGHTS_FILE, full_hash=not cached) + return path + + +def verify_esm_weights(weights_path, full_hash=False): + """Check that ``weights_path`` holds the pinned ESM3 weights (``ESM_WEIGHTS_SHA256``). + + In the Hugging Face cache layout the file is a symlink to ``blobs/``, so the + blob name already is the content hash and the check costs nothing. When the layout + hides it (``local_dir`` downloads, copied snapshots, filesystems without symlinks), or + when ``full_hash`` is set, the file is hashed, once per process. + Returns ``"blob-name"`` or ``"sha256"``, the way the file was verified. + """ + weights_path = Path(weights_path) + resolved = Path(os.path.realpath(weights_path)) + if not full_hash and resolved.parent.name == "blobs" and _SHA256_HEX.fullmatch(resolved.name): + _check_esm_sha256(weights_path, resolved.name) + return "blob-name" + _check_esm_sha256(weights_path, _sha256_once(resolved)) + return "sha256" + + +def _sha256_once(resolved): + """The sha256 of the file ``resolved``, hashed at most once per process for one file identity.""" + stat = resolved.stat() + key = (str(resolved), stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns) + if key not in _hashed: + _hashed[key] = _file_sha256(resolved) + return _hashed[key] + + +def _check_esm_sha256(weights_path, sha256): + if sha256 != ESM_WEIGHTS_SHA256: + raise EsmWeightsMismatchError( + f"ESM3 weights {weights_path} have sha256 {sha256}, expected {ESM_WEIGHTS_SHA256} " + f"({ESM_REPO_ID}@{ESM_REVISION}). The file is corrupt or not the pinned release: " + f"remove it and run download-models again." + ) + + +def _file_sha256(path, chunk_size=1 << 24): + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download_esm3_snapshot(*, cache_dir=None) -> Path: + """``download-models``: fetch the pinned ESM3 snapshot (``ESM_REPO_ID`` at ``ESM_REVISION``, the snapshot + :func:`data_root` serves) into the Hugging Face cache (``cache_dir``, else ``HF_HUB_CACHE`` / ``HF_HOME``) and + verify the weights against ``ESM_WEIGHTS_SHA256`` by a full sha256 (once per process). Returns the snapshot folder. + + Download only: no torch, no model, no device. A cached snapshot is not downloaded again (with ``HF_HUB_OFFLINE=1`` + nothing touches the network). A cached weights file whose content differs from the pin (a truncated or corrupted + blob, whose name alone would pass) is downloaded once more (``force_download``); it is refused + (:class:`EsmWeightsMismatchError`) if that fails or the fresh file still differs. + """ + try: + snapshot = Path(snapshot_download(repo_id=ESM_REPO_ID, revision=ESM_REVISION, cache_dir=cache_dir)) + except LocalEntryNotFoundError as e: # offline or Hub unreachable, and not cached + raise FileNotFoundError(f"ESM3 snapshot {ESM_REPO_ID}@{ESM_REVISION} is not in the Hugging Face cache and " + f"could not be downloaded ({e})") from e + weights = snapshot / ESM_WEIGHTS_FILE + if not weights.is_file(): + raise FileNotFoundError( + f"ESM3 weights {ESM_WEIGHTS_FILE} are missing from the cached snapshot {snapshot} " + f"({ESM_REPO_ID}@{ESM_REVISION}); run download-models with the Hugging Face Hub reachable " + "(HF_HUB_OFFLINE unset)") + try: + verify_esm_weights(weights, full_hash=True) + return snapshot + except EsmWeightsMismatchError as mismatch: + corrupt = mismatch + try: + hf_hub_download(repo_id=ESM_REPO_ID, filename=ESM_WEIGHTS_FILE, revision=ESM_REVISION, cache_dir=cache_dir, + force_download=True) + except (OSError, ValueError) as e: # offline, Hub unreachable (requests errors are OSErrors), a full disk + raise EsmWeightsMismatchError(f"{corrupt} Downloading it again failed ({type(e).__name__}: {e}).") from e + got = _sha256_once(Path(os.path.realpath(weights))) + if got != ESM_WEIGHTS_SHA256: + raise EsmWeightsMismatchError( + f"ESM3 weights {ESM_WEIGHTS_FILE} ({ESM_REPO_ID}@{ESM_REVISION}) have sha256 {got} after a fresh " + f"download, expected {ESM_WEIGHTS_SHA256}: the file at the pinned revision is not the pinned release") + return snapshot + + def structure_encoder(device: torch.device | str = "cpu"): + import torch + from esm.models.vqvae import StructureTokenEncoder with torch.device(device): model = StructureTokenEncoder( d_model=1024, n_heads=1, v_heads=128, n_layers=2, d_out=128, n_codes=4096 ).eval() state_dict = torch.load( - data_root() / "data/weights/esm3_structure_encoder_v0.pth", map_location=device + data_root() / "data/weights/esm3_structure_encoder_v0.pth", map_location=device, weights_only=True ) model.load_state_dict(state_dict) return model @@ -39,6 +167,8 @@ def structure_encoder(device: torch.device | str = "cpu"): def get_model_tokenizers(): + from esm.tokenization import TokenizerCollection, EsmSequenceTokenizer, StructureTokenizer, \ + SecondaryStructureTokenizer, SASADiscretizingTokenizer, InterProQuantizedTokenizer, ResidueAnnotationsTokenizer class CustomAnnotationsTokenizer(ResidueAnnotationsTokenizer): def __init__(self, csv_path: str | None = None, max_annotations: int = 16): @@ -59,6 +189,8 @@ def __init__(self, csv_path: str | None = None, max_annotations: int = 16): def esm_open(device: torch.device | str = "cpu"): + import torch + from esm.models.esm3 import ESM3 with torch.device(device): model = ESM3( d_model=1536, @@ -71,7 +203,63 @@ def esm_open(device: torch.device | str = "cpu"): tokenizers=get_model_tokenizers(), ).eval() state_dict = torch.load( - data_root() / "data/weights/esm3_sm_open_v1.pth", map_location=device + data_root() / ESM_WEIGHTS_FILE, map_location=device, weights_only=True ) model.load_state_dict(state_dict) - return model \ No newline at end of file + return model + + +# One re-entrant lock per ESM3 model. geometric_attention_off flips a flag on the model itself, +# so a forward on the same model in another thread must not run while the flag is off. +_MODEL_LOCKS = weakref.WeakKeyDictionary() +_MODEL_LOCKS_GUARD = threading.Lock() +_SHARED_LOCK = threading.RLock() # for objects that cannot be weakly referenced + + +def _model_lock(model): + with _MODEL_LOCKS_GUARD: + try: + lock = _MODEL_LOCKS.get(model) + if lock is None: + lock = _MODEL_LOCKS[model] = threading.RLock() + except TypeError: + lock = _SHARED_LOCK + return lock + + +@contextmanager +def geometric_attention_off(model, off=True): + """Switch off ESM3's geometric attention (block 0) for the duration of the block, then restore it. + + Without coordinates every frame is masked, so geometric attention adds exactly zero to the + residue stream, but it still allocates an O(L^2) tensor per head: ESM3 ran out of memory at + 2,750 aa on a 40 GB A100, and fits 16,000 aa in about 10 GiB with it off. The embeddings are + bit-identical (checked on GPU for 300-2,000 aa and on CPU for 300-1,000 aa). + + The flag belongs to the model, which other threads may share, so the block holds the model's + lock (also with ``off=False``): a forward on the same model in another thread that goes + through :func:`per_residue_embedding` waits, instead of running without geometric attention. + """ + with _model_lock(model): + blocks = [block for block in model.transformer.blocks if block.use_geom_attn] if off else [] + for block in blocks: + block.use_geom_attn = False + try: + yield + finally: + for block in blocks: + block.use_geom_attn = True + + +def per_residue_embedding(model, esm_prot): + """ESM3 per-residue embedding of one protein, (L + 2, 1536) with BOS and EOS. + + Sequence-only proteins (no coordinates) run without geometric attention; proteins with + coordinates take ESM3's path unchanged. Every call holds the model's lock, so calls on one + model from several threads run one at a time (calls on different models do not wait). + """ + from esm.sdk.api import SamplingConfig + with geometric_attention_off(model, off=esm_prot.coordinates is None): + return model.forward_and_sample( + model.encode(esm_prot), SamplingConfig(return_per_residue_embeddings=True) + ).per_residue_embedding diff --git a/src/foldmatch/utils/fasta.py b/src/foldmatch/utils/fasta.py index 9c98a53..4f131b7 100644 --- a/src/foldmatch/utils/fasta.py +++ b/src/foldmatch/utils/fasta.py @@ -77,6 +77,14 @@ def iter_fasta_offsets(fasta_file: Path) -> Iterator[Tuple[int, str, int]]: yield header_offset, name, length +def read_id_at(fh: IO, offset: int) -> str: + """The id of the record whose header (``>``) line starts at ``offset`` of an open handle: the first token of + the header, as :func:`read_record_at` reads it. Leaves ``fh`` just after the header line.""" + fh.seek(offset) + header = fh.readline().strip() + return header[1:].split()[0] + + def read_record_at(fh: IO, offset: int) -> Tuple[str, str]: """Read a single ``(id, sequence)`` record from an open handle at ``offset``. @@ -85,9 +93,7 @@ def read_record_at(fh: IO, offset: int) -> Tuple[str, str]: The caller owns ``fh`` (it is re-``seek``-ed on each call, so the trailing header that terminates the record need not be consumed). """ - fh.seek(offset) - header = fh.readline().strip() - name = header[1:].split()[0] + name = read_id_at(fh, offset) chunks: list = [] while True: line = fh.readline() diff --git a/src/foldmatch/utils/model.py b/src/foldmatch/utils/model.py index 2db1e10..8c1f9bf 100644 --- a/src/foldmatch/utils/model.py +++ b/src/foldmatch/utils/model.py @@ -1,15 +1,39 @@ -import torch +"""Model downloads from Hugging Face: the aggregator, ESM3 and the esm-codes scoring bundles. -from huggingface_hub import hf_hub_download -from foldmatch.model.residue_embedding_aggregator import ResidueEmbeddingAggregator -from foldmatch.utils.esm.loaders import esm_open +torch and ESM3 are imported only by the functions that build models, so the esm-codes bundle loader +(:func:`get_esm_codes_bundle`) and the download-only functions behind ``download-models`` +(:func:`download_aggregator_weights`, :func:`foldmatch.utils.esm.loaders.download_esm3_snapshot`, +:func:`download_esm_codes_bundles`) work in processes without torch. +""" +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Dict, Iterable, Optional + +from huggingface_hub import constants as hf_constants, hf_hub_download +from huggingface_hub.errors import ( + EntryNotFoundError, LocalEntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError, +) + +from foldmatch.search import scoring_system +from foldmatch.search.scoring_system import BundleMismatchError, BundleUnavailableError, EsmCodesSystem REPO_ID = "rcsb/rcsb-embedding-model" FILE_NAME = "rcsb-embedding-model.pt" REVISION = "410606e40b1bb7968ce318c41009355c3ac32503" +# sha256 of FILE_NAME at REVISION (its Hugging Face LFS blob name), checked by download-models. +AGGREGATOR_SHA256 = "0dc20e4fb5f5767bd1c30c9e67b011186c2c68d8eef596ffad121388c567b687" + + +class AggregatorWeightsMismatchError(ValueError): + """The aggregator weights file is not the pinned one (``AGGREGATOR_SHA256``). A ``ValueError``, so the + command-line tools report it as a one-line ``Error: ...``.""" def get_aggregator_model(device=None): + import torch + from foldmatch.model.residue_embedding_aggregator import ResidueEmbeddingAggregator model_path = hf_hub_download( repo_id=REPO_ID, filename=FILE_NAME, @@ -24,7 +48,159 @@ def get_aggregator_model(device=None): def get_residue_model(device=None): + import torch + from foldmatch.utils.esm.loaders import esm_open if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") return esm_open(device) + +def download_aggregator_weights(*, cache_dir=None) -> Path: + """``download-models``: fetch the pinned aggregator weights (``REPO_ID``/``FILE_NAME`` at ``REVISION``, the file + :func:`get_aggregator_model` loads) into the Hugging Face cache (``cache_dir``, else ``HF_HUB_CACHE`` / + ``HF_HOME``) and verify their sha256 (``AGGREGATOR_SHA256``, a full hash). Returns the file's path. + + Download only: no torch, no model, no device. A cached file is not downloaded again (no network at a commit + revision). A cached file whose sha256 differs is downloaded once more (``force_download``); it is refused + (:class:`AggregatorWeightsMismatchError`) if that fails or the fresh file still differs. + """ + where = f"{REPO_ID}@{REVISION}" + try: + path = Path(hf_hub_download(repo_id=REPO_ID, filename=FILE_NAME, revision=REVISION, cache_dir=cache_dir)) + except LocalEntryNotFoundError as e: # offline or Hub unreachable, and not cached + raise FileNotFoundError(f"aggregator weights {FILE_NAME} ({where}) are not in the Hugging Face cache and " + f"could not be downloaded ({e})") from e + got = _file_sha256(path) + if got == AGGREGATOR_SHA256: + return path + corrupt = (f"aggregator weights {path} ({where}) have sha256 {got}, expected {AGGREGATOR_SHA256}: the cached " + "copy is corrupt or was replaced.") + try: + path = Path(hf_hub_download(repo_id=REPO_ID, filename=FILE_NAME, revision=REVISION, cache_dir=cache_dir, + force_download=True)) + except (OSError, ValueError) as e: # offline, Hub unreachable (requests errors are OSErrors), a full disk + raise AggregatorWeightsMismatchError(f"{corrupt} Downloading it again failed ({type(e).__name__}: {e}).") from e + got = _file_sha256(path) + if got != AGGREGATOR_SHA256: + raise AggregatorWeightsMismatchError( + f"aggregator weights {FILE_NAME} ({where}) have sha256 {got} after a fresh download, expected " + f"{AGGREGATOR_SHA256}: the file at the pinned revision is not the pinned one") + return path + + +def release_mps_cache(device): + """On the Apple GPU (MPS), wait for the batch's kernels and return the allocator's cached + memory to the system. Without it an ESM3 pass grew to 83 GB and slowed to 41 residues/s.""" + import torch + if torch.device(device).type == "mps": + torch.mps.synchronize() + torch.mps.empty_cache() + + +# ===================================================================================================================== +# esm-codes bundles +# ===================================================================================================================== +def _file_sha256(path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 24), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _fetch_file(system: EsmCodesSystem, filename: str, *, cache_dir, local_files_only: bool, + force_download: bool = False) -> str: + """One bundle file through ``hf_hub_download`` at the system's commit; every failure becomes a bundle error.""" + where = f"{system.bundle_repo}@{system.bundle_revision}" + cache = cache_dir or hf_constants.HF_HUB_CACHE + try: + return hf_hub_download(repo_id=system.bundle_repo, filename=filename, revision=system.bundle_revision, + cache_dir=cache_dir, local_files_only=local_files_only, force_download=force_download) + except LocalEntryNotFoundError as e: + if local_files_only: + mode = "cached-only use was requested (local_files_only=True)" + elif hf_constants.HF_HUB_OFFLINE: + mode = "offline mode is on (HF_HUB_OFFLINE=1)" + else: + mode = "the Hugging Face Hub is unreachable" + raise BundleUnavailableError( + f"esm-codes bundle {where}: {filename} is not in the Hugging Face cache ({cache}) and {mode}. Run " + "`fm-embedding download-models` once where the Hub is reachable, with the same HF_HOME / HF_HUB_CACHE." + ) from e + except (EntryNotFoundError, RevisionNotFoundError, RepositoryNotFoundError) as e: + raise BundleMismatchError(f"esm-codes bundle {where} has no {filename} ({type(e).__name__}); the " + f"registry entry {system.name} does not match the Hub") from e + except PermissionError as e: + raise BundleUnavailableError( + f"esm-codes bundle {where}: {filename} is not in the Hugging Face cache ({cache}) and the cache is not " + f"writable ({e}). Run `fm-embedding download-models` with a writable HF_HOME / HF_HUB_CACHE; a read-only " + "cache works once it holds the bundle.") from e + except OSError as e: # HTTP errors (requests' exceptions are OSErrors), a full disk, ... + raise BundleUnavailableError(f"esm-codes bundle {where}: {filename} could not be fetched into {cache} " + f"({type(e).__name__}: {e})") from e + except ValueError as e: + if not force_download: + raise + # hf_hub_download refuses a forced download offline and wraps a failed one in a ValueError + raise BundleUnavailableError(f"esm-codes bundle {where}: {filename} could not be downloaded again into " + f"{cache} ({e})") from e + + +def get_esm_codes_bundle(system: EsmCodesSystem, *, cache_dir=None, local_files_only: bool = False, + redownload_corrupt: bool = False) -> Dict[str, Path]: + """The files of ``system``'s bundle, ``{file name: local path}``, every one sha256-verified. + + ``system`` must be a registry entry, exactly as registered (:func:`scoring_system.registered`): the repo, the + revision and the file sha256s come only from the registry (``system.bundle_repo``, ``system.bundle_revision``, + ``system.bundle_files``), and a system from elsewhere (a manifest) that differs from its entry is refused before + any Hub call. Each file goes through ``hf_hub_download`` at that commit: a cached file is returned without any + network access or write (so ``HF_HUB_OFFLINE=1`` and read-only caches work), a missing one is downloaded into + the cache (``cache_dir``, else ``HF_HUB_CACHE`` / ``HF_HOME``) unless ``local_files_only``. Every call hashes + every file (about 26 MB for esm-codes-v1, tens of ms) and writes no marker files, since the cache may be shared + or read-only. + + A cached file whose sha256 differs is refused (:class:`BundleMismatchError`, naming the cache blob to delete); + with ``redownload_corrupt`` (``download-models``) it is downloaded once more with ``force_download`` and checked + again, and refused only if the Hub's file itself differs from the pin. + + Refused: anything but an :class:`EsmCodesSystem`, an unpinned revision (None, a branch, a short hash), a system + that is not its registry entry (:class:`UnknownScoringSystemError`), :class:`BundleUnavailableError` when a file is + neither cached nor downloadable (offline, cached-only, Hub unreachable, a read-only cache without the bundle), and + :class:`BundleMismatchError` when a file is missing at the pinned revision or its sha256 differs. + """ + if not isinstance(system, EsmCodesSystem): + raise TypeError(f"get_esm_codes_bundle needs a registry EsmCodesSystem, got {type(system).__name__}") + if not isinstance(system.bundle_revision, str) or not scoring_system.is_commit(system.bundle_revision): + raise ValueError(f"{system.name}: bundle revision {system.bundle_revision!r} is not a 40-hex commit; " + "an unpinned bundle download is refused") + scoring_system.registered(system) + where = f"{system.bundle_repo}@{system.bundle_revision}" + files: Dict[str, Path] = {} + for filename, sha256 in system.bundle_files: + path = _fetch_file(system, filename, cache_dir=cache_dir, local_files_only=local_files_only) + got = _file_sha256(path) + if got != sha256 and redownload_corrupt and not local_files_only: + path = _fetch_file(system, filename, cache_dir=cache_dir, local_files_only=False, force_download=True) + got = _file_sha256(path) + if got != sha256: + raise BundleMismatchError( + f"esm-codes bundle file {filename} ({where}) has sha256 {got} after a fresh download, expected " + f"{sha256} for {system.name}: the file at the pinned revision differs from the registry entry") + if got != sha256: + blob = Path(path).resolve() + linked = f" (the cache blob {path} links to)" if blob != Path(path).absolute() else "" + raise BundleMismatchError( + f"esm-codes bundle file {path} ({where}) has sha256 {got}, expected {sha256} for {system.name}: the " + "cached copy is corrupt or was replaced. Run `fm-embedding download-models`, which downloads such a " + f"file again, or delete {blob}{linked} and fetch again.") + files[filename] = Path(path) + return files + + +def download_esm_codes_bundles(names: Optional[Iterable[str]] = None, *, cache_dir=None) -> Dict[str, Dict[str, Path]]: + """Fetch and verify the bundles of the named systems (default: every registered system), ``{name: files}``. + Unknown names are refused (listing the known ones) before anything is downloaded; a corrupt cached file is + downloaded again (``redownload_corrupt``).""" + names = list(dict.fromkeys(names)) if names else list(scoring_system.known_names()) + systems = [scoring_system.get(n) for n in names] + return {s.name: get_esm_codes_bundle(s, cache_dir=cache_dir, redownload_corrupt=True) for s in systems} diff --git a/src/foldmatch/utils/output_files.py b/src/foldmatch/utils/output_files.py new file mode 100644 index 0000000..564ce7c --- /dev/null +++ b/src/foldmatch/utils/output_files.py @@ -0,0 +1,626 @@ +"""The files of one run, compared as the folder that holds them compares names (P9.N). + +One run never writes two of its outputs to one file, never writes an output onto one of its own inputs, and never +writes outside its output folder. The checks here refuse such a run (:class:`RefusalError`) before any model is built, +on every rank alike: they read the inputs' names and the folders, and write nothing but the probe of a folder (a +temporary file, removed at once). A file that exists because an earlier run wrote it is not their concern: a rerun into +the same folder replaces its own earlier files, as before. + +Names are compared as the folder's file system compares them: exactly on a case-sensitive one; case-folded and +Unicode-normalised (canonical caseless matching) on a folder that takes 'A' and 'a', or the NFC and NFD spellings of +'é', for one name (macOS APFS by default), with full case folding ('ß' as 'ss') where the folder folds so (APFS does), +which :func:`probe_folding` finds out per folder. Folders are compared as files (device and inode), never as strings: +a symbolic link and, on macOS, ``SHARED`` and ``shared`` are one folder; and an input and an output's file that +exists are one file when either is a link to the other from another folder. +""" +import glob +import os +import re +import stat +import tempfile +import unicodedata +from typing import Iterable, List, NamedTuple, Optional, Sequence, Tuple + +from foldmatch.errors import RefusalError + +#: The longest file name, in bytes of UTF-8, that the usual file systems take (ext4, XFS, APFS, Lustre, GPFS). +MAX_FILE_NAME_BYTES = 255 +#: The digits reserved for the rank in -.parquet and -.json.gz. +RANK_DIGITS = 6 +#: The suffix of the one file per process of each format that writes one. +STEM_SUFFIXES = {"parquet": ".parquet", "json": ".json.gz"} +#: The formats that write one file per item, .. +ITEM_FORMATS = ("csv", "pt") +#: How many examples a refusal shows. +SHOWN = 5 + + +def shown(items: Sequence, describe=str, more: str = "more") -> str: + """The first SHOWN items, described and joined by '; ', and "and N more" for the rest.""" + listed = "; ".join(describe(item) for item in items[:SHOWN]) + return listed + (f"; and {len(items) - SHOWN} {more}" if len(items) > SHOWN else "") + + +def format_value(out_format) -> str: + """An OutFormat (a str Enum) or its value, as its value.""" + return getattr(out_format, "value", out_format) + + +# ===================================================================================================================== +# D1: a name that becomes a file name +# ===================================================================================================================== +def _missing(name) -> bool: + """Whether a name is missing: None, or NaN (pandas' missing value, numpy's or float's), or pandas.NA.""" + if name is None: + return True + try: + return bool(name != name) + except TypeError: # pandas.NA, whose comparisons are NA + return True + + +def name_problem(name, suffix: str) -> Optional[str]: + """Why ```` cannot be one file of the output folder, or None: an empty or missing name (None, NaN), + '.', '..', a name that holds '/' (or the system's separators) or NUL, or a file name of more than + MAX_FILE_NAME_BYTES bytes in UTF-8. Another scalar, such as an integer id of a Parquet file, is judged as the + writers spell it in the file name (``f"{name}"``).""" + if _missing(name): + return "is empty" + if not isinstance(name, str): + name = f"{name}" + if name == "": + return "is empty" + if name in (".", ".."): + return f"is {name!r}" + for separator in dict.fromkeys(("/", os.sep, os.altsep)): + if separator is not None and separator in name: + return f"holds {separator!r}" + if "\0" in name: + return "holds a NUL character" + size = len((name + suffix).encode("utf-8", "surrogatepass")) + if size > MAX_FILE_NAME_BYTES: + return f"makes a file name of {size} bytes" + return None + + +def _usable_rule(suffix: str) -> str: + return (f"that is not empty, without '/' or NUL, other than '.' and '..', of at most " + f"{MAX_FILE_NAME_BYTES - len(suffix.encode())} bytes in UTF-8") + + +def refuse_unusable_names(names: Sequence, suffix: str, *, source: str, labels: Optional[Sequence] = None, + what: str = "embedding name", given: str = "those rows", pattern: Optional[str] = None): + """Refuse (RefusalError) names that cannot name the file ```` in the output folder (name_problem), + naming the first ones (with their label, "row 3", say, when given); ``pattern`` is the file as the message names + it, ```` by default.""" + bad = [] + for k, name in enumerate(names): + problem = name_problem(name, suffix) + if problem is not None: + bad.append((name, problem, None if labels is None else labels[k])) + if bad: + def describe(item): + name, problem, label = item + return f"{name!r}{'' if label is None else f' ({label})'} {problem}" + raise RefusalError( + f"{source}: {len(bad)} {what}(s) that cannot name a file {pattern or '' + suffix} in the output " + f"folder: {shown(bad, describe)}. Each {what} names its own file there: give {given} " + f"{'an' if what[0] in 'aeiou' else 'a'} {what} {_usable_rule(suffix)}." + ) + + +def refuse_unusable_stem(stem, what: str = "The output name"): + """Refuse (RefusalError) a name that cannot name the files ``-.parquet`` and ``-.json.gz`` + of the output folder (a 6-digit rank reserved), whatever the format: name_problem's rules.""" + suffix = "-" + "0" * RANK_DIGITS + max(STEM_SUFFIXES.values(), key=len) + # The name itself ('', '.', '..', '/', NUL), else its length once a rank and a suffix are added + problem = name_problem(stem, "") + if problem is None: + problem = name_problem(stem, suffix) + if problem is not None: + problem += " once a rank and a suffix are added" + if problem is not None: + raise RefusalError( + f"{what} {stem!r} {problem}: it names the files -.parquet or -.json.gz of the " + f"output folder. Give a name {_usable_rule(suffix)}." + ) + + +# ===================================================================================================================== +# How a folder compares names +# ===================================================================================================================== +class Folding(NamedTuple): + """How a folder compares file names: ``case``, 'A' and 'a' name one file; ``unicode``, the NFC and NFD spellings of + 'é' name one file; ``full``, with ``case``, 'ß' and 'ss' name one file too: full case folding (str.casefold, as + macOS APFS folds), rather than simple case folding, one character for one (a folder that keeps 'ß' and 'ss' + apart).""" + case: bool = False + unicode: bool = False + full: bool = False + + def describe(self) -> str: + """What the folder does not tell apart, for a refusal ('' for an exact folder).""" + told = [part for part, on in (("upper and lower case", self.case), ("Unicode spellings (NFC and NFD)", + self.unicode)) if on] + return " or ".join(told) + + +EXACT = Folding() + + +def _names_it(spelling: str, path: str) -> bool: + try: + return os.path.samefile(spelling, path) + except OSError: + return False + + +def probe_folding(folder) -> Folding: + """How ``folder`` compares names: a uniquely named temporary file is created in it (mkstemp) and removed, and its + swapped-case (but for 'ß'), NFD and 'ß' as 'ss' spellings are tried. A folder in which no file can be created (it + cannot be written, though it may hold files that can be) is probed without writing, on the names it holds + (_probe_names); one that cannot be probed at all (it does not exist, say) compares exactly.""" + for prefix, unicode_too in ((".foldmatch-probe-Aéß-", True), (".foldmatch-probe-A-", False)): + try: + fd, path = tempfile.mkstemp(prefix=prefix, dir=folder) + except (OSError, UnicodeError): + continue + try: + os.close(fd) + head, name = os.path.split(path) + case = _names_it(os.path.join(head, _swap_case(name)), path) + unicode = unicode_too and _names_it(os.path.join(head, unicodedata.normalize("NFD", name)), path) + full = case and unicode_too and _names_it(os.path.join(head, name.replace("ß", "ss")), path) + return Folding(case=case, unicode=unicode, full=full) + finally: + try: + os.remove(path) + except OSError: + pass + return _probe_names(folder) + + +def _swap_case(name: str) -> str: + """The name in swapped case, but for 'ß' (whose upper case is 'SS').""" + return "".join(c if c == "ß" else c.swapcase() for c in name) + + +def _other_form(name: str) -> str: + """The name in NFD, else in NFC (the name itself when it is both).""" + nfd = unicodedata.normalize("NFD", name) + return nfd if nfd != name else unicodedata.normalize("NFC", name) + + +def _sharp_s(name: str) -> str: + """The name with its first 'ß' as 'ss', else its first 'ss' (in any case) as 'ß' (the name itself without + either).""" + if "ß" in name: + return name.replace("ß", "ss", 1) + at = name.lower().find("ss") + return name if at < 0 else name[:at] + "ß" + name[at + 2:] + + +def _probe_names(folder) -> Folding: + """How a folder in which no file can be created compares names, found out without writing, on names that exist: + the folder's entries, then, for what they leave undecided, the folder's own name in its parent (when both are on + one device: a folder may fold otherwise than its parent, as a case-folding ext4 folder does). A spelling of such a + name in swapped case, in NFD (or NFC), or with 'ss' for 'ß' (or 'ß' for 'ss'), that names the same file and is not + an entry of the folder that holds the name (a sibling of the folder that links to it, say) shows that the folder + takes both spellings for one name; one that does not, that it does not. What no name shows is taken on the safe + side, as the folders that fold case fold: Unicode spellings as one name where case is folded (APFS, HFS+, + case-folding ext4), and case folded fully. A folder without such names (one that does not exist, say) compares + exactly.""" + found = {"case": None, "unicode": None, "full": None} + + def candidates(): + try: + entries = os.listdir(folder) + except (OSError, ValueError): + entries = [] + held = set(entries) + for entry in entries: + yield os.fspath(folder), entry, held + real = os.path.realpath(os.fspath(folder)) + parent, base = os.path.split(real) + here, there = file_id(real), file_id(parent) + if base and here is not None and there is not None and here[0] == there[0]: + try: + siblings = set(os.listdir(parent)) + except (OSError, ValueError): + return # a spelling could be a sibling that links to the folder + yield parent, base, siblings + + for head, name, others in candidates(): + path = os.path.join(head, name) + if file_id(path) is None: # a broken symbolic link, say + continue + for part, spell in (("case", _swap_case), ("unicode", _other_form), ("full", _sharp_s)): + if found[part] is not None or (part == "full" and not found["case"]): + continue + other = spell(name) + if other != name and other not in others: + found[part] = _names_it(os.path.join(head, other), path) + if found["case"] is False: + found["full"] = False + if None not in found.values(): + break + if found["case"] is None and found["unicode"] is None: + return EXACT + case = bool(found["case"]) + unicode = case if found["unicode"] is None else found["unicode"] + full = case if found["full"] is None else found["full"] + return Folding(case=case, unicode=unicode, full=case and full) + + +def _fold_case(name: str, full: bool) -> str: + """Full case folding (casefold: 'ß' is 'ss', 'ff' is 'ff'), or simple case folding: each character's one-character + fold (casefold), else its lower case.""" + if name.isascii(): + return name.lower() + if full: + return name.casefold() + return "".join(c.casefold() if len(c.casefold()) == 1 else c.lower() for c in name) + + +def name_key(name: str, folding: Folding) -> str: + """The name as a folder of this folding compares it. Where the folder folds case and Unicode spellings, as + canonical caseless matching compares (Unicode's D145): decomposed (NFD), case folded, then composed (NFC). Case + folding does not keep canonical equivalence, so 'ᾷ' and 'ᾼ' with U+0342 fold to one name, as APFS takes them, + only once decomposed. Simple case folding (``full`` False) is Unicode's, which differs from the older tables of + HFS+ and exFAT for a few letters outside ASCII ('σ' and 'ς', 'µ' and 'μ').""" + if folding.unicode: + name = unicodedata.normalize("NFD", name) + if folding.case: + name = _fold_case(name, folding.full) + if folding.unicode: + name = unicodedata.normalize("NFC", name) + return name + + +def file_id(path) -> Optional[Tuple[int, int]]: + """A file's (device, inode), or None when it cannot be read.""" + try: + stat = os.stat(path) + except (OSError, ValueError): + return None + return stat.st_dev, stat.st_ino + + +def same_folder(a, b) -> bool: + """Whether two folders are one, compared as files (an existing folder and a missing one are not).""" + first = file_id(a) + return first is not None and first == file_id(b) + + +class InFolder: + """Whether paths are in ``folder`` (their parent folder is it, as a file), the answer kept per parent.""" + + def __init__(self, folder): + self.id = file_id(folder) + self._parents = {} + + def __call__(self, path) -> bool: + parent = os.path.dirname(os.fspath(path)) or os.curdir + if parent not in self._parents: + self._parents[parent] = self.id is not None and file_id(parent) == self.id + return self._parents[parent] + + +# ===================================================================================================================== +# The files of one output +# ===================================================================================================================== +class OutputFiles: + """The files that one output of a run writes in its folder, by the names that the folder compares: ``.`` per item, ``..`` (or ``-``) per item of a name whose items are known only once its + structure is read, and ``-.`` per process.""" + + def __init__(self, folder, folding: Optional[Folding] = None): + self.folder = os.fspath(folder) + self.folding = probe_folding(self.folder) if folding is None else folding + self.files = {} # key -> (name, label, file name) + self.stems = {} # (stem key, suffix) -> stem + self.prefixes = {} # (separator, suffix) -> {prefix key: (prefix, label)} + self._existing = None + self.linked = False # whether a file of existing() is a symbolic link or a hard link + + def key(self, name: str) -> str: + return name_key(name, self.folding) + + def add_files(self, names: Sequence, suffix: str, labels: Optional[Sequence] = None) -> List[tuple]: + """Add ```` per name: the repeats, as (file name, [(name, label), ...]) per file written twice.""" + groups = {} + for k, name in enumerate(names): + file = f"{name}{suffix}" + key = self.key(file) + label = None if labels is None else labels[k] + if key in self.files: + groups.setdefault(key, [self.files[key][:2]]).append((name, label)) + else: + self.files[key] = (name, label, file) + return [(f"{entries[0][0]}{suffix}", entries) for entries in groups.values()] + + def add_stem(self, stem: str, suffix: str): + self.stems[(self.key(stem), suffix)] = stem + + def add_prefixes(self, prefixes: Sequence, separator: str, suffix: str, + labels: Optional[Sequence] = None) -> List[tuple]: + """Add ```` per prefix, for the IDs known only once its structure is read: the + prefixes that the folder takes for one, as (files, [(prefix, label), ...]) per prefix given twice (their IDs + are not known yet, and those of one structure, or chain A of two, would give one file twice).""" + index = self.prefixes.setdefault((separator, suffix), {}) + groups = {} + for k, prefix in enumerate(prefixes): + key = self.key(prefix) + label = None if labels is None else labels[k] + if key in index: + groups.setdefault(key, [index[key]]).append((prefix, label)) + else: + index[key] = (prefix, label) + return [(f"{entries[0][0]}{separator}{suffix}", entries) for entries in groups.values()] + + def holds(self, name: str) -> Optional[str]: + """The output that writes the file ``name`` of the folder, described, or None. A lookup per separator in the + name, whatever the number of prefixes.""" + key = self.key(name) + if key in self.files: + return f"the output {self.files[key][0]!r}" + for (stem, suffix), given in self.stems.items(): + # The rank as the writers spell it, in ASCII digits ('\d' takes any Unicode digit) + if key.endswith(suffix) and re.fullmatch(re.escape(stem) + r"-[0-9]+", key[:len(key) - len(suffix)]): + return f"a file -{suffix} of the output name {given!r}" + for (separator, suffix), index in self.prefixes.items(): + if not key.endswith(suffix): + continue + head = key[:len(key) - len(suffix)] + at = head.find(separator) + while at >= 0: + # , an ID of at least one character + if 0 < at and at + len(separator) < len(head) and head[:at] in index: + given = index[head[:at]][0] + return f"a file {given}{separator}{suffix} of the output {given!r}" + at = head.find(separator, at + 1) + return None + + def existing(self) -> dict: + """The files of the folder that the outputs would write and that exist already, as an earlier run left them: + {(device, inode): (path, the output described)}, read once (one listing of the folder, and one lstat per such + file, and a stat more per symbolic link). ``linked`` tells whether any of them is a symbolic link or a hard + link (a file of more than one name), which may be an input of another folder.""" + if self._existing is None: + self._existing = {} + try: + entries = os.listdir(self.folder) + except (OSError, ValueError): + entries = [] + for entry in entries: + output = self.holds(entry) + if output is None: + continue + path = os.path.join(self.folder, entry) + try: + link = os.lstat(path) + except (OSError, ValueError): + continue + if stat.S_ISLNK(link.st_mode): + found = file_id(path) + if found is None: # a broken symbolic link + continue + self.linked = True + else: + found = (link.st_dev, link.st_ino) + self.linked = self.linked or link.st_nlink > 1 + self._existing.setdefault(found, (path, output)) + return self._existing + + def shared_with(self, other: "OutputFiles") -> List[str]: + """The files that this output and another one in the same folder would both write.""" + shared = [self.files[key][2] for key in self.files if key in other.files] + return shared + [f"{stem}-{suffix}" for (key, suffix), stem in self.stems.items() + if (key, suffix) in other.stems] + + +def _folded_by(names: Sequence[str], folding: Folding) -> List[str]: + """What the folder does not tell apart that makes names that differ one name: upper and lower case, Unicode + spellings, or both.""" + for part, alone in (("upper and lower case", Folding(case=folding.case, full=folding.full)), + ("Unicode spellings (NFC and NFD)", Folding(unicode=folding.unicode))): + if alone != EXACT and len({name_key(name, alone) for name in names}) == 1: + return [part] + return [part for part, on in (("upper and lower case", folding.case), + ("Unicode spellings (NFC and NFD)", folding.unicode)) if on] + + +def repeated_files_message(source: str, repeats: List[tuple], folding: Folding, what: str = "embedding name", + given: str = "those rows", folded_advice: Optional[str] = None) -> str: + """The refusal of names that give one file twice (OutputFiles.add_files' and add_prefixes' repeats). Names that + differ, and that the folder only takes for one, are told so, with what can keep them apart: ``folded_advice``, + whose ``{apart}`` is what the folder does not tell apart, or by default another folder, Parquet or JSON, or other + names.""" + def describe(repeat): + file, entries = repeat + names = " and ".join(f"{name!r}{'' if label is None else f' ({label})'}" for name, label in entries) + return f"{names} give {file}" + folded = [] + for _, entries in repeats: + names = list(dict.fromkeys(f"{name}" for name, _ in entries)) + if len(names) > 1: + folded += [part for part in _folded_by(names, folding) if part not in folded] + folded.sort(key=lambda part: part.startswith("Unicode")) + apart = " or ".join(folded) + note = f" (the output folder does not tell {apart} apart)" if folded else "" + found = (f"{source}: {len(repeats)} file(s) of the output folder would each be written for more than one " + f"{what}{note}: {shown(repeats, describe)}. Each one would replace the one before") + if not folded: + return f"{found}: give each its own {what}." + advice = (folded_advice or "write them to a folder of a file system that tells {apart} apart or as parquet or " + "json (which keep each {what} as given), or give {given} {what}s that differ otherwise") + return f"{found}, as the folder takes those names for one: {advice.format(apart=apart, what=what, given=given)}." + + +def _linked_id(path) -> Optional[Tuple[int, int]]: + """The (device, inode) of the file that ``path`` names when it may be another folder's file: through a symbolic + link, or as a hard link (a file of more than one name); else None.""" + try: + link = os.lstat(path) + except (OSError, ValueError, TypeError): + return None + if stat.S_ISLNK(link.st_mode): + return file_id(path) + return (link.st_dev, link.st_ino) if link.st_nlink > 1 else None + + +def refuse_outputs_over_inputs(outputs: OutputFiles, inputs: Iterable[Tuple[str, object]], *, what: str = "run"): + """Refuse (RefusalError) an input of the run (described, path) that is in the output folder under the name of one + of its outputs, or, from another folder, one file with a file of the output folder that one of its outputs would + write, through a symbolic or hard link either way (compared as files, with the files that exist: an input + exists): the run would replace it.""" + in_folder = InFolder(outputs.folder) + found, others = [], [] + for described, path in inputs: + if path is None: + continue + output = outputs.holds(os.path.basename(os.fspath(path))) if in_folder(path) else None + if output is not None: + found.append((described, os.fspath(path), output)) + else: + others.append((described, path)) + # Only an output's file that exists can be an input reached through a link (none in a new folder). Where none of + # them is a link, only an input that is a link can be one (_linked_id); else any input can (file_id) + existing = outputs.existing() if others else {} + if existing: + identify = file_id if outputs.linked else _linked_id + for described, path in others: + file = existing.get(identify(path)) + if file is not None: + found.append((described, f"{os.fspath(path)} (which is {file[0]})", file[1])) + if found: + raise RefusalError( + f"{len(found)} input(s) of this {what} would be replaced by its output, in the output folder " + f"{outputs.folder}: {shown(found, lambda f: f'{f[0]} {f[1]} by {f[2]}')}. An output never replaces an " + f"input of its run: write the output to another folder or under other names, or move the input." + ) + + +def check_outputs(out_folder, out_format, out_name, *, names: Optional[Sequence] = None, + labels: Optional[Sequence] = None, source: str = "The input", prefixes: Optional[Sequence] = None, + separator: str = ".", inputs: Iterable[Tuple[str, object]] = (), + folding: Optional[Folding] = None, stem_what: str = "The output name", + what: str = "embedding name", given: str = "those rows", + folded_advice: Optional[str] = None) -> Optional[OutputFiles]: + """The checks of one output of a run, before any model is built (RefusalError): the output name ``out_name`` names + files of the folder, whatever the format (D1); for csv and pt, each of ``names`` names one file ``.`` + (D1) of its own, as the folder compares names (D3), and each of ``prefixes`` names the files + ``.`` of items known only once their structure is read, prefixes that the folder + takes for one refused too; and no input (described, path) is one of the output files (D2). ``stem_what`` names the + output name in a refusal, ``what`` a name and ``given`` what holds them, ``folded_advice`` what keeps names apart + that the folder takes for one (repeated_files_message); ``folding`` is the folder's, when known. The output's + files, or None without ``out_folder``.""" + if out_folder is None: + return None + fmt = format_value(out_format) + outputs = OutputFiles(out_folder, folding) + if out_name is not None: + refuse_unusable_stem(out_name, stem_what) + if fmt in STEM_SUFFIXES: + outputs.add_stem(out_name, STEM_SUFFIXES[fmt]) + if fmt in ITEM_FORMATS: + suffix = f".{fmt}" + repeats = [] + if names is not None: + refuse_unusable_names(names, suffix, source=source, labels=labels, what=what, given=given) + repeats = outputs.add_files(names, suffix, labels) + if prefixes is not None and not repeats: + refuse_unusable_names(prefixes, separator + suffix, source=source, labels=labels, + pattern=f"{separator}{suffix}") + repeats = outputs.add_prefixes(prefixes, separator, suffix, labels) + if repeats: + raise RefusalError(repeated_files_message(source, repeats, outputs.folding, what, given, folded_advice)) + refuse_outputs_over_inputs(outputs, inputs) + return outputs + + +def refuse_assemblies_over_residues(names: Sequence, structure_names: Iterable, *, out_folder, out_format, + res_folder, res_format, source: str, labels: Optional[Sequence] = None): + """Refuse (RefusalError) assembly embeddings written in the format of the residue embeddings they are computed + from, to their folder, under a name ``.`` for a structure name of the list (D2): + residue embeddings are named ``.``, and the chain IDs are known only once the structure + is read, so such a name could be a residue embedding's. Names such as ``1acb-1`` are allowed.""" + fmt = format_value(out_format) + if out_folder is None or res_folder is None or fmt != format_value(res_format) or \ + not same_folder(out_folder, res_folder): + return + folding = probe_folding(out_folder) + structures = {name_key(name, folding) for name in structure_names if isinstance(name, str)} + found = [] + for k, name in enumerate(names): + key = name_key(name, folding) if isinstance(name, str) else "" + if any(key[:i] in structures for i, c in enumerate(key) if c == "."): + found.append(f"{name!r}" + ("" if labels is None else f" ({labels[k]})")) + if found: + raise RefusalError( + f"{source}: {len(found)} assembly embedding name(s) of the form .<...>, written as {fmt} " + f"files to the folder of the residue embeddings ({os.fspath(out_folder)}): {shown(found)}. Residue " + f"embeddings are named ..{fmt}, and the chain IDs are known only once the " + f"structure is read, so such an assembly embedding could replace a residue embedding it is computed from: " + f"name the assembly otherwise (-, the default, say), or write the assembly " + f"embeddings to another folder or in another format." + ) + + +def residue_files(res_folder, structure_names: Iterable, res_format) -> List[Tuple[str, str]]: + """The residue embedding files that a run on the structures named reads, as inputs (described, path): the files + ``..`` of ``res_folder`` that exist (the chains are known only once each + structure is read). One listing of the folder.""" + if res_folder is None: + return [] + index = OutputFiles(res_folder) + index.add_prefixes([name for name in dict.fromkeys(structure_names) if isinstance(name, str) and name], ".", + f".{format_value(res_format)}") + try: + entries = sorted(os.listdir(res_folder)) + except (OSError, ValueError): + return [] + return [("the residue embedding file", os.path.join(os.fspath(res_folder), entry)) for entry in entries + if index.holds(entry) is not None] + + +#: The files of a database that the commands read, ````: the index and its metadata and inverted lists, +#: the sequence store, and the residue-code store (``cli.search._SEQUENCE_DB_SUFFIXES``, ``search.residue_code_store``). +DATABASE_SUFFIXES = (".index", ".metadata", ".invlists", ".sequences", ".rescodes.bin", ".rescodes.idx", + ".rescodes.json") +#: The files of a database's residue-code store, ``.rescodes.*`` (``search.split_run._CODE_STORE_GLOB``). +CODE_STORE_GLOB = ".rescodes.*" + + +def database_files(db_path) -> List[str]: + """The existing files of a database (DATABASE_SUFFIXES, then any other ``.rescodes.*``).""" + db_path = os.fspath(db_path) + db_dir, index_name = os.path.split(db_path) + files = [db_path + suffix for suffix in DATABASE_SUFFIXES] + pattern = os.path.join(glob.escape(db_dir or os.curdir), glob.escape(index_name) + CODE_STORE_GLOB) + files += sorted(glob.glob(pattern)) + return [f for f in dict.fromkeys(files) if os.path.isfile(f)] + + +def refuse_output_file_over_inputs(output_file, *, files: Iterable[Tuple[str, object]] = (), + databases: Iterable[Tuple[str, str]] = (), option: str = "--output-file"): + """Refuse (RefusalError) an output file that is one of the command's inputs (P9.N D6), before any work: one of + ``files`` (described, path), or a file of one of ``databases`` (described, path) that the commands read + (database_files). Compared as files (os.path.samefile): an input exists, so only an existing output file can be + one, and a symbolic link to it, or on macOS another case of its name, is. Any other file, such as the results of + an earlier command named ``.hits.tsv`` beside the database, is not an input.""" + out = os.fspath(output_file) + if not os.path.isfile(out): + return + found = [] + for described, path in files: + if path is not None and _names_it(out, os.fspath(path)): + found.append(f"{described} {os.fspath(path)}") + for described, db_path in databases: + found.extend(f"the file {path} of {described} {os.fspath(db_path)}" for path in database_files(db_path) + if _names_it(out, path)) + if found: + raise RefusalError( + f"{option} {out} is {'; and '.join(found)}: the results would replace an input of this command. Write " + f"them to another file." + ) diff --git a/src/foldmatch/utils/sequence.py b/src/foldmatch/utils/sequence.py new file mode 100644 index 0000000..94437dd --- /dev/null +++ b/src/foldmatch/utils/sequence.py @@ -0,0 +1,312 @@ +"""Protein sequence normalisation and the FASTA pre-flight. + +One function, :func:`normalize_sequence`, decides what string a FASTA record +stands for: the sequence ESM3 tokenises, the one the sequence store keeps, the +one Stage-2 aligns and the one :func:`seq_sha1` hashes. Keeping them identical +is what lets an embedding, a stored sequence and a residue index line up. + +:func:`preflight_fasta` is the read-only pass every rank makes over a FASTA +file before ESM3 is loaded. It is the scan the dataset needs anyway (the +record offsets), extended with the counts and problems that decide whether +the run may start, so a refusal stops every rank at the same point. + +No torch or esm imports, so lightweight modules can use it. +""" +import hashlib +import heapq +import logging +import string +from array import array +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator, List, Tuple + +import numpy as np + +from foldmatch.utils.fasta import read_record_at + +logger = logging.getLogger(__name__) + +STANDARD_AMINO_ACIDS = "ACDEFGHIKLMNPQRSTVWY" +_STANDARD_BYTES = STANDARD_AMINO_ACIDS.encode("ascii") + +# ASCII whitespace only (string.whitespace). Python's str.split() also treats +# \x1c-\x1f as whitespace, so it is used only as a fast test for "none at all". +_DELETE_WHITESPACE = str.maketrans("", "", string.whitespace) +_UPPER_ASCII_DELETE_WHITESPACE = str.maketrans( + string.ascii_lowercase, string.ascii_uppercase, string.whitespace +) + +# Ids quoted per problem in a pre-flight message. +_EXAMPLE_IDS = 5 + + +def normalize_sequence(sequence: str, allow_non_ascii: bool = False) -> str: + """Remove all ASCII whitespace and uppercase ASCII letters. + + Every other character is kept as it is, one for one, so the ESM3 + tokenizer still sees one token per residue: ``_`` is ````, + ``* J 0-9`` are ```` and ``- . |`` are tokens of their own. Lowercase + letters, which ESM3 would read as ````, become residues. + + Non-ASCII input raises ``ValueError``. With ``allow_non_ascii`` it is kept + unchanged instead (only ASCII letters are uppercased): the plain build and + query paths warn about such input rather than refuse it. + """ + if sequence.isascii(): + # On an ASCII string str.upper() changes ASCII letters only. + sequence = sequence.upper() + elif allow_non_ascii: + return sequence.translate(_UPPER_ASCII_DELETE_WHITESPACE) + else: + raise ValueError("Sequence contains non-ASCII characters") + if len(sequence.split()) != 1: + sequence = sequence.translate(_DELETE_WHITESPACE) + return sequence + + +def seq_sha1(sequence: str) -> bytes: + """SHA-1 (20 bytes) of the ASCII bytes of ``normalize_sequence(sequence)``.""" + return hashlib.sha1(normalize_sequence(sequence).encode("ascii")).digest() + + +def _scan_records(fasta_file: Path) -> Iterator[Tuple[int, str, int, int, bool, bool]]: + """Single pass yielding, per record, ``(header_byte_offset, id, length, + nonstandard, non_ascii, angle_bracket)``. + + The record rules are those of :func:`foldmatch.utils.fasta.iter_fasta_offsets` + (id = first token of the header, blank lines and empty records skipped; a + header without an id raises ``ValueError``, where the readers would fail + with an ``IndexError``), but every figure is about the *normalised* + sequence: ``length`` residues, + of which ``nonstandard`` are outside the 20 standard amino acids, and + whether it holds a non-ASCII character or a ``<``/``>``. + + Uses explicit ``tell``/``readline`` rather than ``for line in f`` because + the latter's read-ahead buffering makes ``tell()`` offsets unreliable. + """ + with open(fasta_file, 'r') as f: + header_offset = None + name = None + length = nonstandard = 0 + non_ascii = angle_bracket = False + while True: + pos = f.tell() + line = f.readline() + if not line: + break + stripped = line.strip() + if not stripped: + continue + if stripped.startswith('>'): + if name is not None and length > 0: + yield header_offset, name, length, nonstandard, non_ascii, angle_bracket + header_offset = pos + words = stripped[1:].split() + if not words: + raise ValueError( + f"{fasta_file}: the header line at byte {pos} is empty ('>' without an id); every " + f"record needs an id" + ) + name = words[0] + length = nonstandard = 0 + non_ascii = angle_bracket = False + continue + if stripped.isascii(): + residues = stripped.upper() + if len(residues.split()) != 1: + residues = residues.translate(_DELETE_WHITESPACE) + nonstandard += len(residues.encode("ascii").translate(None, _STANDARD_BYTES)) + else: + residues = stripped.translate(_UPPER_ASCII_DELETE_WHITESPACE) + nonstandard += sum(c not in STANDARD_AMINO_ACIDS for c in residues) + non_ascii = True + length += len(residues) + if '<' in residues or '>' in residues: + angle_bracket = True + if name is not None and length > 0: + yield header_offset, name, length, nonstandard, non_ascii, angle_bracket + + +def scan_fasta(fasta_file: Path) -> Iterator[Tuple[int, str, int]]: + """Single pass yielding ``(header_byte_offset, id, length)`` per record, + ``length`` being the length of the normalised sequence (see :func:`_scan_records`).""" + for offset, name, length, *_ in _scan_records(fasta_file): + yield offset, name, length + + +def _id_hash(name: str) -> int: + # Deterministic across processes (unlike hash()), so every rank finds the same duplicates. + return int.from_bytes( + hashlib.blake2b(name.encode("utf-8", "surrogatepass"), digest_size=8).digest(), "little" + ) + + +@dataclass(frozen=True) +class FastaPreflight: + """What :func:`preflight_fasta` found in one FASTA file. + + Counts over all records are ``entries``; every other figure is over the + *kept* records, those with at least ``min_res_n`` residues, which are the + ones that get embedded and stored. + """ + path: Path + min_res_n: int + entries: int + entries_kept: int # one embedding each + residues: int + duplicates: int # kept records whose id repeats an earlier kept record + unique_residues: int # residues over the first record of each id (what the store keeps) + nonstandard_entries: int + nonstandard_residues: int + non_ascii_entries: int + angle_bracket_entries: int # records containing '<' or '>' + max_length: int + duplicate_ids: Tuple[str, ...] # a few examples of each problem, in file order + non_ascii_ids: Tuple[str, ...] + angle_bracket_ids: Tuple[str, ...] + offsets: np.ndarray = field(repr=False, compare=False) # header offsets of the kept records + + @property + def unique_entries(self) -> int: + """Kept records with distinct ids: the rows of the sequence store.""" + return self.entries_kept - self.duplicates + + def problems(self) -> List[str]: + """One message per kind of input a strict run refuses.""" + found = [] + if self.duplicates: + found.append( + f"{self.duplicates} record(s) repeat an earlier id " + f"(e.g. {', '.join(self.duplicate_ids)})" + ) + if self.angle_bracket_entries: + found.append( + f"{self.angle_bracket_entries} sequence(s) contain '<' or '>' " + f"(e.g. {', '.join(self.angle_bracket_ids)}); ESM3 reads a bracketed " + f"name such as '' as a single token, so residues and positions " + f"no longer line up" + ) + if self.non_ascii_entries: + found.append( + f"{self.non_ascii_entries} sequence(s) contain non-ASCII characters " + f"(e.g. {', '.join(self.non_ascii_ids)})" + ) + return found + + def check(self, strict: bool = False) -> None: + """Refuse (strict) or warn about (plain) every :meth:`problems` entry. + + Strict raises one ``ValueError`` naming all of them. A plain run logs + them and goes ahead exactly as before: duplicates keep their first + sequence in the store, and the other characters reach ESM3 unchanged. + """ + found = self.problems() + if strict and found: + raise ValueError(f"{self.path}: " + "; ".join(found) + ".") + for problem in found: + logger.warning(f"{self.path}: {problem}.") + logger.info( + f"Pre-flight {self.path}: {self.entries_kept} of {self.entries} entries with " + f">= {self.min_res_n} residues, {self.residues} residues, longest {self.max_length}" + ) + + +def preflight_fasta(fasta_file: Path, min_res_n: int = 0) -> FastaPreflight: + """Read-only pass over a FASTA file: record offsets plus counts and problems. + + Deterministic, so every rank that runs it reaches the same verdict. + Duplicate ids are found from sorted 64-bit id hashes, 8 bytes per record, + with an exact comparison of the ids whose hashes collide, so memory stays + bounded however large the file. + """ + offsets = array('q') + hashes = array('Q') + entries = entries_kept = residues = max_length = 0 + nonstandard_entries = nonstandard_residues = non_ascii_entries = angle_bracket_entries = 0 + non_ascii_ids: List[str] = [] + angle_bracket_ids: List[str] = [] + for offset, name, length, nonstandard, non_ascii, angle_bracket in _scan_records(fasta_file): + entries += 1 + if min_res_n > 0 and length < min_res_n: + continue + entries_kept += 1 + residues += length + max_length = max(max_length, length) + offsets.append(offset) + hashes.append(_id_hash(name)) + if nonstandard: + nonstandard_entries += 1 + nonstandard_residues += nonstandard + if non_ascii: + non_ascii_entries += 1 + if len(non_ascii_ids) < _EXAMPLE_IDS: + non_ascii_ids.append(name) + if angle_bracket: + angle_bracket_entries += 1 + if len(angle_bracket_ids) < _EXAMPLE_IDS: + angle_bracket_ids.append(name) + + offsets = np.frombuffer(offsets, dtype=np.int64) if entries_kept else np.empty(0, dtype=np.int64) + duplicates, duplicate_residues, duplicate_ids = _find_duplicates(fasta_file, offsets, hashes) + return FastaPreflight( + path=Path(fasta_file), + min_res_n=min_res_n, + entries=entries, + entries_kept=entries_kept, + residues=residues, + duplicates=duplicates, + unique_residues=residues - duplicate_residues, + nonstandard_entries=nonstandard_entries, + nonstandard_residues=nonstandard_residues, + non_ascii_entries=non_ascii_entries, + angle_bracket_entries=angle_bracket_entries, + max_length=max_length, + duplicate_ids=duplicate_ids, + non_ascii_ids=tuple(non_ascii_ids), + angle_bracket_ids=tuple(angle_bracket_ids), + offsets=offsets, + ) + + +def _find_duplicates(fasta_file: Path, offsets: np.ndarray, hashes: array) -> Tuple[int, int, Tuple[str, ...]]: + """``(duplicates, their residues, example ids)`` among the kept records. + + A record is a duplicate when its id equals that of an earlier kept record + (the store keeps the first). Only records whose 64-bit id hash occurs more + than once are re-read, to compare the actual ids and measure the repeats. + """ + if len(hashes) < 2: + return 0, 0, () + hashes = np.frombuffer(hashes, dtype=np.uint64) + ordered = np.sort(hashes) + repeated = ordered[1:][ordered[1:] == ordered[:-1]] + del ordered + if repeated.size == 0: + return 0, 0, () + # File-order positions of the records whose hash is shared, regrouped by + # hash; the stable sort keeps file order within each group. + candidates = np.flatnonzero(np.isin(hashes, repeated)) + candidates = candidates[np.argsort(hashes[candidates], kind='stable')] + + duplicates = duplicate_residues = 0 + examples: List[Tuple[int, str]] = [] # max-heap (negated positions) of the first few + first_seen: set = set() + previous_hash = None + with open(fasta_file, 'r') as fh: + for position in candidates.tolist(): + if hashes[position] != previous_hash: + previous_hash = hashes[position] + first_seen.clear() + fh.seek(int(offsets[position])) + name = fh.readline().strip()[1:].split()[0] + if name not in first_seen: + first_seen.add(name) + continue + _, sequence = read_record_at(fh, int(offsets[position])) + duplicates += 1 + duplicate_residues += len(normalize_sequence(sequence, allow_non_ascii=True)) + heapq.heappush(examples, (-position, name)) + if len(examples) > _EXAMPLE_IDS: + heapq.heappop(examples) + return duplicates, duplicate_residues, tuple(dict.fromkeys(name for _, name in sorted(examples, reverse=True))) diff --git a/src/foldmatch/utils/structure_parser.py b/src/foldmatch/utils/structure_parser.py index 0e80fb6..ed31db0 100644 --- a/src/foldmatch/utils/structure_parser.py +++ b/src/foldmatch/utils/structure_parser.py @@ -3,7 +3,7 @@ from pathlib import Path import numpy as np -from biotite.structure import filter_amino_acids, filter_polymer, chain_iter, get_chains, get_residues, \ +from biotite.structure import filter_amino_acids, filter_polymer, get_chain_starts, get_residue_count, \ filter_peptide_backbone, residue_iter, array from biotite.structure.io.pdb import PDBFile, get_structure as get_pdb_structure, get_assembly as get_pdb_assembly, list_assemblies as list_pdb_assemblies from biotite.structure.io.pdbx import CIFFile, get_structure, get_assembly, BinaryCIFFile, list_assemblies @@ -12,10 +12,11 @@ def get_structure_from_src( src_structure, structure_format="mmcif", - chain_id=None, + *, assembly_id=None ): try: + __rewind(src_structure) if __is_path(src_structure) and Path(src_structure).suffix == ".gz": if structure_format == "mmcif" or structure_format == "pdb": src_structure = gzip.open(src_structure, "rt") @@ -24,7 +25,7 @@ def get_structure_from_src( else: raise RuntimeError(f"Unknown file format {structure_format}") if structure_format == "pdb": - pdb_file = PDBFile.read(src_structure) + pdb_file = PDBFile.read(src_structure) if assembly_id is None else _IndexedPDBFile.read(src_structure) structure = __get_pdb_structure(pdb_file, assembly_id) elif structure_format == "mmcif": cif_file = CIFFile.read(src_structure) @@ -37,24 +38,41 @@ def get_structure_from_src( except Exception as e: raise RuntimeError(f"Error reading structure from {src_structure}: {e}") - if chain_id is not None: - return structure[structure.chain_id == chain_id] - return structure +def protein_chain_iter(structure, min_res_n=0, chain_id=None): + """Each protein chain of the structure, in order, as one AtomArray. A protein chain is the amino-acid atoms of the + chain_iter pieces with one chain ID (in an assembly, of one copy: one sym_id), joined in file order, without the + pieces whose amino acids are all HETATM records if it has amino acids in ATOM records; it counts if they hold + polymer amino-acid residues, at least min_res_n of them. With chain_id, the protein chains of that chain ID alone, + reading only its pieces.""" + return (atom_ch for atom_ch, _ in protein_chains_with_sizes(structure, min_res_n, chain_id)) + + +def protein_chains_with_sizes(structure, min_res_n=0, chain_id=None): + """protein_chain_iter's protein chains, each with its size: (AtomArray, the number of polymer amino-acid residues + that min_res_n is tested against, counted piece by piece and summed over the pieces kept).""" + starts = get_chain_starts(structure, add_exclusive_stop=True) + if chain_id is not None: + # The atoms with that chain ID, cut where its pieces end (a selection alone would join pieces that other + # chains' records separate): a few milliseconds on the largest structures + atoms = np.flatnonzero(structure.chain_id == chain_id) + piece_start = np.zeros(structure.array_length() + 1, dtype=bool) + piece_start[starts] = True + structure = structure[atoms] + starts = np.append(np.flatnonzero(piece_start[atoms]), len(atoms)) + return __protein_chains(structure, starts, min_res_n) + + def get_protein_chains(structure, min_res_n=0): - chain_ids = [] - for atom_ch in chain_iter(structure): - atom_res = atom_ch[filter_polymer(atom_ch)] - atom_res = atom_res[filter_amino_acids(atom_res)] - if len(atom_res) > 0 and len(get_residues(atom_res)[0]) >= min_res_n: - chain_ids.append(str(get_chains(atom_res)[0])) - return tuple(chain_ids) + """The chain ID of each protein chain of the structure (protein_chain_iter): in an assembly, once per copy.""" + return tuple(str(atom_ch.chain_id[0]) for atom_ch in protein_chain_iter(structure, min_res_n)) def get_assemblies(structure, structure_format="mmcif"): try: + __rewind(structure) if __is_path(structure) and Path(structure).suffix == ".gz": if structure_format == "mmcif" or structure_format == "pdb": structure = gzip.open(structure, "rt") @@ -86,15 +104,67 @@ def get_backbone_atoms(atom_ch): return np.array([(lambda x: [a.coord for a in x])(r) for r in residue_iter(atom_ch[filter_peptide_backbone(atom_ch)])]) +def __protein_chains(structure, starts, min_res_n): + # The protein chains, and their sizes, of the structure whose chain_iter pieces start at starts (exclusive stop) + aa = filter_amino_acids(structure) + # The amino-acid atoms, and those of ATOM records, of every piece at once: the thousands of water and ligand pieces + # of a large structure cost no Python loop + n_aa = np.diff(np.concatenate(([0], np.cumsum(aa)))[starts]) + n_atom = np.diff(np.concatenate(([0], np.cumsum(aa & ~structure.hetero)))[starts]) + pieces = np.flatnonzero(n_aa > 0) + keys = structure.chain_id[starts[pieces]].tolist() + if "sym_id" in structure.get_annotation_categories(): + # In an assembly, each copy of a chain is a chain of its own: one sym_id (get_structure_from_src gives each copy + # of a chain its own, in PDB files too) + keys = list(zip(keys, structure.sym_id[starts[pieces]].tolist())) + chains = {} + for key, k in zip(keys, pieces.tolist()): + chains.setdefault(key, []).append(k) + # A piece whose amino acids are all HETATM records (a free amino-acid ligand) is left out if the chain has amino + # acids in ATOM records, and a chain made only of HETATM amino acids keeps them all; the chains come in the order + # of their first piece kept + for kept in sorted(([k for k in ch if n_atom[k] > 0] or ch for ch in chains.values()), key=lambda kept: kept[0]): + # Polymer amino-acid residues counted piece by piece, as each piece was counted before + n_pol = 0 + for k in kept: + piece = structure[starts[k]:starts[k + 1]] + piece = piece[filter_polymer(piece)] + n_pol += get_residue_count(piece[filter_amino_acids(piece)]) + if n_pol > 0 and n_pol >= min_res_n: + atoms = np.concatenate([starts[k] + np.flatnonzero(aa[starts[k]:starts[k + 1]]) for k in kept]) + yield structure[atoms], n_pol + + +class _IndexedPDBFile(PDBFile): + """A PDB file read for an assembly: its structure holds each atom's index in the asymmetric unit (au_index), which + biotite's get_assembly keeps, as it builds the assembly from get_structure: the atoms of each REMARK 350 list, in + file order, once for each of its operators.""" + + def get_structure(self, *args, **kwargs): + structure = super().get_structure(*args, **kwargs) + structure.set_annotation("au_index", np.arange(structure.array_length())) + return structure + + def __get_pdb_structure(pdb_file, assembly_id=None): - return get_pdb_structure( - pdb_file, - model=1 - ) if assembly_id is None else get_pdb_assembly( + if assembly_id is None: + return get_pdb_structure( + pdb_file, + model=1 + ) + assembly = get_pdb_assembly( pdb_file, assembly_id=assembly_id, model=1 ) + # biotite numbers the copies (sym_id) of each REMARK 350 list ('APPLY THE FOLLOWING TO CHAINS') from 0, so a chain + # in two lists has two copies with one sym_id. The index in the asymmetric unit (_IndexedPDBFile) rises within a + # copy, and two copies of a chain repeat its atoms, so it stops rising between them: counting where it stops gives + # each copy of a chain a sym_id of its own (biotite's, in an assembly of one list) + starts = np.concatenate(([0], np.flatnonzero(np.diff(assembly.au_index) <= 0) + 1, [assembly.array_length()])) + assembly.set_annotation("sym_id", np.repeat(np.arange(len(starts) - 1), np.diff(starts))) + assembly.del_annotation("au_index") + return assembly def __rename_atom(atom): @@ -117,6 +187,13 @@ def __get_structure(cif_file, assembly_id=None): use_author_fields=False ) +def __rewind(src): + # A stream (a downloaded file) may be read more than once: its assemblies, then each assembly. + seekable = getattr(src, "seekable", None) + if seekable is not None and seekable(): + src.seek(0) + + def __is_path(val): if isinstance(val, (str, os.PathLike)): return True diff --git a/src/foldmatch/utils/structure_provider.py b/src/foldmatch/utils/structure_provider.py index 1c2ad1f..4249555 100644 --- a/src/foldmatch/utils/structure_provider.py +++ b/src/foldmatch/utils/structure_provider.py @@ -2,26 +2,28 @@ class StructureProvider: + """The last structure read, kept so that consecutive requests for it (one per chain of an entry) read the file + once. It is keyed by the source name, the format and the assembly: two assemblies of one entry are two + structures.""" def __init__(self): - self.__src_name = None - self.__structure = None + self.__cached = None def get_structure( self, src_name, src_structure, structure_format="mmcif", - chain_id=None, + *, assembly_id=None ): - if src_name != self.__src_name: - self.__src_name = src_name - self.__structure = get_structure_from_src( + key = (src_name, structure_format, assembly_id) + cached = self.__cached + if cached is None or cached[0] != key: + cached = (key, get_structure_from_src( src_structure=src_structure, structure_format=structure_format, assembly_id=assembly_id - ) - if chain_id is not None: - return self.__structure[self.__structure.chain_id == chain_id] - return self.__structure + )) + self.__cached = cached + return cached[1] diff --git a/src/foldmatch/utils/tensor_digest.py b/src/foldmatch/utils/tensor_digest.py new file mode 100644 index 0000000..f015601 --- /dev/null +++ b/src/foldmatch/utils/tensor_digest.py @@ -0,0 +1,32 @@ +"""Content digests of named arrays (``foldmatch-tensor-digest/1``), independent of the file container. + +numpy only (no torch), so the Stage-2 side can check the panel and the E-value table of an esm-codes bundle without +torch; :func:`foldmatch.model.residue_code_encoder.tensor_digest` applies the same digest to torch tensors. +""" +from __future__ import annotations + +import hashlib +import json +from typing import Mapping + +import numpy as np + +DIGEST_METHOD = "foldmatch-tensor-digest/1" + + +def tensor_digest(named: Mapping[str, object]) -> str: + """sha256 over the ASCII tag ``foldmatch-tensor-digest/1`` and a newline, then, for every name in sorted order, + an 8-byte little-endian length and the compact sorted-key JSON ``{"dtype", "name", "shape"}`` (numpy dtype + string, little-endian), and an 8-byte little-endian length and the raw C-order little-endian bytes.""" + h = hashlib.sha256(DIGEST_METHOD.encode() + b"\n") + for name in sorted(named): + a = np.ascontiguousarray(np.asarray(named[name])) + a = a.astype(a.dtype.newbyteorder("<"), copy=False) + head = json.dumps({"dtype": a.dtype.str, "name": name, "shape": list(a.shape)}, sort_keys=True, + separators=(",", ":")).encode() + raw = a.tobytes(order="C") + h.update(len(head).to_bytes(8, "little")) + h.update(head) + h.update(len(raw).to_bytes(8, "little")) + h.update(raw) + return h.hexdigest() diff --git a/src/foldmatch/writer/batch_writer.py b/src/foldmatch/writer/batch_writer.py index 795875d..3cb92ed 100644 --- a/src/foldmatch/writer/batch_writer.py +++ b/src/foldmatch/writer/batch_writer.py @@ -1,3 +1,4 @@ +import numpy as np import torch import pandas as pd import pyarrow as pa @@ -8,6 +9,16 @@ from lightning.pytorch.callbacks import BasePredictionWriter +from foldmatch.errors import RefusalError +from foldmatch.utils.output_files import name_key, name_problem, probe_folding +from foldmatch.search.residue_shards import ( + SEQ_SHA1, SEQ_SHA1_BYTES, SEQ_SHA1_TYPE, check_residue_output, large_binary_array, pass_record, shard_schema, +) + +#: Default byte budget of a writer with a residue column or residue embeddings: buffered rows are written once they +#: hold this many bytes. +DEFAULT_BYTE_BUDGET = 256 * 1024 * 1024 + class CoreBatchWriter(BasePredictionWriter): def __init__( @@ -19,6 +30,9 @@ def __init__( super().__init__(write_interval) self.out_path = output_path self.postfix = postfix + # The files this writer has written in this run, by the name the folder compares (claim_file) + self._written = {} + self._folding = None def write_on_batch_end( self, @@ -32,7 +46,10 @@ def write_on_batch_end( ): if prediction is None: return - embeddings, dom_ids = prediction + # A prediction is (embeddings, ids) optionally followed by per-item + # payloads (3- and 4-tuples). These writers store the embedding only, + # so any extra elements are dropped. + embeddings, dom_ids = prediction[0], prediction[1] deque(map( self._write_embedding, embeddings, @@ -42,6 +59,39 @@ def write_on_batch_end( def file_name(self, dom_id): return f'{self.out_path}/{dom_id}.{self.postfix}' + def claim_file(self, dom_id) -> str: + """The file of ``dom_id``, once this writer has checked that it may write it (P9.N D4): the name must make one + file of the output folder (name_problem), and one that this writer has not written in this run, compared as + the folder compares names (chains A and a of one structure are one file on a case-insensitive folder). + Otherwise RefusalError, naming the file: the run stops, the first file kept and this one not written. + + A backstop for names known only during the run (the chains of structures read whole); the inference functions + refuse the names they know before any model is built. Kept per writer, so per process: two ranks that write + one file are not seen.""" + suffix = f".{self.postfix}" + problem = name_problem(dom_id, suffix) + if problem is not None: + raise RefusalError( + f"The embedding {dom_id!r} cannot be written to {self.out_path}: its name {problem}, so it cannot name " + f"the file {suffix} there. Write the embeddings as parquet or json, which keep each under its " + f"id, or give it another name." + ) + if self._folding is None: + self._folding = probe_folding(self.out_path) + key = name_key(f"{dom_id}{suffix}", self._folding) + first = self._written.get(key) + if first is not None: + folded = self._folding.describe() + note = f" (the folder does not tell {folded} apart)" if folded and first != dom_id else "" + raise RefusalError( + f"{self.file_name(first)} would be written twice by this run, for {first!r} and then for " + f"{dom_id!r}{note}: the first is kept, {dom_id!r} is not written, and the run stops. Each embedding of " + f"a run needs a file of its own: write them as parquet or json, which keep each under its id, or, " + f"for names that differ only in case, to a folder of a case-sensitive file system." + ) + self._written[key] = dom_id + return self.file_name(dom_id) + @abstractmethod def _write_embedding(self, embedding, dom_id): pass @@ -58,7 +108,7 @@ def __init__( def _write_embedding(self, embedding, dom_id): pd.DataFrame(embedding.to('cpu').numpy()).to_csv( - self.file_name(dom_id), + self.claim_file(dom_id), index=False, header=False ) @@ -78,7 +128,7 @@ def __init__( def _write_embedding(self, embedding, dom_id): torch.save( embedding.to(self.device), - self.file_name(dom_id) + self.claim_file(dom_id) ) @@ -135,24 +185,86 @@ def on_predict_end(self, trainer, pl_module): ) class ParquetBatchWriter(BasePredictionWriter): + """Per-rank Parquet shard ``{output_path}/{df_id}-{rank}.parquet``, written in row groups. + + Without ``residue_output`` the schema is the legacy ``id | embedding`` (``list``, no metadata) and a + prediction's elements beyond ``(embeddings, ids)`` are dropped, so plain passes write the same bytes as before. A + row holds one embedding vector (a chain, assembly or sequence embedding), so a tensor with more than one axis + longer than 1 is refused. + + With ``residue_embeddings=True`` (``fm-embedding ... residue``, ``inference residue-embedding``) a row holds the + residue embeddings of one chain, a 2-D ``[L, D]`` tensor (ESM3's BOS and EOS rows included), as one list of D + values per residue: ``embedding`` is ``list>``, built from offsets, with no metadata. + D is stored in the schema, fixed at the first row group (the writer opens lazily; :attr:`schema` is None until + then), so every embedding must be 2-D and of one D. Not with ``residue_output``, whose ``embedding`` holds chain + embeddings. + + With ``residue_output`` (``"residue_codes"`` for builds, ``"query_encoding"`` for queries) and the scoring + ``system``, the schema, fixed here, adds the payload column (``large_binary``) and ``seq_sha1`` + (``fixed_size_binary(20)``), with the metadata ``foldmatch.residue_output``, ``foldmatch.system``, + ``foldmatch.encoding_digest`` and ``foldmatch.system_digest`` (:mod:`foldmatch.search.residue_shards`), and the + pass record ``foldmatch.pass`` (``min_res_n`` and ``esm_revision`` given here; the device type, its ESM3 + precision and the number of ranks added at the start of the pass, before any row is written). Every prediction + must then be ``(embeddings, ids, payloads, seq_sha1s)``: a writer with a column refuses one without it, and + refuses at the start of the pass a module that does not produce it. + + Buffered rows are written as one row group once ``buffer_size`` rows (16,384) or ``byte_budget`` bytes are + buffered, whichever comes first. The byte budget defaults to 256 MB with a residue column (query encodings take + 128 B per residue) or with ``residue_embeddings`` (6 KiB per residue at D = 1536, so 16,384 chains would buffer + tens of GB), and to none otherwise, which keeps the plain row groups, and so the plain bytes, unchanged (chain + embeddings take 6 KiB a row, so 16,384 rows stay below 256 MB anyway). + """ + def __init__( self, output_path, df_id, write_interval="batch", buffer_size=16384, + residue_output=None, + system=None, + byte_budget=None, + min_res_n=None, + esm_revision=None, + residue_embeddings=False, ): super().__init__(write_interval) self.out_path = output_path self.df_id = df_id self.buffer_size = buffer_size + self.residue_output = check_residue_output(residue_output) + if residue_embeddings and self.residue_output is not None: + raise ValueError(f"a {self.residue_output} writer stores chain embeddings, not residue embeddings") + self.residue_embeddings = bool(residue_embeddings) + if self.residue_output is None and (system is not None or min_res_n is not None or esm_revision is not None): + raise ValueError("a scoring system and a pass record are only used by a writer with a residue output") + if self.residue_output is not None and system is None: + raise ValueError(f"a {self.residue_output} writer needs the scoring system (its schema metadata)") + self.system = system + if byte_budget is None and (self.residue_output is not None or self.residue_embeddings): + byte_budget = DEFAULT_BYTE_BUDGET + if byte_budget is not None and int(byte_budget) <= 0: + raise ValueError(f"byte_budget must be > 0, got {byte_budget!r}") + self.byte_budget = None if byte_budget is None else int(byte_budget) self._writer = None self._id_buffer = [] self._emb_buffer = [] - self._schema = pa.schema([ - ('id', pa.string()), - ('embedding', pa.list_(pa.float32())) - ]) + self._payload_buffer = [] + self._sha1_buffer = [] + self._buffered_bytes = 0 + # Fixed here (the pass record is completed in on_predict_start, before the first row group; residue + # embeddings at the first row group, once D is known): every row group of a rank has exactly this schema and + # metadata. + self._pass = None if self.residue_output is None else {"min_res_n": min_res_n, "esm_revision": esm_revision} + self._schema = None if self.residue_embeddings else shard_schema( + self.residue_output, system, None if self._pass is None else pass_record(**self._pass)) + # Residue embeddings: D, set by the first table (its id kept for the refusal of another D). + self._residue_dim = None + self._residue_dim_id = None + + @property + def schema(self): + return self._schema def _get_writer(self, rank): if self._writer is None: @@ -160,6 +272,64 @@ def _get_writer(self, rank): self._writer = pq.ParquetWriter(path, self._schema, compression='snappy') return self._writer + def on_predict_start(self, trainer, pl_module): + # Every rank has the same module and writer, so a mismatch is refused by all of them, before any batch. + produced = getattr(pl_module, "residue_output", None) + if self.residue_output is not None and produced != self.residue_output: + raise ValueError( + f"this Parquet writer stores {self.residue_output}, but the module produces " + f"{produced or 'no residue output'}" + ) + if self.residue_output is not None and self._writer is None: + # The pass record's device and ranks, known now: the device the module runs on and the world size. + device = getattr(getattr(pl_module, "device", None), "type", None) + world_size = getattr(trainer, "world_size", None) + self._schema = shard_schema(self.residue_output, self.system, pass_record( + **self._pass, accelerator=device, world_size=None if world_size is None else int(world_size))) + + def _residue_columns(self, prediction, dom_ids): + if len(prediction) < 4: + raise ValueError( + f"this Parquet writer stores {self.residue_output}: a prediction must be (embeddings, ids, payloads, " + f"seq_sha1s), got a {len(prediction)}-tuple" + ) + payloads, digests = prediction[2], prediction[3] + if len(payloads) != len(dom_ids) or len(digests) != len(dom_ids): + raise ValueError( + f"{len(dom_ids)} ids but {len(payloads)} {self.residue_output} payloads and {len(digests)} seq_sha1s" + ) + payloads = [bytes(p) for p in payloads] + digests = [bytes(d) for d in digests] + bad = [i for i, d in zip(dom_ids, digests) if len(d) != SEQ_SHA1_BYTES] + if bad: + raise ValueError(f"seq_sha1 must be {SEQ_SHA1_BYTES} bytes (e.g. for {bad[0]!r})") + return payloads, digests + + def _check_shapes(self, embeddings, dom_ids): + """Refuse an embedding this writer cannot store, before any of the batch is buffered: a row holds one vector + (a tensor with at most one axis longer than 1) or, with ``residue_embeddings``, one ``[L, D]`` table (D > 0), + of the first table's D.""" + for emb, dom_id in zip(embeddings, dom_ids): + shape = list(emb.shape) + if not self.residue_embeddings: + if sum(n > 1 for n in shape) > 1: + raise RefusalError( + f"this Parquet writer stores one embedding vector per row, but {dom_id} has shape {shape}: " + f"residue embeddings are written with residue_embeddings=True" + ) + elif len(shape) != 2 or shape[1] == 0: + raise RefusalError( + f"this Parquet writer stores residue embeddings, one [L, D] table per row, but {dom_id} has shape " + f"{shape}" + ) + elif self._residue_dim is None: + self._residue_dim, self._residue_dim_id = shape[1], dom_id + elif shape[1] != self._residue_dim: + raise RefusalError( + f"this Parquet writer stores residue embeddings of one dimension, but {dom_id} has {shape[1]} " + f"values per residue and {self._residue_dim_id} has {self._residue_dim}" + ) + def write_on_batch_end( self, trainer, @@ -172,32 +342,61 @@ def write_on_batch_end( ): if prediction is None: return - embeddings, dom_ids = prediction + # Without a residue column, extra tuple elements beyond (embeddings, ids) + # are dropped: this writer's columns are fixed at construction. + embeddings, dom_ids = prediction[0], prediction[1] + if self.residue_output is not None: + payloads, digests = self._residue_columns(prediction, dom_ids) + self._check_shapes(embeddings, dom_ids) self._id_buffer.extend(dom_ids) # Keep flattened float32 arrays (not Python lists) in the buffer — far # cheaper in memory until we flush. .flatten() copies, so the buffer # never aliases tensors that Lightning may reuse on the next batch. - self._emb_buffer.extend( - emb.detach().to('cpu', torch.float32).numpy().flatten() - for emb in embeddings - ) - if len(self._id_buffer) >= self.buffer_size: + # A residue table is kept flattened too: _flush counts its rows. + flat = [emb.detach().to('cpu', torch.float32).numpy().flatten() for emb in embeddings] + self._emb_buffer.extend(flat) + if self.byte_budget is not None: + self._buffered_bytes += sum(a.nbytes for a in flat) + sum(len(i) for i in dom_ids) + if self.residue_output is not None: + self._payload_buffer.extend(payloads) + self._sha1_buffer.extend(digests) + self._buffered_bytes += sum(len(p) for p in payloads) + SEQ_SHA1_BYTES * len(digests) + if len(self._id_buffer) >= self.buffer_size or ( + self.byte_budget is not None and self._buffered_bytes >= self.byte_budget): self._flush(trainer.global_rank) def _flush(self, rank): """Write the buffered rows as a single Parquet row group.""" if not self._id_buffer: return - table = pa.table( - { - 'id': self._id_buffer, - 'embedding': pa.array(self._emb_buffer, type=pa.list_(pa.float32())), - }, - schema=self._schema, - ) + if self.residue_embeddings: + embedding = self._residue_embedding_array() + else: + embedding = pa.array(self._emb_buffer, type=pa.list_(pa.float32())) + columns = { + 'id': self._id_buffer, + 'embedding': embedding, + } + if self.residue_output is not None: + columns[self.residue_output] = large_binary_array(self._payload_buffer) + columns[SEQ_SHA1] = pa.array(self._sha1_buffer, type=SEQ_SHA1_TYPE) + table = pa.table(columns, schema=self._schema) self._get_writer(rank).write_table(table) self._id_buffer.clear() self._emb_buffer.clear() + self._payload_buffer.clear() + self._sha1_buffer.clear() + self._buffered_bytes = 0 + + def _residue_embedding_array(self): + """The buffered residue tables as ``list>``: one list of D values per residue, + grouped into rows by offsets. The first call fixes the schema, with D.""" + dim = self._residue_dim + if self._schema is None: + self._schema = pa.schema([('id', pa.string()), ('embedding', pa.list_(pa.list_(pa.float32(), dim)))]) + offsets = np.concatenate(([0], np.cumsum([flat.size // dim for flat in self._emb_buffer]))) + residues = pa.FixedSizeListArray.from_arrays(pa.array(np.concatenate(self._emb_buffer)), dim) + return pa.ListArray.from_arrays(pa.array(offsets, type=pa.int32()), residues) def on_predict_end(self, trainer, pl_module): # Flush whatever didn't reach a full buffer before closing. diff --git a/tests/blosum62_reference.py b/tests/blosum62_reference.py new file mode 100644 index 0000000..ca7fa1d --- /dev/null +++ b/tests/blosum62_reference.py @@ -0,0 +1,156 @@ +"""Test oracle for the BLOSUM62 Stage 2: the per-pair biotite path it replaced. Never used in production. + +Up to FoldMatch 0.10.0's native kernel (P9.S3), Stage 2 aligned each (query, candidate) pair with biotite 1.7.1 +``align_optimal(..., local=True, max_number=1)`` and derived every output column from biotite's trace. The functions +below are that code (``alignment._to_protein``, ``_positional_query``, ``_derive_alignment_extras`` and ``_align`` of +release 562c718), with the per-process globals turned into arguments. The kernel path +(``alignment._align_query``) must give the same :class:`~foldmatch.search.alignment.AlignmentMetrics`, field for +field. +""" +from __future__ import annotations + +from typing import Optional, Tuple + +import numpy as np + +from foldmatch.search import alignment, karlin_altschul + + +def blosum62(): + import biotite.sequence.align as align + return align.SubstitutionMatrix.std_protein_matrix() + + +def to_protein(sequence: str): + """``(ProteinSequence, sanitized string, length)``; ``(None, "", 0)`` for an empty sequence.""" + import biotite.sequence as seq + clean = alignment.sanitize_sequence(sequence) + if not clean: + return None, "", 0 + return seq.ProteinSequence(clean), clean, len(clean) + + +def positional_query(query_protein, bias: np.ndarray, matrix=None): + """The query as a sequence over its own positions with an ``L x |protein alphabet|`` substitution matrix: the + BLOSUM62 row of each residue plus its bias (biotite has no per-position scoring hook).""" + import biotite.sequence as seq + import biotite.sequence.align as align + matrix = matrix if matrix is not None else blosum62() + n = len(query_protein) + alphabet = seq.Alphabet(list(range(n))) + positional = seq.GeneralSequence(alphabet, list(range(n))) + scores = matrix.score_matrix()[query_protein.code] + np.asarray(bias)[:, None] + return positional, align.SubstitutionMatrix(alphabet, matrix.get_alphabet2(), scores.astype(np.int32)) + + +def biotite_alignment(query: str, subject: str, gap: Tuple[int, int] = (11, 1), bias: Optional[np.ndarray] = None): + """biotite's local alignment of two sanitized strings with positive gap costs ``gap``; with ``bias`` the query is + scored position-specifically and the real query re-attached (identity is measured on residues).""" + import biotite.sequence.align as align + q_protein, _, _ = to_protein(query) + s_protein, _, _ = to_protein(subject) + penalty = (-abs(gap[0]), -abs(gap[1])) + matrix = blosum62() + if bias is None: + return align.align_optimal(q_protein, s_protein, matrix, gap_penalty=penalty, local=True, max_number=1)[0] + positional, pmatrix = positional_query(q_protein, bias, matrix) + aln = align.align_optimal(positional, s_protein, pmatrix, gap_penalty=penalty, local=True, max_number=1)[0] + return align.Alignment([q_protein, s_protein], aln.trace, aln.score) + + +def _count_runs(mask: np.ndarray) -> int: + if mask.size == 0: + return 0 + return int(mask[0]) + int(np.count_nonzero(mask[1:] & ~mask[:-1])) + + +def _run_length_encode(ops: str) -> str: + if not ops: + return "" + out = [] + prev, count = ops[0], 1 + for ch in ops[1:]: + if ch == prev: + count += 1 + else: + out.append(f"{count}{prev}") + prev, count = ch, 1 + out.append(f"{count}{prev}") + return "".join(out) + + +def _derive_alignment_extras(trace, query_str: str, subject_str: str, needs): + q_idx = trace[:, 0] + s_idx = trace[:, 1] + q_gap = q_idx == -1 + s_gap = s_idx == -1 + both = ~q_gap & ~s_gap + q_arr = np.frombuffer(query_str.encode("ascii"), dtype=np.uint8) + s_arr = np.frombuffer(subject_str.encode("ascii"), dtype=np.uint8) + q_res = q_idx[~q_gap] + s_res = s_idx[~s_gap] + q_start, q_end = int(q_res.min()) + 1, int(q_res.max()) + 1 + t_start, t_end = int(s_res.min()) + 1, int(s_res.max()) + 1 + matches = q_arr[q_idx[both]] == s_arr[s_idx[both]] + n_ident = int(np.count_nonzero(matches)) + mismatch = int(both.sum()) - n_ident + gap_open = _count_runs(q_gap) + _count_runs(s_gap) + cigar = q_aln = t_aln = None + if needs.cigar: + ops = np.empty(trace.shape[0], dtype="U1") + ops[both] = "M" + ops[q_gap] = "D" + ops[s_gap] = "I" + cigar = _run_length_encode("".join(ops.tolist())) + if needs.qaln: + q_aln = "".join(query_str[i] if i != -1 else "-" for i in q_idx) + if needs.taln: + t_aln = "".join(subject_str[i] if i != -1 else "-" for i in s_idx) + return n_ident, mismatch, gap_open, q_start, q_end, t_start, t_end, cigar, q_aln, t_aln + + +def reference_metrics(query: str, subject: str, *, gap: Tuple[int, int] = (11, 1), comp_bias_scale=None, + needs=None, lam=None, k=None, alp=None, db_residues=None) -> Optional[alignment.AlignmentMetrics]: + """The old ``_align`` of one (raw) query and subject: None when either sanitizes to nothing (no hit). ``alp`` + (calibrated) or ``lam``/``k`` (sampled, biotite's EValueEstimator) give the significance columns.""" + import biotite.sequence.align as align + needs = needs if needs is not None else alignment._FieldNeeds(False, False, False, False, False) + q_protein, query_str, query_len = to_protein(query) + s_protein, subject_str, subject_len = to_protein(subject) + if q_protein is None or s_protein is None: + return None + bias = None if comp_bias_scale is None else alignment.local_composition_bias(query, comp_bias_scale) + aln = biotite_alignment(query, subject, gap, bias) + score = int(aln.score) + lam_k = (alp.lam, alp.k) if alp is not None else (lam, k) + bit_score = alignment._bit_score(score, *lam_k) if lam_k[0] is not None else None + q_seq = query_str if needs.qseq else None + t_seq = subject_str if needs.tseq else None + trace = aln.trace + aln_len = int(trace.shape[0]) + if aln_len == 0: + return alignment.AlignmentMetrics( + 0.0, 0.0, 0.0, 0.0, 0, score, bit_score=bit_score, query_len=query_len, subject_len=subject_len, + cigar=("" if needs.cigar else None), q_aln=("" if needs.qaln else None), + t_aln=("" if needs.taln else None), q_seq=q_seq, t_seq=t_seq) + q_aligned = int((trace[:, 0] != -1).sum()) + s_aligned = int((trace[:, 1] != -1).sum()) + (n_ident, mismatch, gap_open, q_start, q_end, t_start, t_end, + cigar, q_aln, t_aln) = _derive_alignment_extras(trace, query_str, subject_str, needs) + pvalue = evalue = None + if alp is not None: + pvalue = karlin_altschul.pvalue_from_evalue(karlin_altschul.evalue(score, query_len, subject_len, alp)) + if db_residues: + evalue = karlin_altschul.evalue(score, query_len, db_residues, alp) + elif lam is not None: + est = align.EValueEstimator(lam, k) + pvalue = alignment._pvalue_from_evalue(alignment._evalue_from_log(est.log_evalue(score, query_len, + subject_len))) + if db_residues: + evalue = alignment._evalue_from_log(est.log_evalue(score, query_len, db_residues)) + return alignment.AlignmentMetrics( + identity_aln=n_ident / aln_len, identity_shorter=n_ident / min(query_len, subject_len), + query_coverage=q_aligned / query_len, subject_coverage=s_aligned / subject_len, aln_len=aln_len, + score=score, bit_score=bit_score, pvalue=pvalue, evalue=evalue, query_len=query_len, + subject_len=subject_len, n_ident=n_ident, mismatch=mismatch, gap_open=gap_open, q_start=q_start, + q_end=q_end, t_start=t_start, t_end=t_end, cigar=cigar, q_aln=q_aln, t_aln=t_aln, q_seq=q_seq, t_seq=t_seq) diff --git a/tests/click_messages.py b/tests/click_messages.py new file mode 100644 index 0000000..0f0f116 --- /dev/null +++ b/tests/click_messages.py @@ -0,0 +1,11 @@ +"""Click's usage errors as patterns that do not depend on its wording, which changes between releases: an unknown +option is "No such option: --x" in click 8.3 and "No such option '--x'." in click 8.5.""" +from __future__ import annotations + +import re + + +def no_such_option(option: str) -> str: + """A pattern of click's error for the unknown ``option``, in either wording.""" + name = re.escape(option) + return rf"No such option(: {name}\b| '{name}')" diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..da405f7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +import os + +import pytest + +# Plain help and error text from every CLI, whatever the environment. Typer forces a terminal, and rich then writes +# colour codes, when GITHUB_ACTIONS, FORCE_COLOR or PY_COLORS is set (as on GitHub Actions); the codes split the words +# that the tests look for ('--min-res-n'). TERMINAL_WIDTH, when set, would override the COLUMNS that the help tests +# give the CLI. typer reads both once, when typer.rich_utils is first imported (at the first help or error), so they +# are set here, before any test runs; the CLIs that tests run as subprocesses inherit them. +os.environ["_TYPER_FORCE_DISABLE_TERMINAL"] = "1" +os.environ.pop("TERMINAL_WIDTH", None) + + +@pytest.fixture(scope="session") +def fmsw_libs(): + """The native Smith-Waterman kernel libraries of this machine, built in place by the developer build + (``python -m foldmatch._native.build``) when they are missing or stale. Without a C compiler the tests that need + the kernel are skipped, unless FOLDMATCH_REQUIRE_NATIVE=1 (CI), which makes it a failure.""" + from foldmatch._native import build + try: + return build.build(only_if_stale=True, quiet=True) + except build.NativeBuildError as e: + if os.environ.get("FOLDMATCH_REQUIRE_NATIVE") == "1": + pytest.fail(f"native kernel build failed: {e}") + pytest.skip(f"native kernel unavailable (no C compiler?): {e}") diff --git a/tests/esm_codes_bundle.py b/tests/esm_codes_bundle.py new file mode 100644 index 0000000..1b18b84 --- /dev/null +++ b/tests/esm_codes_bundle.py @@ -0,0 +1,54 @@ +"""The esm-codes-v1 bundle for tests. + +By default the bundle comes the way FoldMatch gets it: ``get_esm_codes_bundle(get(DEFAULT_ESM_CODES))``, i.e. the +registry entry's repo, revision and sha256s, downloaded into the Hugging Face cache once and verified on every call. +``FOLDMATCH_ESM_CODES_BUNDLE`` (a directory holding the bundle files) overrides it for the encoder tests, to test a +bundle before its upload. + +Tests are skipped only when offline mode is on (``HF_HUB_OFFLINE=1``) and the bundle is not cached. Every other +failure is an error, never a skip: with offline mode off, a Hub that cannot be reached (an outage, a firewall, an +HTTP 5xx) fails the tests instead of turning them green without the acceptance checks, and a sha256 mismatch or a +file missing at the pinned revision always fails. +""" +import functools +import os +import unittest +from pathlib import Path + +from huggingface_hub import constants as hf_constants + +BUNDLE_ENV = "FOLDMATCH_ESM_CODES_BUNDLE" + + +@functools.lru_cache(maxsize=None) +def _fetch(): + """One fetch per test session: ``(files, None)``, or ``(None, error)`` (kept, so that a failing fetch is not + repeated, with its network timeouts, by every test class).""" + from foldmatch.search import scoring_system as ss + from foldmatch.utils.model import get_esm_codes_bundle + try: + return get_esm_codes_bundle(ss.get(ss.DEFAULT_ESM_CODES)), None + except Exception as e: # noqa: BLE001 (classified by registry_files: skip only offline and not cached) + return None, e + + +def registry_files(): + """``{file name: path}`` of the registry bundle (sha256-verified). ``unittest.SkipTest`` only when offline mode + is on and the bundle is not cached; any other failure raises.""" + from foldmatch.search import scoring_system as ss + files, error = _fetch() + if error is None: + return files + if isinstance(error, ss.BundleUnavailableError) and hf_constants.HF_HUB_OFFLINE: + raise unittest.SkipTest(f"offline (HF_HUB_OFFLINE=1) and the esm-codes-v1 bundle is not cached: {error}") + raise RuntimeError(f"the esm-codes-v1 bundle could not be fetched ({type(error).__name__}); with offline mode " + f"off this is a test failure, not a skip: {error}") from error + + +def bundle_dir() -> Path: + """The directory holding the bundle files: ``$FOLDMATCH_ESM_CODES_BUNDLE`` if set, else the registry bundle's + Hugging Face snapshot directory.""" + d = os.environ.get(BUNDLE_ENV) + if d: + return Path(d) + return Path(registry_files()["bundle.json"]).parent diff --git a/tests/esm_codes_reference.py b/tests/esm_codes_reference.py new file mode 100644 index 0000000..3325b44 --- /dev/null +++ b/tests/esm_codes_reference.py @@ -0,0 +1,437 @@ +"""Test oracle for the esm-codes Stage-2 kernel (PLAN section 7, P3.6). numpy only; never used in production. + +* ``normalize_scores`` and ``smith_waterman`` (with ``Alignment``, ``needleman_wunsch`` and ``_align``) are a VERBATIM + copy of ``rcsb-embedding-search/src/scripts/esm3_needleman_wunsch.py`` (md5 7ec19b7e5ff89dc7b44d2512aed3997e), lines + 113-125 and 138-311: the float64 harness the research results were validated against. :func:`check_verbatim` + proves the blocks below are unchanged (sha256 of each block, recorded when they were copied). +* The S5 zero-norm rule and the S7 degenerate rule are thin wrappers around it (:func:`reference_qprep64`, + :func:`reference_rowcol_s7`), with the same expression order as the harness, so they are bit-identical to it + whenever nothing is degenerate. This is exactly how the FoldMatch fixtures F1 / F-deg were computed + (``make_foldmatch_fixtures.py`` ``ref_pair64``). +* :func:`sw_margins64` re-runs the same float64 recurrences keeping every matrix, to measure how close the oracle's + best-cell choice and traceback decisions are to a tie (the S16 path clause). +* :func:`reference_encode` (S2/S3 with the research fp16 casts, float64 arithmetic otherwise) and + :func:`reference_evalue` (S11) complete the reference chain. +""" +from __future__ import annotations + +import hashlib +import math +import os +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +ORACLE_SOURCE = "rcsb-embedding-search/src/scripts/esm3_needleman_wunsch.py" +ORACLE_SOURCE_MD5 = "7ec19b7e5ff89dc7b44d2512aed3997e" +VERBATIM_BLOCKS = { # name: (first line, last line, sha256 of the block text including its final newline) + "normalize_scores": (113, 125, "ad201b13c2e3b168901bb3816269f08ad8e3cf37e4b4ee60eab2b11de5078662"), + "smith_waterman": (138, 311, "a49b1365509fe984544dd3a2cdac9886570fa41e9f3e7b0165736a333eb2c622"), +} +S7_EPS = 1e-6 # S7: a row / column with population std < 1e-6 contributes exactly 0 +QPREP_EPS = 1e-12 # S5: |q_i| < 1e-12 -> zero row + + +# ---- BEGIN VERBATIM normalize_scores ---- +def normalize_scores(sim: np.ndarray, mode: str = "rowcol") -> np.ndarray: + sim = np.asarray(sim, dtype=np.float64) + eps = 1e-8 + if mode == "none": + return sim + if mode == "zscore": + return (sim - sim.mean()) / (sim.std() + eps) + if mode == "rowcol": + z_row = (sim - sim.mean(axis=1, keepdims=True)) / (sim.std(axis=1, keepdims=True) + eps) + z_col = (sim - sim.mean(axis=0, keepdims=True)) / (sim.std(axis=0, keepdims=True) + eps) + return 0.5 * (z_row + z_col) + raise ValueError(f"unknown normalization mode: {mode!r}") + +# ---- END VERBATIM normalize_scores ---- + + +# ---- BEGIN VERBATIM smith_waterman ---- +# ============================================================================= +# Needleman-Wunsch (global) and Smith-Waterman (local), affine gaps +# ============================================================================= +@dataclass +class Alignment: + score: float + pairs: list[tuple[int | None, int | None]] # (index in A, index in B), None = gap + + def gapped(self, seq_a: str, seq_b: str) -> tuple[str, str]: + top = "".join("-" if i is None else seq_a[i] for i, _ in self.pairs) + bottom = "".join("-" if j is None else seq_b[j] for _, j in self.pairs) + return top, bottom + + def aligned_pairs(self) -> list[tuple[int, int]]: + return [(i, j) for i, j in self.pairs if i is not None and j is not None] + + +def needleman_wunsch( + scores: np.ndarray, + gap_open: float, + gap_extend: float | None = None, + free_end_gaps: bool = False, +) -> Alignment: + """Global alignment of the full sequences (Needleman & Wunsch 1970). + + scores[i, j] score for aligning residue i of A with residue j of B (shape n x m) + gap_open penalty (positive) for the first position of a gap + gap_extend penalty for each further position; None -> gap_open (linear gaps) + free_end_gaps leading/trailing gaps cost nothing (semi-global) + """ + return _align(scores, gap_open, gap_extend, local=False, free_end_gaps=free_end_gaps) + + +def smith_waterman( + scores: np.ndarray, + gap_open: float, + gap_extend: float | None = None, +) -> Alignment: + """Local alignment: the highest-scoring pair of segments (Smith & Waterman 1981). + + Only meaningful when unrelated residue pairs score below zero on average, so shift + z-scored matrices down first (LOCAL_SCORE_OFFSET). Returns an empty alignment + (no pairs, score 0) when no residue pair scores above zero. + """ + return _align(scores, gap_open, gap_extend, local=True, free_end_gaps=False) + + +def _align(scores, gap_open, gap_extend, local: bool, free_end_gaps: bool) -> Alignment: + """Dynamic programming shared by both algorithms (Gotoh 1982 affine-gap recurrences). + + States: M = residue pair, X = A residue vs gap, Y = gap vs B residue, H = max(M, X, Y); + Smith-Waterman adds 0 as a fourth option ("start a new alignment here"). + A gap of length L costs gap_open + (L - 1) * gap_extend. + Each DP row is filled with numpy: M and X need only the previous row, and the + within-row Y recurrence is solved with a running maximum, so Python loops over rows only. + """ + S = np.asarray(scores, dtype=np.float64) + if S.ndim != 2: + raise ValueError("scores must be a 2-D matrix") + n, m = S.shape + go = float(gap_open) + ge = go if gap_extend is None else float(gap_extend) + if not go >= ge >= 0: + raise ValueError("need gap_open >= gap_extend >= 0") + + cols = np.arange(m + 1) + + def edge_score(k): # first row / column: a leading gap of length k + if local or free_end_gaps: + return np.zeros_like(k, dtype=np.float64) + return np.where(k > 0, -(go + (k - 1) * ge), 0.0) + + # Traceback flags + h_from_y = np.zeros((n + 1, m + 1), dtype=bool) # H[i,j] = Y[i,j] (else V = max(M, X[, 0])) + v_from_x = np.zeros((n + 1, m + 1), dtype=bool) # V[i,j] = X[i,j] (else M, diagonal) + x_extend = np.zeros((n + 1, m + 1), dtype=bool) # X[i,j] extends X[i-1,j] (else opens from H[i-1,j]) + y_extend = np.zeros((n + 1, m + 1), dtype=bool) # Y[i,j] extends Y[i,j-1] (else opens from V[i,j-1]) + start = np.zeros((n + 1, m + 1), dtype=bool) # the alignment starts here: traceback stops + + # Row 0 + H_prev = edge_score(cols) + X_prev = np.full(m + 1, -np.inf) + last_col = np.empty(n + 1) + last_col[0] = H_prev[m] + if local: + start[0, :] = True + else: + start[0, 0] = True + h_from_y[0, 1:] = True # leading gap in A + y_extend[0, 2:] = True + best_score, best_i, best_j = 0.0, 0, 0 # Smith-Waterman: best cell so far + + for i in range(1, n + 1): + # X: A[i-1] against a gap (vertical move) + x_open = H_prev - go + x_ext = X_prev - ge + X = np.maximum(x_open, x_ext) + x_extend[i] = x_ext > x_open + + # M: A[i-1] paired with B[j-1] (diagonal move) + M = np.empty(m + 1) + M[0] = -np.inf + M[1:] = H_prev[:-1] + S[i - 1] + + V = np.maximum(M, X) + v_from_x[i] = X > M + if local: + # Smith-Waterman: a prefix scoring <= 0 is dropped and a new alignment starts + start[i] = V <= 0 + V = np.maximum(V, 0.0) + X[0] = -np.inf + else: + # Column 0: leading gap in B + V[0] = X[0] = edge_score(np.array(i)) + v_from_x[i, 0] = True + x_extend[i, 0] = i > 1 + + # Y: gap against B[j-1] (horizontal move) + # Y[j] = max(V[j-1] - go, Y[j-1] - ge) = max_{k= gap_extend) + Y = np.empty(m + 1) + Y[0] = -np.inf + if m: + Y[1:] = np.maximum.accumulate(V[:-1] + ge * cols[:-1]) - go - ge * cols[:-1] + y_extend[i, 1:] = (Y[:-1] - ge) > (V[:-1] - go) + + H = np.maximum(V, Y) + h_from_y[i] = Y > V + + if local: + j_max = int(np.argmax(H)) + if H[j_max] > best_score: + best_score, best_i, best_j = float(H[j_max]), i, j_max + H_prev, X_prev = H, X + last_col[i] = H[m] + + # Where the traceback starts + rev: list[tuple[int | None, int | None]] = [] # built in reverse + if local: + i, j, score = best_i, best_j, best_score + elif free_end_gaps: + j_end = m - int(np.argmax(H_prev[::-1])) # best cell in the last row (ties -> j = m) + i_end = n - int(np.argmax(last_col[::-1])) # best cell in the last column (ties -> i = n) + if H_prev[j_end] >= last_col[i_end]: + i, j, score = n, j_end, float(H_prev[j_end]) + else: + i, j, score = i_end, m, float(last_col[i_end]) + rev.extend((None, jj) for jj in range(m - 1, j - 1, -1)) # free trailing gap in A + rev.extend((ii, None) for ii in range(n - 1, i - 1, -1)) # free trailing gap in B + else: + i, j, score = n, m, float(H_prev[m]) + + # Traceback; ties prefer the diagonal + state = "H" + while True: + if state == "H": + state = "Y" if h_from_y[i, j] else "V" + elif state == "V": + if start[i, j]: + break + state = "X" if v_from_x[i, j] else "M" + elif state == "M": + rev.append((i - 1, j - 1)) + i, j, state = i - 1, j - 1, "H" + elif state == "X": + rev.append((i - 1, None)) + state = "X" if x_extend[i, j] else "H" + i -= 1 + else: # "Y" + rev.append((None, j - 1)) + state = "Y" if y_extend[i, j] else "V" + j -= 1 + rev.reverse() + return Alignment(score=score, pairs=rev) +# ---- END VERBATIM smith_waterman ---- + + +def _verbatim_text(name: str) -> str: + text = Path(__file__).read_text() + begin, end = f"# ---- BEGIN VERBATIM {name} ----\n", f"# ---- END VERBATIM {name} ----" + return text[text.index(begin) + len(begin):text.index(end)] + + +def check_verbatim(source: str | os.PathLike | None = None) -> bool: + """True when every verbatim block hashes to its recorded sha256 and, if the oracle source file is given (or + found via $FOLDMATCH_ORACLE_SOURCE), that file has md5 7ec19b7e... and its line ranges equal the blocks.""" + for name, (_a, _b, sha) in VERBATIM_BLOCKS.items(): + if hashlib.sha256(_verbatim_text(name).encode()).hexdigest() != sha: + return False + source = source or os.environ.get("FOLDMATCH_ORACLE_SOURCE") + if source: + data = Path(source).read_bytes() + if hashlib.md5(data).hexdigest() != ORACLE_SOURCE_MD5: + return False + lines = data.decode().split("\n") + for name, (a, b, _sha) in VERBATIM_BLOCKS.items(): + if "\n".join(lines[a - 1:b]) + "\n" != _verbatim_text(name): + return False + return True + + +# ============================================================================= +# S5 / S7 wrappers and the float64 pair oracle +# ============================================================================= +def reference_qprep64(q16) -> np.ndarray: + """harness query rows: fp64(q) / |q| (unit rows). S5 adds: |q_i| < 1e-12 -> zero row; NaN / Inf is an error.""" + Q = np.asarray(q16, dtype=np.float64) + if not np.isfinite(Q).all(): + raise FloatingPointError("non-finite query encoding (S5: error)") + nrm = np.linalg.norm(Q, axis=1, keepdims=True) + zero = nrm < QPREP_EPS + return np.where(zero, 0.0, Q / np.where(zero, 1.0, nrm)) + + +def decode64(codes) -> np.ndarray: + """t = (unpackbits(codes)[:, :64] * 2 - 1) / 8 (unit rows), float64.""" + return (np.unpackbits(np.asarray(codes, np.uint8), axis=1)[:, :64].astype(np.float64) * 2 - 1) / 8.0 + + +def reference_rowcol_s7(sim): + """normalize_scores(sim, 'rowcol') with S7; same expression order as the verbatim harness (bit-identical to it + when nothing is degenerate). Returns (z, degenerate rows, degenerate columns).""" + sim = np.asarray(sim, dtype=np.float64) + eps = 1e-8 + mu, sd = sim.mean(axis=1, keepdims=True), sim.std(axis=1, keepdims=True) + nu, tau = sim.mean(axis=0, keepdims=True), sim.std(axis=0, keepdims=True) + z_row = (sim - mu) / (sd + eps) + z_col = (sim - nu) / (tau + eps) + dr, dc = sd < S7_EPS, tau < S7_EPS + if dr.any(): + z_row = np.where(dr, 0.0, z_row) + if dc.any(): + z_col = np.where(dc, 0.0, z_col) + return 0.5 * (z_row + z_col), int(dr.sum()), int(dc.sum()) + + +def aln_path(aln: Alignment) -> np.ndarray: + """Alignment.pairs -> int32 [k, 2] (query i, target j), -1 = gap, forward order.""" + return np.array([(-1 if i is None else i, -1 if j is None else j) for i, j in aln.pairs], np.int32).reshape(-1, 2) + + +def reference_pair64(q16, codes, offset: float = 1.0, gap_open: float = 5.0, gap_extend: float = 0.5, + s7: bool = True, path: bool = True, margins: bool = False) -> dict: + """float64 oracle for one pair: S_f (+ forward path), S_r = the same on the row-reversed query (S9). + ``margins`` adds the best-cell and traceback margins of the forward run (:func:`sw_margins64`).""" + Q, T = reference_qprep64(q16), decode64(codes) + res = {} + for d, Qd in (("f", Q), ("r", Q[::-1])): + sim = Qd @ T.T + if s7: + z, nr, nc = reference_rowcol_s7(sim) + else: + z, nr, nc = normalize_scores(sim, "rowcol"), -1, -1 + a = smith_waterman(z - offset, gap_open, gap_extend) + res[f"S_{d}"] = float(a.score) + res[f"ndeg_rows_{d}"], res[f"ndeg_cols_{d}"] = nr, nc + if d == "f": + res["z"] = z + if path: + res["path"] = aln_path(a) + if margins: + mg = sw_margins64(z - offset, gap_open, gap_extend) + res.update(best_margin=mg["best_margin"], tb_margin=mg["tb_margin"]) + return res + + +def _gap(a: float, b: float) -> float: + if not (math.isfinite(a) and math.isfinite(b)): + return math.inf + return abs(a - b) + + +def sw_margins64(scores, gap_open: float, gap_extend: float) -> dict: + """The verbatim local recurrences (``_align``, local=True) keeping H, X, V (before the clip), M and Y, then the + oracle traceback with each decision's distance from a tie. Returns score, best cell (1-based), path, + ``best_margin`` (best H minus the best H of any other cell; 0 for an empty alignment) and ``tb_margin`` (the + smallest decision gap along the traceback: Y vs V, the V <= 0 start test, X vs M, and the X / Y extend tests).""" + S = np.asarray(scores, dtype=np.float64) + n, m = S.shape + go, ge = float(gap_open), float(gap_extend) + cols = np.arange(m + 1) + H = np.zeros((n + 1, m + 1)); X = np.full((n + 1, m + 1), -np.inf); V = np.zeros((n + 1, m + 1)) + Vpre = np.full((n + 1, m + 1), -np.inf); M = np.full((n + 1, m + 1), -np.inf); Y = np.full((n + 1, m + 1), -np.inf) + H_prev, X_prev = np.zeros(m + 1), np.full(m + 1, -np.inf) + best, bi, bj = 0.0, 0, 0 + for i in range(1, n + 1): + Xi = np.maximum(H_prev - go, X_prev - ge) + Mi = np.empty(m + 1) + Mi[0] = -np.inf + Mi[1:] = H_prev[:-1] + S[i - 1] + Vi = np.maximum(Mi, Xi) + Vpre[i] = Vi + Vi = np.maximum(Vi, 0.0) + Xi[0] = -np.inf + Yi = np.empty(m + 1) + Yi[0] = -np.inf + if m: + Yi[1:] = np.maximum.accumulate(Vi[:-1] + ge * cols[:-1]) - go - ge * cols[:-1] + Hi = np.maximum(Vi, Yi) + H[i], X[i], V[i], M[i], Y[i] = Hi, Xi, Vi, Mi, Yi + j_max = int(np.argmax(Hi)) + if Hi[j_max] > best: + best, bi, bj = float(Hi[j_max]), i, j_max + H_prev, X_prev = Hi, Xi + if best > 0.0: + Hf = H.ravel().copy() + Hf[bi * (m + 1) + bj] = -np.inf + best_margin = best - float(Hf.max()) + else: + best_margin = 0.0 + tb = math.inf + i, j, state, rev = bi, bj, "H", [] + while best > 0.0: + if state == "H": + tb = min(tb, _gap(Y[i, j], V[i, j])) + state = "Y" if Y[i, j] > V[i, j] else "V" + elif state == "V": + tb = min(tb, abs(Vpre[i, j]) if math.isfinite(Vpre[i, j]) else math.inf) + if Vpre[i, j] <= 0 or i == 0 or j == 0: + break + tb = min(tb, _gap(X[i, j], M[i, j])) + state = "X" if X[i, j] > M[i, j] else "M" + elif state == "M": + rev.append((i - 1, j - 1)) + i, j, state = i - 1, j - 1, "H" + elif state == "X": + rev.append((i - 1, -1)) + tb = min(tb, _gap(X[i - 1, j] - ge, H[i - 1, j] - go)) + state = "X" if (X[i - 1, j] - ge) > (H[i - 1, j] - go) else "H" + i -= 1 + else: + rev.append((-1, j - 1)) + tb = min(tb, _gap(Y[i, j - 1] - ge, V[i, j - 1] - go)) + state = "Y" if (Y[i, j - 1] - ge) > (V[i, j - 1] - go) else "V" + j -= 1 + path = np.array(rev[::-1], np.int32).reshape(-1, 2) + return dict(score=best, best=(bi, bj), path=path, best_margin=best_margin, tb_margin=tb) + + +# ============================================================================= +# S2/S3 reference encoder and S11 E-values +# ============================================================================= +def _gelu_exact(x: np.ndarray) -> np.ndarray: + try: + from scipy.special import erf + except ImportError: # pragma: no cover + erf = np.frompyfunc(math.erf, 1, 1) + return 0.5 * x * (1.0 + np.asarray(erf(x / math.sqrt(2.0)), dtype=np.float64)) + + +def _mlp64(xw: np.ndarray, w: dict) -> np.ndarray: + """f(x) = lin(x) + fc2(GELU_exact(fc1(x))) in float64; w holds numpy 'lin.weight', 'lin.bias', 'fc1.*', 'fc2.*' + (nn.Linear layout: weight [out, in]).""" + g = {k: np.asarray(v, np.float64) for k, v in w.items()} + h = xw @ g["fc1.weight"].T + g["fc1.bias"] + return xw @ g["lin.weight"].T + g["lin.bias"] + _gelu_exact(h) @ g["fc2.weight"].T + g["fc2.bias"] + + +def reference_encode(rows16, dims, offset, M, f_q: dict, f_t: dict): + """Reference S2-S5 encoder for ESM3 rows ``rows16`` fp16 [L, 1536] (BOS / EOS already removed): + xs = fp32(x16[:, dims]) - fp32(offset); xw = fp32(fp16(xs @ M)) (S2, product in float64 here); + h = f(xw) in float64 (S3); codes = packbits(h_t >= 0) (S4); q = fp16(h_q) (S5). + Returns (codes uint8 [L, 8], q float16 [L, 64], h_t float64 [L, 64]). Bits with |h_t| ~ 0 may differ from an + fp32 encoder (see the fixtures' margin_t).""" + x16 = np.asarray(rows16, np.float16) + xs = x16[:, np.asarray(dims)].astype(np.float32) - np.asarray(offset, np.float32) + xw = (xs.astype(np.float64) @ np.asarray(M, np.float64)).astype(np.float16).astype(np.float64) + h_t, h_q = _mlp64(xw, f_t), _mlp64(xw, f_q) + return np.packbits(h_t >= 0, axis=1), h_q.astype(np.float16), h_t + + +def reference_lam_lnk(m, centres, lam, lnK): + """S11: lambda(m), lnK(m) linear in ln m between the centres, clamped at both ends (D15).""" + x = np.log(np.asarray(m, np.float64)) + lc = np.log(np.asarray(centres, np.float64)) + return np.interp(x, lc, np.asarray(lam, np.float64)), np.interp(x, lc, np.asarray(lnK, np.float64)) + + +def reference_evalue(S_prime, m, N, centres, lam, lnK): + """S11: lnE = (lnK(m) + ln m + ln N) - lambda(m) S' and bits = (lambda(m) S' - lnK(m)) / ln 2 (float64).""" + la, lk = reference_lam_lnk(m, centres, lam, lnK) + Sp = np.asarray(S_prime, np.float64) + lnE = (lk + np.log(np.asarray(m, np.float64)) + np.log(np.asarray(N, np.float64))) - la * Sp + return lnE, (la * Sp - lk) / math.log(2.0) diff --git a/tests/resources/chain_embeddings/1acb.A.pt b/tests/resources/chain_embeddings/1acb.A.pt new file mode 100644 index 0000000..ccfeaa8 Binary files /dev/null and b/tests/resources/chain_embeddings/1acb.A.pt differ diff --git a/tests/resources/chain_embeddings/1acb.B.pt b/tests/resources/chain_embeddings/1acb.B.pt new file mode 100644 index 0000000..cdd18d2 Binary files /dev/null and b/tests/resources/chain_embeddings/1acb.B.pt differ diff --git a/tests/resources/chain_embeddings/2uzi.A.pt b/tests/resources/chain_embeddings/2uzi.A.pt new file mode 100644 index 0000000..7db1b4a Binary files /dev/null and b/tests/resources/chain_embeddings/2uzi.A.pt differ diff --git a/tests/resources/chain_embeddings/2uzi.B.pt b/tests/resources/chain_embeddings/2uzi.B.pt new file mode 100644 index 0000000..ecd2b20 Binary files /dev/null and b/tests/resources/chain_embeddings/2uzi.B.pt differ diff --git a/tests/resources/chain_embeddings/2uzi.C.pt b/tests/resources/chain_embeddings/2uzi.C.pt new file mode 100644 index 0000000..335411d Binary files /dev/null and b/tests/resources/chain_embeddings/2uzi.C.pt differ diff --git a/tests/resources/esm_codes/FIXTURES.json b/tests/resources/esm_codes/FIXTURES.json new file mode 100644 index 0000000..1d469b9 --- /dev/null +++ b/tests/resources/esm_codes/FIXTURES.json @@ -0,0 +1,1163 @@ +{ + "fixture_set": "FoldMatch esm-codes-v1 test fixtures v1 (PLAN P0.8)", + "created_utc": "2026-09-28T02:35:33Z", + "system": "esm-codes-v1", + "calibration_version": "foldmatch-evalue-M8_s0-D_fmr-pnlU50s1000-M3t999-v1", + "table_key": "D_fmr/pnl_uniref50/M3t999", + "setting": { + "offset": 1.0, + "gap_open": 5.0, + "gap_extend": 0.5, + "s7_eps": 1e-06, + "qprep_eps": 1e-12 + }, + "generator": { + "path": "/Users/joan/devel/rcsb-embedding-search/src/scripts/esm_codes/make_foldmatch_fixtures.py", + "md5": "5f047ac6e5356da40ce4c0bde63cd89b", + "sha256": "bd1e5f414864e24938bee6e0d4924e9b31ccbe8926fe6c2b6f47e29a6a027636", + "stages": [ + "esm3 (foldmatch env)", + "encode --tag ref (research env)", + "encode --tag fm (foldmatch env, cross-check)", + "build (research env)" + ] + }, + "environments": { + "esm3": { + "python": "3.12.13", + "executable": "/Users/joan/app/miniconda3/envs/foldmatch/bin/python", + "numpy": "2.4.4", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "machine": "arm64", + "threads_env": { + "OMP_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": null + }, + "torch": "2.11.0", + "esm": "3.2.3", + "foldmatch_dist_version": "0.8.0", + "foldmatch_imported_from": "/Users/joan/devel/rcsb-embedding-model/src/foldmatch", + "foldmatch_git": { + "head": "3c3625c83f8ec84adb71948ce9c4f040094814ee", + "branch": "js/dev-esm-code-alignments", + "dirty_files": [ + "?? tests/resources/esm_codes/" + ] + }, + "hf_hub_offline": "1" + }, + "encode_ref": { + "python": "3.12.12", + "executable": "/Users/joan/app/miniconda3/envs/rcsb-embedding-search/bin/python", + "numpy": "2.4.4", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "machine": "arm64", + "threads_env": { + "OMP_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": "1" + }, + "torch": "2.9.0", + "torch_threads": 4, + "repeat_threads": 1 + }, + "encode_fm": { + "python": "3.12.13", + "executable": "/Users/joan/app/miniconda3/envs/foldmatch/bin/python", + "numpy": "2.4.4", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "machine": "arm64", + "threads_env": { + "OMP_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": "1" + }, + "torch": "2.11.0", + "torch_threads": 4, + "repeat_threads": 1 + }, + "build": { + "python": "3.12.12", + "executable": "/Users/joan/app/miniconda3/envs/rcsb-embedding-search/bin/python", + "numpy": "2.4.4", + "platform": "macOS-26.6.2-arm64-arm-64bit", + "machine": "arm64", + "threads_env": { + "OMP_NUM_THREADS": "1", + "VECLIB_MAXIMUM_THREADS": "1" + }, + "research_modules": { + "harness": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/harness/harness.py", + "cath_score": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/scripts/cath_score.py", + "swfast": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/swfast.py", + "bench_validate": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/scripts/bench_validate.py", + "libswkern": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/libswkern.dylib", + "libswmulti": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/libswmulti.so" + }, + "vector_lanes": 4 + } + }, + "esm3": { + "snapshot_path": "/Users/joan/.cache/huggingface/hub/models--rcsb--rcsb-esm/snapshots/636acb420daf674df37ab0220f56061a5de2f169", + "snapshot_revision": "636acb420daf674df37ab0220f56061a5de2f169", + "weights_file": "/Users/joan/.cache/huggingface/hub/models--rcsb--rcsb-esm/snapshots/636acb420daf674df37ab0220f56061a5de2f169/data/weights/esm3_sm_open_v1.pth", + "weights_blob_sha256": "5ead5a135c658068db6a4f1b933e72d6110992c4668822e1c0e2dcc53e38acd9", + "param_dtypes": [ + "torch.float32" + ], + "device": "cpu", + "autocast": false, + "precision": "fp32 weights, fp32 compute (CPU)", + "default_torch_threads": 1, + "repeat_torch_threads": 4, + "load_s": 4.6, + "pass_s_default": 3.6, + "pass_s_repeat": 3.4, + "call": "foldmatch.modules.esm_module.EsmModule(get_residue_model(cpu)).predict_step(items, 0) under torch.inference_mode(); per_residue_embedding [L+2, 1536] fp32 -> stored fp16", + "fasta_reader": "foldmatch.dataset.esm_prot_from_fasta.EsmProtFromFasta(test_sequences.fasta)" + }, + "reference_code": { + "oracle": "harness.py normalize_scores + smith_waterman (verbatim copy of esm3_needleman_wunsch.py, check_verbatim=True) + S5 zero-row and S7 degenerate wrappers in this script", + "kernel": "research swkern.c via swfast.swz_ad_batch (probe.py kernel)", + "q90": "cath_score.query_scores (swmulti.c) + np.quantile(f64 S, 0.9) (panel_local.py)", + "evalue": "bench_validate.tabp + lnE = (lnK + ln m + ln N) - lam S'", + "encoders": "sw_common.Whitener + Model (L2m_B8_s0.pt), CPU fp32 torch, each item alone" + }, + "inputs": { + "fixture_candidate": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/inventory_probe/fixture_candidate.npz", + "md5": "d724b769bcf26178cc20a3bf157a64bf", + "expected_md5": "d724b769bcf26178cc20a3bf157a64bf", + "bytes": 892791 + }, + "probe_report": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/inventory_probe/probe_report.json", + "md5": "c35ed1589908808e31afe3dfd562f776", + "expected_md5": "c35ed1589908808e31afe3dfd562f776", + "bytes": 35632 + }, + "probe_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/inventory_probe/probe.py", + "md5": "d286e8ce440bb8aa42f00e00fd410c04", + "expected_md5": null, + "bytes": 8126 + }, + "enc_val": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/throughput/build/data/L2m_B8_s0.val.npz", + "md5": "5aebc14942665748313f22ca737ca26d", + "expected_md5": "5aebc14942665748313f22ca737ca26d", + "bytes": 58436408 + }, + "enc_test": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/throughput/build/data/L2m_B8_s0.test.npz", + "md5": "e570bc26e1a142183aaf85e9a7a57df7", + "expected_md5": "e570bc26e1a142183aaf85e9a7a57df7", + "bytes": 58333080 + }, + "f1_settings_val": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/evalue/f1_settings/results/dom/M8_s0.val.npz", + "md5": "67a6ee1d9d9825978c8cd7a25869c8a9", + "expected_md5": null, + "bytes": 4070309 + }, + "f1_settings_test": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/evalue/f1_settings/results/dom/M8_s0.test.npz", + "md5": "f4225dd075ef82288748749dcbb016ef", + "expected_md5": null, + "bytes": 2562417 + }, + "cath_set": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/data/cath_set.npz", + "md5": "78b6c3fcf64d1ffcc9aa98d81c04c81c", + "expected_md5": "78b6c3fcf64d1ffcc9aa98d81c04c81c", + "bytes": 113054112 + }, + "cath_q90_local": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/results/cath_q90_shipped_local.npy", + "md5": "8d78e468a8922c9a3e60e7801c7c287f", + "expected_md5": null, + "bytes": 46488 + }, + "enc_check": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/sw_mlp/enc_check/L2m_B8_s0.npz", + "md5": "5e646fb02d0161733379a261b47c20ec", + "expected_md5": null, + "bytes": 1267884 + }, + "enc_check_items": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/sw_mlp/enc_check/check_items.json", + "md5": "7015c0081284aa6c53a2cef24ebb2734", + "expected_md5": null, + "bytes": 996 + }, + "encoders": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/sw_mlp/models/L2m_B8_s0.pt", + "md5": "85fdd985087d6813e9994274d4fca6ab", + "expected_md5": "85fdd985087d6813e9994274d4fca6ab", + "bytes": 13891299 + }, + "sw_common": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/sw_mlp/sw_common.py", + "md5": "b8dab0a03907c9b7ed67663d96c55561", + "expected_md5": "b8dab0a03907c9b7ed67663d96c55561", + "bytes": 12622 + }, + "zcw50_clean": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/clean_pairs/zcw50_clean.npz", + "md5": "4c7692ed66e599aca906b41ec7c13f04", + "expected_md5": "4c7692ed66e599aca906b41ec7c13f04", + "bytes": 37689630 + }, + "ship_final": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/results/ship_final.json", + "md5": "c51735f962c04bd8c441b9625c4cf30a", + "expected_md5": "c51735f962c04bd8c441b9625c4cf30a", + "bytes": 36685 + }, + "panel_codes": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/results/shipped_panel_codes.npz", + "md5": "15cc1c7b2d1dd488a5f9b742365ccb13", + "expected_md5": "15cc1c7b2d1dd488a5f9b742365ccb13", + "bytes": 8020960 + }, + "harness": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/harness/harness.py", + "md5": "05f387cb400cc85dbacaef771a275eeb", + "expected_md5": "05f387cb400cc85dbacaef771a275eeb", + "bytes": 55693 + }, + "oracle_script": { + "path": "/Users/joan/devel/rcsb-embedding-search/src/scripts/esm3_needleman_wunsch.py", + "md5": "7ec19b7e5ff89dc7b44d2512aed3997e", + "expected_md5": "7ec19b7e5ff89dc7b44d2512aed3997e", + "bytes": 19740 + }, + "swkern_c": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/throughput/build/swkern.c", + "md5": "438d5a9811153f61e66cee845f1ed037", + "expected_md5": "438d5a9811153f61e66cee845f1ed037", + "bytes": 30590 + }, + "swfast_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/throughput/build/swfast.py", + "md5": "80a813553d42801fce6b77462c5e1d0a", + "expected_md5": "80a813553d42801fce6b77462c5e1d0a", + "bytes": 15466 + }, + "libswkern": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/throughput/build/libswkern.dylib", + "md5": "4f292c5eec269bed401103cd3e5b1cf0", + "expected_md5": null, + "bytes": 50768 + }, + "kdir_swkern_c": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/swkern.c", + "md5": "438d5a9811153f61e66cee845f1ed037", + "expected_md5": "438d5a9811153f61e66cee845f1ed037", + "bytes": 30590 + }, + "kdir_swfast_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/swfast.py", + "md5": "80a813553d42801fce6b77462c5e1d0a", + "expected_md5": "80a813553d42801fce6b77462c5e1d0a", + "bytes": 15466 + }, + "kdir_swmulti_c": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/swmulti.c", + "md5": "b12fbc2c03638659440b12282a4d8670", + "expected_md5": "b12fbc2c03638659440b12282a4d8670", + "bytes": 1951 + }, + "kdir_libswkern": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/libswkern.dylib", + "md5": "4f292c5eec269bed401103cd3e5b1cf0", + "expected_md5": null, + "bytes": 50768 + }, + "kdir_libswmulti": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/kernel/libswmulti.so", + "md5": "d13b12390bc0e82c90e7d0776be77b7e", + "expected_md5": null, + "bytes": 50816 + }, + "cath_score_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit/scripts/cath_score.py", + "md5": "b3a072d44fd59c86e17a250e8891b931", + "expected_md5": null, + "bytes": 13543 + }, + "panel_local_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/scripts/panel_local.py", + "md5": "300be017eabae269186d6f45e8e5625f", + "expected_md5": null, + "bytes": 4400 + }, + "bench_validate_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/scripts/bench_validate.py", + "md5": "60f0977e37a5bd5c2477a26039dcac77", + "expected_md5": null, + "bytes": 10883 + }, + "rlib_py": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/evalue/reconcile/rlib.py", + "md5": "6bf2586b154391a9cdb9e9c5b2a280bc", + "expected_md5": null, + "bytes": 9363 + }, + "bench_queries": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/evalue/encode/out/queries.L2m_B8_s0.npz", + "md5": "e821497f59dda06a3899c18dce37286c", + "expected_md5": "e821497f59dda06a3899c18dce37286c", + "bytes": 26590375 + }, + "bench_q90_local": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/results/bench_q90_shipped_local.npy", + "md5": "8666161687ac6fc6b82f41bf8ee6eb2b", + "expected_md5": null, + "bytes": 11848 + }, + "bench_q90_nersc": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/panel/fit2/results/bench_q90_shipped.npy", + "md5": "4969b813673919047431f7550c1fd82e", + "expected_md5": null, + "bytes": 11848 + }, + "test_fasta": { + "path": "/Users/joan/devel/rcsb-embedding-model/tests/resources/fasta/test_sequences.fasta", + "md5": "0e874a7584d82ba93af9a9cb8df0b227", + "expected_md5": null, + "bytes": 325 + }, + "pt_1acb_A": { + "path": "/Users/joan/devel/rcsb-embedding-model/tests/resources/embeddings/1acb.A.pt", + "md5": "d3c93088f2048a2e83869133d6a0238e", + "expected_md5": null, + "bytes": 1494103 + }, + "pt_2uzi_A": { + "path": "/Users/joan/devel/rcsb-embedding-model/tests/resources/embeddings/2uzi.A.pt", + "md5": "fe5006f58f4f61331d9b3a481aed76d9", + "expected_md5": null, + "bytes": 713815 + }, + "val_emb": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/embcomp/val.npz", + "md5": "9d56740364d2306d64dab46c285cbf05", + "note": "md5 recorded by the encode stage" + }, + "work/stage_esm3.npz": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/p0/fixtures/work/stage_esm3.npz", + "md5": "eeb5bacce47a1586a28477d6c22657a4", + "bytes": 1188265 + }, + "work/stage_esm3.json": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/p0/fixtures/work/stage_esm3.json", + "md5": "9a848f1bcc991c6e5aa0cb1b51ffced9", + "bytes": 21507 + }, + "work/stage_encode_ref.npz": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/p0/fixtures/work/stage_encode_ref.npz", + "md5": "a4b85796f7ab2b1cc003d2e765f0967a", + "bytes": 219952 + }, + "work/stage_encode_ref.json": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/p0/fixtures/work/stage_encode_ref.json", + "md5": "5c698e1705c0ff85655b4fdd049d06d1", + "bytes": 5999 + }, + "work/stage_encode_fm.npz": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/p0/fixtures/work/stage_encode_fm.npz", + "md5": "c30aed4b018eb8f1a6b361bb39220ae7", + "bytes": 219956 + }, + "work/stage_encode_fm.json": { + "path": "/private/tmp/claude-501/-Users-joan-devel-rcsb-embedding-search/d9266e69-77ea-4121-a428-2e6f3bcc4ca2/scratchpad/p0/fixtures/work/stage_encode_fm.json", + "md5": "d4fff87a16ed9e84948dad61d747fc7a", + "bytes": 5986 + } + }, + "outputs": { + "pairs_v1.npz": { + "fixture": "F1", + "content": "50 CATH pairs: q fp16, codes, sequences, S_f, S_r, forward paths, Q90, lnE at N=1e9", + "bytes": 911304, + "sha256": "9ccf7f840e11ff612359896a4e3b73d43f0192578dfbbdea0f4d54fd9042c48e", + "arrays": { + "pair_ids": [ + [ + 50, + 2 + ], + " 1 ulp) while every difference is <= 0.5 ulp of the ROW's max |q| (q_diff_stats).", + "margin_t = per residue min_k |h_t,k| (float64 re-evaluation). Bits with |h_t| < 1e-4: F2 3, F2b 1, F2c 1, Fdeg 0; an encoder with a different fp32 summation order may flip exactly those.", + "F-deg expected values come from the float64 oracle WITH S5/S7. The research fp32 kernel (no S7) is wrong on degenerate rows/columns (affine-coefficient cancellation): e.g. case n1 kernel 0.5 vs oracle 0.776228; kept only as a diagnostic (real_q90_kernel_noS7).", + "Real ESM3 distinct codes per item (length): allX30 28/30, His6 6/6, polyA50 38/50, Met1 1/1, GS 2/2, angiotensinI 10/10.", + "F-tok: S15 upper-cases '' to '', which ESM3 does not treat as special, so every normalised case is length-preserving; the multi-character collapse only happens on raw (un-normalised) input.", + "F4: m = 32.58 and 594.92 lie just outside the table (centres 32.5844 / 594.9176). High-m rows carry both the research clamp and an UNCAPPED ln-m extrapolation; D15 decides.", + "F2b rows are not stored: load tests/resources/embeddings/{1acb.A,2uzi.A}.pt with map_location='cpu'." + ], + "plan_tolerances": { + "F1": "paths 50/50 identical; S_f/S_r rel <= 1e-5; Q90 abs <= 1e-3; lnE abs <= 1e-3 at N=1e9; 3 pairs survive E <= 1e-3", + "F2": "code bits >= 0.99999; q within 1 fp16 ulp", + "F2b/F2c": "codes identical to the reference encoder", + "F3": "Q90 abs <= 1e-3", + "F4": "rel <= 1e-12", + "F-deg": "synthetic: abs <= 1e-5 vs reference with S7; no NaN; NaN q raises" + }, + "summary": { + "F1": { + "path_eq_f64": 50, + "cand_path_equal": 50, + "max_rel_Sf_kernel_vs_f64": 2.433880152048529e-06, + "max_rel_Sr_kernel_vs_f64": 5.932650543868647e-07, + "n_lnE_le_ln_1e_3": 3, + "max_abs_q90_research_vs_f64": 2.7822562529422612e-05 + }, + "F2": { + "bits": 20416, + "bits_equal_ref_vs_nersc": 20416, + "q_ref_vs_nersc": { + "n": 20416, + "bit_equal": 20353, + "max_elem_ulp": 9, + "n_elem_ulp_gt1": 6, + "max_abs": 0.0078125, + "max_abs_over_row_ulp": 0.5, + "n_abs_gt_elem_ulp": 6 + } + }, + "F3": { + "q90_bit_equal_stored_local": 20, + "max_abs_q90_vs_nersc_score2": 9.441375738106217e-06, + "max_abs_q90_research_vs_f64": 2.340890854668487e-05 + } + } +} diff --git a/tests/resources/esm_codes/degenerate_v1.npz b/tests/resources/esm_codes/degenerate_v1.npz new file mode 100644 index 0000000..e788313 Binary files /dev/null and b/tests/resources/esm_codes/degenerate_v1.npz differ diff --git a/tests/resources/esm_codes/encoder_rows_v1.npz b/tests/resources/esm_codes/encoder_rows_v1.npz new file mode 100644 index 0000000..d58ed57 Binary files /dev/null and b/tests/resources/esm_codes/encoder_rows_v1.npz differ diff --git a/tests/resources/esm_codes/evalue_grid_v1.json b/tests/resources/esm_codes/evalue_grid_v1.json new file mode 100644 index 0000000..fb7e083 --- /dev/null +++ b/tests/resources/esm_codes/evalue_grid_v1.json @@ -0,0 +1,3143 @@ +{ + "fixture": "F4", + "system": "esm-codes-v1", + "calibration_version": "foldmatch-evalue-M8_s0-D_fmr-pnlU50s1000-M3t999-v1", + "table_key": "D_fmr/pnl_uniref50/M3t999", + "source": { + "path": "panel/fit2/results/ship_final.json", + "md5": "c51735f962c04bd8c441b9625c4cf30a", + "function": "panel/fit2/scripts/bench_validate.py tabp (np.interp(ln m, ln centre, table))" + }, + "table": { + "center_len": [ + 32.58437573116228, + 50.34161310783617, + 80.8700900525453, + 149.85910754690784, + 276.49344608131264, + 594.9175724688861 + ], + "lam": [ + 0.18775162858061797, + 0.1227859873586807, + 0.10488523441851651, + 0.08222582327223689, + 0.04784353519669357, + 0.041619283794126286 + ], + "lnK": [ + -13.42382627313042, + -13.969792295008805, + -13.73906582316063, + -13.454607332784638, + -14.484498724905178, + -15.878915750325046 + ] + }, + "formula": { + "lnE": "(lnK(m) + ln m + ln N) - lam(m) * S' (float64, left to right)", + "bits": "(lam(m) * S' - lnK(m)) / ln 2", + "render_bits": "floor(bits + 0.5)", + "E": "exp(lnE) when finite and > 0, else null (render E from lnE)" + }, + "rules": { + "default_columns": "lam/lnK/lnE/bits/render_bits: research rule = np.interp in ln m, CLAMPED at both ends (S11 low clamp is normative; the high-m clamp is only valid if D15 keeps it)", + "extrapolated": "rows with m > last centre also carry 'extrapolated': lam and lnK continued linearly in ln m with the last segment's slope (centres 5 -> 6), UNCAPPED; D15 (P0.7) decides between the clamp and a capped extrapolation (cap = validated length, replace m by min(m, cap) in the extrapolation only; ln m and ln N still use the true m)", + "note": "m = 32.58 < centre 1 (32.5844) and m = 594.92 > centre 6 (594.9176): both are just outside the table" + }, + "last_segment_slopes": { + "lam": -0.008123154584299705, + "lnK": -1.819827690088633 + }, + "tolerance": "plan: E arithmetic rel <= 1e-12 vs ship_final.json", + "rows": [ + { + "m": 10.0, + "N": 1.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -10.182483037233283, + "E": 3.782716596147686e-05, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 10.0, + "N": 1.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -11.121241180136373, + "E": 1.4794705463712727e-05, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 10.0, + "N": 1.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -13.430586211677975, + "E": 1.469503143865571e-06, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 10.0, + "N": 1.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -26.14137146658581, + "E": 4.43554125195739e-12, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 10.0, + "N": 360000.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 2.6113762731990096, + "E": 13.617779746131662, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 10.0, + "N": 360000.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 1.6726181302959198, + "E": 5.326093966936582, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 10.0, + "N": 360000.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -0.6367269012456815, + "E": 0.5290211317916058, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 10.0, + "N": 360000.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -13.347512156153519, + "E": 1.5967948507046574e-06, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 10.0, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 10.540782799713128, + "E": 37827.165961476836, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 10.0, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 9.602024656810038, + "E": 14794.705463712717, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 10.0, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 7.292679625268436, + "E": 1469.50314386557, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 10.0, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -5.418105629639401, + "E": 0.004435541251957379, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 10.0, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 15.145952985701221, + "E": 3782716.596147692, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 10.0, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 14.207194842798131, + "E": 1479470.5463712749, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 10.0, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 11.89784981125653, + "E": 146950.31438655732, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 10.0, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -0.8129354436513072, + "E": 0.44355412519573884, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58, + "N": 1.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -9.00136952705343, + "E": 0.00012324090670249143, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58, + "N": 1.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -9.94012766995652, + "E": 4.8201150400775996e-05, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58, + "N": 1.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -12.249472701498123, + "E": 4.787641242714024e-06, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58, + "N": 1.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -24.96025795640596, + "E": 1.4450993398877105e-11, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58, + "N": 360000.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 3.7924897833788616, + "E": 44.366726412896895, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58, + "N": 360000.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 2.8537316404757718, + "E": 17.352414144279358, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58, + "N": 360000.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 0.5443866089341705, + "E": 1.7235508473770491, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58, + "N": 360000.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -12.166398645973667, + "E": 5.202357623595766e-06, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 11.72189630989298, + "E": 123240.90670249135, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 10.78313816698989, + "E": 48201.150400775965, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 8.473793135448288, + "E": 4787.64124271402, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -4.236992119459549, + "E": 0.01445099339887712, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 16.327066495881073, + "E": 12324090.670249162, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 15.388308352977983, + "E": 4820115.040077607, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 13.078963321436381, + "E": 478764.1242714031, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "below_first_centre", + "at_centre": null, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 0.3681780665285448, + "E": 1.4450993398877152, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 50.0, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": -9.430208837545111, + "E": 8.026243637586091e-05, + "bits": 19.248771714248335, + "render_bits": 19 + }, + { + "m": 50.0, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": -10.049223300248483, + "E": 4.3219304450450216e-05, + "bits": 20.141820809829035, + "render_bits": 20 + }, + { + "m": 50.0, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": -11.571998878498778, + "E": 9.426375642671067e-06, + "bits": 22.33872158495756, + "render_bits": 22 + }, + { + "m": 50.0, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": -19.953454703502437, + "E": 2.1593583846883692e-09, + "bits": 34.43060633912025, + "render_bits": 34 + }, + { + "m": 50.0, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 3.363650472887181, + "E": 28.894477095309913, + "bits": 19.248771714248335, + "render_bits": 19 + }, + { + "m": 50.0, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 2.744636010183809, + "E": 15.558949602162075, + "bits": 20.141820809829035, + "render_bits": 20 + }, + { + "m": 50.0, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 1.2218604319335138, + "E": 3.393495231361582, + "bits": 22.33872158495756, + "render_bits": 22 + }, + { + "m": 50.0, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": -7.159595393070145, + "E": 0.0007773690184878128, + "bits": 34.43060633912025, + "render_bits": 34 + }, + { + "m": 50.0, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 11.2930569994013, + "E": 80262.43637586085, + "bits": 19.248771714248335, + "render_bits": 19 + }, + { + "m": 50.0, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 10.674042536697927, + "E": 43219.30445045018, + "bits": 20.141820809829035, + "render_bits": 20 + }, + { + "m": 50.0, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 9.151266958447632, + "E": 9426.37564267106, + "bits": 22.33872158495756, + "render_bits": 22 + }, + { + "m": 50.0, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 0.769811133443973, + "E": 2.1593583846883675, + "bits": 34.43060633912025, + "render_bits": 34 + }, + { + "m": 50.0, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 15.898227185389393, + "E": 8026243.637586104, + "bits": 19.248771714248335, + "render_bits": 19 + }, + { + "m": 50.0, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 15.27921272268602, + "E": 4321930.445045028, + "bits": 20.141820809829035, + "render_bits": 20 + }, + { + "m": 50.0, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 13.756437144435726, + "E": 942637.5642671081, + "bits": 22.33872158495756, + "render_bits": 22 + }, + { + "m": 50.0, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.12380289254067442, + "lnK": -13.96124630567663, + "lnE": 5.374981319432067, + "E": 215.93583846883723, + "bits": 34.43060633912025, + "render_bits": 34 + }, + { + "m": 100.0, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": -8.55055407630605, + "E": 0.00019343789050801683, + "bits": 18.979698152514374, + "render_bits": 19 + }, + { + "m": 100.0, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": -9.035982327522271, + "E": 0.00011904817349047369, + "bits": 19.68002308325142, + "render_bits": 20 + }, + { + "m": 100.0, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": -10.230135825514179, + "E": 3.6066870613633354e-05, + "bits": 21.402822412864552, + "render_bits": 21 + }, + { + "m": 100.0, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": -16.802834346981825, + "E": 5.0422196756740654e-08, + "bits": 30.885221975044153, + "render_bits": 31 + }, + { + "m": 100.0, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 4.243305234126243, + "E": 69.63764058288605, + "bits": 18.979698152514374, + "render_bits": 19 + }, + { + "m": 100.0, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 3.7578769829100214, + "E": 42.85734245657052, + "bits": 19.68002308325142, + "render_bits": 20 + }, + { + "m": 100.0, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 2.563723484918115, + "E": 12.984073420908016, + "bits": 21.402822412864552, + "render_bits": 21 + }, + { + "m": 100.0, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": -4.008975036549534, + "E": 0.018151990832426617, + "bits": 30.885221975044153, + "render_bits": 31 + }, + { + "m": 100.0, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 12.172711760640361, + "E": 193437.8905080167, + "bits": 18.979698152514374, + "render_bits": 19 + }, + { + "m": 100.0, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 11.68728350942414, + "E": 119048.17349047361, + "bits": 19.68002308325142, + "render_bits": 20 + }, + { + "m": 100.0, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 10.493130011432232, + "E": 36066.87061363333, + "bits": 21.402822412864552, + "render_bits": 21 + }, + { + "m": 100.0, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 3.9204314899645842, + "E": 50.42219675674057, + "bits": 30.885221975044153, + "render_bits": 31 + }, + { + "m": 100.0, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 16.777881946628458, + "E": 19343789.05080178, + "bits": 18.979698152514374, + "render_bits": 19 + }, + { + "m": 100.0, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 16.292453695412235, + "E": 11904817.349047408, + "bits": 19.68002308325142, + "render_bits": 20 + }, + { + "m": 100.0, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 15.098300197420327, + "E": 3606687.061363347, + "bits": 21.402822412864552, + "render_bits": 21 + }, + { + "m": 100.0, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": null, + "lam": 0.09708565024324443, + "lnK": -13.641152513510363, + "lnE": 8.525601675952679, + "E": 5042.219675674072, + "bits": 30.885221975044153, + "render_bits": 31 + }, + { + "m": 594.92, + "N": 1.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -9.282392388630017, + "E": 9.304824939665783e-05, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": -9.282399980060136, + "E": 9.3047543030056e-05, + "bits": 22.6082242881305, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 1.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -9.490488807600649, + "E": 7.556715645498318e-05, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": -9.490496233300496, + "E": 7.556659531804448e-05, + "bits": 22.908443720707943, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 1.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -10.002405998268403, + "E": 4.5290828910951144e-05, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": -10.00241301627178, + "E": 4.529051106087626e-05, + "bits": 23.64698352484844, + "render_bits": 24 + } + }, + { + "m": 594.92, + "N": 1.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -12.82003151113075, + "E": 2.7060203413082675e-06, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": -12.82003628514624, + "E": 2.706007422756077e-06, + "bits": 27.71195464194698, + "render_bits": 28 + } + }, + { + "m": 594.92, + "N": 360000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 3.5114669218022754, + "E": 33.4973697827968, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 3.511459330372156, + "E": 33.49711549082014, + "bits": 22.6082242881305, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 360000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 3.303370502831644, + "E": 27.204176323793945, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 3.303363077131797, + "E": 27.20397431449601, + "bits": 22.908443720707943, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 360000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 2.791453312163891, + "E": 16.304698407942425, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 2.791446294160514, + "E": 16.304583981915457, + "bits": 23.64698352484844, + "render_bits": 24 + } + }, + { + "m": 594.92, + "N": 360000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -0.026172200698458603, + "E": 0.9741673228709753, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": -0.026176974713947754, + "E": 0.9741626721921881, + "bits": 27.71195464194698, + "render_bits": 28 + } + }, + { + "m": 594.92, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.440873448316394, + "E": 93048.24939665776, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 11.440865856886274, + "E": 93047.54303005592, + "bits": 22.6082242881305, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.232777029345762, + "E": 75567.15645498312, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 11.232769603645915, + "E": 75566.59531804442, + "bits": 22.908443720707943, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 10.720859838678008, + "E": 45290.82891095112, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 10.720852820674631, + "E": 45290.511060876226, + "bits": 23.64698352484844, + "render_bits": 24 + } + }, + { + "m": 594.92, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 7.903234325815659, + "E": 2706.020341308263, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 7.90322955180017, + "E": 2706.007422756075, + "bits": 27.71195464194698, + "render_bits": 28 + } + }, + { + "m": 594.92, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 16.046043634304485, + "E": 9304824.939665781, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 16.046036042874366, + "E": 9304754.303005597, + "bits": 22.6082242881305, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 15.837947215333855, + "E": 7556715.64549833, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 15.837939789634008, + "E": 7556659.531804459, + "bits": 22.908443720707943, + "render_bits": 23 + } + }, + { + "m": 594.92, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 15.326030024666101, + "E": 4529082.891095121, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 15.326023006662725, + "E": 4529051.1060876325, + "bits": 23.64698352484844, + "render_bits": 24 + } + }, + { + "m": 594.92, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 12.508404511803754, + "E": 270602.0341308271, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.04161925064807181, + "lnK": -15.878923176024893, + "lnE": 12.508399737788263, + "E": 270600.7422756081, + "bits": 27.71195464194698, + "render_bits": 28 + } + }, + { + "m": 1000.0, + "N": 1.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -8.763064052372277, + "E": 0.00015640464162687052, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": -9.729252652149034, + "E": 5.9516757158428265e-05, + "bits": 24.00212883747329, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 1.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -8.971160471342909, + "E": 0.00012702070270789883, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": -9.916255983612249, + "E": 4.936563697495186e-05, + "bits": 24.271917616404988, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 1.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -9.483077662010663, + "E": 7.612927605552198e-05, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": -10.376284179011758, + "E": 3.116284061119119e-05, + "bits": 24.93559801257696, + "render_bits": 25 + } + }, + { + "m": 1000.0, + "N": 1.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -12.30070317487301, + "E": 4.548544915800892e-06, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": -12.908309287023691, + "E": 2.4773793083526225e-06, + "bits": 28.58853807931212, + "render_bits": 29 + } + }, + { + "m": 1000.0, + "N": 360000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 4.0307952580600155, + "E": 56.30567098567338, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 3.0646066582832585, + "E": 21.42603257703417, + "bits": 24.00212883747329, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 360000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 3.8226988390893837, + "E": 45.72745297484357, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 2.8776033268200436, + "E": 17.771629310982668, + "bits": 24.271917616404988, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 360000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 3.3107816484216306, + "E": 27.406539379987933, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 2.4175751314205347, + "E": 11.218622620028832, + "bits": 24.93559801257696, + "render_bits": 25 + } + }, + { + "m": 1000.0, + "N": 360000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 0.49315613555928106, + "E": 1.6374761696883195, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": -0.11444997659139755, + "E": 0.8918565510069448, + "bits": 28.58853807931212, + "render_bits": 29 + } + }, + { + "m": 1000.0, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.960201784574133, + "E": 156404.6416268704, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 10.994013184797376, + "E": 59516.75715842822, + "bits": 24.00212883747329, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.752105365603501, + "E": 127020.70270789873, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 10.807009853334161, + "E": 49365.63697495183, + "bits": 24.271917616404988, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.240188174935748, + "E": 76129.27605552193, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 10.346981657934652, + "E": 31162.840611191168, + "bits": 24.93559801257696, + "render_bits": 25 + } + }, + { + "m": 1000.0, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 8.4225626620734, + "E": 4548.544915800889, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 7.81495654992272, + "E": 2477.379308352623, + "bits": 28.58853807931212, + "render_bits": 29 + } + }, + { + "m": 1000.0, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 16.565371970562225, + "E": 15640464.162687048, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 15.59918337078547, + "E": 5951675.715842836, + "bits": 24.00212883747329, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 16.357275551591595, + "E": 12702070.2707899, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 15.412180039322255, + "E": 4936563.697495194, + "bits": 24.271917616404988, + "render_bits": 24 + } + }, + { + "m": 1000.0, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 15.845358360923841, + "E": 7612927.60555221, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 14.952151843922746, + "E": 3116284.0611191234, + "bits": 24.93559801257696, + "render_bits": 25 + } + }, + { + "m": 1000.0, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 13.027732848061493, + "E": 454854.4915800899, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.03740066629264301, + "lnK": -16.824011262594386, + "lnE": 12.420126735910813, + "E": 247737.93083526264, + "bits": 28.58853807931212, + "render_bits": 29 + } + }, + { + "m": 5000.0, + "N": 1.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -7.153626139938176, + "E": 0.0007820232081343535, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": -11.114082983023629, + "E": 1.490098882644944e-05, + "bits": 28.32194478318606, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 1.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -7.361722558908808, + "E": 0.0006351035135394948, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": -11.23571774970417, + "E": 1.3194404104887974e-05, + "bits": 28.49742665787576, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 1.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -7.873639749576561, + "E": 0.0003806463802776106, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": -11.534939275738301, + "E": 9.782267248954973e-06, + "bits": 28.92911206961243, + "render_bits": 29 + } + }, + { + "m": 5000.0, + "N": 1.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -10.691265262438911, + "E": 2.2742724579004442e-05, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": -13.181874016592825, + "E": 1.8844510180462134e-06, + "bits": 31.305136652911006, + "render_bits": 31 + } + }, + { + "m": 5000.0, + "N": 360000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 5.640233170494117, + "E": 281.5283549283672, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 1.6797763274086637, + "E": 5.364355977521796, + "bits": 28.32194478318606, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 360000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 5.432136751523485, + "E": 228.6372648742181, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 1.5581415607281226, + "E": 4.74998547775967, + "bits": 28.49742665787576, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 360000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 4.920219560855732, + "E": 137.03269689993982, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 1.2589200346939917, + "E": 3.521616209623791, + "bits": 28.92911206961243, + "render_bits": 29 + } + }, + { + "m": 5000.0, + "N": 360000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 2.1025940479933825, + "E": 8.187380848441606, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": -0.38801470616053324, + "E": 0.6784023664966364, + "bits": 31.305136652911006, + "render_bits": 31 + } + }, + { + "m": 5000.0, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 13.569639697008235, + "E": 782023.2081343528, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 9.609182853922782, + "E": 14900.988826449431, + "bits": 28.32194478318606, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 13.361543278037603, + "E": 635103.5135394944, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 9.48754808724224, + "E": 13194.404104887964, + "bits": 28.49742665787576, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 12.849626087369849, + "E": 380646.38027761003, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 9.18832656120811, + "E": 9782.267248954968, + "bits": 28.92911206961243, + "render_bits": 29 + } + }, + { + "m": 5000.0, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 10.0320005745075, + "E": 22742.724579004425, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 7.541391820353585, + "E": 1884.451018046212, + "bits": 31.305136652911006, + "render_bits": 31 + } + }, + { + "m": 5000.0, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 18.174809882996325, + "E": 78202320.81343518, + "bits": 22.608213336011914, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 14.214353039910876, + "E": 1490098.8826449462, + "bits": 28.32194478318606, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 17.966713464025695, + "E": 63510351.35394946, + "bits": 22.9084330076876, + "render_bits": 23, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 14.092718273230334, + "E": 1319440.4104887994, + "bits": 28.49742665787576, + "render_bits": 28 + } + }, + { + "m": 5000.0, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 17.454796273357942, + "E": 38064638.02776109, + "bits": 23.646973400009774, + "render_bits": 24, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 13.793496747196203, + "E": 978226.7248954988, + "bits": 28.92911206961243, + "render_bits": 29 + } + }, + { + "m": 5000.0, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "above_last_centre", + "at_centre": null, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 14.637170760495593, + "E": 2274272.457900448, + "bits": 27.711947754498507, + "render_bits": 28, + "extrapolated": { + "lam": 0.0243269533361082, + "lnK": -19.75291094112041, + "lnE": 12.146562006341679, + "E": 188445.10180462163, + "bits": 31.305136652911006, + "render_bits": 31 + } + }, + { + "m": 32.58437573116228, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -9.001235228792464, + "E": 0.00012325745885337925, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58437573116228, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -9.939993371695554, + "E": 4.8207624166149424e-05, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58437573116228, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -12.249338403237156, + "E": 4.788284257783991e-06, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58437573116228, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -24.960123658144994, + "E": 1.4452934272484879e-11, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58437573116228, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 3.7926240816398287, + "E": 44.372685187216504, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58437573116228, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 2.853865938736739, + "E": 17.35474469981379, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58437573116228, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 0.5445209071951376, + "E": 1.7237823328022375, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58437573116228, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -12.1662643477127, + "E": 5.203056338094565e-06, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58437573116228, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 11.722030608153947, + "E": 123257.45885337917, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58437573116228, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 10.783272465250857, + "E": 48207.62416614939, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58437573116228, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 8.473927433709255, + "E": 4788.284257783987, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58437573116228, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": -4.236857821198582, + "E": 0.014452934272484895, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 32.58437573116228, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 16.32720079414204, + "E": 12325745.885337943, + "bits": 18.01214587663981, + "render_bits": 18 + }, + { + "m": 32.58437573116228, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 15.38844265123895, + "E": 4820762.41661495, + "bits": 19.366487594000233, + "render_bits": 19 + }, + { + "m": 32.58437573116228, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 13.079097619697349, + "E": 478828.42577839986, + "bits": 22.69816821870687, + "render_bits": 23 + }, + { + "m": 32.58437573116228, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 0, + "lam": 0.18775162858061797, + "lnK": -13.42382627313042, + "lnE": 0.3683123647895119, + "E": 1.4452934272484925, + "bits": 41.03595507176696, + "render_bits": 41 + }, + { + "m": 50.34161310783617, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": -9.437030324765853, + "E": 7.971679036632087e-05, + "bits": 19.268436390992935, + "render_bits": 19 + }, + { + "m": 50.34161310783617, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": -10.050960261559258, + "E": 4.3144299350077684e-05, + "bits": 20.154150066258055, + "render_bits": 20 + }, + { + "m": 50.34161310783617, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": -11.56122790607103, + "E": 9.528455638332041e-06, + "bits": 22.333005707410244, + "render_bits": 22 + }, + { + "m": 50.34161310783617, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": -19.873839250253713, + "E": 2.3383056518544432e-09, + "bits": 34.32556887049994, + "render_bits": 34 + }, + { + "m": 50.34161310783617, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 3.3568289856664384, + "E": 28.698044531875485, + "bits": 19.268436390992935, + "render_bits": 19 + }, + { + "m": 50.34161310783617, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 2.742899048873035, + "E": 15.531947766027965, + "bits": 20.154150066258055, + "render_bits": 20 + }, + { + "m": 50.34161310783617, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 1.2326314043612623, + "E": 3.4302440297995345, + "bits": 22.333005707410244, + "render_bits": 22 + }, + { + "m": 50.34161310783617, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": -7.0799799398214205, + "E": 0.0008417900346675994, + "bits": 34.32556887049994, + "render_bits": 34 + }, + { + "m": 50.34161310783617, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 11.286235512180557, + "E": 79716.79036632081, + "bits": 19.268436390992935, + "render_bits": 19 + }, + { + "m": 50.34161310783617, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 10.672305575387153, + "E": 43144.29935007766, + "bits": 20.154150066258055, + "render_bits": 20 + }, + { + "m": 50.34161310783617, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 9.16203793087538, + "E": 9528.455638332034, + "bits": 22.333005707410244, + "render_bits": 22 + }, + { + "m": 50.34161310783617, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 0.8494265866926973, + "E": 2.338305651854441, + "bits": 34.32556887049994, + "render_bits": 34 + }, + { + "m": 50.34161310783617, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 15.89140569816865, + "E": 7971679.036632099, + "bits": 19.268436390992935, + "render_bits": 19 + }, + { + "m": 50.34161310783617, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 15.277475761375246, + "E": 4314429.935007775, + "bits": 20.154150066258055, + "render_bits": 20 + }, + { + "m": 50.34161310783617, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 13.767208116863474, + "E": 952845.5638332056, + "bits": 22.333005707410244, + "render_bits": 22 + }, + { + "m": 50.34161310783617, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 1, + "lam": 0.1227859873586807, + "lnK": -13.969792295008805, + "lnE": 5.454596772680791, + "E": 233.83056518544464, + "bits": 34.32556887049994, + "render_bits": 34 + }, + { + "m": 80.8700900525453, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": -8.821795610413943, + "E": 0.00014748329978162824, + "bits": 19.064695091730535, + "render_bits": 19 + }, + { + "m": 80.8700900525453, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": -9.346221782506525, + "E": 8.729461485714379e-05, + "bits": 19.821282129520885, + "render_bits": 20 + }, + { + "m": 80.8700900525453, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": -10.636310165854278, + "E": 2.4027533263566533e-05, + "bits": 21.682486242485147, + "render_bits": 22 + }, + { + "m": 80.8700900525453, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": -17.737040535987845, + "E": 1.9810761515951334e-08, + "bits": 31.926674734166497, + "render_bits": 32 + }, + { + "m": 80.8700900525453, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 3.97206370001835, + "E": 53.09398792138618, + "bits": 19.064695091730535, + "render_bits": 19 + }, + { + "m": 80.8700900525453, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 3.4476375279257674, + "E": 31.426061348571764, + "bits": 19.821282129520885, + "render_bits": 20 + }, + { + "m": 80.8700900525453, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 2.157549144578014, + "E": 8.649911974883944, + "bits": 21.682486242485147, + "render_bits": 22 + }, + { + "m": 80.8700900525453, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": -4.943181225555554, + "E": 0.007131874145742467, + "bits": 31.926674734166497, + "render_bits": 32 + }, + { + "m": 80.8700900525453, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 11.901470226532467, + "E": 147483.29978162813, + "bits": 19.064695091730535, + "render_bits": 19 + }, + { + "m": 80.8700900525453, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 11.377044054439885, + "E": 87294.61485714374, + "bits": 19.821282129520885, + "render_bits": 20 + }, + { + "m": 80.8700900525453, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 10.086955671092133, + "E": 24027.533263566518, + "bits": 21.682486242485147, + "render_bits": 22 + }, + { + "m": 80.8700900525453, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 2.986225300958564, + "E": 19.810761515951285, + "bits": 31.926674734166497, + "render_bits": 32 + }, + { + "m": 80.8700900525453, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 16.50664041252056, + "E": 14748329.978162846, + "bits": 19.064695091730535, + "render_bits": 19 + }, + { + "m": 80.8700900525453, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 15.982214240427979, + "E": 8729461.485714393, + "bits": 19.821282129520885, + "render_bits": 20 + }, + { + "m": 80.8700900525453, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 14.692125857080226, + "E": 2402753.326356657, + "bits": 21.682486242485147, + "render_bits": 22 + }, + { + "m": 80.8700900525453, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 2, + "lam": 0.10488523441851651, + "lnK": -13.73906582316063, + "lnE": 7.5913954869466576, + "E": 1981.076151595133, + "bits": 31.926674734166497, + "render_bits": 32 + }, + { + "m": 149.85910754690784, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": -8.03378264675053, + "E": 0.0003243191011516261, + "bits": 18.817761338777338, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": -8.444911763111715, + "E": 0.0002149915650113198, + "bits": 19.41089527611668, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": -9.45628938936023, + "E": 7.819620903821901e-05, + "bits": 20.870004761971465, + "render_bits": 21 + }, + { + "m": 149.85910754690784, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": -15.022977624890666, + "E": 2.989535505691687e-07, + "bits": 28.90103827354616, + "render_bits": 29 + }, + { + "m": 149.85910754690784, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 4.760076663681762, + "E": 116.75487641458537, + "bits": 18.817761338777338, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 4.348947547320577, + "E": 77.39696340407512, + "bits": 19.41089527611668, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 3.3375699210720633, + "E": 28.15063525375884, + "bits": 20.870004761971465, + "render_bits": 21 + }, + { + "m": 149.85910754690784, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": -2.2291183144583737, + "E": 0.10762327820490071, + "bits": 28.90103827354616, + "render_bits": 29 + }, + { + "m": 149.85910754690784, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 12.68948319019588, + "E": 324319.10115162586, + "bits": 18.817761338777338, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 12.278354073834695, + "E": 214991.56501131965, + "bits": 19.41089527611668, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 11.266976447586181, + "E": 78196.20903821895, + "bits": 20.870004761971465, + "render_bits": 21 + }, + { + "m": 149.85910754690784, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 5.700288212055744, + "E": 298.95355056916844, + "bits": 28.90103827354616, + "render_bits": 29 + }, + { + "m": 149.85910754690784, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 17.294653376183973, + "E": 32431910.115162656, + "bits": 18.817761338777338, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 16.88352425982279, + "E": 21499156.501132015, + "bits": 19.41089527611668, + "render_bits": 19 + }, + { + "m": 149.85910754690784, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 15.872146633574275, + "E": 7819620.903821913, + "bits": 20.870004761971465, + "render_bits": 21 + }, + { + "m": 149.85910754690784, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 3, + "lam": 0.08222582327223689, + "lnK": -13.454607332784638, + "lnE": 10.305458398043838, + "E": 29895.355056916913, + "bits": 28.90103827354616, + "render_bits": 29 + }, + { + "m": 276.49344608131264, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": -8.62309393138197, + "E": 0.00017990278588565501, + "bits": 20.551596325348882, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": -8.862311607365438, + "E": 0.00014162729876953143, + "bits": 20.896714480183213, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": -9.450787090284768, + "E": 7.862765384759307e-05, + "bits": 21.74570514107567, + "render_bits": 22 + }, + { + "m": 276.49344608131264, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": -12.689794423100924, + "E": 3.082423489622083e-06, + "bits": 26.418604957532523, + "render_bits": 26 + }, + { + "m": 276.49344608131264, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 4.170765379050322, + "E": 64.76500291883575, + "bits": 20.551596325348882, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 3.9315477030668546, + "E": 50.985827557031314, + "bits": 20.896714480183213, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 3.3430722201475236, + "E": 28.30595538513348, + "bits": 21.74570514107567, + "render_bits": 22 + }, + { + "m": 276.49344608131264, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 0.10406488733136898, + "E": 1.1096724562639502, + "bits": 26.418604957532523, + "render_bits": 26 + }, + { + "m": 276.49344608131264, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 12.10017190556444, + "E": 179902.78588565488, + "bits": 20.551596325348882, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 11.860954229580972, + "E": 141627.29876953133, + "bits": 20.896714480183213, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 11.272478746661642, + "E": 78627.65384759301, + "bits": 21.74570514107567, + "render_bits": 22 + }, + { + "m": 276.49344608131264, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 8.033471413845486, + "E": 3082.423489622081, + "bits": 26.418604957532523, + "render_bits": 26 + }, + { + "m": 276.49344608131264, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 16.705342091552534, + "E": 17990278.58856553, + "bits": 20.551596325348882, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 16.466124415569066, + "E": 14162729.876953164, + "bits": 20.896714480183213, + "render_bits": 21 + }, + { + "m": 276.49344608131264, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 15.877648932649736, + "E": 7862765.384759319, + "bits": 21.74570514107567, + "render_bits": 22 + }, + { + "m": 276.49344608131264, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 4, + "lam": 0.04784353519669357, + "lnK": -14.484498724905178, + "lnE": 12.63864159983358, + "E": 308242.3489622088, + "bits": 26.418604957532523, + "render_bits": 26 + }, + { + "m": 594.9175724688861, + "N": 1.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -9.282396469071196, + "E": 9.304786971952393e-05, + "bits": 22.608213336011914, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 1.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -9.490492888041828, + "E": 7.556684810827526e-05, + "bits": 22.9084330076876, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 1.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -10.002410078709582, + "E": 4.5290644104764847e-05, + "bits": 23.646973400009774, + "render_bits": 24 + }, + { + "m": 594.9175724688861, + "N": 1.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -12.82003559157193, + "E": 2.7060092995739614e-06, + "bits": 27.711947754498507, + "render_bits": 28 + }, + { + "m": 594.9175724688861, + "N": 360000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 3.5114628413610958, + "E": 33.4972330990286, + "bits": 22.608213336011914, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 360000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 3.3033664223904644, + "E": 27.20406531897909, + "bits": 22.9084330076876, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 360000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 2.7914492317227113, + "E": 16.30463187771536, + "bits": 23.646973400009774, + "render_bits": 24 + }, + { + "m": 594.9175724688861, + "N": 360000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": -0.026176281139638213, + "E": 0.9741633478466251, + "bits": 27.711947754498507, + "render_bits": 28 + }, + { + "m": 594.9175724688861, + "N": 1000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.440869367875214, + "E": 93047.86971952386, + "bits": 22.608213336011914, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 1000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 11.232772948904582, + "E": 75566.8481082752, + "bits": 22.9084330076876, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 1000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 10.720855758236828, + "E": 45290.64410476481, + "bits": 23.646973400009774, + "render_bits": 24 + }, + { + "m": 594.9175724688861, + "N": 1000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 7.90323024537448, + "E": 2706.009299573957, + "bits": 27.711947754498507, + "render_bits": 28 + }, + { + "m": 594.9175724688861, + "N": 100000000000.0, + "S_prime": -5.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 16.046039553863306, + "E": 9304786.97195239, + "bits": 22.608213336011914, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 100000000000.0, + "S_prime": 0.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 15.837943134892676, + "E": 7556684.8108275365, + "bits": 22.9084330076876, + "render_bits": 23 + }, + { + "m": 594.9175724688861, + "N": 100000000000.0, + "S_prime": 12.3, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 15.326025944224922, + "E": 4529064.410476491, + "bits": 23.646973400009774, + "render_bits": 24 + }, + { + "m": 594.9175724688861, + "N": 100000000000.0, + "S_prime": 80.0, + "region": "inside", + "at_centre": 5, + "lam": 0.041619283794126286, + "lnK": -15.878915750325046, + "lnE": 12.508400431362574, + "E": 270600.9299573965, + "bits": 27.711947754498507, + "render_bits": 28 + } + ] +} diff --git a/tests/resources/esm_codes/fasta_rows_codes_v1.npz b/tests/resources/esm_codes/fasta_rows_codes_v1.npz new file mode 100644 index 0000000..d515f25 Binary files /dev/null and b/tests/resources/esm_codes/fasta_rows_codes_v1.npz differ diff --git a/tests/resources/esm_codes/ood_rows_codes_v1.npz b/tests/resources/esm_codes/ood_rows_codes_v1.npz new file mode 100644 index 0000000..8759182 Binary files /dev/null and b/tests/resources/esm_codes/ood_rows_codes_v1.npz differ diff --git a/tests/resources/esm_codes/pairs_v1.npz b/tests/resources/esm_codes/pairs_v1.npz new file mode 100644 index 0000000..5330876 Binary files /dev/null and b/tests/resources/esm_codes/pairs_v1.npz differ diff --git a/tests/resources/esm_codes/q90_queries_v1.npz b/tests/resources/esm_codes/q90_queries_v1.npz new file mode 100644 index 0000000..abc5137 Binary files /dev/null and b/tests/resources/esm_codes/q90_queries_v1.npz differ diff --git a/tests/resources/esm_codes/tokenizer_cases_v1.json b/tests/resources/esm_codes/tokenizer_cases_v1.json new file mode 100644 index 0000000..a7faccd --- /dev/null +++ b/tests/resources/esm_codes/tokenizer_cases_v1.json @@ -0,0 +1,1165 @@ +{ + "fixture": "F-tok", + "spec": "S15 normalize_sequence + G9 strict refusals; ids from the ESM3 tokenizer FoldMatch ships (esm.utils.encoding.tokenize_sequence == ESM3.encode(ESMProtein(sequence=s)).sequence)", + "esm": "3.2.3", + "tokenizer_vocab": { + "0": "", + "1": "", + "2": "", + "3": "", + "4": "L", + "5": "A", + "6": "G", + "7": "V", + "8": "S", + "9": "E", + "10": "R", + "11": "T", + "12": "I", + "13": "D", + "14": "P", + "15": "K", + "16": "Q", + "17": "N", + "18": "F", + "19": "Y", + "20": "M", + "21": "H", + "22": "W", + "23": "C", + "24": "X", + "25": "B", + "26": "U", + "27": "Z", + "28": "O", + "29": ".", + "30": "-", + "31": "|", + "32": "" + }, + "fields": { + "normalized": "S15 transcription: null = refused (non-ASCII); ASCII whitespace removed; ASCII upper", + "strict_refusal": "non_ascii | angle_bracket ('<' or '>' after normalisation) | null", + "tokens_raw / tokens_normalized": "ids incl. BOS (0) and EOS (2); *_str = token strings", + "length_preserved": "len(tokens_normalized) - 2 == len(normalized)", + "n_nonstandard": "characters of normalized outside the 20 standard amino acids" + }, + "cases": [ + { + "id": "mask_literal", + "raw": "ACDE", + "raw_is_ascii": true, + "normalized": "ACDE", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 5, + 23, + 32, + 13, + 9, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "C", + "", + "D", + "E", + "" + ], + "n_residue_tokens_raw": 5, + "tokens_normalized": [ + 0, + 5, + 23, + 3, + 20, + 5, + 8, + 15, + 3, + 13, + 9, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "", + "M", + "A", + "S", + "K", + "", + "D", + "E", + "" + ], + "len_normalized": 10, + "n_residue_tokens_normalized": 10, + "length_preserved": true, + "n_nonstandard": 2, + "n_unk": 2, + "n_mask": 0 + }, + { + "id": "mask_literal_lowercase", + "raw": "acde", + "raw_is_ascii": true, + "normalized": "ACDE", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 3, + 3, + 32, + 3, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "", + "", + "", + "", + "", + "" + ], + "n_residue_tokens_raw": 5, + "tokens_normalized": [ + 0, + 5, + 23, + 3, + 20, + 5, + 8, + 15, + 3, + 13, + 9, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "", + "M", + "A", + "S", + "K", + "", + "D", + "E", + "" + ], + "len_normalized": 10, + "n_residue_tokens_normalized": 10, + "length_preserved": true, + "n_nonstandard": 2, + "n_unk": 2, + "n_mask": 0 + }, + { + "id": "mask_short", + "raw": "_", + "raw_is_ascii": true, + "normalized": "_", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 32, + 2 + ], + "tokens_raw_str": [ + "", + "", + "" + ], + "n_residue_tokens_raw": 1, + "tokens_normalized": [ + 0, + 32, + 2 + ], + "tokens_normalized_str": [ + "", + "", + "" + ], + "len_normalized": 1, + "n_residue_tokens_normalized": 1, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 0, + "n_mask": 1 + }, + { + "id": "mask_short_inside", + "raw": "AC_DE", + "raw_is_ascii": true, + "normalized": "AC_DE", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 5, + 23, + 32, + 13, + 9, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "C", + "", + "D", + "E", + "" + ], + "n_residue_tokens_raw": 5, + "tokens_normalized": [ + 0, + 5, + 23, + 32, + 13, + 9, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "", + "D", + "E", + "" + ], + "len_normalized": 5, + "n_residue_tokens_normalized": 5, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 0, + "n_mask": 1 + }, + { + "id": "stop", + "raw": "*", + "raw_is_ascii": true, + "normalized": "*", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "", + "" + ], + "n_residue_tokens_raw": 1, + "tokens_normalized": [ + 0, + 3, + 2 + ], + "tokens_normalized_str": [ + "", + "", + "" + ], + "len_normalized": 1, + "n_residue_tokens_normalized": 1, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 1, + "n_mask": 0 + }, + { + "id": "trailing_stop", + "raw": "MKT*", + "raw_is_ascii": true, + "normalized": "MKT*", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 20, + 15, + 11, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "M", + "K", + "T", + "", + "" + ], + "n_residue_tokens_raw": 4, + "tokens_normalized": [ + 0, + 20, + 15, + 11, + 3, + 2 + ], + "tokens_normalized_str": [ + "", + "M", + "K", + "T", + "", + "" + ], + "len_normalized": 4, + "n_residue_tokens_normalized": 4, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 1, + "n_mask": 0 + }, + { + "id": "J", + "raw": "J", + "raw_is_ascii": true, + "normalized": "J", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "", + "" + ], + "n_residue_tokens_raw": 1, + "tokens_normalized": [ + 0, + 3, + 2 + ], + "tokens_normalized_str": [ + "", + "", + "" + ], + "len_normalized": 1, + "n_residue_tokens_normalized": 1, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 1, + "n_mask": 0 + }, + { + "id": "digits", + "raw": "0123456789", + "raw_is_ascii": true, + "normalized": "0123456789", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "n_residue_tokens_raw": 10, + "tokens_normalized": [ + 0, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 3, + 2 + ], + "tokens_normalized_str": [ + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "len_normalized": 10, + "n_residue_tokens_normalized": 10, + "length_preserved": true, + "n_nonstandard": 10, + "n_unk": 10, + "n_mask": 0 + }, + { + "id": "space", + "raw": "AC DE", + "raw_is_ascii": true, + "normalized": "ACDE", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 5, + 23, + 3, + 13, + 9, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "C", + "", + "D", + "E", + "" + ], + "n_residue_tokens_raw": 5, + "tokens_normalized": [ + 0, + 5, + 23, + 13, + 9, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "D", + "E", + "" + ], + "len_normalized": 4, + "n_residue_tokens_normalized": 4, + "length_preserved": true, + "n_nonstandard": 0, + "n_unk": 0, + "n_mask": 0 + }, + { + "id": "whitespace_mixed", + "raw": " A\tC\nD\rE\u000bF\f ", + "raw_is_ascii": true, + "normalized": "ACDEF", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 3, + 5, + 3, + 23, + 3, + 13, + 3, + 9, + 3, + 18, + 3, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "", + "A", + "", + "C", + "", + "D", + "", + "E", + "", + "F", + "", + "", + "" + ], + "n_residue_tokens_raw": 12, + "tokens_normalized": [ + 0, + 5, + 23, + 13, + 9, + 18, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "D", + "E", + "F", + "" + ], + "len_normalized": 5, + "n_residue_tokens_normalized": 5, + "length_preserved": true, + "n_nonstandard": 0, + "n_unk": 0, + "n_mask": 0 + }, + { + "id": "gap_dot_bar", + "raw": "-.|", + "raw_is_ascii": true, + "normalized": "-.|", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 30, + 29, + 31, + 2 + ], + "tokens_raw_str": [ + "", + "-", + ".", + "|", + "" + ], + "n_residue_tokens_raw": 3, + "tokens_normalized": [ + 0, + 30, + 29, + 31, + 2 + ], + "tokens_normalized_str": [ + "", + "-", + ".", + "|", + "" + ], + "len_normalized": 3, + "n_residue_tokens_normalized": 3, + "length_preserved": true, + "n_nonstandard": 3, + "n_unk": 0, + "n_mask": 0 + }, + { + "id": "lowercase", + "raw": "acde", + "raw_is_ascii": true, + "normalized": "ACDE", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 3, + 3, + 3, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "", + "", + "", + "", + "" + ], + "n_residue_tokens_raw": 4, + "tokens_normalized": [ + 0, + 5, + 23, + 13, + 9, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "D", + "E", + "" + ], + "len_normalized": 4, + "n_residue_tokens_normalized": 4, + "length_preserved": true, + "n_nonstandard": 0, + "n_unk": 0, + "n_mask": 0 + }, + { + "id": "mixed_case", + "raw": "AcDe", + "raw_is_ascii": true, + "normalized": "ACDE", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 5, + 3, + 13, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "", + "D", + "", + "" + ], + "n_residue_tokens_raw": 4, + "tokens_normalized": [ + 0, + 5, + 23, + 13, + 9, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "D", + "E", + "" + ], + "len_normalized": 4, + "n_residue_tokens_normalized": 4, + "length_preserved": true, + "n_nonstandard": 0, + "n_unk": 0, + "n_mask": 0 + }, + { + "id": "nonstandard_real_tokens", + "raw": "XBZUO", + "raw_is_ascii": true, + "normalized": "XBZUO", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 24, + 25, + 27, + 26, + 28, + 2 + ], + "tokens_raw_str": [ + "", + "X", + "B", + "Z", + "U", + "O", + "" + ], + "n_residue_tokens_raw": 5, + "tokens_normalized": [ + 0, + 24, + 25, + 27, + 26, + 28, + 2 + ], + "tokens_normalized_str": [ + "", + "X", + "B", + "Z", + "U", + "O", + "" + ], + "len_normalized": 5, + "n_residue_tokens_normalized": 5, + "length_preserved": true, + "n_nonstandard": 5, + "n_unk": 0, + "n_mask": 0 + }, + { + "id": "non_ascii", + "raw": "ACD\u00c9", + "raw_is_ascii": false, + "normalized": null, + "normalize_refuses": true, + "strict_refusal": "non_ascii", + "tokens_raw": [ + 0, + 5, + 23, + 13, + 3, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "C", + "D", + "", + "" + ], + "n_residue_tokens_raw": 4 + }, + { + "id": "lt_only", + "raw": "A", + "A", + "", + "C", + "" + ], + "n_residue_tokens_raw": 3, + "tokens_normalized": [ + 0, + 5, + 3, + 23, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "", + "C", + "" + ], + "len_normalized": 3, + "n_residue_tokens_normalized": 3, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 1, + "n_mask": 0 + }, + { + "id": "gt_only", + "raw": "A>C", + "raw_is_ascii": true, + "normalized": "A>C", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 5, + 3, + 23, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "", + "C", + "" + ], + "n_residue_tokens_raw": 3, + "tokens_normalized": [ + 0, + 5, + 3, + 23, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "", + "C", + "" + ], + "len_normalized": 3, + "n_residue_tokens_normalized": 3, + "length_preserved": true, + "n_nonstandard": 1, + "n_unk": 1, + "n_mask": 0 + }, + { + "id": "special_cls", + "raw": "A", + "raw_is_ascii": true, + "normalized": "A", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 0, + 5, + 2 + ], + "tokens_raw_str": [ + "", + "", + "A", + "" + ], + "n_residue_tokens_raw": 2, + "tokens_normalized": [ + 0, + 3, + 23, + 4, + 8, + 3, + 5, + 2 + ], + "tokens_normalized_str": [ + "", + "", + "C", + "L", + "S", + "", + "A", + "" + ], + "len_normalized": 6, + "n_residue_tokens_normalized": 6, + "length_preserved": true, + "n_nonstandard": 2, + "n_unk": 2, + "n_mask": 0 + }, + { + "id": "special_eos", + "raw": "A", + "raw_is_ascii": true, + "normalized": "A", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 5, + 2, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "", + "" + ], + "n_residue_tokens_raw": 2, + "tokens_normalized": [ + 0, + 5, + 3, + 9, + 28, + 8, + 3, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "", + "E", + "O", + "S", + "", + "" + ], + "len_normalized": 6, + "n_residue_tokens_normalized": 6, + "length_preserved": true, + "n_nonstandard": 3, + "n_unk": 2, + "n_mask": 0 + }, + { + "id": "special_pad", + "raw": "AC", + "raw_is_ascii": true, + "normalized": "AC", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 5, + 1, + 23, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "", + "C", + "" + ], + "n_residue_tokens_raw": 3, + "tokens_normalized": [ + 0, + 5, + 3, + 14, + 5, + 13, + 3, + 23, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "", + "P", + "A", + "D", + "", + "C", + "" + ], + "len_normalized": 7, + "n_residue_tokens_normalized": 7, + "length_preserved": true, + "n_nonstandard": 2, + "n_unk": 2, + "n_mask": 0 + }, + { + "id": "special_unk", + "raw": "AC", + "raw_is_ascii": true, + "normalized": "AC", + "normalize_refuses": false, + "strict_refusal": "angle_bracket", + "tokens_raw": [ + 0, + 5, + 3, + 23, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "", + "C", + "" + ], + "n_residue_tokens_raw": 3, + "tokens_normalized": [ + 0, + 5, + 3, + 26, + 17, + 15, + 3, + 23, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "", + "U", + "N", + "K", + "", + "C", + "" + ], + "len_normalized": 7, + "n_residue_tokens_normalized": 7, + "length_preserved": true, + "n_nonstandard": 3, + "n_unk": 2, + "n_mask": 0 + }, + { + "id": "standard20", + "raw": "ACDEFGHIKLMNPQRSTVWY", + "raw_is_ascii": true, + "normalized": "ACDEFGHIKLMNPQRSTVWY", + "normalize_refuses": false, + "strict_refusal": null, + "tokens_raw": [ + 0, + 5, + 23, + 13, + 9, + 18, + 6, + 21, + 12, + 15, + 4, + 20, + 17, + 14, + 16, + 10, + 8, + 11, + 7, + 22, + 19, + 2 + ], + "tokens_raw_str": [ + "", + "A", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "K", + "L", + "M", + "N", + "P", + "Q", + "R", + "S", + "T", + "V", + "W", + "Y", + "" + ], + "n_residue_tokens_raw": 20, + "tokens_normalized": [ + 0, + 5, + 23, + 13, + 9, + 18, + 6, + 21, + 12, + 15, + 4, + 20, + 17, + 14, + 16, + 10, + 8, + 11, + 7, + 22, + 19, + 2 + ], + "tokens_normalized_str": [ + "", + "A", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "K", + "L", + "M", + "N", + "P", + "Q", + "R", + "S", + "T", + "V", + "W", + "Y", + "" + ], + "len_normalized": 20, + "n_residue_tokens_normalized": 20, + "length_preserved": true, + "n_nonstandard": 0, + "n_unk": 0, + "n_mask": 0 + } + ] +} diff --git a/tests/test_alignment.py b/tests/test_alignment.py index 0ba149b..5de2891 100644 --- a/tests/test_alignment.py +++ b/tests/test_alignment.py @@ -1,15 +1,26 @@ +import contextlib import csv import math +import multiprocessing +import multiprocessing.pool +import random import re +import signal +import subprocess +import sys import tempfile +import types import unittest from pathlib import Path +from unittest import mock import numpy as np +import pytest -from foldmatch.search import alignment, output +import blosum62_reference as ref +from foldmatch.search import alignment, output, swint, swkern from foldmatch.search.alignment import _chunk_candidate_tasks -from foldmatch.types.api_types import FormatMode +from foldmatch.types.api_types import FormatMode, SignificanceMode from foldmatch.search.output import ( DEFAULT_OUTPUT_FIELDS, SUPPORTED_OUTPUT_FIELDS, @@ -54,15 +65,19 @@ def _signature(results): class TestChunkCandidateTasks(unittest.TestCase): + """The cell dispatch: contiguous candidate slices of about equal DP cells.""" + def test_preserves_all_pairs_in_order(self): q1 = [(f"a{i}", "SEQ", float(i)) for i in range(10)] q2 = [(f"b{i}", "SEQ", float(i)) for i in range(3)] tasks = [("q1", "AAAA", q1), ("q2", "BBBB", q2)] - chunks = _chunk_candidate_tasks(tasks, worker_budget=2) # -> chunk_size 2 + # 13 pairs of 12 cells: budget ceil(156 / (4 * 1)) = 39 cells, 3 pairs. + chunks, cells = _chunk_candidate_tasks(tasks, worker_budget=1, floor_cells=1) - # No chunk mixes queries and none exceeds the computed chunk size. - self.assertTrue(all(len(c) <= 2 for _, _, c in chunks)) + # No chunk mixes queries and none exceeds the budget. + self.assertTrue(all(len(c) <= 3 for _, _, c in chunks)) + self.assertEqual(cells, [12 * len(c) for _, _, c in chunks]) # Concatenating each query's chunks reproduces its candidate list in order. regrouped = {} for qid, _, cand in chunks: @@ -73,18 +88,44 @@ def test_preserves_all_pairs_in_order(self): def test_single_query_is_split_across_many_chunks(self): cand = [(f"s{i}", "SEQ", float(i)) for i in range(64)] tasks = [("q", "AAAA", cand)] - chunks = _chunk_candidate_tasks(tasks, worker_budget=8) + chunks, _ = _chunk_candidate_tasks(tasks, worker_budget=8, floor_cells=1) # A lone query must fan out to more than one chunk so the pool can spread # it — this is the whole point of the change. self.assertGreater(len(chunks), 1) - def test_empty_and_no_pairs(self): - self.assertEqual(_chunk_candidate_tasks([], worker_budget=4), []) - tasks = [("q", "AAAA", [])] - # No pairs to align: returned as-is (caller re-seeds it to []). - self.assertEqual(_chunk_candidate_tasks(tasks, worker_budget=4), tasks) - + def test_the_floor_keeps_small_work_together(self): + cand = [(f"s{i}", "SEQ", float(i)) for i in range(64)] + chunks, cells = _chunk_candidate_tasks([("q", "AAAA", cand)], worker_budget=8) + self.assertEqual((len(chunks), cells), (1, [64 * 12])) + with mock.patch.object(alignment, "_TASK_CELLS_FLOOR", 1): + self.assertGreater(len(_chunk_candidate_tasks([("q", "AAAA", cand)], worker_budget=8)[0]), 1) + def test_empty_and_no_pairs(self): + self.assertEqual(_chunk_candidate_tasks([], worker_budget=4), ([], [])) + # No pairs to align: no task (the caller re-seeds the query to []). + self.assertEqual(_chunk_candidate_tasks([("q", "AAAA", [])], worker_budget=4), ([], [])) + + def test_slices_are_contiguous_within_budget_and_cover_every_candidate(self): + rng = random.Random(2) + tasks = [] + for q in range(30): + n = rng.randint(1, 3000) + cands = [(f"q{q}t{i}", "A" * rng.randint(1, 3000), 0.5) for i in range(rng.randint(0, 200))] + tasks.append((f"q{q}", "M" * n, cands)) + chunks, cells = _chunk_candidate_tasks(tasks, worker_budget=8) + total = sum(len(q) * len(s) for _, q, c in tasks for _, s, _ in c) + budget = max(alignment._TASK_CELLS_FLOOR, -(-total // (4 * 8))) + rebuilt = {} + for (qid, qseq, cands), c in zip(chunks, cells): + self.assertTrue(cands) + self.assertEqual(c, sum(len(qseq) * len(s) for _, s, _ in cands)) + self.assertTrue(c <= budget or len(cands) == 1) + rebuilt.setdefault(qid, []).extend(cands) + self.assertEqual(rebuilt, {q: c for q, _, c in tasks if c}) + self.assertEqual([q for q, _, _ in chunks], sorted([q for q, _, _ in chunks], key=lambda x: int(x[1:]))) + + +@pytest.mark.usefixtures("fmsw_libs") class TestAlignCandidatesParallelism(unittest.TestCase): def _run(self, prefilter, queries, num_workers): return alignment.align_candidates( @@ -107,7 +148,12 @@ def test_serial_and_parallel_agree_single_query(self): prefilter = {"q1": (sids, [1.0 / i for i in range(1, 9)])} serial = self._run(prefilter, queries, num_workers=1) - parallel = self._run(prefilter, queries, num_workers=2) + # These pairs are far below the slice floor: without it the query is cut + # into slices, queued longest first. + with mock.patch.object(alignment, "_TASK_CELLS_FLOOR", 1), \ + self.assertLogs(alignment.logger, "INFO") as logs: + parallel = self._run(prefilter, queries, num_workers=2) + self.assertTrue(any("across 2 CPU workers" in line for line in logs.output), logs.output) self.assertEqual(_signature(serial), _signature(parallel)) # All eight candidates survive the (disabled) filters. @@ -138,6 +184,68 @@ def test_query_with_no_candidates_maps_to_empty(self): self.assertEqual(len(res["q1"]), 2) +class StartMethodTest(unittest.TestCase): + """The Stage-2 pool's start method is pinned to the pre-3.14 platform default (G12). + + ``mp.Pool`` follows the interpreter's global default, which Python 3.14 changes + to 'forkserver' on Linux and which an embedding application can change with + ``set_start_method``. The pool goes through ``mp.get_context`` with the method + that was the default before ('fork' on Linux, 'spawn' on macOS), so neither + changes how the workers start. + """ + + def _run(self, num_workers): + return alignment.align_candidates( + query_sequences={"q1": _STORE["s1"], "q2": _STORE["s5"]}, + prefilter_results={ + "q1": (["s1", "s2", "s3", "s6", "s8"], [0.9, 0.8, 0.7, 0.6, 0.5]), + "q2": (["s5", "s7", "s4"], [0.95, 0.55, 0.2]), + }, + fetch_subject_sequences=_fetch, + min_seq_identity=0.0, + min_coverage=0.0, + max_evalue=None, + num_workers=num_workers, + compute_significance=False, + ) + + def test_platform_defaults(self): + for platform, method in (("linux", "fork"), ("darwin", "spawn"), + ("win32", "spawn"), ("freebsd14", "fork")): + with self.subTest(platform=platform), mock.patch.object(alignment.sys, "platform", platform): + self.assertEqual(alignment._pool_start_method(), method) + + @unittest.skipIf(sys.version_info >= (3, 14), "Python 3.14 changed the POSIX default") + def test_is_the_interpreter_default_before_3_14(self): + # A fresh interpreter, so nothing in this process can have set the method. + default = subprocess.run( + [sys.executable, "-c", "import multiprocessing; print(multiprocessing.get_start_method())"], + check=True, capture_output=True, text=True, + ).stdout.strip() + self.assertEqual(alignment._pool_start_method(), default) + + @unittest.skipIf(sys.platform == "win32", "no forkserver on Windows") + @pytest.mark.usefixtures("fmsw_libs") + def test_pool_ignores_the_global_default(self): + recorded = [] + + class SpyPool(multiprocessing.pool.Pool): + def __init__(self, *args, context=None, **kwargs): + recorded.append(context.get_start_method()) + super().__init__(*args, context=context, **kwargs) + + saved = multiprocessing.get_start_method(allow_none=True) + # Python 3.14's POSIX default, and never the pinned method. + multiprocessing.set_start_method("forkserver", force=True) + try: + with mock.patch("multiprocessing.pool.Pool", SpyPool): + pooled = self._run(num_workers=2) + finally: + multiprocessing.set_start_method(saved, force=True) + self.assertEqual(recorded, [alignment._pool_start_method()]) + self.assertEqual(_signature(pooled), _signature(self._run(num_workers=1))) + + class TestParseFormatOutput(unittest.TestCase): def test_default_when_none(self): self.assertEqual(parse_format_output(None), list(DEFAULT_OUTPUT_FIELDS)) @@ -170,6 +278,7 @@ def test_every_supported_field_is_renderable(self): set(output._FIELD_RENDERERS), ) + @pytest.mark.usefixtures("fmsw_libs") def test_every_supported_field_materializes_when_requested_alone(self): """Guard the registry (output.py) against the compute gate (alignment.py). @@ -180,6 +289,10 @@ def test_every_supported_field_materializes_when_requested_alone(self): never be materialized and would silently render as an empty string in every output file. Requesting each field *on its own* is what exercises the gate; asking for them all at once would mask a missing entry. + + The esm-codes-only fields (``rawrev``, ``q90``, ``rawadj``, ``ecal``) are + the exception: BLOSUM62 never computes them, so they render empty here + (tests/test_code_alignment.py covers them under esm-codes). """ for field in SUPPORTED_OUTPUT_FIELDS: with self.subTest(field=field): @@ -196,6 +309,9 @@ def test_every_supported_field_materializes_when_requested_alone(self): output_fields=[field], ) hit = res["q1"][0] + if field in output.ESM_CODES_OUTPUT_FIELDS: + self.assertEqual(output.format_row("q1", hit, [field]), "") + continue self.assertNotEqual( output.format_row("q1", hit, [field]), "", f"column '{field}' rendered empty when requested alone — " @@ -216,6 +332,7 @@ def _gap_runs(gapped): return len(re.findall(r"-+", gapped)) +@pytest.mark.usefixtures("fmsw_libs") class TestOutputFieldComputation(unittest.TestCase): """The mmseqs-style columns must satisfy their defining invariants.""" @@ -302,6 +419,7 @@ def sig(results): ) +@pytest.mark.usefixtures("fmsw_libs") class TestDefaultFormatGating(unittest.TestCase): def test_heavy_fields_not_materialized_by_default(self): res = alignment.align_candidates( @@ -324,10 +442,11 @@ def test_heavy_fields_not_materialized_by_default(self): self.assertEqual(m.q_start, 1) +@pytest.mark.usefixtures("fmsw_libs") class TestNoAlignmentCandidateDropped(unittest.TestCase): def test_empty_alignment_is_not_reported_even_at_zero_thresholds(self): # Two sequences that share no positive-scoring local alignment produce an - # empty biotite trace (aln_len == 0). Such a candidate is not a hit and + # empty alignment (aln_len == 0). Such a candidate is not a hit and # must be dropped rather than emitted as a malformed position-0 row — # even with the identity/coverage thresholds relaxed all the way to 0. store = {"far": "PPPPPPPPPP"} @@ -345,6 +464,7 @@ def test_empty_alignment_is_not_reported_even_at_zero_thresholds(self): self.assertEqual(res["q"], []) +@pytest.mark.usefixtures("fmsw_libs") class TestMaxEvalueFilter(unittest.TestCase): """The E-value threshold drops weak hits and self-enables significance.""" @@ -394,6 +514,76 @@ def test_requires_a_search_space(self): ) +@pytest.mark.usefixtures("fmsw_libs") +class EmptyDatabaseTest(unittest.TestCase): + """An empty or unbuilt sequence store must say so, not raise a TypeError. + + ``SequenceStore.total_residues()`` is ``COALESCE(SUM(length), 0)``, so an empty + store arrives here as ``0`` rather than ``None``. That slipped past the ``is None`` + guard, left the database E-value unset, and surfaced as ``'<=' not supported + between instances of 'NoneType' and 'float'`` from inside the result loop. + """ + + COMMON = dict( + query_sequences={"q1": _STORE["s1"]}, + prefilter_results={"q1": (["s1"], [0.9])}, + fetch_subject_sequences=_fetch, + num_workers=1, + ) + + def test_empty_database_with_evalue_filter_is_explained(self): + for size in (0, -1): + with self.subTest(size=size): + with self.assertRaises(ValueError) as caught: + alignment.align_candidates(max_evalue=1e-3, subject_db_size=size, **self.COMMON) + message = str(caught.exception) + self.assertIn("non-empty database", message) + self.assertIn(f"subject_db_size is {size}", message) + self.assertIn("max_evalue=None", message) + self.assertIn("--max-evalue inf", message) + # A build refuses an existing database, so the advice says to delete it first. + self.assertIn("Delete the database's files and rebuild it", message) + + def test_refused_before_any_work(self): + """The check runs up front, before the subject sequences are even fetched.""" + def fetch(ids): + raise AssertionError("subject sequences fetched before the size check") + common = dict(self.COMMON, fetch_subject_sequences=fetch) + with self.assertRaisesRegex(ValueError, "non-empty database"): + alignment.align_candidates(max_evalue=1e-3, subject_db_size=0, **common) + + def test_missing_database_size_still_explained(self): + with self.assertRaisesRegex(ValueError, "requires subject_db_size"): + alignment.align_candidates(max_evalue=1e-3, subject_db_size=None, **self.COMMON) + + def test_empty_database_without_evalue_filter_still_works(self): + """Ranking without E-values is legitimate on an empty search space.""" + hits = alignment.align_candidates(max_evalue=None, subject_db_size=0, + compute_significance=True, **self.COMMON)["q1"] + self.assertEqual(len(hits), 1) + self.assertIsNone(hits[0].metrics.evalue) + self.assertIsNotNone(hits[0].metrics.bit_score) + + def test_check_evalue_search_space(self): + """The check align_candidates runs, callable on its own (the CLI runs it before ESM3).""" + check = alignment.check_evalue_search_space + for no_filter in (None, float("inf"), float("nan")): + with self.subTest(max_evalue=no_filter): + self.assertIsNone(check(no_filter, 0)) + self.assertIsNone(check(no_filter, None)) + self.assertEqual(check(1e-3, 10), 1e-3) + with self.assertRaisesRegex(ValueError, "non-empty database"): + check(1e-3, 0) + with self.assertRaisesRegex(ValueError, "requires subject_db_size"): + check(1e-3, None) + + def test_normal_database_size_is_unaffected(self): + hits = alignment.align_candidates(max_evalue=1e-3, subject_db_size=len(_STORE["s1"]), + **self.COMMON)["q1"] + self.assertEqual(len(hits), 1) + self.assertIsNotNone(hits[0].metrics.evalue) + + class TestThresholdDefaultsMatchTheCli(unittest.TestCase): """The library's Stage-2 thresholds are the CLI's documented defaults. @@ -409,6 +599,8 @@ class TestThresholdDefaultsMatchTheCli(unittest.TestCase): ) def test_defaults_are_in_step(self): + """``query sequences`` resolves these defaults per ``--scoring`` (its typer default is None): its BLOSUM62 + defaults are the ones that must equal the library's.""" import inspect from foldmatch.cli import search as cli @@ -418,11 +610,15 @@ def test_defaults_are_in_step(self): cmd = inspect.signature(command).parameters for name in self.SHARED_DEFAULTS: with self.subTest(command=command.__name__, threshold=name): + default = cmd[name].default + if command is cli.query_database_from_fasta and name in cli.BLOSUM62_QUERY_DEFAULTS: + self.assertIsNone(default) + default = cli.BLOSUM62_QUERY_DEFAULTS[name] self.assertEqual( - lib[name].default, cmd[name].default, + lib[name].default, default, f"align_candidates {name}={lib[name].default!r} but " f"--{name.replace('_', '-')} defaults to " - f"{cmd[name].default!r}; update both (and the README).", + f"{default!r}; update both (and the README).", ) @@ -508,6 +704,7 @@ def test_shape_and_zero_scale(self): self.assertFalse(alignment.local_composition_bias(_STORE["s1"], 0.0).any()) +@pytest.mark.usefixtures("fmsw_libs") class TestCompBiasMatchesMmseqs(unittest.TestCase): """Stage-2 scores against real mmseqs 18-8cc5c on the same pairs. @@ -599,6 +796,283 @@ def signature(hits): ) +@pytest.mark.usefixtures("fmsw_libs") +class IdentityVectorisedTest(unittest.TestCase): + """``identity_aln``/``identity_shorter`` come from the kernel's column tally (G3). + + They once came from biotite's ``get_sequence_identity``, which loops over + every alignment column in Python; the kernel counts the same matches, so the + two must give the same floats, not merely close ones. Checked against + biotite's alignment of the pair and against a naive per-column loop on the + mmseqs fixture pairs (with and without the compositional bias correction, + whose position-specific profile changes the alignment) and on 1,000 random + alignments. + """ + + ALL_NEEDS = alignment._FieldNeeds(True, True, True, True, True) + + def tearDown(self): + alignment._worker_init(11, 1, None, None, None) + + @staticmethod + def _naive_identity(trace, query, subject): + matches = 0 + for i, j in trace.tolist(): + if i != -1 and j != -1 and query[i] == subject[j]: + matches += 1 + return matches / len(trace), matches / min(len(query), len(subject)) + + def _check(self, query, subject, comp_bias_scale=None): + """Align one pair through the Stage-2 task and directly with biotite, and compare. + + Returns the metrics, or ``None`` for an empty alignment. + """ + import biotite.sequence.align as align + alignment._worker_init(11, 1, None, None, None, needs=self.ALL_NEEDS, + comp_bias_scale=comp_bias_scale) + _, hits = alignment._align_query(("q", query, [("s", subject, 0.5)])) + metrics = hits[0].metrics + q_clean, s_clean = alignment.sanitize_sequence(query), alignment.sanitize_sequence(subject) + + # The same alignment, built directly with biotite (the oracle). + bias = None if comp_bias_scale is None else alignment.local_composition_bias(query, comp_bias_scale) + aln = ref.biotite_alignment(q_clean, s_clean, (11, 1), bias) + trace = aln.trace + self.assertEqual(metrics.score, int(aln.score)) + self.assertEqual(metrics.aln_len, len(trace)) + if len(trace) == 0: + self.assertEqual((metrics.identity_aln, metrics.identity_shorter), (0.0, 0.0)) + return None + # Same trace, column for column. + self.assertEqual(metrics.q_aln, "".join(q_clean[i] if i != -1 else "-" for i in trace[:, 0])) + self.assertEqual(metrics.t_aln, "".join(s_clean[j] if j != -1 else "-" for j in trace[:, 1])) + + expected = (align.get_sequence_identity(aln, mode="all"), + align.get_sequence_identity(aln, mode="shortest")) + got = (metrics.identity_aln, metrics.identity_shorter) + # Exact float equality on purpose: the output must not move by one ulp. + self.assertEqual(got, expected) + self.assertEqual(got, self._naive_identity(trace, q_clean, s_clean)) + return metrics + + def test_mmseqs_pairs(self): + with open(_RESOURCES / "alignment" / "mmseqs_comp_bias_pairs.tsv") as f: + rows = list(csv.DictReader(f, delimiter="\t")) + for scale in (None, 1.0, 0.5): + for r in rows: + with self.subTest(query=r["query"], target=r["target"], comp_bias_scale=scale): + self.assertIsNotNone(self._check(r["query_sequence"], r["target_sequence"], scale)) + + def test_random_alignments(self): + rng = np.random.default_rng(20261012) + standard = list("ACDEFGHIKLMNPQRSTVWY") + alphabets = ( + standard, + standard + list("BZX*"), # the rest of biotite's protein alphabet + standard + list("UOJBZX*-.0acdy"), # letters sanitize_sequence uppercases or maps to X + ) + found = [] + for case in range(1000): + alphabet = alphabets[case % 3] + query = "".join(rng.choice(alphabet, int(rng.integers(1, 150)))) + if case % 3 == 2: + subject = "".join(rng.choice(alphabet, int(rng.integers(1, 150)))) + else: + # A mutated copy: substitutions at a random rate, then a few indels. + rate = rng.random() + chars = [str(rng.choice(alphabet)) if rng.random() < rate else c for c in query] + for _ in range(int(rng.integers(0, 5))): + p = int(rng.integers(0, len(chars) + 1)) + if rng.random() < 0.5: + del chars[p:p + int(rng.integers(1, 6))] + else: + chars[p:p] = list(rng.choice(alphabet, int(rng.integers(1, 6)))) + subject = "".join(chars) or "A" + with self.subTest(case=case): + metrics = self._check(query, subject, (None, None, None, 1.0)[case % 4]) + if metrics is not None: + found.append(metrics) + # The set has to exercise the computation, not just empty alignments. + self.assertGreater(len(found), 800) + self.assertTrue(any(m.identity_aln == 1.0 for m in found)) + self.assertTrue(any(m.identity_aln < 0.3 for m in found)) + self.assertTrue(any(m.gap_open > 0 for m in found)) + self.assertTrue(any(m.identity_shorter != m.identity_aln for m in found)) + + +@pytest.mark.usefixtures("fmsw_libs") +class NativeKernelOnlyTest(unittest.TestCase): + """Stage 2 aligns on the native kernel only (P9.S3): biotite's aligner is never called, a kernel that cannot be + loaded is an error (no fallback), and a worker whose initializer fails fails Stage 2 instead of hanging it.""" + + COMMON = dict( + query_sequences={"q1": _STORE["s1"], "q2": _STORE["s5"]}, + prefilter_results={"q1": (["s1", "s2", "s3"], [0.9, 0.8, 0.7]), "q2": (["s5", "s7"], [0.9, 0.5])}, + fetch_subject_sequences=_fetch, + min_seq_identity=0.0, + min_coverage=0.0, + max_evalue=None, + ) + + def tearDown(self): + alignment._worker_init(11, 1, None, None, None) + + def test_biotite_aligns_nothing(self): + import biotite.sequence.align as align + with mock.patch.object(align, "align_optimal", side_effect=AssertionError("biotite align_optimal")), \ + mock.patch.object(align, "get_sequence_identity", + side_effect=AssertionError("biotite get_sequence_identity")): + for kw in ({}, {"comp_bias_corr": True}, {"output_fields": list(output.SUPPORTED_OUTPUT_FIELDS), + "subject_db_size": 1000}): + with self.subTest(**{k: str(v)[:20] for k, v in kw.items()}): + res = alignment.align_candidates(num_workers=1, **{**self.COMMON, **kw}) + self.assertEqual(len(res["q1"]), 3) + + def test_a_kernel_that_cannot_be_loaded_is_an_error(self): + def fetch(ids): + raise AssertionError("subject sequences fetched before the kernel was loaded") + missing = swkern.KernelError("native kernel library _fmsw_x not found; build the native kernel with " + "`python -m foldmatch._native.build`") + for kw in ({"num_workers": 1}, {"num_workers": 2}, {"prefilter_results": {}}): + with self.subTest(**kw), mock.patch.object(swint, "load", side_effect=missing): + with self.assertRaises(swkern.KernelError) as ctx: + alignment.align_candidates(**{**self.COMMON, "fetch_subject_sequences": fetch, **kw}) + self.assertIs(ctx.exception, missing) + + def test_initializer_errors_are_raised_by_tasks(self): + task = ("q", "MKTAYIAKQRQ", [("s", "MKTAYIAKQRQ", 0.5)]) + alignment._pool_worker_init(11, 1, None, None, None, None, None, None, "bogus") + with self.assertRaisesRegex(swkern.KernelError, "bogus"): + alignment._align_query(task) + with self.assertRaisesRegex(swkern.KernelError, "bogus"): + alignment._align_indexed((0, task)) + alignment._worker_init(11, 1, None, None, None) # a good initialisation clears it + self.assertEqual(len(alignment._align_query(task)[1]), 1) + with self.assertRaisesRegex(swkern.KernelError, "bogus"): # in-process: raised at once + alignment._worker_init(11, 1, None, None, None, None, None, None, "bogus") + + @unittest.skipIf(sys.platform == "win32", "no SIGALRM") + def test_a_worker_that_cannot_start_fails_stage2(self): + """The workers load the ISA the parent loaded; here they are told one that does not exist, so their + initializer fails. multiprocessing.Pool would start new workers for ever; the tasks raise the error.""" + real_load = swint.load + + def load(isa=None, **kw): + # the parent's call (no ISA) gets a kernel the workers cannot load; theirs (fork) go to the real one + return types.SimpleNamespace(isa="bogus") if isa is None else real_load(isa, **kw) + + with mock.patch.object(swint, "load", load), _deadline(120): + with self.assertRaisesRegex(swkern.KernelError, "bogus"): + alignment.align_candidates(num_workers=2, **self.COMMON) + + +@contextlib.contextmanager +def _deadline(seconds: float): + """TimeoutError in this (the main) thread after ``seconds``: a wait that must end fails instead.""" + def expired(*_): + raise TimeoutError(f"still waiting after {seconds} s") + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +class GapCostsTest(unittest.TestCase): + """The gap costs of the BLOSUM62 Stage 2 (P9.S3): what the kernel accepts, and 11/1 for calibrated + significance, checked by check_gap_costs and by align_candidates before any sequence is read.""" + + def test_the_kernel_range(self): + check = alignment.check_gap_costs + self.assertEqual(check(11, 1), (11, 1)) + self.assertEqual(check(-11, -1), (11, 1)) # the sign is ignored, as it always was + self.assertEqual(check(np.int64(5), 0), (5, 0)) + self.assertEqual(check(1, swint.MAX_GAP), (1, swint.MAX_GAP)) + self.assertEqual(check(swint.MAX_GAP, 1), (swint.MAX_GAP, 1)) + # whole floats are their integers, as biotite took them (the release accepted 11.0) + self.assertEqual(check(11.0, 1.0), (11, 1)) + self.assertEqual(check(np.float64(-11.0), np.float32(0.0)), (11, 0)) + for go, ge, name in ((0, 1, "gap_open"), (swint.MAX_GAP + 1, 1, "gap_open"), + (-swint.MAX_GAP - 1, 1, "gap_open"), (11, swint.MAX_GAP + 1, "gap_extend"), + (0.0, 1, "gap_open")): + with self.subTest(gap_open=go, gap_extend=ge), self.assertRaisesRegex(ValueError, f"^{name} must be"): + check(go, ge) + for go, ge, name in ((11.5, 1, "gap_open"), (11, 0.5, "gap_extend"), (float("nan"), 1, "gap_open"), + (float("inf"), 1, "gap_open"), (True, 1, "gap_open"), (11, "1", "gap_extend"), + (None, 1, "gap_open"), (11, np.bool_(True), "gap_extend")): + with self.subTest(gap_open=go, gap_extend=ge), \ + self.assertRaisesRegex(TypeError, f"^{name} must be a whole number"): + check(go, ge) + # each bound gives its own reason + with self.assertRaisesRegex(ValueError, r"between 1 and 1048576 \(its sign is ignored\), got 0: .*" + r"end-cell rule needs a gap to cost at least 1 to open"): + check(0, 1) + for go, ge in ((swint.MAX_GAP + 1, 1), (11, swint.MAX_GAP + 1)): + with self.subTest(gap_open=go, gap_extend=ge), \ + self.assertRaisesRegex(ValueError, r"got 1048577: 1048576 is the most the Smith-Waterman " + r"kernel takes \(FMSWI_MAX_GAP\)\.$"): + check(go, ge) + + def test_calibrated_significance_needs_11_1(self): + check = alignment.check_gap_costs + for mode in (SignificanceMode.default, "default"): + with self.assertRaisesRegex(ValueError, "No calibrated statistics for BLOSUM62 with gaps 5/2") as ctx: + check(5, 2, mode, significance=True) + self.assertNotIn("E-value filtering", str(ctx.exception)) + with self.assertRaisesRegex(ValueError, r"gaps 10/1.*E-value filtering \(max_evalue=0.001\) requires"): + check(10, 1, max_evalue=1e-3) + self.assertEqual(check(-11, -1, significance=True, max_evalue=1e-3), (11, 1)) + self.assertEqual(check(11.0, 1.0, significance=True, max_evalue=1e-3), (11, 1)) + # Fine without significance, or with the sampled statistics. + for kw in ({}, {"max_evalue": float("inf")}, {"max_evalue": None}, {"max_evalue": float("nan")}, + {"significance_mode": SignificanceMode.sampled, "significance": True, "max_evalue": 1e-3}): + with self.subTest(**{k: str(v) for k, v in kw.items()}): + self.assertEqual(check(5, 2, **kw), (5, 2)) + + def test_align_candidates_checks_before_any_work(self): + def fetch(ids): + raise AssertionError("subject sequences fetched before the gap check") + common = dict(query_sequences={"q1": _STORE["s1"]}, prefilter_results={"q1": (["s1"], [0.9])}, + fetch_subject_sequences=fetch, subject_db_size=1000, num_workers=1) + cases = [ # (options, exception, message) + (dict(gap_open=0, max_evalue=None, compute_significance=False), ValueError, + "gap_open must be between 1 and"), + (dict(gap_extend=swint.MAX_GAP + 1, max_evalue=None), ValueError, "gap_extend must be between 0 and"), + (dict(gap_open=10.5, max_evalue=None, compute_significance=False), TypeError, + "gap_open must be a whole number"), + (dict(gap_open=5, gap_extend=2, max_evalue=None), ValueError, "No calibrated statistics"), + (dict(gap_open=5, gap_extend=2, compute_significance=False), ValueError, "E-value filtering"), + ] + for options, exception, message in cases: + for prefilter in (common["prefilter_results"], {}): + for workers in (1, 4, None): # serial, a pool, the default pool + with self.subTest(**options, queries=len(prefilter), workers=workers), \ + mock.patch.object(swint, "load", side_effect=AssertionError("kernel loaded")), \ + mock.patch.object(alignment, "_expand_cpu_affinity", + side_effect=AssertionError("worker pool set up")), \ + mock.patch.object(multiprocessing, "get_context", + side_effect=AssertionError("worker pool started")): + with self.assertRaisesRegex(exception, message): + alignment.align_candidates(**{**common, "prefilter_results": prefilter, **options, + "num_workers": workers}) + + @pytest.mark.usefixtures("fmsw_libs") + def test_align_candidates_aligns_with_the_absolute_costs(self): + common = dict(query_sequences={"q1": _STORE["s1"]}, + prefilter_results={"q1": (["s1", "s5", "s7"], [0.9, 0.8, 0.7])}, + fetch_subject_sequences=_fetch, min_seq_identity=0.0, min_coverage=0.0, max_evalue=None, + num_workers=1, output_fields=["cigar", "raw"], compute_significance=False) + for go, ge in ((11, 1), (3, 3), (1, 0)): + pos = alignment.align_candidates(gap_open=go, gap_extend=ge, **common) + neg = alignment.align_candidates(gap_open=-go, gap_extend=-ge, **common) + self.assertEqual(pos, neg) + # whole floats, which the release (biotite) accepted, are their integers + self.assertEqual(alignment.align_candidates(gap_open=float(go), gap_extend=np.float64(ge), **common), pos) + + +@pytest.mark.usefixtures("fmsw_libs") class TestWriteAlignedResults(unittest.TestCase): def test_tsv_no_header(self): fields = ["query", "target", "fident", "alnlen", "cigar"] @@ -660,6 +1134,7 @@ def test_default_format_leads_with_ids_then_embedding_score(self): ) +@pytest.mark.usefixtures("fmsw_libs") class TestFormatMode(unittest.TestCase): """--format-mode selects the file layout; the column set is chosen elsewhere.""" diff --git a/tests/test_assembly_lists.py b/tests/test_assembly_lists.py new file mode 100644 index 0000000..465b542 --- /dev/null +++ b/tests/test_assembly_lists.py @@ -0,0 +1,438 @@ +"""P9.A5: an assembly list row of more than 4 fields is refused. + +The assembly lists of inference assembly-embedding and complete-embedding (--src-file, --src-assembly-file) and of the +library's ResidueAssemblyEmbeddingFromTensorFile and ResidueAssemblyEmbeddingFromParquet are CSV without a header, of +structure name, structure file or URL, assembly ID and embedding name, read by read_assembly_list. A row of more than 4 +fields is refused (RefusalError), from a file and from a stream, naming the file or the stream and the row, as +read_chain_list refuses one, and so by both commands before any model is built: complete-embedding now reads its +assembly list before its residue step (a pipe, /dev/stdin or /dev/fd/N, which can be read once, into memory, its text +given to the assembly step: P9.N). Before, pandas read the leading fields of a first such row as the index, which +shifted the columns of every row (a trailing comma was enough), stopped at a later one with a ParserError that named no +file, and refused a stream row of 5 elements with its ValueError; complete-embedding read its assembly list only in its +assembly step, after its residue and chain steps had written their files. Lists of at most 4 fields per row are read as +before, but for P9.N's D5: NA is text, an empty field '' (before, NaN), and a missing or empty embedding name is +-. + +ESM3 and the aggregator are never loaded: the commands run with stub models (_esm3, _aggregate, as in +test_inference_follow_ups.py) or with model getters that fail if called.""" +import errno +import io +import logging +import os +import re +import shutil +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import torch +import typer +from esm.sdk.api import ESMProtein +from esm.utils.structure.protein_chain import ProteinChain +from typer.testing import CliRunner + +from foldmatch.cli import inference +from foldmatch.dataset.residue_assembly_embedding_from_parquet import ResidueAssemblyEmbeddingFromParquet +from foldmatch.dataset.residue_assembly_embedding_from_tensor_file import ResidueAssemblyEmbeddingFromTensorFile +from foldmatch.errors import RefusalError +# Imported before any test patches their model getters +from foldmatch.inference import assembly_inferece, chain_inference, esm_inference, structure_inference # noqa: F401 +from foldmatch.types.api_types import SrcLocation, StructureFormat +from foldmatch.utils.structure_parser import filter_residues, get_structure_from_src, protein_chain_iter, \ + rename_atom_attr + +_RESOURCES = Path(__file__).parent / "resources" +_CIF, _CIF2 = _RESOURCES / "pdb" / "1acb.cif", _RESOURCES / "pdb" / "2uzi.cif" +_EMBEDDINGS = _RESOURCES / "embeddings" +_CPU = ("--accelerator", "cpu", "--devices", "1") +_COLUMNS = ["stream_name", "stream", "assembly_id", "item_name"] + +# What a refusal of an assembly list row of more than 4 fields says after what it found +_RULE = ("An assembly list row holds at most 4 fields (structure name, structure file or URL, assembly ID, embedding " + "name): ") +_REMOVE_THE_EXTRA_FIELDS = "remove the extra fields (a comma after the embedding name makes a 5th)." + +# (case, assembly list, what the refusal found, after the file's path: a regular expression) +_LISTS = ( + ("first row, a trailing comma", f"1acb,{_CIF},1,1acb-1,\n2uzi,{_CIF2},1,2uzi-1\n", + re.escape("its first row has 5 fields")), + ("first row, a 5th field of text", f"1acb,{_CIF},1,1acb-1,extra\n2uzi,{_CIF2},1,2uzi-1\n", + re.escape("its first row has 5 fields")), + ("first row of 6 fields", f"1acb,{_CIF},1,1acb-1,,extra\n2uzi,{_CIF2},1,2uzi-1\n", + re.escape("its first row has 6 fields")), + ("a later row of 5 fields", f"1acb,{_CIF},1,1acb-1\n2uzi,{_CIF2},1,2uzi-1,\n", + r"a row has more than 4 fields \(pandas: [^()]*Expected 4 fields in line 2, saw 5\)"), + ("a first row of 5, a later row of 6", f"1acb,{_CIF},1,1acb-1,\n2uzi,{_CIF2},1,2uzi-1,,\n", + r"a row has more than 4 fields \(pandas: [^()]*Expected 5 fields in line 2, saw 6\)"), + # pandas read this list's first column as the index, so it gave the files of the list without it + ("an index column", f"0,1acb,{_CIF},1,1acb-1\n1,2uzi,{_CIF2},1,2uzi-1\n", + re.escape("its first row has 5 fields")), +) +_READERS = ("read_assembly_list", "ResidueAssemblyEmbeddingFromTensorFile", "ResidueAssemblyEmbeddingFromParquet") +_STREAM_RULE = ("An assembly list row holds at most 4 fields (structure name, structure file or URL, assembly ID, " + "embedding name): remove the extra elements.") + + +def _invoke(*args): + """inference in-process (exit codes as in a shell, the exception kept), the logging setup restored after.""" + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(inference.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + +def _esm3(model, protein): + """Stands in for ESM3 in the inference modules: L + 2 rows (BOS and EOS, zeros), each residue's row its letter, its + position and the chain's length (as in test_inference_follow_ups.py).""" + rows = [[float(ord(aa)), float(k), float(len(protein))] for k, aa in enumerate(protein.sequence)] + return torch.tensor([[0.0] * 3] + rows + [[0.0] * 3]) + + +def _aggregate(x, x_mask): + """Stands in for the aggregator: the sum of each item's rows.""" + return x.masked_fill(x_mask.unsqueeze(-1), 0.0).sum(dim=1) + + +def _csv(residues) -> str: + """The embedding of these residue embeddings as CsvBatchWriter writes it: _aggregate's vector, one value a line.""" + return pd.DataFrame(residues.sum(dim=0).numpy()).to_csv(index=False, header=False) + + +def _protein(chain_id: str) -> ESMProtein: + """Archive 1acb's protein chain, read as every path reads it.""" + atoms = next(protein_chain_iter(get_structure_from_src(str(_CIF), StructureFormat.mmcif), chain_id=chain_id)) + return ESMProtein.from_protein_chain(ProteinChain.from_atomarray(rename_atom_attr(filter_residues(atoms)))) + + +def _read_assembly_list(*args): + """foldmatch.dataset.utils.read_assembly_list, imported when called.""" + from foldmatch.dataset.utils import read_assembly_list + return read_assembly_list(*args) + + +def _expected(rows: list, index: list) -> pd.DataFrame: + """A frame of the assembly list columns, as strings (None: a stream's missing element).""" + return pd.DataFrame(rows, index=pd.Index(index, dtype="int64"), columns=_COLUMNS, dtype=str) + + +def _write_once(fifo: Path, text: str, stop: threading.Event): + """The writer of the named pipe fifo, until stop is set: its first reader reads text, a later one an end of file at + once, as from a pipe whose writer has exited (rather than a wait for another writer).""" + while not stop.is_set(): + try: + # Opened only once a reader has opened it, so that stop always ends the loop + fd = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK) + except OSError as e: + if e.errno != errno.ENXIO: + raise + stop.wait(0.01) + continue + os.set_blocking(fd, True) + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(text) + text = "" + stop.wait(0.01) + + +class _AssemblyListCase(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + # Residue embeddings in Parquet, for ResidueAssemblyEmbeddingFromParquet, which reads them after the list + self.parquet = self.tmp / "residues.parquet" + pq.write_table(pa.table({"id": ["x.A"], "embedding": [[[0.0, 1.0, 2.0]]]}, + schema=pa.schema([("id", pa.string()), + ("embedding", pa.list_(pa.list_(pa.float32(), 3)))])), + self.parquet) + + def assembly_list(self, case: str, text: str) -> Path: + path = self.tmp / f"{case.replace(',', '').replace(' ', '_')}.csv" + path.write_text(text, encoding="utf-8", newline="") + return path + + def readers(self, src, src_location: SrcLocation = SrcLocation.file) -> dict: + """read_assembly_list and both readers (_READERS), each a callable that reads src, by name.""" + return { + "read_assembly_list": lambda: _read_assembly_list(src, src_location), + "ResidueAssemblyEmbeddingFromTensorFile": lambda: ResidueAssemblyEmbeddingFromTensorFile( + src_stream=src, res_embedding_location=self.tmp, src_location=src_location), + "ResidueAssemblyEmbeddingFromParquet": lambda: ResidueAssemblyEmbeddingFromParquet( + src_stream=src, parquet_path=self.parquet, src_location=src_location), + } + + def assert_refused(self, refusal: BaseException, source: str, found: str): + """The refusal names the source and what it found, says that a row holds at most 4 fields, and what to do.""" + self.assertIsInstance(refusal, RefusalError) + self.assertRegex(str(refusal), f"^{re.escape(source)}: {found}\\. " + f"{re.escape(_RULE + _REMOVE_THE_EXTRA_FIELDS)}$") + + +class LongerRowTest(_AssemblyListCase): + """D1: read_assembly_list and both readers refuse a row of more than 4 fields, from a path, an open file (named by + its path) and a text buffer ("The assembly list"), and from a stream; any other ParserError too.""" + + def test_a_file_row_of_more_than_4_fields(self): + """A first row with a trailing comma, with a 5th field of text or with 6 fields, every row with an index column + (pandas took the first one or two fields as the index), and a later row of 5, or of 6 after a first row of 5 + (pandas' ParserError, which names the line, quoted and kept as the cause).""" + for case, text, found in _LISTS: + path = self.assembly_list(case, text) + for source, make, named in (("path", lambda: path, str(path)), + ("open file", lambda: open(path, encoding="utf-8"), str(path)), + ("text buffer", lambda: io.StringIO(text), "The assembly list")): + for reader in _READERS: + with self.subTest(case=case, source=source, reader=reader): + src = make() + try: + with self.assertRaises(RefusalError) as refused: + self.readers(src)[reader]() + finally: + if hasattr(src, "close"): + src.close() + self.assert_refused(refused.exception, named, found) + if "later row" in case: + self.assertIsInstance(refused.exception.__cause__, pd.errors.ParserError) + + def test_a_stream_row_of_more_than_4_elements(self): + """A stream: how many rows have more than 4 elements, and up to 5 of them, counted from 1 (before, pandas' + ValueError named no row).""" + for rows, found in ( + ([("1acb", str(_CIF), "1", "1acb-1"), ("2uzi", str(_CIF2), "1", "2uzi-1", "")], + "1 row(s) of more than 4 elements (row(s) 2"), + ([("1acb", str(_CIF), "1", "1acb-1"), ("2uzi", str(_CIF2), "1", "2uzi-1", "", "x")], + "1 row(s) of more than 4 elements (row(s) 2"), + ([("w", "f", "1")] + [("x", "f", "1", "x-1", "z")] * 5 + [("y",) * 6] * 2, + "7 row(s) of more than 4 elements (row(s) 2, 3, 4, 5, 6 and 2 more")): + for reader, read in self.readers(rows, SrcLocation.stream).items(): + with self.subTest(rows=len(rows), longest=max(map(len, rows)), reader=reader): + with self.assertRaises(RefusalError) as refused: + read() + self.assertEqual(str(refused.exception), + f"The assembly list stream: {found}, counted from 1). {_STREAM_RULE}") + + def test_another_parser_error_is_not_claimed_as_extra_fields(self): + """Another pandas ParserError (an unclosed quote) is refused too, naming the file and quoting pandas, without + saying that a row has more than 4 fields (before, pandas' bare ParserError).""" + path = self.assembly_list("unclosed quote", f'1acb,"{_CIF},1,1acb-1\n2uzi,{_CIF2},1,2uzi-1\n') + for reader, read in self.readers(path).items(): + with self.subTest(reader=reader): + with self.assertRaises(RefusalError) as refused: + read() + self.assertIsInstance(refused.exception.__cause__, pd.errors.ParserError) + self.assertRegex(str(refused.exception), + f"^{re.escape(str(path))}: pandas could not read it as CSV \\([^()]*EOF inside " + f"string[^()]*\\)\\. " + re.escape( + "An assembly list is CSV without a header, of at most 4 fields per row (structure " + "name, structure file or URL, assembly ID, embedding name): correct it where " + "pandas' message points.") + "$") + self.assertNotIn("more than 4", str(refused.exception)) + + +class CommandTest(_AssemblyListCase): + """D1 and D3: inference assembly-embedding and complete-embedding refuse such a list (exit status 1, the + RefusalError) before any model is built (the model getters fail if called) and write nothing; complete-embedding + reads an assembly list in a regular file before its residue step, and so refuses a list that gives one structure + name to two files before any model too.""" + + def setUp(self): + super().setUp() + self.never = mock.Mock(side_effect=AssertionError("a model was built")) + for getter in ("esm_inference.get_residue_model", "structure_inference.get_residue_model", + "structure_inference.get_aggregator_model", "chain_inference.get_aggregator_model"): + self.enterContext(mock.patch(f"foldmatch.inference.{getter}", self.never)) + self.chains = self.assembly_list("chains", f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n") + + def folder(self, name: str) -> Path: + return Path(tempfile.mkdtemp(prefix=f"{name}-", dir=self.tmp)) + + def complete_embedding(self, assemblies: Path): + """inference complete-embedding on the conventional chain list and these assemblies: its result, and the files + written in its residue, chain and assembly folders.""" + folders = [self.folder(name) for name in ("residues", "chains", "assemblies")] + result = _invoke("complete-embedding", "--src-chain-file", self.chains, "--src-assembly-file", assemblies, + "--output-res-path", folders[0], "--output-chain-path", folders[1], + "--output-assembly-path", folders[2], *_CPU) + return result, [f for folder in folders for f in folder.iterdir()] + + def test_assembly_embedding_refuses_it_before_any_model(self): + """Before, the reader built shifted columns (the aggregator was then built, and the first item stopped on the + assembly ID read as the structure file; 6 fields were refused as a shared name, before any model), or stopped on + pandas' ParserError; an index column worked.""" + for case, text, found in _LISTS: + with self.subTest(case=case): + path = self.assembly_list(case, text) + out = self.folder("out") + result = _invoke("assembly-embedding", "--src-file", path, "--res-embedding-location", _EMBEDDINGS, + "--output-folder", out, *_CPU) + self.assertEqual(result.exit_code, 1, result.output) + self.assert_refused(result.exception, str(path.resolve()), found) + self.assertEqual(list(out.iterdir()), []) + self.never.assert_not_called() + + def test_complete_embedding_refuses_it_before_any_model(self): + """Before, the residue and chain steps ran and wrote their files, and the assembly step then failed or, for an + index column, worked (on a chain list of both entries: this test's has 1acb only, so it stopped at 2uzi).""" + for case, text, found in _LISTS: + with self.subTest(case=case): + path = self.assembly_list(case, text) + result, written = self.complete_embedding(path) + self.assertEqual(result.exit_code, 1, result.output) + self.assert_refused(result.exception, str(path.resolve()), found) + self.assertEqual(written, []) + self.never.assert_not_called() + + def test_complete_embedding_refuses_a_shared_name_before_any_model(self): + """A list that gives the name 1acb to two files: refused (refuse_shared_names) before the residue step, where + before it was refused by the assembly step, after the residue and chain steps.""" + copy = Path(shutil.copyfile(_CIF, self.tmp / "1acb-copy.cif")) + path = self.assembly_list("shared name", f"1acb,{_CIF},1,1acb-1\n1acb,{copy},1,1acb-x\n") + result, written = self.complete_embedding(path) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError) + self.assertIn(f"'1acb' names {min(str(_CIF), str(copy))} and {max(str(_CIF), str(copy))}", + str(result.exception)) + self.assertEqual(written, []) + self.never.assert_not_called() + + def test_the_help_describes_the_columns(self): + """D4: the help of assembly-embedding --src-file and complete-embedding --src-assembly-file.""" + commands = typer.main.get_command(inference.app).commands + for command, option in (("assembly-embedding", "src_file"), ("complete-embedding", "src_assembly_file")): + with self.subTest(command=command): + self.assertEqual(next(p.help for p in commands[command].params if p.name == option), + "CSV file of 4 columns, no header: Structure Name | Structure File Path or URL | " + "Assembly Id | Embedding Name, - when the 4th column is " + "missing or empty. Each row needs its own Embedding Name.") + + +class AsBeforeTest(_AssemblyListCase): + """Lists of at most 4 fields per row read as before, but for P9.N's D5: the frames recorded from the readers of + 9763f44 (pandas' call unchanged but for keep_default_na=False, then sorted by structure name), NA as text, an empty + field as '' and a missing or empty embedding name as - (before, NaN); and + complete-embedding's files with stub models, its assembly list in a regular file, a pipe or /dev/fd/N.""" + + # (case, assembly list, the rows of the readers' frame, its index) + LISTS = ( + ("rows of 2, 3 and 4 fields", "c,f.cif,3,c-3\nb,f.cif\na,g.cif,2\n", + [["a", "g.cif", "2", "a-2"], ["b", "f.cif", "", "b-"], ["c", "f.cif", "3", "c-3"]], [2, 1, 0]), + ("an empty 4th field and NA", "b,f.cif,1,\nNA,g.cif,NA,a-1\na,NA,2,NA\n", + [["NA", "g.cif", "NA", "a-1"], ["a", "NA", "2", "NA"], ["b", "f.cif", "1", "b-1"]], [1, 2, 0]), + ("a byte-order mark and CRLF", "b,f.cif,1,b-1\r\na,g.cif,2,a-2\r\n", + [["a", "g.cif", "2", "a-2"], ["b", "f.cif", "1", "b-1"]], [1, 0]), + ("quoted fields", 'b,"f,1.cif",1,"n,1"\n"a\nz",g.cif,"2",\n', + [["a\nz", "g.cif", "2", "a\nz-2"], ["b", "f,1.cif", "1", "n,1"]], [1, 0]), + ("blank lines", "\nb,f.cif,1,b-1\n\n \n\t\na,g.cif,2,a-2\n\n", + [["a", "g.cif", "2", "a-2"], ["b", "f.cif", "1", "b-1"]], [1, 0]), + ("an empty file", "", [], []), + ) + STREAM = [("b", "f.cif", "1"), ("a", "g.cif", "2", "a-2"), ("c", None, "3", None)] + + def test_the_readers_read_them_as_before(self): + """Both readers' frames, from a path, an open file and a text buffer, and from a stream of 3 and 4 elements.""" + for case, text, rows, index in self.LISTS: + path = self.assembly_list(case, text) + for source, make in (("path", lambda: path), ("open file", lambda: open(path, encoding="utf-8")), + ("text buffer", lambda: io.StringIO(text))): + for reader in _READERS[1:]: + with self.subTest(case=case, source=source, reader=reader): + src = make() + try: + data = self.readers(src)[reader]().data + finally: + if hasattr(src, "close"): + src.close() + pd.testing.assert_frame_equal(data, _expected(rows, index), check_exact=True) + for reader in _READERS[1:]: + with self.subTest(stream=reader): + data = self.readers(self.STREAM, SrcLocation.stream)[reader]().data + pd.testing.assert_frame_equal(data, _expected([["a", "g.cif", "2", "a-2"], ["b", "f.cif", "1", "b-1"], + ["c", None, "3", "c-3"]], [1, 0, 2]), check_exact=True) + + def test_read_assembly_list_reads_them_as_pandas_did(self): + """read_assembly_list: the frame of the readers' pandas call on 9763f44, in input order, its index pandas' + (0, 1, ...); a stream as pandas reads its rows, inside a list, a tuple or a generator.""" + for case, text, rows, index in self.LISTS: + path = self.assembly_list(case, text) + unsorted = [row for _, row in sorted(zip(index, rows))] + for source, make in (("path", lambda: path), ("text buffer", lambda: io.StringIO(text))): + with self.subTest(case=case, source=source): + data = _read_assembly_list(make()) + pd.testing.assert_frame_equal(data, _expected(unsorted, sorted(index)), check_exact=True) + self.assertIsInstance(data.index, pd.RangeIndex) + for container in (list, tuple, iter): + with self.subTest(stream=container.__name__): + data = _read_assembly_list(container(self.STREAM), SrcLocation.stream) + pd.testing.assert_frame_equal(data, _expected([["b", "f.cif", "1", "b-1"], ["a", "g.cif", "2", "a-2"], + ["c", None, "3", "c-3"]], [0, 1, 2]), check_exact=True) + + def assert_complete_embedding(self, assemblies: Path): + """complete-embedding on a chain list of archive 1acb's chains A and B and these assemblies (1acb's assembly 1, + named 1acb-1), stub models: the residue files, chain embeddings and assembly embedding that the stubs give + through the unchanged readers and writers.""" + with mock.patch("foldmatch.inference.esm_inference.get_residue_model", return_value="ESM3"), \ + mock.patch("foldmatch.inference.chain_inference.get_aggregator_model", return_value=_aggregate), \ + mock.patch("foldmatch.modules.esm_module.per_residue_embedding", _esm3): + residues_of = {c: _esm3(None, _protein(c)) for c in "AB"} + chains = self.assembly_list("chains", f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n") + folders = [Path(tempfile.mkdtemp(prefix=f"{name}-", dir=self.tmp)) + for name in ("residues", "chains", "assemblies")] + result = _invoke("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", assemblies, + "--output-res-path", folders[0], "--output-chain-path", folders[1], + "--output-assembly-path", folders[2], *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + written = {f.name: torch.load(f, weights_only=True) for f in folders[0].iterdir()} + self.assertEqual(sorted(written), ["1acb.A.pt", "1acb.B.pt"]) + for c in "AB": + self.assertTrue(torch.equal(written[f"1acb.{c}.pt"], residues_of[c]), c) + self.assertEqual({f.name: f.read_text() for f in folders[1].iterdir()}, + {f"1acb.{c}.csv": _csv(residues_of[c]) for c in "AB"}) + self.assertEqual({f.name: f.read_text() for f in folders[2].iterdir()}, + {"1acb-1.csv": _csv(torch.cat([residues_of["A"], residues_of["B"]]))}) + + def test_complete_embedding_on_conventional_lists(self): + """complete-embedding on a chain list and an assembly list of 4 fields per row, stub models: the residue files, + chain embeddings and assembly embedding that the stubs give through the unchanged readers and writers, as + before.""" + self.assert_complete_embedding(self.assembly_list("assemblies", f"1acb,{_CIF},1,1acb-1\n")) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "a named pipe needs os.mkfifo") + def test_complete_embedding_on_an_assembly_list_in_a_pipe(self): + """D3': an assembly list given as a pipe (a named pipe here, or <(...) on macOS) can be read once: + complete-embedding reads it once, into memory, before its residue step, and gives its text to its assembly + step, which writes the assembly embedding, as before (read from the pipe a second time, the assembly step + would find it empty and write nothing, with exit status 0).""" + fifo = self.tmp / "assemblies.fifo" + os.mkfifo(fifo) + stop = threading.Event() + writer = threading.Thread(target=_write_once, args=(fifo, f"1acb,{_CIF},1,1acb-1\n", stop), daemon=True) + writer.start() + try: + self.assert_complete_embedding(fifo) + finally: + stop.set() + writer.join() + + @unittest.skipUnless(os.path.isdir("/dev/fd"), "/dev/fd/N needs /dev/fd") + def test_complete_embedding_on_an_assembly_list_as_dev_fd(self): + """D3'': on macOS, an assembly list in a regular file given as /dev/fd/N (or as /dev/stdin, which click makes + /dev/fd/0) can be read once, as open shares the descriptor's offset: complete-embedding reads a list under + /dev once, into memory, before its residue step, and its assembly step writes the assembly embedding, as before + (read from the path a second time, the assembly step would find it empty and write nothing, with exit status + 0).""" + fd = os.open(self.assembly_list("assemblies", f"1acb,{_CIF},1,1acb-1\n"), os.O_RDONLY) + try: + self.assert_complete_embedding(Path(f"/dev/fd/{fd}")) + finally: + os.close(fd) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_assembly_structures.py b/tests/test_assembly_structures.py new file mode 100644 index 0000000..39e679b --- /dev/null +++ b/tests/test_assembly_structures.py @@ -0,0 +1,346 @@ +"""P9.A: each assembly of an entry, and each copy of a chain in an assembly, is read as itself. + +StructureProvider kept the last structure read under the entry name alone, so assemblies of an entry read one after +another all got the first one's structure; EsmAssemblyFromStructure also passed the previous assembly's structure as +the source of the next, and read a downloaded file (a stream) to its end while listing its assemblies, before reading +any of them. The fixture is 1acb with a second assembly, chain B alone (the first holds A, B, C and D; A and B are +the protein chains, C and D water), and optionally a third, the water alone. + +EsmAssemblyFromStructure also read each protein chain of an assembly from the first piece of the atoms with its chain +ID, which the copies of a chain made by the assembly's operators share, so every copy got the first copy's atoms +(P9.A2). That fixture is 1acb with one assembly of two copies of A, B, C and D, made by the identity and by a 2-fold +about z then 40 Å along x, in mmCIF and in PDB format (author chain IDs E and I, the operators in REMARK 350).""" +import inspect +import io +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import biotite.structure as struc +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import torch +from biotite.structure.io.pdb import PDBFile +from biotite.structure.io.pdbx import CIFCategory, CIFFile, get_structure + +from foldmatch.dataset import esm_assembly_from_structure +from foldmatch.dataset.esm_assembly_from_structure import EsmAssemblyFromStructure +from foldmatch.dataset.esm_prot_from_chain import EsmProtFromChain +from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure +from foldmatch.dataset.resdiue_assembly_embedding_from_structure import ResidueAssemblyDatasetFromStructure +from foldmatch.dataset.residue_assembly_embedding_from_parquet import ResidueAssemblyEmbeddingFromParquet +from foldmatch.dataset.residue_assembly_embedding_from_tensor_file import ResidueAssemblyEmbeddingFromTensorFile +from foldmatch.dataset.residue_embedding_from_structure import ResidueEmbeddingFromStructure +from foldmatch.types.api_types import SrcLocation, StructureFormat +from foldmatch.utils import structure_provider +from foldmatch.utils.structure_parser import get_assemblies, get_structure_from_src +from foldmatch.utils.structure_provider import StructureProvider + +_RESOURCES = Path(__file__).parent / "resources" +_EMBEDDINGS = _RESOURCES / "embeddings" +_ALL = ["A", "B", "C", "D"] + + +def _two_assemblies(folder: Path, water: bool = False) -> Path: + asym = ["A,B,C,D", "B"] + (["C,D"] if water else []) + ids = [str(k + 1) for k in range(len(asym))] + cif = CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")) + cif.block["pdbx_struct_assembly"] = CIFCategory({ + "id": ids, "details": ["author_and_software_defined_assembly"] * len(ids), "method_details": ["?"] * len(ids), + "oligomeric_details": ["?"] * len(ids), "oligomeric_count": ["?"] * len(ids)}) + cif.block["pdbx_struct_assembly_gen"] = CIFCategory({ + "assembly_id": ids, "oper_expression": ["1"] * len(ids), "asym_id_list": asym}) + path = folder / "1acb.cif" + cif.write(str(path)) + return path + + +# The operators of the two-copy assembly, as a rotation and a translation: the identity, and a 2-fold about z then 40 Å +# along x. +_OPERATORS = ((np.eye(3), np.zeros(3)), (np.diag([-1.0, -1.0, 1.0]), np.array([40.0, 0.0, 0.0]))) + + +def _two_copies(folder: Path) -> Path: + """1acb with one assembly: each of _OPERATORS applied to A, B, C and D.""" + cif = CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")) + operators = {"id": ["1", "2"], "type": ["identity operation", "point symmetry operation"]} + for i in range(3): + for j in range(3): + operators[f"matrix[{i + 1}][{j + 1}]"] = [str(rotation[i, j]) for rotation, _ in _OPERATORS] + operators[f"vector[{i + 1}]"] = [str(translation[i]) for _, translation in _OPERATORS] + cif.block["pdbx_struct_oper_list"] = CIFCategory(operators) + cif.block["pdbx_struct_assembly_gen"] = CIFCategory( + {"assembly_id": ["1"], "oper_expression": ["1,2"], "asym_id_list": ["A,B,C,D"]}) + path = folder / "1acb.cif" + cif.write(str(path)) + return path + + +def _two_copies_pdb(folder: Path, renumber: bool = False) -> Path: + """The same assembly in PDB format: 1acb's asymmetric unit with author fields (chains E and I, each with its + waters), as biotite writes it, after REMARK 300 and 350 records applying _OPERATORS to E and I. With renumber, E's + residues 1-146 are numbered 1001-1146, so its numbering decreases and chain_iter cuts it in two pieces, of 144 and + 97 residues, with one chain ID.""" + atoms = get_structure(CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")), model=1, use_author_fields=True) + if renumber: + atoms.res_id[(atoms.chain_id == "E") & ~atoms.hetero & (atoms.res_id <= 146)] += 1000 + remarks = ["REMARK 300", "REMARK 300 BIOMOLECULE: 1", "REMARK 350", "REMARK 350 BIOMOLECULE: 1", + "REMARK 350 APPLY THE FOLLOWING TO CHAINS: E, I"] + for k, (rotation, translation) in enumerate(_OPERATORS): + for i in range(3): + row = "".join(f"{value:10.6f}" for value in rotation[i]) + remarks.append(f"REMARK 350 BIOMT{i + 1} {k + 1:3d}{row}{translation[i]:15.5f}") + pdb = PDBFile() + pdb.set_structure(atoms) + text = io.StringIO() + pdb.write(text) + path = folder / "1acb.pdb" + path.write_text("\n".join(remarks) + "\n" + text.getvalue()) + return path + + +def _asymmetric_unit(path: Path, structure_format: StructureFormat = StructureFormat.mmcif) -> dict: + """The chain-level reader's ESMProteins of the asymmetric unit, by chain ID.""" + return {name.split(".")[-1]: protein for protein, name in EsmProtFromStructure( + src_stream=[("1acb", str(path), "1acb")], src_location=SrcLocation.stream, structure_format=structure_format)} + + +def _copy(protein, operator: int) -> tuple: + """A copy as the assembly should hold it: an asymmetric-unit chain, moved by one of _OPERATORS.""" + return protein.sequence, protein.coordinates, operator + + +def _chains(structure) -> list: + return [str(chain) for chain in struc.get_chains(structure)] + + +def _embedding(chain: str) -> torch.Tensor: + return torch.load(_EMBEDDINGS / f"1acb.{chain}.pt", map_location="cpu") + + +class StructureProviderTest(unittest.TestCase): + + def setUp(self): + self.cif = _two_assemblies(Path(self.enterContext(tempfile.TemporaryDirectory()))) + + def test_assemblies_of_one_entry(self): + provider = StructureProvider() + for assembly_id, chains in (("1", _ALL), ("2", ["B"]), ("1", _ALL)): + with self.subTest(assembly_id=assembly_id): + self.assertEqual(_chains(provider.get_structure("1acb", str(self.cif), assembly_id=assembly_id)), + chains) + self.assertEqual(_chains(get_structure_from_src(str(self.cif), assembly_id=assembly_id)), chains) + + def test_the_same_request_reads_the_file_once(self): + provider = StructureProvider() + with mock.patch.object(structure_provider, "get_structure_from_src", wraps=get_structure_from_src) as read: + provider.get_structure("1acb", str(self.cif), assembly_id="1") + again = provider.get_structure("1acb", str(self.cif), assembly_id="1") + self.assertEqual(read.call_count, 1) + self.assertEqual(_chains(again), _ALL) + provider.get_structure("1acb", str(self.cif), assembly_id="2") + provider.get_structure("1acb", str(self.cif), assembly_id="1") + self.assertEqual(read.call_count, 3) + + def test_chain_id_is_gone(self): + """chain_id, which selected a chain's atoms, is gone: a call that passes it fails, by name or by position + (assembly_id, which took its place, is keyword-only, so a chain ID cannot land in it).""" + provider = StructureProvider() + for call in (lambda: get_structure_from_src(str(self.cif), "mmcif", chain_id="B"), + lambda: provider.get_structure("1acb", str(self.cif), "mmcif", chain_id="B")): + with self.assertRaisesRegex(TypeError, "unexpected keyword argument 'chain_id'"): + call() + with self.assertRaisesRegex(TypeError, "takes from 1 to 2 positional arguments but 3 were given"): + get_structure_from_src(str(self.cif), "mmcif", "B") + with self.assertRaisesRegex(TypeError, "takes from 3 to 4 positional arguments but 5 were given"): + provider.get_structure("1acb", str(self.cif), "mmcif", "B") + + def test_a_stream_is_read_from_its_start(self): + """A downloaded file is a stream: its assemblies are listed, then each assembly is read from it (and the + listing works on a stream already read).""" + stream = io.StringIO(self.cif.read_text()) + self.assertEqual(_chains(get_structure_from_src(stream)), _ALL) + assemblies = get_assemblies(stream) + self.assertEqual([str(a) for a in assemblies], ["1", "2"]) + provider = StructureProvider() + self.assertEqual([_chains(provider.get_structure("1acb", stream, assembly_id=a)) for a in assemblies], + [_ALL, ["B"]]) + + def test_each_dataset_has_its_own_provider(self): + """The default was one StructureProvider per dataset class, built when the class was defined and shared by + every instance of that class.""" + for cls in (EsmAssemblyFromStructure, ResidueAssemblyDatasetFromStructure, + ResidueAssemblyEmbeddingFromTensorFile, ResidueAssemblyEmbeddingFromParquet, EsmProtFromStructure, + ResidueEmbeddingFromStructure, EsmProtFromChain): + with self.subTest(dataset=cls.__name__): + self.assertIsNone(inspect.signature(cls.__init__).parameters["structure_provider"].default) + + +class AssemblyDatasetTest(unittest.TestCase): + """Each assembly dataset on the two-assembly entry: assembly 2 holds chain B alone, read after assembly 1.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.cif = _two_assemblies(self.tmp) + self.expected = {"1acb-1": torch.cat([_embedding("A"), _embedding("B")]), "1acb-2": _embedding("B")} + + def assert_assemblies(self, dataset): + items = {name: tensor for tensor, name in (dataset[i] for i in range(len(dataset)))} + self.assertEqual(sorted(items), sorted(self.expected)) + for name, tensor in self.expected.items(): + with self.subTest(item=name): + self.assertTrue(torch.equal(items[name].float(), tensor.float()), (name, tuple(items[name].shape))) + + def test_from_tensor_files(self): + self.assert_assemblies(ResidueAssemblyEmbeddingFromTensorFile( + src_stream=[("1acb", str(self.cif), "1", "1acb-1"), ("1acb", str(self.cif), "2", "1acb-2")], + res_embedding_location=_EMBEDDINGS, src_location=SrcLocation.stream)) + + def test_every_assembly_of_a_structure(self): + self.assert_assemblies(ResidueAssemblyDatasetFromStructure( + src_stream=[("1acb", str(self.cif), "1acb")], res_embedding_location=_EMBEDDINGS, + src_location=SrcLocation.stream)) + + def test_from_parquet(self): + ids = ["1acb.A", "1acb.B"] + embeddings = [_embedding(i[-1]).numpy().tolist() for i in ids] # one list of D values per residue + kind = pa.list_(pa.list_(pa.float32(), _embedding("A").shape[1])) + parquet = self.tmp / "residues.parquet" + pq.write_table(pa.table({"id": ids, "embedding": embeddings}, + schema=pa.schema([("id", pa.string()), ("embedding", kind)])), parquet) + self.assert_assemblies(ResidueAssemblyEmbeddingFromParquet( + src_stream=[("1acb", str(self.cif), "1", "1acb-1"), ("1acb", str(self.cif), "2", "1acb-2")], + parquet_path=parquet, src_location=SrcLocation.stream)) + + def test_esm_proteins_of_every_assembly(self): + """The ESM3 inputs of each assembly (fm-search build structures --granularity assembly, fm-embedding + from-structures assembly), from a file and from a download.""" + url = "https://example.org/1acb.cif" + for source in (str(self.cif), url): + with self.subTest(source=source), mock.patch.object( + esm_assembly_from_structure, "stringio_from_url", + side_effect=lambda u: io.StringIO(self.cif.read_text())) as download: + items = dict((name, proteins) for proteins, name in EsmAssemblyFromStructure( + src_stream=[("1acb", source, "1acb")], src_location=SrcLocation.stream)) + self.assertEqual(download.call_count, int(source == url)) + self.assertEqual(sorted(items), ["1acb-1", "1acb-2"]) + self.assertEqual(len(items["1acb-1"]), 2) + self.assertEqual([p.sequence for p in items["1acb-2"]], [items["1acb-1"][1].sequence]) + self.assertNotEqual(items["1acb-1"][0].sequence, items["1acb-1"][1].sequence) + + def test_an_assembly_without_protein_chains_is_skipped(self): + """The water-only assembly: skipped with a warning (before, it got assembly 1's chains; a first assembly + without protein chains stopped the build).""" + folder = self.tmp / "water" + folder.mkdir() + cif = _two_assemblies(folder, water=True) + with self.assertLogs(esm_assembly_from_structure.logger, level="WARNING") as logs: + items = [name for _, name in EsmAssemblyFromStructure(src_stream=[("1acb", str(cif), "1acb")], + src_location=SrcLocation.stream)] + self.assertEqual(items, ["1acb-1", "1acb-2"]) + self.assertEqual(len(logs.records), 1) + self.assertIn("1acb-3 has no protein chain", logs.output[0]) + + +class AssemblyCopiesTest(unittest.TestCase): + """EsmAssemblyFromStructure on the two-copy assembly: each copy is read from its own atoms, its chain's in the + asymmetric unit moved by its operator (before, every copy got the first copy's); and the stored-embedding + datasets, which repeat each chain's stored residue embeddings for each of its copies.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.cif = _two_copies(self.tmp) + self.chains = _asymmetric_unit(self.cif) + + def proteins(self, source: str, structure_format: StructureFormat = StructureFormat.mmcif, + min_res_n: int = 0) -> list: + items = list(EsmAssemblyFromStructure(src_stream=[("1acb", source, "1acb")], src_location=SrcLocation.stream, + structure_format=structure_format, min_res_n=min_res_n)) + self.assertEqual([name for _, name in items], ["1acb-1"]) + return items[0][0] + + def assert_copies(self, proteins, copies): + """The ESMProteins are these _copy()s, in order: their atom37 coordinates (NaN where an atom is missing) are + the operator applied here, and the identity's copies are bit for bit the asymmetric unit's.""" + self.assertEqual([protein.sequence for protein in proteins], [sequence for sequence, _, _ in copies]) + for k, (protein, (_, coordinates, operator)) in enumerate(zip(proteins, copies)): + with self.subTest(item=k): + if operator == 0: + self.assertTrue(np.array_equal(protein.coordinates.numpy(), coordinates.numpy(), equal_nan=True)) + rotation, translation = _OPERATORS[operator] + np.testing.assert_allclose(protein.coordinates.numpy(), + coordinates.numpy().astype(np.float64) @ rotation.T + translation, + rtol=0, atol=1e-4) + + def test_copies_in_order(self): + """A, B, A, B, with their chains' sequences (as before).""" + a, b = self.chains["A"], self.chains["B"] + self.assertEqual([protein.sequence for protein in self.proteins(str(self.cif))], [a.sequence, b.sequence] * 2) + + def test_first_copies_are_the_chains_of_the_asymmetric_unit(self): + """The identity's copies are bit for bit the chain-level reader's ESMProteins (as before).""" + for protein, chain in zip(self.proteins(str(self.cif)), "AB"): + with self.subTest(chain=chain): + self.assertEqual(protein.sequence, self.chains[chain].sequence) + self.assertTrue(np.array_equal(protein.coordinates.numpy(), self.chains[chain].coordinates.numpy(), + equal_nan=True)) + + def test_each_copy_from_its_own_atoms(self): + """From a file and from a download.""" + a, b = self.chains["A"], self.chains["B"] + url = "https://example.org/1acb.cif" + for source in (str(self.cif), url): + with self.subTest(source=source), mock.patch.object( + esm_assembly_from_structure, "stringio_from_url", + side_effect=lambda u: io.StringIO(self.cif.read_text())): + self.assert_copies(self.proteins(source), [_copy(a, 0), _copy(b, 0), _copy(a, 1), _copy(b, 1)]) + + def test_min_res_n_between_the_chain_lengths(self): + """min_res_n 64, above B's 63 residues and below A's 241: the two copies of A, in order.""" + a = self.chains["A"] + self.assert_copies(self.proteins(str(self.cif), min_res_n=64), [_copy(a, 0), _copy(a, 1)]) + + def test_pdb_format(self): + """The assembly in PDB format, read with StructureFormat.pdb; and with E's numbering decreasing, which cuts E + in two pieces with one chain ID: each copy of E is read whole, its pieces joined, as in the plain file (P9.A3; + before, each piece was read as itself, and before P9.A2 both got the first piece).""" + pdb = _two_copies_pdb(self.tmp) + chains = _asymmetric_unit(pdb, StructureFormat.pdb) + e, i = chains["E"], chains["I"] + renumbered = self.tmp / "renumbered" + renumbered.mkdir() + for path in (pdb, _two_copies_pdb(renumbered, renumber=True)): + with self.subTest(file=str(path.relative_to(self.tmp))): + self.assert_copies(self.proteins(str(path), StructureFormat.pdb), + [_copy(e, 0), _copy(i, 0), _copy(e, 1), _copy(i, 1)]) + + def test_stored_embeddings_of_each_copy(self): + """The stored-embedding datasets (inference assembly-embedding, complete-embedding) repeat each chain's stored + residue embeddings for each of its copies, in the build's order: A, B, A, B (as before).""" + ids = ["1acb.A", "1acb.B"] + embeddings = [_embedding(i[-1]).numpy().tolist() for i in ids] # one list of D values per residue + kind = pa.list_(pa.list_(pa.float32(), _embedding("A").shape[1])) + parquet = self.tmp / "residues.parquet" + pq.write_table(pa.table({"id": ids, "embedding": embeddings}, + schema=pa.schema([("id", pa.string()), ("embedding", kind)])), parquet) + assembly = [("1acb", str(self.cif), "1", "1acb-1")] + expected = torch.cat([_embedding("A"), _embedding("B")] * 2) + for dataset in ( + ResidueAssemblyEmbeddingFromTensorFile(src_stream=assembly, res_embedding_location=_EMBEDDINGS, + src_location=SrcLocation.stream), + ResidueAssemblyEmbeddingFromParquet(src_stream=assembly, parquet_path=parquet, + src_location=SrcLocation.stream), + ResidueAssemblyDatasetFromStructure(src_stream=[("1acb", str(self.cif), "1acb")], + res_embedding_location=_EMBEDDINGS, + src_location=SrcLocation.stream)): + with self.subTest(dataset=type(dataset).__name__): + self.assertEqual(len(dataset), 1) + tensor, name = dataset[0] + self.assertEqual(name, "1acb-1") + self.assertTrue(torch.equal(tensor.float(), expected.float()), tuple(tensor.shape)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_batch_writer.py b/tests/test_batch_writer.py new file mode 100644 index 0000000..8d5b294 --- /dev/null +++ b/tests/test_batch_writer.py @@ -0,0 +1,106 @@ +import os +import tempfile +import unittest +from types import SimpleNamespace + +import pandas as pd +import pyarrow.parquet as pq +import torch + +from foldmatch.writer.batch_writer import ( + CoreBatchWriter, + CsvBatchWriter, + JsonStorage, + ParquetBatchWriter, + TensorBatchWriter, +) + +_EMBEDDINGS = [torch.arange(4, dtype=torch.float32), torch.arange(4, 8, dtype=torch.float32)] +_IDS = ("a", "b") +# A module with a residue head returns per-item payloads after (embeddings, ids); +# the plain writers must store exactly what a 2-tuple would have given them. +_PREDICTIONS = { + 2: (_EMBEDDINGS, _IDS), + 3: (_EMBEDDINGS, _IDS, [b"\x00" * 8, b"\x01" * 8]), + 4: (_EMBEDDINGS, _IDS, [b"\x00" * 8, b"\x01" * 8], [b"\x02" * 20, b"\x03" * 20]), +} +_TRAINER = SimpleNamespace(global_rank=0) + + +def _write(writer, prediction): + writer.write_on_batch_end(_TRAINER, None, prediction, None, None, 0, 0) + + +class _RecordingWriter(CoreBatchWriter): + def __init__(self): + super().__init__(output_path=None, postfix="none") + self.written = [] + + def _write_embedding(self, embedding, dom_id): + self.written.append((dom_id, embedding.tolist())) + + +class CoreBatchWriterTupleTest(unittest.TestCase): + """CoreBatchWriter (and every writer built on it) accepts 2-, 3- and 4-tuples.""" + + def test_core_writer_3_tuple(self): + writer = _RecordingWriter() + _write(writer, _PREDICTIONS[3]) + self.assertEqual(writer.written, [("a", [0.0, 1.0, 2.0, 3.0]), ("b", [4.0, 5.0, 6.0, 7.0])]) + + def test_core_writer_tuple_sizes_write_the_same(self): + expected = None + for size, prediction in _PREDICTIONS.items(): + with self.subTest(size=size): + writer = _RecordingWriter() + _write(writer, prediction) + if expected is None: + expected = writer.written + self.assertEqual(writer.written, expected) + + def test_none_prediction_is_ignored(self): + writer = _RecordingWriter() + _write(writer, None) + self.assertEqual(writer.written, []) + + def test_file_writers_accept_3_and_4_tuples(self): + for size in (3, 4): + with self.subTest(size=size), tempfile.TemporaryDirectory() as tmp: + _write(CsvBatchWriter(tmp), _PREDICTIONS[size]) + _write(TensorBatchWriter(tmp), _PREDICTIONS[size]) + for dom_id, emb in zip(_IDS, _EMBEDDINGS): + csv = pd.read_csv(os.path.join(tmp, f"{dom_id}.csv"), header=None) + self.assertEqual(csv[0].tolist(), emb.tolist()) + self.assertTrue(torch.equal(torch.load(os.path.join(tmp, f"{dom_id}.pt")), emb)) + + def test_json_storage_accepts_3_and_4_tuples(self): + for size in (3, 4): + with self.subTest(size=size), tempfile.TemporaryDirectory() as tmp: + writer = JsonStorage(tmp, "run") + _write(writer, _PREDICTIONS[size]) + writer.on_predict_end(_TRAINER, None) + df = pd.read_json(os.path.join(tmp, "run-0.json.gz"), orient="records") + self.assertEqual(df["id"].tolist(), list(_IDS)) + + +class ParquetBatchWriterTupleTest(unittest.TestCase): + """The Parquet writer keeps its id + embedding schema whatever the tuple size.""" + + def _table(self, prediction): + with tempfile.TemporaryDirectory() as tmp: + writer = ParquetBatchWriter(tmp, "run") + _write(writer, prediction) + writer.on_predict_end(_TRAINER, None) + return pq.read_table(os.path.join(tmp, "run-0.parquet")) + + def test_tuple_sizes_write_identical_tables(self): + reference = self._table(_PREDICTIONS[2]) + self.assertEqual(reference.column_names, ["id", "embedding"]) + self.assertEqual(reference.column("id").to_pylist(), list(_IDS)) + for size in (3, 4): + with self.subTest(size=size): + self.assertTrue(self._table(_PREDICTIONS[size]).equals(reference)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_chain_pieces.py b/tests/test_chain_pieces.py new file mode 100644 index 0000000..aa1a023 --- /dev/null +++ b/tests/test_chain_pieces.py @@ -0,0 +1,696 @@ +"""P9.A3: a chain that one chain ID cuts into pieces is read whole, and once, on every path. + +In a PDB file (and in an mmCIF or BinaryCIF file not laid out as the archive's) one chain ID can cover several of +biotite's chain_iter pieces: where the chain's residue numbers decrease (a tag numbered 998-999 before 1, an insert +numbered from 1001), and where records of other chains lie between its records (its free amino-acid HETATM ligands +listed after every chain, say). The database (EsmProtFromStructure, EsmProtFromChain) embedded the first piece of the +atoms with the chain's ID, once for each piece listed; the query (FoldMatch.residue_embedding_by_chain) embedded every +piece and kept the last; FoldMatch.residue_embedding and the assembly paths embedded the pieces apart. Every path now +reads the protein chains of structure_parser.protein_chain_iter: for each chain ID (in an assembly, each copy), the +amino-acid atoms of its pieces joined in file order, without the pieces whose amino acids are all HETATM records when +it has amino acids in ATOM records, kept if their polymer residues reach the threshold together. + +The fixtures are 1acb with author fields (chain E, 241 residues numbered 1-245, and chain I, 63 numbered 8-70, each +with its waters), edited, and written as PDB text (with REMARK 350 for assemblies) or as mmCIF by biotite. ESM3 is +never loaded: the paths' ESMProteins are compared, and FoldMatch embeds through a stub.""" +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import biotite.structure as struc +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import torch +from biotite.structure.io.pdb import PDBFile +from biotite.structure.io.pdbx import CIFCategory, CIFFile, get_structure, set_structure +from esm.sdk.api import ESMProtein +from esm.utils.structure.protein_chain import ProteinChain + +from foldmatch import FoldMatch +from foldmatch.dataset.esm_assembly_from_structure import EsmAssemblyFromStructure +from foldmatch.dataset.esm_prot_from_chain import EsmProtFromChain +from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure +from foldmatch.dataset.residue_assembly_embedding_from_parquet import ResidueAssemblyEmbeddingFromParquet +from foldmatch.dataset.residue_assembly_embedding_from_tensor_file import ResidueAssemblyEmbeddingFromTensorFile +from foldmatch.dataset.residue_embedding_from_structure import ResidueEmbeddingFromStructure +from foldmatch.search.embedding_computer import compute_from_structure_file +from foldmatch.types.api_types import Accelerator, Granularity, SrcLocation, StructureFormat +from foldmatch.utils import structure_parser +from foldmatch.utils.structure_parser import filter_residues, get_protein_chains, get_structure_from_src, \ + protein_chain_iter, rename_atom_attr + +_RESOURCES = Path(__file__).parent / "resources" +_PDB, _MMCIF = StructureFormat.pdb, StructureFormat.mmcif + +_ATOMS = get_structure(CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")), model=1, use_author_fields=True) +_E = _ATOMS[(_ATOMS.chain_id == "E") & ~_ATOMS.hetero] +_I = _ATOMS[(_ATOMS.chain_id == "I") & ~_ATOMS.hetero] +_WATER = _ATOMS[_ATOMS.hetero] + +# The operators of the assemblies, as a rotation and a translation: the identity, and a 2-fold about z then 40 Å +# along x. +_OPERATORS = ((np.eye(3), np.zeros(3)), (np.diag([-1.0, -1.0, 1.0]), np.array([40.0, 0.0, 0.0]))) + + +def _residues(atoms) -> list: + starts = list(struc.get_residue_starts(atoms)) + [len(atoms)] + return [atoms[a:b] for a, b in zip(starts[:-1], starts[1:])] + + +def _renumbered(atoms, first: int, last: int, shift: int): + """A copy of the atoms, with the residues numbered first to last numbered shift more.""" + atoms = atoms.copy() + atoms.res_id[(atoms.res_id >= first) & (atoms.res_id <= last)] += shift + return atoms + + +def _hetero(residues, chain_id: str, res_id: int = None, shift: float = 40.0): + """A copy of the residues as HETATM records with this chain ID, moved away (and numbered res_id, res_id + 1, + ...).""" + copies = [] + for k, residue in enumerate(residues): + residue = residue.copy() + residue.chain_id[:] = chain_id + residue.hetero[:] = True + if res_id is not None: + residue.res_id[:] = res_id + k + residue.coord += shift + copies.append(residue) + return struc.concatenate(copies) + + +def _glu(k: int): + return [residue for residue in _residues(_E) if residue.res_name[0] == "GLU"][k] + + +# The asymmetric units, by name: 1acb's E, I and waters, edited, each with all of E and I, as the plain one. +_FIXTURES = { + "plain": lambda: _E + _I + _WATER, + # E's first two residues, 1 and 2, numbered 998 and 999: a tag before the chain's numbering + "tag": lambda: _renumbered(_E, 1, 2, 997) + _I + _WATER, + # E's residues 50-80 numbered 1050-1080: an insert numbered above 1000, then back + "insert": lambda: _renumbered(_E, 50, 80, 1000) + _I + _WATER, + # E's records in two blocks, I's between them, the numbering rising throughout + "blocks": lambda: _E[_E.res_id <= 100] + _I + _E[_E.res_id > 100] + _WATER, + # free amino-acid HETATM groups with chain ID E listed after I: GLU 500, and GLU 600 and 601, consecutive + "ligands": lambda: _E + _I + _hetero([_glu(0)], "E", 500) + _hetero([_glu(0), _glu(1)], "E", 600) + _WATER, + # a free amino acid with chain ID I listed before E + "ligand first": lambda: _hetero([_glu(0)], "I", 500) + _E + _I + _WATER, +} + + +def _write_pdb(atoms, path: Path, remarks=()) -> Path: + pdb = PDBFile() + pdb.set_structure(atoms) + pdb.write(str(path)) + if remarks: + path.write_text("".join(f"{line}\n" for line in remarks) + path.read_text()) + return path + + +def _remarks(*lists) -> list: + """REMARK 300 and 350 records of an assembly 1 that applies, for each (chain IDs, operators) list, those of + _OPERATORS to those chains.""" + remarks = ["REMARK 300", "REMARK 300 BIOMOLECULE: 1", "REMARK 350", "REMARK 350 BIOMOLECULE: 1"] + for chains, operators in lists: + remarks.append(f"REMARK 350 APPLY THE FOLLOWING TO CHAINS: {', '.join(chains)}") + for k in operators: + rotation, translation = _OPERATORS[k] + for i in range(3): + row = "".join(f"{value:10.6f}" for value in rotation[i]) + remarks.append(f"REMARK 350 BIOMT{i + 1} {k + 1:3d}{row}{translation[i]:15.5f}") + return remarks + + +def _assembly(cif: CIFFile, asym_ids: str): + """Assembly 1 of the file: each of _OPERATORS applied to the asym IDs.""" + operators = {"id": ["1", "2"], "type": ["identity operation", "point symmetry operation"]} + for i in range(3): + for j in range(3): + operators[f"matrix[{i + 1}][{j + 1}]"] = [str(rotation[i, j]) for rotation, _ in _OPERATORS] + operators[f"vector[{i + 1}]"] = [str(translation[i]) for _, translation in _OPERATORS] + cif.block["pdbx_struct_assembly"] = CIFCategory({ + "id": ["1"], "details": ["author_and_software_defined_assembly"], "method_details": ["?"], + "oligomeric_details": ["?"], "oligomeric_count": ["?"]}) + cif.block["pdbx_struct_oper_list"] = CIFCategory(operators) + cif.block["pdbx_struct_assembly_gen"] = CIFCategory( + {"assembly_id": ["1"], "oper_expression": ["1,2"], "asym_id_list": [asym_ids]}) + + +def _write_cif(atoms, path: Path, asym_ids: str = None) -> Path: + """The atoms as biotite writes them in mmCIF (an asym ID per chain ID), with _assembly(asym_ids).""" + cif = CIFFile() + set_structure(cif, atoms) + if asym_ids is not None: + _assembly(cif, asym_ids) + cif.write(str(path)) + return path + + +def _protein(atoms) -> ESMProtein: + """The ESMProtein of a chain's atoms, processed as every path processes a chain.""" + return ESMProtein.from_protein_chain(ProteinChain.from_atomarray(rename_atom_attr(filter_residues(atoms)))) + + +class _Esm3: + """Stands in for ESM3 in FoldMatch: keeps the ESMProteins it is given, and returns L + 2 rows (BOS and EOS + included), as per_residue_embedding does.""" + + def __init__(self): + self.proteins = [] + + def __call__(self, model, protein): + self.proteins.append(protein) + return torch.zeros(len(protein) + 2, 4) + + +class _Paths(unittest.TestCase): + """The chain-level and assembly paths on a file, FoldMatch's ESM3 and aggregator stubbed.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.esm3 = _Esm3() + self.enterContext(mock.patch("foldmatch.foldmatch._load_res_model", return_value="ESM3")) + self.enterContext(mock.patch("foldmatch.foldmatch._load_model", return_value=lambda x: x.mean(dim=0))) + self.enterContext(mock.patch("foldmatch.foldmatch.per_residue_embedding", self.esm3)) + + def pdb(self, name: str, *lists) -> Path: + """One of _FIXTURES as a PDB file, with an assembly of these _remarks lists if given.""" + return _write_pdb(_FIXTURES[name](), self.tmp / f"{name.replace(' ', '_')}.pdb", + _remarks(*lists) if lists else ()) + + def build(self, path: Path, structure_format: StructureFormat, min_res_n: int = 0) -> list: + """EsmProtFromStructure (fm-search build structures --granularity chain, fm-embedding from-structures): + (name, ESMProtein) of each chain, in order.""" + return [(name, protein) for protein, name in EsmProtFromStructure( + src_stream=[("x", str(path), "x")], src_location=SrcLocation.stream, structure_format=structure_format, + min_res_n=min_res_n)] + + def chain(self, path: Path, structure_format: StructureFormat, chain_id: str) -> ESMProtein: + """EsmProtFromChain (inference residue-embedding, structure-embedding, complete-embedding).""" + protein, name = EsmProtFromChain(src_stream=[("x", str(path), chain_id, f"x.{chain_id}")], + src_location=SrcLocation.stream, structure_format=structure_format)[0] + self.assertEqual(name, f"x.{chain_id}") + return protein + + def query(self, path: Path, structure_format: StructureFormat, min_res: int = 10, chain_id: str = None) -> dict: + """FoldMatch.residue_embedding_by_chain (fm-search query structure --granularity chain): the ESMProtein of + each chain embedded, by chain ID, in order; each embedded once.""" + self.esm3.proteins.clear() + embeddings = FoldMatch(min_res=min_res).residue_embedding_by_chain(str(path), structure_format, chain_id) + self.assertEqual(len(self.esm3.proteins), len(embeddings), "ESM3 passes") + return {str(c): protein for c, protein in zip(embeddings, self.esm3.proteins)} + + def api(self, path: Path, structure_format: StructureFormat, min_res: int = 10, chain_id: str = None, + assembly_id: str = None) -> list: + """FoldMatch.residue_embedding: the ESMProteins embedded, in order; the rows returned are theirs, BOS and EOS + included, once each.""" + self.esm3.proteins.clear() + rows = FoldMatch(min_res=min_res).residue_embedding(str(path), structure_format, chain_id, assembly_id) + self.assertEqual(rows.shape[0], sum(len(protein) + 2 for protein in self.esm3.proteins)) + return list(self.esm3.proteins) + + def assembly_build(self, path: Path, structure_format: StructureFormat, min_res_n: int = 10) -> list: + """EsmAssemblyFromStructure (fm-search build structures --granularity assembly): assembly 1's ESMProteins.""" + items = list(EsmAssemblyFromStructure(src_stream=[("x", str(path), "x")], src_location=SrcLocation.stream, + structure_format=structure_format, min_res_n=min_res_n)) + self.assertEqual([name for _, name in items], ["x-1"]) + return items[0][0] + + def assembly_query(self, path: Path, structure_format: StructureFormat, min_res: int = 10) -> list: + """FoldMatch.residue_embedding_by_assembly (fm-search query structure --granularity assembly).""" + self.esm3.proteins.clear() + embeddings = FoldMatch(min_res=min_res).residue_embedding_by_assembly(str(path), structure_format, "1") + self.assertEqual(embeddings["1"].shape[0], sum(len(protein) + 2 for protein in self.esm3.proteins)) + return list(self.esm3.proteins) + + def assert_same(self, protein, expected, msg=None): + """The same residues: the sequence, and the atom37 coordinates bit for bit (NaN where an atom is missing).""" + self.assertEqual(protein.sequence, expected.sequence, msg) + self.assertTrue(np.array_equal(protein.coordinates.numpy(), expected.coordinates.numpy(), equal_nan=True), + msg) + + def assert_same_proteins(self, proteins: list, expected: list, msg: str): + self.assertEqual(len(proteins), len(expected), msg) + for k, (protein, other) in enumerate(zip(proteins, expected)): + self.assert_same(protein, other, f"{msg}, chain {k}") + + def assert_paths(self, path: Path, structure_format: StructureFormat, expected: dict, min_res: int = 10): + """Every chain-level path gives these chains, each once, in order, with these residues: the build, the + chain lookup, the query and residue_embedding, of every chain and of each alone.""" + names = [f"x.{c}" for c in expected] + built = self.build(path, structure_format, min_res) + self.assertEqual([name for name, _ in built], names) + for (name, protein), chain in zip(built, expected): + self.assert_same(protein, expected[chain], f"build {name}") + self.assert_same(self.chain(path, structure_format, chain), expected[chain], f"chain lookup {name}") + query = self.query(path, structure_format, min_res, chain) + self.assertEqual(list(query), [chain]) + self.assert_same(query[chain], expected[chain], f"query of {name} alone") + api = self.api(path, structure_format, min_res, chain) + self.assertEqual(len(api), 1) + self.assert_same(api[0], expected[chain], f"residue_embedding of {name} alone") + query = self.query(path, structure_format, min_res) + self.assertEqual(list(query), list(expected)) + api = self.api(path, structure_format, min_res) + self.assertEqual(len(api), len(expected)) + for (chain, protein), from_query, from_api in zip(expected.items(), query.values(), api): + self.assert_same(from_query, protein, f"query {chain}") + self.assert_same(from_api, protein, f"residue_embedding {chain}") + + +class ChainPiecesTest(_Paths): + """The chain-level paths on asymmetric units in PDB format (and one biotite-written mmCIF).""" + + def setUp(self): + super().setUp() + plain = get_structure_from_src(str(self.pdb("plain")), _PDB) + self.plain = {c: _protein(plain[plain.chain_id == c]) for c in ("E", "I")} + + def test_one_chain_per_chain_id_on_every_path(self): + """E cut where its numbering drops (a tag first, an insert in the middle), cut by I's records, or followed + by its free amino-acid ligands after I (or I's before E): E and I once each, in that order, with the + residues of the plain file's E and I (the ligands left out), on every path, at thresholds 0 and 10.""" + for name in ("tag", "insert", "blocks", "ligands", "ligand first"): + path = self.pdb(name) + for min_res in (0, 10): + with self.subTest(fixture=name, min_res=min_res): + self.assert_paths(path, _PDB, self.plain, min_res) + + def test_the_rule_on_the_atoms(self): + """protein_chain_iter yields each chain's amino-acid atoms of ATOM records, in file order (the tag and the + insert keep their numbers), and get_protein_chains lists E and I once.""" + for name in ("plain", "tag", "insert", "blocks", "ligands", "ligand first"): + with self.subTest(fixture=name): + structure = get_structure_from_src(str(self.pdb(name)), _PDB) + chains = list(protein_chain_iter(structure)) + self.assertEqual(get_protein_chains(structure), ("E", "I")) + self.assertEqual(get_protein_chains(structure, 64), ("E",)) + self.assertEqual([chain.chain_id[0] for chain in chains], ["E", "I"]) + for chain in chains: + atoms = structure[(structure.chain_id == chain.chain_id[0]) & ~structure.hetero] + self.assertTrue(np.array_equal(chain.res_id, atoms.res_id)) + self.assertTrue(np.array_equal(chain.coord, atoms.coord)) + + def test_one_chain_id_alone(self): + """With a chain ID, protein_chain_iter reads that ID's pieces alone (the chain lookup's way) and yields the + chains that the whole iteration yields with that ID: one in an asymmetric unit, one per copy in an assembly; + none for an ID without a protein chain.""" + for name, lists in (("plain", ()), ("tag", ()), ("blocks", ()), ("ligands", ()), ("ligand first", ()), + ("insert", ((("E", "I"), (0, 1)),)), ("plain", ((("E", "I"), (0,)), (("E",), (1,))))): + path = self.pdb(name, *lists) + for assembly_id in (None, "1") if lists else (None,): + structure = get_structure_from_src(str(path), _PDB, assembly_id=assembly_id) + chains = list(protein_chain_iter(structure, 10)) + for c in ("E", "I", "Z"): + with self.subTest(fixture=name, assembly_id=assembly_id, chain=c): + alone = list(protein_chain_iter(structure, 10, c)) + same = [chain for chain in chains if chain.chain_id[0] == c] + self.assertEqual(len(alone), len(same)) + for chain, other in zip(alone, same): + self.assertTrue(np.array_equal(chain.res_id, other.res_id)) + self.assertTrue(np.array_equal(chain.coord, other.coord)) + + def test_a_chain_made_only_of_hetatm_amino_acids_is_kept(self): + """Chain P: I's first 12 residues as HETATM records, after I. It is a chain of its own on every path, with + those 12 residues.""" + path = _write_pdb(_E + _I + _hetero(_residues(_I)[:12], "P", shift=60.0) + _WATER, self.tmp / "hetero.pdb") + structure = get_structure_from_src(str(path), _PDB) + expected = dict(self.plain, P=_protein(structure[structure.chain_id == "P"])) + self.assertEqual(len(expected["P"]), 12) + self.assertEqual(expected["P"].sequence, self.plain["I"].sequence[:12]) + self.assert_paths(path, _PDB, expected) + + def test_a_residue_of_a_hetatm_record_inside_a_chain(self): + """E's methionines as selenomethionines (HETATM MSE) inside its ATOM records: kept by every path, read as X + (residue_embedding dropped them, as it did not rename HETATM records to ATOM).""" + atoms = _E.copy() + met = atoms.res_name == "MET" + atoms.res_name[met] = "MSE" + atoms.hetero[met] = True + path = _write_pdb(atoms + _I + _WATER, self.tmp / "mse.pdb") + structure = get_structure_from_src(str(path), _PDB) + expected = dict(self.plain, E=_protein(structure[structure.chain_id == "E"])) + self.assertEqual(len(expected["E"]), len(self.plain["E"])) + self.assertEqual(expected["E"].sequence.count("X"), len(_residues(_E[_E.res_name == "MET"]))) + self.assert_paths(path, _PDB, expected) + + def test_pieces_under_the_threshold_that_reach_it_together(self): + """Chain S as two pieces of 6 residues (I's residues 8-13, then 14-19 numbered 1-6): a chain of 12 residues, + kept at min_res_n and min_res 10 (and 12), not at 13.""" + residues = _residues(_I) + second = struc.concatenate(residues[6:12]).copy() + second.res_id = second.res_id - second.res_id[0] + 1 + pieces = [struc.concatenate(residues[:6]).copy(), second] + for piece in pieces: + piece.chain_id[:] = "S" + piece.coord += 80.0 + path = _write_pdb(_E + _I + pieces[0] + pieces[1] + _WATER, self.tmp / "small.pdb") + structure = get_structure_from_src(str(path), _PDB) + expected = dict(self.plain, S=_protein(structure[structure.chain_id == "S"])) + self.assertEqual(expected["S"].sequence, self.plain["I"].sequence[:12]) + self.assert_paths(path, _PDB, expected) + self.assertEqual(get_protein_chains(structure, 12), ("E", "I", "S")) + self.assertEqual(get_protein_chains(structure, 13), ("E", "I")) + self.assertEqual([name for name, _ in self.build(path, _PDB, 13)], ["x.E", "x.I"]) + self.assertEqual(list(self.query(path, _PDB, 13)), ["E", "I"]) + + def test_free_amino_acids_left_out_are_not_counted(self): + """Chain I cut to its first 9 residues, then E, then two consecutive free GLU of chain I (HETATM 600 and 601, + a piece of their own that filter_polymer counts as 2 polymer residues): left out, they do not count towards + the threshold, so I is a chain of 9 residues on every path at 9 (before, the build read it with the two GLU), + and not a chain at 10.""" + short = struc.concatenate(_residues(_I)[:9]).copy() + path = _write_pdb(short + _E + _hetero([_glu(0), _glu(1)], "I", 600) + _WATER, self.tmp / "short.pdb") + structure = get_structure_from_src(str(path), _PDB) + expected = {"I": _protein(structure[(structure.chain_id == "I") & ~structure.hetero]), "E": self.plain["E"]} + self.assertEqual(len(expected["I"]), 9) + self.assert_paths(path, _PDB, expected, min_res=9) + self.assertEqual(get_protein_chains(structure, 10), ("E",)) + self.assertEqual([name for name, _ in self.build(path, _PDB, 10)], ["x.E"]) + self.assertEqual(list(self.query(path, _PDB, 10)), ["E"]) + + def test_a_lone_amino_acid_ligand_is_not_a_chain(self): + """A free GLU with an asym ID of its own (G) in mmCIF, as the archive gives free amino acids: no chain at + threshold 0 (it has no polymer residue), so the build and the query take A and B only (the query embedded it + before).""" + label = get_structure(CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")), model=1, use_author_fields=False) + glu = _hetero([_glu(0)], "G") + path = _write_cif(label[np.isin(label.chain_id, ["A", "B"])] + glu + label[np.isin(label.chain_id, ["C", "D"])], + self.tmp / "ligand.cif") + structure = get_structure_from_src(str(path), _MMCIF) + self.assertEqual(sorted(set(structure.chain_id.tolist())), ["A", "B", "C", "D", "G"]) + self.assertEqual(get_protein_chains(structure), ("A", "B")) + expected = {c: _protein(structure[structure.chain_id == c]) for c in ("A", "B")} + self.assert_paths(path, _MMCIF, expected, min_res=0) + + +class AssemblyPiecesTest(_Paths): + """The assembly paths: the build (EsmAssemblyFromStructure) and the query (residue_embedding_by_assembly) read + each copy of a chain as one chain, its pieces joined, and take the same residues.""" + + def setUp(self): + super().setUp() + plain = get_structure_from_src(str(self.pdb("plain")), _PDB) + self.plain = {c: _protein(plain[plain.chain_id == c]) for c in ("E", "I")} + + def assert_copies(self, path: Path, structure_format: StructureFormat, chains: list): + """The build's chains have these chains' sequences, in order, the identity's copies their residues bit for + bit; and the query and residue_embedding of the assembly read the same residues.""" + built = self.assembly_build(path, structure_format) + self.assertEqual([protein.sequence for protein in built], [self.plain[c].sequence for c in chains]) + for protein, chain in zip(built, chains[:len(set(chains))]): + self.assert_same(protein, self.plain[chain], f"first copy of {chain}") + self.assert_same_proteins(self.assembly_query(path, structure_format), built, "query") + self.assert_same_proteins(self.api(path, structure_format, assembly_id="1"), built, "residue_embedding") + + def test_one_chain_per_copy(self): + """E with an insert numbered 1050-1080, two copies of E and I: in PDB format (REMARK 350, two BIOMT + operators) and in mmCIF as biotite writes it (asym IDs E and I): E, I, E, I, each E whole (before, the build + and the query read E's two pieces apart in each copy).""" + pdb = self.pdb("insert", (("E", "I"), (0, 1))) + cif = _write_cif(_FIXTURES["insert"](), self.tmp / "insert.cif", "E,I") + for path, structure_format in ((pdb, _PDB), (cif, _MMCIF)): + with self.subTest(file=path.name): + self.assertEqual(get_protein_chains(get_structure_from_src(str(path), structure_format, + assembly_id="1")), ("E", "I", "E", "I")) + self.assert_copies(path, structure_format, ["E", "I", "E", "I"]) + + def test_a_chain_in_two_remark_350_lists(self): + """REMARK 350 applies the identity to E and I, then the 2-fold to E alone. biotite numbers the copies of each + list from 0, so the two copies of E share its sym_id; the reader numbers them apart: two chains, E, I, E (as + before).""" + path = self.pdb("plain", (("E", "I"), (0,)), (("E",), (1,))) + self.assertEqual(get_protein_chains(get_structure_from_src(str(path), _PDB, assembly_id="1")), ("E", "I", "E")) + self.assert_copies(path, _PDB, ["E", "I", "E"]) + + def test_two_lists_with_the_numbers_rising_from_one_copy_into_the_next(self): + """A chain in two REMARK 350 lists of one operator each, whose first amino acid is numbered above its last: E + with its tag (998-999, then 3-245), alone, followed by its waters, or after I in the file; and I after a free + amino acid of its own (GLU 500). biotite gives both copies sym_id 0, and as the numbers rise from the first copy + into the second, chain_iter runs the two together: each copy is still a chain of its own, with the residues of + the plain file's chain (before, chain_iter's pieces were read apart, the first piece of the second copy run into + the last piece of the first).""" + tag = _renumbered(_E, 1, 2, 997) + for name, atoms, lists, chains in ( + ("tag", tag + _I, ((("E",), (0,)), (("E",), (1,))), ["E", "E"]), + ("tag with waters", tag + _I + _WATER[_WATER.chain_id == "E"], ((("E",), (0,)), (("E",), (1,))), + ["E", "E"]), + ("I first", _I + tag, ((("I", "E"), (0,)), (("E",), (1,))), ["I", "E", "E"]), + ("ligand first", _hetero([_glu(0)], "I", 500) + _E + _I, ((("I",), (0,)), (("I",), (1,))), + ["I", "I"])): + with self.subTest(fixture=name): + path = _write_pdb(atoms, self.tmp / f"{name.replace(' ', '_')}_two_lists.pdb", _remarks(*lists)) + assembly = get_structure_from_src(str(path), _PDB, assembly_id="1") + self.assertEqual(get_protein_chains(assembly), tuple(chains)) + self.assertEqual([struc.get_residue_count(chain) for chain in protein_chain_iter(assembly)], + [{"E": 241, "I": 63}[c] for c in chains]) + self.assert_copies(path, _PDB, chains) + + def test_each_copy_of_a_chain_has_a_sym_id_of_its_own(self): + """A PDB assembly as the reader gives it: biotite's, but for the sym_ids. With E and I in one REMARK 350 list + and E alone in another, each of one operator, biotite gives every atom sym_id 0, and the reader gives the + second copy of E sym_id 1, all its atoms; with one list of two operators, the sym_ids are biotite's (0, then + 1). The asymmetric unit has none, and the index in the asymmetric unit that numbers the copies is not kept.""" + two_lists, one_list = ((("E", "I"), (0,)), (("E",), (1,))), ((("E", "I"), (0, 1)),) + for lists, biotite_sym_ids in ((two_lists, [0, 0]), (one_list, [0, 1])): + path = self.pdb("tag", *lists) + with self.subTest(lists=len(lists)): + assembly = get_structure_from_src(str(path), _PDB, assembly_id="1") + biotite = PDBFile.read(str(path)).get_assembly("1", model=1) + self.assertEqual(set(assembly.get_annotation_categories()), set(biotite.get_annotation_categories())) + for category in biotite.get_annotation_categories(): + if category != "sym_id": + self.assertTrue(np.array_equal(assembly.get_annotation(category), + biotite.get_annotation(category)), category) + self.assertTrue(np.array_equal(assembly.coord, biotite.coord)) + au = get_structure_from_src(str(path), _PDB) + self.assertFalse({"sym_id", "au_index"} & set(au.get_annotation_categories())) + # The atoms of each copy: E and I, then E alone (two lists), or E and I twice (one list) + sizes = [np.isin(au.chain_id, chains).sum() for chains, operators in lists for _ in operators] + self.assertTrue(np.array_equal(assembly.sym_id, np.repeat([0, 1], sizes))) + self.assertTrue(np.array_equal(biotite.sym_id, np.repeat(biotite_sym_ids, sizes))) + + def test_the_pinned_biotite_keeps_the_index_in_the_assembly(self): + """The reader numbers the copies of a PDB assembly by each atom's index in the asymmetric unit, which its PDB + file class adds in get_structure, and which biotite's get_assembly (biotite 1.7.1, the release's pin) keeps: + the assembly is the asymmetric unit's atoms of each REMARK 350 list, in file order, once for each operator, + and biotite numbers the copies of each list from 0. A biotite that builds assemblies otherwise fails here.""" + path = self.pdb("tag", (("E", "I"), (0, 1)), (("E",), (1,))) + au = PDBFile.read(str(path)).get_structure(model=1) + e_i, e = np.flatnonzero(np.isin(au.chain_id, ["E", "I"])), np.flatnonzero(au.chain_id == "E") + assembly = structure_parser._IndexedPDBFile.read(str(path)).get_assembly("1", model=1) + self.assertTrue(np.array_equal(assembly.au_index, np.concatenate([e_i, e_i, e]))) + self.assertTrue(np.array_equal(assembly.sym_id, np.repeat([0, 1, 0], [len(e_i), len(e_i), len(e)]))) + self.assertTrue(np.array_equal(get_structure_from_src(str(path), _PDB, assembly_id="1").sym_id, + np.repeat([0, 1, 2], [len(e_i), len(e_i), len(e)]))) + + def test_a_copy_restarting_at_its_first_residue(self): + """E's second part (from its first cysteine after residue 60) numbered again from 1, as its first part is, so + both start with cysteine 1: the asymmetric unit has one E, and so has each copy of the assembly (before, each + copy of E was read as its two pieces).""" + residues = _residues(_E) + k = next(i for i in range(60, len(residues)) if residues[i].res_name[0] == "CYS") + self.assertEqual((residues[0].res_id[0], residues[0].res_name[0]), (1, "CYS")) + second = struc.concatenate(residues[k:]).copy() + second.res_id = second.res_id - second.res_id[0] + 1 + path = _write_pdb(struc.concatenate(residues[:k]) + second + _I + _WATER, self.tmp / "restart.pdb", + _remarks((("E", "I"), (0, 1)))) + self.assert_paths(path, _PDB, self.plain) + self.assert_copies(path, _PDB, ["E", "I", "E", "I"]) + + def test_a_copy_repeating_its_first_piece(self): + """E followed by a copy of its first 20 residues, numbered from 1 as E is: the asymmetric unit has one E of 261 + residues, and so has each copy of the assembly, in the build, the query and residue_embedding, as each copy has + a sym_id of its own (before, each copy of E was read as its two pieces).""" + repeat = struc.concatenate(_residues(_E)[:20]).copy() + repeat.coord += 60.0 + path = _write_pdb(_E + repeat + _I + _WATER, self.tmp / "repeat.pdb", _remarks((("E", "I"), (0, 1)))) + structure = get_structure_from_src(str(path), _PDB) + expected = dict(self.plain, E=_protein(structure[structure.chain_id == "E"])) + sequence = self.plain["E"].sequence + self.assertEqual(expected["E"].sequence, sequence + sequence[:20]) + self.assert_paths(path, _PDB, expected) + built = self.assembly_build(path, _PDB) + self.assertEqual([protein.sequence for protein in built], + [expected["E"].sequence, self.plain["I"].sequence] * 2) + self.assert_same(built[0], expected["E"], "first copy of E") + self.assert_same_proteins(self.assembly_query(path, _PDB), built, "query") + self.assert_same_proteins(self.api(path, _PDB, assembly_id="1"), built, "residue_embedding") + + def test_a_copy_whose_later_piece_differs_by_a_name_or_an_insertion_code(self): + """The same file, with the first residue of E's later piece made a serine (SG named OG: as many atoms, numbered + alike) or given insertion code A: each copy of E is one chain of 261 residues in the build, the query and + residue_embedding too (before, each copy of E was read as its two pieces).""" + def serine(atoms, first): + atoms.res_name[first] = "SER" + atoms.atom_name[first & (atoms.atom_name == "SG")] = "OG" + atoms.element[first & (atoms.element == "S")] = "O" + + def insertion_code(atoms, first): + atoms.ins_code[first] = "A" + + for edit in (serine, insertion_code): + with self.subTest(edit=edit.__name__): + repeat = struc.concatenate(_residues(_E)[:20]).copy() + repeat.coord += 60.0 + edit(repeat, repeat.res_id == repeat.res_id[0]) + path = _write_pdb(_E + repeat + _I + _WATER, self.tmp / f"{edit.__name__}.pdb", + _remarks((("E", "I"), (0, 1)))) + self.assertEqual(get_protein_chains(get_structure_from_src(str(path), _PDB, assembly_id="1")), + ("E", "I", "E", "I")) + built = self.assembly_build(path, _PDB) + self.assertEqual([len(protein) for protein in built], [261, 63, 261, 63]) + self.assert_same_proteins(self.assembly_query(path, _PDB), built, "query") + self.assert_same_proteins(self.api(path, _PDB, assembly_id="1"), built, "residue_embedding") + + def test_archive_layout_copies_of_an_asym_in_two_rows(self): + """1acb's mmCIF with asym A in two pdbx_struct_assembly_gen rows (the identity on A-D, the 2-fold on A): + A, B, A (as before).""" + cif = CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")) + _assembly(cif, "A,B,C,D") + cif.block["pdbx_struct_assembly_gen"] = CIFCategory( + {"assembly_id": ["1", "1"], "oper_expression": ["1", "2"], "asym_id_list": ["A,B,C,D", "A"]}) + path = self.tmp / "1acb.cif" + cif.write(str(path)) + structure = get_structure_from_src(str(path), _MMCIF) + chains = {c: _protein(structure[structure.chain_id == c]) for c in ("A", "B")} + built = self.assembly_build(path, _MMCIF) + self.assertEqual([protein.sequence for protein in built], [chains[c].sequence for c in "ABA"]) + self.assert_same_proteins(self.assembly_query(path, _MMCIF), built, "query") + + +class ArchiveLayoutTest(_Paths): + """Archive mmCIF, each protein chain one piece: every path reads what it read before, the chain's chain_iter + piece.""" + + def test_every_path_as_before(self): + for name in ("1acb", "2uzi"): + path = _RESOURCES / "pdb" / f"{name}.cif" + structure = get_structure_from_src(str(path), _MMCIF) + expected = {c: _protein(next(struc.chain_iter(structure[structure.chain_id == c]))) + for c in get_protein_chains(structure, 10)} + with self.subTest(file=path.name): + self.assertEqual(list(expected), {"1acb": ["A", "B"], "2uzi": ["A", "B", "C"]}[name]) + self.assert_paths(path, _MMCIF, expected) + if name == "1acb": + self.assert_same_proteins(self.assembly_build(path, _MMCIF), list(expected.values()), "build") + self.assert_same_proteins(self.assembly_query(path, _MMCIF), list(expected.values()), "query") + + +class StoredEmbeddingsTest(_Paths): + """The datasets of stored residue embeddings list each chain once (in an assembly, once per copy), as the build + writes one file per chain: ..pt, and one Parquet row per chain.""" + + def setUp(self): + super().setUp() + self.embeddings = {"E": torch.full((243, 4), 1.0), "I": torch.full((65, 4), 2.0)} + for chain, tensor in self.embeddings.items(): + torch.save(tensor, self.tmp / f"x.{chain}.pt") + + def test_chains(self): + """ResidueEmbeddingFromStructure (chain_inference from structures) on E cut by its tag: x.E and x.I, the + build's names (before, x.E twice).""" + path = self.pdb("tag") + dataset = ResidueEmbeddingFromStructure(src_stream=[("x", str(path), "x")], res_embedding_location=self.tmp, + src_location=SrcLocation.stream, structure_format=_PDB) + items = [dataset[i] for i in range(len(dataset))] + self.assertEqual([name for _, name in items], ["x.E", "x.I"]) + self.assertEqual([name for _, name in items], [name for name, _ in self.build(path, _PDB)]) + for tensor, name in items: + self.assertTrue(torch.equal(tensor, self.embeddings[name[-1]]), name) + + def test_assembly_copies(self): + """The tensor-file and Parquet assembly datasets (inference assembly-embedding, complete-embedding) on two + copies of E (cut by its insert) and I: E, I, E, I (before, E's file twice for each copy).""" + path = self.pdb("insert", (("E", "I"), (0, 1))) + kind = pa.list_(pa.list_(pa.float32(), 4)) + parquet = self.tmp / "residues.parquet" + pq.write_table(pa.table({"id": ["x.E", "x.I"], + "embedding": [self.embeddings[c].numpy().tolist() for c in "EI"]}, + schema=pa.schema([("id", pa.string()), ("embedding", kind)])), parquet) + assembly = [("x", str(path), "1", "x-1")] + expected = torch.cat([self.embeddings[c] for c in "EIEI"]) + for dataset in ( + ResidueAssemblyEmbeddingFromTensorFile(src_stream=assembly, res_embedding_location=self.tmp, + src_location=SrcLocation.stream, structure_format=_PDB), + ResidueAssemblyEmbeddingFromParquet(src_stream=assembly, parquet_path=parquet, + src_location=SrcLocation.stream, structure_format=_PDB)): + with self.subTest(dataset=type(dataset).__name__): + tensor, name = dataset[0] + self.assertEqual(name, "x-1") + self.assertTrue(torch.equal(tensor.float(), expected), tuple(tensor.shape)) + + +class NotAProteinChainTest(_Paths): + """A requested chain ID with no protein chain: water (1acb's C), absent (Z), or a lone free amino acid (G); and a + requested assembly with none.""" + + def setUp(self): + super().setUp() + label = get_structure(CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")), model=1, use_author_fields=False) + self.ligand = _write_cif(label + _hetero([_glu(0)], "G"), self.tmp / "ligand.cif") + self.cases = ((_RESOURCES / "pdb" / "1acb.cif", "C"), (_RESOURCES / "pdb" / "1acb.cif", "Z"), + (self.ligand, "G")) + + def test_chain_lookup(self): + """EsmProtFromChain refuses it with an IOError, printed as OSError, that names the structure (the input's + first column), the chain and the file (it returned the lone amino acid, and failed on the others with other + messages).""" + for path, chain in self.cases: + with self.subTest(chain=chain): + with self.assertRaises(IOError) as refused: + EsmProtFromChain(src_stream=[("1acb", str(path), chain, f"item.{chain}")], + src_location=SrcLocation.stream, structure_format=_MMCIF)[0] + self.assertEqual(type(refused.exception).__name__, "OSError") + self.assertEqual(str(refused.exception), + f"No protein chain found for 1acb.{chain} in structure {path}.") + + def test_query(self): + """fm-search query structure --chain-id: today's message, at --min-res 0 too (the lone amino acid was + embedded).""" + for path, chain in self.cases: + with self.subTest(chain=chain): + with self.assertRaises(ValueError) as refused: + list(compute_from_structure_file(path, _MMCIF, Granularity.chain, chain_id=chain, min_res=0, + accelerator=Accelerator.cpu)) + self.assertEqual(str(refused.exception), + f"Chain {chain} not found or does not meet minimum residue requirements") + + def test_residue_embedding(self): + """FoldMatch.residue_embedding with that chain ID, or with no chain long enough: a ValueError that says so + (before, torch.cat's on an empty list).""" + for path, chain in self.cases: + with self.subTest(chain=chain): + with self.assertRaises(ValueError) as refused: + FoldMatch(min_res=0).residue_embedding(str(path), _MMCIF, chain) + self.assertEqual(str(refused.exception), + f"No protein chain {chain} of at least 0 residues found in the structure") + with self.assertRaises(ValueError) as refused: + FoldMatch(min_res=242).residue_embedding(str(_RESOURCES / "pdb" / "1acb.cif"), _MMCIF) + self.assertEqual(str(refused.exception), "No protein chain of at least 242 residues found in the structure") + + def test_assembly_query(self): + """fm-search query structure --granularity assembly on an assembly with no chain long enough (1acb's, at + --min-res 242) or with no protein chain (two copies of 1acb's waters, at 0): residue_embedding_by_assembly + gives an empty dict, as residue_embedding_by_chain does, so the query refuses the assembly with its own + message, and nothing is embedded (before, torch.cat's error on an empty list).""" + water = CIFFile.read(str(_RESOURCES / "pdb" / "1acb.cif")) + _assembly(water, "C,D") + water.write(str(self.tmp / "water.cif")) + for path, min_res in ((_RESOURCES / "pdb" / "1acb.cif", 242), (self.tmp / "water.cif", 0)): + with self.subTest(file=path.name): + self.esm3.proteins.clear() + self.assertEqual(FoldMatch(min_res=min_res).residue_embedding_by_assembly(str(path), _MMCIF, "1"), {}) + with self.assertRaises(ValueError) as refused: + list(compute_from_structure_file(path, _MMCIF, Granularity.assembly, assembly_id="1", + min_res=min_res, accelerator=Accelerator.cpu)) + self.assertEqual(str(refused.exception), + "Assembly 1 not found or does not meet minimum residue requirements") + self.assertEqual(self.esm3.proteins, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_blosum62_stage2.py b/tests/test_cli_blosum62_stage2.py new file mode 100644 index 0000000..566db53 --- /dev/null +++ b/tests/test_cli_blosum62_stage2.py @@ -0,0 +1,399 @@ +"""P9.S6: the BLOSUM62 Stage 2's checks before Stage 1, in ``fm-search query sequences`` and ``fm-search query db``. + +Whenever Stage 2 runs, both commands refuse, before the ESM3 pass (``query sequences``) or the FAISS search (``query +db``): + +* gap costs the native kernel cannot take (``--gap-open`` below 1, either above 2^20; the sign is ignored), and, + under ``--significance-mode default``, gap costs other than 11/1 when the run needs significance (an ``evalue`` or + ``bits`` column, or a finite ``--max-evalue``): usage errors (exit status 2, D20); +* under ``--significance-mode sampled``, a ``--significance-sample-size`` below 2 when the run needs significance + (typer's ``min=2`` on the command line): a usage error; +* a native kernel that cannot be used: a refusal (exit status 1) with the rebuild command. + +The ESM3 pass is the fake one of test_cli_residue_codes (the shards the real pass writes); the database is built from +the fixture FASTA with it once per class. Tests that run Stage 2 use the fixture ``fmsw_libs``; the refusals do not +need the kernel. +""" +import hashlib +import itertools +import math +import tempfile +import unittest +from contextlib import contextmanager +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest +import typer + +from foldmatch.cli import search +from foldmatch.errors import RefusalError +from foldmatch.search import alignment, output, swint, swkern +from foldmatch.types.api_types import SignificanceMode +from test_cli_residue_codes import _build_args, _esm3_spy, fake_chain_predict, invoke, launch_env + +_TEST_PATH = Path(__file__).parent +_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +MAX_GAP = swint.MAX_GAP + + +def _sha256(path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def _no_pass(): + """The ESM3 pass must not be reached.""" + return mock.patch("foldmatch.search.embedding_computer.chain_predict", + mock.Mock(side_effect=AssertionError("the ESM3 pass was reached"))) + + +def _no_search(): + """FAISS must not load the database (the first step of both commands' search).""" + return mock.patch("foldmatch.search.embedding_database.EmbeddingDatabase.load", + mock.Mock(side_effect=AssertionError("the database was loaded"))) + + +@contextmanager +def _no_kernel(): + """No kernel library can be found, as in an install built without a C compiler.""" + with mock.patch.dict(swkern._KERNELS, clear=True), mock.patch("foldmatch._native.find_library", return_value=None): + yield + + +class _Blosum62Case(unittest.TestCase): + """A database of the fixture FASTA with a sequence store (no residue codes), built once per class with the fake + pass; a temporary folder per test (``self.tmp``, query shards in ``self.qshards``), no launch variables.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._dbs = tempfile.TemporaryDirectory() + dbs = Path(cls._dbs.name) + (dbs / "shards").mkdir() + with launch_env(FOLDMATCH_RUN_ID="build"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + result = invoke(*_build_args(_FASTA, dbs / "db", dbs / "shards")) + assert result.exit_code == 0, (result.output, result.exception) + cls.db = dbs / "db" + + @classmethod + def tearDownClass(cls): + cls._dbs.cleanup() + super().tearDownClass() + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.qshards = self.tmp / "qshards" + self.qshards.mkdir() + self.enterContext(launch_env()) + + def args(self, command, *extra, out=None): + out = out or self.tmp / "out.tsv" + if command == "sequences": + return ("query", "sequences", "--db-path", self.db, "--fasta-file", _FASTA, "--tmp-embedding-folder", + self.qshards, "--output-file", out, "--accelerator", "cpu", "--devices", "1", *extra) + return ("query", "db", "--query-db-path", self.db, "--subject-db-path", self.db, "--output-file", out, *extra) + + def run_query(self, command, *extra, out=None, run_id="q"): + """``query sequences`` (fake pass) or ``query db``, to the end; returns ``(result, output path)``.""" + out = out or self.tmp / "out.tsv" + with launch_env(FOLDMATCH_RUN_ID=run_id), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + return invoke(*self.args(command, *extra, out=out)), out + + def before_stage1(self, command, *extra, rank_env=None): + """``command`` with ``extra``, which must stop before Stage 1: neither ESM3 nor the pass is reached, FAISS + does not load the database, and nothing is written.""" + with launch_env(**(rank_env or {})), _esm3_spy() as esm3, _no_pass() as predict, _no_search() as load: + result = invoke(*self.args(command, *extra)) + esm3.assert_not_called() + predict.assert_not_called() + load.assert_not_called() + self.assertEqual(list(self.qshards.iterdir()), []) + self.assertFalse((self.tmp / "out.tsv").exists()) + return result + + def usage_error(self, result, *texts): + """Exit status 2 (typer.BadParameter) naming every one of ``texts`` (typer's error panel borders dropped).""" + self.assertEqual(result.exit_code, 2, result.output) + flat = " ".join(result.output.translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + for text in texts: + self.assertIn(text, flat) + return flat + + def refused(self, result, cause=None): + """Exit status 1 with a RefusalError (D20); returns its message.""" + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError, repr(result.exception)) + if cause is not None: + self.assertIsInstance(result.exception.__cause__, cause, repr(result.exception.__cause__)) + return str(result.exception) + + +class RefusedBeforeStage1Test(_Blosum62Case): + """Each refusal stops the command before Stage 1, under both commands (and on any rank of ``query sequences``).""" + + COMMANDS = ("sequences", "db") + + def test_gap_costs_outside_the_kernel_range(self): + cases = [ # (options, flag, text) + (("--gap-open", "0"), "--gap-open", "must be between 1 and 1048576 (the sign is ignored)"), + (("--gap-open", "0", "--significance-mode", "sampled", "--max-evalue", "inf"), "--gap-open", "got 0"), + (("--gap-open", str(MAX_GAP + 1), "--significance-mode", "sampled"), "--gap-open", + f"got {MAX_GAP + 1}"), + (("--gap-open", str(-MAX_GAP - 1), "--significance-mode", "sampled"), "--gap-open", + f"got {-MAX_GAP - 1}"), + (("--gap-extend", str(MAX_GAP + 1), "--significance-mode", "sampled"), "--gap-extend", + "must be between 0 and 1048576"), + (("--gap-extend", str(-MAX_GAP - 1), "--max-evalue", "inf", "--format-output", "query,target"), + "--gap-extend", "The default is 1."), + ] + for command in self.COMMANDS: + for extra, flag, text in cases: + with self.subTest(command=command, extra=extra): + self.usage_error(self.before_stage1(command, *extra), flag, text, "Smith-Waterman kernel") + + def test_calibrated_significance_needs_11_1(self): + """Under --significance-mode default, any run that needs significance: the message names what needs it and + the ways out that apply.""" + use = "Use --gap-open 11 --gap-extend 1; or --significance-mode sampled" + cases = [ # (options, texts) + (("--gap-open", "10"), # the default columns hold evalue and bits; the default --max-evalue is 1e-3 + ("the evalue and bits columns of --format-output and the E-value filter --max-evalue 0.001", + "gap costs given are 10/1", use, "or --max-evalue inf and a --format-output without evalue and bits")), + (("--gap-open", "10", "--max-evalue", "inf"), + ("for the evalue and bits columns of --format-output;", use, + "or a --format-output without evalue and bits, to run")), + (("--gap-extend", "2", "--format-output", "query,target,fident"), + ("for the E-value filter --max-evalue 0.001;", "given are 11/2", use, "or --max-evalue inf, to run")), + (("--gap-open", "5", "--gap-extend", "2", "--max-evalue", "inf", "--format-output", "query,bits"), + ("for the bits column of --format-output;", "given are 5/2")), + (("--gap-open", "-10", "--gap-extend", "-1", "--significance-mode", "default"), ("given are 10/1",)), + ] + for command in self.COMMANDS: + for extra, texts in cases: + with self.subTest(command=command, extra=extra): + self.usage_error(self.before_stage1(command, *extra), "'--gap-open' / '--gap-extend'", + "--significance-mode default has calibrated statistics only for the gap costs " + "11/1", *texts) + + def test_a_significance_sample_below_2(self): + """--significance-sample-size of 1 or less gave infinite or NaN sampled statistics, or failed after Stage 1 + (a negative size); the option now has min=2.""" + for command in self.COMMANDS: + for size in ("1", "0", "-5"): + with self.subTest(command=command, size=size): + self.usage_error(self.before_stage1(command, "--significance-mode", "sampled", + "--significance-sample-size", size), + "--significance-sample-size", f"{size} is not in the range x>=2") + + def test_every_rank_refuses_alike(self): + for rank_env in ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}): + with self.subTest(rank_env=rank_env): + self.usage_error(self.before_stage1("sequences", "--gap-open", "0", rank_env=rank_env), "--gap-open") + self.usage_error(self.before_stage1("sequences", "--gap-open", "10", rank_env=rank_env), + "--significance-mode default") + + def test_a_kernel_that_cannot_be_used(self): + """Stage 2 has no other aligner: an install without the kernel is refused before Stage 1, with the rebuild + command, as --scoring esm-codes is.""" + for command, rank_env in (("sequences", {}), ("sequences", {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}), + ("db", {})): + with self.subTest(command=command, rank_env=rank_env): + with _no_kernel(): + message = self.refused(self.before_stage1(command, rank_env=rank_env), swkern.KernelError) + self.assertIn("Stage 2 (sequence identity) needs the native Smith-Waterman kernel", message) + self.assertIn("python -m foldmatch._native.build", message) # the rebuild command + self.assertIn("--no-seq-identity runs an embedding-only search", message) + + def test_the_option_checks_come_first(self): + """A usage error (exit status 2) is reported before a refusal of the installation (exit status 1).""" + for command in self.COMMANDS: + with self.subTest(command=command), _no_kernel(): + self.usage_error(self.before_stage1(command, "--gap-open", "0"), "--gap-open") + + def test_query_db_checks_format_output_before_the_search(self): + """query db now parses --format-output before its search whenever Stage 2 runs, as query sequences does.""" + for extra, text in ((("--format-output", "query,target,not_a_field"), "not_a_field"), + (("--format-output", ","), "Empty --format-output")): + with self.subTest(extra=extra): + self.usage_error(self.before_stage1("db", *extra), text, "--format-output") + + +@pytest.mark.usefixtures("fmsw_libs") +class AcceptedTest(_Blosum62Case): + """The combinations the checks let through run Stage 2 to the end, with the options given.""" + + def run_stage2(self, command, *extra, out=None, run_id="q"): + with mock.patch.object(alignment, "align_candidates", wraps=alignment.align_candidates) as blosum: + result, out = self.run_query(command, *extra, out=out, run_id=run_id) + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + blosum.assert_called_once() + self.assertTrue(out.read_text().splitlines()) + return blosum.call_args.kwargs, out + + def test_other_gap_costs_without_calibrated_significance(self): + cases = [ # (options, (gap_open, gap_extend, significance_mode) given to align_candidates) + (("--gap-open", "5", "--gap-extend", "2", "--significance-mode", "sampled", + "--significance-sample-size", "20"), (5, 2, SignificanceMode.sampled)), + (("--gap-open", "5", "--gap-extend", "2", "--max-evalue", "inf", "--format-output", + "query,target,fident,raw"), (5, 2, SignificanceMode.default)), + (("--gap-open", "1", "--gap-extend", "0", "--max-evalue", "inf", "--format-output", "query,target,raw"), + (1, 0, SignificanceMode.default)), + (("--gap-open", str(MAX_GAP), "--gap-extend", str(MAX_GAP), "--max-evalue", "inf", "--format-output", + "query,target,raw"), (MAX_GAP, MAX_GAP, SignificanceMode.default)), + ] + for command in ("sequences", "db"): + for k, (extra, want) in enumerate(cases): + with self.subTest(command=command, extra=extra): + kwargs, _ = self.run_stage2(command, *extra, out=self.tmp / f"{command}{k}.tsv", run_id=f"q{k}") + self.assertEqual((kwargs["gap_open"], kwargs["gap_extend"], kwargs["significance_mode"]), want) + + def test_the_sign_is_ignored(self): + """Negative gap costs are their absolute values, as before; -11/-1 is 11/1, also for calibrated significance + (P9.S3: it used to be refused there).""" + for command in ("sequences", "db"): + with self.subTest(command=command): + _, plain = self.run_stage2(command, out=self.tmp / f"{command}-plain.tsv", run_id="p") + kwargs, negative = self.run_stage2(command, "--gap-open", "-11", "--gap-extend", "-1", + out=self.tmp / f"{command}-neg.tsv", run_id="n") + self.assertEqual((kwargs["gap_open"], kwargs["gap_extend"]), (-11, -1)) + self.assertEqual(_sha256(negative), _sha256(plain)) + _, a = self.run_stage2(command, "--gap-open", "3", "--gap-extend", "2", "--max-evalue", "inf", + "--format-output", "query,target,raw,cigar", out=self.tmp / f"{command}-a.tsv", + run_id="a") + _, b = self.run_stage2(command, "--gap-open", "-3", "--gap-extend", "-2", "--max-evalue", "inf", + "--format-output", "query,target,raw,cigar", out=self.tmp / f"{command}-b.tsv", + run_id="b") + self.assertEqual(_sha256(a), _sha256(b)) + + def test_an_embedding_only_search_needs_neither(self): + """Without Stage 2 the gap options are unused, as before, and the kernel is not needed.""" + for command in ("sequences", "db"): + with self.subTest(command=command), _no_kernel(), \ + mock.patch.object(alignment, "align_candidates", side_effect=AssertionError("Stage 2")): + result, out = self.run_query(command, "--no-seq-identity", "--gap-open", "0", "--gap-extend", + str(MAX_GAP + 1), out=self.tmp / f"{command}.tsv") + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + self.assertTrue(out.read_text().splitlines()) + + +class PythonFunctionTest(_Blosum62Case): + """``query_database_from_database`` called from Python, where typer parses nothing: the same refusals before the + search (typer.BadParameter), and a whole float gap cost runs as its integer, as it did before P9.S3.""" + + def call(self, name, **kw): + out = self.tmp / f"{name}.tsv" + search.query_database_from_database(query_db_path=str(self.db), subject_db_path=str(self.db), + output_file=str(out), **kw) + return out + + def test_refusals_before_the_search(self): + both = ["--gap-open", "--gap-extend"] + cases = [ # (options, param_hint, text) + (dict(gap_open=10.5), "--gap-open", "must be a whole number, got 10.5"), + (dict(gap_extend=True), "--gap-extend", "must be a whole number, got True"), + (dict(gap_open=0.0), "--gap-open", "must be between 1 and 1048576"), + (dict(significance_mode="bogus"), "--significance-mode", "'bogus' is not a significance mode"), + (dict(gap_open=10.0), both, "the gap costs given are 10/1"), + (dict(gap_open=np.int64(-10), significance_mode="default"), both, "the gap costs given are 10/1"), + (dict(significance_mode="sampled", significance_sample_size=-5), "--significance-sample-size", + "must be at least 2 with --significance-mode sampled, got -5"), + (dict(significance_mode=SignificanceMode.sampled, significance_sample_size=1, max_evalue=math.inf), + "--significance-sample-size", "got 1"), # the default columns hold evalue and bits + ] + for kw, param, text in cases: + with self.subTest(**{k: repr(v) for k, v in kw.items()}), _no_search() as load: + with self.assertRaises(typer.BadParameter) as ctx: + self.call("refused", **kw) + load.assert_not_called() + self.assertEqual(ctx.exception.param_hint, param) + self.assertIn(text, ctx.exception.format_message()) + + @pytest.mark.usefixtures("fmsw_libs") + def test_whole_floats_and_unused_sample_sizes_run(self): + """gap_open=11.0 (the release took it, through biotite) is 11 again, with calibrated significance; a sample + size the run does not use is not checked.""" + ints = self.call("ints", gap_open=11, gap_extend=1) + self.assertTrue(ints.read_text().splitlines()) + floats = self.call("floats", gap_open=11.0, gap_extend=np.float64(1.0)) + self.assertEqual(_sha256(floats), _sha256(ints)) + unused = self.call("unused", significance_sample_size=0) # --significance-mode default + self.assertEqual(_sha256(unused), _sha256(ints)) + + +class GapCostRuleTest(unittest.TestCase): + """The command line refuses exactly the combinations ``align_candidates`` refuses (check_gap_costs), so nothing + it lets through fails after Stage 1.""" + + def test_the_command_line_and_align_candidates_agree(self): + """Over the kernel range, its bounds and both signs, whole and other floats and a bool (what the Python + functions may be given), both modes, the significance columns and E-value filters.""" + gaps = (-MAX_GAP - 1, -MAX_GAP, -11, -10, -2, -1, 0, 1, 2, 10, 11, 12, MAX_GAP, MAX_GAP + 1, 11.0, -1.0, 0.0, + 10.5, np.int64(11), True) + fields = (list(output.DEFAULT_OUTPUT_FIELDS), ["query", "target", "fident"], ["query", "bits"], ["evalue"], + ["evalue", "query", "evalue"]) + max_evalues = (None, math.inf, math.nan, 1e-3, 0.0, 10.0) + refused = 0 + for go, ge, mode, cols, max_evalue in itertools.product(gaps, gaps, SignificanceMode, fields, max_evalues): + try: + alignment.check_gap_costs(go, ge, mode, output.needs_significance(cols), max_evalue) + library = True + except (ValueError, TypeError): + library = False + try: + search._check_blosum62_gap_costs(go, ge, mode, cols, max_evalue) + cli = True + except typer.BadParameter: + cli = False + if cli != library: + self.fail(f"gaps {go}/{ge}, {mode.value}, {cols}, max_evalue={max_evalue}: command line " + f"{'accepts' if cli else 'refuses'}, align_candidates {'accepts' if library else 'refuses'}") + refused += not cli + self.assertGreater(refused, 0) + + def test_each_refusal_names_its_own_cause(self): + """Only the 11/1 rule gets the 11/1 message (a float gap cost, a bool or an unknown mode used to get it, with + empty slots); a refusal names one option.""" + check = search._check_blosum62_gap_costs + for args in ((11.0, 1.0, SignificanceMode.sampled, ["query"], 1e-3), + (11.0, 1, SignificanceMode.default, ["query", "target"], math.inf), + (11.0, 1, SignificanceMode.default, ["query"], None), (-11.0, -1.0, "default", ["evalue"], 1e-3)): + with self.subTest(args=args): + check(*args) # accepted: 11/1, or no significance + cases = [ # (args, param_hint, text) + ((11.5, 1, SignificanceMode.sampled, ["query"], 1e-3), "--gap-open", "must be a whole number, got 11.5"), + ((True, 1, SignificanceMode.default, ["query"], None), "--gap-open", "must be a whole number, got True"), + ((11, "1", SignificanceMode.default, ["query"], None), "--gap-extend", "must be a whole number, got '1'"), + ((10, 1, "bogus", ["query"], None), "--significance-mode", "'bogus' is not a significance mode"), + ((10.0, 1, "default", ["query"], 1e-3), ["--gap-open", "--gap-extend"], + "this run needs them for the E-value filter --max-evalue 0.001; the gap costs given are 10/1"), + ] + for args, param, text in cases: + with self.subTest(args=args), self.assertRaises(typer.BadParameter) as ctx: + check(*args) + self.assertEqual(ctx.exception.param_hint, param) + self.assertIn(text, ctx.exception.format_message()) + self.assertNotIn("needs them for ;", ctx.exception.format_message()) + + def test_the_sample_size_check(self): + check = search._check_significance_sample_size + for args in ((1, SignificanceMode.sampled, ["evalue"], None), (0, "sampled", ["query"], 1e-3), + (-5, SignificanceMode.sampled, ["bits"], math.inf)): + with self.subTest(args=args), self.assertRaisesRegex(typer.BadParameter, "must be at least 2"): + check(*args) + for args in ((2, SignificanceMode.sampled, ["evalue"], 1e-3), (1, SignificanceMode.default, ["evalue"], 1e-3), + (0, SignificanceMode.sampled, ["query"], None), + (-5, SignificanceMode.sampled, ["query"], math.inf)): + with self.subTest(args=args): + check(*args) # 2 or more, or not sampled + + def test_a_repeated_column_is_named_once(self): + with self.assertRaises(typer.BadParameter) as ctx: + search._check_blosum62_gap_costs(10, 1, SignificanceMode.default, ["evalue", "query", "evalue"], None) + self.assertIn("for the evalue column of --format-output;", ctx.exception.format_message()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_esm_codes_query.py b/tests/test_cli_esm_codes_query.py new file mode 100644 index 0000000..b37be5b --- /dev/null +++ b/tests/test_cli_esm_codes_query.py @@ -0,0 +1,1349 @@ +"""P6.1-P6.3: ``fm-search query sequences --scoring blosum62|esm-codes`` (plan section 8 P6; D7, D8, D11, D14, D20). + +Most tests replace the ESM3 pass by a fake that writes the shards the real pass writes (same dataset, same writer), +with the F2c reference codes (builds) and query encodings (queries) of the fixture FASTA: the command-line +orchestration, the refusals, the query-encoding memmap and the real esm-codes Stage 2 (kernel, registry bundle) run +without ESM3. ``RealEsm3SmokeTest`` is the end-to-end smoke of the plan (CPU, ESM3 from the cache) and +``RealEsm3DegenerateTest`` runs the real-ESM3 F-deg items through the command line (D14, S13, S13b). +P7.0a: ``MultiRankOrderTest`` (fake pass on two simulated ranks) and the slow ``TwoRankQuerySmoke`` (real ESM3, two +gloo ranks) check that a multi-rank query writes the bytes of a 1-rank query, under both scorings. +""" +import dataclasses +import gc +import io +import json +import logging +import math +import os +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +import weakref +import zlib +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import pytest +import torch + +import esm_codes_bundle +from foldmatch.cli import search +from foldmatch.errors import RefusalError +from foldmatch.search import code_alignment, output, swkern +from foldmatch.search import scoring_system as ss +from foldmatch.search.residue_code_store import ( + IncompatibleCodeStoreError, ResidueCodeStore, ResidueCodeStoreError, +) +from foldmatch.types.api_types import ScoringChoice, SignificanceMode +from foldmatch.utils.sequence import STANDARD_AMINO_ACIDS +from test_cli_residue_codes import _DDP_ENV, _DDP_ENV_PREFIXES, _LAUNCH_ENV, _SRC, _build_args, _esm3_spy, _f2c, \ + _sha1, _sha256, _write, fake_chain_predict, invoke, launch_env + +_TEST_PATH = Path(__file__).parent +_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_F2C = _TEST_PATH / "resources" / "esm_codes" / "fasta_rows_codes_v1.npz" +_FDEG = _TEST_PATH / "resources" / "esm_codes" / "degenerate_v1.npz" +SYSTEM = ss.get(ss.DEFAULT_ESM_CODES) +DOMAIN = SYSTEM.calibrated_domain +ALL_FIELDS = ",".join(output.SUPPORTED_OUTPUT_FIELDS) +DEFAULT_FIELDS = output.DEFAULT_OUTPUT_FIELDS +#: Numeric output columns (finite whenever they are filled). +NUMERIC_FIELDS = ("embscore", "evalue", "gapopen", "pident", "fident", "nident", "qstart", "qend", "qlen", "tstart", + "tend", "tlen", "alnlen", "raw", "bits", "mismatch", "qcov", "tcov", "rawrev", "q90", "rawadj", + "ecal") +_DB_SUFFIXES = (".index", ".metadata", ".sequences") + + +# ===================================================================================================================== +# helpers +# ===================================================================================================================== +def _f2c_q() -> dict: + """F2c: ``{sequence: reference query encoding fp16 [L, 64]}`` of the fixture FASTA.""" + z = np.load(_F2C, allow_pickle=False) + offsets = z["offsets"] + return {str(s): z["q"][offsets[k]:offsets[k + 1]] for k, s in enumerate(z["sequences"])} + + +def _fake_q(name: str, sequence: str) -> np.ndarray: + """The F2c query encodings of the fixture's sequences; pseudo-random fp16 rows (fixed per id) for any other.""" + reference = _f2c_q().get(sequence) + if reference is not None: + return reference + rng = np.random.default_rng(zlib.crc32(name.encode())) + return rng.normal(size=(len(sequence), 64)).astype(np.float16) + + +def fake_query_pass(calls=None, payload=_fake_q, seq_sha1=None): + """Stand-in for ``chain_complete_inference.predict`` in a query: :func:`fake_chain_predict` with the query + encodings as payloads; ``seq_sha1(name, sequence)``, if given, replaces the seq_sha1 the shard records.""" + if seq_sha1 is None: + return fake_chain_predict(calls, payload=payload) + + def predict(**kwargs): + from foldmatch.writer import batch_writer + real = batch_writer.ParquetBatchWriter.write_on_batch_end + + def write(self, trainer, module, prediction, *args): + embeddings, names, payloads, _ = prediction + sequences = {name: seq for name, seq in _fasta_records(kwargs["src_stream"].path)} + sha1s = tuple(seq_sha1(name, sequences[name]) for name in names) + return real(self, trainer, module, (embeddings, names, payloads, sha1s), *args) + + with mock.patch.object(batch_writer.ParquetBatchWriter, "write_on_batch_end", write): + fake_chain_predict(calls, payload=payload)(**kwargs) + return predict + + +def _fasta_records(path): + from foldmatch.utils.fasta import iter_fasta + from foldmatch.utils.sequence import normalize_sequence + return [(name, normalize_sequence(seq)) for name, seq in iter_fasta(path)] + + +def _query_args(db, fasta, shards, out, *extra): + return ("query", "sequences", "--db-path", db, "--fasta-file", fasta, "--tmp-embedding-folder", shards, + "--output-file", out, "--accelerator", "cpu", "--devices", "1", *extra) + + +def _esm(*extra): + return ("--scoring", "esm-codes", *extra) + + +def _rows(path, fields=DEFAULT_FIELDS): + """The rows of a headless TSV output as dicts over ``fields``.""" + out = [] + for line in Path(path).read_text().splitlines(): + if line: + cols = line.split("\t") + assert len(cols) == len(fields), (line, fields) + out.append(dict(zip(fields, cols))) + return out + + +def _no_pass(): + """The ESM3 pass must not be reached.""" + return mock.patch("foldmatch.search.embedding_computer.chain_predict", + mock.Mock(side_effect=AssertionError("the ESM3 pass was reached"))) + + +def _clear(folder: Path): + for path in folder.iterdir(): + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() + + +class _Case(unittest.TestCase): + """A temporary folder per test (``self.tmp``, query shards in ``self.qshards``), no launch variables.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.qshards = self.tmp / "qshards" + self.qshards.mkdir() + self.enterContext(launch_env()) + + def refused(self, result, cause=None): + """Exit status 1 with a RefusalError (D20); returns its message.""" + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError, repr(result.exception)) + if cause is not None: + self.assertIsInstance(result.exception.__cause__, cause, repr(result.exception.__cause__)) + return str(result.exception) + + def usage_error(self, result, *texts): + """Exit status 2 (typer.BadParameter) naming every one of ``texts`` (typer's error panel borders dropped).""" + self.assertEqual(result.exit_code, 2, result.output) + flat = " ".join(result.output.translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + for text in texts: + self.assertIn(text, flat) + return flat + + +@pytest.mark.usefixtures("fmsw_libs") +class _DbCase(_Case): + """Two databases of the fixture FASTA built once per class with the fake pass: ``codes`` (``--residue-codes``, + F2c codes) and ``plain``. ``database(kind)`` copies one into the test's folder.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() # unittest.SkipTest only when offline and not cached + swkern.load() + cls._dbs = tempfile.TemporaryDirectory() + cls.dbs = Path(cls._dbs.name) + for kind, extra in (("codes", ("--residue-codes",)), ("plain", ())): + (cls.dbs / kind).mkdir() + shards = cls.dbs / f"{kind}-shards" + shards.mkdir() + with launch_env(FOLDMATCH_RUN_ID=f"build-{kind}"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + result = invoke(*_build_args(_FASTA, cls.dbs / kind / "db", shards, *extra)) + assert result.exit_code == 0, (result.output, result.exception) + + @classmethod + def tearDownClass(cls): + cls._dbs.cleanup() + super().tearDownClass() + + def database(self, kind="codes", name="db") -> Path: + """A copy of the class's ``kind`` database in ``self.tmp/kind``; returns its ``--db-path``.""" + folder = self.tmp / kind + if not folder.exists(): + shutil.copytree(self.dbs / kind, folder) + return folder / name + + def query(self, db, *extra, fasta=_FASTA, out=None, run_id="q", predict=None, calls=None): + """``query sequences`` with the fake query pass; returns ``(result, output path)``.""" + out = out or self.tmp / "out.tsv" + with launch_env(FOLDMATCH_RUN_ID=run_id), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", predict or fake_query_pass(calls)): + result = invoke(*_query_args(db, fasta, self.qshards, out, *extra)) + return result, out + + +def _edit_manifest(db: Path, edit): + path = Path(f"{db}.rescodes.json") + manifest = json.loads(path.read_text()) + edit(manifest) + path.write_text(json.dumps(manifest)) + + +# ===================================================================================================================== +# P6.1: the options +# ===================================================================================================================== +class OptionTest(unittest.TestCase): + + def _command(self, *path): + import typer + command = typer.main.get_command(search.app) + for name in path: + command = command.commands[name] + return command + + def _param(self, path, name): + return next(p for p in self._command(*path).params if p.name == name) + + def test_scoring_option(self): + param = self._param(("query", "sequences"), "scoring") + self.assertEqual(param.opts, ["--scoring"]) + self.assertEqual(list(param.type.choices), ["blosum62", "esm-codes"]) + self.assertIs(param.default, ScoringChoice.blosum62) + for path in (("query", "db"), ("query", "structure"), ("query", "embedding")): + with self.subTest(command=" ".join(path)): + self.assertNotIn("scoring", [p.name for p in self._command(*path).params]) + + def test_align_workers_help_gives_each_scorings_default(self): + """The esm-codes pool default (P7.2d, P7.2j) is not the BLOSUM62 one; query db is BLOSUM62 only.""" + text = " ".join(self._param(("query", "sequences"), "align_workers").help.split()) + self.assertIn("Default under --scoring blosum62: all CPUs on the node", text) + self.assertIn("under esm-codes: up to one worker per physical core", text) + self.assertIn("only as many as the work needs", text) + self.assertIn("Defaults to all CPUs", self._param(("query", "db"), "align_workers").help) + + def test_scoring_dependent_defaults_are_resolved_per_scoring(self): + """The typer defaults are None (resolved per scoring) and the help states both scorings' defaults.""" + for name in ("min_seq_identity", "min_coverage", "max_evalue", "gap_open", "gap_extend", "comp_bias_corr", + "comp_bias_corr_scale", "significance_mode", "significance_sample_size"): + with self.subTest(option=name): + param = self._param(("query", "sequences"), name) + self.assertIsNone(param.default) + for name, text in (("min_seq_identity", "Default: 0.3 under --scoring blosum62, 0 under esm-codes"), + ("min_coverage", "Default: 0.8 under --scoring blosum62, 0 under esm-codes"), + ("max_evalue", "(default 1e-3)"), ("gap_open", "refused with --scoring esm-codes"), + ("gap_extend", "refused with --scoring esm-codes"), + ("comp_bias_corr", "refused with --scoring esm-codes"), + ("comp_bias_corr_scale", "Default: 1.0. BLOSUM62 scoring only: refused with --scoring " + "esm-codes"), + ("significance_mode", "refused with --scoring esm-codes"), + ("significance_sample_size", "Default: 500. BLOSUM62 scoring only: refused with --scoring " + "esm-codes")): + with self.subTest(option=name): + self.assertIn(text, self._param(("query", "sequences"), name).help) + # query db keeps its options and defaults. + for name, default in (("min_seq_identity", 0.3), ("min_coverage", 0.8), ("max_evalue", 1e-3), + ("gap_open", 11), ("gap_extend", 1), ("comp_bias_corr", False), + ("comp_bias_corr_scale", 1.0), ("significance_sample_size", 500)): + with self.subTest(command="query db", option=name): + self.assertEqual(self._param(("query", "db"), name).default, default) + + def test_output_fields_in_the_help(self): + """--format-output lists the esm-codes fields in query sequences and query db alike (P6.5, CHANGELOG).""" + for path in (("query", "sequences"), ("query", "db")): + with self.subTest(command=" ".join(path)): + text = self._param(path, "format_output").help + self.assertIn(", ".join(output.SUPPORTED_OUTPUT_FIELDS), text) + self.assertIn("Only 'query sequences --scoring esm-codes' fills rawrev, q90, rawadj, ecal", text) + help_text = invoke(*path, "--help").output + for field in output.ESM_CODES_OUTPUT_FIELDS: + self.assertIn(field, help_text) + self.assertEqual(output.DEFAULT_OUTPUT_FIELDS[-2:], ("evalue", "bits")) # unchanged default columns + + +class ResolveOptionsTest(unittest.TestCase): + """:func:`search.resolve_query_stage2_options` (P6.1, D8, D20).""" + + def test_blosum62_defaults_are_the_old_ones(self): + o = search.resolve_query_stage2_options("blosum62") + self.assertEqual((o.scoring, o.min_seq_identity, o.min_coverage, o.max_evalue, o.gap_open, o.gap_extend, + o.comp_bias_corr, o.comp_bias_corr_scale, o.significance_mode, o.significance_sample_size), + (ScoringChoice.blosum62, 0.3, 0.8, 1e-3, 11, 1, False, 1.0, SignificanceMode.default, 500)) + self.assertEqual(search.resolve_query_stage2_options(), o) + + def test_esm_codes_defaults(self): + o = search.resolve_query_stage2_options("esm-codes") + self.assertEqual((o.scoring, o.min_seq_identity, o.min_coverage, o.max_evalue), + (ScoringChoice.esm_codes, 0.0, 0.0, 1e-3)) + self.assertEqual((o.gap_open, o.gap_extend, o.comp_bias_corr, o.comp_bias_corr_scale, o.significance_mode, + o.significance_sample_size), (None,) * 6) + + def test_explicit_values_are_honoured(self): + for scoring in ("blosum62", "esm-codes"): + with self.subTest(scoring=scoring): + o = search.resolve_query_stage2_options(scoring, min_seq_identity=0.5, min_coverage=0.25, + max_evalue=float("inf"), seq_identity=True) + self.assertEqual((o.min_seq_identity, o.min_coverage, o.max_evalue), (0.5, 0.25, float("inf"))) + o = search.resolve_query_stage2_options("blosum62", gap_open=10, gap_extend=2, comp_bias_corr=True, + comp_bias_corr_scale=0.5, significance_mode="sampled", + significance_sample_size=50, min_seq_identity=0.0) + self.assertEqual((o.gap_open, o.gap_extend, o.comp_bias_corr, o.comp_bias_corr_scale, o.significance_mode, + o.significance_sample_size, o.min_seq_identity), + (10, 2, True, 0.5, SignificanceMode.sampled, 50, 0.0)) + + def test_blosum62_options_are_refused_under_esm_codes(self): + import typer + cases = {"--gap-open": dict(gap_open=11), "--gap-extend": dict(gap_extend=1), + "--comp-bias-corr/--no-comp-bias-corr": dict(comp_bias_corr=False), + "--comp-bias-corr-scale": dict(comp_bias_corr_scale=1.0), + "--significance-mode": dict(significance_mode=SignificanceMode.default), + "--significance-sample-size": dict(significance_sample_size=500), + "--no-seq-identity": dict(seq_identity=False)} + for flag, kwargs in cases.items(): + with self.subTest(flag=flag), self.assertRaises(typer.BadParameter) as ctx: + search.resolve_query_stage2_options("esm-codes", **kwargs) + self.assertIn(flag, str(ctx.exception)) + with self.assertRaises(typer.BadParameter) as ctx: + search.resolve_query_stage2_options("esm-codes", gap_open=5, significance_mode="sampled", + seq_identity=False) + self.assertIn("--gap-open, --significance-mode are BLOSUM62 options", str(ctx.exception)) + self.assertIn("--no-seq-identity", str(ctx.exception)) + with self.assertRaises(typer.BadParameter) as ctx: # P7.0b: a NaN scale is refused as given, not checked + search.resolve_query_stage2_options("esm-codes", comp_bias_corr_scale=float("nan"), + significance_sample_size=0) + self.assertIn("--comp-bias-corr-scale, --significance-sample-size are BLOSUM62 options", str(ctx.exception)) + + def test_esm_codes_filter_values_are_checked(self): + import typer + for kwargs in (dict(min_seq_identity=1.5), dict(min_coverage=-0.1), dict(min_coverage=float("nan")), + dict(max_evalue=-1.0), dict(max_evalue=float("nan"))): + with self.subTest(**kwargs), self.assertRaises(typer.BadParameter): + search.resolve_query_stage2_options("esm-codes", **kwargs) + # blosum62 keeps today's behaviour: no check here. + search.resolve_query_stage2_options("blosum62", min_seq_identity=1.5, max_evalue=-1.0) + + +class RefusedOptionsCliTest(_Case): + """Each refused option exits 2 before anything is read: the database path does not even exist.""" + + def run_esm(self, *extra): + with _esm3_spy() as esm3, _no_pass() as predict, \ + mock.patch.object(search, "resolve_esm_codes_database", + side_effect=AssertionError("the database was read")) as resolve: + result = invoke(*_query_args(self.tmp / "missing" / "db", _FASTA, self.qshards, self.tmp / "o.tsv", + *_esm(*extra))) + esm3.assert_not_called() + predict.assert_not_called() + resolve.assert_not_called() + return result + + def test_each_refused_option(self): + for extra, flag in ((("--gap-open", "11"), "--gap-open"), (("--gap-extend", "2"), "--gap-extend"), + (("--comp-bias-corr",), "--comp-bias-corr"), (("--no-comp-bias-corr",), "--comp-bias-corr"), + (("--significance-mode", "sampled"), "--significance-mode"), + (("--comp-bias-corr-scale", "1.0"), "--comp-bias-corr-scale"), + (("--comp-bias-corr-scale", "0.5"), "--comp-bias-corr-scale"), + (("--significance-sample-size", "500"), "--significance-sample-size"), + (("--significance-sample-size", "100"), "--significance-sample-size"), + (("--no-seq-identity",), "--no-seq-identity")): + with self.subTest(extra=extra): + self.usage_error(self.run_esm(*extra), flag, "--scoring") + + def test_option_values(self): + for extra, text in ((("--min-seq-identity", "2"), "--min-seq-identity"), + (("--min-coverage", "-0.5"), "--min-coverage"), (("--max-evalue", "-1"), "--max-evalue"), + (("--align-workers", "-1"), "--align-workers"), + (("--format-output", "query,target,not_a_field"), "not_a_field")): + with self.subTest(extra=extra): + self.usage_error(self.run_esm(*extra), text) + + def test_format_output_under_blosum62_is_checked_before_esm3(self): + """P7.0c: under blosum62 too, an unknown or empty --format-output is a usage error (exit status 2) before the + query's ESM3 pass, whenever Stage 2 runs (the sequence store found, or --seq-identity).""" + db = self.tmp / "plain" / "db" + db.parent.mkdir() + from foldmatch.search.sequence_store import SequenceStore + with mock.patch.object(SequenceStore, "exists", return_value=True), \ + mock.patch.object(search, "_check_stage2_stores") as stores: + for extra, text in ((("--format-output", "query,target,not_a_field"), "not_a_field"), + (("--format-output", ","), "Empty --format-output"), + (("--seq-identity", "--format-output", "query,bogus"), "bogus")): + with self.subTest(extra=extra), _esm3_spy() as esm3, _no_pass() as predict: + result = invoke(*_query_args(db, _FASTA, self.qshards, self.tmp / "o.tsv", *extra)) + self.usage_error(result, text, "--format-output") + esm3.assert_not_called() + predict.assert_not_called() + stores.assert_not_called() + self.assertFalse((self.tmp / "o.tsv").exists()) + + def test_seq_identity_on_is_accepted(self): + """--seq-identity (Stage 2 on) is what esm-codes does anyway: it reaches the database resolution.""" + with mock.patch.object(search, "resolve_esm_codes_database", + side_effect=RefusalError("stop here")) as resolve, _esm3_spy() as esm3: + result = invoke(*_query_args(self.tmp / "db", _FASTA, self.qshards, self.tmp / "o.tsv", + *_esm("--seq-identity"))) + self.assertEqual(result.exit_code, 1) + resolve.assert_called_once() + esm3.assert_not_called() + + +# ===================================================================================================================== +# P6.2: resolution before ESM3, every refusal on every rank alike +# ===================================================================================================================== +class RefusalTest(_DbCase): + + def run_esm(self, db, *extra, fasta=_FASTA, rank_env=None, prepare_ok=True): + """An esm-codes query that must be refused before ESM3 (and before the pass); ``prepare_ok=False``: before + the bundle is prepared (step 5) too.""" + prepare = mock.patch.object(ss, "prepare", wraps=ss.prepare) if prepare_ok else \ + mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle prepared")) + with launch_env(**(rank_env or {})), _esm3_spy() as esm3, _no_pass() as predict, prepare as prep: + result = invoke(*_query_args(db, fasta, self.qshards, self.tmp / "o.tsv", *_esm(*extra))) + esm3.assert_not_called() + predict.assert_not_called() + if not prepare_ok: + prep.assert_not_called() + self.assertEqual(list(self.qshards.iterdir()), []) + self.assertFalse((self.tmp / "o.tsv").exists()) + return result + + def _ranks(self): + return ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}) + + def test_no_sequence_store(self): + db = self.database("plain") + Path(f"{db}.sequences").unlink() + for rank_env in self._ranks(): + with self.subTest(rank_env=rank_env): + message = self.refused(self.run_esm(db, rank_env=rank_env, prepare_ok=False)) + self.assertIn("has no sequence store", message) + self.assertIn("build sequences --residue-codes", message) + self.assertFalse(Path(f"{db}.sequences").exists()) # nothing was created by the check + + def test_built_without_residue_codes(self): + db = self.database("plain") + for rank_env in self._ranks(): + with self.subTest(rank_env=rank_env): + message = self.refused(self.run_esm(db, rank_env=rank_env, prepare_ok=False), FileNotFoundError) + self.assertIn("was built without --residue-codes", message) + # BLOSUM62 needs the kernel too: offered without saying it works (step 4 has not loaded it) + self.assertIn("--scoring blosum62 can query this database (its Stage 2 needs the native kernel too)", + message) + self.assertNotIn("still works", message) + self.assertNotIn("partial", message) + + def test_partial_build(self): + db = self.database("codes") + Path(f"{db}.rescodes.json").unlink() + for rank_env in self._ranks(): + with self.subTest(rank_env=rank_env): + message = self.refused(self.run_esm(db, rank_env=rank_env, prepare_ok=False), ResidueCodeStoreError) + self.assertIn("are a partial build", message) + self.assertNotIn("built without", message) + self.assertIn("--from-shards", message) + + def test_unreadable_manifest(self): + db = self.database("codes") + Path(f"{db}.rescodes.json").write_text("{not json") + message = self.refused(self.run_esm(db, prepare_ok=False), IncompatibleCodeStoreError) + self.assertIn("not readable JSON", message) + + def test_unknown_system(self): + db = self.database("codes") + _edit_manifest(db, lambda m: m.update(system_name="esm-codes-v99")) + for rank_env in self._ranks(): + with self.subTest(rank_env=rank_env): + message = self.refused(self.run_esm(db, rank_env=rank_env, prepare_ok=False), + ss.UnknownScoringSystemError) + self.assertIn("'esm-codes-v99'", message) + self.assertIn("database was built by a newer FoldMatch; upgrade", message) + + def test_a_finite_max_evalue_needs_residues(self): + """Step 6: a finite --max-evalue needs a database with residues (N of the E-values): refused on an empty + database, which the commands cannot build (written here through the stores' API); --max-evalue inf passes.""" + from foldmatch.search.sequence_store import SequenceStore + folder = self.tmp / "empty" + folder.mkdir() + sequence_store = SequenceStore(folder, "db") + sequence_store.create(_write(folder / "empty.fasta", "")) + store = ResidueCodeStore(folder, "db") + store.create([([], [], [])], SYSTEM) + store.validate(sequence_store, 0, preflight=None) + db = folder / "db" + self.assertEqual(store.manifest()["total_residues"], 0) + message = self.refused(self.run_esm(db), ValueError) + self.assertIn("needs a non-empty database", message) + self.assertEqual(search.resolve_esm_codes_database(str(db), float("inf")).total_residues, 0) + + def test_incompatible_stores(self): + """check_compatible against the system and the sequence store meta (step 3), each field on its own.""" + cases = { + "manifest version": (lambda m: m.update(manifest_version=2), "unknown manifest_version 2"), + "encoding digest": (lambda m: m.update(encoding_digest="0" * 16), "encoding_digest"), + "ESM3 weights": (lambda m: m["esm"].update(weights_sha256="0" * 64), "built with ESM3 weights " + "0" * 64), + "entries": (lambda m: m.update(entries=3), "3 entries, the sequence store 2"), + "residues": (lambda m: m.update(total_residues=304), "304 residues, the sequence store 303"), + } + for case, (edit, text) in cases.items(): + with self.subTest(case=case): + shutil.rmtree(self.tmp / "codes", ignore_errors=True) + db = self.database("codes") + _edit_manifest(db, edit) + message = self.refused(self.run_esm(db, prepare_ok=False), IncompatibleCodeStoreError) + self.assertIn(text, message) + if case != "manifest version": # a newer manifest asks for an upgrade instead + self.assertIn("--scoring blosum62 can query this database (its Stage 2 needs the native kernel " + "too)", message) + + def test_missing_kernel(self): + db = self.database("codes") + for rank_env in self._ranks(): + with self.subTest(rank_env=rank_env), mock.patch.dict(swkern._KERNELS, clear=True), \ + mock.patch("foldmatch._native.find_library", return_value=None): + message = self.refused(self.run_esm(db, rank_env=rank_env, prepare_ok=False), swkern.KernelError) + self.assertIn("native Smith-Waterman kernel", message) + self.assertIn("python -m foldmatch._native.build", message) # the rebuild command + # BLOSUM62's Stage 2 aligns on the same kernel (P9.S3): it is no way out here + self.assertNotIn("still works", message) + self.assertIn("--scoring blosum62 needs it too", message) + + def test_a_bundle_problem(self): + db = self.database("codes") + with _esm3_spy() as esm3, _no_pass(), mock.patch.object( + ss, "prepare", side_effect=ss.BundleUnavailableError("not cached")) as prepare: + result = invoke(*_query_args(db, _FASTA, self.qshards, self.tmp / "o.tsv", *_esm())) + self.assertIn("not cached", self.refused(result, ss.BundleUnavailableError)) + prepare.assert_called_once() + esm3.assert_not_called() + + def test_one_fetch_then_cached_only(self): + """The process that starts the run may download the bundle; a rank Lightning starts, or a multi-node run, + finds it cached; the pass gets the prepared registry system and writes query encodings.""" + db = self.database("codes") + stop = RuntimeError("stop at the pass") + for rank_env, extra, local_only in (({}, (), False), ({"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}, (), True), + ({}, ("--num-nodes", "2"), True)): + with self.subTest(rank_env=rank_env, extra=extra), launch_env(**rank_env), \ + mock.patch.object(ss, "prepare", wraps=ss.prepare) as prepare, \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", side_effect=stop) as predict: + result = invoke(*_query_args(db, _FASTA, self.qshards, self.tmp / "o.tsv", *_esm(*extra))) + self.assertIs(result.exception.__cause__ or result.exception, stop) + prepare.assert_called_once_with(SYSTEM, local_files_only=local_only) + kwargs = predict.call_args.kwargs + self.assertEqual(kwargs["residue_output"], "query_encoding") + self.assertIs(kwargs["scoring_system"].system, SYSTEM) + + def test_query_fasta_the_method_cannot_take(self): + """Duplicate ids, '<' / '>' and non-ASCII sequences: refused by the strict query pre-flight, before the + database is read; a blosum62 query of the same file is not strict.""" + db = self.database("codes") + for text, what in ((">a\nACDE\n>b\nKLM\n>a\nACDE\n", "repeat an earlier id"), + (">a\nACDE\n", "'<' or '>'"), (">a\nACDE>\n", "'<' or '>'"), + (">a\nACDÉ\n", "non-ASCII")): + for rank_env in self._ranks(): + with self.subTest(what=what, rank_env=rank_env), \ + mock.patch.object(search, "resolve_esm_codes_database", + side_effect=AssertionError("the database was read")): + fasta = _write(self.tmp / "bad.fasta", text) + self.assertIn(what, self.refused(self.run_esm(db, fasta=fasta, rank_env=rank_env, + prepare_ok=False), ValueError)) + fasta = _write(self.tmp / "dup.fasta", ">a\nACDE\n>a\nACDE\n") + with _esm3_spy() as esm3: + result = invoke(*_query_args(db, fasta, self.qshards, self.tmp / "o.tsv")) + self.assertIsInstance(result.exception, AssertionError) # the blosum62 pass was reached + esm3.assert_called_once() + + def test_query_encodings_of_an_earlier_run_in_the_way(self): + db = self.database("codes") + with launch_env(FOLDMATCH_RUN_ID="same"): + (self.qshards / "query_encodings_same.f16").touch() + with _esm3_spy() as esm3, _no_pass(): + result = invoke(*_query_args(db, _FASTA, self.qshards, self.tmp / "o.tsv", *_esm())) + self.assertIn("query_encodings_same.f16", self.refused(result, FileExistsError)) + esm3.assert_not_called() + self.assertEqual([p.name for p in self.qshards.iterdir()], ["query_encodings_same.f16"]) + + def test_shards_of_the_same_run_id(self): + db = self.database("codes") + with launch_env(FOLDMATCH_RUN_ID="r"): + (self.qshards / "emb_r-0.parquet").touch() + with _esm3_spy() as esm3: + result = invoke(*_query_args(db, _FASTA, self.qshards, self.tmp / "o.tsv", *_esm())) + self.assertIn("same run id", self.refused(result, FileExistsError)) + esm3.assert_not_called() + self.assertEqual([p.name for p in self.qshards.iterdir()], ["emb_r-0.parquet"]) # not this run's: kept + + def test_the_faiss_count_after_the_index_loads(self): + """Rank 0 compares the live FAISS count with the code store once the index is loaded (after the pass); the + run's shards and query encodings are removed all the same.""" + from foldmatch.search.faiss_database import FaissEmbeddingDatabase + load = FaissEmbeddingDatabase.load_database + + def one_more(self_, *args, **kwargs): + out = load(self_, *args, **kwargs) + self_.chain_ids = list(self_.chain_ids) + ["phantom"] + return out + + db = self.database("codes") + with mock.patch.object(FaissEmbeddingDatabase, "load_database", one_more): + result, out = self.query(db, *_esm()) + message = self.refused(result, IncompatibleCodeStoreError) + self.assertIn("has 2 entries, the FAISS index 3", message) + self.assertEqual(list(self.qshards.iterdir()), []) + self.assertFalse(out.exists()) + + def test_exit_status_of_a_refusal(self): + """fm-search prints this refusal on one line and exits 1 (D20).""" + db = self.database("plain") + argv = ["fm-search", *map(str, _query_args(db, _FASTA, self.qshards, self.tmp / "o.tsv", *_esm()))] + stderr = self.enterContext(mock.patch("sys.stderr", new_callable=io.StringIO)) + root = logging.getLogger() + handlers = root.handlers[:] + try: + with mock.patch.object(sys, "argv", argv), _esm3_spy(), self.assertRaises(SystemExit) as ctx: + search.main() + finally: + root.handlers[:] = handlers + self.assertEqual(ctx.exception.code, 1) + errors = [line for line in stderr.getvalue().splitlines() if line.startswith("Error: ")] + self.assertEqual(len(errors), 1, stderr.getvalue()) + self.assertIn("was built without --residue-codes", errors[0]) + + +# ===================================================================================================================== +# P6.1-P6.3: the query (fake ESM3 pass, real bundle and kernel) +# ===================================================================================================================== +class QueryTest(_DbCase): + + def assert_self_hits(self, rows): + lengths = {name: len(seq) for name, seq in _fasta_records(_FASTA)} + by_query = {} + for row in rows: + by_query.setdefault(row["query"], []).append(row) + self.assertEqual(set(by_query), set(lengths)) + for qid, hits in by_query.items(): + top = hits[0] + self.assertEqual(top["target"], qid) + self.assertEqual((top["qstart"], top["qend"]), (top["tstart"], top["tend"])) # main diagonal + self.assertEqual(top["gapopen"], "0") + self.assertEqual(int(top["alnlen"]), int(top["qend"]) - int(top["qstart"]) + 1) + self.assertGreaterEqual(float(top["fident"]), 0.99) + self.assertGreaterEqual(int(top["alnlen"]) / lengths[qid], 0.9) # qcov = tcov here + self.assertLessEqual(float(top["evalue"]), 1e-3) + + def test_esm_codes_query(self): + """Default columns, no empty cell; each query's top hit is itself on the diagonal; Stage 2 got the defaults + of D8, N = the manifest's 303 residues, the run's query encodings; BLOSUM62 was not used.""" + db = self.database("codes") + calls = [] + with mock.patch.object(code_alignment, "align_candidates_esm_codes", + wraps=code_alignment.align_candidates_esm_codes) as esm, \ + mock.patch("foldmatch.search.alignment.align_candidates", + side_effect=AssertionError("BLOSUM62 Stage 2")) as blosum: + result, out = self.query(db, *_esm("--align-workers", "0"), calls=calls) + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + blosum.assert_not_called() + esm.assert_called_once() + args, kwargs = esm.call_args + query_index, prefilter, code_store, _fetch, prepared, db_residues, _pool = args + self.assertEqual(db_residues, 303) + self.assertEqual((code_store.base, prepared.system), (db, SYSTEM)) + self.assertEqual((kwargs["min_seq_identity"], kwargs["min_coverage"], kwargs["max_evalue"]), (0.0, 0.0, 1e-3)) + self.assertEqual(kwargs["output_needs"], list(DEFAULT_FIELDS)) + self.assertEqual(kwargs["query_sequences"], dict(_fasta_records(_FASTA))) + self.assertEqual(sorted(query_index.entries), ["1acb_E", "2uzi_A"]) + self.assertEqual(query_index.path.name, "query_encodings_q.f16") + self.assertEqual(calls[0]["residue_output"], "query_encoding") + rows = _rows(out) + self.assertTrue(rows) + for row in rows: + self.assertTrue(all(row.values()), row) + self.assert_self_hits(rows) + # The run's shards and its query encodings are gone. + self.assertEqual(list(self.qshards.iterdir()), []) + + def test_blosum62_is_unchanged(self): + """Without --scoring: the BLOSUM62 Stage 2 with its old defaults; on a database with codes it writes the + same bytes as on one without; the esm-codes fields render empty.""" + outs = {} + for kind in ("codes", "plain"): + db = self.database(kind) + with mock.patch("foldmatch.search.alignment.align_candidates", + wraps=__import__("foldmatch.search.alignment", fromlist=["x"]).align_candidates) as blosum, \ + mock.patch.object(code_alignment, "align_candidates_esm_codes", + side_effect=AssertionError("esm-codes Stage 2")): + result, outs[kind] = self.query(db, out=self.tmp / f"{kind}.tsv", run_id=f"b{kind}") + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + kwargs = blosum.call_args.kwargs + self.assertEqual({k: kwargs[k] for k in ("min_seq_identity", "min_coverage", "max_evalue", "gap_open", + "gap_extend", "comp_bias_corr", "comp_bias_corr_scale", + "significance_mode", "significance_sample_size")}, + dict(min_seq_identity=0.3, min_coverage=0.8, max_evalue=1e-3, gap_open=11, gap_extend=1, + comp_bias_corr=False, comp_bias_corr_scale=1.0, + significance_mode=SignificanceMode.default, significance_sample_size=500)) + self.assertEqual(_sha256(outs["codes"]), _sha256(outs["plain"])) + self.assertTrue(_rows(outs["codes"])) + result, out = self.query(self.database("codes"), "--format-output", ALL_FIELDS, run_id="ball") + self.assertEqual(result.exit_code, 0, result.output) + rows = _rows(out, output.SUPPORTED_OUTPUT_FIELDS) + self.assertTrue(rows) + for row in rows: + self.assertEqual([row[f] for f in output.ESM_CODES_OUTPUT_FIELDS], [""] * 4) + self.assertNotIn(".", row["raw"]) # the integer BLOSUM62 score + + def test_blosum62_takes_the_scale_and_the_sample_size(self): + """P7.0b: --comp-bias-corr-scale and --significance-sample-size reach the BLOSUM62 Stage 2; a NaN scale is + still the ValueError (exit status 1) it was, raised before the pre-flight.""" + from foldmatch.search import alignment + with mock.patch.object(alignment, "align_candidates", wraps=alignment.align_candidates) as blosum: + result, _ = self.query(self.database("codes"), "--comp-bias-corr", "--comp-bias-corr-scale", "0.5", + "--significance-mode", "sampled", "--significance-sample-size", "20", + "--gap-open", "3", "--max-evalue", "inf") + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + kwargs = blosum.call_args.kwargs + self.assertEqual((kwargs["comp_bias_corr_scale"], kwargs["significance_sample_size"]), (0.5, 20)) + with mock.patch.object(search, "_fasta_preflight", side_effect=AssertionError("pre-flight")) as preflight: + result, _ = self.query(self.database("codes"), "--comp-bias-corr-scale", "nan", predict=mock.Mock()) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, ValueError) + self.assertIn("--comp-bias-corr-scale must be between 0 and 1", str(result.exception)) + preflight.assert_not_called() + + def test_an_embedding_only_search_ignores_format_output(self): + """P7.0c: without Stage 2 (--no-seq-identity) --format-output is not used, and an unknown field is ignored as + before: the query/target/embscore rows.""" + result, out = self.query(self.database("plain"), "--no-seq-identity", "--format-output", "query,not_a_field") + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + rows = _rows(out, output.EMBEDDING_OUTPUT_FIELDS) + self.assertEqual({row["query"] for row in rows}, {"1acb_E", "2uzi_A"}) + + def test_explicit_values_are_honoured(self): + db = self.database("codes") + with mock.patch.object(code_alignment, "align_candidates_esm_codes", + wraps=code_alignment.align_candidates_esm_codes) as esm: + result, _ = self.query(db, *_esm("--min-seq-identity", "0.5", "--min-coverage", "0.7", "--max-evalue", + "10", "--align-workers", "0")) + self.assertEqual(result.exit_code, 0, result.output) + kwargs = esm.call_args.kwargs + self.assertEqual((kwargs["min_seq_identity"], kwargs["min_coverage"], kwargs["max_evalue"]), (0.5, 0.7, 10.0)) + + def test_every_field_and_the_header(self): + db = self.database("codes") + result, out = self.query(db, *_esm("--format-output", ALL_FIELDS, "--format-mode", "tsv", "--max-evalue", + "inf", "--threshold", "-2", "--align-workers", "0")) + self.assertEqual(result.exit_code, 0, result.output) + lines = out.read_text().splitlines() + self.assertEqual(lines[0], ALL_FIELDS.replace(",", "\t")) + rows = [dict(zip(output.SUPPORTED_OUTPUT_FIELDS, line.split("\t"))) for line in lines[1:]] + self.assertEqual(len(rows), 4) # every candidate traced back, each pair aligned + for row in rows: + self.assertTrue(all(row.values()), row) + for field in NUMERIC_FIELDS: + self.assertTrue(math.isfinite(float(row[field])), (field, row[field])) + self.assertEqual(row["ecal"], "1") + raw, rawrev, q90, rawadj = (float(row[f]) for f in ("raw", "rawrev", "q90", "rawadj")) + self.assertAlmostEqual(rawadj, raw - rawrev - q90, delta=2e-3) + self.assertEqual(len(row["qaln"]), int(row["alnlen"])) + + def test_serial_and_pool_give_the_same_bytes(self): + """S16 on one machine: the default worker pool (spawn) and the in-process run write identical files.""" + db = self.database("codes") + outs = [] + for k, extra in enumerate((("--align-workers", "0"), (), ("--align-workers", "3"))): + result, out = self.query(db, *_esm("--format-output", ALL_FIELDS, "--max-evalue", "inf", *extra), + out=self.tmp / f"o{k}.tsv", run_id=f"p{k}") + self.assertEqual(result.exit_code, 0, result.output) + outs.append(_sha256(out)) + self.assertEqual(len(set(outs)), 1) + + def test_queries_without_candidates_are_skipped_before_the_panel(self): + db = self.database("codes") + stats = {} + real = code_alignment.align_candidates_esm_codes + + def align(*args, **kwargs): + return real(*args, stats=stats, **kwargs) + + with mock.patch.object(code_alignment, "align_candidates_esm_codes", align): + result, out = self.query(db, *_esm("--threshold", "2", "--align-workers", "0")) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual((stats["queries"], stats["queries_without_candidates"], stats["panel_pairs"], + stats["round1_tasks"]), (2, 2, 0, 0)) + self.assertEqual(out.read_text(), "") + + def test_queries_outside_the_calibrated_domain(self): + """A 9-aa query: aligned, its hits reported with ecal=0, and one warning gives the counts (S13).""" + db = self.database("codes") + fasta = _write(self.tmp / "short.fasta", _FASTA.read_text() + ">short\nACDEFGHIK\n") + with self.assertLogs("foldmatch.search.code_alignment", "WARNING") as logs: + result, out = self.query(db, *_esm("--format-output", "query,target,ecal,evalue", "--max-evalue", + "inf", "--threshold", "-2", "--align-workers", "0"), fasta=fasta) + self.assertEqual(result.exit_code, 0, result.output) + rows = _rows(out, ("query", "target", "ecal", "evalue")) + short = [r for r in rows if r["query"] == "short"] + self.assertTrue(short) + self.assertEqual({r["ecal"] for r in short}, {"0"}) + self.assertEqual({r["ecal"] for r in rows if r["query"] != "short"}, {"1"}) + warnings = [r.getMessage() for r in logs.records if r.levelno == logging.WARNING] + self.assertEqual(len(warnings), 1) + self.assertIn("1 queries shorter than 21 aa", warnings[0]) + + def test_an_encoding_that_does_not_match_its_query_is_an_error(self): + """A missing encoding (empty payload), a length or a seq_sha1 that differs from the query's: an error after + the pass (exit 1, not a refusal), with the run's files removed.""" + db = self.database("codes") + + def short_payload(name, sequence): + q = _fake_q(name, sequence) + return q[:-1] if name == "2uzi_A" else q + + def empty_payload(name, sequence): + return np.empty((0, 64), np.float16) if name == "2uzi_A" else _fake_q(name, sequence) + + cases = { + "length": (fake_query_pass(payload=short_payload), "its sequence has 58 residues, its encoding 57 rows"), + "empty": (fake_query_pass(payload=empty_payload), "without an encoding"), + "seq_sha1": (fake_query_pass(seq_sha1=lambda name, s: _sha1(s + "A") if name == "1acb_E" else _sha1(s)), + "seq_sha1 differs"), + } + for case, (predict, text) in cases.items(): + with self.subTest(case=case): + result, out = self.query(db, *_esm("--align-workers", "0"), predict=predict, run_id=f"e{case}") + self.assertEqual(result.exit_code, 1, result.output) + self.assertNotIsInstance(result.exception, RefusalError) + self.assertIsInstance(result.exception, ValueError) + self.assertIn(text, str(result.exception)) + self.assertEqual(list(self.qshards.iterdir()), []) + self.assertFalse(out.exists()) + + def test_the_run_files_are_removed_after_a_stage_2_failure(self): + db = self.database("codes") + with mock.patch.object(code_alignment, "align_candidates_esm_codes", + side_effect=RuntimeError("stage 2 failed")): + result, _ = self.query(db, *_esm()) + self.assertEqual(str(result.exception), "stage 2 failed") + self.assertEqual(list(self.qshards.iterdir()), []) + + def test_a_failed_pass_deletes_the_shard_it_wrote(self): + """The query shard holds the query encodings (128 B per residue): a process whose pass fails deletes the shard + it wrote, and the error goes on (not a refusal).""" + db = self.database("codes") + write = fake_query_pass() + + def failing(**kwargs): + write(**kwargs) + raise RuntimeError("the ESM3 pass failed after writing its shard") + + result, out = self.query(db, *_esm(), predict=failing, run_id="f") + self.assertEqual(result.exit_code, 1, result.output) + self.assertEqual(str(result.exception), "the ESM3 pass failed after writing its shard") + self.assertNotIsInstance(result.exception, RefusalError) + self.assertEqual(list(self.qshards.iterdir()), []) + self.assertFalse(out.exists()) + + def test_a_rank_other_than_0_leaves_after_the_pass(self): + """Only rank 0 searches, aligns and writes: a rank > 0 (LOCAL_RANK=1; its process group simulated through + _is_rank_zero) returns after the pass without loading the index, running Stage 2, writing the output or + deleting the run's shards, which rank 0 reads.""" + db = self.database("codes") + out = self.tmp / "out.tsv" + from foldmatch.search.embedding_database import EmbeddingDatabase + with launch_env(LOCAL_RANK=1, FOLDMATCH_RUN_ID="r1"), \ + mock.patch.object(search, "_is_rank_zero", return_value=False), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_query_pass()), \ + mock.patch.object(EmbeddingDatabase, "load", side_effect=AssertionError("index loaded")) as load, \ + mock.patch.object(code_alignment, "align_candidates_esm_codes", + side_effect=AssertionError("Stage 2 ran")) as esm: + result = invoke(*_query_args(db, _FASTA, self.qshards, out, *_esm())) + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + load.assert_not_called() + esm.assert_not_called() + self.assertFalse(out.exists()) + self.assertEqual([p.name for p in self.qshards.iterdir()], ["emb_r1-0.parquet"]) + + +# ===================================================================================================================== +# P7.0a: a multi-rank query writes the queries in input order +# ===================================================================================================================== +def _order_fasta(path: Path) -> Path: + """Five queries cut from the fixture's two chains: two ranks deal them 0, 2, 4 | 1, 3 (DDP's round robin), so + rank order (qA, qB, qC, 1acb_E, 2uzi_A) is not input order.""" + seqs = dict(_fasta_records(_FASTA)) + a, b = seqs["1acb_E"], seqs["2uzi_A"] + records = (("qA", a[:120]), ("1acb_E", a), ("qB", b[4:]), ("2uzi_A", b), ("qC", a[100:])) + path.write_text("".join(f">{name}\n{seq}\n" for name, seq in records)) + return path + + +#: The query order of :func:`_order_fasta`: input order, and the order two ranks write without the fix. +_INPUT_ORDER = ["qA", "1acb_E", "qB", "2uzi_A", "qC"] +_RANK_ORDER = ["qA", "qB", "qC", "1acb_E", "2uzi_A"] +#: Options that give every query rows: every candidate (``--threshold -1``), no identity, coverage or E-value filter. +_EVERY_ROW = ("--threshold", "-1", "--min-seq-identity", "0", "--min-coverage", "0", "--max-evalue", "inf") +_ORDER_CASES = { + "blosum62": (*_EVERY_ROW, "--format-output", ALL_FIELDS), + "blosum62-embedding-only": ("--threshold", "-1", "--no-seq-identity"), + "esm-codes": _esm(*_EVERY_ROW, "--format-output", ALL_FIELDS, "--align-workers", "0"), +} + + +class MultiRankOrderTest(_DbCase): + """P7.0a: a 2-rank ``query sequences`` (the fake pass deals the queries round-robin to two shards, as DDP's sampler + does) writes the same bytes as a 1-rank one, under blosum62 (Stage 2, and embedding-only) and esm-codes; each + query keeps its hits. Without the reordering the 2-rank file lists rank 0's queries first.""" + + def setUp(self): + super().setUp() + self.fasta = _order_fasta(self.tmp / "order.fasta") + + def run_query(self, case, ranks, tag): + result, out = self.query(self.database("codes"), *_ORDER_CASES[case], fasta=self.fasta, + out=self.tmp / f"{case}-{tag}.tsv", run_id=f"o-{tag}", + predict=fake_chain_predict(payload=_fake_q, ranks=ranks)) + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + self.assertEqual(list(self.qshards.iterdir()), []) + return out + + @staticmethod + def query_order(out): + return list(dict.fromkeys(line.split("\t")[0] for line in out.read_text().splitlines())) + + def test_two_ranks_write_the_bytes_of_one(self): + for case in _ORDER_CASES: + with self.subTest(case=case): + one, two = self.run_query(case, 1, "1"), self.run_query(case, 2, "2") + self.assertEqual(self.query_order(one), _INPUT_ORDER) # every query has rows + self.assertEqual(_sha256(two), _sha256(one)) + if case != "blosum62-embedding-only": + rows = _rows(one, output.SUPPORTED_OUTPUT_FIELDS) + self.assertEqual(len(rows), 10) # 5 queries x 2 targets + blank = [row["rawrev"] == "" for row in rows] + self.assertEqual(set(blank), {case == "blosum62"}) + + def test_without_the_reordering_two_ranks_write_rank_order(self): + """The control: with ``in_input_order`` bypassed (the rows keyed by their ids in read order), the 2-rank file + has the same rows per query, in rank order.""" + from foldmatch.search.embedding_computer import EmbeddingComputer + + def read_order(self_, results): + ids = [qid for batch, _ in self_.get_embedding_batches() for qid in batch] + return {ids[row]: value for row, value in results.items()} + + for case in _ORDER_CASES: + with self.subTest(case=case): + one = self.run_query(case, 1, "1") + with mock.patch.object(EmbeddingComputer, "in_input_order", read_order): + two = self.run_query(case, 2, "2") + self.assertEqual(self.query_order(two), _RANK_ORDER) + self.assertNotEqual(_sha256(two), _sha256(one)) + self.assertEqual(sorted(two.read_text().splitlines()), sorted(one.read_text().splitlines())) + + +def _dup_fasta(path: Path, last=None) -> Path: + """A repeated query id (accepted, with a warning, under blosum62): p0, x(1acb_E), x(``last``, default 2uzi_A), + p3. Two ranks deal p0, x(last) | x(1acb_E), p3, so rank 0's shard, read first, holds the later x.""" + seqs = dict(_fasta_records(_FASTA)) + a, b = seqs["1acb_E"], seqs["2uzi_A"] + path.write_text(f">p0\n{a[:120]}\n>x\n{a}\n>x\n{b if last is None else last}\n>p3\n{b[4:]}\n") + return path + + +class RepeatedIdMultiRankTest(_DbCase): + """P7.0a (review): a repeated query id under blosum62. The fake pass fixes each embedding by its sequence, as + ESM3 does, so the two x records have different embeddings: a 2-rank run writes the bytes of a 1-rank run, whose + x is its last record for Stage 1 (the embedding a search keeps) and for Stage 2 (the sequence aligned).""" + + _CASES = {"embedding-only": ("--no-seq-identity",), + "stage-2": ("--max-evalue", "inf", "--min-seq-identity", "0", "--min-coverage", "0", + "--format-output", ALL_FIELDS)} + + def run_query(self, fasta, case, ranks, *extra): + result, out = self.query(self.database("plain"), "--threshold", "-1", *self._CASES[case], *extra, + fasta=fasta, out=self.tmp / f"dup-{case}-{ranks}.tsv", run_id=f"dup-{ranks}", + predict=fake_chain_predict(ranks=ranks, by_sequence=True)) + self.assertEqual(result.exit_code, 0, (result.output, result.exception)) + self.assertEqual(list(self.qshards.iterdir()), []) + return out + + def test_two_ranks_write_the_bytes_of_one(self): + fasta = _dup_fasta(self.tmp / "dup.fasta") + single = _dup_fasta(self.tmp / "single.fasta").read_text().replace( + f">x\n{dict(_fasta_records(_FASTA))['1acb_E']}\n", "") + (self.tmp / "single.fasta").write_text(single) # p0, x(2uzi_A), p3: x's last record only + for case in self._CASES: + with self.subTest(case=case): + one, two = self.run_query(fasta, case, 1), self.run_query(fasta, case, 2) + self.assertEqual(_sha256(two), _sha256(one)) + lines = one.read_text().splitlines() + self.assertEqual(list(dict.fromkeys(line.split("\t")[0] for line in lines)), ["p0", "x", "p3"]) + # x's rows are those of a FASTA holding only its last record + last = self.run_query(self.tmp / "single.fasta", case, 1) + x_rows = [line for line in lines if line.startswith("x\t")] + self.assertEqual(x_rows, [line for line in last.read_text().splitlines() if line.startswith("x\t")]) + + def test_stage_2_aligns_the_record_stage_1_searched(self): + """The last x record is shorter than --min-res-n, so the pass embeds the earlier one: Stage 2 aligns that + record's sequence too (before, it aligned the last record's, whose embedding was never computed).""" + a = dict(_fasta_records(_FASTA))["1acb_E"] + fasta = _dup_fasta(self.tmp / "short.fasta", last="ACDEFG") + for ranks in (1, 2): + with self.subTest(ranks=ranks): + out = self.run_query(fasta, "stage-2", ranks, "--min-res-n", "20") + rows = [row for row in _rows(out, output.SUPPORTED_OUTPUT_FIELDS) if row["query"] == "x"] + self.assertEqual(len(rows), 2) + self.assertTrue(all(row["qseq"] == a for row in rows), rows) + self.assertEqual({row["qlen"] for row in rows}, {str(len(a))}) + + +# ===================================================================================================================== +# real ESM3 (CPU): the end-to-end smoke and the F-deg items through the command line +# ===================================================================================================================== +def _track_models(): + """Patches recording a weak reference to every ESM3 model and aggregator the pass builds, and whether they are + still alive when Stage 2 starts. Returns ``(patches, refs, alive_at_stage_2)``.""" + from foldmatch.inference import chain_complete_inference as cci + refs, alive = [], [] + + def track(fn): + def build(*args, **kwargs): + model = fn(*args, **kwargs) + refs.append(weakref.ref(model)) + return model + return build + + real_align = code_alignment.align_candidates_esm_codes + + def align(*args, **kwargs): + alive.append([ref() is not None for ref in refs]) + return real_align(*args, **kwargs) + + patches = (mock.patch.object(cci, "get_residue_model", track(cci.get_residue_model)), + mock.patch.object(cci, "get_aggregator_model", track(cci.get_aggregator_model)), + mock.patch.object(code_alignment, "align_candidates_esm_codes", align)) + return patches, refs, alive + + +@pytest.mark.usefixtures("fmsw_libs") +class RealEsm3SmokeTest(unittest.TestCase): + """The plan's end-to-end smoke (CPU, the 2-sequence fixture, ESM3 from the cache): build with --residue-codes, then + query the same FASTA with --scoring esm-codes --threshold 0; a blosum62 query on that database writes the same + bytes as on a database built without codes. The commands' wall times are printed (acceptance: the smoke within + 2x the current query sequences test).""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() + swkern.load() + gc.collect() + cls._tmp = tempfile.TemporaryDirectory() + cls.tmp = Path(cls._tmp.name) + cls.results, cls.seconds = {}, {} + patches, cls.model_refs, cls.alive = _track_models() + steps = ( + ("build codes", _build_args(_FASTA, cls.tmp / "codes" / "db", cls.tmp / "bshards", "--residue-codes")), + ("query esm-codes", _query_args(cls.tmp / "codes" / "db", _FASTA, cls.tmp / "qshards", + cls.tmp / "esm.tsv", *_esm("--threshold", "0"))), + ("query blosum62 codes db", _query_args(cls.tmp / "codes" / "db", _FASTA, cls.tmp / "qshards", + cls.tmp / "blosum_codes.tsv")), + ("build plain", _build_args(_FASTA, cls.tmp / "plain" / "db", cls.tmp / "bshards")), + ("query blosum62 plain db", _query_args(cls.tmp / "plain" / "db", _FASTA, cls.tmp / "qshards", + cls.tmp / "blosum_plain.tsv")), + ) + for folder in ("codes", "plain", "bshards", "qshards"): + (cls.tmp / folder).mkdir() + with launch_env(), patches[0], patches[1], patches[2]: + for k, (name, args) in enumerate(steps): + os.environ["FOLDMATCH_RUN_ID"] = f"smoke{k}" + t0 = time.perf_counter() + cls.results[name] = invoke(*args) + cls.seconds[name] = time.perf_counter() - t0 + gc.collect() + print("\nP6 smoke wall times (s): " + ", ".join(f"{k} {v:.1f}" for k, v in cls.seconds.items()), + file=sys.stderr) + + @classmethod + def tearDownClass(cls): + cls._tmp.cleanup() + super().tearDownClass() + + def setUp(self): + for name, result in self.results.items(): + self.assertEqual(result.exit_code, 0, f"{name}: {result.output[-4000:]} {result.exception!r}") + + def test_each_query_finds_itself(self): + rows = _rows(self.tmp / "esm.tsv") + self.assertTrue(rows) + for row in rows: + self.assertTrue(all(row.values()), row) # the default 13 columns, no empty cell + QueryTest.assert_self_hits(self, rows) + manifest = ResidueCodeStore(self.tmp / "codes", "db").manifest() + self.assertEqual((manifest["entries"], manifest["total_residues"]), (2, 303)) # N = 303 + + def test_blosum62_on_a_database_with_codes_is_byte_identical(self): + self.assertEqual(_sha256(self.tmp / "blosum_codes.tsv"), _sha256(self.tmp / "blosum_plain.tsv")) + self.assertTrue(_rows(self.tmp / "blosum_codes.tsv")) + for suffix in _DB_SUFFIXES: + with self.subTest(suffix=suffix): + self.assertEqual(_sha256(self.tmp / "codes" / f"db{suffix}"), _sha256(self.tmp / "plain" / f"db{suffix}")) + + def test_the_models_are_freed_before_stage_2(self): + self.assertEqual(len(self.alive), 1) + self.assertEqual(len(self.alive[0]), 4) # ESM3 and the aggregator of the build and of the query pass + self.assertEqual(self.alive[0], [False] * 4) + + def test_query_files_are_removed(self): + self.assertEqual(list((self.tmp / "qshards").iterdir()), []) + + +def _fdeg_items(): + z = np.load(_FDEG, allow_pickle=False) + return [(str(i), str(s)) for i, s in zip(z["real_ids"], z["real_sequences"])] + + +#: Two longer chains (F3 queries of 439 and 609 aa) whose best cell against the length-1 target M is clearly positive +#: (S_f 0.27 / 0.26 on the Mac CPU, where 1acb_E's is only 0.016), so the length-1 target is aligned and reported. +_F3_CHAINS = ("Q8X1D8_1", "P17169_1") + + +def _f3_items(ids): + z = np.load(_TEST_PATH / "resources" / "esm_codes" / "q90_queries_v1.npz", allow_pickle=False) + sequences = {str(i): str(s) for i, s in zip(z["ids"], z["sequences"])} + return [(i, sequences[i]) for i in ids] + + +@pytest.mark.usefixtures("fmsw_libs") +class RealEsm3DegenerateTest(unittest.TestCase): + """The real-ESM3 F-deg items through the command line (plan section 8 P6 Tests; D14, S13, S13b): a database of the + six items, the fixture's two chains and two longer F3 chains, built with --residue-codes, queried with the same + FASTA under --max-evalue inf. Every output is finite and every hit's ecal is S13's; all-X, poly-A, HHHHHH, GS and + the 10-aa peptide are aligned and reported; the single-residue query M returns no hits and is counted in the + warning, as is a synthetic identical-row query (poly-A with its encoding rows made identical); the length-1 + target M is aligned by the long chains and gets ecal=0.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() + swkern.load() + gc.collect() + cls._tmp = tempfile.TemporaryDirectory() + cls.tmp = Path(cls._tmp.name) + cls.sequences = dict(_fdeg_items() + _fasta_records(_FASTA) + _f3_items(_F3_CHAINS)) + cls.fasta = cls.tmp / "fdeg.fasta" + cls.fasta.write_text("".join(f">{name}\n{seq}\n" for name, seq in cls.sequences.items())) + for folder in ("db", "bshards", "qshards"): + (cls.tmp / folder).mkdir() + cls.db = cls.tmp / "db" / "fdeg" + cls.results, cls.warnings = {}, {} + from foldmatch.search.embedding_computer import EmbeddingComputer + real_build = EmbeddingComputer.build_query_encodings + + def identical_rows(self_, dst, system): # poly-A's encoding rows made identical (a synthetic S13b query) + index = real_build(self_, dst, system) + entry = index.entries["polyA50"] + rows = np.memmap(index.path, dtype=np.float16, mode="r+", shape=(index.rows, index.width)) + rows[entry.offset:entry.offset + entry.length] = rows[entry.offset] + rows.flush() + del rows + return index + + query = ("--scoring", "esm-codes", "--max-evalue", "inf", "--threshold", "-2", "--format-output", ALL_FIELDS) + with launch_env(): + os.environ["FOLDMATCH_RUN_ID"] = "fdeg-build" + cls.results["build"] = invoke(*_build_args(cls.fasta, cls.db, cls.tmp / "bshards", "--residue-codes")) + gc.collect() + for name, patch in (("real", None), ("identical", identical_rows)): + os.environ["FOLDMATCH_RUN_ID"] = f"fdeg-{name}" + handler = _Records() + logger = logging.getLogger("foldmatch.search.code_alignment") + logger.addHandler(handler) + try: + with (mock.patch.object(EmbeddingComputer, "build_query_encodings", patch) if patch + else mock.patch.dict(os.environ)): + cls.results[name] = invoke(*_query_args(cls.db, cls.fasta, cls.tmp / "qshards", + cls.tmp / f"{name}.tsv", *query)) + finally: + logger.removeHandler(handler) + cls.warnings[name] = [r.getMessage() for r in handler.records if r.levelno == logging.WARNING] + gc.collect() + + @classmethod + def tearDownClass(cls): + cls._tmp.cleanup() + super().tearDownClass() + + def setUp(self): + for name, result in self.results.items(): + self.assertEqual(result.exit_code, 0, f"{name}: {result.output[-4000:]} {result.exception!r}") + + def rows(self, name): + return _rows(self.tmp / f"{name}.tsv", output.SUPPORTED_OUTPUT_FIELDS) + + def distinct_codes(self) -> dict: + store = ResidueCodeStore(self.tmp / "db", "fdeg") + codes = np.asarray(store.open_memmap()) + return {name: len(np.unique(codes[e.offset:e.offset + e.length], axis=0)) + for name, e in store.lookup(self.sequences).items()} + + def expected_ecal(self, qid, tid, distinct): + def nonstandard(seq): + return sum(c not in STANDARD_AMINO_ACIDS for c in seq) / len(seq) > DOMAIN.max_nonstandard_fraction + q, t = self.sequences[qid], self.sequences[tid] + inside = (DOMAIN.query_min_length <= len(q) <= DOMAIN.query_max_length and not nonstandard(q) + and len(t) <= DOMAIN.target_max_length and not nonstandard(t) + and distinct[tid] >= DOMAIN.target_min_distinct_codes) + return "1" if inside else "0" + + def test_outputs_are_finite_and_ecal_follows_s13(self): + distinct = self.distinct_codes() + self.assertEqual(distinct["Met1"], 1) + for name in ("real", "identical"): + rows = self.rows(name) + self.assertTrue(rows) + for row in rows: + with self.subTest(run=name, query=row["query"], target=row["target"]): + self.assertTrue(all(row.values()), row) + for field in NUMERIC_FIELDS: + self.assertTrue(math.isfinite(float(row[field])), (field, row[field])) + self.assertGreater(int(row["alnlen"]), 0) + self.assertEqual(row["ecal"], self.expected_ecal(row["query"], row["target"], distinct)) + + def test_d14_items_are_aligned_and_reported(self): + rows = self.rows("real") + queries = {row["query"] for row in rows} + for qid in ("allX30", "polyA50", "His6", "GS", "angiotensinI"): + self.assertIn(qid, queries) + for qid in ("allX30", "His6", "GS", "angiotensinI"): # outside S13 (non-standard, or shorter than 21 aa) + self.assertEqual({row["ecal"] for row in rows if row["query"] == qid}, {"0"}, qid) + self.assertIn("1", {row["ecal"] for row in rows if row["query"] == "polyA50"}) + + def test_a_length_1_target_gets_ecal_0(self): + on_met1 = [row for row in self.rows("real") if row["target"] == "Met1"] + self.assertTrue({row["query"] for row in on_met1} & set(_F3_CHAINS), "no query aligned to the length-1 target") + self.assertEqual({row["ecal"] for row in on_met1}, {"0"}) + + def test_fully_degenerate_queries_report_no_hits(self): + """S13b, even with --max-evalue inf: M (one residue) in both runs, poly-A when its rows are identical; each + counted in the run's one warning.""" + self.assertNotIn("Met1", {row["query"] for row in self.rows("real")}) + identical = {row["query"] for row in self.rows("identical")} + self.assertNotIn("Met1", identical) + self.assertNotIn("polyA50", identical) + for name, want in (("real", "1 fully degenerate queries"), ("identical", "2 fully degenerate queries")): + with self.subTest(run=name): + self.assertEqual(len(self.warnings[name]), 1, self.warnings[name]) + self.assertIn(want, self.warnings[name][0]) + self.assertIn("Met1", self.warnings[name][0]) + self.assertIn("polyA50", self.warnings["identical"][0]) + # The other queries' hits are those of the real run. + real = [row for row in self.rows("real") if row["query"] != "polyA50"] + self.assertEqual([row for row in self.rows("identical")], real) + + +@unittest.skipUnless(os.environ.get("FOLDMATCH_SLOW_TESTS") == "1", + "slow: real ESM3 on one and on two CPU ranks; set FOLDMATCH_SLOW_TESTS=1 to run") +@pytest.mark.usefixtures("fmsw_libs") +class TwoRankQuerySmoke(unittest.TestCase): + """P7.0a with real ESM3 (CPU, gloo): ``query sequences --devices 2``, which Lightning re-runs for rank 1, writes + the same bytes as ``--devices 1``, under blosum62 and esm-codes, on a database of the fixture built with + --residue-codes. Every command runs as a subprocess with the same environment, the torch thread count included + (Lightning would otherwise give each of the two ranks half the cores), so the runs' ESM3 numbers agree.""" + + CASES = ("blosum62", "esm-codes") + + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() + swkern.load() + cls._tmp = tempfile.TemporaryDirectory() + cls.tmp = tmp = Path(cls._tmp.name) + fasta = _order_fasta(tmp / "order.fasta") + for folder in ("db", "bshards", "qshards"): + (tmp / folder).mkdir() + env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env["PYTHONPATH"] = os.pathsep.join([_SRC] + [p for p in [os.environ.get("PYTHONPATH")] if p]) + # One torch thread in every run: Lightning gives each DDP rank cpu_count // 2 unless OMP_NUM_THREADS is set, + # and on macOS a value above 1 set at start can deadlock the FAISS flat search (torch and FAISS each bring + # an OpenMP runtime). + env["OMP_NUM_THREADS"] = "1" + cls.runs, cls.seconds = {}, {} + + def run(name, args): + t0 = time.perf_counter() + cls.runs[name] = subprocess.run([sys.executable, "-m", "foldmatch.cli.search", *map(str, args)], env=env, + capture_output=True, text=True, timeout=1200) + cls.seconds[name] = time.perf_counter() - t0 + + run("build", _build_args(_FASTA, tmp / "db" / "db", tmp / "bshards", "--residue-codes")) + for case in cls.CASES: + for devices in (1, 2): + args = list(_query_args(tmp / "db" / "db", fasta, tmp / "qshards", tmp / f"{case}-{devices}.tsv", + *_ORDER_CASES[case])) + args[args.index("--devices") + 1] = devices + run(f"{case} {devices}", args) + print("\nP7.0a two-rank smoke wall times (s): " + ", ".join(f"{k} {v:.1f}" for k, v in cls.seconds.items()), + file=sys.stderr) + + @classmethod + def tearDownClass(cls): + cls._tmp.cleanup() + super().tearDownClass() + + def setUp(self): + for name, run in self.runs.items(): + self.assertEqual(run.returncode, 0, f"{name}: {run.stderr[-4000:]}") + + def test_two_ranks_write_the_bytes_of_one(self): + for case in self.CASES: + with self.subTest(case=case): + one, two = self.tmp / f"{case}-1.tsv", self.tmp / f"{case}-2.tsv" + self.assertEqual(MultiRankOrderTest.query_order(one), _INPUT_ORDER) + self.assertEqual(len(one.read_text().splitlines()), 10) + self.assertEqual(_sha256(two), _sha256(one)) + + def test_the_second_run_had_two_ranks(self): + for case in self.CASES: + with self.subTest(case=case): + self.assertIn("Streaming embeddings from 1 Parquet shard(s)", self.runs[f"{case} 1"].stderr) + self.assertIn("Streaming embeddings from 2 Parquet shard(s)", self.runs[f"{case} 2"].stderr) + self.assertEqual(list((self.tmp / "qshards").iterdir()), []) + + +class _Records(logging.Handler): + def __init__(self): + super().__init__(logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_help.py b/tests/test_cli_help.py new file mode 100644 index 0000000..e806cb7 --- /dev/null +++ b/tests/test_cli_help.py @@ -0,0 +1,141 @@ +"""CLI help text that states what the code does (G10).""" +import unittest + +import click +import pytest +import typer + +from foldmatch.cli import embedding, inference +from foldmatch.cli.search import app + + +def _option_help(path, param_name): + command = typer.main.get_command(app) + for name in path: + command = command.commands[name] + return next(p.help for p in command.params if p.name == param_name) + + +class MinCoverageHelpTest(unittest.TestCase): + + def test_help_says_both_sides_must_reach_the_value(self): + for path in (("query", "sequences"), ("query", "db")): + with self.subTest(command=" ".join(path)): + text = _option_help(path, "min_coverage") + self.assertIn("both", text) + self.assertIn("either side is below", text) + self.assertNotIn("AND", text) + + def test_align_candidates_docstring_says_the_same(self): + from foldmatch.search import alignment + doc = " ".join(alignment.align_candidates.__doc__.split()) + self.assertIn("min_coverage: drop hits whose query or subject coverage is below this", doc) + self.assertIn("min(qcov, tcov) >= min_coverage", doc) + self.assertNotIn("*and*", doc) + + @pytest.mark.usefixtures("fmsw_libs") + def test_filter_drops_a_hit_with_one_side_below(self): + """The behaviour the help describes: min(qcov, tcov) >= --min-coverage.""" + from foldmatch.search import alignment + query = "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVQAPILSRVGDGTQDNLSGAEKAVQVKVKALPDAQ" + subject = query + "GSHMASMTGGQQMGRGSEFELRRQACGRTRAPPPPPLRSGC" * 2 # tcov ~ 0.44 + kwargs = dict( + query_sequences={"q": query}, + prefilter_results={"q": (["s"], [0.9])}, + fetch_subject_sequences=lambda ids: {"s": subject}, + min_seq_identity=0.0, + max_evalue=None, + num_workers=1, + compute_significance=False, + ) + kept = alignment.align_candidates(min_coverage=0.4, **kwargs)["q"] + self.assertEqual(len(kept), 1) + metrics = kept[0].metrics + self.assertGreaterEqual(metrics.query_coverage, 0.99) + self.assertLess(metrics.subject_coverage, 0.8) + self.assertEqual(alignment.align_candidates(min_coverage=0.8, **kwargs)["q"], []) + + +def _commands(command, path=()): + """Every command under a click command, by path.""" + if isinstance(command, click.Group): + for name, sub in command.commands.items(): + yield from _commands(sub, (*path, name)) + else: + yield path, command + + +def _inference_help(name): + """The help of an inference command and of each of its options, by parameter name.""" + command = typer.main.get_command(inference.app).commands[name] + return command.help, {p.name: p.help for p in command.params} + + +class LogLevelHelpTest(unittest.TestCase): + + def test_every_log_level_help_is_about_logging(self): + """--log-level read "Number of nodes to use for inference of embeddings." in 10 commands.""" + seen = 0 + for typer_app in (app, inference.app, embedding.app): + for path, command in _commands(typer.main.get_command(typer_app)): + for param in command.params: + if param.name == "log_level": + seen += 1 + with self.subTest(command=" ".join(path)): + self.assertRegex(param.help, r"^Logging (verbosity )?level\.$") + self.assertGreaterEqual(seen, 24) + + def test_no_other_option_reads_like_the_old_text(self): + for typer_app in (app, inference.app, embedding.app): + for path, command in _commands(typer.main.get_command(typer_app)): + for param in command.params: + with self.subTest(command=" ".join(path), param=param.name): + self.assertNotIn("inference of embeddings", getattr(param, "help", None) or "") + + +class InferenceOutputHelpTest(unittest.TestCase): + """The help of the inference commands says what they write (test_inference_outputs.py runs them).""" + + def test_structure_embedding_says_parquet(self): + """It said a pandas DataFrame file, an option out-df-name that does not exist, and a JSON file.""" + text, options = _inference_help("structure-embedding") + for help_text in (text, options["output_folder"], options["output_name"]): + self.assertIn("-.parquet", help_text) + for help_text in (text, options["output_folder"], options["output_name"]): + for stale in ("DataFrame", "out-df-name", "JSON"): + self.assertNotIn(stale, help_text) + + def test_the_output_format_commands_say_so(self): + """Each said one format, the default or none of them, whatever --output-format; complete-embedding a single + JSON file.""" + for name, default, folders in (("residue-embedding", "torch tensor files by default", + {"output_folder": "chain"}), + ("chain-embedding", "csv files by default", {"output_folder": "chain"}), + ("assembly-embedding", "csv files by default", {"output_folder": "assembly"}), + ("complete-embedding", "csv files by default", + {"output_chain_path": "chain", "output_assembly_path": "assembly"})): + with self.subTest(command=name): + text, options = _inference_help(name) + self.assertIn("in the format of output-format", text) + self.assertIn(default, text) + for folder, item in folders.items(): + self.assertIn("in the format of output-format", options[folder]) + self.assertIn(f"one file per {item} for csv and pt", options[folder]) + self.assertIn("one file per process for parquet and json", options[folder]) + self.assertNotIn("single JSON file", options[folder]) + + def test_the_residue_file_formats_are_named(self): + """chain-embedding and assembly-embedding also read csv residue embeddings.""" + for name in ("chain-embedding", "assembly-embedding"): + with self.subTest(command=name): + self.assertIn("stored as torch tensor or csv files (res-embedding-format)", _inference_help(name)[0]) + + def test_chain_embedding_does_not_list_structure_format(self): + self.assertNotIn("structure_format", _inference_help("chain-embedding")[1]) + for name in ("residue-embedding", "structure-embedding", "assembly-embedding", "complete-embedding"): + with self.subTest(command=name): + self.assertIn("structure_format", _inference_help(name)[1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_logging.py b/tests/test_cli_logging.py new file mode 100644 index 0000000..809f5bc --- /dev/null +++ b/tests/test_cli_logging.py @@ -0,0 +1,267 @@ +"""``set_log_level``, the logging set-up of every ``fm-search``, ``fm-embedding`` and ``inference`` command +(``--log-level info|warn|debug``).""" +import json +import logging +import os +import re +import subprocess +import sys +import tempfile +import unittest +import warnings +from contextlib import contextmanager +from pathlib import Path + +import foldmatch +from foldmatch.cli import args_utils +from foldmatch.cli.args_utils import RankFilter, set_log_level +from foldmatch.types.api_types import LogLevel + +_SRC = str(Path(foldmatch.__file__).resolve().parent.parent) + +#: The format of ``--log-level warn`` and ``--log-level debug``. +_DETAILED = '> %(asctime)s - [proc-%(rank)s] - %(levelname)s - %(filename)s:%(lineno)d - %(message)s' +_LIGHTNING_LOGGERS = ("lightning.pytorch", "lightning.fabric.utilities.distributed") + + +@contextmanager +def _logging_restored(): + """The root logger's handlers and level, the Lightning loggers' levels, the warnings filters and what a deferred + ``set_log_level`` recorded, as they were.""" + root = logging.getLogger() + handlers, level = list(root.handlers), root.level + levels = {name: logging.getLogger(name).level for name in _LIGHTNING_LOGGERS} + recorded = dict(args_utils._ROOT_BEFORE_LIGHTNING) + try: + with warnings.catch_warnings(): + yield root + finally: + args_utils._ROOT_BEFORE_LIGHTNING.clear() + args_utils._ROOT_BEFORE_LIGHTNING.update(recorded) + for handler in list(root.handlers): + root.removeHandler(handler) + for handler in handlers: + root.addHandler(handler) + root.setLevel(level) + for name, value in levels.items(): + logging.getLogger(name).setLevel(value) + + +class SetLogLevelTest(unittest.TestCase): + + def configured(self, level): + """The root logger's level and its one handler's format after ``set_log_level(level)``.""" + with _logging_restored() as root: + set_log_level(level) + self.assertEqual(len(root.handlers), 1, root.handlers) + [handler] = root.handlers + self.assertTrue(any(isinstance(f, RankFilter) for f in handler.filters)) + return root.level, handler.formatter._fmt + + def test_every_level_sets_up_logging(self): + """P9.L1: each value of ``--log-level`` has its configuration; ``warn`` had none (the branch tested + ``'warning'``), so WARNINGs printed through Python's last-resort handler, without the format.""" + expected = {LogLevel.info: (logging.INFO, '[proc-%(rank)s] %(message)s'), + LogLevel.warn: (logging.WARNING, _DETAILED), + LogLevel.debug: (logging.DEBUG, _DETAILED)} + self.assertEqual(set(expected), set(LogLevel)) + for level, configuration in expected.items(): + for value in (level, level.value): + with self.subTest(level=value): + self.assertEqual(self.configured(value), configuration) + + def test_an_unknown_level_is_refused(self): + for level in ("warning", "WARN", "", None): + with self.subTest(level=level), _logging_restored() as root: + before = list(root.handlers) + with self.assertRaisesRegex(ValueError, "unknown log level .*: one of info, warn, debug"): + set_log_level(level) + self.assertEqual(root.handlers, before) + + +# ===================================================================================================================== +# P9.L2: set_log_level(..., defer_lightning=True), and Lightning imported before, by or after set_log_level +# ===================================================================================================================== +#: A sitecustomize: any import of torch, Lightning, ESM, FAISS, transformers or huggingface_hub ends the process with +#: status 97, naming the module (as ``test_cli_split_build._NO_HEAVY_IMPORTS``). +_NO_HEAVY_IMPORTS = r""" +import os, sys +_BLOCKED = ("torch", "lightning", "lightning_fabric", "pytorch_lightning", "esm", "faiss", "transformers", + "huggingface_hub") +class _NoHeavyImports: + def find_spec(self, name, path=None, target=None): + if name.partition(".")[0] in _BLOCKED: + sys.stderr.write(f"FORBIDDEN IMPORT: {name}\n") + sys.stderr.flush() + os._exit(97) + return None +sys.meta_path.insert(0, _NoHeavyImports()) +""" + +#: Sets up logging at $DRIVER_LEVEL as $DRIVER_MODE says, then logs FoldMatch's records, Lightning's own (a Trainer's +#: INFO lines among them), a warning; prints the logging, warnings and progress-bar state as JSON. The mode and level +#: come from the environment, so that every run has the same sys.argv (Lightning's srun hint, where srun exists, +#: quotes it). +_LIGHTNING_DRIVER = r""" +import json, logging, os, sys, warnings +mode, level = os.environ["DRIVER_MODE"], os.environ["DRIVER_LEVEL"] +from foldmatch.cli.args_utils import set_log_level +if mode.startswith("configured before"): + logging.basicConfig(level=logging.WARNING, format="configured before: %(message)s") +if mode == "imported before": + import lightning +elif "deferred" in mode: + set_log_level(level, defer_lightning=True) + if mode == "deferred, imported meanwhile": + import lightning +set_log_level(level) +sys.stderr.write("=== set up\n") +sys.stderr.flush() +logging.debug("fm debug") +logging.info("fm info") +logging.warning("fm warning") +logging.getLogger("lightning.pytorch.core").error("a lightning error") +logging.getLogger("lightning.fabric.utilities.distributed").info("Initializing distributed: GLOBAL_RANK: 0") +warnings.warn("a user warning") +import lightning +lightning.Trainer(accelerator="cpu", devices=1, logger=False, enable_checkpointing=False, enable_progress_bar=False, + enable_model_summary=False) +from huggingface_hub.utils import are_progress_bars_disabled +import transformers.utils.logging as transformers_logging +def handlers(logger): + return [(type(h).__name__, h.level, getattr(h.formatter, "_fmt", None), [type(f).__name__ for f in h.filters]) + for h in logger.handlers] +loggers = {name or "root": {"level": logging.getLogger(name).level, "propagate": logging.getLogger(name).propagate, + "handlers": handlers(logging.getLogger(name))} + for name in ("", "lightning", "lightning.pytorch", "lightning.fabric", + "lightning.fabric.utilities.distributed")} +print(json.dumps({"loggers": loggers, "warnings": [repr(f) for f in warnings.filters], + "hub bars disabled": are_progress_bars_disabled(), + "transformers tqdm": transformers_logging._tqdm_active, + "env": {k: os.environ.get(k) for k in ("KMP_DUPLICATE_LIB_OK", "PYTORCH_NVML_BASED_CUDA_CHECK", + "HF_HUB_DISABLE_PROGRESS_BARS")}})) +""" + +#: The deferred set-up alone: FoldMatch's records and a warning, then whether huggingface_hub was imported. +_DEFERRED_DRIVER = r""" +import json, logging, os, sys, warnings +from foldmatch.cli.args_utils import set_log_level +set_log_level(os.environ["DRIVER_LEVEL"], defer_lightning=os.environ["DRIVER_MODE"] == "deferred") +logging.debug("fm debug") +logging.info("fm info") +logging.warning("fm warning") +warnings.warn("a user warning") +print(json.dumps({"HF_HUB_DISABLE_PROGRESS_BARS": os.environ.get("HF_HUB_DISABLE_PROGRESS_BARS")})) +""" + +_TIME = re.compile(r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3}") + + +def _line(driver, statement): + """The line number of ``statement`` in ``driver``, as the records it logs carry it (``:N``).""" + return driver.splitlines().index(statement) + 1 + + +class LightningLoggingTest(unittest.TestCase): + """P9.L2: the log output and the logging state are the same whether Lightning is imported by ``set_log_level`` + (as before), before it, or by a second ``set_log_level`` after a ``set_log_level(..., defer_lightning=True)``; and + when Lightning was imported between the two, from the second call on. At ``info`` Lightning's INFO lines (a + Trainer's 'GPU available: ...') stay hidden, at ``debug`` they show; the rank prefix is the launcher's.""" + + MODES = ("by set_log_level", "imported before", "deferred", "deferred, imported meanwhile") + CONFIGURED_BEFORE = ("configured before, by set_log_level", "configured before, deferred") + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + + def run_driver(self, driver, mode, level, hook=False): + (self.tmp / "site").mkdir(exist_ok=True) + (self.tmp / "site" / "sitecustomize.py").write_text(_NO_HEAVY_IMPORTS) + path = ([str(self.tmp / "site")] if hook else []) + [_SRC] + env = {k: v for k, v in os.environ.items() if k not in ("HF_HUB_DISABLE_PROGRESS_BARS", "PYTHONWARNINGS")} + env = {k: v for k, v in env.items() if k not in ("RANK", "LOCAL_RANK", "SLURM_PROCID", "JSM_NAMESPACE_RANK")} + # GLOBAL_RANK gives the records' rank prefix; Lightning, which reads the others, logs as rank 0. + env.update(PYTHONPATH=os.pathsep.join(path), GLOBAL_RANK="3", HF_HUB_OFFLINE="1", DRIVER_MODE=mode, + DRIVER_LEVEL=level) + result = subprocess.run([sys.executable, "-c", driver], env=env, cwd=self.tmp, capture_output=True, + text=True, timeout=600) + self.assertNotIn("FORBIDDEN IMPORT", result.stderr) + self.assertEqual(result.returncode, 0, result.stderr) + return _TIME.sub("TIME", result.stderr), json.loads(result.stdout) + + def test_lightning_imported_before_by_or_after_set_log_level(self): + for level in LogLevel: + runs = {mode: self.run_driver(_LIGHTNING_DRIVER, mode, level.value) for mode in self.MODES} + stderr, state = runs["by set_log_level"] + lines = stderr.splitlines() + with self.subTest(level=level): + self.assertEqual(lines[0], "=== set up", stderr) + self.assertIn("a lightning error", lines) # Lightning's own handler, as before + if level is LogLevel.info: + self.assertEqual(lines[1:3], ["[proc-3] fm info", "[proc-3] fm warning"], stderr) + for hidden in ("GPU available", "Initializing distributed", "fm debug", "a user warning"): + self.assertNotIn(hidden, stderr) + else: + warning, info = (_line(_LIGHTNING_DRIVER, f'logging.{name}("fm {name}")') + for name in ("warning", "info")) + self.assertIn(f"> TIME - [proc-3] - WARNING - :{warning} - fm warning", lines) + self.assertIn("Initializing distributed: GLOBAL_RANK: 0", lines) + self.assertTrue(any(line.startswith("GPU available: ") for line in lines), stderr) + self.assertIn("UserWarning: a user warning", stderr) + self.assertEqual(f"> TIME - [proc-3] - INFO - :{info} - fm info" in lines, + level is LogLevel.debug, stderr) + self.assertEqual(state["loggers"]["lightning.pytorch"]["propagate"], False) + self.assertEqual(state["loggers"]["lightning.pytorch"]["level"], + logging.ERROR if level is LogLevel.info else logging.INFO) + self.assertEqual(state["hub bars disabled"], level is LogLevel.info) + for mode in self.MODES[1:]: + with self.subTest(level=level, mode=mode): + other, other_state = runs[mode] + self.assertEqual(other_state, state) + if mode == "deferred, imported meanwhile": # what that import logged is not compared + self.assertEqual(other.partition("=== set up")[2], stderr.partition("=== set up")[2]) + else: + self.assertEqual(other, stderr) + + def test_a_root_logger_configured_before(self): + """With the root logger configured before ``set_log_level`` (in process), Lightning gives its loggers no + handler of their own, and a deferred set-up imports it with the root logger at the level it found: the same + output and state as ``set_log_level(level)``.""" + for level in LogLevel: + (stderr, state), (other, other_state) = (self.run_driver(_LIGHTNING_DRIVER, mode, level.value) + for mode in self.CONFIGURED_BEFORE) + with self.subTest(level=level): + self.assertEqual((state["loggers"]["lightning.pytorch"]["propagate"], + state["loggers"]["lightning.pytorch"]["handlers"]), (True, [])) + self.assertEqual("GPU available" in stderr, level is not LogLevel.info, stderr) + self.assertEqual((other, other_state), (stderr, state)) + + def test_the_deferred_set_up_imports_nothing_heavy(self): + """With ``defer_lightning=True`` the records come in the format of the level, the rank prefix included, as + after ``set_log_level(level)``, with nothing imported that imports torch (the hook; ``RankFilter`` imported + torch.distributed at the first record before P9.L2), and the progress bars are left alone (no + HF_HUB_DISABLE_PROGRESS_BARS).""" + for level in LogLevel: + with self.subTest(level=level): + deferred, state = self.run_driver(_DEFERRED_DRIVER, "deferred", level.value, hook=True) + self.assertEqual(state, {"HF_HUB_DISABLE_PROGRESS_BARS": None}) + self.assertEqual(deferred, self.run_driver(_DEFERRED_DRIVER, "by set_log_level", level.value)[0]) + self.assertIn("[proc-3] fm warning" if level is LogLevel.info else "- [proc-3] - WARNING - ", deferred) + + def test_the_progress_bars_when_huggingface_hub_is_imported(self): + """At ``info`` the deferred set-up disables the Hugging Face progress bars as before when huggingface_hub is + already imported.""" + from huggingface_hub.utils import are_progress_bars_disabled, enable_progress_bars + try: + with _logging_restored(): + enable_progress_bars() + set_log_level(LogLevel.debug, defer_lightning=True) + self.assertFalse(are_progress_bars_disabled()) + set_log_level(LogLevel.info, defer_lightning=True) + self.assertTrue(are_progress_bars_disabled()) + finally: + enable_progress_bars() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_residue_codes.py b/tests/test_cli_residue_codes.py new file mode 100644 index 0000000..62b325e --- /dev/null +++ b/tests/test_cli_residue_codes.py @@ -0,0 +1,948 @@ +"""P5.6-P5.7: ``fm-search build sequences --residue-codes`` and ``fm-search build residue-codes --from-shards``. + +Most tests replace the ESM3 pass by a fake that writes the shard the real pass writes (same dataset, same writer) +with the F2c reference codes for the fixture FASTA, so the command-line orchestration, the refusals and the code +store run without ESM3. ``RealEsm3CliTest`` runs the real CPU pass through the command line and compares the codes +with F2c (gate A4-xcpu). The bundle comes from the scoring-system registry (``esm_codes_bundle``: skipped only when +offline and not cached). +""" +import dataclasses +import gc +import hashlib +import io +import json +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +import zlib +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import torch +from typer.testing import CliRunner + +import esm_codes_bundle +import foldmatch +from foldmatch.cli import search +from foldmatch.errors import RefusalError +from foldmatch.search import scoring_system as ss +from foldmatch.search.residue_code_store import CodeStoreValidationError, ResidueCodeStore, entries_digest + +_TEST_PATH = Path(__file__).parent +_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_F2C = _TEST_PATH / "resources" / "esm_codes" / "fasta_rows_codes_v1.npz" +_SRC = str(Path(foldmatch.__file__).resolve().parent.parent) +_LAUNCH_ENV = ("FOLDMATCH_RUN_ID", "SLURM_JOB_ID", "SLURM_STEP_ID", "SLURM_RESTART_COUNT", "SLURM_PROCID", + "SLURM_NTASKS", "SLURM_STEP_NUM_TASKS", "RANK", "LOCAL_RANK", "NODE_RANK", "WORLD_SIZE") + +SYSTEM = ss.get(ss.DEFAULT_ESM_CODES) +# Another encoding (a different target encoder): the shards of another system. +OTHER_ENCODING = dataclasses.replace(SYSTEM, name="esm-codes-test-other", target_encoder_sha256="0" * 64) + +#: Plan section 8, P5.6: the help text of --residue-codes, verbatim. +PLAN_HELP = ("Also compute 8-byte per-residue ESM3 codes in the same pass, enabling `query sequences --scoring " + "esm-codes`. Adds 8 B per residue plus up to about 100 B per entry for the index, and a one-time ~26 MB " + "download. Sequence (FASTA) databases only.") +#: A4_THRESHOLDS.json (sha256 7eabf36e...), gate A4-xcpu: CPU Mac vs CPU Linux code bits, also the F2c threshold. +A4_XCPU_BITS = 0.999 +_DB_SUFFIXES = (".index", ".metadata", ".sequences") + + +@contextmanager +def launch_env(**values): + """Exactly ``values`` among the launch variables; the rest unset (only those variables are restored).""" + saved = {name: os.environ.get(name) for name in _LAUNCH_ENV} + try: + for name in _LAUNCH_ENV: + os.environ.pop(name, None) + os.environ.update({k: str(v) for k, v in values.items()}) + yield + finally: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _f2c(): + """F2c: ``{sequence: reference codes [L, 8]}``, the ids and sequences in file order, all codes ``[303, 8]``.""" + z = np.load(_F2C, allow_pickle=False) + offsets = z["offsets"] + ids = [str(i) for i in z["ids"]] + sequences = [str(s) for s in z["sequences"]] + by_sequence = {s: z["codes"][offsets[k]:offsets[k + 1]] for k, s in enumerate(sequences)} + return by_sequence, ids, sequences, z["codes"] + + +def _sha1(sequence: str) -> bytes: + return hashlib.sha1(sequence.encode("ascii")).digest() + + +def _sha256(path) -> str: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def _bits_equal(a, b): + return int((np.unpackbits(a, axis=1) == np.unpackbits(b, axis=1)).sum()), int(a.size * 8) + + +def _write(path: Path, text: str) -> Path: + path.write_text(text, encoding="utf-8") + return path + + +def _fake_codes(name: str, sequence: str) -> np.ndarray: + """The F2c reference codes of the fixture's sequences; pseudo-random codes (fixed per id) for any other.""" + reference = _f2c()[0].get(sequence) + if reference is not None: + return reference + return np.random.default_rng(zlib.crc32(name.encode())).integers(0, 256, (len(sequence), 8), dtype=np.uint8) + + +def fake_chain_predict(calls=None, payload=_fake_codes, system=None, ranks=1, device="cpu", by_sequence=False): + """Stand-in for ``chain_complete_inference.predict``: the shards the pass writes, from the same dataset and + writer (with the writer's start-of-pass hook, which records the pass), with an 8-d chain embedding fixed per id + (``by_sequence``: per sequence, as ESM3 gives it, so two records with one id differ) and, for a residue pass, + ``payload(name, sequence)`` as the residue codes (``system`` overrides the scoring system the shards are written + for). With ``ranks`` > 1 the entries go round-robin to that many rank shards, as DDP's sampler deals them; + ``device`` is the device type the module 'runs' on.""" + def predict(**kwargs): + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + from foldmatch.inference.chain_complete_inference import residue_shard_writer + from foldmatch.writer.batch_writer import ParquetBatchWriter + if calls is not None: + calls.append(kwargs) + dataset = EsmProtFromFasta(kwargs["src_stream"], min_res_n=kwargs["min_res_n"]) + residue_output = kwargs.get("residue_output") + for rank in range(ranks): + items = [dataset[i] for i in range(rank, len(dataset), ranks)] + names = tuple(name for _, name in items) + keys = [prot.sequence for prot, _ in items] if by_sequence else names + embeddings = [torch.from_numpy(np.random.default_rng(zlib.crc32(key.encode())).normal(size=8) + .astype(np.float32)) for key in keys] + trainer = SimpleNamespace(global_rank=rank, world_size=ranks) + if residue_output is None: + writer = ParquetBatchWriter(kwargs["out_folder"], kwargs["out_name"]) + prediction = (embeddings, names) + else: + shard_system = system or kwargs["scoring_system"].system + writer = residue_shard_writer(kwargs["out_folder"], kwargs["out_name"], residue_output, shard_system, + min_res_n=kwargs["min_res_n"]) + writer.on_predict_start(trainer, SimpleNamespace(residue_output=residue_output, + device=torch.device(device))) + payloads = tuple(np.ascontiguousarray(payload(name, prot.sequence)).tobytes() for prot, name in items) + prediction = (embeddings, names, payloads, tuple(_sha1(prot.sequence) for prot, _ in items)) + writer.write_on_batch_end(trainer, None, prediction, None, None, 0, 0) + writer.on_predict_end(trainer, None) + return predict + + +def _esm3_spy(): + """Patch the ESM3 loader of the pass; any call means ESM3 was reached.""" + return mock.patch("foldmatch.inference.chain_complete_inference.get_residue_model", + mock.Mock(side_effect=AssertionError("ESM3 was loaded"))) + + +def invoke(*args): + """``fm-search`` in-process (exit codes as in a shell, the exception kept), the logging setup restored after.""" + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(search.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + +def _build_args(fasta, db, shards, *extra): + return ("build", "sequences", "--fasta-file", fasta, "--output-db", db, "--tmp-embedding-folder", shards, + "--accelerator", "cpu", "--devices", "1", *extra) + + +class _Tmp(unittest.TestCase): + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.shards = self.tmp / "shards" + self.shards.mkdir() + self.enterContext(launch_env()) + + def refused(self, result, cause=None): + """The command exited 1 with a RefusalError (D20); returns its message.""" + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError, repr(result.exception)) + if cause is not None: + self.assertIsInstance(result.exception.__cause__, cause, repr(result.exception.__cause__)) + return str(result.exception) + + def usage_error(self, result, text): + """The command exited 2 (typer.BadParameter or a click usage error) naming ``text`` (in typer's error panel, + whose borders are dropped).""" + self.assertEqual(result.exit_code, 2, result.output) + self.assertIn(text, " ".join(result.output.translate({ord(c): " " for c in "│╭╮╰╯─"}).split())) + + +class _WithBundle(_Tmp): + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() # unittest.SkipTest only when offline and not cached + + +# ===================================================================================================================== +# P5.6: the option +# ===================================================================================================================== +class OptionTest(unittest.TestCase): + + def _command(self, *path): + import typer + command = typer.main.get_command(search.app) + for name in path: + command = command.commands[name] + return command + + def test_help_text_and_default(self): + param = next(p for p in self._command("build", "sequences").params if p.name == "residue_codes") + self.assertEqual(param.help, PLAN_HELP) + self.assertEqual(param.opts, ["--residue-codes"]) + self.assertEqual(param.secondary_opts, ["--no-residue-codes"]) + self.assertIs(param.default, False) + output = CliRunner().invoke(search.app, ["build", "sequences", "--help"], env={"COLUMNS": "400"}).output + help_text = " ".join(output.translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + self.assertIn(f"--residue-codes --no-residue-codes {PLAN_HELP} [default: no-residue-codes]", help_text) + + +# ===================================================================================================================== +# P5.6: refusals, every one before ESM3 and on every rank alike +# ===================================================================================================================== +class BuildRefusalTest(_Tmp): + + def build(self, fasta=_FASTA, *extra): + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", + side_effect=AssertionError("bundle fetched")) as prepare: + result = invoke(*_build_args(fasta, self.tmp / "db", self.shards, "--residue-codes", *extra)) + esm3.assert_not_called() + prepare.assert_not_called() + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), []) + self.assertEqual(list(self.shards.iterdir()), []) + return result + + def test_fasta_the_method_cannot_take(self): + for text, what in ((">a\nACDE\n>b\nKLM\n>a\nACDE\n", "repeat an earlier id"), + (">a\nACDE\n", "'<' or '>'"), (">a\nACDE>\n", "'<' or '>'"), + (">a\nACDÉ\n", "non-ASCII")): + for rank_env in ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}): + with self.subTest(what=what, text=text, rank_env=rank_env), launch_env(**rank_env): + fasta = _write(self.tmp / "bad.fasta", text) + self.assertIn(what, self.refused(self.build(fasta), ValueError)) + + def test_a_header_without_an_id(self): + """'>' alone: refused by the pre-flight before ESM3, with the file named (it used to be an IndexError); a + plain build refuses it the same way, as a ValueError.""" + fasta = _write(self.tmp / "h.fasta", ">\nACDE\n>b\nKLMN\n") + message = self.refused(self.build(fasta), ValueError) + self.assertIn(f"{fasta}: the header line at byte 0 is empty", message) + with _esm3_spy() as esm3: + result = invoke(*_build_args(fasta, self.tmp / "db", self.shards)) + self.assertEqual(result.exit_code, 1) + self.assertIs(type(result.exception), ValueError) + self.assertIn("is empty ('>' without an id)", str(result.exception)) + esm3.assert_not_called() + + def test_a_plain_build_of_the_same_fasta_is_not_strict(self): + fasta = _write(self.tmp / "dup.fasta", ">a\nACDE\n>a\nACDE\n") + with _esm3_spy() as esm3: + result = invoke(*_build_args(fasta, self.tmp / "db", self.shards)) + self.assertIsInstance(result.exception, AssertionError) # the pass was reached + esm3.assert_called_once() + + def test_existing_outputs(self): + for suffix in (".index", ".metadata", ".invlists", ".sequences", ".rescodes.bin", ".rescodes.idx", + ".rescodes.json", ".rescodes.json.tmp"): + for rank_env in ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}): + with self.subTest(suffix=suffix, rank_env=rank_env), launch_env(**rank_env): + existing = self.tmp / f"db{suffix}" + existing.touch() + try: + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare") as prepare: + result = invoke(*_build_args(_FASTA, self.tmp / "db", self.shards, "--residue-codes")) + finally: + existing.unlink() + self.assertIn(str(existing), self.refused(result, FileExistsError)) + esm3.assert_not_called() + prepare.assert_not_called() + + def test_shards_of_the_same_run_id(self): + with launch_env(FOLDMATCH_RUN_ID="r"): + (self.shards / "emb_r-0.parquet").touch() + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare") as prepare: + result = invoke(*_build_args(_FASTA, self.tmp / "db", self.shards, "--residue-codes")) + self.assertIn("same run id", self.refused(result, FileExistsError)) + esm3.assert_not_called() + prepare.assert_not_called() + + def test_a_bundle_problem(self): + with _esm3_spy() as esm3, mock.patch.object( + ss, "prepare", side_effect=ss.BundleUnavailableError("not cached")) as prepare: + result = invoke(*_build_args(_FASTA, self.tmp / "db", self.shards, "--residue-codes")) + self.assertIn("not cached", self.refused(result, ss.BundleUnavailableError)) + prepare.assert_called_once() + esm3.assert_not_called() + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), []) + + def test_one_fetch_then_cached_only(self): + """The process that starts the run may download the bundle; a rank Lightning starts, or a multi-node run, + finds it cached (P4 outcome: prepare(system, local_files_only=True) as the every-rank pre-flight).""" + stop = RuntimeError("stop before the pass") + for rank_env, extra, local_only in (({}, (), False), ({"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}, (), True), + ({}, ("--num-nodes", "2"), True)): + with self.subTest(rank_env=rank_env, extra=extra), launch_env(**rank_env), \ + mock.patch.object(ss, "prepare") as prepare, \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", side_effect=stop) as predict: + result = invoke(*_build_args(_FASTA, self.tmp / "db", self.shards, "--residue-codes", *extra)) + self.assertIs(result.exception, stop) + prepare.assert_called_once_with(SYSTEM, local_files_only=local_only) + kwargs = predict.call_args.kwargs + self.assertEqual(kwargs["residue_output"], "residue_codes") + self.assertIs(kwargs["scoring_system"], prepare.return_value) + + def test_exit_status_of_a_refusal(self): + """fm-search prints this refusal on one line and exits 1 (D20).""" + fasta = _write(self.tmp / "dup.fasta", ">a\nACDE\n>a\nACDE\n") + argv = ["fm-search", *map(str, _build_args(fasta, self.tmp / "db", self.shards, "--residue-codes"))] + stderr = self.enterContext(mock.patch("sys.stderr", new_callable=io.StringIO)) + root = logging.getLogger() + handlers = root.handlers[:] + try: + with mock.patch.object(sys, "argv", argv), _esm3_spy(), self.assertRaises(SystemExit) as ctx: + search.main() + finally: + root.handlers[:] = handlers + self.assertEqual(ctx.exception.code, 1) + errors = [line for line in stderr.getvalue().splitlines() if line.startswith("Error: ")] + self.assertEqual(len(errors), 1, stderr.getvalue()) + self.assertIn("repeat an earlier id", errors[0]) + + +# ===================================================================================================================== +# P5.6: the build (fake ESM3 pass, real bundle) +# ===================================================================================================================== +class BuildTest(_WithBundle): + + def build(self, *extra, fasta=_FASTA, db=None, calls=None, run_id="p56", **fake): + db = db or self.tmp / "db" + with launch_env(FOLDMATCH_RUN_ID=run_id), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(calls, **fake)): + return invoke(*_build_args(fasta, db, self.shards, *extra)) + + def test_the_store_is_built_and_validated(self): + calls = [] + result = self.build("--residue-codes", calls=calls) + self.assertEqual(result.exit_code, 0, result.output) + _, ids, sequences, codes = _f2c() + store = ResidueCodeStore(self.tmp, "db") + self.assertTrue(store.exists()) + manifest = json.loads(store.manifest_path.read_text()) + self.assertEqual((manifest["entries"], manifest["total_residues"], manifest["min_res_n"]), (2, 303, 0)) + self.assertEqual(manifest, store.check_compatible()) + self.assertEqual(store.bin_path.stat().st_size, 8 * 303) + np.testing.assert_array_equal(store.open_memmap(), codes) + self.assertEqual(manifest["bin_sha256"], hashlib.sha256(codes.tobytes()).hexdigest()) + self.assertEqual(manifest["entries_digest"], entries_digest([(i, _sha1(s)) for i, s in zip(ids, sequences)])) + self.assertEqual({i: e[:2] for i, e in store.lookup(ids).items()}, {"1acb_E": (0, 245), "2uzi_A": (245, 58)}) + self.assertEqual(manifest["cross_checks"], { + "preflight_entries": 2, "preflight_residues": 303, "sequence_store_entries": 2, + "sequence_store_residues": 303, "faiss_entries": 2, "seq_sha1_mismatches": 0, "length_mismatches": 0}) + build = manifest["build"] + self.assertEqual((build["run_id"], build["from_shards"], build["accelerator"], build["devices"], + build["esm_precision"], build["encoder_precision"]), + ("p56", False, "cpu", 1, "fp32", "fp32-no-tf32")) + from foldmatch.search.residue_shards import read_shard_info + record = read_shard_info(self.shards / "emb_p56-0.parquet").pass_record + for key in ("foldmatch_version", "torch", "esm", "platform"): # the pass's, from the shards + self.assertEqual(build[key], record[key], key) + self.assertEqual(build["foldmatch_version"], foldmatch.__version__) + self.assertEqual((record["min_res_n"], record["esm_revision"]), (0, manifest["esm"]["revision"])) + self.assertEqual(manifest["system_name"], SYSTEM.name) + self.assertEqual(manifest["encoding_digest"], SYSTEM.encoding_digest()) + # The pass got the prepared registry system; the shards are kept (D13). + self.assertEqual(calls[0]["residue_output"], "residue_codes") + self.assertIs(calls[0]["scoring_system"].system, SYSTEM) + self.assertEqual([p.name for p in self.shards.iterdir()], ["emb_p56-0.parquet"]) + for suffix in _DB_SUFFIXES: + self.assertTrue((self.tmp / f"db{suffix}").exists()) + + def test_a_plain_build_is_unchanged(self): + """Without the option: the pass is called without residue keywords, no bundle is fetched, no code store is + written, and the database files are byte-identical to those of a --residue-codes build.""" + calls = [] + with mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched")): + result = self.build(calls=calls, db=self.tmp / "plain" / "db") + self.assertEqual(result.exit_code, 0, result.output) + self.assertNotIn("residue_output", calls[0]) + self.assertNotIn("scoring_system", calls[0]) + self.assertEqual(sorted(p.name for p in (self.tmp / "plain").iterdir()), + ["db.index", "db.metadata", "db.sequences"]) + result = self.build("--residue-codes", db=self.tmp / "codes" / "db", run_id="p56b") + self.assertEqual(result.exit_code, 0, result.output) + for suffix in _DB_SUFFIXES: + with self.subTest(suffix=suffix): + self.assertEqual(_sha256(self.tmp / "plain" / f"db{suffix}"), + _sha256(self.tmp / "codes" / f"db{suffix}")) + result = self.build("--no-residue-codes", db=self.tmp / "off" / "db", run_id="p56c") + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(sorted(p.name for p in (self.tmp / "off").iterdir()), + ["db.index", "db.metadata", "db.sequences"]) + + def test_min_res_n(self): + result = self.build("--residue-codes", "--min-res-n", "100") + self.assertEqual(result.exit_code, 0, result.output) + manifest = ResidueCodeStore(self.tmp, "db").check_compatible() + self.assertEqual((manifest["entries"], manifest["total_residues"], manifest["min_res_n"]), (1, 245, 100)) + + def test_the_build_block_describes_the_pass(self): + """The device type, its ESM3 precision and the ranks come from the pass (the shards' record): two ranks on + CUDA give accelerator 'cuda', devices 2 and bf16 autocast.""" + result = self.build("--residue-codes", ranks=2, device="cuda") + self.assertEqual(result.exit_code, 0, result.output) + build = ResidueCodeStore(self.tmp, "db").check_compatible()["build"] + self.assertEqual((build["accelerator"], build["devices"], build["esm_precision"], build["encoder_precision"]), + ("cuda", 2, "bf16-autocast", "fp32-no-tf32")) + self.assertEqual(sorted(p.name for p in self.shards.iterdir()), ["emb_p56-0.parquet", "emb_p56-1.parquet"]) + + def test_the_live_faiss_count_is_validated(self): + """The direct build gives validate() the live FAISS index's count: one too many fails check (e) and only + the code store is removed.""" + from foldmatch.search.embedding_database import EmbeddingDatabase + create_db = EmbeddingDatabase.create_db + + def one_more(self_, *args, **kwargs): + out = create_db(self_, *args, **kwargs) + self_.db.chain_ids.append("phantom") + return out + + with mock.patch.object(EmbeddingDatabase, "create_db", one_more): + result = self.build("--residue-codes") + message = self.refused(result, CodeStoreValidationError) + self.assertEqual(set(result.exception.__cause__.failures), {"e"}) + self.assertIn("the FAISS index has 3 entries, the code store 2", message) + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), ["db.index", "db.metadata", "db.sequences"]) + + def test_every_refusal_of_a_residue_codes_build_is_a_refusal_error(self): + """D20: a refused step of a --residue-codes build, in the pass or on rank 0 after it, exits 1 with a + RefusalError (the step's error is its cause); a plain build raises what it always raised.""" + def missing_rank(**kwargs): # rank 1's shard is lost + fake_chain_predict(ranks=2)(**kwargs) + next(Path(kwargs["out_folder"]).glob("*-1.parquet")).unlink() + + cases = { + "the pass": (mock.Mock(side_effect=ValueError("the pass refused")), "the pass refused"), + "the shard count": (missing_rank, "a rank's shard is missing"), + } + for case, (predict, text) in cases.items(): + for extra in (("--residue-codes",), ()): + with self.subTest(case=case, extra=extra): + for path in [*self.shards.iterdir(), *self.tmp.glob("db*")]: + path.unlink() + with launch_env(FOLDMATCH_RUN_ID="d20"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", predict): + result = invoke(*_build_args(_FASTA, self.tmp / "db", self.shards, *extra)) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIn(text, str(result.exception)) + if extra: + self.refused(result, ValueError) + else: + self.assertIs(type(result.exception), ValueError) + + def test_cleanup_embeddings_comes_after_the_store(self): + result = self.build("--residue-codes", "--cleanup-embeddings") + self.assertEqual(result.exit_code, 0, result.output) + self.assertTrue(ResidueCodeStore(self.tmp, "db").exists()) + self.assertEqual(list(self.shards.iterdir()), []) + + def test_a_validation_failure_keeps_the_database_and_the_shards(self): + """An entry the pass could not encode (an empty payload, P5.1) fails check (c) after the pass: the code store + is removed, the database still serves BLOSUM62 and the shards stay, even with --cleanup-embeddings.""" + def payload(name, sequence): + return np.empty((0, 8), np.uint8) if name == "2uzi_A" else _fake_codes(name, sequence) + + result = self.build("--residue-codes", "--cleanup-embeddings", payload=payload) + message = self.refused(result, CodeStoreValidationError) + self.assertEqual(result.exception.__cause__.failures["c"], (1, ("2uzi_A",))) + self.assertIn("empty payload", message) + # The command resolves --tmp-embedding-folder: on macOS the default TMPDIR is under /var -> /private/var. + self.assertIn(f"kept in {self.shards.resolve()} (run id p56)", message) + # The pass wrote the empty payload, so the advice is a fresh build, not --from-shards. + self.assertIn("rebuilding from the same shards would fail the same way", message) + self.assertNotIn("--from-shards", message) + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), ["db.index", "db.metadata", "db.sequences"]) + self.assertEqual([p.name for p in self.shards.iterdir()], ["emb_p56-0.parquet"]) + + + +# ===================================================================================================================== +# P5.7: build residue-codes --from-shards (fake ESM3 pass, real bundle) +# ===================================================================================================================== +def _store_files(db_folder: Path, name: str = "db") -> dict: + return {p.name: _sha256(p) for p in sorted(db_folder.glob(f"{name}.rescodes.*"))} + + +class FromShardsTest(_WithBundle): + + def setUp(self): + super().setUp() + self.db = self.tmp / "db1" + result = self.build(self.db / "db", run_id="p57") + self.assertEqual(result.exit_code, 0, result.output) + self.manifest = ResidueCodeStore(self.db, "db").manifest() + + def build(self, db, *extra, run_id, fasta=_FASTA, shards=None, **fake): + with launch_env(FOLDMATCH_RUN_ID=run_id), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(**fake)): + return invoke(*_build_args(fasta, db, shards or self.shards, "--residue-codes", *extra)) + + def copy_database(self, name="db2") -> Path: + """The database of ``self.db`` without its code store, in a new folder.""" + folder = self.tmp / name + folder.mkdir() + for suffix in _DB_SUFFIXES: + shutil.copyfile(self.db / f"db{suffix}", folder / f"db{suffix}") + return folder + + def from_shards(self, folder, *extra, shards=None, name="db"): + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched")): + result = invoke("build", "residue-codes", folder, name, "--from-shards", shards or self.shards, *extra) + esm3.assert_not_called() + return result + + def test_reproduces_the_direct_build(self): + folder = self.copy_database() + result = self.from_shards(folder) + self.assertEqual(result.exit_code, 0, result.output) + store = ResidueCodeStore(folder, "db") + manifest = store.check_compatible() + for key in ("bin_sha256", "entries_digest", "entries", "total_residues", "system_digest", "encoding_digest", + "esm", "bundle", "nonstandard_entries"): + with self.subTest(key=key): + self.assertEqual(manifest[key], self.manifest[key]) + self.assertEqual(store.bin_path.read_bytes(), ResidueCodeStore(self.db, "db").bin_path.read_bytes()) + self.assertEqual(manifest["build"]["run_id"], "p57") + self.assertEqual((manifest["cross_checks"]["preflight_entries"], manifest["cross_checks"]["faiss_entries"]), + (None, 2)) + self.assertEqual([p.name for p in self.shards.iterdir()], ["emb_p57-0.parquet"]) # read, never changed + # The manifest describes the pass that computed the codes, as the direct build's does. + self.assertEqual(manifest["min_res_n"], self.manifest["min_res_n"]) + self.assertEqual((manifest["build"]["from_shards"], self.manifest["build"]["from_shards"]), (True, False)) + for key in set(manifest["build"]) - {"from_shards", "created_utc"}: + self.assertEqual(manifest["build"][key], self.manifest["build"][key], key) + self.assertEqual(manifest["build"]["accelerator"], "cpu") + + def test_the_pass_is_described_whatever_rebuilds_the_store(self): + """The manifest's min_res_n and build block come from the shards: a store rebuilt from the shards of a + two-rank CUDA build with --min-res-n 100 says so (there is no --min-res-n to get wrong), not what the + rebuilding process runs on.""" + shards = self.tmp / "cuda" + shards.mkdir() + db = self.tmp / "cuda_db" + result = self.build(db / "db", "--min-res-n", "100", run_id="gpu", shards=shards, ranks=2, device="cuda") + self.assertEqual(result.exit_code, 0, result.output) + direct = ResidueCodeStore(db, "db").manifest() + ResidueCodeStore(db, "db").remove() + with mock.patch("foldmatch.search.residue_shards._dist_version", return_value="9.9.9"), \ + mock.patch("foldmatch.search.residue_code_store._dist_version", return_value="9.9.9"), \ + mock.patch("platform.machine", return_value="other-machine"): + result = self.from_shards(db, shards=shards) + self.assertEqual(result.exit_code, 0, result.output) + rebuilt = ResidueCodeStore(db, "db").check_compatible() + self.assertEqual((rebuilt["min_res_n"], rebuilt["entries"]), (100, 1)) + self.assertEqual((rebuilt["build"]["accelerator"], rebuilt["build"]["devices"], + rebuilt["build"]["esm_precision"]), ("cuda", 2, "bf16-autocast")) + for key in set(rebuilt["build"]) - {"from_shards", "created_utc"}: + self.assertEqual(rebuilt["build"][key], direct["build"][key], key) + self.assertNotEqual(rebuilt["build"]["torch"], "9.9.9") + self.assertEqual((rebuilt["esm"], rebuilt["bin_sha256"]), (direct["esm"], direct["bin_sha256"])) + + def test_shards_that_do_not_record_their_pass(self): + """Shards without a pass record (an earlier development build of the writer) are refused, not given a + guessed --min-res-n.""" + import pyarrow.parquet as pq + from foldmatch.search.residue_shards import META_PASS + folder = self.copy_database() + old = self.tmp / "old" + old.mkdir() + table = pq.read_table(self.shards / "emb_p57-0.parquet") + meta = {k: v for k, v in table.schema.metadata.items() if k != META_PASS} + pq.write_table(table.replace_schema_metadata(meta), old / "emb_old-0.parquet") + message = self.refused(self.from_shards(folder, shards=old)) + self.assertIn("do not record the --min-res-n", message) + self.assertEqual(_store_files(folder), {}) + + def test_a_complete_store_is_refused_unless_force(self): + before = _store_files(self.db) + self.assertIn("--force", self.refused(self.from_shards(self.db))) + self.assertEqual(_store_files(self.db), before) + result = self.from_shards(self.db, "--force") + self.assertEqual(result.exit_code, 0, result.output) + manifest = ResidueCodeStore(self.db, "db").check_compatible() + self.assertEqual((manifest["bin_sha256"], manifest["entries_digest"]), + (self.manifest["bin_sha256"], self.manifest["entries_digest"])) + self.assertEqual(sorted(_store_files(self.db)), ["db.rescodes.bin", "db.rescodes.idx", "db.rescodes.json"]) + + def test_force_keeps_the_store_when_the_new_codes_fail_validation(self): + """--force with shards that pass the footer and count checks but are not this database's (other ids, a + payload that is not whole codes, an empty payload): refused, and the valid store is untouched (review + finding: it used to be deleted before validation).""" + def short(name, sequence): + return _fake_codes(name, sequence).tobytes()[:-1] if name == "2uzi_A" else _fake_codes(name, sequence) + + def empty(name, sequence): + return np.empty((0, 8), np.uint8) if name == "2uzi_A" else _fake_codes(name, sequence) + + foreign = _write(self.tmp / "foreign.fasta", ">x\nACDEFGHIK\n>y\nLMNPQRST\n") # 2 entries, other ids + cases = { + "other ids": ({"fasta": foreign}, "(a) 2 sequence-store entr(ies) have no residue codes"), + "a payload of 463 bytes": ({"payload": short}, "not a multiple of 8 bytes"), + "an empty payload": ({"payload": empty}, "(c) 1 entr(ies) have codes of another length"), + } + before = _store_files(self.db) + for k, (case, (fake, text)) in enumerate(cases.items()): + with self.subTest(case=case): + shards = self.tmp / f"bad{k}" + shards.mkdir() + self.build(self.tmp / f"other{k}" / "db", run_id=f"bad{k}", shards=shards, **fake) + message = self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", shards=shards), text) + self.assertEqual(_store_files(self.db), before) + if case != "a payload of 463 bytes": + self.assertIn("existing residue-code store", message) + + def test_the_faiss_count_is_checked_before_anything_is_written(self): + """A FAISS index with one entry less than the sequence store and the shards: refused by the count check + before create() runs, with --force too.""" + import pickle + from foldmatch.search.residue_code_store import ResidueCodeStore as Store + folder = self.tmp / "dbf" + shutil.copytree(self.db, folder) + with open(folder / "db.metadata", "rb") as f: + meta = pickle.load(f) + meta["embedding_ids"] = meta["embedding_ids"][:-1] + with open(folder / "db.metadata", "wb") as f: + pickle.dump(meta, f) + before = _store_files(folder) + with mock.patch.object(Store, "create", side_effect=AssertionError("create() ran")) as create: + message = self.refused(self.from_shards(folder, "--force")) + create.assert_not_called() + self.assertIn("1 in its FAISS index", message) + self.assertIn("Nothing was changed", message) + self.assertEqual(_store_files(folder), before) + + def test_unreadable_database_files_are_named(self): + folder = self.copy_database() + data = (folder / "db.metadata").read_bytes() + (folder / "db.metadata").write_bytes(data[: len(data) // 2]) + message = self.refused(self.from_shards(folder), ValueError) + self.assertIn(f"{folder / 'db.metadata'} is not readable FAISS database metadata", message) + folder = self.copy_database("db3") + (folder / "db.sequences").write_bytes(b"not a database" * 100) + message = self.refused(self.from_shards(folder)) + self.assertIn(f"{folder / 'db.sequences'} is not a readable sequence store", message) + self.assertEqual(_store_files(folder), {}) + + def test_an_io_error_is_a_refusal(self): + """A full disk while the new codes are written: one 'Error:' line (exit 1, D20) naming the cause, and the old + store untouched (--force).""" + import errno + full = OSError(errno.ENOSPC, "No space left on device", str(self.db / "db.rescodes.bin.tmp")) + with mock.patch("foldmatch.search.residue_code_store.ResidueCodeStore.create", side_effect=full): + message = self.assert_refused_and_unchanged(self.from_shards(self.db, "--force"), "No space left", + cause=OSError) + self.assertIn("db.rescodes.bin.tmp", message) + + def test_stray_files_in_the_shard_folder(self): + """A copy named emb_copy[1]-0.parquet is no shard of any run (its name is no run id) and is ignored; an + unreadable shard of another run is refused naming the file, and --run-id gets past it.""" + shutil.copyfile(self.shards / "emb_p57-0.parquet", self.shards / "emb_copy[1]-0.parquet") + folder = self.copy_database() + result = self.from_shards(folder) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(ResidueCodeStore(folder, "db").manifest()["bin_sha256"], self.manifest["bin_sha256"]) + (self.shards / "emb_zzz-0.parquet").write_bytes(b"") + folder = self.copy_database("db3") + message = self.refused(self.from_shards(folder)) + self.assertIn(f"{(self.shards / 'emb_zzz-0.parquet').resolve()}: not a readable Parquet embedding shard", + message) + self.assertIn("--run-id", message) + result = self.from_shards(folder, "--run-id", "p57") + self.assertEqual(result.exit_code, 0, result.output) + + def test_a_partial_store_is_replaced(self): + folder = self.copy_database() + (folder / "db.rescodes.bin").write_bytes(b"stale") + result = self.from_shards(folder) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(ResidueCodeStore(folder, "db").manifest()["bin_sha256"], self.manifest["bin_sha256"]) + + def assert_refused_and_unchanged(self, result, *texts, cause=None): + """Refused before anything was written: with --force too, the existing store is untouched.""" + message = self.refused(result, cause) + for text in texts: + self.assertIn(text, message) + self.assertEqual(ResidueCodeStore(self.db, "db").manifest(), self.manifest) + return message + + def test_shards_of_another_encoding(self): + other = self.tmp / "other" + other.mkdir() + self.assertEqual(self.build(self.tmp / "x" / "db", run_id="other", shards=other, + system=OTHER_ENCODING).exit_code, 1) # the build itself refuses them + from foldmatch.search.residue_shards import ResidueShardError + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", shards=other), "stale", + "encoding_digest", cause=ResidueShardError) + + def test_counts_must_agree(self): + other = self.tmp / "other" + other.mkdir() + self.assertEqual(self.build(self.tmp / "x" / "db", "--min-res-n", "100", run_id="short", + shards=other).exit_code, 0) + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", shards=other), + "hold 1 entries", "2 in its sequence store", "2 in its FAISS index", + "Nothing was changed") + + def test_a_missing_rank(self): + """A rank's shard is missing: the shards hold fewer entries than the database.""" + shards = self.tmp / "two" + shards.mkdir() + fasta = _write(self.tmp / "three.fasta", ">x\nACDEFGHIK\n>y\nLMNPQRST\n>z\nVWYACDEF\n") + result = self.build(self.tmp / "three" / "db", fasta=fasta, run_id="two", shards=shards, ranks=2) + self.assertEqual(result.exit_code, 0, result.output) + manifest = ResidueCodeStore(self.tmp / "three", "db").manifest() + self.assertEqual((manifest["entries"], manifest["build"]["devices"]), (3, 2)) + (shards / "emb_two-1.parquet").unlink() + message = self.refused(self.from_shards(self.tmp / "three", "--force", shards=shards)) + self.assertIn("1 shard(s) of run two", message) + self.assertIn("hold 2 entries", message) + self.assertIn("a rank's shard is missing", message) + self.assertEqual(ResidueCodeStore(self.tmp / "three", "db").manifest(), manifest) + + def test_an_extra_shard(self): + """A shard of another run with the same id mixed in: more entries than the database.""" + other = self.tmp / "other" + other.mkdir() + fasta = _write(self.tmp / "one.fasta", ">x\nACDEFGHIK\n") + self.assertEqual(self.build(self.tmp / "y" / "db", fasta=fasta, run_id="p57", shards=other).exit_code, 0) + shutil.copyfile(self.shards / "emb_p57-0.parquet", other / "emb_p57-1.parquet") + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", shards=other), + "2 shard(s) of run p57", "hold 3 entries") + + def test_shards_without_codes(self): + plain = self.tmp / "plain" + plain.mkdir() + with launch_env(FOLDMATCH_RUN_ID="plain"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + self.assertEqual(invoke(*_build_args(_FASTA, self.tmp / "p" / "db", plain)).exit_code, 0) + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", shards=plain), + "hold residue codes", "plain") + from foldmatch.search.residue_shards import ResidueShardError + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", "--run-id", "plain", shards=plain), + "without --residue-codes", cause=ResidueShardError) + + def test_no_shards(self): + empty = self.tmp / "empty" + empty.mkdir() + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", shards=empty), "No embedding shards", + "--cleanup-embeddings") + + def test_several_runs_need_a_run_id(self): + self.assertEqual(self.build(self.tmp / "z" / "db", run_id="p57-1").exit_code, 0) + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force"), "2 runs", "p57 (1 shard(s))", + "p57-1 (1 shard(s))", "--run-id") + for run_id in ("p57", "p57-1"): + with self.subTest(run_id=run_id): + folder = self.copy_database(f"db_{run_id}") + result = self.from_shards(folder, "--run-id", run_id) + self.assertEqual(result.exit_code, 0, result.output) + manifest = ResidueCodeStore(folder, "db").manifest() + self.assertEqual(manifest["build"]["run_id"], run_id) + self.assertEqual(manifest["bin_sha256"], self.manifest["bin_sha256"]) + self.assert_refused_and_unchanged(self.from_shards(self.db, "--force", "--run-id", "p5"), + "No embedding shards of run p5", "p57, p57-1") + + def test_the_database_must_be_a_sequence_database(self): + folder = self.copy_database() + (folder / "db.sequences").unlink() + self.assertIn("no sequence store", self.refused(self.from_shards(folder))) + folder = self.copy_database("db3") + (folder / "db.index").unlink() + self.assertIn("Database files not found", self.refused(self.from_shards(folder), ValueError)) + self.assertEqual(_store_files(folder), {}) + + def test_flag_refusals_exit_2(self): + self.usage_error(self.from_shards(self.db, name="a/db"), "not a database name") + self.usage_error(self.from_shards(self.db, "--run-id", "a b"), "is not a run id") + self.usage_error(self.from_shards(self.db, "--min-res-n", "0"), "No such option") # it comes from the shards + self.usage_error(invoke("build", "residue-codes", self.db, "db"), "--from-shards") + self.usage_error(invoke("build", "residue-codes", self.tmp / "nope", "db", "--from-shards", self.shards), + "does not exist") + self.assertEqual(ResidueCodeStore(self.db, "db").manifest(), self.manifest) + + def test_shard_order_is_the_builds(self): + """--from-shards reads a run's shards in the order the build does (sorted by path, so rank 10 before 2).""" + from foldmatch.search.embedding_computer import EmbeddingComputer, run_shard_files, shard_run_ids + folder = self.tmp / "many" + folder.mkdir() + for rank in range(12): + (folder / f"emb_r-1-{rank}.parquet").touch() + (folder / "emb_r-0.parquet").touch() + (folder / "emb_r-1-x.parquet").touch() + (folder / "emb_r[1]-0.parquet").touch() # not a run id: no run's shard + with launch_env(FOLDMATCH_RUN_ID="r-1"): + self.assertEqual(EmbeddingComputer(folder).shard_files(), run_shard_files(folder, "r-1")) + self.assertEqual([p.name for p in run_shard_files(folder, "r-1")][:4], + ["emb_r-1-0.parquet", "emb_r-1-1.parquet", "emb_r-1-10.parquet", "emb_r-1-11.parquet"]) + self.assertEqual(len(run_shard_files(folder, "r-1")), 12) + self.assertEqual([p.name for p in run_shard_files(folder, "r")], ["emb_r-0.parquet"]) + self.assertEqual(shard_run_ids(folder), ["r", "r-1"]) + self.assertEqual([p.name for p in run_shard_files(folder, "r[1]")], ["emb_r[1]-0.parquet"]) # escaped glob + + +# ===================================================================================================================== +# real ESM3 through the command line (CPU) +# ===================================================================================================================== +class RealEsm3CliTest(unittest.TestCase): + """``fm-search build sequences --residue-codes`` on ``test_sequences.fasta`` with the real CPU pass: the manifest, + the codes against F2c (A4-xcpu), the plain build's files, and --from-shards.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() + gc.collect() # earlier tests' ESM3 models, if only reference cycles still hold them + cls._tmp = tempfile.TemporaryDirectory() + cls.tmp = Path(cls._tmp.name) + cls.shards = cls.tmp / "shards" + cls.shards.mkdir() + cls.results = {} + for kind, extra in (("codes", ("--residue-codes",)), ("plain", ())): + with launch_env(FOLDMATCH_RUN_ID=f"real-{kind}"): + cls.results[kind] = invoke(*_build_args(_FASTA, cls.tmp / kind / "db", cls.shards, "--batch-size", + "2", *extra)) + gc.collect() # the pass's ESM3 (5.6 GB) is only reachable through reference cycles + + @classmethod + def tearDownClass(cls): + cls._tmp.cleanup() + super().tearDownClass() + + def setUp(self): + for kind, result in self.results.items(): + self.assertEqual(result.exit_code, 0, f"{kind}: {result.output[-4000:]} {result.exception!r}") + + def test_manifest_and_f2c_codes(self): + _, ids, sequences, codes_ref = _f2c() + store = ResidueCodeStore(self.tmp / "codes", "db") + manifest = store.check_compatible() + self.assertEqual((manifest["entries"], manifest["total_residues"]), (2, 303)) + self.assertEqual(store.bin_path.stat().st_size, 8 * 303) + entries = store.lookup(ids) + self.assertEqual([entries[i][:2] for i in ids], [(0, 245), (245, 58)]) + self.assertEqual([entries[i].seq_sha1 for i in ids], [_sha1(s) for s in sequences]) + codes = np.asarray(store.open_memmap()) + equal, n = _bits_equal(codes, codes_ref) + print(f"\nCLI build F2c CPU code bits: {equal}/{n} = {equal / n:.6f}", file=sys.stderr) + self.assertGreaterEqual(equal / n, A4_XCPU_BITS, f"F2c code bits {equal}/{n}") + + def test_a_build_without_codes_writes_the_same_database_files(self): + for suffix in _DB_SUFFIXES: + with self.subTest(suffix=suffix): + self.assertEqual(_sha256(self.tmp / "plain" / f"db{suffix}"), + _sha256(self.tmp / "codes" / f"db{suffix}")) + self.assertEqual(sorted(p.name for p in (self.tmp / "plain").iterdir()), + ["db.index", "db.metadata", "db.sequences"]) + + def test_from_shards_reproduces_the_direct_build(self): + folder = self.tmp / "rebuilt" + folder.mkdir() + for suffix in _DB_SUFFIXES: + shutil.copyfile(self.tmp / "codes" / f"db{suffix}", folder / f"db{suffix}") + with _esm3_spy() as esm3: + result = invoke("build", "residue-codes", folder, "db", "--from-shards", self.shards) + self.assertEqual(result.exit_code, 0, result.output) + esm3.assert_not_called() + direct = ResidueCodeStore(self.tmp / "codes", "db").manifest() + rebuilt = ResidueCodeStore(folder, "db").check_compatible() + self.assertEqual((rebuilt["bin_sha256"], rebuilt["entries_digest"]), + (direct["bin_sha256"], direct["entries_digest"])) + self.assertEqual(rebuilt["build"]["run_id"], "real-codes") + + +_TWO_RANK_CHILD_ENV = "FOLDMATCH_TEST_PREPARE_LOG" +# Rendezvous state this process may hold from an earlier in-process DDP pass (Lightning sets MASTER_ADDR/MASTER_PORT): +# a child that inherited MASTER_PORT would try to join that process group, or find its port taken. +_DDP_ENV_PREFIXES = ("MASTER_", "PL_", "TORCHELASTIC_") +_DDP_ENV = ("WORLD_SIZE", "LOCAL_WORLD_SIZE", "GROUP_RANK", "ROLE_RANK") +_SITECUSTOMIZE = r""" +import json, os +from foldmatch.search import scoring_system as ss +_prepare = ss.prepare +def _spy(system, **kwargs): + rank = max(int(os.environ.get(k) or 0) for k in ("RANK", "LOCAL_RANK", "NODE_RANK")) + with open(os.path.join(os.environ["FOLDMATCH_TEST_PREPARE_LOG"], f"prepare-{rank}.json"), "w") as f: + json.dump(kwargs, f) + return _prepare(system, **kwargs) +ss.prepare = _spy +""" + + +@unittest.skipUnless(os.environ.get("FOLDMATCH_SLOW_TESTS") == "1", + "slow: two ESM3 ranks on CPU; set FOLDMATCH_SLOW_TESTS=1 to run") +class TwoRankCliSmoke(unittest.TestCase): + """``fm-search build sequences --residue-codes --devices 2`` as a command (CPU, gloo): Lightning re-runs the + command for rank 1, both ranks pass the pre-flight (rank 1 checks the bundle cached-only), and rank 0 builds a + store whose codes are F2c's.""" + + def test_two_rank_build(self): + _, ids, _, codes_ref = _f2c() + esm_codes_bundle.registry_files() + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + (tmp / "shards").mkdir() + (tmp / "site").mkdir() + (tmp / "log").mkdir() + (tmp / "site" / "sitecustomize.py").write_text(_SITECUSTOMIZE) + env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env["PYTHONPATH"] = os.pathsep.join([str(tmp / "site"), _SRC] + + [p for p in [os.environ.get("PYTHONPATH")] if p]) + env[_TWO_RANK_CHILD_ENV] = str(tmp / "log") + command = [sys.executable, "-m", "foldmatch.cli.search", + *map(str, _build_args(_FASTA, tmp / "db" / "idx", tmp / "shards", "--residue-codes"))] + command[command.index("--devices") + 1] = "2" + run = subprocess.run(command, env=env, capture_output=True, text=True, timeout=1800) + self.assertEqual(run.returncode, 0, run.stderr[-4000:]) + calls = {p.name: json.loads(p.read_text()) for p in (tmp / "log").iterdir()} + store = ResidueCodeStore(tmp / "db", "idx") + manifest = store.check_compatible() + codes = np.asarray(store.open_memmap()).copy() + entries = store.lookup(ids) + shards = sorted(p.name for p in (tmp / "shards").iterdir()) + self.assertEqual(calls, {"prepare-0.json": {"local_files_only": False}, + "prepare-1.json": {"local_files_only": True}}) + self.assertEqual((manifest["entries"], manifest["total_residues"], manifest["build"]["devices"]), (2, 303, 2)) + self.assertEqual(len(shards), 2) + self.assertEqual(manifest["build"]["run_id"], shards[0][len("emb_"):-len("-0.parquet")]) + z = np.load(_F2C, allow_pickle=False) + for k, name in enumerate(ids): + offset, length, _ = entries[name] + got = codes[offset:offset + length] + equal, n = _bits_equal(got, codes_ref[z["offsets"][k]:z["offsets"][k + 1]]) + self.assertGreaterEqual(equal / n, A4_XCPU_BITS, f"{name} code bits {equal}/{n}") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli_search.py b/tests/test_cli_search.py index afbf293..96f5699 100644 --- a/tests/test_cli_search.py +++ b/tests/test_cli_search.py @@ -5,6 +5,8 @@ import tempfile from pathlib import Path +import pytest + from foldmatch.search.output import CLUSTER_OUTPUT_FIELDS, DEFAULT_OUTPUT_FIELDS from foldmatch.types.api_types import StructureFormat, Accelerator, Granularity, FormatMode @@ -342,10 +344,10 @@ def test_14_query_database_assembly(self): def test_17_build_database_from_embeddings(self): - """Test building a database from pre-computed embedding .pt files.""" + """Test building a database from pre-computed chain embedding .pt files.""" from foldmatch.cli.search import build_database_from_embeddings - embedding_dir = Path(f"{self.__test_path}/resources/embeddings") + embedding_dir = Path(f"{self.__test_path}/resources/chain_embeddings") output_db = os.path.join(self.__temp_dir, "test_from_embeddings") build_database_from_embeddings( @@ -391,10 +393,10 @@ def test_18_build_database_from_fasta(self): self.assertEqual(len(db.chain_ids), 2) def test_21_query_database_from_embedding(self): - """Test querying the database with a pre-computed embedding file.""" + """Test querying the database with pre-computed chain embedding files: each query finds itself first.""" from foldmatch.cli.search import build_database_from_embeddings, query_database_from_embedding - embedding_dir = Path(f"{self.__test_path}/resources/embeddings") + embedding_dir = Path(f"{self.__test_path}/resources/chain_embeddings") subject_db = os.path.join(self.__temp_dir, "test_query_from_emb_db") build_database_from_embeddings( embedding_folder=embedding_dir, @@ -403,8 +405,8 @@ def test_21_query_database_from_embedding(self): use_gpu_index=False ) - # Use one of the pre-computed .pt chain embedding files as the query - embedding_folder = Path(f"{self.__test_path}/resources/embeddings") + # The same pre-computed .pt chain embedding files as the queries + embedding_folder = Path(f"{self.__test_path}/resources/chain_embeddings") output_file = os.path.join(self.__temp_dir, "query_from_embedding_results.csv") query_database_from_embedding( @@ -420,16 +422,22 @@ def test_21_query_database_from_embedding(self): self.assertTrue(os.path.exists(output_file)) with open(output_file, 'r') as f: lines = [ln.rstrip("\n") for ln in f if ln.strip()] - # No header: every row is a result, query id in the first column. - self.assertGreater(len(lines), 0) - query_ids = {ln.split("\t")[0] for ln in lines} - self.assertIn("1acb.A", query_ids) + # No header: every row is a result (query, hit, score), each query's best hit first. + top_hits = {} + for ln in lines: + query_id, hit_id, score = ln.split("\t") + top_hits.setdefault(query_id, (hit_id, float(score))) + self.assertEqual(sorted(top_hits), ["1acb.A", "1acb.B", "2uzi.A", "2uzi.B", "2uzi.C"]) + for query_id, (hit_id, score) in top_hits.items(): + with self.subTest(query=query_id): + self.assertEqual(hit_id, query_id) + self.assertGreater(score, 0.9999) def test_22_query_database_from_fasta(self): """Test querying the database with sequences from a FASTA file.""" from foldmatch.cli.search import build_database_from_embeddings, query_database_from_fasta - embedding_dir = Path(f"{self.__test_path}/resources/embeddings") + embedding_dir = Path(f"{self.__test_path}/resources/chain_embeddings") subject_db = os.path.join(self.__temp_dir, "test_query_from_fasta_db") build_database_from_embeddings( embedding_folder=embedding_dir, @@ -547,6 +555,7 @@ def test_24_similarity_graph(self): nodes = root.findall('.//g:node', ns) self.assertGreater(len(nodes), 0) + @pytest.mark.usefixtures("fmsw_libs") # Stage 2 runs on the native kernel def test_25_stage2_format_output_tsv(self): """Stage-2 sequence-identity search emits mmseqs-style TSV columns. @@ -629,7 +638,10 @@ def test_25_stage2_format_output_tsv(self): self.assertEqual(cigar_len, int(alnlen)) def test_26_stage2_rejects_unknown_format_field(self): - """An unknown --format-output field fails with a clear error.""" + """An unknown --format-output field fails with a clear error (a usage error, exit status 2), before the + query's ESM3 pass (P7.0c).""" + from unittest import mock + import typer from foldmatch.cli.search import ( build_database_from_fasta, query_database_from_fasta, @@ -644,7 +656,9 @@ def test_26_stage2_rejects_unknown_format_field(self): accelerator=Accelerator.cpu, use_gpu_index=False, ) - with self.assertRaises(ValueError) as ctx: + with self.assertRaises(typer.BadParameter) as ctx, \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", + side_effect=AssertionError("the query's ESM3 pass was reached")): query_database_from_fasta( db_path=subject_db, fasta_file=fasta_file, @@ -658,6 +672,7 @@ def test_26_stage2_rejects_unknown_format_field(self): ) self.assertIn("not_a_field", str(ctx.exception)) + @pytest.mark.usefixtures("fmsw_libs") # Stage 2 runs on the native kernel def test_27_fasta_min_res_n_filters_index_store_and_queries(self): """--min-res-n drops short sequences from the index, the sequence store and the embedded queries alike. diff --git a/tests/test_cli_split_build.py b/tests/test_cli_split_build.py new file mode 100644 index 0000000..b7bdd0f --- /dev/null +++ b/tests/test_cli_split_build.py @@ -0,0 +1,3774 @@ +"""P8b: the split sequence build (``build sequences-embed``, ``build sequences-index``, ``build sequences-seal``) and +the phases of ``build sequences`` it shares with the one-shot build. + +No ESM3 except in ``RealEsm3SplitTest`` and ``TwoRankSplitSmoke`` (so a run of this module loads ESM3 on CPU): elsewhere +the pass is ``fake_chain_predict`` and the spies are those of ``test_cli_residue_codes``. ``OneShotPhasesTest`` +(P8b.1) checks that ``build sequences`` runs its phases in the old order with the old arguments, the run binding and +the pre-flight of ``EmbeddingComputer``, the index-phase log lines and the CPU-affinity message. ``EmbedOptionTest``, +``EmbedStepTest`` and ``EmbedRefusalTest`` (P8b.3) check step 1, ``build sequences-embed``: its options, the shards and +the run record it leaves, and each of its refusals E1-E12 at its place. Two ranks are the fake pass's two shards plus a +patched ``_gather_rank_facts`` that returns two ranks' facts. ``IndexOptionTest``, ``IndexStepTest`` and +``IndexRefusalTest`` (P8b.4) check step 2, ``build sequences-index``: its options, that the two steps build the +one-shot's database (``assert_same_database``), its threads, and each of its refusals I1-I24 at its place; step 2 runs +under the ESM3 spy with ``scoring_system.prepare`` refusing. ``SealOptionTest``, ``SealTest`` and ``SealRefusalTest`` +(P8b.5) check the recovery, ``build sequences-seal``, under the same spies: its options, that the shards a failed +one-shot kept or a step 1 left without a record are sealed and then built into the one-shot's database, and each of its +refusals S1-S11, which write nothing. ``LaunchChecksBeforeTorchTest`` (P9.I1) runs step 2 and the seal as commands +with a hook that ends the process at any import of torch or Lightning: I2 and I3 refuse before it; ``CommandLineTest`` +(P9.I2) checks the command line their advice ends with. ``RealEsm3SplitTest`` runs both builds with the real CPU pass, +and ``TwoRankSplitSmoke`` (slow) both builds on two real ranks as commands. +""" +import dataclasses +import errno +import gc +import glob +import hashlib +import json +import logging +import os +import re +import shlex +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import time +import unittest +from contextlib import ExitStack, chdir, nullcontext +from datetime import datetime, timedelta, timezone +from pathlib import Path +from types import MappingProxyType +from unittest import mock + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +from typer.testing import CliRunner + +import esm_codes_bundle +import foldmatch +import foldmatch.search.code_alignment as code_alignment +from foldmatch.cli import search +from foldmatch.search import alignment, split_run +from foldmatch.search import scoring_system as ss +from foldmatch.search.embedding_computer import EmbeddingComputer +from foldmatch.search.embedding_database import EmbeddingDatabase +from foldmatch.search.faiss_database import FaissEmbeddingDatabase +from foldmatch.search.residue_code_store import CodeStoreValidationError, ResidueCodeStore +from foldmatch.search.residue_shards import META_PASS, pass_record, read_shard_info +from foldmatch.search.sequence_store import SequenceStore +from foldmatch.types.api_types import Accelerator, IndexConfig, IndexType, LogLevel, Strategy +from foldmatch.utils.esm.loaders import ESM_REVISION +from foldmatch.utils.sequence import preflight_fasta +from click_messages import no_such_option +from test_cli_residue_codes import A4_XCPU_BITS, OTHER_ENCODING, PLAN_HELP, _DDP_ENV, _DDP_ENV_PREFIXES, \ + _LAUNCH_ENV, _SITECUSTOMIZE, _SRC, _TWO_RANK_CHILD_ENV, _bits_equal, _esm3_spy, _f2c, _fake_codes, _Tmp, \ + _WithBundle, _build_args, _write, fake_chain_predict, invoke, launch_env + +_TEST_PATH = Path(__file__).parent +_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +SYSTEM = ss.get(ss.DEFAULT_ESM_CODES) +#: Non-default pass and index options, so that the spies see each one threaded through. +_OPTIONS = ("--min-res-n", "100", "--batch-size", "2", "--ivf-nlist", "256", "--load-batch-size", "1000", + "--cleanup-embeddings") + + +def _faiss_libs() -> str: + """The faiss-cpu wheel's library folder, beside the installed faiss package (the macOS wheels have none).""" + import faiss + return os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(faiss.__file__))), "faiss_cpu.libs") + + +# ===================================================================================================================== +# P8b.1: the phases of build sequences +# ===================================================================================================================== +class OneShotPhasesTest(_WithBundle): + + def build(self, *extra, run_id="p8b1", db="db"): + with launch_env(FOLDMATCH_RUN_ID=run_id), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + return invoke(*_build_args(_FASTA, self.tmp / db, self.shards, *extra)) + + def test_the_phases_run_in_order_with_the_old_arguments(self): + """Pre-flight, pass, index phase, cleanup: each once, in this order, with the command's arguments; the pass, + create_db and the sequence store get exactly the arguments they got before the phases were functions, and + the index phase gets no run record and no hook.""" + for codes in (False, True): + with self.subTest(codes=codes): + for path in [*self.shards.iterdir(), *self.tmp.glob("db*")]: + path.unlink() + extra = _OPTIONS + (("--residue-codes",) if codes else ()) + manager = mock.Mock() + spies = { + "preflight": mock.patch.object(search, "_sequence_build_preflight", + wraps=search._sequence_build_preflight), + "run_pass": mock.patch.object(search, "_run_sequence_pass", wraps=search._run_sequence_pass), + "index": mock.patch.object(search, "_build_sequence_database", + wraps=search._build_sequence_database), + "cleanup": mock.patch.object(EmbeddingComputer, "remove_shards", autospec=True, + side_effect=EmbeddingComputer.remove_shards), + "compute": mock.patch.object(EmbeddingComputer, "compute_from_fasta", autospec=True, + side_effect=EmbeddingComputer.compute_from_fasta), + "create_db": mock.patch.object(EmbeddingDatabase, "create_db", autospec=True, + side_effect=EmbeddingDatabase.create_db), + "store": mock.patch.object(SequenceStore, "create", autospec=True, + side_effect=SequenceStore.create), + } + prepare = (mock.patch.object(ss, "prepare", wraps=ss.prepare) if codes else + mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched"))) + with ExitStack() as stack, prepare: + for name, patcher in spies.items(): + manager.attach_mock(stack.enter_context(patcher), name) + result = self.build(*extra) + self.assertEqual(result.exit_code, 0, result.output) + + self.assertEqual([c[0] for c in manager.mock_calls], + ["preflight", "run_pass", "compute", "index", "create_db", "store", "cleanup"]) + calls = {c[0]: c for c in manager.mock_calls} + self.assertEqual(calls["preflight"], mock.call.preflight( + _FASTA.resolve(), 100, str(self.tmp / "db"), self.shards.resolve(), 1, codes)) + run = calls["run_pass"].args[0] + self.assertIsInstance(run, search._SequencePass) + self.assertEqual(run.residue_codes, codes) + self.assertEqual(calls["run_pass"], mock.call.run_pass( + run, fasta_file=_FASTA.resolve(), min_res_n=100, batch_size=2, num_workers=0, num_nodes=1, + accelerator=Accelerator.cpu, devices=["1"], strategy=Strategy.auto)) + residue = {"residue_output": "residue_codes", "scoring_system": run.prepared} if codes else {} + self.assertEqual(calls["compute"], mock.call.compute( + run.embedding_computer, fasta_file=_FASTA.resolve(), min_res_n=100, batch_size=2, + num_workers=0, num_nodes=1, accelerator=Accelerator.cpu, devices=1, strategy=Strategy.auto, + preflight=run.preflight, **residue)) + self.assertEqual(calls["index"], mock.call.index( + run.embedding_computer, run.preflight, fasta_file=_FASTA.resolve(), output_db=str(self.tmp / "db"), + min_res_n=100, system=SYSTEM if codes else None, + index=search._IndexOptions(False, IndexType.auto, 256, None, 1000), refusals=run.refusals, + omp_env_at_start=os.environ.get("OMP_NUM_THREADS"))) + self.assertNotIn("split", calls["index"].kwargs) + self.assertNotIn("after_sequence_store", calls["index"].kwargs) + create_db = calls["create_db"].kwargs + self.assertEqual((create_db["use_gpu_index"], create_db["index_type"], create_db["index_config"]), + (False, IndexType.auto, IndexConfig(nlist=256, nprobe=None))) + self.assertEqual(calls["store"].kwargs, {"fasta_file": _FASTA.resolve(), "min_res_n": 100}) + self.assertEqual(calls["cleanup"], mock.call.cleanup(run.embedding_computer)) + self.assertEqual(run.preflight.entries_kept, 1) + self.assertEqual(ResidueCodeStore(self.tmp, "db").exists(), codes) + self.assertEqual(list(self.shards.iterdir()), []) + + def test_the_index_phase_hook(self): + """``after_sequence_store`` runs once the sequence store has passed its check and before the code store, inside + the refusal wrapper it is given; a refused store check never reaches it. Without ``system`` the phase builds + no code store and returns None.""" + prepared = ss.prepare(SYSTEM, local_files_only=True) + with launch_env(FOLDMATCH_RUN_ID="hook"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + computer = EmbeddingComputer(self.shards) + computer.compute_from_fasta(_FASTA, accelerator=Accelerator.cpu, residue_output="residue_codes", + scoring_system=prepared) + seen = [] + + def build(db, hook, refusals=search._refusals, system=SYSTEM): + return search._build_sequence_database( + computer, computer.preflight, fasta_file=_FASTA, output_db=str(self.tmp / db), min_res_n=0, + system=system, index=search._IndexOptions(False, IndexType.flat, 256, None, 1000), refusals=refusals, + after_sequence_store=hook) + + def files(db): + return sorted(p.name for p in self.tmp.glob(f"{db}.*")) + + manifest = build("db", lambda: seen.append(files("db"))) + self.assertEqual(seen, [["db.index", "db.metadata", "db.sequences"]]) + self.assertEqual(manifest["entries"], 2) + self.assertTrue(ResidueCodeStore(self.tmp, "db").exists()) + self.assertIsNone(build("plain", lambda: seen.append(files("plain")), system=None)) + self.assertEqual(seen[-1], files("plain")) + self.assertEqual(files("plain"), ["plain.index", "plain.metadata", "plain.sequences"]) + with mock.patch.object(search, "_check_store_matches_preflight", side_effect=ValueError("store mismatch")), \ + self.assertRaises(search.RefusalError): + build("db2", lambda: seen.append("reached")) + self.assertNotIn("reached", seen) + + def stat_fails(): + raise FileNotFoundError("the FASTA is gone") + + for refusals, raised in ((search._refusals, search.RefusalError), (search.nullcontext, FileNotFoundError)): + with self.subTest(refusals=refusals.__name__): + for path in self.tmp.glob("db3.*"): + path.unlink() + with self.assertRaises(raised) as ctx: + build("db3", stat_fails, refusals=refusals) + self.assertIn("the FASTA is gone", str(ctx.exception)) + self.assertFalse(ResidueCodeStore(self.tmp, "db3").exists()) + + def test_the_mode_of_a_pass(self): + """``_SequencePass``: the residue keywords and the refusal wrapper of each mode (D20).""" + plain = search._SequencePass(None, None, None, False) + self.assertEqual(plain.residue, {}) + with self.assertRaises(ValueError) as ctx, plain.refusals(): + raise ValueError("plain") + self.assertIs(type(ctx.exception), ValueError) + prepared = object() + codes = search._SequencePass(None, None, prepared, True) + self.assertEqual(codes.residue, {"residue_output": "residue_codes", "scoring_system": prepared}) + with self.assertRaises(search.RefusalError) as ctx, codes.refusals(): + raise FileExistsError("codes") + self.assertIsInstance(ctx.exception.__cause__, FileExistsError) + + def test_a_fresh_index_config_per_build(self): + """An ivf_pq build sets its config's nprobe: each build gets a new one.""" + options = search._IndexOptions(False, IndexType.ivf_pq, 256, None, 1000) + first = options.index_config() + first.nprobe = 4 + self.assertIsNot(options.index_config(), first) + self.assertEqual(options.index_config(), IndexConfig(nlist=256, nprobe=None)) + + def test_a_computer_bound_to_a_run(self): + """``run`` binds the computer to that run's shards without drawing an id; a malformed id is refused.""" + (self.shards / "emb_x-0.parquet").touch() + (self.shards / "emb_x-1-0.parquet").touch() + with mock.patch("foldmatch.search.embedding_computer.run_id", + side_effect=AssertionError("run_id() called")), \ + launch_env(LOCAL_RANK=1): # a rank > 0 without an id: run_id() would refuse + computer = EmbeddingComputer(self.shards, run="x") + self.assertEqual((computer.run_id, computer.out_name), ("x", "emb_x")) + self.assertEqual(computer.shard_files(), [self.shards / "emb_x-0.parquet"]) + for bad in ("a b", "", "-x", "a/b", "a*"): + with self.subTest(run=bad), self.assertRaises(ValueError) as ctx: + EmbeddingComputer(self.shards, run=bad) + self.assertIn("is not a run id", str(ctx.exception)) + with launch_env(FOLDMATCH_RUN_ID="drawn"): + self.assertEqual(EmbeddingComputer(self.shards).run_id, "drawn") + + def test_attach_preflight_sets_what_the_pass_set(self): + with launch_env(FOLDMATCH_RUN_ID="a"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + passed = EmbeddingComputer(self.shards) + passed.compute_from_fasta(_FASTA, min_res_n=100, accelerator=Accelerator.cpu) + bound = EmbeddingComputer(self.shards, run="a") + preflight = bound.attach_preflight(_FASTA, 100) + self.assertIs(bound.preflight, preflight) + self.assertEqual((bound.expected_count, bound.preflight), (passed.expected_count, passed.preflight)) + self.assertEqual(bound.preflight.offsets.tolist(), passed.preflight.offsets.tolist()) + self.assertEqual(bound.expected_count, 1) + bound.check_embedding_count() # the pass's shard, as rank 0 checks it + self.assertEqual(bound.input_ids(), ["1acb_E"]) + # A given pre-flight is taken as it is, after its path and min_res_n are matched. + other = EmbeddingComputer(self.shards, run="a") + self.assertIs(other.attach_preflight(str(_FASTA), 100, passed.preflight), passed.preflight) + copy = Path(shutil.copyfile(_FASTA, self.tmp / "copy.fasta")) + for fasta, min_res_n in ((_FASTA, 0), (copy, 100)): + with self.subTest(fasta=fasta, min_res_n=min_res_n): + fresh = EmbeddingComputer(self.shards, run="a") + with self.assertRaises(ValueError) as ctx: + fresh.attach_preflight(fasta, min_res_n, passed.preflight) + self.assertIn("does not match", str(ctx.exception)) + self.assertEqual((fresh.expected_count, fresh.preflight), (None, None)) + # strict refuses what a plain pre-flight only logs; a residue pass is strict. + dup = _write(self.tmp / "dup.fasta", ">a\nACDE\n>a\nACDE\n") + self.assertEqual(EmbeddingComputer(self.shards, run="b").attach_preflight(dup, 0).duplicates, 1) + with self.assertRaises(ValueError) as ctx: + EmbeddingComputer(self.shards, run="b").attach_preflight(dup, 0, strict=True) + self.assertIn("repeat an earlier id", str(ctx.exception)) + with launch_env(FOLDMATCH_RUN_ID="c"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", + side_effect=AssertionError("the pass ran")), self.assertRaises(ValueError) as ctx: + EmbeddingComputer(self.shards).compute_from_fasta(dup, residue_output="residue_codes") + self.assertIn("repeat an earlier id", str(ctx.exception)) + + def logged_build(self, *extra, **kwargs): + """The build's root-logger records (``set_log_level`` would replace assertLogs' handler).""" + with mock.patch.object(search, "set_log_level"), self.assertLogs(level="INFO") as logs: + result = self.build(*extra, **kwargs) + self.assertEqual(result.exit_code, 0, result.output) + return logs + + def test_index_phase_log_lines(self): + logs = self.logged_build("--residue-codes") + messages = [record.getMessage() for record in logs.records] + phase = [m for m in messages if m.startswith("Index phase: ")] + self.assertEqual(len(phase), 1, messages) + import faiss + omp = os.environ.get("OMP_NUM_THREADS", "unset") + self.assertRegex(phase[0], rf"^Index phase: FAISS {re.escape(faiss.__version__)}, " + rf"{faiss.omp_get_max_threads()} OpenMP thread\(s\); \d+ of {os.cpu_count()} CPUs " + rf"allowed \({code_alignment._physical_cpus()} physical cores\); " + rf"OMP_NUM_THREADS={omp}$") + for what in ("FAISS index", "Sequence store", "Residue-code store"): + with self.subTest(what=what): + self.assertEqual(len([m for m in messages if re.fullmatch(rf"{what} built in \d+\.\d s", m)]), 1, + messages) + order = [next(k for k, m in enumerate(messages) if m.startswith(prefix)) + for prefix in ("Index phase: ", "FAISS index built", "Sequence store built", "Residue-code store " + "written", "Residue-code store built")] + self.assertEqual(order, sorted(order)) + self.assertFalse([m for m in messages if "stay allocated" in m]) + plain = [record.getMessage() for record in self.logged_build(run_id="plain", db="plain").records] + self.assertFalse([m for m in plain if m.startswith("Residue-code store built")]) + + def test_the_multi_node_hint(self): + messages = [record.getMessage() for record in self.logged_build("--num-nodes", "2").records] + hint = [m for m in messages if "stay allocated" in m] + self.assertEqual(len(hint), 1, messages) + self.assertIn("the other 1 node(s) stay allocated", hint[0]) + self.assertIn("'build sequences-embed' and then 'build sequences-index'", hint[0]) + self.assertLess(messages.index(hint[0]), + next(k for k, m in enumerate(messages) if m.startswith("Index phase"))) + # Ranks > 0 return after the pass: no index phase, no hint. + with mock.patch.object(search, "_is_rank_zero", return_value=False), \ + mock.patch.object(search, "set_log_level"), self.assertLogs(level="INFO") as logs: + result = self.build("--num-nodes", "2", run_id="rank1", db="rank1") + self.assertEqual(result.exit_code, 0, result.output) + messages = [record.getMessage() for record in logs.records] + self.assertFalse([m for m in messages if "stay allocated" in m or m.startswith("Index phase")], messages) + self.assertEqual(sorted(p.name for p in self.tmp.glob("rank1*")), []) + + def test_the_thread_warning(self): + """A WARNING when FAISS has fewer OpenMP threads than the usable physical cores, naming the cause; never on + macOS, where foldmatch pins OMP_NUM_THREADS=1 against the second libomp. OMP_NUM_THREADS unset when the + command started (``omp_env_at_start=None``) and set at the index phase was set by the pass: Lightning's DDP + launcher, which the INFO line and the WARNING name, with the remedy. With the variable unset, the cause is a + limit set inside the process (OpenMP's default, one thread per allowed CPU, is never below the physical + cores).""" + pools = [{"user_api": "blas", "internal_api": "openblas", "num_threads": 4, + "filepath": "/opt/lib/libscipy_openblas64_-ff651d7f.so"}] + not_given = search._OMP_ENV_NOT_RECORDED + lightning = ("OMP_NUM_THREADS=2, which Lightning's DDP launcher set during the ESM3 pass (it was unset at the " + "start; Lightning sets it to the allowed CPUs // devices per node): the index phase may be slower " + "than this node allows. To choose FAISS's threads, export OMP_NUM_THREADS before the run (it also " + "sets each rank's torch threads in the pass), or build in two steps ('build sequences-embed', " + "then 'build sequences-index' as its own job).") + inside = ("an OpenMP thread limit set inside this process or by another OpenMP variable (OMP_NUM_THREADS is " + "unset)") + cases = (("linux", "2", not_given, "OMP_NUM_THREADS=2: the index phase may be slower than this node allows."), + ("linux", "2", "2", "OMP_NUM_THREADS=2: the index phase may be slower than this node allows."), + ("linux", "2", None, lightning), + ("linux", None, not_given, inside), ("linux", None, None, inside), + ("darwin", "2", None, None), ("darwin", "2", not_given, None), ("darwin", None, not_given, None)) + for platform, omp_env, at_start, cause in cases: + env = {k: v for k, v in os.environ.items() if k != "OMP_NUM_THREADS"} + if omp_env is not None: + env["OMP_NUM_THREADS"] = omp_env + args = () if at_start is not_given else (at_start,) + with self.subTest(platform=platform, omp_env=omp_env, at_start=args), \ + mock.patch.dict(os.environ, env, clear=True), mock.patch.object(sys, "platform", platform), \ + mock.patch("faiss.omp_get_max_threads", return_value=2), \ + mock.patch.object(code_alignment, "_physical_cpus", return_value=8), \ + mock.patch("threadpoolctl.threadpool_info", return_value=pools), \ + self.assertLogs(level="INFO") as logs: + search._log_index_phase(*args) + messages = [(record.levelname, record.getMessage()) for record in logs.records] + self.assertIn(("INFO", "Thread pool: openblas, 4 thread(s), libscipy_openblas64_-ff651d7f.so"), messages) + self.assertIn("2 OpenMP thread(s)", messages[0][1]) + self.assertIn("(8 physical cores)", messages[0][1]) + set_by_pass = at_start is None and omp_env is not None + suffix = (" (set during the ESM3 pass by Lightning: the allowed CPUs // devices per node; unset at the " + "start)") + self.assertTrue(messages[0][1].endswith(f"OMP_NUM_THREADS={omp_env or 'unset'}" + + (suffix if set_by_pass else "")), messages[0][1]) + warnings = [m for level, m in messages if level == "WARNING"] + if cause is not None: + self.assertEqual(len(warnings), 1, messages) + self.assertEqual(warnings[0], f"FAISS has 2 OpenMP thread(s) for 8 usable physical cores, because of " + f"{cause}" + ("" if cause.endswith(".") else + ": the index phase may be slower than this node allows.")) + self.assertEqual("Lightning" in warnings[0], set_by_pass) + else: + self.assertEqual(warnings, []) + with mock.patch.object(sys, "platform", "linux"), mock.patch("faiss.omp_get_max_threads", return_value=8), \ + mock.patch.object(code_alignment, "_physical_cpus", return_value=8), \ + mock.patch("threadpoolctl.threadpool_info", return_value=pools), \ + self.assertLogs(level="INFO") as logs: + search._log_index_phase(None) + self.assertEqual([r for r in logs.records if r.levelno >= logging.WARNING], []) + + def test_omp_num_threads_as_the_command_found_it(self): + """The one-shot reads OMP_NUM_THREADS before its pass and gives that value to the index phase: a value the pass + set (as Lightning's DDP launcher does when the variable is unset) is logged as set during the pass; one set + before the command is not.""" + real = search._run_sequence_pass + + def setting(*args, **kwargs): # what Lightning's launcher does during a multi-device pass + real(*args, **kwargs) + os.environ.setdefault("OMP_NUM_THREADS", "3") + + for at_start in (None, "3"): + with self.subTest(at_start=at_start), mock.patch.dict(os.environ), \ + mock.patch.object(search, "_run_sequence_pass", side_effect=setting), \ + mock.patch.object(search, "_log_index_phase", wraps=search._log_index_phase) as phase: + os.environ.pop("OMP_NUM_THREADS", None) + if at_start is not None: + os.environ["OMP_NUM_THREADS"] = at_start + messages = [record.getMessage() for record in self.logged_build( + run_id=f"omp{at_start}", db=f"omp{at_start}").records] + self.assertEqual(phase.call_args, mock.call(at_start)) + line = next(m for m in messages if m.startswith("Index phase: ")) + self.assertEqual(line.endswith("OMP_NUM_THREADS=3 (set during the ESM3 pass by Lightning: the allowed CPUs " + "// devices per node; unset at the start)"), at_start is None, line) + self.assertTrue(line.endswith("OMP_NUM_THREADS=3") or at_start is None, line) + + def test_threadpoolctl_failures_do_not_fail_a_build(self): + with mock.patch("threadpoolctl.threadpool_info", side_effect=RuntimeError("cannot inspect")), \ + self.assertLogs(level="INFO") as logs: + search._log_index_phase() + self.assertFalse([r for r in logs.records if r.getMessage().startswith("Thread pool: ")]) + + def test_a_single_threaded_faiss_blas(self): + """One line when FAISS's BLAS runs on one thread by build, a WARNING (INFO on macOS) naming the library and the + pinned faiss-cpu: the faiss-cpu 1.13.2 wheel's OpenBLAS in faiss_cpu.libs (threading layer disabled), FAISS's + own BLAS whose threading layer threadpoolctl does not report, any BLAS built without threads; the first one + only. None for a threaded BLAS (MKL at 128 threads, numpy's OpenBLAS limited to one thread, the faiss-cpu + 1.15.1 wheel's OpenMP OpenBLAS at one thread or more), and none when threadpoolctl finds no BLAS.""" + import faiss + package = os.path.dirname(os.path.realpath(faiss.__file__)) + libs = os.path.join(os.path.dirname(package), "faiss_cpu.libs") + + def blas(path, layer, threads, api="openblas"): + runtime = {"user_api": "blas", "internal_api": api, "num_threads": threads, "filepath": path, + "version": "0.3.15"} + if layer is not None: + runtime["threading_layer"] = layer + return runtime + + gomp = {"user_api": "openmp", "internal_api": "openmp", "num_threads": 8, "version": None, + "filepath": os.path.join(libs, "libgomp-e985bcbb.so.1.0.0")} + wheel = blas(os.path.join(libs, "libopenblas-r0-11edc3fa.3.15.so"), "disabled", 1) + serial = blas("/opt/conda/lib/libopenblasp-r0.3.21.so", "disabled", 1) + pinned = os.path.join(libs, "libopenblaso-r0-d77a1985.3.15.so") + cases = { + "the faiss-cpu 1.13.2 wheel": ([gomp, wheel], "libopenblas-r0-11edc3fa.3.15.so"), + "in the package, no threading layer": ( + [blas(os.path.join(package, ".dylibs", "libopenblas.0.dylib"), None, 1)], "libopenblas.0.dylib"), + "in faiss_cpu.libs, an unknown layer": ([blas(os.path.join(libs, "libopenblas.so"), "unknown", 1)], + "libopenblas.so"), + "in faiss_cpu.libs, no threading layer, 8 threads": ( + [blas(os.path.join(libs, "libopenblas.so"), None, 8)], None), + "built without threads elsewhere": ([serial], "libopenblasp-r0.3.21.so"), + "two of them": ([wheel, serial], "libopenblas-r0-11edc3fa.3.15.so"), + "MKL at 128 threads": ([gomp, blas("/envs/faisscf/lib/libmkl_rt.so.2", "intel", 128, api="mkl")], None), + "numpy's OpenBLAS limited to one thread": ( + [blas("/sp/numpy.libs/libscipy_openblas64_-ff651d7f.so", "pthreads", 1)], None), + "the faiss-cpu 1.15.1 wheel's OpenMP OpenBLAS at one thread": ([blas(pinned, "openmp", 1)], None), + "the faiss-cpu 1.15.1 wheel's OpenMP OpenBLAS": ([gomp, blas(pinned, "openmp", 64)], None), + "no BLAS": ([gomp], None), + "nothing found": ([], None), + } + text = ("FAISS's BLAS, {}, is single-threaded, so the ivf_pq training and adds run on one core whatever " + "FAISS's thread count: install the faiss-cpu 1.15.1 FoldMatch pins (pip install faiss-cpu==1.15.1), " + "whose wheel bundles an OpenMP OpenBLAS, or a FAISS built on MKL (README, 'FAISS threads').") + for case, (pools, name) in cases.items(): + for platform in ("linux", "darwin"): + with self.subTest(case=case, platform=platform), mock.patch.object(sys, "platform", platform), \ + mock.patch("faiss.omp_get_max_threads", return_value=8), \ + mock.patch.object(code_alignment, "_physical_cpus", return_value=8), \ + mock.patch("threadpoolctl.threadpool_info", return_value=pools), \ + self.assertLogs(level="INFO") as logs: + search._log_index_phase(None) + lines = [(record.levelname, record.getMessage()) for record in logs.records + if "single-threaded" in record.getMessage()] + self.assertEqual(lines, [] if name is None else + [("INFO" if platform == "darwin" else "WARNING", text.format(name))]) + self.assertEqual(len([record for record in logs.records if record.getMessage().startswith( + "Thread pool: ")]), len(pools)) + self.assertEqual([record.getMessage() for record in logs.records if record.levelno >= logging.WARNING + and "single-threaded" not in record.getMessage()], []) # FAISS has its 8 threads + + def widen(self, *args): + """``_expand_cpu_affinity(*args)`` widening 1 CPU to 4 (affinity calls mocked); its log lines.""" + with mock.patch.object(os, "cpu_count", return_value=4), \ + mock.patch.object(os, "sched_setaffinity", create=True) as setaffinity, \ + mock.patch.object(os, "sched_getaffinity", create=True, side_effect=[{0}, {0, 1, 2, 3}]), \ + self.assertLogs("foldmatch.search.alignment", level="INFO") as logs: + self.assertEqual(alignment._expand_cpu_affinity(*args), 4) + setaffinity.assert_called_once_with(0, range(4)) + return logs.output + + def test_the_default_affinity_message_is_unchanged(self): + self.assertEqual(self.widen(), + ["INFO:foldmatch.search.alignment:Widened CPU affinity for alignment: 1 -> 4 CPUs"]) + + def test_the_affinity_message_names_its_purpose(self): + self.assertEqual(self.widen("the index phase"), + ["INFO:foldmatch.search.alignment:Widened CPU affinity for the index phase: 1 -> 4 CPUs"]) + + +# ===================================================================================================================== +# P8b.3: build sequences-embed +# ===================================================================================================================== +#: Plan P8b (CLI), verbatim: the command help, and the help texts step 1 does not take from 'build sequences'. The +#: command help's sbatch example also gives --kill-on-invalid-dep=yes (work/p8b/deviations.md, P8b.3). +EMBED_HELP = ( + "Step 1 of a split 'build sequences' for multi-node GPU runs: the pre-flight and the ESM3 pass (chain embeddings, " + "and residue codes with --residue-codes) on every rank, writing the embedding shards and, last, a run record into " + "--shards-folder. No database file is written. Then build the database with 'build sequences-index' as its own " + "one-process job on a CPU node (e.g. sbatch --dependency=afterok: --kill-on-invalid-dep=yes), so that " + "these nodes are released." +) +EMBED_OUTPUT_DB_HELP = ( + "Path of the database 'build sequences-index' will build (as --output-db of 'build sequences'). Refused if any of " + "its files exist; recorded in the run record." +) +SHARDS_FOLDER_HELP = ( + "Folder for this step's output: the embedding shards (emb_-.parquet) and, written last, the run " + "record (emb_.run.json) that 'build sequences-index --from-shards' reads. It must be on storage every node " + "can write and that outlives the job (e.g. $SCRATCH), with room for about 6 KiB per entry plus 8 B per residue " + "with --residue-codes. Use one folder per build." +) +#: Five entries: two and three ranks get different deals. +_FIVE = ">e0\nACDEFGHIKL\n>e1 second\nKLMNPQRSTV\n>e2\nVWYACDEFGH\n>e3\nIKLMNPQRST\n>e4\nACDEFGHIKLMN\n" +#: A rank that starts the run, and a rank Lightning or srun starts (it shares the run id it is given). +_RANK_ENVS = ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}) + + +def _embed_args(fasta, db, shards, *extra): + return ("build", "sequences-embed", "--fasta-file", fasta, "--output-db", db, "--shards-folder", shards, + "--accelerator", "cpu", "--devices", "1", *extra) + + +def _ranks(n, change=None): + """A ``_gather_rank_facts`` of an ``n``-rank world: rank 0's facts, and a copy for every other rank, rank 1's + updated with ``change(facts)``.""" + def gather(facts): + others = [dict(facts) for _ in range(n - 1)] + if change is not None: + others[0].update(change(facts)) + return [facts, *others] + return gather + + +def _flat(output: str) -> str: + """Typer's help output with the panel borders dropped and the whitespace collapsed.""" + return " ".join(output.translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +def _command(*path): + import typer + command = typer.main.get_command(search.app) + for name in path: + command = command.commands[name] + return command + + +def _rewrite(path, change): + """Rewrite the shard at ``path`` as ``change(table)``.""" + pq.write_table(change(pq.read_table(path)), path) + + +def _with_pass(**changes): + """A table change: the shard's pass record (``foldmatch.pass``) with ``changes``.""" + def change(table): + meta = dict(table.schema.metadata) + record = {**json.loads(meta[META_PASS]), **changes} + meta[META_PASS] = json.dumps(record, sort_keys=True, separators=(",", ":")).encode("ascii") + return table.replace_schema_metadata(meta) + return change + + +class EmbedOptionTest(_Tmp): + + def test_the_command_help(self): + self.assertEqual(_command("build", "sequences-embed").help, EMBED_HELP) + output = CliRunner().invoke(search.app, ["build", "sequences-embed", "--help"], env={"COLUMNS": "400"}).output + for text in (EMBED_HELP, EMBED_OUTPUT_DB_HELP, SHARDS_FOLDER_HELP, PLAN_HELP): + with self.subTest(text=text[:40]): + self.assertIn(text, _flat(output)) + + def test_the_options(self): + """Exactly the step-1 options; each one the type, default and help of 'build sequences' unless the plan gives + its own help (--output-db, --shards-folder, which is --tmp-embedding-folder renamed). --min-res-n refuses a + negative value (a record holds no negative count).""" + embed = {param.name: param for param in _command("build", "sequences-embed").params} + oneshot = {param.name: param for param in _command("build", "sequences").params} + self.assertEqual(sorted(embed), sorted([ + "fasta_file", "output_db", "shards_folder", "min_res_n", "accelerator", "num_nodes", "devices", "strategy", + "batch_size", "num_workers", "residue_codes", "log_level"])) + own_help = {"output_db": EMBED_OUTPUT_DB_HELP, "shards_folder": SHARDS_FOLDER_HELP} + + def info(param, *drop): + out = param.to_info_dict() + for key in ("name", "opts", "help", *drop): + out.pop(key) + return out + + for name, param in embed.items(): + with self.subTest(name=name): + same = oneshot["tmp_embedding_folder" if name == "shards_folder" else name] + self.assertEqual(param.help, own_help.get(name, same.help)) + if name == "min_res_n": + self.assertEqual(info(param, "type"), info(same, "type")) + self.assertEqual((param.type.min, param.type.max, param.default), (0, None, 0)) + else: + self.assertEqual(info(param), info(same)) + self.assertEqual(embed["shards_folder"].opts, ["--shards-folder"]) + self.assertTrue(embed["shards_folder"].required) + self.assertEqual(embed["residue_codes"].help, PLAN_HELP) + codes = embed["residue_codes"] + self.assertEqual((codes.opts, codes.secondary_opts, codes.default), + (["--residue-codes"], ["--no-residue-codes"], False)) + + def test_options_another_command_owns(self): + """The index phase's options (step 2 takes them), --cleanup-embeddings (the shards are this step's output), + --tmp-embedding-folder (renamed --shards-folder) and step 2's run options: exit 2, nothing read or written.""" + for option in (("--use-gpu-index",), ("--index-type", "flat"), ("--ivf-nlist", "256"), ("--ivf-nprobe", "4"), + ("--load-batch-size", "1000"), ("--cleanup-embeddings",), + ("--tmp-embedding-folder", self.shards), ("--from-shards", self.shards), ("--run-id", "x")): + with self.subTest(option=option[0]), _esm3_spy() as esm3: + result = invoke(*_embed_args(_FASTA, self.tmp / "db", self.shards, *option)) + self.usage_error(result, option[0]) + self.assertRegex(_flat(result.output), no_such_option(option[0])) + esm3.assert_not_called() + self.usage_error(invoke(*_embed_args(_FASTA, self.tmp / "db", self.shards, "--min-res-n", "-1")), + "--min-res-n") + self.usage_error(invoke("build", "sequences-embed", "--fasta-file", _FASTA, "--output-db", self.tmp / "db"), + "--shards-folder") + self.assertEqual(list(self.shards.iterdir()), []) + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), []) + + +class EmbedStepTest(_WithBundle): + + def embed(self, *extra, ranks=1, run_id="p8b3", fasta=_FASTA, db=None, shards=None, gather=None, **fake): + """Step 1 with the fake pass on ``ranks`` ranks (``gather``, default :func:`_ranks`, stands in for the + all-gather of more than one rank).""" + db = db or self.tmp / "db" + with launch_env(FOLDMATCH_RUN_ID=run_id), ExitStack() as stack: + stack.enter_context(mock.patch("foldmatch.search.embedding_computer.chain_predict", + fake_chain_predict(ranks=ranks, **fake))) + if gather is not None or ranks > 1: + stack.enter_context(mock.patch.object(search, "_gather_rank_facts", + side_effect=gather or _ranks(ranks))) + return invoke(*_embed_args(fasta, db, shards or self.shards, *extra)) + + def logged_embed(self, *extra, **kwargs): + """The step's root-logger messages (``set_log_level`` would replace assertLogs' handler).""" + with mock.patch.object(search, "set_log_level"), self.assertLogs(level="INFO") as logs: + result = self.embed(*extra, **kwargs) + self.assertEqual(result.exit_code, 0, result.output) + return [record.getMessage() for record in logs.records] + + def test_the_run_record(self): + """Plain and codes, on 1 and 2 ranks, with --min-res-n 0 and 100: the shards and the record are the only files + (no database file), and every field of the record holds what the run did.""" + stat = split_run.fasta_stat(_FASTA) + sha256 = hashlib.sha256(_FASTA.read_bytes()).hexdigest() + system = {"name": SYSTEM.name, "digest": SYSTEM.digest(), "encoding_digest": SYSTEM.encoding_digest()} + for codes in (False, True): + for ranks in (1, 2): + for min_res_n in (0, 100): + name = f"{'codes' if codes else 'plain'}{ranks}-{min_res_n}" + with self.subTest(codes=codes, ranks=ranks, min_res_n=min_res_n): + shards = self.tmp / name + shards.mkdir() + db = self.tmp / f"db-{name}" + result = self.embed("--min-res-n", min_res_n, *(("--residue-codes",) if codes else ()), + ranks=ranks, run_id=name, db=db, shards=shards) + self.assertEqual(result.exit_code, 0, result.output) + kept = 2 if min_res_n == 0 else 1 + deal = split_run.dealt_rows(kept, ranks) + names = [f"emb_{name}-{rank}.parquet" for rank in sorted(deal)] + self.assertEqual(sorted(p.name for p in shards.iterdir()), names + [f"emb_{name}.run.json"]) + self.assertEqual(sorted(p.name for p in self.tmp.glob(f"db-{name}*")), []) + path = shards / f"emb_{name}.run.json" + record = json.loads(path.read_text()) + self.assertEqual(split_run.read_run_record(path), record) + created = datetime.strptime(record.pop("created_utc"), "%Y-%m-%dT%H:%M:%SZ") + self.assertLess(abs(datetime.now(timezone.utc).replace(tzinfo=None) - created), + timedelta(minutes=5)) + run_pass = record.pop("pass") + self.assertEqual(record, { + "kind": "foldmatch-sequence-run", "record_version": 1, "run_id": name, + "command": "build sequences-embed", "sealed": False, + "foldmatch_version": foldmatch.__version__, + "fasta": {"path": str(_FASTA.resolve()), "bytes": stat.bytes, "mtime_ns": stat.mtime_ns, + "sha256": sha256}, + "min_res_n": min_res_n, "strict": codes, + "preflight": split_run.preflight_counts(preflight_fasta(_FASTA, min_res_n)), + "output_db": os.path.abspath(db), + "residue_output": "residue_codes" if codes else None, + "system": system if codes else None, + "world_size": ranks, + "options": {"accelerator": "cpu", "devices": ["1"], "num_nodes": 1, "strategy": "auto", + "batch_size": 1, "num_workers": 0}, + "shards": [{"name": n, "rank": rank, "rows": deal[rank], + "bytes": (shards / n).stat().st_size} for rank, n in zip(sorted(deal), names)], + "rows": kept, + }) + counts = record["preflight"] + self.assertEqual((counts["entries"], counts["entries_kept"], counts["residues"]), + (2, 2, 303) if min_res_n == 0 else (2, 1, 245)) + if codes: + self.assertEqual(run_pass, read_shard_info(shards / names[0]).pass_record) + self.assertEqual((run_pass["accelerator"], run_pass["world_size"], run_pass["min_res_n"], + run_pass["esm_revision"]), ("cpu", ranks, min_res_n, ESM_REVISION)) + else: + self.assertEqual(run_pass, pass_record(min_res_n=min_res_n, esm_revision=ESM_REVISION, + world_size=ranks)) + self.assertIsNone(run_pass["accelerator"]) + + def test_the_pass_of_a_cuda_run(self): + """A codes run records its shards' pass (the device type and its ESM3 precision); a plain one records no + device.""" + for codes in (True, False): + with self.subTest(codes=codes): + shards = self.tmp / f"cuda-{codes}" + shards.mkdir() + result = self.embed(*(("--residue-codes",) if codes else ()), ranks=2, device="cuda", shards=shards, + run_id=f"cuda-{codes}", db=self.tmp / f"cuda-{codes}-db") + self.assertEqual(result.exit_code, 0, result.output) + run_pass = split_run.read_run_record(shards / f"emb_cuda-{codes}.run.json")["pass"] + if codes: + self.assertEqual((run_pass["accelerator"], run_pass["esm_precision"], run_pass["world_size"]), + ("cuda", "bf16-autocast", 2)) + else: + self.assertEqual((run_pass["accelerator"], run_pass["esm_precision"], run_pass["world_size"]), + (None, None, 2)) + + def test_every_rank_gathers_its_facts_after_the_pass(self): + """After the pass and before the rank-0 check, every rank gathers what it saw: its FASTA stat before the + pre-flight and after the pass, no stat error, the size, its pre-flight counts and the pre-flight's seconds.""" + manager = mock.Mock() + with mock.patch.object(search, "_run_sequence_pass", wraps=search._run_sequence_pass) as run_pass, \ + mock.patch.object(search, "_gather_rank_facts", wraps=search._gather_rank_facts) as gather, \ + mock.patch.object(search, "_is_rank_zero", wraps=search._is_rank_zero) as rank_zero: + for name, spy in (("run_pass", run_pass), ("gather", gather), ("rank_zero", rank_zero)): + manager.attach_mock(spy, name) + result = self.embed("--residue-codes") + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual([c[0] for c in manager.mock_calls], ["run_pass", "gather", "rank_zero"]) + facts = gather.call_args.args[0] + self.assertEqual(sorted(facts), ["after", "before", "error", "preflight", "seconds", "size"]) + stat = split_run.fasta_stat(_FASTA) + self.assertEqual((facts["before"], facts["after"], facts["error"], facts["size"]), + (stat, stat, None, stat.bytes)) + self.assertEqual(facts["preflight"], split_run.preflight_counts(preflight_fasta(_FASTA, 0))) + self.assertGreaterEqual(facts["seconds"], 0) + + def test_gather_rank_facts(self): + """``[facts]`` without a process group; ``all_gather_object`` over the group's ranks with one.""" + self.assertEqual(search._gather_rank_facts({"rank": 0}), [{"rank": 0}]) + import torch.distributed as dist + + def all_gather_object(out, obj): + out[:] = [obj, {"rank": 1}] + + with mock.patch.object(dist, "is_available", return_value=True), \ + mock.patch.object(dist, "is_initialized", return_value=True), \ + mock.patch.object(dist, "get_world_size", return_value=2), \ + mock.patch.object(dist, "all_gather_object", side_effect=all_gather_object) as gathered: + self.assertEqual(search._gather_rank_facts({"rank": 0}), [{"rank": 0}, {"rank": 1}]) + self.assertEqual(gathered.call_args.args, ([{"rank": 0}, {"rank": 1}], {"rank": 0})) + + def test_ranks_above_zero_write_no_record(self): + """A rank above 0 gathers its facts and exits 0 once the pass is over: no record, and nothing new logged.""" + with mock.patch.object(search, "_is_rank_zero", return_value=False), \ + mock.patch.object(search, "_gather_rank_facts", wraps=search._gather_rank_facts) as gather, \ + mock.patch.object(search, "set_log_level"), self.assertLogs(level="INFO") as logs: + result = self.embed("--residue-codes") + self.assertEqual(result.exit_code, 0, result.output) + gather.assert_called_once() + self.assertEqual([p.name for p in self.shards.iterdir()], ["emb_p8b3-0.parquet"]) + messages = [record.getMessage() for record in logs.records] + self.assertFalse([m for m in messages if m.startswith(("Pre-flight on", "FASTA sha256", "Run p8b3", + "Build the database", "Add --index-type"))], messages) + + def test_the_log_lines(self): + """Rank 0 logs the ranks' pre-flight times, the FASTA hash, the run, the next command and the index hint, in + this order.""" + size = _FASTA.stat().st_size + cases = ((("--residue-codes",), 1, f"2 entries in 1 shard(s) of 1 rank(s), residue codes of {SYSTEM.name}"), + ((), 2, "2 entries in 2 shard(s) of 2 rank(s)")) + for extra, ranks, run in cases: + with self.subTest(ranks=ranks): + for path in self.shards.iterdir(): + path.unlink() + messages = self.logged_embed(*extra, ranks=ranks) + prefixes = ("Pre-flight on ", "FASTA sha256 of ", "Run p8b3 complete: ", "Build the database with: ", + "Add --index-type") + lines = [[m for m in messages if m.startswith(prefix)] for prefix in prefixes] + self.assertEqual([len(found) for found in lines], [1] * 5, messages) + lines = [found[0] for found in lines] + self.assertRegex(lines[0], rf"^Pre-flight on {ranks} rank\(s\): min \d+\.\d s, max \d+\.\d s$") + self.assertRegex(lines[1], rf"^FASTA sha256 of {re.escape(str(_FASTA.resolve()))} \({size} bytes\) " + rf"in \d+\.\d s$") + self.assertEqual(lines[2], f"Run p8b3 complete: {run}; record " + f"{self.shards.resolve() / 'emb_p8b3.run.json'}") + self.assertIn(f"auto picks hnsw at {FaissEmbeddingDatabase.N_EMBEDDINGS_DB_THR:,} entries or more", + lines[4]) + self.assertIn("about 12 KiB per entry at peak; ivf_pq is bounded", lines[4]) + order = [messages.index(line) for line in lines] + self.assertEqual(order, sorted(order)) + self.assertFalse([m for m in messages if "stay allocated" in m or m.startswith("Index phase")]) + + def test_the_next_command(self): + """The logged next command, split as a shell splits it, is step 2 on this run: the FASTA, the database path the + record holds (absolute, also for a relative --output-db), the shards folder and the run id, with paths that + hold spaces. (Running it needs 'build sequences-index', P8b.4.)""" + folder = self.tmp / "with space" + folder.mkdir() + fasta = Path(shutil.copyfile(_FASTA, folder / "my seqs.fasta")) + shards = folder / "shard s" + shards.mkdir() + for db, run_id in ((folder / "my db", "abs"), (Path("rel dir") / "my db", "rel")): + with self.subTest(db=str(db)), chdir(folder): + messages = self.logged_embed(fasta=fasta, db=db, shards=shards, run_id=run_id) + line = next(m for m in messages if m.startswith("Build the database with: ")) + argv = shlex.split(line[len("Build the database with: "):]) + record = split_run.read_run_record(shards / f"emb_{run_id}.run.json") + self.assertEqual(record["output_db"], str(folder.resolve() / db) if run_id == "rel" else str(db)) + self.assertEqual(argv, ["fm-search", "build", "sequences-index", "--fasta-file", str(fasta.resolve()), + "--output-db", record["output_db"], "--from-shards", str(shards.resolve()), + "--run-id", run_id]) + + +class EmbedRefusalTest(_WithBundle): + + def refused_before_esm3(self, *extra, fasta=_FASTA, codes=False, prepare=None, cause=None): + """Step 1 with the real pass under the ESM3 spy, refused before ESM3 (exit 1, a RefusalError), the bundle + untouched unless ``prepare`` (a patch, or ``nullcontext()`` for the real one) says otherwise: no file written, + the shards folder unchanged. Returns the message.""" + before = {p.name: p.read_bytes() for p in self.shards.iterdir()} + args = _embed_args(fasta, self.tmp / "db", self.shards, *extra, *(("--residue-codes",) if codes else ())) + with _esm3_spy() as esm3, prepare or mock.patch.object(ss, "prepare", + side_effect=AssertionError("bundle fetched")): + result = invoke(*args) + esm3.assert_not_called() + message = self.refused(result, cause) + self.assertEqual({p.name: p.read_bytes() for p in self.shards.iterdir()}, before) + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), []) + return message + + @staticmethod + def every_rank(modes=(False, True), envs=_RANK_ENVS): + """``(codes, rank env)`` for each mode, on rank 0 and on a rank started with the run id (by default).""" + return [(codes, rank_env) for codes in modes for rank_env in envs] + + def test_e1_fasta_the_pre_flight_refuses(self): + """A header without an id in either mode; duplicate ids, '<'/'>' and non-ASCII under codes. A plain run takes + those, as 'build sequences' does (the pass is reached).""" + cases = [(">\nACDE\n>b\nKLMN\n", "is empty", False), (">\nACDE\n>b\nKLMN\n", "is empty", True), + (">a\nACDE\n>b\nKLM\n>a\nACDE\n", "repeat an earlier id", True), + (">a\nACDE\n", "'<' or '>'", True), (">a\nACDÉ\n", "non-ASCII", True)] + for text, what, codes in cases: + fasta = _write(self.tmp / "bad.fasta", text) + for _, rank_env in self.every_rank(modes=(codes,)): + with self.subTest(what=what, codes=codes, rank_env=rank_env), launch_env(**rank_env): + self.assertIn(what, self.refused_before_esm3(fasta=fasta, codes=codes, cause=ValueError)) + with launch_env(FOLDMATCH_RUN_ID="dup"), _esm3_spy() as esm3: + result = invoke(*_embed_args(_write(self.tmp / "dup.fasta", ">a\nACDE\n>a\nACDE\n"), self.tmp / "db", + self.shards)) + self.assertIsInstance(result.exception, AssertionError) # the pass was reached + esm3.assert_called_once() + + def test_e2_an_output_exists(self): + for suffix in (".index", ".metadata", ".invlists", ".sequences", ".rescodes.bin", ".rescodes.idx", + ".rescodes.json", ".rescodes.json.tmp"): + existing = self.tmp / f"db{suffix}" + for codes, rank_env in self.every_rank(): + with self.subTest(suffix=suffix, codes=codes, rank_env=rank_env), launch_env(**rank_env): + existing.touch() + try: + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare") as prepare: + result = invoke(*_embed_args(_FASTA, self.tmp / "db", self.shards, + *(("--residue-codes",) if codes else ()))) + finally: + existing.unlink() + message = self.refused(result, FileExistsError) + self.assertIn(f"Database already exists at {self.tmp / 'db'} (found: {existing})", message) + self.assertIn("Choose a different --output-db", message) + esm3.assert_not_called() + prepare.assert_not_called() + self.assertEqual(list(self.shards.iterdir()), []) + + def test_e3_nothing_to_build(self): + for codes, rank_env in self.every_rank(): + with self.subTest(codes=codes, rank_env=rank_env), launch_env(**rank_env): + self.assertIn("there is nothing to build", + self.refused_before_esm3("--min-res-n", "1000", codes=codes, cause=ValueError)) + + def test_e4_no_run_id(self): + """A malformed FOLDMATCH_RUN_ID; a rank above 0 with no run id to share.""" + cases = [*self.every_rank(envs=({"FOLDMATCH_RUN_ID": "a b"}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "a b"})), + *self.every_rank(envs=({"LOCAL_RANK": 1},))] + for codes, rank_env in cases: + with self.subTest(codes=codes, rank_env=rank_env), launch_env(**rank_env): + what = "is not a valid run id" if "FOLDMATCH_RUN_ID" in rank_env else "has no run id to share" + self.assertIn(what, self.refused_before_esm3(codes=codes, cause=ValueError)) + + def test_e5_shards_of_the_same_run_id(self): + (self.shards / "emb_r-0.parquet").write_bytes(b"an earlier run's shard") + for codes, rank_env in self.every_rank(envs=({"FOLDMATCH_RUN_ID": "r"}, _RANK_ENVS[1])): + with self.subTest(codes=codes, rank_env=rank_env), launch_env(**rank_env): + self.assertIn("same run id", self.refused_before_esm3(codes=codes, cause=FileExistsError)) + + def test_e6_a_record_of_the_same_run_id(self): + """A record, or a staged .tmp, of an earlier run with this id: refused after the bundle check, before ESM3.""" + for name, what in (("emb_r.run.json", "a run record of an earlier run with the same run id"), + ("emb_r.run.json.tmp", "stopped while it wrote its run record")): + (self.shards / name).write_text("{}\n") + for codes, rank_env in self.every_rank(envs=({"FOLDMATCH_RUN_ID": "r"}, _RANK_ENVS[1])): + with self.subTest(name=name, codes=codes, rank_env=rank_env), launch_env(**rank_env): + message = self.refused_before_esm3(codes=codes, prepare=nullcontext()) + self.assertIn(what, message) + self.assertIn(str(self.shards.resolve() / name), message) + (self.shards / name).unlink() + + def test_e7_the_bundle(self): + """A bundle that cannot be fetched; a bundle missing from the cache of a rank that must find it cached (a rank + above 0, every rank of a multi-node run): the refusal names download-models.""" + for _, rank_env in self.every_rank(modes=(True,)): + with self.subTest(rank_env=rank_env), launch_env(**rank_env): + unavailable = mock.patch.object(ss, "prepare", side_effect=ss.BundleUnavailableError("not cached")) + self.assertIn("not cached", self.refused_before_esm3(codes=True, prepare=unavailable, + cause=ss.BundleUnavailableError)) + prepare = ss.prepare + empty = self.tmp / "empty-hf-cache" + empty.mkdir() + + def empty_cache(system, *, local_files_only): + self.assertTrue(local_files_only) + return prepare(system, cache_dir=empty, local_files_only=local_files_only) + + for extra, rank_env in (((), _RANK_ENVS[1]), (("--num-nodes", "2"), {})): + with self.subTest(extra=extra, rank_env=rank_env), launch_env(**rank_env): + message = self.refused_before_esm3(*extra, codes=True, cause=ss.BundleUnavailableError, + prepare=mock.patch.object(ss, "prepare", side_effect=empty_cache)) + self.assertIn("download-models", message) + + def test_e8_the_residue_pass(self): + """The pass's own refusal before ESM3 loads (check_residue_pass): a prepared system that is not this + FoldMatch's registry entry.""" + foreign = ss.PreparedSystem(system=OTHER_ENCODING, files={}, bundle={}, encoders_file="e", scoring_file="s") + for _, rank_env in self.every_rank(modes=(True,)): + with self.subTest(rank_env=rank_env), launch_env(**rank_env): + message = self.refused_before_esm3(codes=True, cause=ss.UnknownScoringSystemError, + prepare=mock.patch.object(ss, "prepare", return_value=foreign)) + self.assertIn(OTHER_ENCODING.name, message) + + # -- rank 0, after the pass --------------------------------------------------------------------------------------- + def post_pass(self, *extra, codes=True, ranks=1, gather=None, predict=None, fasta=_FASTA, foreign=(), **fake): + """Step 1 with the fake pass (``predict`` wraps or replaces it), refused by rank 0 after the pass: exit 1, a + RefusalError saying that no record was written and where the shards are kept; no record and no .tmp but the + ``foreign`` files the pass put there; the shards as the pass left them, byte for byte; no database file. + Returns the message.""" + for path in self.shards.iterdir(): + path.unlink() + written = {} + base = predict or fake_chain_predict(ranks=ranks, **fake) + + def recording(**kwargs): + base(**kwargs) + written.update({p.name: p.read_bytes() for p in Path(kwargs["out_folder"]).glob("*.parquet")}) + + with launch_env(FOLDMATCH_RUN_ID="e"), ExitStack() as stack: + stack.enter_context(mock.patch("foldmatch.search.embedding_computer.chain_predict", recording)) + if gather is not None or ranks > 1: + stack.enter_context(mock.patch.object(search, "_gather_rank_facts", + side_effect=gather or _ranks(ranks))) + result = invoke(*_embed_args(fasta, self.tmp / "db", self.shards, *extra, + *(("--residue-codes",) if codes else ()))) + message = self.refused(result) + self.assertTrue(written) + self.assertEqual({p.name: p.read_bytes() for p in self.shards.glob("*.parquet")}, written) + self.assertEqual(sorted(p.name for p in self.shards.iterdir() if p.suffix != ".parquet"), sorted(foreign)) + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), []) + self.assertIn(f"No run record was written; the shards are kept in {self.shards.resolve()} as run e.", message) + return message + + def assert_seal_hint(self, message, fasta=_FASTA, present=True): + hint = (f"Once the cause is fixed, 'build sequences-seal' can record them: " + shlex.join([ + "fm-search", "build", "sequences-seal", "--fasta-file", str(Path(fasta).resolve()), "--from-shards", + str(self.shards.resolve()), "--run-id", "e", "--min-res-n", "0"])) + if present: + self.assertTrue(message.endswith(hint), message) + else: + self.assertNotIn("sequences-seal", message) + + def test_e9_the_shards_are_not_the_deal(self): + """A lost shard (the embedding count), a rank of the world with no shard (W = 3 gathered for 2 shards), and + rows that are not the deal (the right count): refused before any footer is read.""" + five = _write(self.tmp / "five.fasta", _FIVE) + + def lost(**kwargs): + fake_chain_predict(ranks=2)(**kwargs) + (Path(kwargs["out_folder"]) / f"{kwargs['out_name']}-1.parquet").unlink() + + def blocks(**kwargs): # rank 0 holds entries 0, 2; rank 1 entries 4, 1, 3: five rows, not the deal + fake_chain_predict(ranks=2)(**kwargs) + first, second = (Path(kwargs["out_folder"]) / f"{kwargs['out_name']}-{rank}.parquet" for rank in (0, 1)) + t0, t1 = pq.read_table(first), pq.read_table(second) + pq.write_table(t0.slice(0, 2), first) + pq.write_table(pa.concat_tables([t0.slice(2), t1.replace_schema_metadata(t0.schema.metadata)]), second) + + for codes in (False, True): + with self.subTest(case="a lost shard", codes=codes), \ + mock.patch.object(split_run, "check_plain_footers", side_effect=AssertionError("footers read")), \ + mock.patch.object(split_run, "check_codes_footers", side_effect=AssertionError("footers read")): + message = self.post_pass(codes=codes, ranks=2, predict=lost, fasta=five) + self.assertIn("Found 3 embeddings in 1 shard(s)", message) + self.assertIn("a rank's shard is missing", message) + self.assert_seal_hint(message, present=False) + with self.subTest(case="a dropped rank", codes=codes): + message = self.post_pass(codes=codes, ranks=2, gather=_ranks(3), fasta=five) + self.assertIn("are not the deal of its 5 entries over the 3 rank(s) of the pass (no shard of rank(s) " + "2; rank 0 has 3 row(s), not 2): the shards cannot be indexed.", message) + self.assert_seal_hint(message, present=False) + with self.subTest(case="rows not the deal", codes=codes): + message = self.post_pass(codes=codes, ranks=2, predict=blocks, fasta=five) + self.assertIn("(rank 0 has 2 row(s), not 3; rank 1 has 3 row(s), not 2)", message) + + def test_e10_the_footers(self): + """Codes: a shard of another system's encoding, a plain shard among codes shards, a pass record with another + --min-res-n or world size; plain: a shard that is not plain. Each names the shard.""" + def pass_then(change, rank=0, ranks=1): + def predict(**kwargs): + fake_chain_predict(ranks=ranks)(**kwargs) + _rewrite(Path(kwargs["out_folder"]) / f"{kwargs['out_name']}-{rank}.parquet", change) + return predict + + def plain(table): + return table.select(["id", "embedding"]).replace_schema_metadata(None) + + def tagged(table): + return table.replace_schema_metadata({b"foldmatch.residue_output": b"residue_codes"}) + + cases = { + "a stale system": ({"system": OTHER_ENCODING}, ("stale", "encoding_digest")), + "a plain shard mixed in": ({"predict": pass_then(plain, rank=1, ranks=2), "ranks": 2}, + ("mixed embedding shards", "emb_e-1.parquet")), + "pass.min_res_n": ({"predict": pass_then(_with_pass(min_res_n=5))}, + ("do not record a pass with --min-res-n 0 on 1 rank(s)", + "emb_e-0.parquet (min_res_n 5, world_size 1)")), + "pass.world_size": ({"predict": pass_then(_with_pass(world_size=3))}, + ("emb_e-0.parquet (min_res_n 0, world_size 3)",)), + "a plain run's shard that is not plain": ({"codes": False, "predict": pass_then(tagged)}, + ("not plain embedding shards", "emb_e-0.parquet")), + } + for case, (kwargs, texts) in cases.items(): + with self.subTest(case=case): + message = self.post_pass(**kwargs) + for text in texts: + self.assertIn(text, message) + self.assert_seal_hint(message, present=False) + + def test_e11_a_fasta_modified_during_the_pass(self): + fasta = Path(shutil.copyfile(_FASTA, self.tmp / "copy.fasta")) + size = fasta.stat().st_size + + def appending(**kwargs): + fake_chain_predict()(**kwargs) + with open(fasta, "a") as f: + f.write(">late\nACDE\n") + + for codes in (False, True): + with self.subTest(codes=codes): + shutil.copyfile(_FASTA, fasta) + message = self.post_pass(codes=codes, predict=appending, fasta=fasta) + self.assertIn(f"The FASTA was modified during the pass (rank 0: before {size} bytes, mtime_ns", message) + self.assertIn(f", after {size + 11} bytes, mtime_ns", message) + self.assert_seal_hint(message, fasta=fasta) + + def test_e11_another_rank_saw_another_fasta(self): + """Rank 1's stat changed during its pass (inode, mtime or size), its FASTA has another size, its pre-flight + counted otherwise, or its stat failed.""" + def changed(**fields): + return lambda facts: {"after": facts["after"]._replace(**{k: getattr(facts["after"], k) + v + for k, v in fields.items()})} + + def other_size(facts): + grown = facts["before"]._replace(bytes=facts["before"].bytes + 1) + return {"before": grown, "after": grown, "size": grown.bytes} + + size = _FASTA.stat().st_size + cases = { + "an inode": (changed(ino=1), "The FASTA was modified during the pass (rank 1: before"), + "an mtime": (changed(mtime_ns=1), "The FASTA was modified during the pass (rank 1: before"), + "a size": (changed(bytes=1), f", after {size + 1} bytes"), + "another size": (other_size, f"The ranks did not read the same FASTA (rank 1: {size + 1} bytes, rank 0: " + f"{size})"), + "other counts": (lambda facts: {"preflight": {**facts["preflight"], "residues": 7}}, + "(rank 1: residues 7, rank 0: residues 303)"), + "a stat error": (lambda facts: {"before": None, "size": None, + "error": "stat before the pre-flight: [Errno 2] No such file"}, + "could not be stat'ed on every rank (rank 1: stat before the pre-flight: [Errno 2] No " + "such file)"), + } + for case, (change, text) in cases.items(): + for codes in (False, True): + with self.subTest(case=case, codes=codes): + message = self.post_pass(codes=codes, ranks=2, gather=_ranks(2, change)) + self.assertIn(text, message) + self.assert_seal_hint(message) + + def test_e11_rank_zero_cannot_stat_or_hash(self): + stat = split_run.fasta_stat + calls = [] + + def second_fails(path): + calls.append(path) + if len(calls) == 2: + raise FileNotFoundError(errno.ENOENT, "No such file or directory", str(path)) + return stat(path) + + with mock.patch.object(split_run, "fasta_stat", side_effect=second_fails): + message = self.post_pass() + self.assertEqual(len(calls), 3) # before the pre-flight, after the pass (failed), after the hash + self.assertIn("(rank 0: stat after the pass: [Errno 2] No such file or directory", message) + self.assert_seal_hint(message) + with mock.patch.object(split_run, "fasta_sha256", side_effect=OSError(errno.EIO, "Input/output error")): + message = self.post_pass() + self.assertIn("Input/output error", message) + self.assert_seal_hint(message) + + def test_e11_a_write_during_rank_zeros_hash(self): + """Rank 0's bracket closes after the hash: a write while rank 0 hashes the FASTA (its mtime moved once the hash + is done) is refused, so the record never pairs a sha256 with a stat taken before it.""" + fasta = Path(shutil.copyfile(_FASTA, self.tmp / "copy.fasta")) + real = split_run.fasta_sha256 + + def written_while_hashed(path): + out = real(path) + stat = os.stat(path) + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10 ** 9)) + return out + + for codes in (False, True): + with self.subTest(codes=codes), mock.patch.object(split_run, "fasta_sha256", + side_effect=written_while_hashed): + message = self.post_pass(codes=codes, fasta=fasta) + self.assertIn("The FASTA was modified during the pass (rank 0: before", message) + self.assert_seal_hint(message, fasta=fasta) + + def test_e11_node_local_copies_are_allowed(self): + """Another inode on rank 1, the same before and after its pass: its own copy of the FASTA, accepted.""" + def copy(facts): + return {key: facts[key]._replace(ino=facts[key].ino + 1) for key in ("before", "after")} + + with launch_env(FOLDMATCH_RUN_ID="copy"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(ranks=2)), \ + mock.patch.object(search, "_gather_rank_facts", side_effect=_ranks(2, copy)): + result = invoke(*_embed_args(_FASTA, self.tmp / "db", self.shards, "--residue-codes")) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(split_run.read_run_record(self.shards / "emb_copy.run.json")["world_size"], 2) + + def test_e12_the_record_cannot_be_written(self): + """A full disk while the record is staged: its .tmp is removed, the shards kept.""" + with mock.patch.object(split_run, "_write_all", + side_effect=OSError(errno.ENOSPC, "No space left on device")): + message = self.post_pass() + self.assertIn("No space left on device", message) + self.assertIn(f"The run record {self.shards.resolve() / 'emb_e.run.json'} cannot be written", message) + self.assert_seal_hint(message) + + def test_e12_a_record_or_tmp_that_appeared_during_the_pass(self): + """Another process's .tmp or record of this run id, written during the pass: refused, and left as it is.""" + for name, text in (("emb_e.run.json.tmp", "cannot be staged"), ("emb_e.run.json", "File exists")): + def predict(**kwargs): + fake_chain_predict()(**kwargs) + (Path(kwargs["out_folder"]) / name).write_text("another process\n") + + with self.subTest(name=name): + message = self.post_pass(predict=predict, foreign=(name,)) + self.assertIn(text, message) + self.assertIn(name, message) + self.assertEqual((self.shards / name).read_text(), "another process\n") + self.assert_seal_hint(message) + + +# ===================================================================================================================== +# P8b.4: build sequences-index +# ===================================================================================================================== +#: Plan P8b (CLI), verbatim: step 2's command help and the help texts it does not take from 'build sequences'. +INDEX_HELP = ( + "Step 2 of a split 'build sequences': build the database from the shards of one 'build sequences-embed' run, " + "after checking against its run record that they are complete and were computed from this FASTA: the FAISS " + "index, the sequence store and, when the run computed residue codes, the residue-code store, as 'build " + "sequences' builds them after its ESM3 pass. No ESM3, GPU or download. One process: run it as its own job on a " + "CPU node, not inside the GPU job; FAISS uses the node's physical cores unless OMP_NUM_THREADS is set." +) +INDEX_OWN_HELP = { + "fasta_file": "The FASTA file the run embedded. Checked against the run record (size, sha256, pre-flight counts) " + "and against the id of every shard row; it may have moved since.", + "output_db": "Path to save the FAISS database (as for 'build sequences'). Usually the run's --output-db, which " + "the record holds; another path is allowed.", + "from_shards": "Folder holding the shards and the run record of a 'build sequences-embed' run (its " + "--shards-folder).", + "run_id": "Run id of the run to build from (its record is emb_.run.json; 'build sequences-embed' logs " + "it). Needed when the folder holds more than one run, unfinished ones included.", + "residue_codes": "Build the residue-code store from the run's codes (enabling `query sequences --scoring " + "esm-codes`). Default: exactly when the run computed them ('build sequences-embed " + "--residue-codes'). --residue-codes refuses a run without codes; --no-residue-codes leaves the " + "store out (add it later with 'build residue-codes --from-shards').", + "cleanup_embeddings": "Delete the run's record, then its shards, from --from-shards once the database is built. " + "Use it only on the last build from this run. Refused together with --no-residue-codes on a " + "run with residue codes. By default they are kept.", +} +#: SLURM job variables that launch_env leaves alone and step 2's launch checks read. +_JOB_ENV = ("SLURM_JOB_NUM_NODES", "SLURM_GPUS_ON_NODE", "SLURM_JOB_GPUS") +#: The build-block keys in which a split-built code store may differ from the one-shot's (plan P8b, Identical outputs). +_EXEMPT = ("created_utc", "run_id", "split") + + +def _index_args(fasta, db, shards, *extra): + return ("build", "sequences-index", "--fasta-file", fasta, "--output-db", db, "--from-shards", shards, *extra) + + +#: ``sys.argv[0]`` of the ``fm-search`` console script, as step 2 and the seal see it in ``_SplitBuild`` (P9.I2). +_FM_SEARCH = "/opt/foldmatch/bin/fm-search" + + +def _as_fm_search(args): + """``sys.argv`` as the ``fm-search`` console script sets it for ``args``.""" + return mock.patch.object(sys, "argv", [_FM_SEARCH, *map(str, args)]) + + +def _command_line(args) -> str: + """The command line that the launch checks' advice gives for ``args`` run through the console script.""" + return shlex.join(["fm-search", *map(str, args)]) + + +#: What the launch checks' advice says the one-node job's SCRIPT runs next time (P9.I2, review finding 2). +_NEXT_BUILD = ("with the paths and run id of that job's build (export FOLDMATCH_RUN_ID=ID before the sbatch, and pass " + "--run-id \"$FOLDMATCH_RUN_ID\")") + + +def _tree(*folders) -> dict: + """``{path: sha256}`` of every file under ``folders`` (``None`` for a folder), and whether each folder exists.""" + out = {} + for folder in map(Path, folders): + out[str(folder)] = folder.exists() + for path in sorted(folder.rglob("*")): + out[str(path)] = None if path.is_dir() else hashlib.sha256(path.read_bytes()).hexdigest() + return out + + +def _database_files(db) -> dict: + """``{suffix: path}`` of the files of the database at ``db`` (``--output-db``).""" + db = Path(db) + return {path.name[len(db.name):]: path for path in db.parent.glob(f"{glob.escape(db.name)}.*")} + + +def _code_store_index(path): + """``(entries rows, meta rows)`` of a ``.rescodes.idx``.""" + con = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + try: + return (con.execute("SELECT * FROM entries ORDER BY id").fetchall(), + dict(con.execute("SELECT key, value FROM meta").fetchall())) + finally: + con.close() + + +def assert_same_database(a, b, exempt=_EXEMPT): + """Plan P8b (Identical outputs): the databases at ``a`` and ``b`` (``--output-db`` paths) have the same files, each + byte-identical except the residue-code store's index and manifest, which are equal in content: the ``entries`` + rows equal, the ``meta`` rows equal except ``build_json``, which decodes to its own manifest's ``build`` block, + and the manifests equal except the ``build`` keys in ``exempt`` (present on one side or not).""" + files = _database_files(a), _database_files(b) + if sorted(files[0]) != sorted(files[1]): + raise AssertionError(f"{a} and {b} have other files: {sorted(files[0])} != {sorted(files[1])}") + for suffix in sorted(files[0]): + if suffix not in (".rescodes.idx", ".rescodes.json") and \ + files[0][suffix].read_bytes() != files[1][suffix].read_bytes(): + raise AssertionError(f"{files[0][suffix]} and {files[1][suffix]} differ") + if ".rescodes.json" not in files[0]: + return + stores = [] + for side in files: + manifest = json.loads(side[".rescodes.json"].read_text()) + entries, meta = _code_store_index(side[".rescodes.idx"]) + build = json.loads(meta.pop("build_json")) + if build != manifest["build"]: + raise AssertionError(f"{side['.rescodes.idx']}: build_json {build} is not its manifest's build block " + f"{manifest['build']}") + manifest["build"] = {key: value for key, value in build.items() if key not in exempt} + stores.append((entries, meta, manifest)) + for k, what in enumerate(("entries rows", "meta rows", "manifest")): + if stores[0][k] != stores[1][k]: + raise AssertionError(f"the code stores of {a} and {b} differ in their {what}: {stores[0][k]} != " + f"{stores[1][k]}") + + +def _resync(record_path): + """Write the record's shard sizes and rows as the shards now have them (a shard rewritten on purpose).""" + record = json.loads(record_path.read_text()) + for shard in record["shards"]: + path = record_path.parent / shard["name"] + shard.update(bytes=path.stat().st_size, rows=pq.read_metadata(path).num_rows) + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") + + +def _edit_record(record_path, change): + """Rewrite the record as ``change(record)`` (a dict changed in place).""" + record = json.loads(record_path.read_text()) + change(record) + record_path.write_text(json.dumps(record, indent=2, sort_keys=True) + "\n") + + +def _one_residue_changed(path): + """Rewrite the FASTA at ``path`` with the first residue of its first sequence line replaced by another standard + residue: the same size, ids and pre-flight counts, another sha256. Its mtime is moved on by 1 s (as well as by the + write), so that a clock coarser than the test cannot hide the change from a stat bracket.""" + path = Path(path) + stat = path.stat() + data = bytearray(path.read_bytes()) + position = data.index(b"\n", data.index(b">")) + 1 + data[position] = ord("W") if data[position] != ord("W") else ord("G") + before = hashlib.sha256(path.read_bytes()).hexdigest() + path.write_bytes(bytes(data)) + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10 ** 9)) + assert path.stat().st_size == stat.st_size + assert hashlib.sha256(path.read_bytes()).hexdigest() != before + return path + + +class _SplitBuild(_WithBundle): + """Step 1 with the fake pass, step 2 under the ESM3 spy with ``scoring_system.prepare`` refusing (no bundle), and + the one-shot build to compare them with. The database goes into ``out``; the SLURM job variables are unset.""" + + def setUp(self): + super().setUp() + self.enterContext(mock.patch.dict(os.environ)) + for name in _JOB_ENV: + os.environ.pop(name, None) + # On Linux the real widening would change this test process's CPU mask for the rest of the session; tests of + # the widening patch it themselves. + self.enterContext(mock.patch.object(alignment, "_expand_cpu_affinity", return_value=os.cpu_count() or 1)) + self.out = self.tmp / "out" + self.out.mkdir() + self.db = self.out / "db" + + def embed(self, *extra, ranks=1, run_id="run", fasta=_FASTA, db=None, shards=None, gather=None, **fake): + """Step 1 (as ``EmbedStepTest.embed``), which must succeed; returns its record's path.""" + shards = shards or self.shards + with launch_env(FOLDMATCH_RUN_ID=run_id), ExitStack() as stack: + stack.enter_context(mock.patch("foldmatch.search.embedding_computer.chain_predict", + fake_chain_predict(ranks=ranks, **fake))) + if gather is not None or ranks > 1: + stack.enter_context(mock.patch.object(search, "_gather_rank_facts", + side_effect=gather or _ranks(ranks))) + result = invoke(*_embed_args(fasta, db or self.db, shards, *extra)) + self.assertEqual(result.exit_code, 0, result.output) + return Path(shards) / f"emb_{run_id}.run.json" + + def index(self, *extra, fasta=_FASTA, db=None, shards=None, env=None): + """Step 2 under the ESM3 spy, the bundle refused, with ``env`` set and ``sys.argv`` as the console script + sets it; returns the result.""" + args = _index_args(fasta, db or self.db, shards or self.shards, *extra) + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched")), \ + mock.patch.dict(os.environ, {name: str(value) for name, value in (env or {}).items()}), \ + _as_fm_search(args): + result = invoke(*args) + esm3.assert_not_called() + return result + + def built(self, *extra, **kwargs): + """Step 2, which must succeed and leave no build lock.""" + result = self.index(*extra, **kwargs) + self.assertEqual(result.exit_code, 0, f"{result.output} {result.exception!r}") + db = kwargs.get("db") or self.db + self.assertFalse(Path(f"{db}.build.lock").exists()) + return result + + def oneshot(self, *extra, run_id="oneshot", fasta=_FASTA, ranks=1, **fake): + """``build sequences`` on the same input, into its own folders; returns its database path.""" + db = self.tmp / f"{run_id}-db" / "db" + shards = self.tmp / f"{run_id}-shards" + shards.mkdir() + with launch_env(FOLDMATCH_RUN_ID=run_id), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", + fake_chain_predict(ranks=ranks, **fake)): + result = invoke(*_build_args(fasta, db, shards, *extra)) + self.assertEqual(result.exit_code, 0, result.output) + return db + + def logged(self, call): + """``call()``'s root-logger records as ``(level, message)`` (``set_log_level`` would replace assertLogs' + handler).""" + with mock.patch.object(search, "set_log_level"), self.assertLogs(level="INFO") as logs: + call() + return [(record.levelname, record.getMessage()) for record in logs.records] + + def setup_and_warnings(self, call): + """``call()``'s ``set_log_level`` calls, as ``("set_log_level", level)``, and root-logger WARNINGs, as + ``("WARNING", message)``, in the order they came.""" + events = [] + + class Recorder(logging.Handler): + def emit(self, record): + events.append((record.levelname, record.getMessage())) + + root, recorder = logging.getLogger(), Recorder(level=logging.WARNING) + root.addHandler(recorder) + try: + with mock.patch.object(search, "set_log_level", + side_effect=lambda level, **_: events.append(("set_log_level", level))): + call() + finally: + root.removeHandler(recorder) + return events + + +class IndexOptionTest(_Tmp): + + def test_the_command_help(self): + self.assertEqual(_command("build", "sequences-index").help, INDEX_HELP) + output = CliRunner().invoke(search.app, ["build", "sequences-index", "--help"], env={"COLUMNS": "400"}).output + for text in (INDEX_HELP, *INDEX_OWN_HELP.values()): + with self.subTest(text=text[:40]): + self.assertIn(text, _flat(output)) + + def test_the_options(self): + """Exactly step 2's options; the index options and --log-level are the one-shot's (type, default, help), the + others have the plan's help; --residue-codes is a tri-state (default None, D24).""" + index = {param.name: param for param in _command("build", "sequences-index").params} + oneshot = {param.name: param for param in _command("build", "sequences").params} + self.assertEqual(sorted(index), sorted([ + "fasta_file", "output_db", "from_shards", "run_id", "residue_codes", "cleanup_embeddings", + "use_gpu_index", "index_type", "ivf_nlist", "ivf_nprobe", "load_batch_size", "log_level"])) + + def info(param, *drop): + out = param.to_info_dict() + for key in ("name", "opts", "help", *drop): + out.pop(key) + return out + + for name, param in index.items(): + with self.subTest(name=name): + self.assertEqual(param.help, INDEX_OWN_HELP.get(name, oneshot.get(name) and oneshot[name].help)) + if name in oneshot and name != "residue_codes": + self.assertEqual(info(param), info(oneshot[name])) + from_shards = index["from_shards"] + self.assertEqual((from_shards.opts, from_shards.required, from_shards.type.exists, from_shards.type.dir_okay, + from_shards.type.file_okay), (["--from-shards"], True, True, True, False)) + self.assertEqual((index["run_id"].default, index["run_id"].required, index["run_id"].type.name), + (None, False, "text")) + codes = index["residue_codes"] + self.assertEqual((codes.opts, codes.secondary_opts, codes.default, codes.type.name), + (["--residue-codes"], ["--no-residue-codes"], None, "boolean")) + self.assertEqual(index["cleanup_embeddings"].default, False) + + def test_options_another_command_owns(self): + """The pass's options (there is no pass; --min-res-n comes from the record), --shards-folder and + --tmp-embedding-folder (use --from-shards): exit 2, nothing read or written. A malformed --run-id is exit 2 too + (I1).""" + for option in (("--min-res-n", "0"), ("--accelerator", "cpu"), ("--num-nodes", "1"), ("--devices", "1"), + ("--strategy", "auto"), ("--batch-size", "1"), ("--num-workers", "0"), + ("--shards-folder", self.shards), ("--tmp-embedding-folder", self.shards), ("--force",)): + with self.subTest(option=option[0]), _esm3_spy() as esm3: + result = invoke(*_index_args(_FASTA, self.tmp / "db", self.shards, *option)) + self.usage_error(result, option[0]) + self.assertRegex(_flat(result.output), no_such_option(option[0])) + esm3.assert_not_called() + for bad in ("a b", "a/b", "a*"): + with self.subTest(run_id=bad), mock.patch.object(split_run, "find_run") as find_run: + self.usage_error(invoke(*_index_args(_FASTA, self.tmp / "db", self.shards, "--run-id", bad)), + "is not a run id") + find_run.assert_not_called() + self.usage_error(invoke("build", "sequences-index", "--fasta-file", _FASTA, "--output-db", self.tmp / "db"), + "--from-shards") + self.assertEqual(list(self.shards.iterdir()), []) + self.assertEqual(sorted(p.name for p in self.tmp.glob("db*")), []) + + def test_the_from_shards_help_names_step_1(self): + """'build residue-codes' names step 1's shards in its --from-shards help and in its command help.""" + param = next(p for p in _command("build", "residue-codes").params if p.name == "from_shards") + self.assertIn("or those of a 'build sequences-embed --residue-codes' run (its --shards-folder).", param.help) + self.assertIn("from the embedding shards its 'build sequences --residue-codes' run kept, or those of the " + "'build sequences-embed --residue-codes' run it was built from, without running ESM3 again", + _command("build", "residue-codes").help) + + def test_the_load_batch_size_help(self): + """--load-batch-size has one help in every build command: FAISS adds IVF-PQ vectors in chunks of 32768, so a + larger batch changes only the memory (the older text claimed that larger batches use more cores).""" + expected = ("Number of vectors per batch streamed into the FAISS index; larger batches use more memory (FAISS " + "adds IVF-PQ vectors in chunks of 32768 whatever this value).") + for name in ("structures", "embeddings", "sequences", "sequences-index"): + with self.subTest(command=name): + param = next(p for p in _command("build", name).params if p.name == "load_batch_size") + self.assertEqual(param.help, expected) + + +class IndexStepTest(_SplitBuild): + + def test_two_steps_equal_the_one_shot(self): + """Plain and codes, on 1 and 2 ranks, with --min-res-n 0 and 100: step 1 plus step 2 with no extra option build + the one-shot's database (assert_same_database); the code store's manifest says the build was split.""" + for codes in (False, True): + for ranks in (1, 2): + for min_res_n in (0, 100): + name = f"{'codes' if codes else 'plain'}{ranks}-{min_res_n}" + with self.subTest(codes=codes, ranks=ranks, min_res_n=min_res_n): + extra = ("--min-res-n", min_res_n, *(("--residue-codes",) if codes else ())) + shards = self.tmp / f"shards-{name}" + shards.mkdir() + db = self.tmp / f"split-{name}" / "db" + self.embed(*extra, ranks=ranks, run_id=f"split-{name}", shards=shards, db=db) + self.built(shards=shards, db=db) + oneshot = self.oneshot(*extra, run_id=f"one-{name}", ranks=ranks) + assert_same_database(oneshot, db) + self.assertEqual(sorted(_database_files(db)), + sorted([".index", ".metadata", ".sequences", + *((".rescodes.bin", ".rescodes.idx", ".rescodes.json") if codes + else ())])) + if codes: + build = ResidueCodeStore(db.parent, "db").check_compatible()["build"] + self.assertEqual((build["split"], build["from_shards"], build["run_id"], + build["devices"]), (True, False, f"split-{name}", ranks)) + self.assertNotIn("split", + ResidueCodeStore(oneshot.parent, "db").check_compatible()["build"]) + self.assertEqual(len(list(shards.glob("*.parquet"))), len(split_run.dealt_rows( + 2 if min_res_n == 0 else 1, ranks))) # the shards are kept + + def test_hnsw_at_one_thread(self): + """--index-type hnsw: byte-identical too, at one FAISS thread (set explicitly, with OMP_NUM_THREADS=1 so that + step 2 leaves it; setting the variable alone after FAISS started would not change its threads). HNSW over the + fixture's two vectors is the same at any thread count, so this checks the equality, not thread determinism.""" + import faiss + self.embed() + previous = faiss.omp_get_max_threads() + self.addCleanup(faiss.omp_set_num_threads, previous) + faiss.omp_set_num_threads(1) + with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "1"}): + self.built("--index-type", "hnsw") + assert_same_database(self.oneshot("--index-type", "hnsw"), self.db) + self.assertEqual(faiss.omp_get_max_threads(), 1) + + def test_assert_same_database_sees_a_difference(self): + """The helper's own controls: another index, a missing file, another manifest field, another build key (in the + manifest and the index's build_json), a build_json that is not its manifest's, another entry row.""" + self.embed("--residue-codes") + self.built() + oneshot = self.oneshot("--residue-codes") + assert_same_database(oneshot, self.db) + copy = self.tmp / "copy" + cases = { + "another index": (lambda db: Path(f"{db}.index").write_bytes(Path(f"{db}.index").read_bytes() + b"\0"), + "db.index differ"), + "a missing file": (lambda db: Path(f"{db}.sequences").unlink(), "have other files"), + "another manifest field": (lambda db: _edit_record( + Path(f"{db}.rescodes.json"), lambda m: m.update(nonstandard_entries=99)), "differ in their manifest"), + "another build key": (lambda db: ( + _edit_record(Path(f"{db}.rescodes.json"), lambda m: m["build"].update(devices=7)), + _execute(Path(f"{db}.rescodes.idx"), + "UPDATE meta SET value = json_set(value, '$.devices', 7) WHERE key = 'build_json'")), + "differ in their manifest"), + "a build_json that is not its manifest's": (lambda db: _edit_record( + Path(f"{db}.rescodes.json"), lambda m: m["build"].update(devices=7)), + "is not its manifest's build block"), + "another entry": (lambda db: _execute(Path(f"{db}.rescodes.idx"), + "UPDATE entries SET length = length + 1"), "differ in their entries"), + } + for case, (change, text) in cases.items(): + with self.subTest(case=case): + shutil.rmtree(copy, ignore_errors=True) + shutil.copytree(self.out, copy) + change(copy / "db") + with self.assertRaises(AssertionError) as ctx: + assert_same_database(oneshot, copy / "db") + self.assertIn(text, str(ctx.exception)) + assert_same_database(oneshot, self.db, exempt=_EXEMPT) + with self.assertRaises(AssertionError) as ctx: + assert_same_database(oneshot, self.db, exempt=("created_utc", "run_id")) + self.assertIn("'split': True", str(ctx.exception)) + + def test_the_tri_state(self): + """The default follows the run: a code store for a codes run, none for a plain run; --residue-codes on a codes + run builds what the default builds.""" + self.embed("--residue-codes", run_id="codes") + self.built("--run-id", "codes") + self.assertTrue(ResidueCodeStore(self.out, "db").exists()) + self.built("--residue-codes", db=self.tmp / "explicit" / "db") + assert_same_database(self.db, self.tmp / "explicit" / "db") + plain = self.tmp / "plain" + plain.mkdir() + self.embed(run_id="plain", shards=plain, db=self.tmp / "p" / "db") + self.built(shards=plain, db=self.tmp / "p" / "db") + self.assertEqual(sorted(_database_files(self.tmp / "p" / "db")), [".index", ".metadata", ".sequences"]) + + def test_no_residue_codes_on_a_codes_run(self): + """--no-residue-codes on a codes run builds the plain one-shot's database; a later 'build residue-codes + --from-shards --run-id' adds the one-shot's code store (bin_sha256, entries_digest).""" + self.embed("--residue-codes") + self.built("--no-residue-codes") + assert_same_database(self.oneshot(run_id="plain"), self.db) + result = invoke("build", "residue-codes", self.out, "db", "--from-shards", self.shards, "--run-id", "run") + self.assertEqual(result.exit_code, 0, result.output) + direct = ResidueCodeStore(self.oneshot("--residue-codes", run_id="codes").parent, "db").manifest() + added = ResidueCodeStore(self.out, "db").check_compatible() + self.assertEqual((added["bin_sha256"], added["entries_digest"]), + (direct["bin_sha256"], direct["entries_digest"])) + self.assertEqual(sorted(p.name for p in self.shards.iterdir()), ["emb_run-0.parquet", "emb_run.run.json"]) + + def test_no_residue_codes_with_a_system_unknown_here(self): + """A codes run whose system this FoldMatch does not register: the default is refused (I10), and + --no-residue-codes builds the plain database from the shards' own consistency.""" + self.embed("--residue-codes") + with mock.patch.object(ss, "REGISTRY", MappingProxyType({})): + self.assertIn("unknown esm-codes scoring system", self.refused(self.index())) + self.built("--no-residue-codes") + assert_same_database(self.oneshot(run_id="plain"), self.db) + + def test_two_builds_from_one_run(self): + """Two step-2 builds from one run into two folders: the same database, and the shards and record unchanged.""" + self.embed("--residue-codes", ranks=2) + before = _tree(self.shards) + self.built() + self.built(db=self.tmp / "second" / "db") # a folder that does not exist yet + self.assertEqual(_tree(self.shards), before) + assert_same_database(self.db, self.tmp / "second" / "db") + + def test_a_copied_folder_and_a_moved_fasta(self): + """A copy of the shards folder builds the database (the record names its shards by file name), and so does a + copy of the FASTA elsewhere with the same bytes (logged at INFO).""" + self.embed("--residue-codes") + copied = self.tmp / "copied" + shutil.copytree(self.shards, copied) + shutil.rmtree(self.shards) + moved = self.tmp / "elsewhere" / "seqs.fasta" + moved.parent.mkdir() + shutil.copyfile(_FASTA, moved) + messages = self.logged(lambda: self.built(shards=copied, fasta=moved)) + self.assertIn(("INFO", f"{moved.resolve()} has the size and sha256 of the FASTA run run embedded, with another " + f"path and modification time (recorded: {_FASTA.resolve()}, mtime_ns " + f"{_FASTA.stat().st_mtime_ns})"), messages) + assert_same_database(self.oneshot("--residue-codes"), self.db) + + def test_cleanup_embeddings(self): + """--cleanup-embeddings deletes the record, then the shards, after the code store; no lock is left.""" + for codes in (True, False): + with self.subTest(codes=codes): + for path in [*self.shards.iterdir(), *self.out.iterdir()]: + path.unlink() + record = self.embed(*(("--residue-codes",) if codes else ())) + seen = [] + remove_shards = EmbeddingComputer.remove_shards + + def removing(computer): + seen.append((record.exists(), ResidueCodeStore(self.out, "db").exists(), + sorted(p.name for p in self.shards.iterdir()))) + remove_shards(computer) + + with mock.patch.object(EmbeddingComputer, "remove_shards", autospec=True, side_effect=removing): + self.built("--cleanup-embeddings") + self.assertEqual(seen, [(False, codes, ["emb_run-0.parquet"])]) + self.assertEqual(list(self.shards.iterdir()), []) + self.assertEqual(sorted(p.name for p in self.out.iterdir()), + sorted(f"db{s}" for s in (".index", ".metadata", ".sequences", + *((".rescodes.bin", ".rescodes.idx", ".rescodes.json") + if codes else ())))) + + def test_no_residue_codes_and_cleanup_on_a_plain_run(self): + """--no-residue-codes --cleanup-embeddings is refused only on a run with codes (I9): a plain run builds and + deletes its record and shards.""" + self.embed() + self.built("--no-residue-codes", "--cleanup-embeddings") + self.assertEqual(list(self.shards.iterdir()), []) + self.assertEqual(sorted(p.name for p in self.out.iterdir()), ["db.index", "db.metadata", "db.sequences"]) + + def test_the_index_options_reach_the_index_phase(self): + """Step 2's index options go to the index phase as given (the phase mocked: --use-gpu-index needs faiss-gpu), + with the pre-flight bound to the computer, so that get_embedding_batches counts the shard rows again.""" + self.embed() + seen = [] + + def phase(computer, preflight, **kwargs): + seen.append((kwargs["index"], computer.expected_count, computer.preflight is preflight, + preflight.entries_kept)) + + with mock.patch.object(search, "_build_sequence_database", side_effect=phase): + self.built("--index-type", "ivf_pq", "--ivf-nlist", "256", "--ivf-nprobe", "4", "--load-batch-size", "1000", + "--use-gpu-index") + self.assertEqual(seen, [(search._IndexOptions(True, IndexType.ivf_pq, 256, 4, 1000), 2, True, 2)]) + + def test_the_pre_flight_is_the_records(self): + """Step 2's pre-flight runs with the record's min_res_n and strictness: strict for a codes run.""" + for codes in (True, False): + with self.subTest(codes=codes): + for path in [*self.shards.iterdir(), *self.out.iterdir()]: + path.unlink() + self.embed("--min-res-n", "100", *(("--residue-codes",) if codes else ())) + with mock.patch.object(search, "_fasta_preflight", wraps=search._fasta_preflight) as preflight: + self.built() + self.assertEqual([(c.args[1], c.kwargs["strict"]) for c in preflight.call_args_list], [(100, codes)]) + + def test_the_manifest_of_a_split_build(self): + """build.split is set and from_shards false; the ESM3 revision is the shards' pass record's, not this + installation's pin; the other build keys are the one-shot's (they describe the pass).""" + self.embed("--residue-codes") + with mock.patch("foldmatch.search.residue_code_store._installed_esm_revision", + return_value="another-installation"): + self.built() + split = ResidueCodeStore(self.out, "db").check_compatible() + self.assertEqual(split["esm"]["revision"], ESM_REVISION) + self.assertEqual(read_shard_info(self.shards / "emb_run-0.parquet").pass_record["esm_revision"], ESM_REVISION) + direct = ResidueCodeStore(self.oneshot("--residue-codes").parent, "db").manifest() + self.assertEqual({k: v for k, v in split["build"].items() if k not in _EXEMPT}, + {k: v for k, v in direct["build"].items() if k not in _EXEMPT}) + self.assertEqual((split["build"]["split"], split["build"]["from_shards"], split["min_res_n"]), (True, False, 0)) + + def test_the_log_lines(self): + """The FASTA hash and id-check times, the run it builds from, the threads, the end line; a WARNING for another + FoldMatch's run and for a sealed record, an INFO line for another --output-db.""" + record = self.embed("--residue-codes", ranks=2) + messages = self.logged(lambda: self.built()) + db, shards = self.db, self.shards.resolve() + size = _FASTA.stat().st_size + found = [m for _, m in messages] + prefixes = (f"FASTA sha256 of {_FASTA.resolve()} ({size} bytes) in ", "Ids of the 2 shard rows checked " + "against ", f"Building {db} from run run in {shards}: ", "Index phase: ", + f"Database {db} built from run run in ") + lines = [[m for m in found if m.startswith(prefix)] for prefix in prefixes] + self.assertEqual([len(line) for line in lines], [1] * 5, found) + created = split_run.read_run_record(record)["created_utc"] + self.assertEqual(lines[2][0], f"Building {db} from run run in {shards}: 2 entries in 2 shard(s), embedded " + f"{created} by FoldMatch {foldmatch.__version__} on 2 rank(s) (cpu), residue " + f"codes of {SYSTEM.name}") + self.assertRegex(lines[4][0], r" in \d+\.\d s$") + order = [found.index(line[0]) for line in lines] + self.assertEqual(order, sorted(order)) + self.assertFalse([m for level, m in messages if level == "WARNING" and "FoldMatch" in m]) + self.assertFalse([m for m in found if "not the --output-db the run recorded" in m]) + + # Another FoldMatch's step 1: D30. + _edit_record(record, lambda r: r.update(foldmatch_version="0.0.1")) + messages = self.logged(lambda: self.built(db=self.tmp / "older" / "db")) + warnings = [m for level, m in messages if level == "WARNING"] + self.assertTrue([m for m in warnings if m.startswith(f"Run run was embedded by FoldMatch 0.0.1, and this is " + f"FoldMatch {foldmatch.__version__}")], warnings) + self.assertFalse([m for m in warnings if "sequences-seal" in m], warnings) + start = next(m for _, m in messages if m.startswith("Building ") and " from run run " in m) + self.assertIn(f"embedded {created} by FoldMatch 0.0.1 on 2 rank(s) (cpu), residue codes", start) + + # A sealed record holds the seal's FoldMatch; a codes run's pass record holds the pass's, which D30 compares. + def sealed_by(seal_version, pass_version): + def change(record_): + record_.update(foldmatch_version=seal_version, command=split_run.SEAL_COMMAND, sealed=True, + output_db=None, options={}) + record_["pass"]["foldmatch_version"] = pass_version + return change + + for k, (seal_version, pass_version) in enumerate((("0.0.2", "0.0.1"), ("0.0.2", foldmatch.__version__))): + with self.subTest(seal=seal_version, run_pass=pass_version): + _edit_record(record, sealed_by(seal_version, pass_version)) + other = self.tmp / f"sealed{k}" / "db" + messages = self.logged(lambda: self.built("--no-residue-codes", db=other)) + warnings = [m for level, m in messages if level == "WARNING"] + d30 = [m for m in warnings if m.startswith("Run run was embedded by FoldMatch ")] + self.assertEqual(d30, [] if pass_version == foldmatch.__version__ else [ + f"Run run was embedded by FoldMatch 0.0.1, and this is FoldMatch {foldmatch.__version__}: its " + f"record, shard schemas and digests, pre-flight counts and ids were checked, and the database is " + f"built by this FoldMatch."]) + self.assertIn(f"The record of run run was written by 'build sequences-seal' (FoldMatch 0.0.2): it " + f"binds the shards to the FASTA by their entry count, rows and ids (and, with residue " + f"codes, their seq_sha1), not by the pass that wrote them.", warnings) + start = next(m for _, m in messages if m.startswith(f"Building {other} from run")) + self.assertEqual(start, f"Building {other} from run run in {shards}: 2 entries in 2 shard(s), sealed " + f"{created} by FoldMatch 0.0.2; embedded by FoldMatch {pass_version} on 2 " + f"rank(s) (cpu), its residue codes of {SYSTEM.name} left out " + f"(--no-residue-codes); sealed record") + self.assertFalse([m for _, m in messages if "not the --output-db" in m]) # a sealed record names none + _edit_record(record, lambda r: r.update(output_db="/elsewhere/db")) + messages = self.logged(lambda: self.built("--no-residue-codes", db=self.tmp / "third" / "db")) + self.assertIn(("INFO", f"Building {self.tmp / 'third' / 'db'}, not the --output-db the run recorded " + f"(/elsewhere/db)"), messages) + plain = self.tmp / "plain" + plain.mkdir() + plain_record = self.embed(run_id="plain", shards=plain, db=self.tmp / "p" / "db") + messages = self.logged(lambda: self.built(shards=plain, db=self.tmp / "p" / "db")) + start = next(m for _, m in messages if m.startswith("Building ") and " from run plain " in m) + self.assertTrue(start.endswith("on 1 rank(s) (device not recorded), no residue codes"), start) + # A sealed plain run: its pass record is the seal's, so the pass's FoldMatch is unknown (no D30). + _edit_record(plain_record, lambda r: r.update(foldmatch_version="0.0.2", command=split_run.SEAL_COMMAND, + sealed=True, output_db=None, options={})) + plain_created = split_run.read_run_record(plain_record)["created_utc"] + messages = self.logged(lambda: self.built(shards=plain, db=self.tmp / "p2" / "db")) + warnings = [m for level, m in messages if level == "WARNING"] + self.assertFalse([m for m in warnings if "was embedded by FoldMatch" in m], warnings) + self.assertIn("The record of run plain was written by 'build sequences-seal' (FoldMatch 0.0.2): it binds the " + "shards to the FASTA by their entry count, rows and ids (and, with residue codes, their " + "seq_sha1), not by the pass that wrote them, whose FoldMatch version is not recorded (shards " + "without residue codes do not record it).", warnings) + start = next(m for _, m in messages if m.startswith("Building ") and " from run plain " in m) + self.assertTrue(start.endswith(f"sealed {plain_created} by FoldMatch 0.0.2; embedded by a FoldMatch it does " + f"not record on 1 rank(s) (device not recorded), no residue codes; sealed " + f"record"), start) + + def test_the_logged_next_command_runs_verbatim(self): + """Step 1's next command, split as a shell splits it (paths with spaces, a relative --output-db recorded + absolute), builds the database step 1 recorded.""" + folder = self.tmp / "with space" + folder.mkdir() + fasta = Path(shutil.copyfile(_FASTA, folder / "my seqs.fasta")) + shards = folder / "shard s" + shards.mkdir() + with chdir(folder): + messages = self.logged(lambda: self.embed("--residue-codes", fasta=fasta, shards=shards, + db=Path("rel dir") / "my db")) + line = next(m for _, m in messages if m.startswith("Build the database with: ")) + argv = shlex.split(line[len("Build the database with: "):]) + self.assertEqual(argv[:3], ["fm-search", "build", "sequences-index"]) + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched")): + result = invoke(*argv[1:]) + esm3.assert_not_called() + self.assertEqual(result.exit_code, 0, result.output) + self.assertTrue(ResidueCodeStore(folder / "rel dir", "my db").check_compatible()) + + def test_launch_warnings(self): + """A one-task srun step of a multi-node job, and a one-node job with GPUs: built, with a WARNING (D34).""" + self.embed() + cases = { + "an srun step": ({"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 2, "SLURM_STEP_ID": 3, + "SLURM_STEP_NUM_TASKS": 1, "SLURM_NTASKS": 8}, + "'build sequences-index' runs in a one-task srun step of SLURM job 42, which holds 2 " + "nodes: the others stay allocated until it ends. Next time, run it as its own one-node " + "CPU job: submit it from the GPU job's script before the job's work starts: sbatch -C cpu " + "-N 1 --dependency=afterok:$SLURM_JOB_ID --kill-on-invalid-dep=yes ... SCRIPT. SCRIPT " + "runs this command, without srun, " + _NEXT_BUILD + ":\n {command}"), + "GPUs": ({"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 1, "SLURM_GPUS_ON_NODE": 4}, + "SLURM job 42 has GPUs (SLURM_GPUS_ON_NODE=4), which 'build sequences-index' does not use: run it " + "as its own job on a CPU node (sbatch -C cpu -N 1 ...); a project with GPU hours only may run it " + "on one GPU node, where ivf_pq fits (flat and hnsw may not)."), + } + for k, (case, (env, text)) in enumerate(cases.items()): + with self.subTest(case=case): + db = self.tmp / f"w{k}" / "db" + messages = self.logged(lambda: self.built(env=env, db=db)) + warnings = [m for level, m in messages if level == "WARNING"] + self.assertIn(text.format(command=_command_line(_index_args(_FASTA, db, self.shards))), warnings) + self.assertFalse([m for m in warnings if "afterok:42" in m], warnings) # the build runs in job 42 + messages = self.logged(lambda: self.built(env={"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 1}, + db=self.tmp / "w" / "db")) + self.assertFalse([m for level, m in messages if level == "WARNING" and "SLURM job" in m]) + + def test_launch_warnings_after_the_logging_setup(self): + """P9.I1: the launch checks run before set_log_level, and their WARNING is logged once it has set up the + logging, so in its format.""" + self.embed() + for k, env in enumerate(({"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 2, "SLURM_STEP_ID": 3, + "SLURM_STEP_NUM_TASKS": 1}, + {"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 1, "SLURM_GPUS_ON_NODE": 4})): + with self.subTest(env=env): + events = self.setup_and_warnings(lambda: self.built(env=env, db=self.tmp / f"w{k}" / "db")) + launch = [event for event in events if "SLURM job 42" in event[1]] + self.assertEqual(len(launch), 1, events) + self.assertEqual(events[:2], [("set_log_level", LogLevel.info), launch[0]]) + + def test_offline_with_an_empty_hf_cache(self): + """Step 2 as a command, with HF_HUB_OFFLINE=1 and an empty Hugging Face cache: the code store is built from + the registry entry alone.""" + self.embed("--residue-codes") + empty = self.tmp / "empty-hf-cache" + empty.mkdir() + env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env.update(HF_HUB_OFFLINE="1", HF_HUB_CACHE=str(empty), HF_HOME=str(empty), + PYTHONPATH=os.pathsep.join([_SRC] + [p for p in [os.environ.get("PYTHONPATH")] if p])) + run = subprocess.run([sys.executable, "-m", "foldmatch.cli.search", *map(str, _index_args( + _FASTA, self.db, self.shards))], env=env, capture_output=True, text=True, timeout=600) + self.assertEqual(run.returncode, 0, run.stderr[-4000:]) + self.assertTrue(ResidueCodeStore(self.out, "db").check_compatible()) + self.assertEqual(list(empty.rglob("models--*")), []) # nothing fetched (transformers writes a version.txt) + + # -- D29: the threads of the index phase (FAISS's thread calls and threadpoolctl's limit mocked) ------------------ + #: threadpoolctl's runtimes as a Linux node with the faiss-cpu 1.15.1 wheel might list them: numpy's and scipy's + #: OpenBLAS (one of them twice), FAISS's OpenMP OpenBLAS in faiss_cpu.libs, and an OpenMP. + POOLS = [{"user_api": "blas", "internal_api": "openblas", "num_threads": 6, + "filepath": "/site-packages/numpy.libs/libscipy_openblas64_-ff651d7f.so"}, + {"user_api": "blas", "internal_api": "openblas", "num_threads": 6, + "filepath": "/site-packages/scipy.libs/libscipy_openblas-68440149.so"}, + {"user_api": "blas", "internal_api": "openblas", "num_threads": 6, + "filepath": "/elsewhere/libscipy_openblas-68440149.so"}, + {"user_api": "blas", "internal_api": "openblas", "num_threads": 6, "threading_layer": "openmp", + "filepath": os.path.join(_faiss_libs(), "libopenblaso-r0-d77a1985.3.15.so")}, + {"user_api": "openmp", "internal_api": "openmp", "num_threads": 6, + "filepath": os.path.join(_faiss_libs(), "libgomp-a34b3233.so.1.0.0")}] + #: The same without FAISS's BLAS (the faiss-cpu 1.13.2 wheel's is listed at 1 thread, an MKL build's not at all). + POOLS_NO_FAISS_BLAS = [pool for pool in POOLS if "libopenblaso" not in pool["filepath"]] + + def threads(self, platform="linux", pools=POOLS, info_error=None, **env): + """``_whole_node_threads`` with ``env`` (the thread variables otherwise unset) on ``platform``, threadpoolctl + listing ``pools`` (or raising ``info_error``): ``(FAISS thread calls inside the block, after it, + threadpool_limits calls, (limit left inside, after), log)``.""" + clean = {k: v for k, v in os.environ.items() if k not in search._BLAS_THREAD_VARS} + clean.update(env) + with mock.patch.dict(os.environ, clean, clear=True), mock.patch.object(sys, "platform", platform), \ + mock.patch.object(code_alignment, "_physical_cpus", return_value=6), \ + mock.patch("faiss.omp_get_max_threads", return_value=3), \ + mock.patch("faiss.omp_set_num_threads") as set_threads, \ + mock.patch("threadpoolctl.threadpool_info", return_value=pools, side_effect=info_error), \ + mock.patch("threadpoolctl.threadpool_limits") as limits, self.assertLogs(level="INFO") as logs: + with search._whole_node_threads(): + inside = list(set_threads.call_args_list) + exited = limits.return_value.__exit__.called + after = list(set_threads.call_args_list) + return inside, after, list(limits.call_args_list), (exited, limits.return_value.__exit__.called), \ + [record.getMessage() for record in logs.records] + + def test_d29_threads_with_the_variables_unset(self): + """FAISS gets the physical cores (the previous count restored after the block), and so does BLAS; the INFO line + names the BLAS libraries threadpoolctl found (the limit reaches no other; the faiss-cpu 1.15.1 wheel's OpenMP + OpenBLAS is one of them), or says it found none.""" + inside, after, limits, exited, log = self.threads() + self.assertEqual(inside, [mock.call(6)]) + self.assertEqual(after, [mock.call(6), mock.call(3)]) + self.assertEqual(limits, [mock.call(limits=6, user_api="blas")]) + self.assertEqual(exited, (False, True)) + self.assertEqual(len(log), 1) + self.assertRegex(log[0], r"^FAISS/BLAS threads set to 6 \(physical cores of the \d+ allowed CPUs\); BLAS " + r"libraries limited: libopenblaso-r0-d77a1985\.3\.15\.so, " + r"libscipy_openblas-68440149\.so, libscipy_openblas64_-ff651d7f\.so$") + *_, log = self.threads(pools=self.POOLS_NO_FAISS_BLAS) + self.assertRegex(log[0], r"^FAISS/BLAS threads set to 6 \(physical cores of the \d+ allowed CPUs\); BLAS " + r"libraries limited: libscipy_openblas-68440149\.so, " + r"libscipy_openblas64_-ff651d7f\.so$") + *_, log = self.threads(pools=[pool for pool in self.POOLS if pool["user_api"] != "blas"]) + self.assertRegex(log[0], r"^FAISS/BLAS threads set to 6 \(physical cores of the \d+ allowed CPUs\); " + r"threadpoolctl found no BLAS library to limit$") + + def test_d29_omp_num_threads_set(self): + inside, after, limits, _, log = self.threads(OMP_NUM_THREADS="2") + self.assertEqual((inside, after, limits), ([], [], [])) + self.assertEqual(log, ["FAISS threads left to OMP_NUM_THREADS=2; BLAS threads left to OMP_NUM_THREADS=2"]) + + def test_d29_a_blas_variable_set(self): + """BLAS is left to the variable, and the INFO line says that FAISS's OpenMP OpenBLAS (the faiss-cpu 1.15.1 + wheel's), which the variable does not govern, runs FAISS's threads; with OMP_NUM_THREADS also set, FAISS is + left to it too and nothing is added.""" + for name in ("OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"): + for pools, follows in ((self.POOLS, ", except FAISS's OpenMP OpenBLAS, libopenblaso-r0-d77a1985.3.15.so, " + "which runs FAISS's OpenMP threads"), + (self.POOLS_NO_FAISS_BLAS, ""), ([], "")): + with self.subTest(name=name, pools=len(pools)): + inside, after, limits, _, log = self.threads(pools=pools, **{name: "1"}) + self.assertEqual((inside, limits), ([mock.call(6)], [])) + self.assertRegex(log[0], rf"^FAISS threads set to 6 \(physical cores of the \d+ allowed CPUs\); " + rf"BLAS threads left to {name}=1{re.escape(follows)}$") + inside, after, limits, _, log = self.threads(OMP_NUM_THREADS="2", OPENBLAS_NUM_THREADS="1") + self.assertEqual((inside, after, limits), ([], [], [])) + self.assertEqual(log, ["FAISS threads left to OMP_NUM_THREADS=2; BLAS threads left to OMP_NUM_THREADS=2, " + "OPENBLAS_NUM_THREADS=1"]) + *_, log = self.threads(info_error=RuntimeError("cannot inspect"), OPENBLAS_NUM_THREADS="1") + self.assertRegex(log[0], r"BLAS threads left to OPENBLAS_NUM_THREADS=1$") + + def test_d29_never_on_macos(self): + for env in ({}, {"OMP_NUM_THREADS": "1"}): + with self.subTest(env=env): + inside, after, limits, _, log = self.threads(platform="darwin", **env) + self.assertEqual((inside, after, limits), ([], [], [])) + self.assertEqual(log, ["FAISS/BLAS threads left as they are on macOS (foldmatch pins " + "OMP_NUM_THREADS)"]) + + def test_d29_restored_on_an_exception(self): + clean = {k: v for k, v in os.environ.items() if k not in search._BLAS_THREAD_VARS} + with mock.patch.dict(os.environ, clean, clear=True), mock.patch.object(sys, "platform", "linux"), \ + mock.patch.object(code_alignment, "_physical_cpus", return_value=6), \ + mock.patch("faiss.omp_get_max_threads", return_value=3), \ + mock.patch("faiss.omp_set_num_threads") as set_threads, \ + mock.patch("threadpoolctl.threadpool_info", return_value=self.POOLS), \ + mock.patch("threadpoolctl.threadpool_limits") as limits, self.assertRaises(RuntimeError): + with search._whole_node_threads(): + raise RuntimeError("the index phase failed") + self.assertEqual(set_threads.call_args_list, [mock.call(6), mock.call(3)]) + self.assertTrue(limits.return_value.__exit__.called) + + def test_d29_step_2_sets_the_threads(self): + """Through the command: the affinity is widened for 'the index phase' before the run is read; FAISS and BLAS + get the physical cores for the index phase, which runs inside the limit, and FAISS's count is restored.""" + self.embed() + clean = {k: v for k, v in os.environ.items() if k not in search._BLAS_THREAD_VARS} + manager = mock.Mock() + with mock.patch.dict(os.environ, clean, clear=True), mock.patch.object(sys, "platform", "linux"), \ + mock.patch.object(code_alignment, "_physical_cpus", return_value=6), \ + mock.patch("faiss.omp_set_num_threads") as set_threads, \ + mock.patch("threadpoolctl.threadpool_limits") as limits, \ + mock.patch("threadpoolctl.threadpool_info", return_value=[]), \ + mock.patch.object(alignment, "_expand_cpu_affinity", return_value=8) as widen, \ + mock.patch.object(split_run, "find_run", wraps=split_run.find_run) as find_run, \ + mock.patch.object(search, "_build_sequence_database", wraps=search._build_sequence_database) as phase: + for name, spy in (("widen", widen), ("find_run", find_run), ("set_threads", set_threads), + ("limits", limits), ("phase", phase)): + manager.attach_mock(spy, name) + self.built() + calls = [c[0] for c in manager.mock_calls if not c[0].startswith("limits().")] + self.assertEqual(calls, ["widen", "find_run", "set_threads", "limits", "phase", "set_threads"]) + self.assertEqual(widen.call_args, mock.call(purpose="the index phase")) + self.assertEqual(set_threads.call_args_list[0], mock.call(6)) + self.assertEqual(limits.call_args, mock.call(limits=6, user_api="blas")) + self.assertEqual(phase.call_args.kwargs["split"], split_run.read_run_record(self.shards / "emb_run.run.json")) + self.assertIs(phase.call_args.kwargs["refusals"], search._refusals) + + +#: A fresh process that loads torch first, as the CLI does, then runs an unmocked ``_whole_node_threads`` block and +#: prints FAISS's OpenMP count and FAISS's OpenBLAS's threads before, inside and after it (FaissWheelTest). +_D29_CHILD = """ +import json, logging, os, sys +import torch +import faiss, threadpoolctl +from foldmatch.cli import search +from foldmatch.search.code_alignment import _physical_cpus +logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stdout, force=True) +def faiss_blas(): + return [r["num_threads"] for r in threadpoolctl.threadpool_info() if r.get("user_api") == "blas" + and os.path.basename(r["filepath"]).startswith("libopenblaso")] +before = (faiss.omp_get_max_threads(), faiss_blas()) +with search._whole_node_threads(): + inside = (faiss.omp_get_max_threads(), faiss_blas()) +after = (faiss.omp_get_max_threads(), faiss_blas()) +print("D29 " + json.dumps({"n": _physical_cpus(), "before": before, "inside": inside, "after": after})) +""" + +# The esm-codes in-process BLAS limit with the wheel: FAISS's count set apart from its OpenBLAS's (no BLAS call since, +# so the OpenBLAS still reports its own count), as after faiss.omp_set_num_threads in the Python API. +_ESM_CODES_LIMIT_CHILD = """ +import json, os +import torch +import faiss, threadpoolctl +from foldmatch.search import code_alignment +def faiss_blas(): + return [r["num_threads"] for r in threadpoolctl.threadpool_info() if r.get("user_api") == "blas" + and os.path.basename(r["filepath"]).startswith("libopenblaso")] +reported = faiss_blas() +k = 2 if reported != [2] else 3 +faiss.omp_set_num_threads(k) +before = (faiss.omp_get_max_threads(), torch.get_num_threads(), faiss_blas()) +with code_alignment._blas_one_thread_here(): + inside = faiss.omp_get_max_threads() +after = (faiss.omp_get_max_threads(), torch.get_num_threads()) +print("ESM " + json.dumps({"k": k, "before": before, "inside": inside, "after": after})) +""" + + +class FaissWheelTest(unittest.TestCase): + """P9: the pinned faiss-cpu wheel, unmocked. Linux only (the macOS wheels use Accelerate, which threadpoolctl does + not list), and only when the installed FAISS is that wheel (not a conda build, not another version).""" + + def setUp(self): + import faiss + if not sys.platform.startswith("linux"): + self.skipTest("the faiss-cpu wheels bundle an OpenBLAS on Linux only") + if faiss.__version__ != search._FAISS_CPU_PIN or not os.path.isdir(_faiss_libs()): + self.skipTest(f"FAISS {faiss.__version__} at {faiss.__file__} is not the faiss-cpu " + f"{search._FAISS_CPU_PIN} wheel") + + def test_the_wheel_blas_is_threaded(self): + """threadpoolctl lists the wheel's BLAS in faiss_cpu.libs as an OpenMP OpenBLAS, and the index phase logs it + without the single-threaded line (a pin to a wheel with a single-threaded BLAS fails here).""" + import threadpoolctl + runtimes = threadpoolctl.threadpool_info() + own = [r for r in runtimes if r.get("user_api") == "blas" + and os.path.dirname(os.path.realpath(r["filepath"])) == os.path.realpath(_faiss_libs())] + self.assertEqual([(r["internal_api"], r.get("threading_layer")) for r in own], [("openblas", "openmp")], + runtimes) + self.assertEqual(search._faiss_openmp_blas(runtimes), os.path.basename(os.path.realpath(own[0]["filepath"]))) + with self.assertLogs(level="INFO") as logs: + search._log_index_phase(None) + messages = [record.getMessage() for record in logs.records] + self.assertFalse([m for m in messages if "single-threaded" in m], messages) + self.assertEqual(len([m for m in messages if m.startswith("Thread pool: openblas, ") + and m.endswith(os.path.basename(own[0]["filepath"]))]), 1, messages) + + def test_d29_with_the_wheel(self): + """D29 unmocked, in a fresh process with torch loaded first: the INFO line names FAISS's OpenMP OpenBLAS + among the libraries limited; inside the block FAISS and that OpenBLAS run the physical cores (at most the + wheel's 128); after it FAISS has its previous count again.""" + env = {k: v for k, v in os.environ.items() if k not in search._BLAS_THREAD_VARS + and k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env["PYTHONPATH"] = os.pathsep.join([_SRC] + [p for p in [os.environ.get("PYTHONPATH")] if p]) + run = subprocess.run([sys.executable, "-c", _D29_CHILD], env=env, capture_output=True, text=True, timeout=600) + self.assertEqual(run.returncode, 0, run.stderr[-4000:]) + lines = run.stdout.splitlines() + result = json.loads(next(line for line in lines if line.startswith("D29 "))[4:]) + n = result["n"] + want = min(n, 128) + limited = [line for line in lines if line.startswith("FAISS/BLAS threads set to ")] + self.assertEqual(len(limited), 1, run.stdout) + self.assertRegex(limited[0], rf"^FAISS/BLAS threads set to {n} \(physical cores of the \d+ allowed CPUs\); " + rf"BLAS libraries limited: (.+, )?libopenblaso-[^,]+\.so(, .+)?$") + self.assertEqual(result["inside"], [want, [want]], result) + self.assertEqual(result["after"][0], result["before"][0], result) + + def test_the_esm_codes_in_process_limit_with_the_wheel(self): + """The esm-codes Stage 2's in-process BLAS limit, unmocked, in a fresh process with torch loaded first: it + reaches FAISS's OpenMP count (1 inside), and afterwards FAISS and torch have their counts of before, although + FAISS's had been set apart from the count its OpenBLAS reports (threadpoolctl's restore alone gives them that + stale count).""" + env = {k: v for k, v in os.environ.items() if k not in search._BLAS_THREAD_VARS + and k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env["PYTHONPATH"] = os.pathsep.join([_SRC] + [p for p in [os.environ.get("PYTHONPATH")] if p]) + run = subprocess.run([sys.executable, "-c", _ESM_CODES_LIMIT_CHILD], env=env, capture_output=True, text=True, + timeout=600) + self.assertEqual(run.returncode, 0, run.stderr[-4000:]) + result = json.loads(next(line for line in run.stdout.splitlines() if line.startswith("ESM "))[4:]) + k, (faiss_before, torch_before, blas) = result["k"], result["before"] + self.assertEqual(faiss_before, k, result) + self.assertEqual(len(blas), 1, result) + self.assertNotEqual(blas[0], k, result) # the OpenBLAS reports another count: the case at issue + self.assertEqual(result["inside"], 1, result) + self.assertEqual(result["after"], [faiss_before, torch_before], result) + + +def _execute(path, sql): + con = sqlite3.connect(path) + try: + con.execute(sql) + con.commit() + finally: + con.close() + + +class IndexRefusalTest(_SplitBuild): + + def setUp(self): + super().setUp() + self.inputs = self.tmp / "in" + self.inputs.mkdir() + self.fasta = Path(shutil.copyfile(_FASTA, self.inputs / "seqs.fasta")) + + def refused_index(self, *extra, cause=None, env=None, shards=None): + """Step 2 refused (exit 1, a RefusalError), before ESM3, leaving the shards folder and the database folder as + they were (no lock, no database file, nothing changed; I22/I23 remove what they wrote). Returns the + message.""" + before = _tree(self.out, shards or self.shards) + result = self.index(*extra, fasta=self.fasta, shards=shards, env=env) + message = self.refused(result, cause) + self.assertEqual(_tree(self.out, shards or self.shards), before) + return message + + def embed_run(self, *extra, **kwargs): + return self.embed(*extra, fasta=self.fasta, **kwargs) + + # -- before the build lock ---------------------------------------------------------------------------------------- + def test_i1_a_malformed_run_id(self): + self.embed_run() + self.usage_error(self.index("--run-id", "a b", fasta=self.fasta), "is not a run id") + + def test_i2_one_of_several_ranks(self): + self.embed_run() + for env, launch in (({"WORLD_SIZE": 2}, "WORLD_SIZE=2"), ({"LOCAL_RANK": 1}, "LOCAL_RANK=1"), + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 0, "SLURM_STEP_NUM_TASKS": 2}, + "SLURM_STEP_NUM_TASKS=2")): + with self.subTest(env=env), mock.patch.object(split_run, "find_run") as find_run, \ + mock.patch.object(alignment, "_expand_cpu_affinity") as widen: + message = self.refused_index(env=env) + find_run.assert_not_called() + widen.assert_not_called() + self.assertEqual(message, f"'build sequences-index' runs as one process ({launch}): launch it once, " + f"directly in the batch script or with srun -n 1. The command:\n " + f"{_command_line(_index_args(self.fasta, self.db, self.shards))}") + + def test_i3_the_batch_step_of_a_multi_node_job(self): + self.embed_run() + with mock.patch.object(split_run, "find_run") as find_run: + message = self.refused_index(env={"SLURM_JOB_ID": 4711, "SLURM_JOB_NUM_NODES": 2, "SLURM_NTASKS": 8}) + find_run.assert_not_called() + self.assertIn("SLURM job 4711 holds 2 nodes", message) + # No dependency on the job being refused: its afterok would never be met. + self.assertTrue(message.endswith( + "Run it as its own one-node CPU job instead: once this job has ended, sbatch -C cpu -N 1 ... SCRIPT (no " + "dependency is needed then); next time, submit it from the GPU job's script before the job's work starts: " + "sbatch -C cpu -N 1 --dependency=afterok:$SLURM_JOB_ID --kill-on-invalid-dep=yes ... SCRIPT. SCRIPT runs " + "this command, without srun, next time " + _NEXT_BUILD + ":\n " + + _command_line(_index_args(self.fasta, self.db, self.shards))), message) + self.assertNotIn("afterok:4711", message) + + def test_i2_i3_before_the_logging_setup(self): + """P9.I1: I2 and I3 refuse before set_log_level (which, until P9.L2, imported Lightning and so torch).""" + self.embed_run() + for env in ({"WORLD_SIZE": 2}, {"SLURM_JOB_ID": 4711, "SLURM_JOB_NUM_NODES": 2, "SLURM_NTASKS": 8}): + with self.subTest(env=env), mock.patch.object(search, "set_log_level", + side_effect=AssertionError("set_log_level")) as setup: + self.refused_index(env=env) + setup.assert_not_called() + + def test_i4_no_run_with_a_record(self): + record = self.embed_run() + empty = self.tmp / "empty" + empty.mkdir() + self.assertIn("No run in", self.refused_index(shards=empty)) + record.unlink() + message = self.refused_index() + self.assertIn("Run run in", message) + self.assertIn("has no run record (1 shard(s), no run record): it did not finish, or its shards were kept by " + "'build sequences'", message) + self.assertIn("'fm-search build sequences-seal --fasta-file FASTA --from-shards", message) + for path in self.shards.iterdir(): + path.unlink() + (self.shards / "emb_run.run.json.tmp").write_text("{}") + message = self.refused_index() + self.assertIn("an unfinished run record (.tmp)", message) + self.assertIn(f"delete {self.shards.resolve() / 'emb_run.run.json.tmp'} first", message) + + def test_i5_several_runs(self): + self.embed_run() + self.embed_run(run_id="run2", db=self.tmp / "other") + (self.shards / "emb_run3-0.parquet").write_bytes(b"a run without a record") + message = self.refused_index() + self.assertIn("holds 3 runs: run (run record, 1 shard(s)); run2 (run record, 1 shard(s)); run3 (1 shard(s), " + "no run record)", message) + self.assertIn("Pass --run-id", message) + self.built("--run-id", "run2", fasta=self.fasta) + + def test_i6_a_run_id_without_a_record(self): + self.embed_run() + message = self.refused_index("--run-id", "nope") + self.assertIn("Run nope has no run record (emb_nope.run.json)", message) + self.assertIn("Runs with a record here: run.", message) + + def test_i7_an_invalid_record(self): + record = self.embed_run() + original = record.read_text() + cases = { + "another kind": (lambda r: r.update(kind="other"), "kind is 'other'"), + "a newer version": (lambda r: r.update(record_version=2), "upgrade FoldMatch"), + "a missing key": (lambda r: r.pop("rows"), "the key rows is missing"), + "another run id": (lambda r: r.update(run_id="other"), "but the file is named for run 'run'"), + } + for case, (change, text) in cases.items(): + with self.subTest(case=case): + record.write_text(original) + _edit_record(record, change) + self.assertIn(text, self.refused_index()) + record.write_text("not json") + self.assertIn("it is not JSON", self.refused_index()) + + def test_i8_residue_codes_on_a_plain_run(self): + self.embed_run() + message = self.refused_index("--residue-codes") + self.assertIn("Run run computed no residue codes", message) + self.assertIn("run step 1 with --residue-codes", message) + + def test_i9_no_residue_codes_and_cleanup_on_a_codes_run(self): + self.embed_run("--residue-codes") + message = self.refused_index("--no-residue-codes", "--cleanup-embeddings") + self.assertIn("would delete the only copy of run run's residue codes", message) + + def test_i10_the_run_system(self): + record = self.embed_run("--residue-codes") + with mock.patch.object(ss, "REGISTRY", MappingProxyType({})): + message = self.refused_index(cause=ss.UnknownScoringSystemError) + self.assertIn(f"unknown esm-codes scoring system '{SYSTEM.name}'", message) + self.assertIn("--no-residue-codes builds the database without its residue codes.", message) + _edit_record(record, lambda r: r["system"].update(digest="0" * 64)) + message = self.refused_index() + self.assertIn(f"with digest {'0' * 64} and encoding_digest {SYSTEM.encoding_digest()}, but this " + f"FoldMatch's {SYSTEM.name} has digest {SYSTEM.digest()}", message) + + # -- inside the build lock, before any database file -------------------------------------------------------------- + def test_i11_a_build_lock(self): + self.embed_run("--residue-codes") + lock = Path(f"{self.db}.build.lock") + lock.write_text('{"pid": 1, "host": "elsewhere"}\n') + Path(f"{self.db}.index").touch() + message = self.refused_index() + self.assertIn(f"{lock} exists ({{\"pid\": 1, \"host\": \"elsewhere\"}})", message) + self.assertIn("files present: db.index.", message) + self.assertIn("delete the lock and these files, then rerun", message) + self.assertNotIn("build residue-codes", message) + Path(f"{self.db}.metadata").touch() + SequenceStore(self.out, "db").create(self.fasta) + message = self.refused_index() + self.assertIn(f"'fm-search build residue-codes {self.out} db --from-shards {self.shards.resolve()} --run-id " + f"run' adds it (delete the lock first).", message) + + def test_i12_an_output_exists(self): + """Refused inside the lock before the shards are read or the FASTA hashed (patched to fail), so not by the + pre-flight's own outputs check, which comes after both.""" + self.embed_run("--residue-codes") + def early(): + stack = ExitStack() + stack.enter_context(mock.patch.object(split_run, "check_run_shards", + side_effect=AssertionError("shards read"))) + stack.enter_context(mock.patch.object(split_run, "fasta_sha256", + side_effect=AssertionError("FASTA hashed"))) + return stack + + for suffix in (".index", ".metadata", ".invlists", ".sequences", ".rescodes.bin", ".rescodes.idx", + ".rescodes.json", ".rescodes.json.tmp"): + with self.subTest(suffix=suffix): + existing = Path(f"{self.db}{suffix}") + existing.touch() + try: + with early(): + message = self.refused_index(cause=FileExistsError) + finally: + existing.unlink() + self.assertIn(f"Database already exists at {self.db} (found: {existing})", message) + self.assertNotIn("build residue-codes", message) + for suffix in (".index", ".metadata"): + Path(f"{self.db}{suffix}").touch() + SequenceStore(self.out, "db").create(self.fasta) + with early(): + message = self.refused_index() + self.assertTrue(message.endswith( + f"The database is complete but for its residue-code store: 'fm-search build residue-codes {self.out} db " + f"--from-shards {self.shards.resolve()} --run-id run' adds it."), message) + + def test_i13_the_recorded_shards(self): + self.embed_run(ranks=2) + shard = self.shards / "emb_run-1.parquet" + data = shard.read_bytes() + shard.unlink() + self.assertIn("missing emb_run-1.parquet", self.refused_index()) + shard.write_bytes(data) + shutil.copyfile(shard, self.shards / "emb_run-7.parquet") + self.assertIn("not in the record emb_run-7.parquet", self.refused_index()) + + def test_i14_a_changed_shard(self): + self.embed_run() + shard = self.shards / "emb_run-0.parquet" + data = shard.read_bytes() + pq.write_table(pq.read_table(shard), shard, compression="none") + message = self.refused_index() + self.assertIn("1 shard(s) of run run in", message) + self.assertIn("changed since the run: emb_run-0.parquet (", message) + shard.write_bytes(b"\0" * len(data)) # the recorded size, no footer + self.assertIn("the footer of a shard of run run cannot be read", self.refused_index()) + + def test_i15_the_footers(self): + """Codes: a pass record with another --min-res-n (the shard's size and rows written into the record); under + --no-residue-codes, shards of another system than the record's; plain: a shard that is not plain.""" + def rewritten(record, change): + _rewrite(self.shards / "emb_run-0.parquet", change) + _resync(record) + + record = self.embed_run("--residue-codes") + rewritten(record, _with_pass(min_res_n=5)) + self.assertIn("emb_run-0.parquet (min_res_n 5, world_size 1)", self.refused_index()) + self.assertIn("emb_run-0.parquet (min_res_n 5, world_size 1)", self.refused_index("--no-residue-codes")) + for path in self.shards.iterdir(): + path.unlink() + record = self.embed_run("--residue-codes") + _edit_record(record, lambda r: r["system"].update(name="esm-codes-test-other")) + message = self.refused_index("--no-residue-codes") + self.assertIn(f"hold the residue codes of {SYSTEM.name}", message) + self.assertIn("not those of esm-codes-test-other", message) + for path in self.shards.iterdir(): + path.unlink() + record = self.embed_run() + rewritten(record, lambda table: table.replace_schema_metadata({b"foldmatch.residue_output": b"x"})) + self.assertIn("are not plain embedding shards", self.refused_index()) + + def test_i16_another_size(self): + self.embed_run() + size = self.fasta.stat().st_size + with open(self.fasta, "a") as f: + f.write(">late\nACDE\n") + with mock.patch.object(split_run, "fasta_sha256", side_effect=AssertionError("hashed")): + message = self.refused_index() + self.assertIn(f"{self.fasta.resolve()} has {size + 11} bytes, but the FASTA run run embedded " + f"({self.fasta.resolve()}) had {size}", message) + + def test_the_folders_a_refused_build_created_are_removed(self): + """A database folder that did not exist is created for the build lock and removed again by a refusal (here + I16); one that existed stays.""" + self.embed_run() + with open(self.fasta, "a") as f: + f.write(">late\nACDE\n") + new = self.tmp / "new" + result = self.index(fasta=self.fasta, db=new / "deeper" / "db") + self.assertIn("bytes, but the FASTA run run embedded", self.refused(result)) + self.assertFalse(new.exists()) + (new / "deeper").mkdir(parents=True) + self.refused(self.index(fasta=self.fasta, db=new / "deeper" / "db")) + self.assertEqual(list((new / "deeper").iterdir()), []) + + def test_i17_another_sha256(self): + self.embed_run() + data = bytearray(self.fasta.read_bytes()) + position = data.index(b"\n", data.index(b">")) + 1 + data[position] = ord("A") if data[position] != ord("A") else ord("C") + self.fasta.write_bytes(bytes(data)) + message = self.refused_index() + self.assertIn("is not the FASTA run run embedded", message) + self.assertIn("though it has its size", message) + + def test_i17_the_given_fasta_is_hashed(self): + """--fasta-file is what step 2 hashes: a same-size copy with one residue changed, given while the recorded + FASTA is still in place, is refused; the recorded one is untouched.""" + self.embed_run() + given = self.tmp / "other" / "seqs.fasta" + given.parent.mkdir() + _one_residue_changed(shutil.copyfile(self.fasta, given)) + recorded = self.fasta.read_bytes() + before = _tree(self.out, self.shards) + message = self.refused(self.index(fasta=given)) + self.assertIn(f"{given.resolve()} (sha256 ", message) + self.assertIn(f"is not the FASTA run run embedded ({self.fasta.resolve()}, sha256", message) + self.assertEqual((self.fasta.read_bytes(), _tree(self.out, self.shards)), (recorded, before)) + + def test_i18_the_pre_flight(self): + """The pre-flight's own refusal and nothing to build (mocked: the FASTA has the recorded bytes).""" + self.embed_run() + from foldmatch.utils import sequence + with mock.patch.object(sequence.FastaPreflight, "check", side_effect=ValueError("a pre-flight refusal")): + self.assertIn("a pre-flight refusal", self.refused_index(cause=ValueError)) + real = sequence.preflight_fasta + + def nothing(path, min_res_n): + return dataclasses.replace(real(path, min_res_n), entries_kept=0) + + with mock.patch.object(sequence, "preflight_fasta", side_effect=nothing): + self.assertIn("there is nothing to build", self.refused_index(cause=ValueError)) + + def test_i19_other_counts(self): + self.embed_run() + from foldmatch.utils import sequence + real = sequence.preflight_fasta + + def counting_otherwise(path, min_res_n): + found = real(path, min_res_n) + return dataclasses.replace(found, residues=found.residues + 1) + + with mock.patch.object(sequence, "preflight_fasta", side_effect=counting_otherwise): + message = self.refused_index() + self.assertIn("counts residues 304, but run run's record holds residues 303", message) + self.assertIn("this FoldMatch's pre-flight counts differently", message) + + def test_i20_ids_not_at_their_dealt_positions(self): + record = self.embed_run() + shard = self.shards / "emb_run-0.parquet" + _rewrite(shard, lambda table: table.set_column(0, table.schema.field("id"), + pa.array(table.column("id").to_pylist()[::-1], + table.schema.field("id").type))) + _resync(record) + message = self.refused_index() + self.assertIn("2 shard row(s) of run run do not hold the id of the entry", message) + self.assertIn("emb_run-0.parquet row 0 holds '2uzi_A', entry 0 is '1acb_E'", message) + + # -- the index phase ---------------------------------------------------------------------------------------------- + def test_i21_a_faiss_refusal(self): + """ivf_pq on the fake 8-d embeddings (PQ m = 64 does not divide 8): nothing written, rerun with other + options.""" + self.embed_run("--residue-codes") + message = self.refused_index("--index-type", "ivf_pq", cause=ValueError) + self.assertIn("must divide embedding dim D=8", message) + self.assertIn("'build sequences-index' can be rerun with other index options, without a new ESM3 pass.", + message) + self.built("--index-type", "flat", fasta=self.fasta) + + def test_i22_the_store_does_not_match_the_pre_flight(self): + """The FASTA grows after FAISS, before the sequence store reads it: the database files are removed.""" + self.embed_run() + create_db = EmbeddingDatabase.create_db + + def then_grows(db, *args, **kwargs): + create_db(db, *args, **kwargs) + with open(self.fasta, "a") as f: + f.write(">late\nACDEFGHIKL\n") + + with mock.patch.object(EmbeddingDatabase, "create_db", then_grows): + message = self.refused_index(cause=ValueError) + self.assertIn("does not match its pre-flight", message) + self.assertIn("The database files it wrote were removed (db.index, db.metadata, db.sequences)", message) + + def test_i23_the_fasta_changed_during_the_build(self): + """The FASTA rewritten with one residue changed (the same size and counts, another sha256) while the sequence + store is built: refused by the stat bracket, the database files removed (also when the FASTA is gone).""" + self.embed_run("--residue-codes") + create = SequenceStore.create + + def rewrite(store, *args, **kwargs): + create(store, *args, **kwargs) + _one_residue_changed(self.fasta) + + with mock.patch.object(SequenceStore, "create", autospec=True, side_effect=rewrite): + message = self.refused_index() + self.assertIn(f"The FASTA changed during this build ({self.fasta.resolve()}: ", message) + self.assertIn("The database files it wrote were removed (db.index, db.metadata, db.sequences).", message) + shutil.copyfile(_FASTA, self.fasta) + os.utime(self.fasta, ns=(0, split_run.read_run_record(self.shards / "emb_run.run.json")["fasta"]["mtime_ns"])) + + def removed(store, *args, **kwargs): + create(store, *args, **kwargs) + self.fasta.rename(self.inputs / "gone.fasta") + + with mock.patch.object(SequenceStore, "create", autospec=True, side_effect=removed): + message = self.refused_index() + self.assertIn("cannot be stat'ed", message) + self.assertIn("The database files it wrote were removed", message) + + def test_i23_the_bracket_opens_before_the_hash(self): + """The bracket starts with the stat taken before the FASTA is hashed: a change during the id check (after the + hash and the pre-flight, before the index phase) is refused at the sequence store, which read the new bytes; + no database file is left.""" + self.embed_run() + check = split_run.check_dealt_ids + + def then_changed(*args): + seconds = check(*args) + _one_residue_changed(self.fasta) + return seconds + + with mock.patch.object(split_run, "check_dealt_ids", side_effect=then_changed): + message = self.refused_index() + self.assertIn(f"The FASTA changed during this build ({self.fasta.resolve()}: ", message) + self.assertEqual(list(self.out.iterdir()), []) + + def test_i24_the_code_store_fails_validation(self): + """An entry with an empty payload: the code store is removed; the database, the record and the shards stay, + also with --cleanup-embeddings; no lock is left.""" + def payload(name, sequence): + return np.empty((0, 8), np.uint8) if name == "2uzi_A" else _fake_codes(name, sequence) + + self.embed_run("--residue-codes", payload=payload) + before = _tree(self.shards) + result = self.index("--cleanup-embeddings", fasta=self.fasta) + message = self.refused(result, CodeStoreValidationError) + self.assertIn("empty payload", message) + self.assertIn(f"This build's shards are kept in {self.shards.resolve()} (run id run).", message) + self.assertEqual(sorted(p.name for p in self.out.iterdir()), ["db.index", "db.metadata", "db.sequences"]) + self.assertEqual(_tree(self.shards), before) + + +# ===================================================================================================================== +# P8b.5: build sequences-seal +# ===================================================================================================================== +#: Plan P8b (CLI), verbatim: the seal's command help and the help texts it does not take from 'build sequences'. +SEAL_HELP = ( + "Recovery: write the run record of embedding shards that have none, so that 'build sequences-index' can build from " + "them: a 'build sequences-embed' run whose rank 0 stopped after the pass (wall time, out of memory, a full disk), " + "or the shards a 'build sequences' run kept when its index phase failed. Checks the shards against the FASTA " + "(entry count, each rank's rows, the shard schema and the id of every row) without ESM3 or a GPU, in one process. " + "The record says it was sealed: it binds the shards by these checks, not by the pass." +) +SEAL_OWN_HELP = { + "fasta_file": "The FASTA file the run embedded.", + "from_shards": "Folder holding the run's shards.", + "run_id": "Run id of the shards to seal (emb_-.parquet).", + "min_res_n": "The --min-res-n the run used. Required for a run without residue codes, whose shards do not record " + "it; for a run with codes it is read from the shards, and a different value is refused.", +} +#: The record fields a seal writes otherwise than step 1 (plan P8b.5, step 10). +_SEALED = {"sealed": True, "command": "build sequences-seal", "options": {}, "output_db": None} + + +def _seal_args(fasta, shards, run_id, *extra): + return ("build", "sequences-seal", "--fasta-file", fasta, "--from-shards", shards, "--run-id", run_id, *extra) + + +def _codes_flag(codes): + return ("--residue-codes",) if codes else () + + +class _Sealing(_SplitBuild): + """``_SplitBuild`` plus the seal, under the ESM3 spy with ``scoring_system.prepare`` refusing, and runs that have + no record.""" + + def seal(self, *extra, fasta=_FASTA, shards=None, run_id="run", env=None): + """``build sequences-seal`` with ``env`` set and ``sys.argv`` as the console script sets it; returns the + result.""" + args = _seal_args(fasta, shards or self.shards, run_id, *extra) + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched")), \ + mock.patch.dict(os.environ, {name: str(value) for name, value in (env or {}).items()}), \ + _as_fm_search(args): + result = invoke(*args) + esm3.assert_not_called() + return result + + def sealed(self, *extra, shards=None, run_id="run", **kwargs): + """The seal, which must succeed; returns its record's path.""" + result = self.seal(*extra, shards=shards, run_id=run_id, **kwargs) + self.assertEqual(result.exit_code, 0, f"{result.output} {result.exception!r}") + return Path(shards or self.shards) / f"emb_{run_id}.run.json" + + def unrecorded(self, *extra, **kwargs): + """Step 1 whose rank 0 stopped after the pass: its record deleted. Returns that record.""" + path = self.embed(*extra, **kwargs) + record = split_run.read_run_record(path) + path.unlink() + return record + + +class SealOptionTest(_Tmp): + + def test_the_command_help(self): + self.assertEqual(_command("build", "sequences-seal").help, SEAL_HELP) + output = CliRunner().invoke(search.app, ["build", "sequences-seal", "--help"], env={"COLUMNS": "400"}).output + for text in (SEAL_HELP, *SEAL_OWN_HELP.values()): + with self.subTest(text=text[:40]): + self.assertIn(text, _flat(output)) + + def test_the_options(self): + """Exactly the seal's options, with the plan's help; --fasta-file and --from-shards are step 2's (required, + must exist), --log-level the one-shot's; --run-id is required; --min-res-n is optional and at least 0.""" + seal = {param.name: param for param in _command("build", "sequences-seal").params} + oneshot = {param.name: param for param in _command("build", "sequences").params} + index = {param.name: param for param in _command("build", "sequences-index").params} + self.assertEqual(sorted(seal), sorted(["fasta_file", "from_shards", "run_id", "min_res_n", "log_level"])) + + def info(param): + out = param.to_info_dict() + for key in ("name", "help"): + out.pop(key) + return out + + for name, param in seal.items(): + with self.subTest(name=name): + self.assertEqual(param.help, SEAL_OWN_HELP[name] if name in SEAL_OWN_HELP else oneshot[name].help) + for name, same in (("fasta_file", index), ("from_shards", index), ("log_level", oneshot)): + with self.subTest(name=name): + self.assertEqual(info(seal[name]), info(same[name])) + run_id = seal["run_id"] + self.assertEqual((run_id.opts, run_id.required, run_id.type.name), (["--run-id"], True, "text")) + min_res_n = seal["min_res_n"] + self.assertEqual((min_res_n.opts, min_res_n.required, min_res_n.default, min_res_n.type.min, + min_res_n.type.max), (["--min-res-n"], False, None, 0, None)) + + def test_options_another_command_owns(self): + """Every pass, index and output option, --residue-codes (the shards say whether they hold codes) and + --cleanup-embeddings: exit 2, nothing read or written. S1: a malformed --run-id, a negative --min-res-n and + no --run-id are exit 2 too.""" + for option in (("--output-db", self.tmp / "db"), ("--shards-folder", self.shards), + ("--tmp-embedding-folder", self.shards), ("--accelerator", "cpu"), ("--num-nodes", "1"), + ("--devices", "1"), ("--strategy", "auto"), ("--batch-size", "1"), ("--num-workers", "0"), + ("--use-gpu-index",), ("--index-type", "flat"), ("--ivf-nlist", "256"), ("--ivf-nprobe", "4"), + ("--load-batch-size", "1000"), ("--residue-codes",), ("--no-residue-codes",), + ("--cleanup-embeddings",), ("--force",)): + with self.subTest(option=option[0]), _esm3_spy() as esm3: + result = invoke(*_seal_args(_FASTA, self.shards, "run", *option)) + self.usage_error(result, option[0]) + self.assertRegex(_flat(result.output), no_such_option(option[0])) + esm3.assert_not_called() + for bad in ("a b", "a/b", "a*"): + with self.subTest(run_id=bad), mock.patch.object(split_run, "refuse_existing_record") as refuse: + self.usage_error(invoke(*_seal_args(_FASTA, self.shards, bad)), "is not a run id") + refuse.assert_not_called() + self.usage_error(invoke(*_seal_args(_FASTA, self.shards, "run", "--min-res-n", "-1")), "--min-res-n") + self.usage_error(invoke("build", "sequences-seal", "--fasta-file", _FASTA, "--from-shards", self.shards), + "--run-id") + self.assertEqual(list(self.shards.iterdir()), []) + + +class SealTest(_Sealing): + + def test_the_shards_a_failed_one_shot_kept(self): + """A 'build sequences' whose index phase fails after the pass (ivf_pq on the fake 8-d embeddings), plain (the + seal given --min-res-n) and codes (the seal reads it from the shards), on 1 and 2 ranks, with --min-res-n 0 and + 100: sealed, then step 2 with a flat index builds the database of a flat one-shot.""" + for codes in (False, True): + for ranks in (1, 2): + for min_res_n in (0, 100): + name = f"{'codes' if codes else 'plain'}{ranks}-{min_res_n}" + with self.subTest(codes=codes, ranks=ranks, min_res_n=min_res_n): + extra = ("--min-res-n", min_res_n, *_codes_flag(codes)) + shards = self.tmp / f"kept-{name}" + shards.mkdir() + with launch_env(FOLDMATCH_RUN_ID=f"kept-{name}"), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", + fake_chain_predict(ranks=ranks)): + failed = invoke(*_build_args(_FASTA, self.tmp / f"failed-{name}" / "db", shards, *extra, + "--index-type", "ivf_pq")) + self.assertEqual(failed.exit_code, 1, failed.output) + self.assertIn("must divide embedding dim D=8", str(failed.exception)) + path = self.sealed(*(() if codes else ("--min-res-n", min_res_n)), shards=shards, + run_id=f"kept-{name}") + record = split_run.read_run_record(path) + kept = 2 if min_res_n == 0 else 1 + self.assertEqual((record["sealed"], record["min_res_n"], record["rows"], record["world_size"]), + (True, min_res_n, kept, ranks if codes else min(ranks, kept))) + db = self.tmp / f"split-{name}" / "db" + self.built("--index-type", "flat", shards=shards, db=db) + assert_same_database(self.oneshot(*extra, "--index-type", "flat", run_id=f"one-{name}", + ranks=ranks), db) + + def test_a_lost_record(self): + """A step-1 run whose record was deleted, plain and codes, on 1 and 2 ranks, with --min-res-n 0 and 100 (one + entry, so one shard): the sealed record is the lost one except sealed, command, options, output_db and + created_utc and, plain, the pass's ESM3 revision (no plain shard records it); the record is the only new + file.""" + for codes in (False, True): + for ranks, min_res_n in ((1, 0), (2, 0), (1, 100)): + name = f"{'codes' if codes else 'plain'}{ranks}-{min_res_n}" + with self.subTest(codes=codes, ranks=ranks, min_res_n=min_res_n): + shards = self.tmp / f"shards-{name}" + shards.mkdir() + lost = self.unrecorded("--min-res-n", min_res_n, *_codes_flag(codes), ranks=ranks, run_id=name, + shards=shards, db=self.tmp / f"db-{name}") + before = _tree(shards) + path = self.sealed(*(() if codes else ("--min-res-n", min_res_n)), shards=shards, run_id=name) + sealed = split_run.read_run_record(path) + self.assertEqual(json.loads(path.read_text()), sealed) + self.assertEqual({k: v for k, v in _tree(shards).items() if k != str(path)}, before) + created = datetime.strptime(sealed["created_utc"], "%Y-%m-%dT%H:%M:%SZ") + self.assertLess(abs(datetime.now(timezone.utc).replace(tzinfo=None) - created), + timedelta(minutes=5)) + expected = {**lost, **_SEALED, "created_utc": sealed["created_utc"]} + if not codes: + self.assertEqual(lost["pass"]["esm_revision"], ESM_REVISION) + expected["pass"] = {**lost["pass"], "esm_revision": None} + self.assertEqual(sealed, expected) + + def test_the_seal_command_step_1_names(self): + """Step 1 on 2 ranks refused after it hashed the FASTA (E12: a full disk while the record is staged): the seal + command its refusal names, run verbatim (paths with spaces), records the shards, and step 2 builds the + one-shot's database. Plain and codes (the command gives --min-res-n in both modes).""" + for codes in (False, True): + name = "codes" if codes else "plain" + with self.subTest(codes=codes): + shards = self.tmp / f"shard s {name}" + shards.mkdir() + with launch_env(FOLDMATCH_RUN_ID=name), \ + mock.patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(ranks=2)), \ + mock.patch.object(search, "_gather_rank_facts", side_effect=_ranks(2)), \ + mock.patch.object(split_run, "_write_all", + side_effect=OSError(errno.ENOSPC, "No space left on device")): + result = invoke(*_embed_args(_FASTA, self.tmp / f"db-{name}", shards, *_codes_flag(codes))) + message = self.refused(result) + hint = "Once the cause is fixed, 'build sequences-seal' can record them: " + argv = shlex.split(message[message.index(hint) + len(hint):]) + self.assertEqual(argv[:3], ["fm-search", "build", "sequences-seal"]) + self.assertEqual(split_run.record_run_ids(shards), []) + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", + side_effect=AssertionError("bundle fetched")): + sealed = invoke(*argv[1:]) + esm3.assert_not_called() + self.assertEqual(sealed.exit_code, 0, f"{sealed.output} {sealed.exception!r}") + self.assertEqual(split_run.read_run_record(shards / f"emb_{name}.run.json")["world_size"], 2) + db = self.tmp / f"split-{name}" / "db" + self.built(shards=shards, db=db) + assert_same_database(self.oneshot(*_codes_flag(codes), run_id=f"one-{name}", ranks=2), db) + + def test_a_world_larger_than_its_entries(self): + """A pass on 3 ranks over 2 entries writes 2 shards: a codes run's sealed record keeps its pass's world of 3, a + plain run's world is its 2 shards (the same deal). Both build the one-shot's database.""" + for codes in (False, True): + name = "codes" if codes else "plain" + with self.subTest(codes=codes): + shards = self.tmp / f"shards-{name}" + shards.mkdir() + lost = self.unrecorded(*_codes_flag(codes), ranks=3, run_id=name, shards=shards, + db=self.tmp / f"x-{name}") + self.assertEqual((lost["world_size"], len(lost["shards"])), (3, 2)) + sealed = split_run.read_run_record(self.sealed(*(() if codes else ("--min-res-n", "0")), + shards=shards, run_id=name)) + self.assertEqual((sealed["world_size"], sealed["pass"]["world_size"]), (3, 3) if codes else (2, 2)) + self.assertEqual(sealed["shards"], lost["shards"]) + db = self.tmp / f"split-{name}" / "db" + self.built(shards=shards, db=db) + assert_same_database(self.oneshot(*_codes_flag(codes), run_id=f"one-{name}", ranks=3), db) + + def test_the_log_lines_and_the_next_command(self): + """The FASTA hash and the id check are timed; then the run sealed, the WARNING that the record binds the + shards by these checks (and seq_sha1 with codes), and step 2's command, which builds the database once DB is + given (paths with spaces, split as a shell splits them).""" + folder = self.tmp / "with space" + folder.mkdir() + fasta = Path(shutil.copyfile(_FASTA, folder / "my seqs.fasta")) + for codes in (True, False): + with self.subTest(codes=codes): + shards = folder / f"shard s {codes}" + shards.mkdir() + self.unrecorded(*_codes_flag(codes), ranks=2, fasta=fasta, shards=shards, db=self.tmp / f"db-{codes}") + messages = self.logged(lambda: self.sealed(*(() if codes else ("--min-res-n", "0")), fasta=fasta, + shards=shards)) + found = [m for _, m in messages] + size = fasta.stat().st_size + prefixes = (f"FASTA sha256 of {fasta.resolve()} ({size} bytes) in ", + f"Ids{' and seq_sha1' if codes else ''} of the 2 shard rows checked against " + f"{fasta.resolve()} in ", "Run run sealed: ", "The record of run run is sealed: ", + "Build the database DB with: ") + lines = [[m for m in found if m.startswith(prefix)] for prefix in prefixes] + self.assertEqual([len(line) for line in lines], [1] * 5, found) + lines = [line[0] for line in lines] + order = [found.index(line) for line in lines] + self.assertEqual(order, sorted(order)) + for line in lines[:2]: + self.assertRegex(line, r" in \d+\.\d s$") + self.assertEqual(lines[2], f"Run run sealed: 2 entries in 2 shard(s) of 2 rank(s)" + f"{f', residue codes of {SYSTEM.name}' if codes else ''}; record " + f"{shards.resolve() / 'emb_run.run.json'}") + self.assertIn(("WARNING", lines[3]), messages) + self.assertIn(f"binds the shards to {fasta.resolve()} by their entry count, each rank's rows and the " + f"id of every row", lines[3]) + self.assertEqual("seq_sha1" in lines[3], codes) + self.assertEqual(" and its seq_sha1, the sha1 of the entry's sequence, not by the pass" in lines[3], + codes) + self.assertTrue(lines[3].endswith(", not by the pass that wrote them, which 'build sequences-seal' did " + "not see."), lines[3]) + argv = shlex.split(lines[4][len("Build the database DB with: "):]) + self.assertEqual(argv, ["fm-search", "build", "sequences-index", "--fasta-file", str(fasta.resolve()), + "--output-db", "DB", "--from-shards", str(shards.resolve()), "--run-id", "run"]) + db = folder / f"my db {codes}" + argv[argv.index("DB")] = str(db) + with _esm3_spy() as esm3, mock.patch.object(ss, "prepare", + side_effect=AssertionError("bundle fetched")): + result = invoke(*argv[1:]) + esm3.assert_not_called() + self.assertEqual(result.exit_code, 0, result.output) + assert_same_database(self.oneshot(*_codes_flag(codes), run_id=f"one-{codes}", fasta=fasta, ranks=2), db) + + def test_offline_with_an_empty_hf_cache(self): + """The seal as a command, with HF_HUB_OFFLINE=1 and an empty Hugging Face cache: a codes run is sealed from its + shards' footers, without the bundle.""" + self.unrecorded("--residue-codes") + empty = self.tmp / "empty-hf-cache" + empty.mkdir() + env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env.update(HF_HUB_OFFLINE="1", HF_HUB_CACHE=str(empty), HF_HOME=str(empty), + PYTHONPATH=os.pathsep.join([_SRC] + [p for p in [os.environ.get("PYTHONPATH")] if p])) + run = subprocess.run([sys.executable, "-m", "foldmatch.cli.search", *map(str, _seal_args( + _FASTA, self.shards, "run"))], env=env, capture_output=True, text=True, timeout=600) + self.assertEqual(run.returncode, 0, run.stderr[-4000:]) + self.assertEqual(split_run.read_run_record(self.shards / "emb_run.run.json")["system"]["name"], SYSTEM.name) + self.assertEqual(list(empty.rglob("models--*")), []) + + def test_a_system_unknown_here(self): + """The seal needs no registry entry: a codes run whose system this FoldMatch does not register is sealed; + step 2 then refuses it (I10) or builds the plain database (--no-residue-codes).""" + self.unrecorded("--residue-codes") + with mock.patch.object(ss, "REGISTRY", MappingProxyType({})): + self.sealed() + self.assertIn("unknown esm-codes scoring system", self.refused(self.index())) + self.built("--no-residue-codes") + assert_same_database(self.oneshot(run_id="plain"), self.db) + + def test_the_foldmatch_of_a_sealed_pass(self): + """Step 1 by another FoldMatch, its record lost, sealed by this one: the sealed record holds the seal's version, + and step 2 takes the pass's from the codes shards' pass record (D30 WARNING, start line). A plain run's pass + version is not recorded anywhere (its pass record is the seal's): no D30 WARNING, and the sealed WARNING says + the version is unknown.""" + for codes in (True, False): + with self.subTest(codes=codes): + for path in [*self.shards.iterdir(), *self.out.iterdir()]: + path.unlink() + with mock.patch.object(foldmatch, "__version__", "0.7.0-old"), \ + mock.patch.object(search, "__version__", "0.7.0-old"): + lost = self.unrecorded(*_codes_flag(codes)) + self.assertEqual((lost["foldmatch_version"], lost["pass"]["foldmatch_version"]), ("0.7.0-old",) * 2) + record = split_run.read_run_record(self.sealed(*(() if codes else ("--min-res-n", "0")))) + self.assertEqual((record["foldmatch_version"], record["pass"]["foldmatch_version"]), + (foldmatch.__version__, "0.7.0-old" if codes else foldmatch.__version__)) + messages = self.logged(lambda: self.built()) + warnings = [m for level, m in messages if level == "WARNING"] + d30 = [m for m in warnings if m.startswith("Run run was embedded by FoldMatch ")] + self.assertEqual([m.split(",")[0] for m in d30], ["Run run was embedded by FoldMatch 0.7.0-old"] + if codes else []) + sealed = next(m for m in warnings if "written by 'build sequences-seal'" in m) + self.assertIn(f"(FoldMatch {foldmatch.__version__})", sealed) + self.assertEqual(sealed.endswith("whose FoldMatch version is not recorded (shards without residue " + "codes do not record it)."), not codes) + start = next(m for _, m in messages if m.startswith(f"Building {self.db} from run run ")) + self.assertIn(f"sealed {record['created_utc']} by FoldMatch {foldmatch.__version__}; embedded by " + + ("FoldMatch 0.7.0-old on" if codes else "a FoldMatch it does not record on"), start) + + def test_launch_warnings_after_the_logging_setup(self): + """P9.I1: a one-task srun step of a multi-node job, and a one-node job with GPUs: sealed, with the WARNING of + D34 logged once set_log_level has set up the logging.""" + cases = { + "an srun step": ({"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 2, "SLURM_STEP_ID": 3, + "SLURM_STEP_NUM_TASKS": 1, "SLURM_NTASKS": 8}, + "'build sequences-seal' runs in a one-task srun step of SLURM job 42, which holds 2 " + "nodes: the others stay allocated until it ends. Next time, run it as its own one-node " + "CPU job: submit it from the GPU job's script before the job's work starts: sbatch -C cpu " + "-N 1 --dependency=afterok:$SLURM_JOB_ID --kill-on-invalid-dep=yes ... SCRIPT. SCRIPT " + "runs this command, without srun, " + _NEXT_BUILD + ":\n {command}"), + "GPUs": ({"SLURM_JOB_ID": 42, "SLURM_JOB_NUM_NODES": 1, "SLURM_JOB_GPUS": "0,1"}, + "SLURM job 42 has GPUs (SLURM_JOB_GPUS=0,1), which 'build sequences-seal' does not use: run it as " + "its own job on a CPU node (sbatch -C cpu -N 1 ...); a project with GPU hours only may run it on " + "one GPU node, where ivf_pq fits (flat and hnsw may not)."), + } + for k, (case, (env, text)) in enumerate(cases.items()): + with self.subTest(case=case): + shards = self.tmp / f"w{k}" + shards.mkdir() + self.unrecorded(shards=shards) + events = self.setup_and_warnings(lambda: self.sealed("--min-res-n", 0, env=env, shards=shards)) + text = text.format(command=_command_line(_seal_args(_FASTA, shards, "run", "--min-res-n", 0))) + self.assertEqual(events[:2], [("set_log_level", LogLevel.info), ("WARNING", text)], events) + + +class SealRefusalTest(_Sealing): + + def setUp(self): + super().setUp() + self.inputs = self.tmp / "in" + self.inputs.mkdir() + self.fasta = Path(shutil.copyfile(_FASTA, self.inputs / "seqs.fasta")) + + def refused_seal(self, *extra, cause=None, env=None, shards=None, run_id="run", fasta=None): + """The seal refused (exit 1, a RefusalError), before ESM3, writing nothing: the shards folder and the database + folder as they were. Returns the message.""" + before = _tree(self.out, shards or self.shards) + result = self.seal(*extra, fasta=fasta or self.fasta, shards=shards, run_id=run_id, env=env) + message = self.refused(result, cause) + self.assertEqual(_tree(self.out, shards or self.shards), before) + return message + + def unrecorded_run(self, *extra, **kwargs): + return self.unrecorded(*extra, **{"fasta": self.fasta, **kwargs}) + + def clear(self): + for path in self.shards.iterdir(): + path.unlink() + + def shard(self, rank=0): + return self.shards / f"emb_run-{rank}.parquet" + + def test_s1_the_flags(self): + self.unrecorded_run() + self.usage_error(self.seal("--min-res-n", "-1", fasta=self.fasta), "--min-res-n") + self.usage_error(self.seal(fasta=self.fasta, run_id="a b"), "is not a run id") + self.assertEqual(split_run.record_run_ids(self.shards), []) + + def test_s2_the_launch(self): + """One of several ranks (I2), and the batch step of a multi-node job (I3): refused before the folder is + read, and before set_log_level (P9.I1).""" + self.unrecorded_run() + command = _command_line(_seal_args(self.fasta, self.shards, "run")) + i2 = f" The command:\n {command}" + i3 = f" SCRIPT. SCRIPT runs this command, without srun, next time {_NEXT_BUILD}:\n {command}" + for env, text, end in (({"WORLD_SIZE": 2}, "'build sequences-seal' runs as one process (WORLD_SIZE=2): launch " + "it once, directly in the batch script or with srun -n 1.", i2), + ({"LOCAL_RANK": 1}, "(LOCAL_RANK=1)", i2), + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 0, "SLURM_STEP_NUM_TASKS": 2}, + "(SLURM_STEP_NUM_TASKS=2)", i2), + ({"SLURM_JOB_ID": 4711, "SLURM_JOB_NUM_NODES": 2, "SLURM_NTASKS": 8}, + "SLURM job 4711 holds 2 nodes, and 'build sequences-seal' runs in one process", i3)): + with self.subTest(env=env), mock.patch.object(split_run, "refuse_existing_record") as refuse, \ + mock.patch.object(search, "set_log_level", side_effect=AssertionError("set_log_level")) as setup: + message = self.refused_seal(env=env) + self.assertIn(text, message) + self.assertTrue(message.endswith(end), message) + refuse.assert_not_called() + setup.assert_not_called() # P9.I1: before set_log_level (until P9.L2 it imported Lightning) + + def test_s3_a_record_or_tmp(self): + """A run with a record: use it. A record whose writer stopped (.tmp only): delete it, then seal again.""" + record = self.embed(fasta=self.fasta) + message = self.refused_seal() + self.assertIn(f"{self.shards.resolve() / 'emb_run.run.json'} exists: run run in {self.shards.resolve()} has " + f"its run record already, so there is nothing to seal.", message) + self.assertIn(f"Use it: 'build sequences-index --from-shards {self.shards.resolve()} --run-id run'", message) + record.rename(self.shards / "emb_run.run.json.tmp") + message = self.refused_seal() + self.assertIn(f"{self.shards.resolve() / 'emb_run.run.json.tmp'} exists", message) + self.assertIn("Delete the .tmp, then run 'build sequences-seal' again.", message) + + def test_s4_no_shards_of_the_run(self): + self.unrecorded_run() + self.embed(fasta=self.fasta, run_id="done", db=self.tmp / "other") + message = self.refused_seal(run_id="nope") + self.assertIn(f"No embedding shards of run nope (emb_nope-.parquet) in {self.shards.resolve()}, so there " + f"is nothing to seal. It holds run done (run record, 1 shard(s)); run run (1 shard(s), no run " + f"record).", message) + empty = self.tmp / "empty" + empty.mkdir() + self.assertTrue(self.refused_seal(shards=empty).endswith("It holds no run.")) + + def test_s5_the_footers(self): + """Plain and codes shards mixed; codes shards of two encodings; a codes shard without a pass record; residue + shards of another kind (query encodings); a plain shard that is not legacy; a shard that is no Parquet file. + Each names the shards.""" + def metadata(change): + def rewrite(table): + meta = dict(table.schema.metadata) + change(meta) + return table.replace_schema_metadata(meta) + return rewrite + + cases = { + "plain and codes": (True, 1, lambda t: t.select(["id", "embedding"]).replace_schema_metadata(None), + ("The 2 shards of run run in", "are mixed: ", "no residue column (a plain pass) (e.g. " + "emb_run-1.parquet)", "residue_codes of")), + "two encodings": (True, 1, metadata(lambda m: m.update({b"foldmatch.encoding_digest": b"0" * 64})), + ("are mixed: ", "encoding_digest " + "0" * 64)), + "no pass record": (True, 0, metadata(lambda m: m.pop(META_PASS)), + ("1 of the 2 shard(s) of run run in", "record no pass (foldmatch.pass): " + "emb_run-0.parquet.")), + "query encodings": (True, None, metadata(lambda m: m.update({b"foldmatch.residue_output": + b"query_encoding"})), + ("hold query_encoding of", f"not the residue codes of {SYSTEM.name}")), + "not legacy": (False, None, lambda t: t.replace_schema_metadata({b"foldmatch.system": b"x"}), + ("are not plain embedding shards", "the metadata foldmatch.system")), + } + for case, (codes, rank, change, texts) in cases.items(): + with self.subTest(case=case): + self.clear() + self.unrecorded_run(*_codes_flag(codes), ranks=2) + for r in ((0, 1) if rank is None else (rank,)): + _rewrite(self.shard(r), change) + message = self.refused_seal(*(() if codes else ("--min-res-n", "0"))) + for text in texts: + self.assertIn(text, message) + with self.subTest(case="no Parquet file"): + self.clear() + self.unrecorded_run("--residue-codes") + self.shard().write_bytes(b"not a shard") + self.assertIn("emb_run-0.parquet: not a readable Parquet embedding shard", self.refused_seal()) + + def test_s6_the_pass(self): + """A plain run without --min-res-n; a codes run with another --min-res-n than its shards'; codes shards that + disagree on their pass's world size or --min-res-n, or record no valid one; a world smaller than the shard + count. Each gives both values.""" + self.unrecorded_run() + self.assertIn("hold no residue codes, so they do not record the --min-res-n of their pass: give the " + "--min-res-n the run used.", self.refused_seal()) + cases = { + "another --min-res-n": ((), ("--min-res-n", "100"), + ("--min-res-n is 100, but the shards of run run in", "record a pass with " + "--min-res-n 0: leave --min-res-n out")), + "world sizes disagree": (((1, _with_pass(world_size=3)),), (), + ("disagree on their pass's world_size: 2 in emb_run-0.parquet, 3 in " + "emb_run-1.parquet.",)), + "min_res_n disagree": (((1, _with_pass(min_res_n=5)),), (), + ("disagree on their pass's min_res_n: 0 in emb_run-0.parquet, 5 in " + "emb_run-1.parquet.",)), + "no valid min_res_n": (((0, _with_pass(min_res_n=None)), (1, _with_pass(min_res_n=None))), (), + ("record a pass without a valid --min-res-n and world size (min_res_n None, " + "world_size 2)",)), + "a world smaller than the shards": (((0, _with_pass(world_size=1)), (1, _with_pass(world_size=1))), (), + ("record a pass on 1 rank(s), but there are 2 of them",)), + } + for case, (changes, extra, texts) in cases.items(): + with self.subTest(case=case): + self.clear() + self.unrecorded_run("--residue-codes", ranks=2) + for rank, change in changes: + _rewrite(self.shard(rank), change) + message = self.refused_seal(*extra) + for text in texts: + self.assertIn(text, message) + + def test_s7_the_pre_flight(self): + """The pre-flight's own refusals (strict for a codes run, a header without an id in either mode) and nothing to + build: as the one-shot.""" + self.unrecorded_run("--residue-codes") + duplicate = _write(self.inputs / "dup.fasta", ">1acb_E\nACDE\n>1acb_E\nACDE\n") + self.assertIn("repeat an earlier id", self.refused_seal(fasta=duplicate, cause=ValueError)) + self.clear() + self.unrecorded_run() + empty_id = _write(self.inputs / "empty-id.fasta", ">\nACDE\n>b\nKLMN\n") + self.assertIn("is empty", self.refused_seal("--min-res-n", "0", fasta=empty_id, cause=ValueError)) + self.assertIn("there is nothing to build", self.refused_seal("--min-res-n", "1000", cause=ValueError)) + + def test_s8_the_shards_are_incomplete(self): + """Rows other than the kept entries (a lost shard, codes or plain; a plain run sealed with another --min-res-n) + and rows that are not the deal.""" + message_end = "The shards are incomplete: run 'build sequences-embed' again" + self.unrecorded_run("--residue-codes", ranks=2) + self.shard(1).unlink() + message = self.refused_seal() + self.assertIn(f"The 1 shard(s) of run run in {self.shards.resolve()} hold 1 row(s), but " + f"{self.fasta.resolve()} has 2 entries with at least 0 residues.", message) + self.assertTrue(message.endswith(f"{message_end}."), message) + self.clear() + self.unrecorded_run(ranks=2) + self.shard(1).unlink() + message = self.refused_seal("--min-res-n", "0") + self.assertIn("hold 1 row(s), but", message) + self.assertTrue(message.endswith(f"{message_end}, or give the --min-res-n the run used."), message) + self.clear() + self.unrecorded_run() + self.assertIn("hold 2 row(s), but", self.refused_seal("--min-res-n", "100")) + self.clear() + five = _write(self.inputs / "five.fasta", _FIVE) + self.unrecorded_run("--residue-codes", ranks=2, fasta=five) + t0, t1 = pq.read_table(self.shard(0)), pq.read_table(self.shard(1)) + pq.write_table(t0.slice(0, 2), self.shard(0)) # rank 0: entries 0, 2; rank 1: 4, 1, 3 + pq.write_table(pa.concat_tables([t0.slice(2), t1.replace_schema_metadata(t0.schema.metadata)]), self.shard(1)) + message = self.refused_seal(fasta=five) + self.assertIn("are not the deal of its 5 entries over the 2 rank(s) of the pass (rank 0 has 2 row(s), not 3; " + "rank 1 has 3 row(s), not 2): the shards cannot be indexed.", message) + self.assertTrue(message.endswith(f"{message_end}."), message) + + def test_s9_ids_not_at_their_dealt_positions(self): + for codes in (False, True): + with self.subTest(codes=codes): + self.clear() + self.unrecorded_run(*_codes_flag(codes)) + _rewrite(self.shard(), lambda table: table.set_column(0, table.schema.field("id"), pa.array( + table.column("id").to_pylist()[::-1], table.schema.field("id").type))) + message = self.refused_seal(*(() if codes else ("--min-res-n", "0"))) + self.assertIn("2 shard row(s) of run run do not hold the id of the entry", message) + self.assertIn("emb_run-0.parquet row 0 holds '2uzi_A', entry 0 is '1acb_E'", message) + + def test_s9_sequences_that_are_not_the_shards(self): + """A FASTA with the run's ids, order and lengths but another residue (the same size and pre-flight counts): + a codes run's seq_sha1 refuses it; a plain run has no seq_sha1, so its seal binds by ids only (D31).""" + for codes in (True, False): + with self.subTest(codes=codes): + self.clear() + shutil.copyfile(_FASTA, self.fasta) + self.unrecorded_run(*_codes_flag(codes)) + _one_residue_changed(self.fasta) + if not codes: + self.sealed("--min-res-n", "0", fasta=self.fasta) + continue + message = self.refused_seal() + self.assertIn("1 shard row(s) of run run hold a seq_sha1 that is not the sha1 of the sequence of the " + f"entry of {self.fasta.resolve()} the pass dealt them, though they hold its id: " + "emb_run-0.parquet row 0 ('1acb_E', entry 0). These shards were not computed from this " + "FASTA's sequences.", message) + + def test_s10_a_write_during_the_hash(self): + """The seal's bracket opens before the hash: a write while the FASTA is hashed (its mtime moved once the hash + is done) is refused, so the record never pairs a sha256 with a stat taken after it.""" + self.unrecorded_run("--residue-codes") + real = split_run.fasta_sha256 + + def written_while_hashed(path): + out = real(path) + stat = os.stat(path) + os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10 ** 9)) + return out + + with mock.patch.object(split_run, "fasta_sha256", side_effect=written_while_hashed): + message = self.refused_seal() + self.assertIn(f"The FASTA changed during the seal ({self.fasta.resolve()}: ", message) + self.assertEqual(split_run.record_run_ids(self.shards), []) + + def test_s10_the_fasta_changed_during_the_seal(self): + """The FASTA touched, or removed, after the id check: refused with both stats.""" + self.unrecorded_run("--residue-codes") + check = split_run.check_dealt_ids + stat = self.fasta.stat() + + def touching(*args, **kwargs): + seconds = check(*args, **kwargs) + os.utime(self.fasta, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10 ** 9)) + return seconds + + with mock.patch.object(split_run, "check_dealt_ids", side_effect=touching): + message = self.refused_seal() + self.assertIn(f"The FASTA changed during the seal ({self.fasta.resolve()}: {stat.st_size} bytes, mtime_ns " + f"{stat.st_mtime_ns}, inode {stat.st_ino} before it was hashed, now {stat.st_size} bytes, " + f"mtime_ns {stat.st_mtime_ns + 10 ** 9}, inode {stat.st_ino})", message) + + def removing(*args, **kwargs): + seconds = check(*args, **kwargs) + self.fasta.rename(self.inputs / "gone.fasta") + return seconds + + with mock.patch.object(split_run, "check_dealt_ids", side_effect=removing): + self.assertIn("now cannot be stat'ed", self.refused_seal()) + + def test_s11_the_record_cannot_be_written(self): + """A full disk while the record is staged: the .tmp is removed. A record of the run written by another process + during the seal: refused, and left as it is.""" + self.unrecorded_run("--residue-codes") + with mock.patch.object(split_run, "_write_all", side_effect=OSError(errno.ENOSPC, "No space left on device")): + message = self.refused_seal() + self.assertIn(f"The run record {self.shards.resolve() / 'emb_run.run.json'} cannot be written", message) + self.assertIn("No space left on device", message) + check = split_run.check_dealt_ids + + def racing(*args, **kwargs): + seconds = check(*args, **kwargs) + (self.shards / "emb_run.run.json").write_text("another process\n") + return seconds + + with mock.patch.object(split_run, "check_dealt_ids", side_effect=racing): + message = self.refused(self.seal(fasta=self.fasta)) + self.assertIn("File exists", message) + self.assertEqual((self.shards / "emb_run.run.json").read_text(), "another process\n") + self.assertEqual(sorted(p.name for p in self.shards.iterdir()), ["emb_run-0.parquet", "emb_run.run.json"]) + + +# ===================================================================================================================== +# P9.I1, P9.I2: the launch checks of step 2 and the seal, before torch, and the command line in their advice +# P9.L2: their later refusals, before torch too +# ===================================================================================================================== +#: A sitecustomize for the commands that ``_HookedCommands`` runs: any import of torch, Lightning, ESM, FAISS, +#: transformers or huggingface_hub ends the process at once with status 97, naming the module and the innermost +#: FoldMatch function that imported it, so that no ``except`` can hide it; one of them already imported when the +#: interpreter starts (by a ``.pth`` file) ends it with status 98. +_NO_HEAVY_IMPORTS = r""" +import os, sys +_BLOCKED = ("torch", "lightning", "lightning_fabric", "pytorch_lightning", "esm", "faiss", "transformers", + "huggingface_hub") +_loaded = sorted(name for name in sys.modules if name.partition(".")[0] in _BLOCKED) +if _loaded: + sys.stderr.write(f"FORBIDDEN IMPORT at start-up: {', '.join(_loaded)}\n") + sys.stderr.flush() + os._exit(98) +class _NoHeavyImports: + def find_spec(self, name, path=None, target=None): + if name.partition(".")[0] in _BLOCKED: + frame = sys._getframe(1) + while frame is not None and not frame.f_globals.get("__name__", "").startswith("foldmatch"): + frame = frame.f_back + where = f" (from {frame.f_globals['__name__']}.{frame.f_code.co_name})" if frame is not None else "" + sys.stderr.write(f"FORBIDDEN IMPORT: {name}{where}\n") + sys.stderr.flush() + os._exit(97) + return None +sys.meta_path.insert(0, _NoHeavyImports()) +""" +#: The console script that pip and uv install for a ``[project.scripts]`` entry point. +_CONSOLE_SCRIPT = """\ +#!{python} +import sys +from {module} import {attr} +if __name__ == '__main__': + sys.argv[0] = sys.argv[0].removesuffix('.exe') + sys.exit({attr}()) +""" + + +def _fm_search_entry_point() -> str: + """``fm-search``'s entry point, from ``pyproject.toml`` when the tests run in a source tree.""" + path = Path(_SRC).parent / "pyproject.toml" + if not path.is_file(): + return "foldmatch.cli.search:main" + import tomllib + return tomllib.loads(path.read_text())["project"]["scripts"]["fm-search"] + + +class _HookedCommands: + """``fm-search`` commands as subprocesses in ``self.tmp``, through an ``fm-search`` console script of the + package's entry point or through ``python -m foldmatch.cli.search``, with ``_NO_HEAVY_IMPORTS`` unless ``hook`` + is false. ``hooked_setup`` needs ``self.tmp`` and the environment the commands get, set.""" + + def hooked_setup(self): + for name in ("site", "bin"): + (self.tmp / name).mkdir() + (self.tmp / "site" / "sitecustomize.py").write_text(_NO_HEAVY_IMPORTS) + module, _, attr = _fm_search_entry_point().partition(":") + self.fm_search = self.tmp / "bin" / "fm-search" + self.fm_search.write_text(_CONSOLE_SCRIPT.format(python=sys.executable, module=module, attr=attr)) + self.env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and k not in _JOB_ENV + and not k.startswith(_DDP_ENV_PREFIXES)} + self.env["HF_HUB_OFFLINE"] = "1" + + @staticmethod + def program(entry) -> list: + """The command line of ``entry`` (``"fm-search"`` or ``"python -m"``) before the arguments.""" + return [sys.executable, "-m", "foldmatch.cli.search"] if entry == "python -m" else ["fm-search"] + + def command_env(self, hook=True, **env) -> dict: + """The environment of the commands, with ``env`` added, and with the hook unless ``hook`` is false.""" + path = ([str(self.tmp / "site")] if hook else []) + [_SRC] + \ + [p for p in [os.environ.get("PYTHONPATH")] if p] + return {**self.env, "PYTHONPATH": os.pathsep.join(path), **env} + + def run_command(self, entry, args, hook=True, **env): + """``args`` through ``entry`` (``"fm-search"`` or ``"python -m"``) in ``self.tmp``, with ``env`` added, and with + the hook unless ``hook`` is false.""" + program = [sys.executable, str(self.fm_search)] if entry == "fm-search" else self.program(entry) + return subprocess.run([*program, *map(str, args)], env=self.command_env(hook, **env), cwd=self.tmp, + capture_output=True, text=True, timeout=600) + + def in_this_folder(self, entry) -> list: + """What the I3 advice's command starts with when run through ``entry`` in ``self.tmp`` with absolute paths: a + cd to ``self.tmp``, except through the console script (P9.I2).""" + return [] if entry == "fm-search" else ["cd", os.path.realpath(self.tmp), "&&"] + + def assert_refused_before_torch(self, result, start, logged=0, command=False): + """Exit status 1 with ``logged`` log lines on stderr, then ``Error: `` and the refusal, which starts with + ``start``: one line, or with ``command`` that line and the command line the launch checks' advice ends with, on + a line of its own indented by four spaces (P9.M). Returns the refusal's lines, the command line without its + indent.""" + self.assertNotIn("FORBIDDEN IMPORT", result.stderr) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stdout, "") + lines = result.stderr.split("\n") + self.assertEqual(len(lines), logged + 2 + command, result.stderr) + self.assertTrue(lines[logged].startswith(f"Error: {start}"), result.stderr) + self.assertEqual(lines[-1], "", result.stderr) + if not command: + return [lines[logged]] + self.assertRegex(lines[logged + 1], "^ [^ ]", result.stderr) + return [lines[logged], lines[logged + 1][4:]] + + +class LaunchChecksBeforeTorchTest(_HookedCommands, unittest.TestCase): + """``build sequences-index`` and ``build sequences-seal`` as commands, through an ``fm-search`` console script of + the package's entry point and through ``python -m foldmatch.cli.search``, with ``_NO_HEAVY_IMPORTS``: I2 and I3 + refuse with exit status 1 and their whole message on stderr, before torch or Lightning is imported (P8b, N3 case + E: I3 came 118 s after the launch, after ``set_log_level``'s Lightning import). Without a launch refusal they + refuse the empty shards folder, still before torch (P9.L2). The WARNING of a one-task srun step of a multi-node + job, without the hook, comes in the logging format ``set_log_level`` sets up.""" + + I3_ENV = {"SLURM_JOB_ID": "4711", "SLURM_JOB_NUM_NODES": "2", "SLURM_NTASKS": "8"} + SRUN_STEP_ENV = {"SLURM_JOB_ID": "42", "SLURM_JOB_NUM_NODES": "2", "SLURM_STEP_ID": "3", + "SLURM_STEP_NUM_TASKS": "1", "SLURM_NTASKS": "8"} + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + (self.tmp / "shards").mkdir() + self.hooked_setup() + (self.tmp / "in put").mkdir() + self.fasta = Path(shutil.copyfile(_FASTA, self.tmp / "in put" / "it's seqs.fasta")) + self.db = self.tmp / "out" / "db" + + def commands(self) -> dict: + return {"build sequences-index": _index_args(self.fasta, self.db, self.tmp / "shards"), + "build sequences-seal": _seal_args(self.fasta, self.tmp / "shards", "run")} + + def assert_nothing_written(self): + self.assertFalse(self.db.parent.exists()) + self.assertEqual(list((self.tmp / "shards").iterdir()), []) + + def test_i3_before_torch(self): + for entry in ("fm-search", "python -m"): + for command, args in self.commands().items(): + with self.subTest(entry=entry, command=command): + result = self.run_command(entry, args, **self.I3_ENV) + error, line = self.assert_refused_before_torch( + result, f"SLURM job 4711 holds 2 nodes, and '{command}' runs in one process on one of them: " + f"the others would stay allocated until it ends. Run it as its own one-node CPU job " + f"instead: ", command=True) + self.assertIn("--dependency=afterok:$SLURM_JOB_ID --kill-on-invalid-dep=yes", error) + self.assertNotIn("afterok:4711", result.stderr) + # P9.I2: the command to run in the one-node job, last, as it was invoked, shell-quoted; P9.M: on + # a line of its own. + self.assertTrue(error.endswith(f" SCRIPT runs this command, without srun, next time " + f"{_NEXT_BUILD}:"), error) + self.assertEqual(shlex.split(line), + [*self.in_this_folder(entry), *self.program(entry), *map(str, args)]) + self.assert_nothing_written() + + def test_the_command_runs_from_another_folder(self): + """P9.I2: with relative paths, the I3 refusal's command starts with a cd to the folder the command ran in, so + that it runs from any folder (sbatch starts the one-node job in the folder it is submitted from). From another + folder it passes the paths' checks and refuses the empty shards folder (P9.L2: still before torch); without + the cd it is refused there with exit status 2. The I2 refusal's command, for the same batch script, has no + cd.""" + self.fm_search.chmod(0o755) + elsewhere = self.tmp / "jobs" + elsewhere.mkdir() + env = self.command_env(PATH=os.pathsep.join([str(self.tmp / "bin"), os.environ.get("PATH", "")])) + relative = {"build sequences-index": _index_args("in put/it's seqs.fasta", "out/db", "shards"), + "build sequences-seal": _seal_args("in put/it's seqs.fasta", "shards", "run")} + for command, args in relative.items(): + with self.subTest(command=command): + result = self.run_command("fm-search", args, **self.I3_ENV) + _, line = self.assert_refused_before_torch( + result, f"SLURM job 4711 holds 2 nodes, and '{command}' runs ", command=True) + self.assertEqual(shlex.split(line), ["cd", os.path.realpath(self.tmp), "&&", "fm-search", *args]) + rerun = subprocess.run(["/bin/sh", "-c", line], cwd=elsewhere, env=env, capture_output=True, text=True, + timeout=600) + self.assert_refused_before_torch(rerun, "No run in " if "index" in command else + "No embedding shards of run run ") + without_cd = subprocess.run(["/bin/sh", "-c", line.split(" && ", 1)[1]], cwd=elsewhere, env=env, + capture_output=True, text=True, timeout=600) + self.assertEqual(without_cd.returncode, 2, without_cd.stderr) + self.assertIn("Invalid value for '--fasta-file'", without_cd.stderr) + i2 = self.run_command("fm-search", args, WORLD_SIZE="2") + _, line = self.assert_refused_before_torch(i2, f"'{command}' runs as one process (WORLD_SIZE=2): ", + command=True) + self.assertEqual(shlex.split(line), ["fm-search", *args]) + self.assert_nothing_written() + self.assertEqual(list(elsewhere.iterdir()), []) + + def test_i2_before_torch(self): + for command, args in self.commands().items(): + for env, launch in (({"WORLD_SIZE": "2"}, "WORLD_SIZE=2"), + ({"SLURM_JOB_ID": "7", "SLURM_STEP_ID": "0", "SLURM_STEP_NUM_TASKS": "4", + "SLURM_JOB_NUM_NODES": "2"}, "SLURM_STEP_NUM_TASKS=4")): + with self.subTest(command=command, env=env): + result = self.run_command("fm-search", args, **env) + error, line = self.assert_refused_before_torch(result, "", command=True) + self.assertEqual(error, f"Error: '{command}' runs as one process ({launch}): launch it once, " + f"directly in the batch script or with srun -n 1. The command:") + self.assertEqual(shlex.split(line), ["fm-search", *map(str, args)]) + self.assert_nothing_written() + + def test_without_a_launch_refusal(self): + """Without a launch refusal, each command, through each entry, goes on and refuses the empty shards folder, + still before torch or Lightning is imported (P9.L2; before it, set_log_level's Lightning import came first: + ``LaterRefusalsBeforeTorchTest`` has the hook's control).""" + starts = {"build sequences-index": f"No run in {os.path.realpath(self.tmp / 'shards')}: no run record", + "build sequences-seal": "No embedding shards of run run (emb_run-.parquet) in "} + for entry in ("fm-search", "python -m"): + for command, args in self.commands().items(): + with self.subTest(entry=entry, command=command): + self.assert_refused_before_torch(self.run_command(entry, args), starts[command]) + self.assert_nothing_written() + + def test_the_srun_step_warning_in_the_logging_format(self): + """Without the hook: the WARNING of a one-task srun step of a multi-node job, in the format of --log-level info, + warn (P9.L1: it had no format) and debug; then the empty shards folder is refused.""" + detailed = r"> \d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} - \[proc-0\] - WARNING - search\.py:\d+ - " + formats = {"info": r"\[proc-0\] ", "warn": detailed, "debug": detailed} + for command, args in self.commands().items(): + for level, prefix in formats.items(): + with self.subTest(command=command, level=level): + result = self.run_command("fm-search", [*args, "--log-level", level], hook=False, + **self.SRUN_STEP_ENV) + self.assertEqual(result.returncode, 1, result.stderr) + stderr = result.stderr.splitlines() + lines = [k for k, line in enumerate(stderr) if "one-task srun step" in line] + self.assertEqual(len(lines), 1, result.stderr) + warning = stderr[lines[0]] + self.assertRegex(warning, "^" + prefix + re.escape( + f"'{command}' runs in a one-task srun step of SLURM job 42, which holds 2 nodes: the others " + f"stay allocated until it ends. Next time, run it as its own one-node CPU job: ")) + self.assertTrue(warning.endswith(f" SCRIPT runs this command, without srun, {_NEXT_BUILD}:"), + warning) + # P9.M: the command on the next line, indented, without the log line's prefix. + self.assertEqual(stderr[lines[0] + 1], + " " + shlex.join(["fm-search", *map(str, args), "--log-level", level])) + self.assertEqual(len([line for line in result.stderr.splitlines() if line.startswith("Error: ")]), + 1, result.stderr) + + + +class LaterRefusalsBeforeTorchTest(_HookedCommands, _Sealing): + """P9.L2: the refusals of step 2 up to its index phase, and every refusal of the seal, come before torch or + Lightning is imported, through the ``fm-search`` console script with ``_NO_HEAVY_IMPORTS`` (before, they came + after ``set_log_level``'s Lightning import: minutes from a cold file system on a compute node). The control: with + nothing to refuse, step 2 reaches its deferred Lightning import after its last check before the index phase; the + seal seals without any of them.""" + + def setUp(self): + super().setUp() + self.hooked_setup() + self.inputs = self.tmp / "in" + self.inputs.mkdir() + self.fasta = Path(shutil.copyfile(_FASTA, self.inputs / "seqs.fasta")) + + def refused_hooked(self, args, text): + """``args`` refused with exit status 1 before torch, Lightning, ESM, FAISS, transformers or huggingface_hub + was imported: log lines, then the ``Error:`` line, which holds ``text``. Returns the log lines.""" + result = self.run_command("fm-search", args) + self.assertNotIn("FORBIDDEN IMPORT", result.stderr) + self.assertEqual((result.returncode, result.stdout), (1, ""), result.stderr) + *logged, error, end = result.stderr.split("\n") + self.assertEqual(end, "", result.stderr) + self.assertTrue(error.startswith("Error: "), result.stderr) + self.assertIn(text, error) + for line in logged: + self.assertTrue(line.startswith("[proc-0] "), result.stderr) + return logged + + def test_step_2_refusals(self): + """I4 (no run record: an empty folder, a run whose record is gone), I6, I8, I11, I12 (inside the build lock), + I16 and I17 (a FASTA that is not the one the run embedded: another size, one residue changed).""" + self.embed(fasta=self.fasta) + unrecorded = self.tmp / "unrecorded" + unrecorded.mkdir() + self.embed(fasta=self.fasta, shards=unrecorded, db=self.tmp / "other").unlink() + empty = self.tmp / "empty" + empty.mkdir() + larger = _write(self.inputs / "larger.fasta", self.fasta.read_text() + ">late\nACDE\n") + changed = _one_residue_changed(shutil.copyfile(self.fasta, self.inputs / "changed.fasta")) + lock = Path(f"{self.db}.build.lock") + cases = [ + ("I4", _index_args(self.fasta, self.db, empty), "No run in ", None), + ("I4", _index_args(self.fasta, self.db, unrecorded), "has no run record", None), + ("I6", _index_args(self.fasta, self.db, self.shards, "--run-id", "nope"), "Run nope has no run record", + None), + ("I8", _index_args(self.fasta, self.db, self.shards, "--residue-codes"), "computed no residue codes", None), + ("I11", _index_args(self.fasta, self.db, self.shards), f"{lock} exists", lock), + ("I12", _index_args(self.fasta, self.db, self.shards), "Database already exists at", + Path(f"{self.db}.index")), + ("I16", _index_args(larger, self.db, self.shards), "bytes, but the FASTA run run embedded", None), + ("I17", _index_args(changed, self.db, self.shards), "is not the FASTA run run embedded", None), + ] + for case, args, text, present in cases: + with self.subTest(case=case, text=text): + if present is not None: + present.write_text("{}\n") + try: + self.refused_hooked(args, text) + finally: + if present is not None: + present.unlink() + self.assertEqual(sorted(p.name for p in self.out.iterdir()), []) + + def test_seal_refusals(self): + """S3 (a run with a record), S4 (no shards of the run), S7 (nothing to build) and S8 (a FASTA with more + entries than the shards hold rows).""" + self.embed(fasta=self.fasta) + unrecorded = self.tmp / "unrecorded" + unrecorded.mkdir() + self.embed(fasta=self.fasta, shards=unrecorded, db=self.tmp / "other").unlink() + larger = _write(self.inputs / "larger.fasta", self.fasta.read_text() + ">late\nACDEFGHIKL\n") + cases = [ + ("S3", _seal_args(self.fasta, self.shards, "run"), "has its run record already"), + ("S4", _seal_args(self.fasta, unrecorded, "nope"), "No embedding shards of run nope"), + ("S7", _seal_args(self.fasta, unrecorded, "run", "--min-res-n", "1000"), "there is nothing to build"), + ("S8", _seal_args(larger, unrecorded, "run", "--min-res-n", "0"), "hold 2 row(s), but"), + ] + for case, args, text in cases: + with self.subTest(case=case): + self.refused_hooked(args, text) + self.assertEqual(sorted(p.name for p in unrecorded.iterdir()), ["emb_run-0.parquet"]) + + def test_the_seal_imports_none_of_them(self): + """A plain run and a residue-code run sealed with the hook: exit status 0, the record written.""" + for codes in (False, True): + with self.subTest(codes=codes): + shards = self.tmp / f"unrecorded-{codes}" + shards.mkdir() + self.embed(*_codes_flag(codes), fasta=self.fasta, shards=shards, db=self.tmp / f"db-{codes}").unlink() + args = _seal_args(self.fasta, shards, "run", *(() if codes else ("--min-res-n", "0"))) + result = self.run_command("fm-search", args) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("FORBIDDEN IMPORT", result.stderr) + self.assertIn("[proc-0] Run run sealed: 2 entries in 1 shard(s) of 1 rank(s)", result.stderr) + self.assertTrue(split_run.read_run_record(shards / "emb_run.run.json")["sealed"]) + + def test_the_hook_stops_step_2_at_its_deferred_lightning_import(self): + """The control: with nothing to refuse, step 2 logs its last line before the index phase, then imports + Lightning in set_log_level's second call, which the hook stops; without the hook it builds the database.""" + self.embed(fasta=self.fasta) + args = _index_args(self.fasta, self.db, self.shards) + result = self.run_command("fm-search", args) + self.assertEqual(result.returncode, 97, result.stderr) + *_, building, forbidden, end = result.stderr.split("\n") + self.assertTrue(building.startswith(f"[proc-0] Building {self.db} from run run in "), result.stderr) + self.assertEqual((forbidden, end), + ("FORBIDDEN IMPORT: lightning (from foldmatch.cli.args_utils._import_lightning)", "")) + Path(f"{self.db}.build.lock").unlink() # the hook ended the process inside the lock + result = self.run_command("fm-search", args, hook=False) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"[proc-0] Database {self.db} built from run run in ", result.stderr) + + def test_set_log_level_and_the_build_time(self): + """In process: step 2 sets up logging without Lightning, then again before EmbeddingComputer, which imports + it; the build's time leaves that import out, as before. The seal sets up logging once, without Lightning.""" + self.embed(fasta=self.fasta) + calls = [] + + def slow_import(level, defer_lightning=False): + calls.append((level, defer_lightning)) + if not defer_lightning: + time.sleep(1.5) + + with mock.patch.object(search, "set_log_level", side_effect=slow_import), \ + self.assertLogs(level="INFO") as logs: + self.built(fasta=self.fasta) + self.assertEqual(calls, [(LogLevel.info, True), (LogLevel.info, False)]) + [built] = [r.getMessage() for r in logs.records if " built from run run in " in r.getMessage()] + self.assertLess(float(built.rsplit(" in ", 1)[1].removesuffix(" s")), 1.5, built) + calls.clear() + unrecorded = self.tmp / "unrecorded" + unrecorded.mkdir() + self.embed(fasta=self.fasta, shards=unrecorded, db=self.tmp / "other").unlink() + with mock.patch.object(search, "set_log_level", side_effect=slow_import): + self.sealed("--min-res-n", "0", fasta=self.fasta, shards=unrecorded) + self.assertEqual(calls, [(LogLevel.info, True)]) + + +class CommandLineTest(unittest.TestCase): + """P9.I2: ``_command_line``, the command line that the launch checks' advice ends with, from ``sys.argv`` and + ``sys.orig_argv``.""" + + ARGS = ["build", "sequences-index", "--fasta-file", "/data/my seqs.fasta", "--output-db", "it's/db", + "--from-shards=/scratch/$USER/shards", "--run-id", "60158326_2_0_ab12cd34", "--index-type", "ivf_pq"] + + def command_line(self, argv, orig_argv=(), command="build sequences-index", **options): + with mock.patch.object(sys, "argv", list(argv)), mock.patch.object(sys, "orig_argv", list(orig_argv)): + return search._command_line(command, **options) + + def test_the_console_script(self): + """``fm-search``, whatever path the console script ran from; every argument as given, shell-quoted.""" + for argv0 in ("/env/bin/fm-search", "fm-search", "./fm-search"): + with self.subTest(argv0=argv0): + line = self.command_line([argv0, *self.ARGS], ["/env/bin/python3.12", argv0, *self.ARGS]) + self.assertEqual(line, "fm-search build sequences-index --fasta-file '/data/my seqs.fasta' " + "--output-db 'it'\"'\"'s/db' '--from-shards=/scratch/$USER/shards' --run-id " + "60158326_2_0_ab12cd34 --index-type ivf_pq") + self.assertEqual(shlex.split(line), ["fm-search", *self.ARGS]) + seal = ["build", "sequences-seal", "--fasta-file", "f.fasta", "--from-shards", "s", "--run-id", "r"] + self.assertEqual(self.command_line(["/env/bin/fm-search", *seal], command="build sequences-seal"), + "fm-search " + shlex.join(seal)) + + def test_python_m(self): + """``python -m foldmatch.cli.search``, with the interpreter and its options as they were typed.""" + program = ["python3", "-X", "importtime", "-m", "foldmatch.cli.search"] + line = self.command_line([search.__file__, *self.ARGS], [*program, *self.ARGS]) + self.assertTrue(line.startswith("python3 -X importtime -m foldmatch.cli.search build sequences-index "), line) + self.assertEqual(shlex.split(line), [*program, *self.ARGS]) + + def test_other_invocations(self): + """What ``sys.orig_argv`` holds before the arguments.""" + cases = { + "the module as a script": ("src/foldmatch/cli/search.py", + ["/usr/bin/python3", "src/foldmatch/cli/search.py"]), + "python -c": ("-c", ["python", "-c", "from foldmatch.cli.search import main; main()"]), + "a wrapper script": ("/home/me/run fm.py", ["python", "/home/me/run fm.py"]), + } + for case, (argv0, program) in cases.items(): + with self.subTest(case=case): + line = self.command_line([argv0, *self.ARGS], [*program, *self.ARGS]) + self.assertEqual(shlex.split(line), [*program, *self.ARGS]) + + def test_fallbacks(self): + """``fm-search`` when ``sys.orig_argv`` does not end with the arguments (``sys.argv`` rewritten), and + ``fm-search ...`` when ``sys.argv`` does not run the command (the app called in-process).""" + line = self.command_line(["/x/search.py", *self.ARGS], ["python", "-m", "foldmatch.cli.search", "build"]) + self.assertEqual(shlex.split(line), ["fm-search", *self.ARGS]) + for argv in (["/venv/bin/pytest", "-q", "tests"], ["/env/bin/fm-search", "build", "sequences-seal"], + ["/env/bin/fm-search"], []): + with self.subTest(argv=argv): + self.assertEqual(self.command_line(argv, argv), "fm-search build sequences-index ...") + with mock.patch.object(sys, "argv", ["/venv/bin/pytest"]): + del sys.argv + self.assertEqual(search._command_line("build sequences-seal"), "fm-search build sequences-seal ...") + + def test_in_another_job(self): + """For another job, which sbatch starts in the folder it is submitted from: a cd to this process's folder first + when the line depends on it, a relative --fasta-file, --output-db or --from-shards (as ``--option PATH`` or + ``--option=PATH``) or a program other than fm-search; not when that folder is gone.""" + absolute = ["build", "sequences-index", "--fasta-file", "/data/s.fasta", "--output-db", "/db/x", + "--from-shards=/scratch/shards", "--run-id", "r1", "--index-type", "ivf_pq"] + fm_search = ["/env/bin/fm-search", *absolute] + with mock.patch.object(os, "getcwd", return_value="/scratch/my run"): + for options in ({}, {"in_another_job": True}): + self.assertEqual(self.command_line(fm_search, **options), "fm-search " + shlex.join(absolute)) + for k, value in ((3, "s.fasta"), (3, "../s.fasta"), (5, "db/x"), (5, "x"), (5, ""), (6, "--from-shards=s")): + argv = list(fm_search) + argv[k + 1] = value + with self.subTest(argv=argv): + line = self.command_line(argv, in_another_job=True) + self.assertEqual(shlex.split(line), ["cd", "/scratch/my run", "&&", "fm-search", *argv[1:]]) + self.assertTrue(line.startswith("cd '/scratch/my run' && fm-search build "), line) + self.assertEqual(self.command_line(argv), shlex.join(["fm-search", *argv[1:]])) + equals = ["/env/bin/fm-search", *absolute[:2], "--fasta-file=s.fasta", *absolute[4:]] + self.assertEqual(shlex.split(self.command_line(equals, in_another_job=True))[:3], + ["cd", "/scratch/my run", "&&"]) + program = ["/usr/bin/python3", "-m", "foldmatch.cli.search"] + self.assertEqual(shlex.split(self.command_line([search.__file__, *absolute], [*program, *absolute], + in_another_job=True)), + ["cd", "/scratch/my run", "&&", *program, *absolute]) + self.assertEqual(self.command_line(["/venv/bin/pytest"], in_another_job=True), + "fm-search build sequences-index ...") + relative = ["/env/bin/fm-search", *absolute[:3], "s.fasta", *absolute[4:]] + with mock.patch.object(os, "getcwd", side_effect=FileNotFoundError(errno.ENOENT, "No such file or directory")): + self.assertEqual(self.command_line(relative, in_another_job=True), shlex.join(["fm-search", *relative[1:]])) + + def test_a_newline(self): + """An argument holding a newline keeps it, inside its quotes: the line runs as given, but on two lines.""" + args = ["build", "sequences-index", "--output-db", "/db/a\nb"] + line = self.command_line(["/env/bin/fm-search", *args]) + self.assertEqual(line.count("\n"), 1, line) + self.assertEqual(shlex.split(line), ["fm-search", *args]) + +# ===================================================================================================================== +# real ESM3 (CPU) +# ===================================================================================================================== +class RealEsm3SplitTest(unittest.TestCase): + """The one-shot ``build sequences`` and the two steps on ``test_sequences.fasta`` with the real CPU pass at + --batch-size 2, plain and codes, with pinned run ids, one build after another.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + esm_codes_bundle.registry_files() + gc.collect() # earlier tests' ESM3 models, if only reference cycles still hold them + cls._tmp = tempfile.TemporaryDirectory() + cls.tmp = Path(cls._tmp.name) + cls.results = {} + with mock.patch.dict(os.environ): + for name in _JOB_ENV: + os.environ.pop(name, None) + for kind, extra in (("codes", ("--residue-codes",)), ("plain", ())): + for step in ("one", "split"): + (cls.tmp / f"{step}-{kind}-shards").mkdir() + with launch_env(FOLDMATCH_RUN_ID=f"one-{kind}"): + cls.results[f"one-{kind}"] = invoke(*_build_args( + _FASTA, cls.tmp / f"one-{kind}" / "db", cls.tmp / f"one-{kind}-shards", "--batch-size", "2", + *extra)) + gc.collect() # the pass's ESM3 (5.6 GB) is only reachable through reference cycles + with launch_env(FOLDMATCH_RUN_ID=f"split-{kind}"): + cls.results[f"embed-{kind}"] = invoke(*_embed_args( + _FASTA, cls.tmp / f"split-{kind}" / "db", cls.tmp / f"split-{kind}-shards", "--batch-size", + "2", *extra)) + gc.collect() + with launch_env(), _esm3_spy(), \ + mock.patch.object(ss, "prepare", side_effect=AssertionError("bundle fetched")): + cls.results[f"index-{kind}"] = invoke(*_index_args( + _FASTA, cls.tmp / f"split-{kind}" / "db", cls.tmp / f"split-{kind}-shards")) + + @classmethod + def tearDownClass(cls): + cls._tmp.cleanup() + super().tearDownClass() + + def setUp(self): + for name, result in self.results.items(): + self.assertEqual(result.exit_code, 0, f"{name}: {result.output[-4000:]} {result.exception!r}") + + def test_two_steps_equal_the_one_shot(self): + _, ids, _, codes_ref = _f2c() + for kind in ("codes", "plain"): + with self.subTest(kind=kind): + columns = ["id", "embedding", *(("residue_codes", "seq_sha1") if kind == "codes" else ())] + tables = [pq.read_table(self.tmp / f"{step}-{kind}-shards" / f"emb_{step}-{kind}-0.parquet", + columns=columns) for step in ("one", "split")] + self.assertTrue(tables[0].equals(tables[1]), + "precondition: the one-shot's and step 1's passes wrote different shard tables") + assert_same_database(self.tmp / f"one-{kind}" / "db", self.tmp / f"split-{kind}" / "db") + if kind == "codes": + store = ResidueCodeStore(self.tmp / f"split-{kind}", "db") + manifest = store.check_compatible() + self.assertEqual((manifest["entries"], manifest["total_residues"], manifest["build"]["split"]), + (2, 303, True)) + equal, n = _bits_equal(np.asarray(store.open_memmap()), codes_ref) + print(f"\nsplit build F2c CPU code bits: {equal}/{n} = {equal / n:.6f}", file=sys.stderr) + self.assertGreaterEqual(equal / n, A4_XCPU_BITS, f"F2c code bits {equal}/{n}") + record = split_run.read_run_record(self.tmp / f"split-{kind}-shards" / f"emb_split-{kind}.run.json") + self.assertEqual((record["rows"], record["preflight"]["residues"], record["world_size"], + record["pass"]["accelerator"]), (2, 303, 1, "cpu" if kind == "codes" else None)) + + +# ===================================================================================================================== +# two real ranks as commands (slow) +# ===================================================================================================================== +@unittest.skipUnless(os.environ.get("FOLDMATCH_SLOW_TESTS") == "1", + "slow: two ESM3 ranks on CPU, twice; set FOLDMATCH_SLOW_TESTS=1 to run") +class TwoRankSplitSmoke(unittest.TestCase): + """``build sequences-embed --residue-codes --devices 2`` as a command (CPU, gloo: Lightning re-runs the command for + rank 1, and the real ``all_gather_object`` runs), then ``build sequences-index``, and a two-rank one-shot ``build + sequences --residue-codes`` to compare with (pattern of ``TwoRankCliSmoke``).""" + + def test_two_rank_split_build(self): + esm_codes_bundle.registry_files() + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + for name in ("shards", "oneshot-shards", "site", "log-embed", "log-oneshot", "empty-hf-cache"): + (tmp / name).mkdir() + (tmp / "site" / "sitecustomize.py").write_text(_SITECUSTOMIZE) + env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and k not in _JOB_ENV + and not k.startswith(_DDP_ENV_PREFIXES)} + env["PYTHONPATH"] = os.pathsep.join([str(tmp / "site"), _SRC] + + [p for p in [os.environ.get("PYTHONPATH")] if p]) + if sys.platform == "darwin": + env["OMP_NUM_THREADS"] = "1" + + def run(args, **extra_env): + command = [sys.executable, "-m", "foldmatch.cli.search", *map(str, args)] + if "--devices" in command: + command[command.index("--devices") + 1] = "2" + return subprocess.run(command, env={**env, **extra_env}, capture_output=True, text=True, + timeout=1800) + + split_db, oneshot_db = tmp / "split" / "db", tmp / "oneshot" / "db" + embed = run(_embed_args(_FASTA, split_db, tmp / "shards", "--residue-codes"), + **{_TWO_RANK_CHILD_ENV: str(tmp / "log-embed")}) + self.assertEqual(embed.returncode, 0, embed.stderr[-4000:]) + records = sorted((tmp / "shards").glob("*.run.json")) + self.assertEqual(len(records), 1, sorted(p.name for p in (tmp / "shards").iterdir())) + record = split_run.read_run_record(records[0]) + calls = {p.name: json.loads(p.read_text()) for p in (tmp / "log-embed").iterdir()} + + refused = run(_index_args(_FASTA, split_db, tmp / "shards"), WORLD_SIZE="2") + self.assertEqual(refused.returncode, 1, refused.stderr[-4000:]) + self.assertIn("runs as one process (WORLD_SIZE=2)", refused.stderr) + self.assertFalse(split_db.parent.exists()) + + index = run(_index_args(_FASTA, split_db, tmp / "shards"), HF_HUB_OFFLINE="1", + HF_HUB_CACHE=str(tmp / "empty-hf-cache")) + self.assertEqual(index.returncode, 0, index.stderr[-4000:]) + oneshot = run(_build_args(_FASTA, oneshot_db, tmp / "oneshot-shards", "--residue-codes"), + **{_TWO_RANK_CHILD_ENV: str(tmp / "log-oneshot")}) + self.assertEqual(oneshot.returncode, 0, oneshot.stderr[-4000:]) + tables = [[pq.read_table(path).select(["id", "embedding", "residue_codes", "seq_sha1"]) + for path in sorted(folder.glob("*.parquet"))] for folder in (tmp / "oneshot-shards", + tmp / "shards")] + same_shards = len(tables[0]) == len(tables[1]) and all(a.equals(b) for a, b in zip(*tables)) + self.assertTrue(same_shards, "precondition: the two passes wrote different shard tables") + assert_same_database(oneshot_db, split_db) + manifest = ResidueCodeStore(split_db.parent, "db").check_compatible() + self.assertEqual(calls, {"prepare-0.json": {"local_files_only": False}, + "prepare-1.json": {"local_files_only": True}}) + self.assertEqual((record["world_size"], len(record["shards"]), record["rows"]), (2, 2, 2)) + self.assertEqual((manifest["entries"], manifest["build"]["devices"], manifest["build"]["split"]), (2, 2, True)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_clustering.py b/tests/test_clustering.py new file mode 100644 index 0000000..bece9d9 --- /dev/null +++ b/tests/test_clustering.py @@ -0,0 +1,1401 @@ +"""EmbeddingClusterer.build_similarity_graph: blocks of chains read with reconstruct_n and searched many at a time give, +on every database type, the expected graph: on hnsw the one-query-per-call graph bit for bit, on flat the same up to +float rounding, and on ivf_pq the normalised decoded query against each code. Also cluster_leiden on python-igraph's +Leiden (P9.U): the call, its private seeding, the cluster ids by size, and its refusals.""" +import hashlib +import logging +import math +import os +import random +import re +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +import weakref +import xml.etree.ElementTree as ET +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +# As the commands run (set_log_level): Lightning first, which brings torch and, through scikit-learn, +# KMP_DUPLICATE_LIB_OK; on macOS, FAISS's OpenMP regions abort a process that loaded torch's libomp without it. +import lightning # noqa: F401 +import faiss +import igraph as ig +import numpy as np +from tqdm import tqdm +from typer.testing import CliRunner + +import foldmatch +from foldmatch.cli import search +from foldmatch.cli.search import cluster_database, similarity_graph +from foldmatch.search import clustering +from foldmatch.search.clustering import EmbeddingClusterer +from foldmatch.search.faiss_database import FaissEmbeddingDatabase +from foldmatch.types.api_types import FormatMode, IndexConfig, IndexType + + +def _clustered_vectors(seed: int, n: int, d: int, n_centers: int, noise: float) -> np.ndarray: + """Unit vectors around ``n_centers`` random centres, at a noise level of 0.5 to 1.5 times ``noise`` each, so scores + spread over the thresholds; the members of a centre are contiguous, so the last chains have edges among + themselves.""" + rng = np.random.default_rng(seed) + centers = rng.standard_normal((n_centers, d)) + centers /= np.linalg.norm(centers, axis=1, keepdims=True) + members = np.sort(rng.integers(0, n_centers, n)) + levels = noise * rng.uniform(0.5, 1.5, (n, 1)) + vectors = centers[members] + levels * rng.standard_normal((n, d)) / math.sqrt(d) + vectors /= np.linalg.norm(vectors, axis=1, keepdims=True) + return vectors.astype(np.float32) + + +def _build(folder: Path, name: str, vectors: np.ndarray, index_type: IndexType, config: IndexConfig = None) -> str: + db = FaissEmbeddingDatabase(folder, name) + db.create_database(iter([([f"chain{i}" for i in range(len(vectors))], vectors)]), index_type=index_type, + index_config=config) + return str(folder / name) + + +def _old_way(index, n: int, threshold: float, max_neighbors: int): + """The graph build of FoldMatch 875b514: one stored vector per FAISS call, searched as it is, and an edge (i, j) + from the search of i when i < j. Returns the ordered edges, their weights, and each chain's k-th score and id.""" + k = min(max_neighbors, n) + edges, weights, kth, kth_ids = [], [], [], [] + for i in range(n): + query = np.ascontiguousarray(index.reconstruct(i), dtype=np.float32).reshape(1, -1) + scores, ids = index.search(query, k) + kth.append(float(scores[0, -1])) + kth_ids.append(int(ids[0, -1])) + for j, score in zip(ids[0], scores[0]): + if i < j and score >= threshold: + edges.append((i, int(j))) + weights.append(float(score)) + return edges, weights, kth, kth_ids + + +def _budget(rows: int, k: int) -> int: + """The result budget that gives blocks of ``rows`` chains at ``k`` neighbours.""" + return rows * k * 12 + + +class _Recorder: + """The database index, recording the rows read and searched in each call.""" + + def __init__(self, index): + self._index = index + self.reads = [] + self.searches = [] + + def reconstruct_n(self, start, n): + self.reads.append((start, n)) + return self._index.reconstruct_n(start, n) + + def search(self, x, k): + self.searches.append((x.copy(), k)) + return self._index.search(x, k) + + def __getattr__(self, name): + return getattr(self._index, name) + + +class _GraphCase(unittest.TestCase): + + def graph(self, db_path: str, threshold: float, max_neighbors: int, rows: int = None, k: int = None): + clusterer = EmbeddingClusterer(db_path) + if rows is None: + graph = clusterer.build_similarity_graph(threshold=threshold, max_neighbors=max_neighbors) + else: + with mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", _budget(rows, k)): + graph = clusterer.build_similarity_graph(threshold=threshold, max_neighbors=max_neighbors) + self.assertEqual(graph.vcount(), len(clusterer.chain_ids)) + self.assertEqual(graph.vs["name"], clusterer.chain_ids) + self.assertTrue(all(type(w) is float for w in graph.es["weight"])) + return graph.get_edgelist(), graph.es["weight"] + + def assert_same_up_to_rounding(self, got, ref, threshold, tol, kth=None): + """Same edge set as ``ref`` except pairs whose score is within ``tol`` of the threshold or of the row's k-th + score; weights of the shared edges within ``tol``.""" + got_w = dict(zip(*got)) + ref_w = dict(zip(*ref)) + self.assertEqual(len(got_w), len(got[0])) + for pair in set(got_w) ^ set(ref_w): + score = got_w.get(pair, ref_w.get(pair)) + near = abs(score - threshold) <= tol or (kth is not None and abs(score - kth[pair[0]]) <= tol) + self.assertTrue(near, f"edge {pair} score {score} only in one graph") + shared = sorted(set(got_w) & set(ref_w)) + self.assertGreater(len(shared), 0) + diff = max(abs(got_w[p] - ref_w[p]) for p in shared) + self.assertLessEqual(diff, tol) + + +class HnswGraphTest(_GraphCase): + """hnsw: the graph equals the one-query-per-call graph bit for bit, ordered edges and weights.""" + + n, d, threshold, max_neighbors = 2000, 1536, 0.6, 40 + # A --max-neighbors small enough that it, not the threshold, limits the edges of many chains. + small_k = 5 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.vectors = _clustered_vectors(seed=3, n=cls.n, d=cls.d, n_centers=150, noise=1.0) + cls.db_path = _build(Path(cls.tmp.name), "hnsw", cls.vectors, IndexType.hnsw) + db = FaissEmbeddingDatabase(Path(cls.tmp.name), "hnsw") + db.load_database() + cls.ref = _old_way(db.index, cls.n, cls.threshold, cls.max_neighbors) + cls.ref_small_k = _old_way(db.index, cls.n, cls.threshold, cls.small_k) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_fixture(self): + edges = self.ref[0] + self.assertGreater(len(edges), 1000) + self.assertNotEqual(self.n % 7, 0) + # The last short block of 7 has edges of its own. + self.assertTrue(any(i >= self.n - self.n % 7 for i, _ in edges)) + # At small_k, many chains have an edge to their k-th hit: --max-neighbors limits their edges. + _, _, kth, kth_ids = self.ref_small_k + saturated = [i for i in range(self.n) if kth_ids[i] > i and kth[i] >= self.threshold] + self.assertGreater(len(saturated), 100) + + def test_same_graph_as_one_query_at_a_time(self): + edges, weights = self.graph(self.db_path, self.threshold, self.max_neighbors) + self.assertEqual(edges, self.ref[0]) + self.assertEqual(weights, self.ref[1]) + + def test_block_sizes(self): + for rows in (1, 7, self.n + 5): + with self.subTest(rows=rows): + edges, weights = self.graph(self.db_path, self.threshold, self.max_neighbors, rows, + self.max_neighbors) + self.assertEqual(edges, self.ref[0]) + self.assertEqual(weights, self.ref[1]) + + def test_max_neighbors_limits_the_edges_as_before(self): + """A chain's k-th hit, when above the threshold, is an edge, as in the one-query-per-call graph.""" + edges, weights = self.graph(self.db_path, self.threshold, self.small_k) + self.assertEqual(edges, self.ref_small_k[0]) + self.assertEqual(weights, self.ref_small_k[1]) + + def test_blocks_follow_the_budget_and_search_the_stored_vectors(self): + """Blocks of budget // (k * 12) chains, at most _GRAPH_MAX_ROWS, a short last one, each read once and searched + as stored.""" + for k, rows, max_rows in ((40, 7, None), (20, 14, None), (1000, 1, None), (1, 13, 13)): + with self.subTest(k=k, max_rows=max_rows): + clusterer = EmbeddingClusterer(self.db_path) + stored = clusterer.db.index.reconstruct_n(0, self.n) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + # At k = 1 the budget alone would give blocks of 280 chains: the row cap makes them 13. + with mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", _budget(7, 40)), \ + mock.patch.object(clustering, "_GRAPH_MAX_ROWS", max_rows or clustering._GRAPH_MAX_ROWS): + clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=k) + starts = list(range(0, self.n, rows)) + expected = [(s, min(rows, self.n - s)) for s in starts] + self.assertEqual(recorder.reads, expected) + self.assertEqual([(len(x), kk) for x, kk in recorder.searches], [(m, k) for _, m in expected]) + searched = np.concatenate([x for x, _ in recorder.searches]) + self.assertTrue(np.array_equal(searched, stored)) + # Normalising the stored vectors again would change them: the comparison above can see it. + renormalised = stored.copy() + faiss.normalize_L2(renormalised) + self.assertFalse(np.array_equal(renormalised, stored)) + + def test_max_neighbors_0_is_refused_after_one_chain(self): + """FAISS refuses k = 0, as before, after one chain is read: not after a block of the budget's size.""" + clusterer = EmbeddingClusterer(self.db_path) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + with self.assertRaises(AssertionError): + clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=0) + self.assertEqual(recorder.reads, [(0, 1)]) + + +class FlatGraphTest(_GraphCase): + """flat across FAISS's BLAS switch: the same graph up to float rounding.""" + + n, d, threshold, max_neighbors = 1500, 256, 0.6, 30 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.vectors = _clustered_vectors(seed=5, n=cls.n, d=cls.d, n_centers=60, noise=1.0) + cls.db_path = _build(Path(cls.tmp.name), "flat", cls.vectors, IndexType.flat) + db = FaissEmbeddingDatabase(Path(cls.tmp.name), "flat") + db.load_database() + cls.ref = _old_way(db.index, cls.n, cls.threshold, cls.max_neighbors) + cls.ref_all = _old_way(db.index, cls.n, cls.threshold, cls.n) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_fixture(self): + self.assertGreater(len(self.ref[0]), 1000) + # One-query searches stay below the BLAS switch; one block of every chain is above it. + self.assertLess(self.d, faiss.cvar.distance_compute_blas_threshold) + self.assertGreaterEqual(self.n * self.d, faiss.cvar.distance_compute_blas_threshold) + self.assertGreater(clustering._GRAPH_RESULT_BYTES // (self.max_neighbors * 12), self.n) + self.assertGreater(clustering._GRAPH_MAX_ROWS, self.n) + + def test_same_graph_across_the_blas_switch(self): + got = self.graph(self.db_path, self.threshold, self.max_neighbors) + self.assert_same_up_to_rounding(got, self.ref[:2], self.threshold, 1e-5, kth=self.ref[2]) + + def test_block_sizes(self): + for rows in (1, 7, self.n + 5): + with self.subTest(rows=rows): + got = self.graph(self.db_path, self.threshold, self.n, rows, self.n) + self.assert_same_up_to_rounding(got, self.ref_all[:2], self.threshold, 1e-6) + + +class IvfPqGraphTest(_GraphCase): + """ivf_pq: the graph builds without a direct map, and a score is the normalised decoded query against the code.""" + + n, d, threshold = 1000, 64, 0.5 + config = dict(nlist=16, m=8, nbits=4, nprobe=16, opq_train_size=1000) + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + vectors = _clustered_vectors(seed=7, n=cls.n, d=cls.d, n_centers=40, noise=0.8) + # Planted exact duplicates: chain 500 + t is chain 10 * t. + cls.duplicates = [(10 * t, 500 + t) for t in range(20)] + for a, b in cls.duplicates: + vectors[b] = vectors[a] + cls.db_path = _build(Path(cls.tmp.name), "pq", vectors, IndexType.ivf_pq, IndexConfig(**cls.config)) + db = FaissEmbeddingDatabase(Path(cls.tmp.name), "pq") + db.load_database() + cls.decoded = db.index.reconstruct_n(0, cls.n) + cls.norms = np.linalg.norm(cls.decoded, axis=1) + # The score of chain i's search for chain j: the unit decoded query against j's code. + cls.expected = (cls.decoded / cls.norms[:, None]) @ cls.decoded.T + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_cluster_and_similarity_graph_run(self): + out = Path(self.tmp.name) + cluster_database(db_path=self.db_path, threshold=self.threshold, output_file=str(out / "clusters.tsv"), + seed=1) + rows = (out / "clusters.tsv").read_text().splitlines() + self.assertEqual([r.split("\t")[0] for r in rows], [f"chain{i}" for i in range(self.n)]) + similarity_graph(db_path=self.db_path, threshold=self.threshold, output_file=str(out / "graph.graphml")) + nodes = ET.parse(out / "graph.graphml").getroot().findall(".//g:node", + {"g": "http://graphml.graphdrawing.org/xmlns"}) + self.assertEqual(len(nodes), self.n) + + def test_weights_are_the_unit_query_against_the_code(self): + edge_list, weights = self.graph(self.db_path, self.threshold, self.n) + edges = np.array(edge_list) + self.assertGreater(len(edges), 1000) + self.assertTrue(np.all(edges[:, 0] < edges[:, 1])) + self.assertTrue(np.all(np.diff(edges[:, 0]) >= 0)) + np.testing.assert_allclose(weights, self.expected[edges[:, 0], edges[:, 1]], rtol=0, atol=1e-5) + # Every chain is scanned (nprobe = nlist, k = n): the edges are the pairs above the threshold. + i, j = np.nonzero(np.triu(self.expected >= self.threshold, k=1)) + ref = (list(zip(i.tolist(), j.tolist())), self.expected[i, j].tolist()) + self.assert_same_up_to_rounding((edge_list, weights), ref, self.threshold, 1e-5) + + def test_duplicates_score_the_code_norm(self): + got = dict(zip(*self.graph(self.db_path, self.threshold, self.n))) + checked = 0 + for a, b in self.duplicates: + # Same code; the reverse OPQ product's last bits can depend on the row's position (OpenBLAS threads). + np.testing.assert_allclose(self.decoded[a], self.decoded[b], rtol=0, atol=1e-6) + norm = float(self.norms[b]) + if norm < self.threshold + 0.01 or abs(norm - norm ** 2) < 1e-3: + continue + self.assertAlmostEqual(got[(a, b)], norm, delta=1e-5) + checked += 1 + self.assertGreater(checked, 5) + + def test_no_direct_map_and_parallel_mode_3(self): + clusterer = EmbeddingClusterer(self.db_path) + clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=50) + ivf = faiss.extract_index_ivf(clusterer.db.index) + self.assertEqual(ivf.direct_map.type, faiss.DirectMap.NoMap) + self.assertEqual(ivf.parallel_mode, 3) + + def test_fill_ids_make_no_edges(self): + """With nprobe < nlist and max_neighbors >= n, FAISS fills rows with -1: no edge comes from them.""" + clusterer = EmbeddingClusterer(self.db_path) + clusterer.db.set_nprobe(1) + queries = self.decoded.copy() + faiss.normalize_L2(queries) + scores, ids = clusterer.db.index.search(queries, self.n) + self.assertTrue((ids == -1).any()) + graph = clusterer.build_similarity_graph(threshold=-10.0, max_neighbors=self.n + 5) + rows, cols = np.nonzero(np.arange(self.n)[:, None] < ids) + self.assertEqual(graph.get_edgelist(), list(zip(rows.tolist(), ids[rows, cols].tolist()))) + self.assertEqual(graph.es["weight"], scores[rows, cols].tolist()) + self.assertGreaterEqual(min(min(e) for e in graph.get_edgelist()), 0) + + def test_block_sizes(self): + """The same graph up to float rounding: the OPQ rotation is a BLAS product, whose last bits can depend on the + number of rows (a block of 1 against more on OpenBLAS; blocks of 1, 2 and 7 against 64 or more on Apple + Accelerate).""" + ref = self.graph(self.db_path, self.threshold, self.n) + for rows in (1, 2, 7, self.n + 5): + with self.subTest(rows=rows): + got = self.graph(self.db_path, self.threshold, self.n, rows, self.n) + self.assert_same_up_to_rounding(got, ref, self.threshold, 1e-6) + + +class EdgeCaseTest(_GraphCase): + + n, d = 60, 16 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + folder = Path(cls.tmp.name) + # Six clusters in the first 8 dimensions, and six chains along the last 8 axes that score 0 with every other + # chain: singletons. + cls.vectors = np.zeros((cls.n, cls.d), dtype=np.float32) + cls.vectors[:54, :8] = _clustered_vectors(seed=11, n=54, d=8, n_centers=6, noise=0.5) + cls.vectors[54:, 8:14] = np.eye(6, dtype=np.float32) + cls.db_path = _build(folder, "flat", cls.vectors, IndexType.flat) + cls.one_path = _build(folder, "one", cls.vectors[:1], IndexType.flat) + # Scores a threshold can equal exactly: 0.6 and 0.8 from chain 0, and 0 with the last chain. + cls.exact_vectors = np.array([[1, 0, 0], [0.6, 0.8, 0], [0.8, 0.6, 0], [0, 0, 1]], dtype=np.float32) + cls.exact_path = _build(folder, "exact", cls.exact_vectors, IndexType.flat) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_max_neighbors_above_n(self): + db = FaissEmbeddingDatabase(Path(self.tmp.name), "flat") + db.load_database() + ref = _old_way(db.index, self.n, 0.5, self.n + 10) + edges, weights = self.graph(self.db_path, 0.5, self.n + 10) + self.assertGreater(len(edges), 0) + self.assertEqual(edges, ref[0]) + self.assertEqual(weights, ref[1]) + + def test_score_equal_to_threshold_makes_an_edge(self): + """An edge needs score >= threshold: the axis chains score exactly 0 with every chain, so at threshold 0 they + make edges, the same as in the one-query-per-call graph.""" + db = FaissEmbeddingDatabase(Path(self.tmp.name), "flat") + db.load_database() + ref = _old_way(db.index, self.n, 0.0, self.n) + self.assertGreater(sum(w == 0.0 for w in ref[1]), 0) + edges, weights = self.graph(self.db_path, 0.0, self.n) + self.assertEqual(edges, ref[0]) + self.assertEqual(weights, ref[1]) + + def test_threshold_equal_to_a_float32_score(self): + """A float32 score equal to the threshold as a Python float makes an edge, as before (NumPy compares the + float32 scores with the threshold in float32): 0, 0.6, the score itself, and a float64 just above it that rounds + to it in float32.""" + db = FaissEmbeddingDatabase(Path(self.tmp.name), "exact") + db.load_database() + stored = db.index.reconstruct_n(0, 4) + score = float(stored[0] @ stored[1]) + for threshold in (0.0, 0.6, score, float(np.nextafter(score, 1.0))): + with self.subTest(threshold=threshold): + ref = _old_way(db.index, 4, threshold, 10) + edges, weights = self.graph(self.exact_path, threshold, 10) + self.assertEqual(edges, ref[0]) + self.assertEqual(weights, ref[1]) + self.assertIn((0, 1), edges) + if threshold == 0.0: + self.assertIn((0, 3), edges) + self.assertEqual(np.float32(np.nextafter(score, 1.0)), np.float32(score)) + + def test_one_chain(self): + clusterer = EmbeddingClusterer(self.one_path) + graph = clusterer.build_similarity_graph(threshold=0.0, max_neighbors=10) + self.assertEqual((graph.vcount(), graph.ecount()), (1, 0)) + self.assertEqual(clusterer.cluster_leiden(seed=1).tolist(), [0]) + stats = clusterer.get_cluster_statistics() + self.assertEqual((stats["n_clusters"], stats["n_singletons"]), (1, 1)) + + def test_threshold_above_every_score(self): + clusterer = EmbeddingClusterer(self.db_path) + graph = clusterer.build_similarity_graph(threshold=2.0, max_neighbors=10) + self.assertEqual((graph.vcount(), graph.ecount()), (self.n, 0)) + self.assertEqual(graph.es["weight"], []) + clusters = clusterer.cluster_leiden(seed=1) + self.assertEqual(clusters.tolist(), list(range(self.n))) + stats = clusterer.get_cluster_statistics() + self.assertEqual((stats["n_clusters"], stats["n_singletons"], stats["max_cluster_size"]), (self.n, self.n, 1)) + self.assertTrue(math.isnan(stats["modularity"])) + clusterer.print_statistics() + + def test_statistics_and_export_as_before(self): + """Cluster sizes from np.bincount give the rows and statistics of the dict count.""" + out = Path(self.tmp.name) / "clusters.tsv" + cluster_database(db_path=self.db_path, threshold=0.5, output_file=str(out), max_neighbors=10, seed=3, + min_cluster_size=2, format_mode=FormatMode.tsv) + clusterer = EmbeddingClusterer(self.db_path) + clusterer.build_similarity_graph(threshold=0.5, max_neighbors=10) + membership = clusterer.cluster_leiden(seed=3) + sizes = {} + for cluster_id in membership: + sizes[cluster_id] = sizes.get(cluster_id, 0) + 1 + self.assertTrue(any(s == 1 for s in sizes.values()) and any(s >= 2 for s in sizes.values())) + expected = ["chain_id\tcluster_id\tcluster_size"] + [ + f"{chain}\t{c}\t{sizes[c]}" for chain, c in zip(clusterer.chain_ids, membership) if sizes[c] >= 2] + self.assertEqual(out.read_text().splitlines(), expected) + stats = clusterer.get_cluster_statistics() + values = list(sizes.values()) + self.assertEqual( + {k: stats[k] for k in ("n_chains", "n_clusters", "n_singletons", "min_cluster_size", "max_cluster_size", + "mean_cluster_size", "median_cluster_size")}, + {"n_chains": self.n, "n_clusters": len(sizes), "n_singletons": sum(1 for s in values if s == 1), + "min_cluster_size": min(values), "max_cluster_size": max(values), + "mean_cluster_size": np.mean(values), "median_cluster_size": np.median(values)}) + self.assertEqual(stats["modularity"], clusterer.graph.modularity(membership, weights="weight")) + + +# P9.Q1: the graph is built with less memory per edge (int32 ids, no concatenation, blocks released as consumed, equal +# weights sharing one float, malloc_trim on glibc), and is K2's graph bit for bit. + +def _k2_way(clusterer: EmbeddingClusterer, threshold: float, max_neighbors: int): + """K2's build_similarity_graph (c40d4e1) verbatim, on ``clusterer``: int64 blocks, np.concatenate, ig.Graph of the + ndarray and weights.tolist(). It reads the module's block budget and row cap, as K2 did.""" + if clusterer.db.index is None: + raise ValueError("Database not loaded. Call load_database() first.") + + n_chains = len(clusterer.chain_ids) + logging.info(f"\nBuilding similarity graph with threshold >= {threshold}") + logging.info(f"Searching up to {max_neighbors} neighbors per chain") + + k = min(max_neighbors, n_chains) + rows = 1 if k < 1 else max(1, min(n_chains, clustering._GRAPH_MAX_ROWS, clustering._GRAPH_RESULT_BYTES // (k * 12))) + + index_to_use = clusterer.db.index + if clusterer.db.is_gpu_index: + logging.info("Moving GPU index to CPU for reconstruction...") + index_to_use = faiss.index_gpu_to_cpu(clusterer.db.index) + + normalize = clusterer.db.index_type == IndexType.ivf_pq + + edge_blocks = [] + weight_blocks = [] + n_edges = 0 + start_time = time.perf_counter() + + logging.info("Performing k-NN search ...") + with tqdm(total=n_chains, desc="Processing chains", unit="chain") as progress: + for start in range(0, n_chains, rows): + end = min(start + rows, n_chains) + embeddings = np.ascontiguousarray(index_to_use.reconstruct_n(start, end - start), dtype=np.float32) + if normalize: + faiss.normalize_L2(embeddings) + scores, indices = clusterer.db.index.search(embeddings, k) + del embeddings + + chains = np.arange(start, end, dtype=np.int64)[:, None] + block_rows, block_cols = np.nonzero((chains < indices) & (scores >= threshold)) + edge_blocks.append(np.stack([block_rows + start, indices[block_rows, block_cols]], axis=1)) + weight_blocks.append(scores[block_rows, block_cols]) + n_edges += len(block_rows) + del scores, indices, chains, block_rows, block_cols + + progress.set_postfix(edges=n_edges, refresh=False) + progress.update(end - start) + + edges = np.concatenate(edge_blocks) if edge_blocks else np.empty((0, 2), dtype=np.int64) + weights = np.concatenate(weight_blocks) if weight_blocks else np.empty(0, dtype=np.float32) + del edge_blocks, weight_blocks + + logging.info(f"Created graph with {n_chains} nodes and {len(edges)} edges") + + clusterer.graph = ig.Graph(n=n_chains, edges=edges, directed=False) + clusterer.graph.es['weight'] = weights.tolist() + del edges, weights + + clusterer.graph.vs['name'] = clusterer.chain_ids + + logging.info(f"Similarity graph built in {time.perf_counter() - start_time:.2f}s") + + return clusterer.graph + + +def _patched(name: str, value): + """A patch of the clustering module's constant ``name``; none when ``value`` is None.""" + if value is None: + return mock.patch.object(clustering, name, getattr(clustering, name)) + return mock.patch.object(clustering, name, value) + + +def _exact_divisor(counts) -> int: + """A chunk size of at least 4 that divides one block's kept-edge count exactly, that block spanning two chunks or + more.""" + for count in sorted((int(c) for c in counts), reverse=True): + for size in range(4, count // 2 + 1): + if count % size == 0: + return size + raise AssertionError(f"no block's edge count has a divisor: {list(counts)}") + + +def _float32_span(weights) -> int: + bits = np.asarray(weights, dtype=np.float32).view(np.uint32).astype(np.int64) + return int(bits.max() - bits.min() + 1) + + +def _shares_equal_weights(weights) -> bool: + """Whether equal weights are one float object: as many objects as distinct values, and some value repeated.""" + distinct = {w.hex() for w in weights} + return len(distinct) < len(weights) and len({id(w) for w in weights}) == len(distinct) + + +class _SameGraphAsK2: + """On a fixture database, the graph equals K2's bit for bit: the same edges in the same order, the same weights by + value and by bits, Python floats, and the same names, at every block size and every chunk of edges and weights + handed to igraph (chunks of 1 and 3 edges, and chunks that divide a block's edge count exactly), with int32 ids + and with the int64 fallback.""" + + index_type = config = None + n = threshold = max_neighbors = None + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.db_path = _build(Path(cls.tmp.name), cls.index_type.value, cls.vectors(), cls.index_type, cls.config) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def budget(self, rows): + """Blocks of ``rows`` chains at the fixture's k; the default budget when ``rows`` is None.""" + return _patched("_GRAPH_RESULT_BYTES", None if rows is None else _budget(rows, min(self.max_neighbors, self.n))) + + def assert_same_list(self, got, ref, what): + """``got == ref``, reported by the first difference (a diff of a million-edge list would not finish).""" + if got != ref: + first = next((i for i, (a, b) in enumerate(zip(got, ref)) if a != b), min(len(got), len(ref))) + self.fail(f"{what} differ: lengths {len(got)} and {len(ref)}, first difference at {first}: " + f"{got[first:first + 3]} against {ref[first:first + 3]}") + + def assert_same_graph(self, graph, ref): + self.assertEqual(graph.vcount(), ref.vcount()) + self.assert_same_list(graph.get_edgelist(), ref.get_edgelist(), "edges") + weights, ref_weights = graph.es["weight"], ref.es["weight"] + self.assert_same_list(weights, ref_weights, "weights") + self.assertTrue(np.asarray(weights, dtype=np.float64).tobytes() + == np.asarray(ref_weights, dtype=np.float64).tobytes(), "weight bits differ") + self.assertTrue(all(type(w) is float for w in weights)) + self.assert_same_list(graph.vs["name"], ref.vs["name"], "names") + + def test_same_graph_as_k2(self): + clusterer = EmbeddingClusterer(self.db_path) + several = self.n // 4 + 1 + refs = {} + for rows in (None, 1, 7, several): + with self.budget(rows): + refs[rows] = _k2_way(clusterer, self.threshold, self.max_neighbors) + self.assertGreater(refs[None].ecount(), 1000) + counts = np.bincount(np.array(refs[several].get_edgelist())[:, 0] // several, minlength=4) + self.assertEqual(len(counts), 4) + divisor = _exact_divisor(counts) + for rows, ref in refs.items(): + for chunk in (None, 1, 3, divisor): + with self.subTest(rows=rows, chunk=chunk), self.budget(rows), _patched("_GRAPH_EDGE_CHUNK", chunk): + graph = clusterer.build_similarity_graph(threshold=self.threshold, + max_neighbors=self.max_neighbors) + self.assert_same_graph(graph, ref) + + def test_int64_ids_when_the_largest_id_is_above_the_int32_limit(self): + clusterer = EmbeddingClusterer(self.db_path) + rows = self.n // 4 + 1 + with self.budget(rows): + ref = _k2_way(clusterer, self.threshold, self.max_neighbors) + graphs = {} + for max_id, dtype in ((self.n - 1, np.int32), (self.n - 2, np.int64)): + with self.subTest(max_id=max_id): + dtypes = [] + edge_tuples = clustering._edge_tuples + + def recording(blocks): + dtypes.extend(b.dtype for b in blocks) + return edge_tuples(blocks) + + with self.budget(rows), _patched("_GRAPH_INT32_MAX_ID", max_id), \ + mock.patch.object(clustering, "_edge_tuples", recording): + graphs[dtype] = clusterer.build_similarity_graph(threshold=self.threshold, + max_neighbors=self.max_neighbors) + self.assertGreater(len(dtypes), 1) + self.assertEqual(set(dtypes), {np.dtype(dtype)}) + self.assert_same_graph(graphs[dtype], ref) + self.assert_same_graph(graphs[np.int64], graphs[np.int32]) + + +class HnswSameGraphAsK2Test(_SameGraphAsK2, unittest.TestCase): + """hnsw (HnswGraphTest's fixture, with 20 vectors repeated): also how the graph is built (no concatenation, igraph + pulling the edges from the generator, freed memory released twice, with every block released before each release) + and the weight table, at its cap and past it.""" + + index_type, n, d, threshold, max_neighbors = IndexType.hnsw, 2000, 1536, 0.6, 40 + + @classmethod + def vectors(cls): + # Equal weights on every platform: a repeated vector has its original's scores. Without the repeats, equal + # float32 scores came by chance: 1 to 4 values of 2,796 edges, by platform (FAISS's SIMD code), and maybe none + # on CI's Linux runner, where the test failed. + vectors = _clustered_vectors(seed=3, n=cls.n, d=cls.d, n_centers=150, noise=1.0) + for t in range(20): + vectors[1000 + t] = vectors[10 * t] + return vectors + + def test_no_concatenation(self): + clusterer = EmbeddingClusterer(self.db_path) + for rows in (None, 7): + with self.subTest(rows=rows), self.budget(rows): + ref = _k2_way(clusterer, self.threshold, self.max_neighbors) + with mock.patch.object(clustering.np, "concatenate", side_effect=AssertionError("np.concatenate")): + graph = clusterer.build_similarity_graph(threshold=self.threshold, + max_neighbors=self.max_neighbors) + self.assert_same_graph(graph, ref) + + def test_igraph_is_given_no_ndarray(self): + clusterer = EmbeddingClusterer(self.db_path) + ref = _k2_way(clusterer, self.threshold, self.max_neighbors) + generators = [] + edge_tuples = clustering._edge_tuples + + def recording(blocks): + generators.append(edge_tuples(blocks)) + return generators[-1] + + with mock.patch.object(clustering.ig, "Graph", wraps=ig.Graph) as graph_class, \ + mock.patch.object(clustering, "_edge_tuples", recording): + graph = clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=self.max_neighbors) + graph_class.assert_called_once() + args = list(graph_class.call_args.args) + list(graph_class.call_args.kwargs.values()) + self.assertIn("edges", graph_class.call_args.kwargs) + self.assertFalse(any(isinstance(a, np.ndarray) for a in args)) + # igraph pulls the edges from _edge_tuples' generator itself: no list or tuple of every edge is made first + self.assertEqual(len(generators), 1) + self.assertIs(graph_class.call_args.kwargs["edges"], generators[0]) + self.assert_same_graph(graph, ref) + + def test_freed_memory_released_after_the_graph_and_after_the_weights(self): + clusterer = EmbeddingClusterer(self.db_path) + calls = [] + + def record(): + calls.append((clusterer.graph, clusterer.graph.ecount(), "weight" in clusterer.graph.es.attributes())) + + with mock.patch.object(clustering, "_release_freed_memory", side_effect=record) as release: + graph = clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=self.max_neighbors) + self.assertEqual(release.call_count, 2) + self.assertGreater(graph.ecount(), 1000) + self.assertEqual(len(calls), 2) + self.assertTrue(all(c[0] is graph for c in calls)) + self.assertEqual([c[1:] for c in calls], [(graph.ecount(), False), (graph.ecount(), True)]) + + def test_every_block_released_before_each_release(self): + """The build hands its own block lists to _edge_tuples and _weight_list and keeps no other reference: every + pair block is dead at the first release (after the igraph build), every weight block at the second.""" + clusterer = EmbeddingClusterer(self.db_path) + refs = {"pairs": [], "weights": []} + edge_tuples, weight_list = clustering._edge_tuples, clustering._weight_list + + def recording(real, key): + def call(blocks): + refs[key].extend(weakref.ref(b) for b in blocks) + return real(blocks) + return call + + alive = [] + + def release(): + alive.append({key: sum(r() is not None for r in rs) for key, rs in refs.items()}) + + with self.budget(7), \ + mock.patch.object(clustering, "_edge_tuples", recording(edge_tuples, "pairs")), \ + mock.patch.object(clustering, "_weight_list", recording(weight_list, "weights")), \ + mock.patch.object(clustering, "_release_freed_memory", side_effect=release): + graph = clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=self.max_neighbors) + self.assertGreater(len(refs["pairs"]), 1) + self.assertEqual(len(refs["weights"]), len(refs["pairs"])) + self.assertGreater(graph.ecount(), 1000) + self.assertEqual(alive, [{"pairs": 0, "weights": 0}, {"pairs": 0, "weights": 0}]) + + def test_release_without_malloc_trim_is_a_no_op(self): + """On this platform, with the C library lookup failing (OSError) and with a C library that has no malloc_trim + (AttributeError), the build gives the same graph; on Linux, malloc_trim(0) is called twice.""" + self.assertIsNone(clustering._release_freed_memory()) + clusterer = EmbeddingClusterer(self.db_path) + ref = _k2_way(clusterer, self.threshold, self.max_neighbors) + linux = SimpleNamespace(platform="linux") + trim = mock.Mock(return_value=1) + cases = { + "this platform": (clustering.sys, clustering.ctypes), + "lookup raises OSError": (linux, SimpleNamespace(CDLL=mock.Mock(side_effect=OSError("no C library")))), + "no malloc_trim": (linux, SimpleNamespace(CDLL=mock.Mock(return_value=object()))), + "this C library on linux": (linux, clustering.ctypes), + "malloc_trim on linux": (linux, SimpleNamespace(CDLL=mock.Mock(return_value=SimpleNamespace( + malloc_trim=trim)))), + "malloc_trim not on linux": (SimpleNamespace(platform="darwin"), SimpleNamespace(CDLL=mock.Mock( + return_value=SimpleNamespace(malloc_trim=trim)))), + } + for case, (platform, c_library) in cases.items(): + with self.subTest(case=case): + trim.reset_mock() + with mock.patch.object(clustering, "sys", platform), mock.patch.object(clustering, "ctypes", c_library): + graph = clusterer.build_similarity_graph(threshold=self.threshold, + max_neighbors=self.max_neighbors) + self.assert_same_graph(graph, ref) + if case == "malloc_trim on linux": + c_library.CDLL.assert_called_with(None) + self.assertEqual(trim.call_args_list, [mock.call(0), mock.call(0)]) + elif case == "malloc_trim not on linux": + c_library.CDLL.assert_not_called() + trim.assert_not_called() + + def test_equal_weights_share_one_float(self): + """The table path, at the default cap and at a cap equal to the weights' float32 span: equal weights are one + object; past the cap (patched to 1), one float an edge as before. The same graph as K2's in all three, at + chunks of 1, 3, a divisor of the edge count, and the default.""" + clusterer = EmbeddingClusterer(self.db_path) + ref = _k2_way(clusterer, self.threshold, self.max_neighbors) + span = _float32_span(ref.es["weight"]) + self.assertLess(span, clustering._GRAPH_WEIGHT_TABLE_MAX) + # Equal weights to share: the repeated vectors' scores. + self.assertLess(len({w.hex() for w in ref.es["weight"]}), ref.ecount()) + # One block: the whole fixture. + self.assertGreaterEqual(clustering._GRAPH_RESULT_BYTES // (self.max_neighbors * 12), self.n) + divisor = _exact_divisor([ref.ecount()]) + for cap, shared in ((None, True), (span, True), (1, False)): + for chunk in (None, 1, 3, divisor): + with self.subTest(cap=cap, chunk=chunk), _patched("_GRAPH_WEIGHT_TABLE_MAX", cap), \ + _patched("_GRAPH_EDGE_CHUNK", chunk): + graph = clusterer.build_similarity_graph(threshold=self.threshold, + max_neighbors=self.max_neighbors) + self.assert_same_graph(graph, ref) + if shared: + self.assertTrue(_shares_equal_weights(graph.es["weight"])) + + +class FlatSameGraphAsK2Test(_SameGraphAsK2, unittest.TestCase): + """flat (FlatGraphTest's fixture): also every pair, negative scores included, past the weight table's cap.""" + + index_type, n, d, threshold, max_neighbors = IndexType.flat, 1500, 256, 0.6, 30 + + @classmethod + def vectors(cls): + return _clustered_vectors(seed=5, n=cls.n, d=cls.d, n_centers=60, noise=1.0) + + def test_every_pair_past_the_weight_table_cap(self): + """Threshold -1 at max_neighbors = n: every pair is an edge, in one block of two chunks, negative scores + among them, so the weights are one float an edge (no table).""" + clusterer = EmbeddingClusterer(self.db_path) + ref = _k2_way(clusterer, -1.0, self.n) + self.assertEqual(ref.ecount(), self.n * (self.n - 1) // 2) + self.assertGreater(ref.ecount(), clustering._GRAPH_EDGE_CHUNK) + self.assertLess(ref.ecount(), 2 * clustering._GRAPH_EDGE_CHUNK) + self.assertGreaterEqual(clustering._GRAPH_RESULT_BYTES // (self.n * 12), self.n) + self.assertLess(min(ref.es["weight"]), 0) + self.assertGreater(_float32_span(ref.es["weight"]), clustering._GRAPH_WEIGHT_TABLE_MAX) + for chunk in (None, 1, 3, _exact_divisor([ref.ecount()])): + with self.subTest(chunk=chunk), _patched("_GRAPH_EDGE_CHUNK", chunk): + graph = clusterer.build_similarity_graph(threshold=-1.0, max_neighbors=self.n) + self.assert_same_graph(graph, ref) + + +class IvfPqSameGraphAsK2Test(_SameGraphAsK2, unittest.TestCase): + """ivf_pq (IvfPqGraphTest's fixture): also the similarity-graph and cluster files equal K2's.""" + + index_type, n, d, threshold, max_neighbors = IndexType.ivf_pq, 1000, 64, 0.5, 1000 + config = IndexConfig(nlist=16, m=8, nbits=4, nprobe=16, opq_train_size=1000) + + @classmethod + def vectors(cls): + vectors = _clustered_vectors(seed=7, n=cls.n, d=cls.d, n_centers=40, noise=0.8) + for t in range(20): + vectors[500 + t] = vectors[10 * t] + return vectors + + def test_files_as_k2(self): + """similarity-graph writes the GraphML of K2's graph, and cluster with a seed the Leiden TSV on it (both + at the commands' default --max-neighbors, 1000).""" + out = Path(self.tmp.name) + similarity_graph(db_path=self.db_path, threshold=self.threshold, output_file=str(out / "q1.graphml")) + cluster_database(db_path=self.db_path, threshold=self.threshold, output_file=str(out / "q1.tsv"), seed=1) + clusterer = EmbeddingClusterer(self.db_path) + ref = _k2_way(clusterer, self.threshold, 1000) + self.assertIs(clusterer.graph, ref) + ref.write_graphml(str(out / "k2.graphml")) + clusterer.cluster_leiden(seed=1) + clusterer.export_clusters(str(out / "k2.tsv")) + self.assertGreater(ref.ecount(), 1000) + self.assertEqual((out / "q1.graphml").read_bytes(), (out / "k2.graphml").read_bytes()) + self.assertEqual((out / "q1.tsv").read_bytes(), (out / "k2.tsv").read_bytes()) + + +class GraphEdgeCaseAsK2Test(unittest.TestCase): + """EdgeCaseTest's fixtures: no edge, one chain, and blocks with no kept edge between blocks with edges, each the + same as K2's graph; a block with no kept edge is not stored.""" + + n, d = 60, 16 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + folder = Path(cls.tmp.name) + vectors = np.zeros((cls.n, cls.d), dtype=np.float32) + vectors[:54, :8] = _clustered_vectors(seed=11, n=54, d=8, n_centers=6, noise=0.5) + vectors[54:, 8:14] = np.eye(6, dtype=np.float32) + cls.db_path = _build(folder, "flat", vectors, IndexType.flat) + cls.one_path = _build(folder, "one", vectors[:1], IndexType.flat) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + assert_same_list = _SameGraphAsK2.assert_same_list + assert_same_graph = _SameGraphAsK2.assert_same_graph + + def test_no_edge(self): + clusterer = EmbeddingClusterer(self.db_path) + ref = _k2_way(clusterer, 2.0, 10) + graph = clusterer.build_similarity_graph(threshold=2.0, max_neighbors=10) + self.assertEqual((graph.vcount(), graph.ecount(), graph.es["weight"]), (self.n, 0, [])) + self.assert_same_graph(graph, ref) + + def test_one_chain(self): + clusterer = EmbeddingClusterer(self.one_path) + ref = _k2_way(clusterer, 0.0, 10) + graph = clusterer.build_similarity_graph(threshold=0.0, max_neighbors=10) + self.assertEqual((graph.vcount(), graph.ecount()), (1, 0)) + self.assert_same_graph(graph, ref) + + def test_blocks_with_no_edge_between_blocks_with_edges(self): + threshold, k = 0.5, 10 + clusterer = EmbeddingClusterer(self.db_path) + gaps = {} + for rows in (1, 2, 7): + with self.subTest(rows=rows), mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", _budget(rows, k)): + ref = _k2_way(clusterer, threshold, k) + with_edges = sorted({i // rows for i, _ in ref.get_edgelist()}) + gaps[rows] = len(set(range(with_edges[0], with_edges[-1])) - set(with_edges)) + stored = [] + edge_tuples, weight_list = clustering._edge_tuples, clustering._weight_list + + def recording(real): + def call(blocks): + stored.append([len(b) for b in blocks]) + return real(blocks) + return call + + with mock.patch.object(clustering, "_edge_tuples", recording(edge_tuples)), \ + mock.patch.object(clustering, "_weight_list", recording(weight_list)): + graph = clusterer.build_similarity_graph(threshold=threshold, max_neighbors=k) + self.assert_same_graph(graph, ref) + counts = np.bincount(np.array(ref.get_edgelist())[:, 0] // rows) + self.assertEqual(stored, [counts[with_edges].tolist()] * 2) + # Blocks of one chain: some with no kept edge lie between blocks with edges. + self.assertGreater(gaps[1], 0) + + +class _PopRecorder(list): + """A list of blocks that records, at each pop, which of the original blocks are still alive.""" + + def __init__(self, blocks): + super().__init__(blocks) + self.refs = [weakref.ref(b) for b in blocks] + self.alive = [] + + def pop(self, *args): + self.alive.append([r() is not None for r in self.refs]) + return super().pop(*args) + + +class GraphPiecesTest(unittest.TestCase): + """The id dtype rule, the edge generator and the weight list on small blocks.""" + + def test_id_dtype(self): + self.assertEqual(clustering._GRAPH_INT32_MAX_ID, 2 ** 31 - 1) + for n_chains, dtype in ((0, np.int32), (1, np.int32), (2 ** 31, np.int32), (2 ** 31 + 1, np.int64), + (2 ** 40, np.int64)): + with self.subTest(n_chains=n_chains): + self.assertIs(clustering._graph_id_dtype(n_chains), dtype) + with _patched("_GRAPH_INT32_MAX_ID", 9): + self.assertIs(clustering._graph_id_dtype(10), np.int32) + self.assertIs(clustering._graph_id_dtype(11), np.int64) + + def test_edge_tuples_release_each_block_once_consumed(self): + blocks = [np.array([[0, 1], [0, 2]], dtype=np.int32), np.array([[1, 2], [1, 3], [1, 4]], dtype=np.int32), + np.array([[3, 4]], dtype=np.int32)] + refs = [weakref.ref(b) for b in blocks] + with _patched("_GRAPH_EDGE_CHUNK", 2): + edges = clustering._edge_tuples(blocks) + got = [next(edges), next(edges)] + self.assertIsNotNone(refs[0]()) + got.append(next(edges)) + self.assertIsNone(refs[0]()) + self.assertLess(len(blocks), 3) + got.extend(edges) + self.assertEqual(len(blocks), 0) + self.assertEqual([r() for r in refs], [None] * 3) + self.assertEqual(got, [(0, 1), (0, 2), (1, 2), (1, 3), (1, 4), (3, 4)]) + self.assertTrue(all(type(e) is tuple and all(type(i) is int for i in e) for e in got)) + self.assertEqual(list(clustering._edge_tuples([])), []) + + def test_weight_list_releases_each_block_once_consumed(self): + for cap in (None, 1): + with self.subTest(cap=cap), _patched("_GRAPH_WEIGHT_TABLE_MAX", cap), _patched("_GRAPH_EDGE_CHUNK", 2): + blocks = _PopRecorder([np.float32([0.5, 0.75, 0.5]), np.float32([0.75]), np.float32([0.25, 0.5])]) + expected = [0.5, 0.75, 0.5, 0.75, 0.25, 0.5] + got = clustering._weight_list(blocks) + self.assertEqual(got, expected) + self.assertEqual(len(blocks), 0) + self.assertEqual(blocks.alive, [[True, True, True], [False, True, True], [False, False, True]]) + self.assertEqual([r() for r in blocks.refs], [None] * 3) + + def test_weight_list_shares_equal_bit_patterns_across_slices_and_blocks(self): + blocks = [np.float32([0.5, 0.75, 0.5]), np.float32([]), np.float32([0.75, 0.5, 1.0])] + expected = [w for b in blocks for w in b.tolist()] + with _patched("_GRAPH_EDGE_CHUNK", 2): + got = clustering._weight_list(list(blocks)) + self.assertEqual(got, expected) + self.assertTrue(all(type(w) is float for w in got)) + self.assertTrue(got[0] is got[2] is got[4]) + self.assertTrue(got[1] is got[3]) + self.assertEqual(clustering._weight_list([]), []) + self.assertEqual(clustering._weight_list([np.float32([])]), []) + + def test_weight_list_keeps_the_sign_of_zero(self): + """0.0 with -0.0 spans past the cap (one float an edge); -0.0 alone takes the table: signs kept either way.""" + for values, shared in (([0.0, -0.0, 0.5], False), ([-0.0, -0.0], True)): + with self.subTest(values=values): + block = np.float32(values) + self.assertEqual(_float32_span(block) <= clustering._GRAPH_WEIGHT_TABLE_MAX, shared) + expected = block.tolist() + got = clustering._weight_list([block]) + self.assertEqual(np.asarray(got, dtype=np.float64).tobytes(), + np.asarray(expected, dtype=np.float64).tobytes()) + self.assertEqual([math.copysign(1.0, w) for w in got], [math.copysign(1.0, w) for w in expected]) + self.assertTrue(all(type(w) is float for w in got)) + if shared: + self.assertIs(got[0], got[1]) + + +# P9.U: cluster_leiden runs python-igraph's community_leiden, a fixed number of iterations, seeded with a private +# random.Random, cluster ids by decreasing size, ties by smallest chain index. + +_SRC = str(Path(foldmatch.__file__).resolve().parent.parent) + + +def _reference_leiden(graph: ig.Graph, seed: int, resolution: float = 1.0, n_iterations: int = 10) -> list: + """P9.U's Leiden call, made here and not through the code under test: python-igraph's community_leiden with its + arguments, seeded with random.seed (Python's global state saved and restored around it), and the cluster ids + renumbered in pure Python by (-size, smallest chain index).""" + state = random.getstate() + try: + random.seed(seed) + membership = graph.community_leiden(objective_function="modularity", weights="weight", resolution=resolution, + beta=0.01, n_iterations=n_iterations).membership + finally: + random.setstate(state) + members = {} + for chain, cluster in enumerate(membership): + members.setdefault(cluster, []).append(chain) + order = sorted(members, key=lambda c: (-len(members[c]), members[c][0])) + new_id = {cluster: rank for rank, cluster in enumerate(order)} + return [new_id[cluster] for cluster in membership] + + +def _as_sets(membership) -> set: + """A partition as a set of frozensets of chain indexes, whatever its labels.""" + members = {} + for chain, cluster in enumerate(membership): + members.setdefault(int(cluster), set()).add(chain) + return {frozenset(m) for m in members.values()} + + +def _flat(text: str) -> str: + """Text on one line, typer's error panel borders and colour codes dropped.""" + return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", text).translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +class _LeidenSpy: + """python-igraph's Graph.community_leiden, recording the keyword arguments of each call while patched.""" + + def __init__(self): + self.calls = [] + + def patch(self): + real, calls = ig.Graph.community_leiden, self.calls + + def community_leiden(graph, *args, **kwargs): + assert not args, args + calls.append(kwargs) + return real(graph, **kwargs) + + return mock.patch.object(ig.Graph, "community_leiden", community_leiden) + + +# Imports the clustering module and the CLI with the removed Leiden package blocked, then runs cluster on a database. +# The package's name is spelled in two pieces so that a grep of the repository for it finds nothing. +_NO_LEIDEN_PACKAGE_CHILD = """ +import sys +sys.modules["leiden" + "alg"] = None +import lightning # noqa: F401 +import foldmatch +assert foldmatch.__file__.startswith(sys.argv[1]), foldmatch.__file__ +import foldmatch.search.clustering +from foldmatch.cli.search import cluster_database +cluster_database(db_path=sys.argv[2], threshold=0.5, max_neighbors=10, seed=3, output_file=sys.argv[3]) +print("cluster ran") +""" + +# Prints the digest of cluster_leiden's array at one seed, in a fresh process. +_SEED_DIGEST_CHILD = """ +import hashlib +import sys +import lightning # noqa: F401 +import foldmatch +assert foldmatch.__file__.startswith(sys.argv[1]), foldmatch.__file__ +from foldmatch.search.clustering import EmbeddingClusterer +clusterer = EmbeddingClusterer(sys.argv[2]) +clusterer.build_similarity_graph(threshold=float(sys.argv[4]), max_neighbors=10) +clusters = clusterer.cluster_leiden(seed=int(sys.argv[3])) +print(clusters.dtype.name, hashlib.sha256(clusters.tobytes()).hexdigest()) +""" + + +class LeidenIgraphTest(unittest.TestCase): + """cluster_leiden: python-igraph's community_leiden with modularity at the resolution, beta 0.01 and a fixed + number of iterations (10 by default); seeded with a private random.Random, Python's global random state untouched + and python-igraph's generator set back to the random module; cluster ids by decreasing size, ties by smallest chain + index, as an int32 array; and the refusals (iterations below 1, negative seeds, negative kept weights).""" + + n, d = 90, 16 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + folder = Path(cls.tmp.name) + # Six well-separated clusters: a chain's few nearest neighbours score above 0, but some pairs of chains of + # different clusters score below 0. + cls.vectors = _clustered_vectors(seed=31, n=cls.n, d=cls.d, n_centers=6, noise=0.3) + cls.db_path = _build(folder, "flat", cls.vectors, IndexType.flat) + # Noisier clusters, kept from threshold 0 at k 10: their partition depends on the seed (the fixture above + # gives one partition whatever the seed), so the seeding tests tell a seeded generator from any other. + cls.sensitive_db_path = _build(folder, "flat_seed_sensitive", _clustered_vectors( + seed=31, n=cls.n, d=cls.d, n_centers=6, noise=2.0), IndexType.flat) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def setUp(self): + self.addCleanup(ig.set_random_number_generator, random) + + def clusterer(self, threshold: float = 0.5, max_neighbors: int = 10) -> EmbeddingClusterer: + clusterer = EmbeddingClusterer(self.db_path) + clusterer.build_similarity_graph(threshold=threshold, max_neighbors=max_neighbors) + return clusterer + + def seed_sensitive_clusterer(self) -> EmbeddingClusterer: + """A clusterer whose Leiden partition depends on the seed (threshold 0, k 10).""" + clusterer = EmbeddingClusterer(self.sensitive_db_path) + clusterer.build_similarity_graph(threshold=0.0, max_neighbors=10) + return clusterer + + def with_graph(self, graph: ig.Graph) -> EmbeddingClusterer: + """A clusterer whose graph a caller set (not built by build_similarity_graph).""" + clusterer = EmbeddingClusterer(self.db_path) + clusterer.graph = graph + return clusterer + + @staticmethod + def planted_graph(): + """A weighted planted partition: four blocks of 40, 30, 20 and 10 nodes, dense and heavy inside a block, + sparse and light between blocks. Returns the graph and its blocks.""" + rng = np.random.default_rng(37) + sizes = (40, 30, 20, 10) + block = np.repeat(np.arange(len(sizes)), sizes) + n = len(block) + pairs = [(i, j) for i in range(n) for j in range(i + 1, n)] + same = np.array([block[i] == block[j] for i, j in pairs]) + keep = rng.random(len(pairs)) < np.where(same, 0.6, 0.02) + weights = np.where(same, rng.uniform(0.8, 1.0, len(pairs)), 0.1) + graph = ig.Graph(n=n, edges=[p for p, k in zip(pairs, keep) if k], directed=False) + graph.es["weight"] = weights[keep].tolist() + return graph, _as_sets(block) + + @staticmethod + def cliques_graph(): + """67 disjoint cliques in node order: 64 of 3 nodes, and 3 of 10 nodes at positions 20, 41 and 65, so + igraph's first-appearance ids are not by size and 64 clusters tie (more than 16, so an unstable sort of the + sizes would misorder ties).""" + cliques, start = [], 0 + for c in range(67): + size = 10 if c in (20, 41, 65) else 3 + cliques.append(list(range(start, start + size))) + start += size + edges = [(a, b) for clique in cliques for i, a in enumerate(clique) for b in clique[i + 1:]] + graph = ig.Graph(n=start, edges=edges, directed=False) + graph.es["weight"] = [1.0] * len(edges) + return graph, cliques + + # The call + + def test_the_call(self): + """The keyword arguments of community_leiden: exactly "modularity", the weights, the resolution, beta 0.01, and + 10 iterations unless given.""" + clusterer = self.clusterer() + spy = _LeidenSpy() + with spy.patch(): + clusterer.cluster_leiden(seed=1) + clusterer.cluster_leiden(resolution=2.5, seed=1) + clusterer.cluster_leiden(n_iterations=3, seed=1) + expected = dict(objective_function="modularity", weights="weight", resolution=1.0, beta=0.01, n_iterations=10) + self.assertEqual(spy.calls, [expected, {**expected, "resolution": 2.5}, {**expected, "n_iterations": 3}]) + self.assertIs(type(spy.calls[0]["objective_function"]), str) + self.assertEqual(clustering._LEIDEN_ITERATIONS, 10) + + def test_planted_partition(self): + graph, blocks = self.planted_graph() + clusterer = self.with_graph(graph) + clusters = clusterer.cluster_leiden(seed=2) + self.assertEqual(_as_sets(clusters), blocks) + self.assertEqual(clusters.tolist(), _reference_leiden(graph, 2)) + stats = clusterer.get_cluster_statistics() + self.assertEqual(stats["modularity"], graph.modularity(clusters, weights="weight")) + self.assertGreater(stats["modularity"], 0.5) + + # Seeding + + def test_same_seed_same_partition_as_global_seeding(self): + """The same seed twice gives the same array, equal to random.seed seeding of the same call, on a fixture + whose partition depends on the seed.""" + clusterer = self.seed_sensitive_clusterer() + partitions = {tuple(clusterer.cluster_leiden(seed=seed).tolist()) for seed in range(4)} + self.assertGreater(len(partitions), 1) + first = clusterer.cluster_leiden(seed=5) + second = clusterer.cluster_leiden(seed=5) + np.testing.assert_array_equal(first, second) + self.assertEqual(first.tolist(), _reference_leiden(clusterer.graph, 5)) + for seed in (0, 1, 2 ** 31, 2 ** 64): + with self.subTest(seed=seed): + self.assertEqual(clusterer.cluster_leiden(seed=seed).tolist(), + _reference_leiden(clusterer.graph, seed)) + self.assertGreater(len(set(first.tolist())), 3) + + def test_same_seed_in_fresh_processes(self): + env = {**os.environ, "PYTHONPATH": os.pathsep.join([_SRC] + [p for p in os.environ.get( + "PYTHONPATH", "").split(os.pathsep) if p and p != _SRC])} + digests = [] + for _ in range(2): + run = subprocess.run([sys.executable, "-c", _SEED_DIGEST_CHILD, _SRC, self.sensitive_db_path, "5", "0"], + env=env, capture_output=True, text=True, timeout=600) + self.assertEqual(run.returncode, 0, run.stderr) + digests.append(run.stdout.split()) + clusters = self.seed_sensitive_clusterer().cluster_leiden(seed=5) + self.assertEqual(digests, [["int32", hashlib.sha256(clusters.tobytes()).hexdigest()]] * 2) + + def test_global_random_state_untouched(self): + """random.getstate() is the same before and after, seeded or not, and a caller's seeded stream goes on.""" + clusterer = self.clusterer() + for seed in (5, None): + with self.subTest(seed=seed): + state = random.getstate() + clusterer.cluster_leiden(seed=seed) + self.assertEqual(random.getstate(), state) + random.seed(7) + expected = [random.random() for _ in range(3)] + random.seed(7) + clusterer.cluster_leiden(seed=1) + self.assertEqual([random.random() for _ in range(3)], expected) + + def test_generator_restored_to_the_random_module(self): + """After the call, and after an error inside it, python-igraph's generator is the random module again.""" + clusterer = self.clusterer() + with mock.patch.object(ig, "set_random_number_generator", wraps=ig.set_random_number_generator) as setter: + clusterer.cluster_leiden(seed=1) + self.assertIsInstance(setter.call_args_list[0].args[0], random.Random) + self.assertEqual(setter.call_args_list[-1], mock.call(random)) + failing = mock.patch.object(ig.Graph, "community_leiden", side_effect=ig.InternalError("inside Leiden")) + with mock.patch.object(ig, "set_random_number_generator", wraps=ig.set_random_number_generator) as setter, \ + failing, self.assertRaisesRegex(ig.InternalError, "inside Leiden"): + clusterer.cluster_leiden(seed=1) + self.assertEqual(setter.call_args_list[-1], mock.call(random)) + self.assertEqual(len(setter.call_args_list), 2) + + # Cluster ids + + def test_ids_by_size_ties_by_smallest_chain_index(self): + graph, cliques = self.cliques_graph() + raw = graph.community_leiden(objective_function="modularity", weights="weight", n_iterations=10).membership + # igraph numbers by first appearance: a 3-node clique is its cluster 0 + self.assertEqual(raw[0], 0) + clusterer = self.with_graph(graph) + clusters = clusterer.cluster_leiden(seed=3) + self.assertIs(clusters, clusterer.clusters) + self.assertIsInstance(clusters, np.ndarray) + self.assertEqual(clusters.dtype, np.int32) + self.assertEqual(_as_sets(clusters), _as_sets(raw)) + self.assertEqual(_as_sets(clusters), {frozenset(c) for c in cliques}) + # The three 10-node cliques first, then the 3-node cliques, each group in node order + by_size = sorted(cliques, key=lambda c: (-len(c), c[0])) + self.assertEqual([int(clusters[c[0]]) for c in by_size], list(range(len(cliques)))) + sizes = np.bincount(clusters) + self.assertTrue(np.all(np.diff(sizes) <= 0), sizes.tolist()) + self.assertEqual(clusters.tolist(), _reference_leiden(graph, 3)) + + # Refusals + + def test_iterations_below_1(self): + clusterer = EmbeddingClusterer(self.db_path) + for n_iterations in (0, -1): + with self.subTest(n_iterations=n_iterations), self.assertRaisesRegex( + ValueError, r"^n_iterations must be at least 1 \(Leiden runs a fixed number of iterations\)\.$"): + clusterer.cluster_leiden(n_iterations=n_iterations, seed=1) + self.assertIsNone(clusterer.clusters) + + def test_seed_values(self): + clusterer = self.clusterer() + with self.assertRaisesRegex(ValueError, r"^--seed must be at least 0\.$"): + clusterer.cluster_leiden(seed=-1) + with self.assertRaises(TypeError): + clusterer.cluster_leiden(seed=5.0) + self.assertIsNone(clusterer.clusters) + clusterer = self.seed_sensitive_clusterer() + expected = clusterer.cluster_leiden(seed=5) + for seed in (np.int64(5), np.int32(5)): + with self.subTest(seed=type(seed).__name__): + np.testing.assert_array_equal(clusterer.cluster_leiden(seed=seed), expected) + + def test_negative_kept_weights_refused(self): + """Every pair of chains (k = n) at threshold -1 keeps negative scores: refused, naming the smallest weight.""" + clusterer = self.clusterer(threshold=-1.0, max_neighbors=self.n) + weights = clusterer.graph.es["weight"] + self.assertEqual(clusterer.graph.ecount(), self.n * (self.n - 1) // 2) + self.assertLess(min(weights), 0) + self.assertEqual(clusterer.min_weight, min(weights)) + message = (f"Leiden needs non-negative edge weights: the graph's smallest weight is {min(weights):.4g} (edges " + f"keep scores at or above the threshold of build_similarity_graph).") + with self.assertRaisesRegex(ValueError, f"^{re.escape(message)}$"): + clusterer.cluster_leiden(seed=1) + self.assertIsNone(clusterer.clusters) + + def test_negative_threshold_with_non_negative_kept_weights_runs(self): + """At a few neighbours a chain, threshold -1 keeps only scores of at least 0: the edges and the partition of + threshold 0.""" + clusterer = self.clusterer(threshold=-1.0, max_neighbors=5) + self.assertGreaterEqual(clusterer.min_weight, 0) + self.assertEqual(clusterer.min_weight, min(clusterer.graph.es["weight"])) + at_zero = self.clusterer(threshold=0.0, max_neighbors=5) + self.assertEqual(clusterer.graph.get_edgelist(), at_zero.graph.get_edgelist()) + np.testing.assert_array_equal(clusterer.cluster_leiden(seed=4), at_zero.cluster_leiden(seed=4)) + + def test_a_kept_weight_of_exactly_0_runs(self): + """Orthogonal unit vectors score exactly 0, which threshold 0 keeps: 0 is not below 0, in the library and the + CLI.""" + vectors = np.zeros((3, self.d), dtype=np.float32) + vectors[[0, 2], 0] = 1.0 + vectors[1, 1] = 1.0 + db_path = _build(Path(self.tmp.name), "flat_zero_score", vectors, IndexType.flat) + clusterer = EmbeddingClusterer(db_path) + clusterer.build_similarity_graph(threshold=0.0, max_neighbors=3) + self.assertEqual(sorted(clusterer.graph.es["weight"]), [0.0, 0.0, 1.0]) + self.assertEqual(clusterer.min_weight, 0.0) + self.assertEqual(len(clusterer.cluster_leiden(seed=1)), 3) + out = Path(self.tmp.name) / "zero_score.tsv" + result = self.invoke("cluster", "--db-path", db_path, "--output-file", out, "--threshold", "0", + "--max-neighbors", 3, "--seed", 1) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + self.assertEqual(len(out.read_text().splitlines()), 3) + + def test_a_graph_set_after_the_build_is_not_judged_by_its_weights(self): + clusterer = self.clusterer(threshold=-1.0, max_neighbors=self.n) + self.assertLess(clusterer.min_weight, 0) + other = self.clusterer(threshold=0.5, max_neighbors=10) + clusterer.graph = other.graph + np.testing.assert_array_equal(clusterer.cluster_leiden(seed=4), other.cluster_leiden(seed=4)) + # A build that fails records nothing + clusterer.db.index = None + with self.assertRaises(ValueError): + clusterer.build_similarity_graph(threshold=-1.0, max_neighbors=self.n) + self.assertIsNone(clusterer.min_weight) + self.assertIsNone(clusterer._min_weight_graph) + + def test_no_edge_records_inf(self): + clusterer = self.clusterer(threshold=2.0, max_neighbors=10) + self.assertEqual(clusterer.graph.ecount(), 0) + self.assertEqual(clusterer.min_weight, float("inf")) + self.assertIs(clusterer._min_weight_graph(), clusterer.graph) + + # Through the CLI + + def invoke(self, *args): + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(search.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + def test_cli_negative_kept_weights(self): + out = Path(self.tmp.name) / "neg.tsv" + base = ("cluster", "--db-path", self.db_path, "--output-file", out, "--threshold", "-1") + clusterer = self.clusterer(threshold=-1.0, max_neighbors=self.n) + for method in ((), ("--method", "leiden")): + with self.subTest(method=method): + result = self.invoke(*base, *method, "--max-neighbors", self.n) + self.assertEqual(result.exit_code, 2, f"{result.output}\n{result.exception!r}") + self.assertIn(f"Invalid value for --threshold: with --method leiden, Leiden's modularity needs " + f"non-negative edge weights, and this --threshold kept a score of " + f"{clusterer.min_weight:.4g}. Use a --threshold of 0 or more, or --method greedy.", + _flat(result.output)) + self.assertFalse(out.exists()) + # Non-negative kept scores at threshold -1: runs, as at threshold 0 + result = self.invoke(*base, "--max-neighbors", 5, "--seed", 4) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + at_zero = self.clusterer(threshold=0.0, max_neighbors=5) + at_zero.cluster_leiden(seed=4) + at_zero.export_clusters(str(Path(self.tmp.name) / "zero.tsv")) + self.assertEqual(out.read_bytes(), (Path(self.tmp.name) / "zero.tsv").read_bytes()) + # Greedy computes no modularity: not refused + result = self.invoke(*base, "--max-neighbors", self.n, "--method", "greedy") + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + self.assertEqual(len(out.read_text().splitlines()), self.n) + + def test_cli_seed_2_to_the_31(self): + out = Path(self.tmp.name) / "big_seed.tsv" + result = self.invoke("cluster", "--db-path", self.sensitive_db_path, "--output-file", out, "--threshold", + "0", "--max-neighbors", 10, "--seed", 2 ** 31) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + clusterer = self.seed_sensitive_clusterer() + expected = _reference_leiden(clusterer.graph, 2 ** 31) + self.assertEqual([int(line.split("\t")[1]) for line in out.read_text().splitlines()], expected) + + # The removed package + + def test_runs_without_the_removed_leiden_package(self): + out = Path(self.tmp.name) / "blocked.tsv" + env = {**os.environ, "PYTHONPATH": os.pathsep.join([_SRC] + [p for p in os.environ.get( + "PYTHONPATH", "").split(os.pathsep) if p and p != _SRC])} + run = subprocess.run([sys.executable, "-c", _NO_LEIDEN_PACKAGE_CHILD, _SRC, self.db_path, str(out)], env=env, + capture_output=True, text=True, timeout=600) + self.assertEqual(run.returncode, 0, run.stderr) + self.assertEqual(run.stdout.splitlines()[-1], "cluster ran") + self.assertEqual(len(out.read_text().splitlines()), self.n) + + def test_no_name_of_the_removed_package_in_the_repository(self): + root = Path(_SRC).parent + if shutil.which("git") is None or not (root / ".git").exists(): + self.skipTest("not a git checkout") + run = subprocess.run(["git", "grep", "-n", "-i", "leiden" + "alg", "--", "src", "tests", "pyproject.toml", + "uv.lock", "README.md"], cwd=root, capture_output=True, text=True, timeout=60) + self.assertEqual((run.returncode, run.stdout), (1, ""), run.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_clustering_greedy.py b/tests/test_clustering_greedy.py new file mode 100644 index 0000000..6122d77 --- /dev/null +++ b/tests/test_clustering_greedy.py @@ -0,0 +1,1072 @@ +"""EmbeddingClusterer.cluster_greedy and fm-search cluster --method greedy: greedy representatives in database order, +computed in streamed blocks that search only the chains no representative has taken at the block's start. On hnsw, and +on flat below FAISS's BLAS switch, the partition equals the sequential rule on one query per chain bit for bit, at every +block size; on ivf_pq it is checked against the result rows the method used. Also the cluster ids (by decreasing size, +ties by representative index), the statistics, the 4-column output, the refusals (with cluster's Leiden options), +and the Leiden output in c40d4e1's format.""" +import logging +import math +import random +import re +import tempfile +import unittest +from collections import Counter +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +# As the commands run (set_log_level): Lightning first, which brings torch and, through scikit-learn, +# KMP_DUPLICATE_LIB_OK; on macOS, FAISS's OpenMP regions abort a process that loaded torch's libomp without it. +import lightning # noqa: F401 +import faiss +import igraph as ig +import numpy as np +import typer +from typer.testing import CliRunner + +from foldmatch.cli import search +from foldmatch.cli.search import cluster_database +from foldmatch.search import clustering +from foldmatch.search.clustering import EmbeddingClusterer +from foldmatch.search.faiss_database import FaissEmbeddingDatabase +from foldmatch.search.output import CLUSTER_OUTPUT_FIELDS, GREEDY_CLUSTER_OUTPUT_FIELDS, format_spec, \ + write_cluster_results +from foldmatch.types.api_types import ClusterMethod, FormatMode, IndexConfig, IndexType + + +def _clustered_vectors(seed: int, n: int, d: int, n_centers: int, noise: float) -> np.ndarray: + """Unit vectors around ``n_centers`` random centres, at a noise level of 0.5 to 1.5 times ``noise`` each; the + members of a centre are contiguous (tests/test_clustering.py's fixture).""" + rng = np.random.default_rng(seed) + centers = rng.standard_normal((n_centers, d)) + centers /= np.linalg.norm(centers, axis=1, keepdims=True) + members = np.sort(rng.integers(0, n_centers, n)) + levels = noise * rng.uniform(0.5, 1.5, (n, 1)) + vectors = centers[members] + levels * rng.standard_normal((n, d)) / math.sqrt(d) + vectors /= np.linalg.norm(vectors, axis=1, keepdims=True) + return vectors.astype(np.float32) + + +def _build(folder: Path, name: str, vectors: np.ndarray, index_type: IndexType, config: IndexConfig = None) -> str: + db = FaissEmbeddingDatabase(folder, name) + db.create_database(iter([([f"chain{i}" for i in range(len(vectors))], vectors)]), index_type=index_type, + index_config=config) + return str(folder / name) + + +def _load(db_path: str): + path = Path(db_path) + db = FaissEmbeddingDatabase(path.parent, path.name) + db.load_database() + return db + + +def _budget(rows: int, k: int) -> int: + """The result budget that gives blocks of ``rows`` chains at ``k`` neighbours.""" + return rows * k * 12 + + +def _block_rows(n: int, k: int) -> int: + """The rows of a block at the module's budget and row cap (build_similarity_graph's formula).""" + return max(1, min(n, clustering._GRAPH_MAX_ROWS, clustering._GRAPH_RESULT_BYTES // (k * 12))) + + +def _one_query_results(index, n: int, k: int) -> list: + """Each chain's result row from a search of its stored vector alone (flat and hnsw), as Python lists.""" + results = [] + for i in range(n): + query = np.ascontiguousarray(index.reconstruct(i), dtype=np.float32).reshape(1, -1) + scores, ids = index.search(query, k) + results.append((scores[0].tolist(), ids[0].tolist())) + return results + + +def _sequential(results: list, n: int, threshold: float, k: int, rows: int, max_neighbors: int = None): + """The sequential rule on full result rows, in plain Python: chains in database order; a chain in a cluster is + passed over; any other becomes a representative and takes every id of its row with id >= 0, a score at least the + threshold (compared in float32, as NumPy compares float32 scores with a Python float) and no cluster yet. Blocks + of ``rows`` chains only count the searched chains: those in no cluster at their block's start. Cluster ids by + decreasing size, ties by representative index. Clusters at the cap: of at least ``max_neighbors`` (default k) + chains.""" + threshold = float(np.float32(threshold)) + label = [-1] * n + searched = saturated = 0 + for start in range(0, n, rows): + end = min(start + rows, n) + searched += sum(1 for i in range(start, end) if label[i] < 0) + for i in range(start, end): + if label[i] >= 0: + continue + label[i] = i + scores, ids = results[i] + for j, score in zip(ids, scores): + if j >= 0 and score >= threshold and label[j] < 0: + label[j] = i + if k < n and ids[-1] >= 0 and scores[-1] >= threshold: + saturated += 1 + members = {} + for i, rep in enumerate(label): + members.setdefault(rep, []).append(i) + reps = sorted(members, key=lambda r: (-len(members[r]), r)) + cluster_of = {rep: c for c, rep in enumerate(reps)} + sizes = [len(members[r]) for r in reps] + return SimpleNamespace(clusters=[cluster_of[label[i]] for i in range(n)], representatives=reps, sizes=sizes, + label=label, n_searched=searched, n_saturated=saturated, + n_at_cap=sum(1 for s in sizes if s >= (max_neighbors or k)), saturated_reps=[ + r for r in reps if k < n and results[r][1][-1] >= 0 + and results[r][0][-1] >= threshold]) + + +class _GreedyCase(unittest.TestCase): + + def greedy(self, db_path: str, threshold: float, max_neighbors: int, rows: int = None): + """cluster_greedy on a fresh clusterer, with blocks of ``rows`` chains (the default budget when None).""" + clusterer = EmbeddingClusterer(db_path) + budget = clustering._GRAPH_RESULT_BYTES if rows is None else _budget(rows, min(max_neighbors, + len(clusterer.chain_ids))) + with mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", budget): + clusters = clusterer.cluster_greedy(threshold=threshold, max_neighbors=max_neighbors) + self.assertIs(clusters, clusterer.clusters) + self.assertEqual(clusterer.method, "greedy") + return clusterer + + def assert_same_as(self, clusterer: EmbeddingClusterer, ref): + self.assertEqual(clusterer.clusters.tolist(), ref.clusters) + self.assertEqual(clusterer.representatives.tolist(), ref.representatives) + self.assertEqual(np.bincount(clusterer.clusters).tolist(), ref.sizes) + stats = clusterer.get_cluster_statistics() + self.assertEqual((stats["n_saturated"], stats["n_at_cap"], stats["n_searched"]), + (ref.n_saturated, ref.n_at_cap, ref.n_searched)) + self.assertEqual((clusterer.n_saturated, clusterer.n_searched), (ref.n_saturated, ref.n_searched)) + + def assert_cluster_ids(self, clusterer: EmbeddingClusterer): + """Ids by non-increasing size, equal sizes in ascending representative index, and representatives[i] the + representative of cluster i (a chain of cluster i).""" + sizes = np.bincount(clusterer.clusters) + reps = clusterer.representatives + self.assertEqual(len(reps), len(sizes)) + self.assertTrue(np.all(np.diff(sizes) <= 0)) + ties = np.diff(sizes) == 0 + self.assertTrue(np.all(np.diff(reps)[ties] > 0)) + self.assertTrue(np.array_equal(clusterer.clusters[reps], np.arange(len(reps)))) + # The representative is the cluster's lowest chain index (every member comes after it in database order). + lowest = np.full(len(reps), len(clusterer.clusters)) + np.minimum.at(lowest, clusterer.clusters, np.arange(len(clusterer.clusters))) + self.assertTrue(np.array_equal(lowest, reps)) + + +class HnswGreedyTest(_GreedyCase): + """hnsw: FAISS results do not depend on the batch, so cluster_greedy equals the sequential rule bit for bit at + every block size.""" + + n, d, threshold, max_neighbors = 2000, 1536, 0.6, 40 + # A --max-neighbors small enough that it, not the threshold, bounds many clusters. + small_k = 5 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + vectors = _clustered_vectors(seed=3, n=cls.n, d=cls.d, n_centers=150, noise=1.0) + cls.db_path = _build(Path(cls.tmp.name), "hnsw", vectors, IndexType.hnsw) + index = _load(cls.db_path).index + cls.results = _one_query_results(index, cls.n, cls.max_neighbors) + cls.results_small_k = _one_query_results(index, cls.n, cls.small_k) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def reference(self, rows=None, small=False): + k = self.small_k if small else self.max_neighbors + return _sequential(self.results_small_k if small else self.results, self.n, self.threshold, k, + rows or _block_rows(self.n, k)) + + def test_fixture(self): + ref = self.reference() + sizes = np.array(ref.sizes) + self.assertGreater(len(sizes), 100) + self.assertGreater(np.count_nonzero(sizes == 1), 10) + self.assertGreater(np.count_nonzero(sizes > 5), 50) + self.assertNotEqual(self.n % 7, 0) + # At small_k many representatives keep their k-th neighbour, some with clusters below k. + small = self.reference(small=True) + self.assertGreater(small.n_saturated, 50) + self.assertGreater(small.n_at_cap, 10) + self.assertTrue(any(small.sizes[small.representatives.index(r)] < self.small_k + for r in small.saturated_reps)) + + def test_same_as_the_sequential_rule(self): + for small in (False, True): + with self.subTest(small_k=small): + k = self.small_k if small else self.max_neighbors + clusterer = self.greedy(self.db_path, self.threshold, k) + self.assert_same_as(clusterer, self.reference(small=small)) + self.assertEqual(clusterer.n_searched, self.n) # one block holds every chain + self.assert_cluster_ids(clusterer) + + def test_block_sizes(self): + """The same partition with blocks of 1, 7 (a short last block) and more than n chains; the searched chains + are those in no cluster at their block's start: as many as clusters with blocks of 1, n with one block.""" + for small in (False, True): + k = self.small_k if small else self.max_neighbors + for rows in (1, 7, self.n + 5): + with self.subTest(small_k=small, rows=rows): + clusterer = self.greedy(self.db_path, self.threshold, k, rows) + ref = self.reference(min(rows, self.n), small=small) + self.assert_same_as(clusterer, ref) + if rows == 1: + self.assertEqual(clusterer.n_searched, len(ref.sizes)) + elif rows > self.n: + self.assertEqual(clusterer.n_searched, self.n) + else: + self.assertLess(len(ref.sizes), clusterer.n_searched) + self.assertLess(clusterer.n_searched, self.n) + + def test_blocks_read_and_searched(self): + """Each block is read once with reconstruct_n, only when it has a chain in no cluster, and one FAISS call + searches those chains' stored vectors, in ascending order.""" + clusterer = EmbeddingClusterer(self.db_path) + stored = clusterer.db.index.reconstruct_n(0, self.n) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + with mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", _budget(7, self.max_neighbors)): + clusterer.cluster_greedy(threshold=self.threshold, max_neighbors=self.max_neighbors) + ref = self.reference(7) + expected_reads, expected_rows = [], [] + for start in range(0, self.n, 7): + end = min(start + 7, self.n) + # A chain is in a cluster from its representative's turn on + todo = [i for i in range(start, end) if ref.label[i] >= start] + if todo: + expected_reads.append((start, end - start)) + expected_rows.append(todo) + self.assertEqual(sum(map(len, expected_rows)), ref.n_searched) + self.assertEqual(recorder.reads, expected_reads) + self.assertLess(len(expected_reads), math.ceil(self.n / 7)) + self.assertEqual(len(recorder.searches), len(expected_rows)) + for (queries, k, _, _), rows in zip(recorder.searches, expected_rows): + self.assertEqual(k, self.max_neighbors) + self.assertTrue(queries.flags.c_contiguous) + self.assertEqual(queries.dtype, np.float32) + self.assertTrue(np.array_equal(queries, stored[rows])) + + def test_row_cap_sets_the_blocks(self): + """With the budget left large (more than n rows at k 40), _GRAPH_MAX_ROWS of 7 makes the blocks 7 chains: + each read once, only when it has a chain in no cluster, and the searched chains and their count as the + reference at 7 rows.""" + self.assertGreater(clustering._GRAPH_RESULT_BYTES // (self.max_neighbors * 12), self.n) + clusterer = EmbeddingClusterer(self.db_path) + stored = clusterer.db.index.reconstruct_n(0, self.n) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + with mock.patch.object(clustering, "_GRAPH_MAX_ROWS", 7): + clusterer.cluster_greedy(threshold=self.threshold, max_neighbors=self.max_neighbors) + ref = self.reference(7) + expected_reads, expected_rows = [], [] + for start in range(0, self.n, 7): + end = min(start + 7, self.n) + todo = [i for i in range(start, end) if ref.label[i] >= start] + if todo: + expected_reads.append((start, end - start)) + expected_rows.append(todo) + self.assertEqual(recorder.reads, expected_reads) + self.assertEqual([len(q) for q, _, _, _ in recorder.searches], [len(r) for r in expected_rows]) + for (queries, _, _, _), rows in zip(recorder.searches, expected_rows): + self.assertTrue(np.array_equal(queries, stored[rows])) + self.assertLess(ref.n_searched, self.n) + self.assert_same_as(clusterer, ref) + + def test_int64_labels_when_the_largest_id_is_above_the_int32_limit(self): + ref = self.reference() + for max_id, dtype in ((self.n - 1, np.int32), (self.n - 2, np.int64)): + with self.subTest(max_id=max_id), mock.patch.object(clustering, "_GRAPH_INT32_MAX_ID", max_id): + clusterer = self.greedy(self.db_path, self.threshold, self.max_neighbors) + self.assertEqual(clusterer.clusters.dtype, dtype) + self.assert_same_as(clusterer, ref) + + +class FlatGreedyTest(_GreedyCase): + """flat below FAISS's BLAS switch: one query per call and one block of every chain give the same scores, so the + same partition bit for bit.""" + + n, d, threshold, max_neighbors = 1500, 64, 0.5, 20 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + vectors = _clustered_vectors(seed=5, n=cls.n, d=cls.d, n_centers=60, noise=1.0) + cls.db_path = _build(Path(cls.tmp.name), "flat", vectors, IndexType.flat) + index = _load(cls.db_path).index + cls.results = _one_query_results(index, cls.n, cls.max_neighbors) + cls.results_all = _one_query_results(index, cls.n, cls.n) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_fixture(self): + # Every FAISS call of the tests stays below the BLAS switch (nq * d < distance_compute_blas_threshold). + self.assertLess(self.n * self.d, faiss.cvar.distance_compute_blas_threshold) + ref = _sequential(self.results, self.n, self.threshold, self.max_neighbors, self.n) + self.assertGreater(len(ref.sizes), 50) + self.assertGreater(ref.n_saturated, 10) + self.assertGreater(ref.n_at_cap, 0) + + def test_same_as_the_sequential_rule(self): + for k, results in ((self.max_neighbors, self.results), (self.n, self.results_all)): + for rows in (None, 1, 7, self.n + 5): + with self.subTest(k=k, rows=rows): + clusterer = self.greedy(self.db_path, self.threshold, k, rows) + ref = _sequential(results, self.n, self.threshold, k, min(rows or self.n, self.n)) + self.assert_same_as(clusterer, ref) + self.assert_cluster_ids(clusterer) + + def test_max_neighbors_above_n(self): + """max_neighbors above n is clamped to n: every search asks for n neighbours, and blocks are sized at k = n.""" + for rows in (None, 7): + with self.subTest(rows=rows): + clusterer = EmbeddingClusterer(self.db_path) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + budget = clustering._GRAPH_RESULT_BYTES if rows is None else _budget(rows, self.n) + with mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", budget): + clusterer.cluster_greedy(threshold=self.threshold, max_neighbors=self.n + 10) + ref = _sequential(self.results_all, self.n, self.threshold, self.n, rows or self.n, self.n + 10) + self.assert_same_as(clusterer, ref) + self.assertEqual(clusterer.n_saturated, 0) + self.assertEqual(clusterer.k, self.n) + self.assertTrue(recorder.searches) + self.assertEqual({k for _, k, _, _ in recorder.searches}, {self.n}) + self.assertEqual({count for _, count in recorder.reads[:-1]}, {rows} if rows else set()) + + +def _blobs(seed: int, sizes, d: int, noise: float, scattered: int = 0) -> tuple: + """Tight blobs of the given sizes around orthogonal centres (the first axes), plus ``scattered`` random unit + vectors in the other axes, in a random order: the vectors and each chain's blob (-1 for scattered chains).""" + rng = np.random.default_rng(seed) + blob = np.repeat(np.arange(len(sizes)), sizes) + vectors = np.zeros((len(blob), d)) + vectors[np.arange(len(blob)), blob] = 1.0 + vectors += noise * rng.standard_normal(vectors.shape) / math.sqrt(d) + if scattered: + other = np.zeros((scattered, d)) + other[:, len(sizes):] = rng.standard_normal((scattered, d - len(sizes))) + vectors = np.concatenate([vectors, other]) + blob = np.concatenate([blob, np.full(scattered, -1)]) + vectors /= np.linalg.norm(vectors, axis=1, keepdims=True) + order = rng.permutation(len(blob)) + return vectors[order].astype(np.float32), blob[order] + + +class PlantedClustersTest(_GreedyCase): + """Well-separated blobs smaller than k, a threshold between the intra- and inter-blob scores: each blob is one + cluster, represented by its lowest chain index, and no representative keeps its k-th neighbour.""" + + d, threshold, max_neighbors = 32, 0.5, 40 + sizes = (30, 1, 7, 12, 25, 3, 18, 9, 30, 2, 14, 1) + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.vectors, cls.blob = _blobs(seed=13, sizes=cls.sizes, d=cls.d, noise=0.1) + cls.n = len(cls.blob) + cls.db_path = _build(Path(cls.tmp.name), "planted", cls.vectors, IndexType.flat) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_fixture(self): + scores = self.vectors @ self.vectors.T + same = self.blob[:, None] == self.blob[None, :] + self.assertGreater(scores[same].min(), self.threshold + 0.2) + self.assertLess(scores[~same].max(), self.threshold - 0.2) + self.assertLess(max(self.sizes), self.max_neighbors) + + def test_each_blob_is_one_cluster(self): + for rows in (None, 1, 7): + with self.subTest(rows=rows): + clusterer = self.greedy(self.db_path, self.threshold, self.max_neighbors, rows) + for b, size in enumerate(self.sizes): + chains = np.flatnonzero(self.blob == b) + self.assertEqual(len(set(clusterer.clusters[chains].tolist())), 1) + cluster = clusterer.clusters[chains[0]] + self.assertEqual(clusterer.representatives[cluster], chains.min()) + self.assertEqual(np.count_nonzero(clusterer.clusters == cluster), size) + self.assertEqual(len(clusterer.representatives), len(self.sizes)) + self.assertEqual(clusterer.n_saturated, 0) + self.assert_cluster_ids(clusterer) + stats = clusterer.get_cluster_statistics() + self.assertEqual((stats["n_clusters"], stats["n_singletons"], stats["max_cluster_size"], + stats["n_at_cap"]), (len(self.sizes), 2, 30, 0)) + + +class _Recorder: + """The database index, recording the rows read (and a copy of what reconstruct_n returned) and each search's + queries and results.""" + + def __init__(self, index): + self._index = index + self.reads = [] + self.read_vectors = [] + self.searches = [] + + def reconstruct_n(self, start, n): + self.reads.append((start, n)) + vectors = self._index.reconstruct_n(start, n) + self.read_vectors.append(vectors.copy()) + return vectors + + def search(self, x, k): + scores, ids = self._index.search(x, k) + self.searches.append((x.copy(), k, scores.copy(), ids.copy())) + return scores, ids + + def __getattr__(self, name): + return getattr(self._index, name) + + +class SizeCapTest(_GreedyCase): + """One blob larger than k: no cluster above k + 1, clusters at the cap, saturated representatives (some with + clusters below k), none at k = n; the WARNING when most representatives are saturated.""" + + d, threshold, k = 32, 0.5, 50 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.vectors, cls.blob = _blobs(seed=17, sizes=(300,), d=cls.d, noise=0.6, scattered=100) + cls.n = len(cls.blob) + cls.db_path = _build(Path(cls.tmp.name), "cap", cls.vectors, IndexType.flat) + index = _load(cls.db_path).index + cls.results = _one_query_results(index, cls.n, cls.k) + cls.results_all = _one_query_results(index, cls.n, cls.n) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_fixture(self): + self.assertLess(self.n * self.d, faiss.cvar.distance_compute_blas_threshold) + ref = _sequential(self.results, self.n, self.threshold, self.k, self.n) + self.assertGreater(ref.n_at_cap, 0) + self.assertGreater(ref.n_saturated, ref.n_at_cap) + self.assertLess(ref.n_saturated, len(ref.sizes) / 2) + + def test_cap_and_saturation(self): + for rows in (None, 1, 7): + with self.subTest(rows=rows): + clusterer = self.greedy(self.db_path, self.threshold, self.k, rows) + ref = _sequential(self.results, self.n, self.threshold, self.k, min(rows or self.n, self.n)) + self.assert_same_as(clusterer, ref) + sizes = np.bincount(clusterer.clusters) + self.assertLessEqual(sizes.max(), self.k + 1) + # A saturated representative whose cluster is below k: its row's earlier ids were taken already. + below = [r for r in ref.saturated_reps if ref.sizes[ref.representatives.index(r)] < self.k] + self.assertGreater(len(below), 0) + + def test_nothing_saturated_at_k_equal_n(self): + """At k = n every chain is in the row: nothing is cut, even when the row's last cell is kept.""" + for threshold in (self.threshold, -1.0): + with self.subTest(threshold=threshold): + clusterer = self.greedy(self.db_path, threshold, self.n) + ref = _sequential(self.results_all, self.n, threshold, self.n, self.n) + self.assert_same_as(clusterer, ref) + self.assertEqual(clusterer.n_saturated, 0) + self.assertEqual(len(clusterer.representatives), 1) + + def test_warning_when_most_representatives_are_saturated(self): + warning = ("Most clusters are bounded by --max-neighbors, not --threshold: a representative or singleton may " + "score at or above --threshold with an earlier representative.") + clusterer = self.greedy(self.db_path, self.threshold, self.k) + with self.assertLogs(level="INFO") as logs: + clusterer.print_statistics() + self.assertNotIn(f"WARNING:root:{warning}", logs.output) + n_clusters = len(clusterer.representatives) + self.assertIn(f"INFO:root:Representatives with all --max-neighbors neighbours at or above --threshold: " + f"{clusterer.n_saturated} of {n_clusters} ({clusterer.n_saturated / n_clusters:.2%}); their " + f"clusters may be cut, and a larger --max-neighbors can change them.", + logs.output) + # Every cell kept: every representative saturated. + clusterer = self.greedy(self.db_path, -1.0, self.k) + self.assertEqual(clusterer.n_saturated, len(clusterer.representatives)) + with self.assertLogs(level="INFO") as logs: + clusterer.print_statistics() + self.assertEqual(logs.output.count(f"WARNING:root:{warning}"), 1) + # The bound: above 50%, not at it. + stats = clusterer.get_cluster_statistics() + for n_saturated, warned in ((2, False), (3, True)): + with self.subTest(n_saturated=n_saturated): + four = dict(stats, n_clusters=4, n_saturated=n_saturated) + with mock.patch.object(clusterer, "get_cluster_statistics", return_value=four), \ + self.assertLogs(level="INFO") as logs: + clusterer.print_statistics() + self.assertEqual(f"WARNING:root:{warning}" in logs.output, warned) + + +class IvfPqGreedyTest(_GreedyCase): + """ivf_pq (no exact reference across batch sizes): invariants against the result rows the method used, recorded by + a spy on the index; the queries are the normalised decoded embeddings.""" + + n, d, threshold = 1000, 64, 0.5 + config = dict(nlist=16, m=8, nbits=4, nprobe=16, opq_train_size=1000) + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + vectors = _clustered_vectors(seed=7, n=cls.n, d=cls.d, n_centers=40, noise=0.8) + # Planted exact duplicates: chain 500 + t is chain 10 * t. + cls.duplicates = [(10 * t, 500 + t) for t in range(20)] + for a, b in cls.duplicates: + vectors[b] = vectors[a] + cls.db_path = _build(Path(cls.tmp.name), "pq", vectors, IndexType.ivf_pq, IndexConfig(**cls.config)) + decoded = _load(cls.db_path).index.reconstruct_n(0, cls.n) + cls.unnormalised = decoded + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def spied(self, threshold, max_neighbors, rows=None, nprobe=None): + clusterer = EmbeddingClusterer(self.db_path) + if nprobe is not None: + clusterer.db.set_nprobe(nprobe) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + budget = clustering._GRAPH_RESULT_BYTES if rows is None else _budget(rows, min(max_neighbors, self.n)) + with mock.patch.object(clustering, "_GRAPH_RESULT_BYTES", budget): + clusterer.cluster_greedy(threshold=threshold, max_neighbors=max_neighbors) + return clusterer, recorder + + def searched_rows(self, recorder): + """Each searched chain's recorded (scores, ids), found by its query: within the block read before the search, + the queries are the normalised decoded rows of the chains searched, in ascending order.""" + self.assertEqual(len(recorder.reads), len(recorder.searches)) + rows = {} + for (start, count), read, (queries, k, scores, ids) in zip(recorder.reads, recorder.read_vectors, + recorder.searches): + block = read.copy() + faiss.normalize_L2(block) + at = 0 + for q, query in enumerate(queries): + while at < count and not np.array_equal(block[at], query): + at += 1 + self.assertLess(at, count, "a query is not a normalised decoded row of its block, in order") + self.assertNotIn(start + at, rows) + rows[start + at] = (scores[q], ids[q]) + at += 1 + return rows + + def assert_invariants(self, clusterer, recorder, threshold): + threshold32 = np.float32(threshold) + rows = self.searched_rows(recorder) + self.assertEqual(len(rows), clusterer.n_searched) + clusters, reps = clusterer.clusters, clusterer.representatives + self.assertEqual(len(clusters), self.n) + self.assertTrue(np.all((clusters >= 0) & (clusters < len(reps)))) + rep_of = reps[clusters] + # Every label a representative, each representative searched and its own representative + self.assertTrue(np.array_equal(rep_of[reps], reps)) + self.assertTrue(set(reps.tolist()) <= set(rows)) + for chain in range(self.n): + rep = int(rep_of[chain]) + scores, ids = rows[rep] + if chain != rep: + # A member is in its representative's row, at a score at least the threshold + at = np.flatnonzero(ids == chain) + self.assertEqual(len(at), 1, f"chain {chain} not in its representative {rep}'s row") + self.assertGreaterEqual(scores[at[0]], threshold32) + for rep in reps.tolist(): + scores, ids = rows[rep] + kept = ids[(ids >= 0) & (scores >= threshold32)] + # Every kept id is taken by the time the representative's turn is over: by it or an earlier one + self.assertTrue(np.all(rep_of[kept] <= rep)) + self.assert_cluster_ids(clusterer) + return rows + + def test_invariants(self): + for max_neighbors, rows in ((self.n, None), (self.n, 7), (self.n, 1), (50, None), (50, 7)): + with self.subTest(max_neighbors=max_neighbors, rows=rows): + clusterer, recorder = self.spied(self.threshold, max_neighbors, rows) + self.assert_invariants(clusterer, recorder, self.threshold) + self.assertGreater(len(clusterer.representatives), 10) + self.assertGreater(np.bincount(clusterer.clusters).max(), 10) + if max_neighbors == self.n: + for a, b in self.duplicates: + self.assertEqual(clusterer.clusters[a], clusterer.clusters[b]) + self.assertEqual(clusterer.n_saturated, 0) + + def test_queries_are_normalised(self): + """The queries are unit vectors (searched_rows finds each among its block's normalised decoded rows), and + normalising matters on this fixture: the decoded embeddings are not unit vectors.""" + norms = np.linalg.norm(self.unnormalised, axis=1) + self.assertGreater(np.abs(norms - 1).max(), 1e-3) + for rows in (None, 7): + with self.subTest(rows=rows): + clusterer, recorder = self.spied(self.threshold, 50, rows) + self.assertEqual(len(self.searched_rows(recorder)), clusterer.n_searched) + searched = np.concatenate([q for q, _, _, _ in recorder.searches]) + np.testing.assert_allclose(np.linalg.norm(searched, axis=1), 1, rtol=0, atol=1e-5) + + def test_fill_ids_are_never_labels(self): + """With nprobe < nlist and max_neighbors >= n, FAISS fills rows with -1 (scored -FLT_MAX): no chain is labelled + from them, even at a threshold of -inf, which every fill's score passes.""" + for threshold in (-10.0, float("-inf")): + with self.subTest(threshold=threshold): + clusterer, recorder = self.spied(threshold, self.n + 5, nprobe=1) + fills = [scores[ids == -1] for _, _, scores, ids in recorder.searches] + self.assertGreater(sum(map(len, fills)), 0) + if threshold == float("-inf"): + self.assertTrue(all(np.all(f >= threshold) for f in fills)) + self.assert_invariants(clusterer, recorder, threshold) + + def test_no_direct_map_and_parallel_mode_3(self): + clusterer = EmbeddingClusterer(self.db_path) + clusterer.cluster_greedy(threshold=self.threshold, max_neighbors=50) + ivf = faiss.extract_index_ivf(clusterer.db.index) + self.assertEqual(ivf.direct_map.type, faiss.DirectMap.NoMap) + self.assertEqual(ivf.parallel_mode, 3) + + +class StatisticsTest(unittest.TestCase): + """The modularity is for Leiden partitions only; greedy adds its own keys and lines.""" + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + vectors, _ = _blobs(seed=19, sizes=(20, 5, 9, 1, 14), d=16, noise=0.3) + cls.n = len(vectors) + cls.db_path = _build(Path(cls.tmp.name), "stats", vectors, IndexType.flat) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_greedy_after_a_graph_then_leiden(self): + clusterer = EmbeddingClusterer(self.db_path) + self.assertIsNone(clusterer.method) + self.assertIsNone(clusterer.representatives) + clusterer.build_similarity_graph(threshold=0.5, max_neighbors=10) + clusterer.cluster_greedy(threshold=0.5, max_neighbors=10) + self.assertIsNotNone(clusterer.graph) + stats = clusterer.get_cluster_statistics() + self.assertIsNone(stats["modularity"]) + self.assertEqual(stats["method"], "greedy") + with self.assertLogs(level="INFO") as logs: + clusterer.print_statistics() + self.assertFalse(any("Modularity" in line for line in logs.output)) + n_clusters = stats["n_clusters"] + self.assertIn(f"INFO:root:Chains searched: {stats['n_searched']} of {self.n} " + f"({stats['n_searched'] / self.n:.2%})", logs.output) + self.assertIn(f"INFO:root:Clusters of at least --max-neighbors chains: {stats['n_at_cap']}", logs.output) + self.assertIn("INFO:root:Method: greedy", logs.output) + sizes = np.bincount(clusterer.clusters) + self.assertEqual(stats["n_at_cap"], np.count_nonzero(sizes >= 10)) + self.assertGreater(stats["n_at_cap"], 0) + self.assertLessEqual(stats["max_cluster_size"], 11) + + out = Path(self.tmp.name) / "greedy.tsv" + clusterer.export_clusters(str(out)) + self.assertEqual({len(line.split("\t")) for line in out.read_text().splitlines()}, {4}) + + clusterer.cluster_leiden(seed=1) + self.assertEqual(clusterer.method, "leiden") + self.assertIsNone(clusterer.representatives) + stats = clusterer.get_cluster_statistics() + self.assertEqual(stats["modularity"], clusterer.graph.modularity(clusterer.clusters, weights="weight")) + self.assertNotIn("method", stats) + self.assertNotIn("n_searched", stats) + with self.assertLogs(level="INFO") as logs: + clusterer.print_statistics() + self.assertTrue(any(line.startswith("INFO:root:Modularity:") for line in logs.output)) + self.assertFalse(any("searched" in line for line in logs.output)) + clusterer.export_clusters(str(out)) + self.assertEqual({len(line.split("\t")) for line in out.read_text().splitlines()}, {3}) + + def test_a_partition_set_without_either_method_keeps_the_modularity(self): + clusterer = EmbeddingClusterer(self.db_path) + clusterer.build_similarity_graph(threshold=0.5, max_neighbors=10) + clusterer.clusters = [0] * self.n + stats = clusterer.get_cluster_statistics() + self.assertEqual(stats["modularity"], clusterer.graph.modularity(clusterer.clusters, weights="weight")) + + +class EdgeCaseTest(_GreedyCase): + + n, d = 60, 16 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + folder = Path(cls.tmp.name) + vectors, _ = _blobs(seed=23, sizes=(10, 10, 10, 10, 10, 4), d=cls.d, noise=0.5, scattered=6) + cls.db_path = _build(folder, "flat", vectors, IndexType.flat) + cls.one_path = _build(folder, "one", vectors[:1], IndexType.flat) + # Scores a threshold can equal exactly: 0.6 and 0.8 from chain 0, and 0 with the last chain. + cls.exact_vectors = np.array([[1, 0, 0], [0.6, 0.8, 0], [0.8, 0.6, 0], [0, 0, 1]], dtype=np.float32) + cls.exact_path = _build(folder, "exact", cls.exact_vectors, IndexType.flat) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_one_chain(self): + clusterer = self.greedy(self.one_path, 0.0, 10) + self.assertEqual(clusterer.clusters.tolist(), [0]) + self.assertEqual(clusterer.representatives.tolist(), [0]) + self.assertEqual((clusterer.n_searched, clusterer.n_saturated), (1, 0)) + stats = clusterer.get_cluster_statistics() + self.assertEqual((stats["n_clusters"], stats["n_singletons"], stats["n_at_cap"]), (1, 1, 0)) + + def test_threshold_above_every_score(self): + clusterer = self.greedy(self.db_path, 2.0, 10) + self.assertEqual(clusterer.clusters.tolist(), list(range(self.n))) + self.assertEqual(clusterer.representatives.tolist(), list(range(self.n))) + self.assertEqual((clusterer.n_searched, clusterer.n_saturated), (self.n, 0)) + stats = clusterer.get_cluster_statistics() + self.assertEqual((stats["n_clusters"], stats["n_singletons"], stats["max_cluster_size"]), + (self.n, self.n, 1)) + self.assertIsNone(stats["modularity"]) + clusterer.print_statistics() + + def test_max_neighbors_above_n(self): + ref = _sequential(_one_query_results(_load(self.db_path).index, self.n, self.n), self.n, 0.5, self.n, self.n, + self.n + 10) + self.assertGreater(len(ref.sizes), 6) + clusterer = self.greedy(self.db_path, 0.5, self.n + 10) + self.assert_same_as(clusterer, ref) + + def test_score_equal_to_threshold_is_taken(self): + """A float32 score equal to the threshold takes the chain: 0 (the axis chain), 0.6, the score itself, and a + float64 just above it that rounds to it in float32, as the graph's edges.""" + stored = _load(self.exact_path).index.reconstruct_n(0, 4) + score = float(stored[0] @ stored[1]) + self.assertEqual(np.float32(np.nextafter(score, 1.0)), np.float32(score)) + for threshold in (0.0, 0.6, score, float(np.nextafter(score, 1.0))): + with self.subTest(threshold=threshold): + clusterer = self.greedy(self.exact_path, threshold, 10) + self.assertEqual(clusterer.clusters[1], clusterer.clusters[0]) + self.assertEqual(clusterer.representatives[clusterer.clusters[1]], 0) + if threshold == 0.0: + self.assertEqual(clusterer.clusters.tolist(), [0, 0, 0, 0]) + + def test_max_neighbors_below_1_is_refused_before_any_read(self): + clusterer = EmbeddingClusterer(self.db_path) + recorder = _Recorder(clusterer.db.index) + clusterer.db.index = recorder + for max_neighbors in (0, -1): + with self.subTest(max_neighbors=max_neighbors): + with self.assertRaisesRegex(ValueError, r"^--max-neighbors must be at least 1\.$"): + clusterer.cluster_greedy(threshold=0.5, max_neighbors=max_neighbors) + self.assertEqual((recorder.reads, recorder.searches), ([], [])) + self.assertIsNone(clusterer.clusters) + + +def _reference_leiden(graph, seed: int, n_iterations: int = 10) -> list: + """P9.U's Leiden call at resolution 1, made here and not through the code under test (tests/test_clustering.py's + reference): python-igraph's community_leiden seeded with random.seed (Python's global state saved and restored), + cluster ids renumbered in pure Python by (-size, smallest chain index).""" + state = random.getstate() + try: + random.seed(seed) + membership = graph.community_leiden(objective_function="modularity", weights="weight", resolution=1.0, + beta=0.01, n_iterations=n_iterations).membership + finally: + random.setstate(state) + members = {} + for chain, cluster in enumerate(membership): + members.setdefault(cluster, []).append(chain) + order = sorted(members, key=lambda c: (-len(members[c]), members[c][0])) + new_id = {cluster: rank for rank, cluster in enumerate(order)} + return [new_id[cluster] for cluster in membership] + + +def _c40d4e1_export(chain_ids, membership, output_file: str, min_cluster_size=None, + format_mode=FormatMode.headless_tsv): + """c40d4e1's export_clusters and write_cluster_results, verbatim but for the file handling: a list of every + (chain_id, cluster_id, cluster_size), then each row joined from its three str().""" + cluster_sizes = np.bincount(membership).tolist() + assignments = [ + (chain_id, cluster_id, cluster_sizes[cluster_id]) + for chain_id, cluster_id in zip(chain_ids, membership) + if min_cluster_size is None or cluster_sizes[cluster_id] >= min_cluster_size + ] + spec = format_spec(format_mode) + lines = [spec.delimiter.join(CLUSTER_OUTPUT_FIELDS)] if spec.header else [] + lines += [spec.delimiter.join((str(chain_id), str(cluster_id), str(cluster_size))) + for chain_id, cluster_id, cluster_size in assignments] + Path(output_file).write_text("".join(line + "\n" for line in lines)) + + +class ClusterCommandTest(unittest.TestCase): + """cluster_database, called directly as test_cli_search's test_23: the greedy rows, and the Leiden file as + c40d4e1 wrote it for the reference membership, at several export chunks, with and without --min-cluster-size.""" + + n, d, threshold, max_neighbors, seed = 600, 128, 0.6, 20, 42 + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + vectors = _clustered_vectors(seed=29, n=cls.n, d=cls.d, n_centers=60, noise=1.0) + cls.db_path = _build(Path(cls.tmp.name), "hnsw", vectors, IndexType.hnsw) + clusterer = EmbeddingClusterer(cls.db_path) + cls.chain_ids = clusterer.chain_ids + graph = clusterer.build_similarity_graph(threshold=cls.threshold, max_neighbors=cls.max_neighbors) + # The Leiden call, made here and not through the code under test: the reference membership + cls.graph = graph + cls.membership = _reference_leiden(graph, cls.seed) + greedy = EmbeddingClusterer(cls.db_path) + greedy.cluster_greedy(threshold=cls.threshold, max_neighbors=cls.max_neighbors) + cls.greedy_clusters = greedy.clusters + cls.greedy_reps = greedy.representatives + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def chunks(self): + # 1, 3, an exact divisor of n, and more than n (None: the default, also more than n) + self.assertEqual(self.n % 120, 0) + self.assertNotEqual(self.n % 7, 0) + return (None, 1, 3, 7, 120, self.n + 5) + + def test_fixture(self): + sizes = np.bincount(self.greedy_clusters) + self.assertGreater(np.count_nonzero(sizes == 1), 5) + self.assertGreater(np.count_nonzero(sizes >= 2), 20) + + def test_greedy_rows(self): + sizes = np.bincount(self.greedy_clusters) + for chunk in self.chunks(): + for min_size in (None, 2): + with self.subTest(chunk=chunk, min_cluster_size=min_size): + out = Path(self.tmp.name) / "greedy.tsv" + with mock.patch.object(clustering, "_EXPORT_CHUNK", chunk or clustering._EXPORT_CHUNK): + cluster_database(db_path=self.db_path, method=ClusterMethod.greedy, threshold=self.threshold, + output_file=str(out), max_neighbors=self.max_neighbors, + min_cluster_size=min_size, format_mode=FormatMode.tsv) + lines = out.read_text().splitlines() + self.assertEqual(lines[0].split("\t"), list(GREEDY_CLUSTER_OUTPUT_FIELDS)) + expected = [ + f"{self.chain_ids[i]}\t{c}\t{sizes[c]}\t{self.chain_ids[self.greedy_reps[c]]}" + for i, c in enumerate(self.greedy_clusters.tolist()) if min_size is None or sizes[c] >= min_size] + self.assertEqual(lines[1:], expected) + rows = [line.split("\t") for line in lines[1:]] + counts = Counter(cluster for _, cluster, _, _ in rows) + for chain, cluster, size, rep in rows: + self.assertEqual(int(size), counts[cluster]) + if rep == chain: + self.assertEqual(self.greedy_reps[int(cluster)], self.chain_ids.index(chain)) + self.assertEqual(sum(1 for chain, _, _, rep in rows if chain == rep), len(counts)) + if min_size: + self.assertTrue(all(int(size) >= 2 for _, _, size, _ in rows)) + self.assertLess(len(rows), self.n) + # Headless: the same rows without the header + with mock.patch.object(clustering, "_EXPORT_CHUNK", chunk or clustering._EXPORT_CHUNK): + cluster_database(db_path=self.db_path, method="greedy", threshold=self.threshold, + output_file=str(out), max_neighbors=self.max_neighbors, + min_cluster_size=min_size) + self.assertEqual(out.read_text().splitlines(), lines[1:]) + + def test_leiden_membership_as_reference(self): + """cluster_leiden returns the reference membership, cluster ids by non-increasing size (0 the largest).""" + clusterer = EmbeddingClusterer(self.db_path) + clusterer.build_similarity_graph(threshold=self.threshold, max_neighbors=self.max_neighbors) + self.assertEqual(clusterer.cluster_leiden(seed=self.seed).tolist(), self.membership) + sizes = np.bincount(self.membership) + self.assertGreater(len(set(sizes.tolist())), 2) + self.assertTrue(np.all(np.diff(sizes) <= 0), sizes.tolist()) + + def test_leiden_file_as_reference(self): + for chunk in self.chunks(): + for min_size in (None, 2): + for format_mode in (FormatMode.headless_tsv, FormatMode.tsv): + with self.subTest(chunk=chunk, min_cluster_size=min_size, format_mode=format_mode.value): + ref = Path(self.tmp.name) / "ref.tsv" + _c40d4e1_export(self.chain_ids, self.membership, str(ref), min_size, format_mode) + for method in (None, ClusterMethod.leiden): + out = Path(self.tmp.name) / "leiden.tsv" + kwargs = {} if method is None else {"method": method} + with mock.patch.object(clustering, "_EXPORT_CHUNK", chunk or clustering._EXPORT_CHUNK): + cluster_database(db_path=self.db_path, threshold=self.threshold, output_file=str(out), + max_neighbors=self.max_neighbors, min_cluster_size=min_size, + seed=self.seed, format_mode=format_mode, **kwargs) + self.assertEqual(out.read_bytes(), ref.read_bytes()) + self.assertGreater(len(ref.read_text().splitlines()), 100) + + def test_methods_route(self): + """greedy calls cluster_greedy and builds no graph; leiden (the default) the graph, then Leiden at resolution + 1.0 when --resolution is not given and 10 iterations when --leiden-iterations is not given.""" + out = str(Path(self.tmp.name) / "routed.tsv") + cases = [ + (dict(), [mock.call.build_similarity_graph(threshold=0.7, max_neighbors=9), + mock.call.cluster_leiden(resolution=1.0, n_iterations=10, seed=None)]), + (dict(method=ClusterMethod.leiden, resolution=2.5, seed=4), + [mock.call.build_similarity_graph(threshold=0.7, max_neighbors=9), + mock.call.cluster_leiden(resolution=2.5, n_iterations=10, seed=4)]), + (dict(leiden_iterations=3), [mock.call.build_similarity_graph(threshold=0.7, max_neighbors=9), + mock.call.cluster_leiden(resolution=1.0, n_iterations=3, seed=None)]), + (dict(method=ClusterMethod.greedy), [mock.call.cluster_greedy(threshold=0.7, max_neighbors=9)]), + ] + for kwargs, calls in cases: + with self.subTest(**{k: str(v) for k, v in kwargs.items()}): + with mock.patch.object(clustering, "EmbeddingClusterer") as made: + # A graph whose kept scores are all at least 0 + made.return_value.min_weight = 0.7 + cluster_database(db_path=self.db_path, threshold=0.7, max_neighbors=9, output_file=out, **kwargs) + clusterer = made.return_value + self.assertEqual([c for c in clusterer.mock_calls if c[0] in ( + "build_similarity_graph", "cluster_leiden", "cluster_greedy")], calls) + + def test_leiden_iterations_reach_igraph(self): + """cluster --leiden-iterations 3 --seed 5 runs python-igraph's Leiden at 3 iterations, and writes the reference + file at 3 iterations, seed 5; without the option, 10 iterations.""" + real = ig.Graph.community_leiden + calls = [] + + def community_leiden(graph, *args, **kwargs): + calls.append(kwargs["n_iterations"]) + return real(graph, *args, **kwargs) + + out = Path(self.tmp.name) / "iterations.tsv" + ref = Path(self.tmp.name) / "iterations_ref.tsv" + base = ("cluster", "--db-path", self.db_path, "--threshold", self.threshold, "--max-neighbors", + self.max_neighbors, "--output-file", out, "--seed", 5) + for extra, n_iterations in ((("--leiden-iterations", 3), 3), ((), 10)): + with self.subTest(extra=extra): + calls.clear() + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + with mock.patch.object(ig.Graph, "community_leiden", community_leiden): + result = CliRunner().invoke(search.app, [str(a) for a in (*base, *extra)]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + self.assertEqual(calls, [n_iterations]) + _c40d4e1_export(self.chain_ids, _reference_leiden(self.graph, 5, n_iterations), str(ref)) + self.assertEqual(out.read_bytes(), ref.read_bytes()) + + +def _flat(text: str) -> str: + """Text on one line, typer's error panel borders and colour codes dropped.""" + return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", text).translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +class ClusterRowLengthTest(unittest.TestCase): + """write_cluster_results refuses rows whose length does not match the fields.""" + + def test_rows_must_match_the_fields(self): + cases = [(GREEDY_CLUSTER_OUTPUT_FIELDS, [("a", 0, 1)], r"^cluster row \('a', 0, 1\) has 3 values for the 4 " + r"fields chain_id, cluster_id, cluster_size, representative$"), + (CLUSTER_OUTPUT_FIELDS, [("a", 0, 1, "a")], r"^cluster row \('a', 0, 1, 'a'\) has 4 values for the " + r"3 fields chain_id, cluster_id, cluster_size$")] + with tempfile.TemporaryDirectory() as d: + for fields, rows, message in cases: + with self.subTest(fields=fields), self.assertRaisesRegex(ValueError, message): + write_cluster_results(rows, str(Path(d) / "c.tsv"), FormatMode.tsv, fields) + + +class RefusalTest(unittest.TestCase): + """Through the CLI: exit status 2 before the database is loaded.""" + + def invoke(self, *args): + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + with mock.patch.object(clustering, "EmbeddingClusterer", + side_effect=AssertionError("the database was loaded")): + return CliRunner().invoke(search.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + def refused(self, *args) -> str: + result = self.invoke(*args) + self.assertEqual(result.exit_code, 2, f"{result.output}\n{result.exception!r}") + return _flat(result.output) + + def test_leiden_options_under_greedy_in_one_message(self): + base = ("cluster", "--db-path", "no/such/db", "--method", "greedy") + greedy = "--method greedy has no resolution or iteration count and is deterministic." + cases = ( + (("--resolution", "1.0"), f"--resolution is a Leiden option: {greedy} Remove it, or use --method leiden."), + (("--seed", "7"), f"--seed is a Leiden option: {greedy} Remove it, or use --method leiden."), + (("--seed", "7", "--resolution", "2"), f"--resolution, --seed are Leiden options: {greedy} Remove them, " + f"or use --method leiden."), + (("--leiden-iterations", "5"), f"--leiden-iterations is a Leiden option: {greedy} Remove it, or use " + f"--method leiden."), + # The greedy refusal comes first: not the minimum's message + (("--leiden-iterations", "0"), f"--leiden-iterations is a Leiden option: {greedy} Remove it, or use " + f"--method leiden."), + (("--seed", "-1"), f"--seed is a Leiden option: {greedy} Remove it, or use --method leiden."), + (("--seed", "3", "--leiden-iterations", "5", "--resolution", "2"), + f"--resolution, --leiden-iterations, --seed are Leiden options: {greedy} Remove them, or use --method " + f"leiden."), + ) + for extra, message in cases: + with self.subTest(extra=extra): + output = self.refused(*base, *extra) + self.assertIn(f"Invalid value for --method: {message}", output) + self.assertEqual(output.count("Leiden option"), 1) + + def test_max_neighbors_below_1(self): + for command in (("cluster",), ("cluster", "--method", "leiden"), ("cluster", "--method", "greedy"), + ("similarity-graph",)): + for value in ("0", "-3"): + with self.subTest(command=command, value=value): + output = self.refused(*command, "--db-path", "no/such/db", "--max-neighbors", value) + self.assertIn("Invalid value for --max-neighbors: --max-neighbors must be at least 1.", output) + + def test_leiden_options_with_leiden_are_not_refused(self): + for method in ((), ("--method", "leiden")): + for extra in (("--resolution", "2", "--seed", "3"), ("--leiden-iterations", "1"), + ("--resolution", "2", "--leiden-iterations", "1", "--seed", "0")): + with self.subTest(method=method, extra=extra): + result = self.invoke("cluster", "--db-path", "no/such/db", *method, *extra) + self.assertEqual(result.exit_code, 1) + self.assertEqual(str(result.exception), "the database was loaded") + + def test_leiden_iterations_below_1(self): + for method in ((), ("--method", "leiden")): + for value in ("0", "-2"): + with self.subTest(method=method, value=value): + output = self.refused("cluster", "--db-path", "no/such/db", *method, "--leiden-iterations", value) + self.assertIn("Invalid value for --leiden-iterations: --leiden-iterations must be at least 1.", + output) + + def test_leiden_iterations_must_be_an_integer(self): + for value in ("2.5", "ten"): + with self.subTest(value=value): + output = self.refused("cluster", "--db-path", "no/such/db", "--leiden-iterations", value) + self.assertIn("--leiden-iterations", output) + self.assertNotIn("must be at least 1", output) + + def test_seed_below_0(self): + for method in ((), ("--method", "leiden")): + for value in ("-1", "-5"): + with self.subTest(method=method, value=value): + output = self.refused("cluster", "--db-path", "no/such/db", *method, "--seed", value) + self.assertIn("Invalid value for --seed: --seed must be at least 0.", output) + + def test_leiden_iterations_checked_before_the_seed(self): + output = self.refused("cluster", "--db-path", "no/such/db", "--leiden-iterations", "0", "--seed", "-1") + self.assertIn("Invalid value for --leiden-iterations: --leiden-iterations must be at least 1.", output) + self.assertNotIn("--seed must be at least 0", output) + + def test_leiden_iterations_help(self): + command = typer.main.get_command(search.app).commands["cluster"] + text = next(p.help for p in command.params if p.name == "leiden_iterations") + self.assertEqual(text, "Leiden iterations, a fixed count (at least 1). Default: 10, which on two measured " + "1M-chain graphs agreed with Leiden run to convergence within seed noise; fewer are " + "faster, with somewhat noisier partitions. --method leiden only.") + self.assertIn(f"Default: {clustering._LEIDEN_ITERATIONS},", text) + + def test_unknown_method(self): + output = self.refused("cluster", "--db-path", "no/such/db", "--method", "components") + self.assertIn("--method", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_code_alignment.py b/tests/test_code_alignment.py new file mode 100644 index 0000000..f5d17f1 --- /dev/null +++ b/tests/test_code_alignment.py @@ -0,0 +1,1715 @@ +"""P6.4: the esm-codes Stage-2 aligner (``foldmatch.search.code_alignment``; plan section 8 P6, method S5-S14, S16). + +No ESM3: the encodings and codes are the fixtures' (F1 ``pairs_v1.npz``, F3 ``q90_queries_v1.npz``, F-deg +``degenerate_v1.npz``); the databases are built in the test with ``SequenceStore`` and ``ResidueCodeStore``, the query +encodings written as the per-run memmap of a ``QueryEncodingIndex``. The panel and the E-value table come from the +registry bundle (``esm_codes_bundle``: skipped only when offline and not cached). + +Tolerances are S16: S_f / S_r within 5e-5 max(|ref|, 1); paths identical unless the float64 path-score margin is below +1e-4 + 2e-6 |S_f|; Q90 and ln E within 1e-4 (F1, at N = 1e9); F3 Q90 within 1e-3 (the plan's bound). On one machine +the output is bit-identical for any task size, block size, query batch and worker count, and round 2 reproduces +round 1's S_f bit for bit. +""" +import contextlib +import dataclasses +import functools +import logging +import math +import os +import pickle +import signal +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +from concurrent.futures.process import BrokenProcessPool +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest + +import foldmatch +from foldmatch.errors import InternalError +from foldmatch.search import alignment, output, swkern +from foldmatch.search import code_statistics as cs +from foldmatch.search import code_alignment as ca +from foldmatch.search import scoring_system as ss +from foldmatch.search import stage2_profile +from foldmatch.search.query_encodings import QueryEncodingEntry, QueryEncodingIndex +from foldmatch.search.residue_code_store import ResidueCodeStore, ResidueCodeStoreError +from foldmatch.search.sequence_store import SequenceStore +from foldmatch.utils.sequence import STANDARD_AMINO_ACIDS, seq_sha1 + +_RES = Path(__file__).parent / "resources" / "esm_codes" +SYSTEM = ss.get(ss.DEFAULT_ESM_CODES) +ESM_REV = "636acb420daf674df37ab0220f56061a5de2f169" # passed explicitly: the code store needs no torch +N_F1 = 10 ** 9 # the fixtures' database size for ln E +ALL_FIELDS = list(output.SUPPORTED_OUTPUT_FIELDS) +_AA = np.frombuffer(STANDARD_AMINO_ACIDS.encode(), np.uint8) + + +# ===================================================================================================================== +# data +# ===================================================================================================================== +@functools.lru_cache(maxsize=None) +def _prepared(): + import esm_codes_bundle + esm_codes_bundle.registry_files() # unittest.SkipTest only when offline and not cached + return ss.prepare(SYSTEM) + + +@functools.lru_cache(maxsize=None) +def _f1(): + z = np.load(_RES / "pairs_v1.npz") + qo, to = z["q_offsets"], z["t_offsets"] + pairs = [] + for p in range(len(z["pair_q"])): + a, b = int(z["pair_q"][p]), int(z["pair_t"][p]) + pairs.append(dict( + qid=str(z["q_ids"][a]), q16=z["q"][qo[a]:qo[a + 1]], qseq=str(z["q_sequences"][a]), + tid=str(z["t_ids"][b]), codes=z["codes"][to[b]:to[b + 1]], tseq=str(z["t_sequences"][b]), + S_f=float(z["S_f"][p]), S_r=float(z["S_r"][p]), q90=float(z["q90"][p]), lnE=float(z["lnE_N1e9"][p]), + path=z["path_flat"][z["path_off"][p]:z["path_off"][p + 1]], n_ident=int(z["n_ident"][p]), + aln_cols=int(z["aln_cols"][p]))) + return pairs + + +def _random_seq(rng, n): + return _AA[rng.integers(0, len(_AA), n)].tobytes().decode("ascii") + + +@contextlib.contextmanager +def _deadline(seconds: float): + """TimeoutError in this (the main) thread after ``seconds``: a test of a wait that must end fails instead.""" + def expired(*_): + raise TimeoutError(f"still waiting after {seconds} s") + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def _run_script(source: str, folder: Path, timeout: float = 120.0): + """Run ``source`` as a script file (spawn workers re-import a main *file*) in its own session, with this tree on + PYTHONPATH; on timeout the session (the script and its workers) is killed. Returns ``(returncode, output)``, + returncode None on a timeout.""" + script = folder / "script.py" + script.write_text(textwrap.dedent(source)) + src = str(Path(foldmatch.__file__).resolve().parents[1]) + paths = [src, str(Path(__file__).resolve().parent), os.environ.get("PYTHONPATH", "")] + env = dict(os.environ, PYTHONPATH=os.pathsep.join(p for p in paths if p)) + proc = subprocess.Popen([sys.executable, str(script)], cwd=folder, env=env, start_new_session=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + out, _ = proc.communicate(timeout=timeout) + return proc.returncode, out + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + out, _ = proc.communicate() + return None, out + + +class _ListHandler(logging.Handler): + def __init__(self): + super().__init__(logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +class World: + """A database (sequence store + residue-code store) and a run's query-encoding memmap in a temporary folder.""" + + def __init__(self, folder: Path, targets, queries, name="db"): + """``targets``: ``[(id, sequence, codes uint8 [m, 8])]``; ``queries``: ``[(id, sequence, q fp16 [n, 64])]``.""" + self.folder = Path(folder) + fasta = self.folder / f"{name}.fasta" + with open(fasta, "w") as f: + for tid, seq, _ in targets: + f.write(f">{tid}\n{seq}\n") + self.seq_store = SequenceStore(self.folder, name) + self.seq_store.create(fasta) + self.store = ResidueCodeStore(self.folder, name) + self.store.create([([t[0] for t in targets], [np.asarray(t[2], np.uint8) for t in targets], + [seq_sha1(t[1]) for t in targets])], SYSTEM, esm_revision=ESM_REV) + self.store.validate(self.seq_store, len(targets), preflight=None) + self.residues = self.store.manifest()["total_residues"] + self.query_index = self.make_query_index(queries, f"{name}.queries.f16") + self.query_sequences = {qid: seq for qid, seq, _ in queries} + + def make_query_index(self, queries, filename): + path = self.folder / filename + entries, rows = {}, 0 + with open(path, "wb") as f: + for qid, seq, q16 in queries: + f.write(np.ascontiguousarray(q16, " World: + if "f1" not in cls._worlds: + f1 = _f1() + cls._worlds["f1"] = World(cls.tmp, [(p["tid"], p["tseq"], p["codes"]) for p in f1], + [(p["qid"], p["qseq"], p["q16"]) for p in f1], name="f1") + return cls._worlds["f1"] + + +def _f1_own_target(): + """Each F1 query against its own target (the fixture pairs).""" + return {p["qid"]: ([p["tid"]], [0.5]) for p in _f1()} + + +def _f1_mixed(queries=None, per_query=8, stride=7): + """Each F1 query against its own target and ``per_query - 1`` others, with decreasing embedding scores.""" + f1 = _f1() + picks = range(len(f1)) if queries is None else queries + out = {} + for a in picks: + tids = [f1[(a + j * stride) % len(f1)]["tid"] for j in range(per_query)] + out[f1[a]["qid"]] = (tids, [1.0 - 0.05 * j for j in range(per_query)]) + return out + + +def _signature(results): + """Everything the output can show, bit for bit: every metrics field (floats as hex) and the rendered row.""" + def plain(v): + return float(v).hex() if isinstance(v, float) else v + sig = {} + for qid, hits in results.items(): + sig[qid] = [(h.subject_id, float(h.emb_score).hex(), + tuple(plain(getattr(h.metrics, f.name)) for f in dataclasses.fields(h.metrics)), + output.format_row(qid, h, ALL_FIELDS)) for h in hits] + return sig + + +def _path_from_hit(m) -> np.ndarray: + """The forward path a hit's CIGAR and start positions describe.""" + i, j, out = m.q_start - 1, m.t_start - 1, [] + for op in swkern.expand_cigar(m.cigar): + if op == "M": + out.append((i, j)) + i, j = i + 1, j + 1 + elif op == "I": + out.append((i, -1)) + i += 1 + else: + out.append((-1, j)) + j += 1 + return np.array(out, np.int32).reshape(-1, 2) + + +def _assert_path_by_margin(tc, got, want, q16, codes, s_f, what): + """S16: the path equals the reference unless the float64 path scores tie within 1e-4 + 2e-6 |S_f|.""" + if np.array_equal(got, want): + return + z = swkern.z64_pair(q16, codes) + gap = abs(swkern.path_score(z, got, swkern.SETTING_D) - swkern.path_score(z, want, swkern.SETTING_D)) + tc.assertLess(gap, swkern.s16_path_margin(s_f), f"{what}: path differs beyond the S16 near-tie margin") + + +# ===================================================================================================================== +# differential: F1 and F3 +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class F1DifferentialTest(_WorldCase): + + def _check_pair(self, p, m): + self.assertLessEqual(abs(m.score - p["S_f"]), 5e-5 * max(abs(p["S_f"]), 1.0), p["qid"]) + self.assertLessEqual(abs(m.score_rev - p["S_r"]), 5e-5 * max(abs(p["S_r"]), 1.0), p["qid"]) + self.assertLessEqual(abs(m.q90 - p["q90"]), 1e-4, p["qid"]) + self.assertLessEqual(abs(m.log_evalue - p["lnE"]), 1e-4, p["qid"]) + self.assertEqual(m.score_adj, (m.score - m.score_rev) - m.q90) + _assert_path_by_margin(self, _path_from_hit(m), p["path"], p["q16"], p["codes"], p["S_f"], p["qid"]) + self.assertEqual((m.aln_len, m.n_ident), (p["aln_cols"], p["n_ident"])) + self.assertTrue(m.ecal) + + def test_within_s16_and_every_candidate_traced_without_an_e_filter(self): + world = self.f1_world() + res, stats = world.align(_f1_own_target(), max_evalue=None) + self.assertTrue(stats["panel"]) + self.assertEqual((stats["candidate_pairs"], stats["survivors"]), (50, 50)) # inf: every candidate traced + for p in _f1(): + hits = res[p["qid"]] + self.assertEqual([h.subject_id for h in hits], [p["tid"]]) + self._check_pair(p, hits[0].metrics) + self.assertEqual(stats["hits"], 50) + + def test_only_three_pairs_survive_at_1e_3(self): + world = self.f1_world() + res, stats = world.align(_f1_own_target(), max_evalue=1e-3) + want = {p["qid"] for p in _f1() if p["lnE"] <= math.log(1e-3)} + self.assertEqual(len(want), 3) + self.assertEqual(stats["survivors"], 3) + self.assertEqual(stats["round2_tasks"], 3) + self.assertEqual({q for q, h in res.items() if h}, want) + for p in _f1(): + if p["qid"] in want: + self._check_pair(p, res[p["qid"]][0].metrics) + self.assertLessEqual(res[p["qid"]][0].metrics.log_evalue, math.log(1e-3)) + self.assertEqual(list(res), [p["qid"] for p in _f1()]) # every query, in prefilter order + + def test_reported_scores_are_round_1_s_f_bit_for_bit(self): + """raw is the S_f behind the E-value: the same bits as a direct score_fwd_rev of the pair.""" + world = self.f1_world() + res, _ = world.align(_f1_mixed(range(0, 50, 5)), max_evalue=None, panel_policy="never") + for qid, hits in res.items(): + p = next(x for x in _f1() if x["qid"] == qid) + for h in hits: + t = next(x for x in _f1() if x["tid"] == h.subject_id) + with ca._blas_one_thread_here(): # the aligner's BLAS thread count (OpenBLAS: 1) + r = swkern.score_fwd_rev(swkern.query_prep(p["q16"]), t["codes"], [0, len(t["codes"])], + swkern.SETTING_D) + self.assertEqual(np.float32(h.metrics.score).view(np.uint32), r.s_fwd[0].view(np.uint32)) + self.assertEqual(np.float32(h.metrics.score_rev).view(np.uint32), r.s_rev[0].view(np.uint32)) + + def test_round_2_score_mismatch_is_an_internal_error(self): + world = self.f1_world() + real = ca._round2_task + + def perturbed(task): + out = list(real(task)) + out[1] = out[1].copy() + out[1][0] = np.nextafter(out[1][0], np.float32(np.inf)) + return tuple(out) + + with mock.patch.object(ca, "_round2_task", perturbed): + with self.assertRaisesRegex(InternalError, "round-2 traceback score"): + world.align(_f1_own_target(), max_evalue=None, panel_policy="never") + + +@pytest.mark.usefixtures("fmsw_libs") +class F3Q90Test(_WorldCase): + + def test_q90_of_the_benchmark_queries(self): + z = np.load(_RES / "q90_queries_v1.npz") + o = z["offsets"] + queries = [(str(z["ids"][k]), str(z["sequences"][k]), z["q"][o[k]:o[k + 1]]) for k in range(len(o) - 1)] + world = self.f1_world() + qi = world.make_query_index(queries, "f3.queries.f16") + target = _f1()[0]["tid"] + res, stats = world.align({q[0]: ([target], [0.5]) for q in queries}, query_index=qi, max_evalue=None, + query_sequences={q[0]: q[1] for q in queries}) + got = np.array([stats["q90"][q[0]] for q in queries]) + self.assertEqual(len(got), 20) + self.assertLessEqual(float(np.max(np.abs(got - z["q90"]))), 1e-3) + self.assertEqual(stats["query_domain"], dict(aligned=20, short=0, long=0, nonstandard=0)) + + +# ===================================================================================================================== +# invariance: serial == spawn pool, task / block / batch sizes +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class InvarianceTest(_WorldCase): + """Output bytes (every metrics field bit for bit, every rendered column) do not depend on the scheduling.""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + # A: the panel on, an E filter that some candidates pass (F1 queries 4 and 7 have significant own pairs; 48 + # and 10 are the shortest); B: no panel, every candidate traced back + cls.workload_a = dict(prefilter=_f1_mixed([4, 7, 9, 48, 10], per_query=12), max_evalue=10.0) + cls.workload_b = dict(prefilter=_f1_mixed(per_query=8), max_evalue=None, panel_policy="never") + world = cls.f1_world() + cls.ref_a = world.align(cls.workload_a["prefilter"], max_evalue=10.0) + cls.ref_b = world.align(cls.workload_b["prefilter"], max_evalue=None, panel_policy="never") + + def _same(self, ref, got): + self.assertEqual(_signature(ref[0]), _signature(got[0])) + + def _run(self, workload, pool=None, **kwargs): + w = dict(workload) + return self.f1_world().align(w.pop("prefilter"), pool, **w, **kwargs) + + def test_references_exercise_both_rounds(self): + _, sa = self.ref_a + self.assertTrue(sa["panel"]) + self.assertGreater(sa["survivors"], 0) + self.assertLess(sa["survivors"], sa["candidate_pairs"]) + _, sb = self.ref_b + self.assertFalse(sb["panel"]) + self.assertEqual(sb["survivors"], sb["candidate_pairs"]) + self.assertGreater(sb["hits"], 50) + + def test_task_cells_block_floats_and_query_batch(self): + for kwargs in (dict(task_cells=1 << 12), dict(task_cells=1 << 16, block_floats=1 << 12), + dict(block_floats=1 << 20), dict(query_batch=1), dict(query_batch=7, task_cells=5000)): + with self.subTest(**kwargs): + got_a = self._run(self.workload_a, **kwargs) + got_b = self._run(self.workload_b, **kwargs) + self._same(self.ref_a, got_a) + self._same(self.ref_b, got_b) + if "task_cells" in kwargs: + self.assertGreater(got_a[1]["round1_tasks"], self.ref_a[1]["round1_tasks"]) + + def test_serial_equals_spawn_pool_bit_for_bit(self): + parent_env = tuple(os.environ.get(k) for k in ca._BLAS_THREAD_VARS) + with ca.CodeAlignmentPool(3) as pool: + got_a = self._run(self.workload_a, pool, task_cells=1 << 14) + got_b = self._run(self.workload_b, pool, task_cells=1 << 14) + self.assertEqual(pool.start_method, "spawn") + self.assertEqual((pool.starts, pool.workers), (1, 3)) # one persistent pool, both rounds, 2 calls + heavy = list(pool._pool.map(eval, ["sorted(m for m in __import__('sys').modules " + "if m.split('.')[0] in ('torch', 'biotite'))"] * 12)) + # the workers were spawned with BLAS on one thread (S6), the parent's environment is left as it was + blas_env = list(pool._pool.map(eval, [f"tuple(__import__('os').environ.get(k) " + f"for k in {ca._BLAS_THREAD_VARS!r})"] * 12)) + self.assertEqual(set(blas_env), {("1",) * len(ca._BLAS_THREAD_VARS)}) + self.assertEqual(tuple(os.environ.get(k) for k in ca._BLAS_THREAD_VARS), parent_env) + self.assertEqual(got_a[1]["workers"], 3) + self._same(self.ref_a, got_a) + self._same(self.ref_b, got_b) + self.assertEqual([m for ms in heavy for m in ms], [], "a Stage-2 worker imported torch or biotite") + + def test_worker_count_does_not_change_the_output(self): + got = self._run(self.workload_b, 2, query_batch=13, task_cells=1 << 14) # a pool for this call only + self.assertEqual(got[1]["workers"], 2) + self._same(self.ref_b, got) + + def test_batches_overlap_in_the_dataflow_without_changing_the_output(self): + """Several batches on one pool (the next batch's round 1 runs while this one's round 2 does), and round-2 + hits built per query as its parts arrive: the same bits as the in-process reference.""" + with ca.CodeAlignmentPool(3) as pool: + got_a = self._run(self.workload_a, pool, query_batch=2, task_cells=1 << 15) + got_b = self._run(self.workload_b, pool, query_batch=7, task_cells=1 << 14) + self.assertEqual(got_a[1]["batches"], 3) + self.assertEqual(got_b[1]["batches"], 8) + self._same(self.ref_a, got_a) + self._same(self.ref_b, got_b) + + def test_repeated_candidates_keep_their_order_in_any_scheduling(self): + """A candidate listed twice gives two hits with equal S14 keys: they stay in candidate order whatever the + order in which a query's round-2 parts arrive.""" + pre = {q: (ids + ids, emb + emb) for q, (ids, emb) in _f1_mixed([4, 7, 9], per_query=5).items()} + ref = self.f1_world().align(pre, None, max_evalue=None) + self.assertEqual(ref[1]["hits"], 30) + with ca.CodeAlignmentPool(3) as pool: + got = self.f1_world().align(pre, pool, max_evalue=None, task_cells=1 << 12) + self.assertGreater(got[1]["round2_tasks"], 3) + self._same(ref, got) + + def test_the_parent_holds_round_2_payload_only_for_queries_in_flight(self): + """In-process every query's round 2 runs right after its round 1 and its hits are built when its last part + arrives, so the parent never holds more than one query's survivors (8 here) of round-2 payload.""" + with mock.patch.dict(os.environ, {stage2_profile.PROFILE_ENV: "1"}): + got = self._run(self.workload_b, None, task_cells=1 << 12) + self._same(self.ref_b, got) + mem = got[1]["profile"]["memory"] + self.assertGreater(got[1]["survivors"], 8 * 10) + self.assertGreater(mem["round2_parts_peak_survivors"], 0) + self.assertLessEqual(mem["round2_parts_peak_survivors"], 8) + + def test_pool_restarts_when_the_worker_setup_changes(self): + world = self.f1_world() + f1 = _f1() + other = world.make_query_index([(p["qid"], p["qseq"], p["q16"]) for p in f1[:10]], "other.queries.f16") + pre = _f1_mixed(range(10), per_query=6) + with ca.CodeAlignmentPool(2) as pool: + a, _ = world.align(pre, pool, max_evalue=None, panel_policy="never", task_cells=1 << 12) + b, _ = world.align(pre, pool, max_evalue=None, panel_policy="never", task_cells=1 << 12, + query_index=other) + self.assertEqual(pool.starts, 2) + self.assertEqual(_signature(a), _signature(b)) + + +# ===================================================================================================================== +# profiling mode (P7.1) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class ProfileTest(_WorldCase): + """FOLDMATCH_STAGE2_PROFILE: the time breakdown is recorded, every task is accounted for, and the hits are the same + bit for bit as without it (in-process and with a spawn pool).""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.prefilter = _f1_mixed([4, 7, 9, 48, 10], per_query=12) + with mock.patch.dict(os.environ, {stage2_profile.PROFILE_ENV: ""}): + cls.ref = cls.f1_world().align(cls.prefilter, max_evalue=10.0) + + def _profiled(self, pool=None, **kwargs): + with mock.patch.dict(os.environ, {stage2_profile.PROFILE_ENV: "1"}): + return self.f1_world().align(self.prefilter, pool, max_evalue=10.0, **kwargs) + + def _check_accounting(self, stats): + p = stats["profile"] + w = p["workers"] + self.assertEqual(w["panel"]["tasks"] + w["candidates"]["tasks"], stats["round1_tasks"]) + self.assertEqual(w["round2"]["tasks"], stats["round2_tasks"]) + self.assertEqual(w["panel"]["pairs"], stats["panel_pairs"]) + self.assertEqual(w["candidates"]["pairs"], stats["candidate_pairs"]) + self.assertEqual(w["round2"]["pairs"], stats["survivors"]) + self.assertEqual(len(p["tasks"]), stats["round1_tasks"] + stats["round2_tasks"]) + for kind in ("panel", "candidates", "round2"): + k = w[kind] + parts = sum(k[c] for c in ("prep", "sim", "kernel", "paths", "distinct", "metrics", "driver", "other")) + self.assertAlmostEqual(parts, k["seconds"], delta=1e-9 + 1e-9 * k["seconds"], msg=kind) + self.assertGreater(k["sim"], 0.0, kind) + self.assertGreater(k["kernel"], 0.0, kind) + self.assertGreater(w["round2"]["distinct"], 0.0) + for phase in ("setup", "collect", "lookup", "plan_r1", "round1", "consume_r1", "statistics", "fetch", + "plan_r2", "round2", "consume_r2", "hits_assemble", "hits_metrics", "hits_objects", + "hits_sort", "finish", "total"): + self.assertIn(phase, p["parent"]) + self.assertGreater(p["memory"]["round2_parts_peak_bytes"], 0) + # the parent holds round-2 payload only for the queries in flight (their hits are built as they complete) + self.assertGreater(p["memory"]["round2_parts_peak_survivors"], 0) + self.assertLessEqual(p["memory"]["round2_parts_peak_survivors"], stats["survivors"]) + self.assertEqual([r["name"] for r in p["ipc"]["rounds"]], ["round1", "round2"]) + return p + + def test_the_mode_is_off_unless_asked_for(self): + self.assertNotIn("profile", self.ref[1]) + for value in ("", "0", "false", "No", " off "): + with mock.patch.dict(os.environ, {stage2_profile.PROFILE_ENV: value}): + self.assertFalse(stage2_profile.profiling_enabled(), value) + for value in ("1", "yes", "true"): + with mock.patch.dict(os.environ, {stage2_profile.PROFILE_ENV: value}): + self.assertTrue(stage2_profile.profiling_enabled(), value) + + def test_in_process_profile_keeps_the_hits_and_accounts_for_every_task(self): + res, stats = self._profiled(task_cells=1 << 14) + self.assertEqual(_signature(res), _signature(self.ref[0])) + p = self._check_accounting(stats) + self.assertEqual(p["pool"]["executors"], 0) + self.assertGreater(p["memory"]["in_process_rss_max_bytes"], 0) + self.assertEqual(p["ipc"]["task_bytes"], 0) # nothing is pickled in-process + + def test_pool_profile_keeps_the_hits_and_records_the_worker_start_up(self): + with ca.CodeAlignmentPool(2) as pool: + res, stats = self._profiled(pool, task_cells=1 << 14) + again, stats2 = self._profiled(pool, task_cells=1 << 14) + self.assertEqual(pool.starts, 1) + self.assertEqual(_signature(res), _signature(self.ref[0])) + self.assertEqual(_signature(again), _signature(self.ref[0])) + p = self._check_accounting(stats) + self.assertEqual((p["pool"]["executors"], p["pool"]["workers"]), (1, 2)) + starts = p["pool"]["worker_starts"] + self.assertTrue(1 <= len(starts) <= 2) + for s in starts: + self.assertGreater(s["init_seconds"], 0.0) + self.assertGreaterEqual(s["ready_after_submit"], 0.0) + self.assertGreater(p["ipc"]["task_bytes"], 0) + self.assertGreater(p["ipc"]["result_bytes"], 0) + self.assertEqual(p["ipc"]["submit_to_start"]["n"], stats["round1_tasks"] + stats["round2_tasks"]) + self.assertGreater(p["memory"]["worker_rss_max_bytes"], 0) + for r in p["ipc"]["rounds"]: + self.assertGreater(r["utilisation"], 0.0) + self.assertLessEqual(r["utilisation"], 1.0 + 1e-6) + # each worker process reports its start-up once (with its first task), and the warm pool starts nothing + pids = [s["pid"] for s in starts + stats2["profile"]["pool"]["worker_starts"]] + self.assertEqual(len(pids), len(set(pids))) + self.assertLessEqual(len(pids), 2) + self.assertEqual(stats2["profile"]["pool"]["executors"], 0) + + def test_switching_the_mode_restarts_the_pool_and_the_hits_stay_the_same(self): + with ca.CodeAlignmentPool(2) as pool: + on, _ = self._profiled(pool, task_cells=1 << 14) + with mock.patch.dict(os.environ, {stage2_profile.PROFILE_ENV: ""}): + off, stats = self.f1_world().align(self.prefilter, pool, max_evalue=10.0, task_cells=1 << 14) + self.assertEqual(pool.starts, 2) + self.assertNotIn("profile", stats) + self.assertEqual(_signature(on), _signature(off)) + + def test_kernel_wrapper_timings_do_not_change_the_results(self): + f1 = _f1() + q16 = f1[3]["q16"] + codes = np.concatenate([p["codes"] for p in f1[:20]]) + offsets = np.concatenate([[0], np.cumsum([len(p["codes"]) for p in f1[:20]])]) + qp = swkern.query_prep(q16) + a = swkern.score_fwd_rev(qp, codes, offsets, swkern.SETTING_D, block_floats=1 << 13) + t1 = {} + b = swkern.score_fwd_rev(qp, codes, offsets, swkern.SETTING_D, block_floats=1 << 13, timings=t1) + for x, y in ((a.s_fwd, b.s_fwd), (a.s_rev, b.s_rev)): + np.testing.assert_array_equal(x.view(np.uint32), y.view(np.uint32)) + self.assertEqual(set(t1), {"sim", "kernel", "blocks"}) + self.assertGreater(t1["blocks"], 1) + c = swkern.align(qp, codes, offsets, swkern.SETTING_D, block_floats=1 << 13) + t2 = {} + d = swkern.align(qp, codes, offsets, swkern.SETTING_D, block_floats=1 << 13, timings=t2) + np.testing.assert_array_equal(c.scores.view(np.uint32), d.scores.view(np.uint32)) + for x, y in ((c.ends, d.ends), (c.path, d.path), (c.path_off, d.path_off)): + np.testing.assert_array_equal(x, y) + self.assertEqual(set(t2), {"sim", "kernel", "blocks", "paths"}) + self.assertEqual(t2["blocks"], t1["blocks"]) + + +# ===================================================================================================================== +# panel policy (D11), ranking (S14), filters +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class PanelPolicyTest(_WorldCase): + + def test_auto_skips_the_panel_without_e_values_and_the_ranking_is_unchanged(self): + world = self.f1_world() + pre = _f1_mixed(range(0, 50, 3), per_query=10) + with_panel, s1 = world.align(pre, max_evalue=None) # evalue / bits requested + without, s2 = world.align(pre, max_evalue=None, output_needs=["query", "target", "raw", "fident"]) + self.assertTrue(s1["panel"]) + self.assertFalse(s2["panel"]) + self.assertEqual(s2["panel_pairs"], 0) + self.assertEqual(s2["q90"], {}) + for qid in pre: + self.assertEqual([h.subject_id for h in with_panel[qid]], [h.subject_id for h in without[qid]]) + for a, b in zip(with_panel[qid], without[qid]): + self.assertEqual((a.metrics.score, a.metrics.score_rev), (b.metrics.score, b.metrics.score_rev)) + self.assertIsNotNone(a.metrics.log_evalue) + for name in ("q90", "score_adj", "log_evalue", "evalue", "bit_score"): + self.assertIsNone(getattr(b.metrics, name), name) + self.assertIsNotNone(b.metrics.ecal) + s = [h.metrics.score - h.metrics.score_rev for h in without[qid]] + self.assertEqual(s, sorted(s, reverse=True)) + + def test_policies(self): + world = self.f1_world() + pre = _f1_mixed(range(3), per_query=3) + _, st = world.align(pre, max_evalue=None, output_needs=["query", "target"], panel_policy="always") + self.assertTrue(st["panel"]) + _, st = world.align(pre, max_evalue=None, output_needs=["query", "q90"]) # q90 needs the panel + self.assertTrue(st["panel"]) + _, st = world.align(pre, max_evalue=float("inf"), output_needs=["query", "target"]) + self.assertFalse(st["panel"]) + _, st = world.align(pre, max_evalue=1.0, output_needs=["query", "target"]) # an E filter needs it + self.assertTrue(st["panel"]) + with self.assertRaisesRegex(ValueError, "cannot filter by E-value"): + world.align(pre, max_evalue=1e-3, panel_policy="never") + with self.assertRaisesRegex(ValueError, "panel_policy"): + world.align(pre, panel_policy="sometimes") + + def test_ties_are_broken_by_embedding_score_then_target_id(self): + p = _f1()[0] + targets = [(tid, p["tseq"], p["codes"]) for tid in ("tC", "tA", "tB", "tD")] + (self.tmp / "ties").mkdir() + world = World(self.tmp / "ties", targets, [(p["qid"], p["qseq"], p["q16"])], name="ties") + res, _ = world.align({p["qid"]: (["tC", "tA", "tB", "tD"], [0.5, 0.5, 0.9, 0.1])}, max_evalue=None) + self.assertEqual([h.subject_id for h in res[p["qid"]]], ["tB", "tA", "tC", "tD"]) + + def test_identity_and_coverage_filters(self): + world = self.f1_world() + pre = _f1_mixed(range(0, 50, 4), per_query=6) + everything, _ = world.align(pre, max_evalue=None, panel_policy="never") + filtered, _ = world.align(pre, max_evalue=None, panel_policy="never", min_seq_identity=0.2, + min_coverage=0.5) + for qid in pre: + want = [h.subject_id for h in everything[qid] + if h.metrics.identity_aln >= 0.2 and min(h.metrics.query_coverage, + h.metrics.subject_coverage) >= 0.5] + self.assertEqual([h.subject_id for h in filtered[qid]], want) + self.assertTrue(all(h.metrics.aln_len > 0 for h in everything[qid])) + self.assertGreater(sum(map(len, everything.values())), sum(map(len, filtered.values()))) + + +# ===================================================================================================================== +# identity, metrics and strings +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class PathMetricsTest(unittest.TestCase): + + def test_identity_along_a_hand_checked_path(self): + # query ACDEFGHIKL vs target ACDXFGHKL: + # A C D E F G H I K L + # A C D X F G H - K L -> 8 identical, 1 mismatch (E/X), 1 gap run (I = target gap), 10 columns + q, t = "ACDEFGHIKL", "ACDXFGHKL" + path = np.array([(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, -1), (8, 7), (9, 8)], np.int32) + cols = ca.path_columns(path, [0, len(path)], np.frombuffer(q.encode(), np.uint8), + np.frombuffer(t.encode(), np.uint8), [0]) + got = {k: int(v[0]) for k, v in cols.items()} + self.assertEqual(got, dict(aln_len=10, n_ident=8, n_aligned=9, mismatch=1, gap_open=1, q_aligned=10, + t_aligned=9, q_start=1, q_end=10, t_start=1, t_end=9)) + self.assertEqual(swkern.cigar_from_path(path), "7M1I2M") + + def test_vectorised_metrics_equal_the_per_path_helper(self): + """path_columns over many paths at once == swkern.path_metrics path by path (F1 paths, 300 random + alignments, and empty paths), with targets concatenated.""" + rng = np.random.default_rng(6) + cases = [(p["path"], p["qseq"], p["tseq"]) for p in _f1()] + cases.append((np.zeros((0, 2), np.int32), _random_seq(rng, 30), _random_seq(rng, 40))) + for _kind, q16, codes in swkern.golden_pairs(300, seed=99): + a = swkern.align(swkern.query_prep(q16), codes, [0, len(codes)], swkern.SETTING_D) + cases.append((a.path_of(0), _random_seq(rng, len(q16)), _random_seq(rng, len(codes)))) + by_query = {} + for path, qseq, tseq in cases: + by_query.setdefault(qseq, []).append((path, tseq)) + for qseq, items in by_query.items(): + path = np.concatenate([np.asarray(p, np.int32).reshape(-1, 2) for p, _ in items]) + path_off = np.concatenate([[0], np.cumsum([len(p) for p, _ in items])]) + t_starts = np.concatenate([[0], np.cumsum([len(t) for _, t in items])[:-1]]) + cols = ca.path_columns(path, path_off, np.frombuffer(qseq.encode(), np.uint8), + np.frombuffer("".join(t for _, t in items).encode(), np.uint8), t_starts) + for k, (p, tseq) in enumerate(items): + pm = swkern.path_metrics(p, qseq, tseq) + self.assertEqual( + tuple(int(cols[f][k]) for f in ("aln_len", "n_ident", "mismatch", "gap_open", "q_aligned", + "t_aligned", "q_start", "q_end", "t_start", "t_end")), + (pm.aln_len, pm.n_ident, pm.mismatch, pm.gap_open, pm.q_aligned, pm.t_aligned, pm.q_start, + pm.q_end, pm.t_start, pm.t_end)) + self.assertTrue(any(len(p) == 0 for p, _, _ in cases)) + + def test_sanitised_bytes_and_nonstandard_count(self): + for b in range(128): + ch = chr(b) + if ch.isspace(): + continue + self.assertEqual(ca.sanitized_bytes("A" + ch + "C").decode(), alignment.sanitize_sequence("A" + ch + "C")) + self.assertEqual(ca.nonstandard_count("ACDXBZ*_-"), 6) + self.assertEqual(ca.nonstandard_count(STANDARD_AMINO_ACIDS), 0) + + def test_vectorised_strings_equal_the_per_path_builders(self): + """_PathStrings over many paths at once == cigar_from_path / _gapped path by path (empty paths included).""" + rng = np.random.default_rng(8) + paths, t_seqs = [], [] + q16 = _f1()[2]["q16"] + qseq = _random_seq(rng, len(q16)) + for p in _f1()[:30]: + a = swkern.align(swkern.query_prep(q16), p["codes"], [0, len(p["codes"])], swkern.SETTING_D) + paths.append(a.path_of(0)) + t_seqs.append(_random_seq(rng, len(p["codes"]))) + paths.insert(3, np.zeros((0, 2), np.int32)) + t_seqs.insert(3, _random_seq(rng, 12)) + paths.append(np.zeros((0, 2), np.int32)) + t_seqs.append(_random_seq(rng, 5)) + path = np.concatenate(paths) + path_off = np.concatenate([[0], np.cumsum([len(p) for p in paths])]) + t_starts = np.concatenate([[0], np.cumsum([len(t) for t in t_seqs])[:-1]]) + q_bytes = np.frombuffer(qseq.encode(), np.uint8) + t_bytes = np.frombuffer("".join(t_seqs).encode(), np.uint8) + strings = ca._PathStrings(path, path_off, q_bytes, t_bytes, t_starts, cigar=True, qaln=True, taln=True) + for k, (p, tseq) in enumerate(zip(paths, t_seqs)): + lo, hi = int(path_off[k]), int(path_off[k + 1]) + self.assertEqual(strings.cigar(lo, hi), swkern.cigar_from_path(p), k) + self.assertEqual(strings.qaln[lo:hi].decode(), ca._gapped(p[:, 0], q_bytes), k) + self.assertEqual(strings.taln[lo:hi].decode(), ca._gapped(p[:, 1], np.frombuffer(tseq.encode(), np.uint8)), + k) + self.assertTrue(any("I" in swkern.cigar_from_path(p) or "D" in swkern.cigar_from_path(p) for p in paths)) + + def test_distinct_codes_are_counted_up_to_the_s13_minimum(self): + """The round-2 distinct-code count is min(distinct codes, cap), so 'fewer than cap' is decided exactly.""" + rng = np.random.default_rng(12) + pool = rng.integers(0, 256, (5, 8), dtype=np.uint8) + cases = [np.zeros((0, 8), np.uint8), pool[:1], np.repeat(pool[:1], 9, axis=0), _f1()[0]["codes"]] + for size in (2, 3, 7, 40): + cases.append(pool[rng.integers(0, len(pool), size)]) + cases.append(np.repeat(pool[rng.integers(0, len(pool), 1)], size, axis=0)) + for codes in cases: + exact = len(np.unique(codes.view(np.uint64))) if len(codes) else 0 + for cap in (1, 2, 3, 5): + got = ca._distinct_codes(codes, cap) + self.assertEqual(got, min(exact, cap), (len(codes), exact, cap)) + self.assertEqual(got < cap, exact < cap) + + +@pytest.mark.usefixtures("fmsw_libs") +class HitFieldsTest(_WorldCase): + + def test_fields_strings_and_identity_on_f1(self): + world = self.f1_world() + res, _ = world.align(_f1_mixed(range(0, 50, 7), per_query=5), max_evalue=None) + seen = 0 + for qid, hits in res.items(): + for h in hits: + m = h.metrics + seen += 1 + p = next(x for x in _f1() if x["qid"] == qid) + t = next(x for x in _f1() if x["tid"] == h.subject_id) + path = _path_from_hit(m) + # identity hand-checked along the path: naive loop over the sanitised sequences + ident = sum(1 for i, j in path.tolist() if i >= 0 and j >= 0 and p["qseq"][i] == t["tseq"][j]) + self.assertEqual(m.n_ident, ident) + self.assertEqual(m.identity_aln, ident / len(path)) + self.assertEqual(m.identity_shorter, ident / min(len(p["qseq"]), len(t["tseq"]))) + self.assertEqual((m.query_len, m.subject_len), (len(p["qseq"]), len(t["tseq"]))) + self.assertEqual(m.q_aln.replace("-", ""), p["qseq"][m.q_start - 1:m.q_end]) + self.assertEqual(m.t_aln.replace("-", ""), t["tseq"][m.t_start - 1:m.t_end]) + self.assertEqual((m.q_seq, m.t_seq), (p["qseq"], t["tseq"])) + self.assertEqual(m.query_coverage, (m.q_end - m.q_start + 1) / m.query_len) + self.assertEqual(m.subject_coverage, (m.t_end - m.t_start + 1) / m.subject_len) + self.assertEqual(m.aln_len, m.n_ident + m.mismatch + m.q_aln.count("-") + m.t_aln.count("-")) + spec = _prepared().for_worker() + self.assertEqual(m.log_evalue, cs.log_evalue(m.score_adj, m.query_len, N_F1, spec)) + self.assertEqual(m.bit_score, cs.bit_score(m.score_adj, m.query_len, spec)) + self.assertEqual(m.evalue, math.exp(m.log_evalue)) + self.assertIsInstance(m.score, float) + self.assertIsNone(m.pvalue) + self.assertGreater(seen, 10) + + def test_strings_are_built_only_when_requested(self): + world = self.f1_world() + res, _ = world.align(_f1_own_target(), max_evalue=None, output_needs=list(output.DEFAULT_OUTPUT_FIELDS)) + for hits in res.values(): + for h in hits: + for name in ("cigar", "q_aln", "t_aln", "q_seq", "t_seq"): + self.assertIsNone(getattr(h.metrics, name), name) + res, _ = world.align(_f1_own_target(), max_evalue=None, output_needs=ca.OutputNeeds(cigar=True)) + self.assertTrue(all(h.metrics.cigar and h.metrics.q_aln is None for hs in res.values() for h in hs)) + + +# ===================================================================================================================== +# degenerate and tiny inputs (F-deg; D14, S13, S13b) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class DegenerateTest(_WorldCase): + + @classmethod + def setUpClass(cls): + super().setUpClass() + z = cls.z = np.load(_RES / "degenerate_v1.npz") + ro = z["real_offsets"] + f1 = _f1()[0] + names = [str(x) for x in z["real_ids"]] + ["base"] + seqs = [str(x) for x in z["real_sequences"]] + [None] + assert np.array_equal(z["base_q"], f1["q16"]) and np.array_equal(z["base_codes"], f1["codes"]) + cls.items = [dict(name=names[i], seq=seqs[i], q16=z["real_q"][ro[i]:ro[i + 1]], + codes=z["real_codes"][ro[i]:ro[i + 1]]) for i in range(len(ro) - 1)] + cls.items.append(dict(name="base", seq=f1["qseq"], tseq=f1["tseq"], q16=f1["q16"], codes=f1["codes"])) + for it in cls.items[:-1]: + it["tseq"] = it["seq"] + # a synthetic query of 40 bitwise-identical rows (S13b) + so = z["syn_q_offsets"] + k = list(z["syn_ids"].astype(str)).index("identical_q_rows") + cls.ident_q = z["syn_q"][so[k]:so[k + 1]] + rng = np.random.default_rng(3) + queries = [(f"q_{it['name']}", it["seq"], it["q16"]) for it in cls.items] + queries.append(("q_identical_rows", _random_seq(rng, len(cls.ident_q)), cls.ident_q)) + cls.world = World(cls.tmp, [(f"t_{it['name']}", it["tseq"], it["codes"]) for it in cls.items], queries, + name="fdeg") + targets = [f"t_{it['name']}" for it in cls.items] + cls.prefilter = {q[0]: (targets, [0.5] * len(targets)) for q in queries} + logs = _ListHandler() + log = logging.getLogger("foldmatch.search.code_alignment") + log.addHandler(logs) + try: + cls.res, cls.stats = cls.world.align(cls.prefilter, max_evalue=None) + finally: + log.removeHandler(logs) + cls.logs = [r.getMessage() for r in logs.records if r.levelno >= logging.WARNING] + + def test_single_residue_and_identical_row_queries_report_no_hits(self): + self.assertEqual(sorted(self.stats["degenerate_queries"]), ["q_Met1", "q_identical_rows"]) + self.assertEqual(self.res["q_Met1"], []) + self.assertEqual(self.res["q_identical_rows"], []) + warning = "\n".join(self.logs) + self.assertIn("2 fully degenerate queries", warning) + self.assertIn("q_Met1", warning) + + def test_pairs_follow_the_fixture_and_d14(self): + z = self.z + names = [it["name"] for it in self.items] + checked = 0 + for p, (a, b) in enumerate(zip(z["pair_q"], z["pair_t"])): + qname, tname = f"q_{names[a]}", f"t_{names[b]}" + if qname == "q_Met1": + continue + hits = {h.subject_id: h.metrics for h in self.res[qname]} + if tname not in hits: + self.assertEqual(z["pair_path_off"][p + 1] - z["pair_path_off"][p], 0, (qname, tname)) + self.assertEqual(z["pair_S_f"][p], 0.0) + continue + m = hits[tname] + for v in (m.score, m.score_rev, m.q90, m.score_adj, m.log_evalue, m.bit_score): + self.assertTrue(math.isfinite(v)) + self.assertLessEqual(abs(m.score - z["pair_S_f"][p]), 5e-5 * max(abs(z["pair_S_f"][p]), 1.0)) + self.assertLessEqual(abs(m.score_rev - z["pair_S_r"][p]), 5e-5 * max(abs(z["pair_S_r"][p]), 1.0)) + self.assertLessEqual(abs(m.q90 - z["pair_q90"][p]), 1e-4, qname) + self.assertLessEqual(abs(m.log_evalue - z["pair_lnE_N1e9"][p]), 1e-4, (qname, tname)) + self.assertEqual(m.ecal, bool(z["pair_ecal_provisional"][p]), (qname, tname)) + checked += 1 + self.assertGreater(checked, 20) + + def test_tiny_and_low_complexity_items_are_aligned_with_ecal_0(self): + for qname in ("q_allX30", "q_polyA50", "q_His6", "q_GS", "q_angiotensinI"): + self.assertTrue(self.res[qname], qname) + for qname in ("q_allX30", "q_His6", "q_GS", "q_angiotensinI"): # non-standard or short queries + self.assertTrue(all(not h.metrics.ecal for h in self.res[qname]), qname) + for qname, hits in self.res.items(): + for h in hits: + if h.subject_id in ("t_Met1", "t_allX30"): # 1 distinct code; non-standard + self.assertFalse(h.metrics.ecal, (qname, h.subject_id)) + self.assertTrue(any(h.metrics.ecal for h in self.res["q_polyA50"])) + self.assertTrue(any(h.metrics.ecal for h in self.res["q_base"])) + qd = self.stats["query_domain"] + self.assertEqual((qd["aligned"], qd["short"], qd["nonstandard"]), (6, 3, 1)) + + def test_synthetic_cases(self): + z = self.z + qo, to = z["syn_q_offsets"], z["syn_t_offsets"] + rng = np.random.default_rng(11) + d = SYSTEM.calibrated_domain + for i, name in enumerate(z["syn_ids"].astype(str)): + with self.subTest(case=name): + q16, codes = z["syn_q"][qo[i]:qo[i + 1]], z["syn_codes"][to[i]:to[i + 1]] + qseq, tseq = _random_seq(rng, len(q16)), _random_seq(rng, len(codes)) + folder = self.tmp / f"syn_{name}" + folder.mkdir() + world = World(folder, [("t", tseq, codes)], [("q", qseq, q16)], name="syn") + if z["syn_expect_error"][i]: + with self.assertRaisesRegex(ValueError, "NaN or Inf"): + world.align({"q": (["t"], [0.5])}, max_evalue=None) + continue + res, stats = world.align({"q": (["t"], [0.5])}, max_evalue=None) + degenerate = len(q16) == 1 or bool(np.all(q16.view(np.uint16) == q16[:1].view(np.uint16))) + self.assertEqual(stats["degenerate_queries"], ["q"] if degenerate else []) + if degenerate: + self.assertEqual(res["q"], []) + continue + if not res["q"]: + self.assertEqual(z["syn_S_f"][i], 0.0) + continue + m = res["q"][0].metrics + self.assertLessEqual(abs(m.score - z["syn_S_f"][i]), 1e-5 * max(abs(z["syn_S_f"][i]), 1.0)) + self.assertLessEqual(abs(m.score_rev - z["syn_S_r"][i]), 1e-5 * max(abs(z["syn_S_r"][i]), 1.0)) + distinct = len(np.unique(codes.view(np.uint64))) + want_ecal = (d.query_min_length <= len(q16) <= d.query_max_length + and len(codes) <= d.target_max_length and distinct >= d.target_min_distinct_codes) + self.assertEqual(m.ecal, want_ecal) + if name in ("m1", "identical_codes"): + self.assertFalse(m.ecal) + + +@pytest.mark.usefixtures("fmsw_libs") +class CalibratedDomainTest(_WorldCase): + """S13 at its limits: queries of 966 and 967 residues, targets of 3,000 and 3,001, and 100-residue sequences with + exactly 1% (inside) and 2% (outside) non-standard residues on either side. The encodings and codes are an F1 + pair's (query 3: 247 x 275, S_f 415.7) tiled to the lengths, so every pair aligns; ecal must be 1 exactly for + the pairs whose query and target are both inside, and the once-per-run warning gives the counts.""" + + QUERIES = {"q966": (966, 0), "q967": (967, 0), "q100": (100, 0), "q100_x1": (100, 1), "q100_x2": (100, 2)} + TARGETS = {"t3000": (3000, 0), "t3001": (3001, 0), "t100": (100, 0), "t100_x1": (100, 1), "t100_x2": (100, 2)} + Q_INSIDE = {"q966", "q100", "q100_x1"} + T_INSIDE = {"t3000", "t100", "t100_x1"} + + @staticmethod + def _tile(rows, n): + return np.ascontiguousarray(np.concatenate([rows] * -(-n // len(rows)))[:n]) + + @staticmethod + def _sequence(rng, n, nonstandard): + seq = list(_random_seq(rng, n)) + for k in range(nonstandard): + seq[10 + 37 * k] = "X" + return "".join(seq) + + @classmethod + def setUpClass(cls): + super().setUpClass() + p = _f1()[3] + rng = np.random.default_rng(966) + targets = [(tid, cls._sequence(rng, m, x), cls._tile(p["codes"], m)) for tid, (m, x) in cls.TARGETS.items()] + queries = [(qid, cls._sequence(rng, n, x), cls._tile(p["q16"], n)) for qid, (n, x) in cls.QUERIES.items()] + cls.world = World(cls.tmp, targets, queries, name="domain") + prefilter = {qid: (list(cls.TARGETS), [0.5] * len(cls.TARGETS)) for qid in cls.QUERIES} + logs = _ListHandler() + log = logging.getLogger("foldmatch.search.code_alignment") + log.addHandler(logs) + try: + cls.res, cls.stats = cls.world.align(prefilter, max_evalue=None, panel_policy="never", + output_needs=["query", "target", "ecal"]) + finally: + log.removeHandler(logs) + cls.warnings = [r.getMessage() for r in logs.records if r.levelno == logging.WARNING] + + def test_the_inputs_sit_on_the_limits(self): + d = SYSTEM.calibrated_domain + self.assertEqual((d.query_min_length, d.query_max_length, d.target_max_length, d.max_nonstandard_fraction), + (21, 966, 3000, 0.01)) + for qid, seq in self.world.query_sequences.items(): + self.assertEqual((len(seq), ca.nonstandard_count(seq)), self.QUERIES[qid]) + + def test_ecal_of_every_pair(self): + for qid in self.QUERIES: + hits = {h.subject_id: h.metrics.ecal for h in self.res[qid]} + self.assertEqual(set(hits), set(self.TARGETS), qid) # every pair aligned + for tid, ecal in hits.items(): + with self.subTest(query=qid, target=tid): + self.assertIs(ecal, qid in self.Q_INSIDE and tid in self.T_INSIDE) + + def test_counts_and_the_warning(self): + self.assertEqual(self.stats["query_domain"], dict(aligned=5, short=0, long=1, nonstandard=1)) + self.assertEqual(self.stats["hit_domain"], dict(hits=25, ecal0=16, query_outside=10, target_long=5, + target_nonstandard=5, target_low_distinct=0)) + self.assertEqual(len(self.warnings), 1) + for text in ("0 queries shorter than 21 aa, 1 longer than 966 aa, 1 with too many non-standard residues", + "16 of 25 hits with ecal=0 (10 for the query, 5 target too long, 5 target non-standard, 0 " + "target with too few distinct codes)"): + self.assertIn(text, self.warnings[0]) + + +# ===================================================================================================================== +# refusals and edge cases +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class EdgeCaseTest(_WorldCase): + + def test_empty_database_gives_no_hits(self): + folder = self.tmp / "empty" + folder.mkdir() + p = _f1()[0] + world = World(folder, [], [(p["qid"], p["qseq"], p["q16"])], name="empty") + self.assertEqual(world.residues, 0) + res, stats = world.align({p["qid"]: ([], [])}, db_residues=0, max_evalue=1e-3) + self.assertEqual(res, {p["qid"]: []}) + self.assertEqual((stats["queries_without_candidates"], stats["candidate_pairs"]), (1, 0)) + + def test_queries_without_candidates_are_skipped_before_the_panel(self): + world = self.f1_world() + f1 = _f1() + pre = {f1[0]["qid"]: ([], []), f1[1]["qid"]: ([f1[1]["tid"]], [0.4])} + res, stats = world.align(pre, max_evalue=None) + self.assertEqual(res[f1[0]["qid"]], []) + self.assertEqual(len(res[f1[1]["qid"]]), 1) + self.assertEqual((stats["queries_without_candidates"], stats["panel_pairs"]), (1, SYSTEM.panel_size)) + self.assertEqual(list(stats["q90"]), [f1[1]["qid"]]) + + def test_oversize_pairs_are_skipped_and_counted(self): + world = self.f1_world() + f1 = _f1() + short = min(f1, key=lambda p: len(p["codes"])) + p = f1[0] + limit = len(p["q16"]) * (len(short["codes"]) + 1) + self.assertGreater(len(p["q16"]) * len(p["codes"]), limit) + handlers = {name: _ListHandler() for name in ("foldmatch.search.code_alignment", "foldmatch.search.swkern")} + swkern.reset_skipped_pairs() # swkern warns once per process: make this the first skip + for name, handler in handlers.items(): + logging.getLogger(name).addHandler(handler) + try: + res, stats = world.align({p["qid"]: ([p["tid"], short["tid"]], [0.9, 0.8])}, max_evalue=None, + panel_policy="never", max_pair_cells=limit) + finally: + for name, handler in handlers.items(): + logging.getLogger(name).removeHandler(handler) + self.assertEqual(stats["skipped_pairs"], 1) + self.assertEqual([h.subject_id for h in res[p["qid"]]], [short["tid"]]) + # one warning, from the parent, with the count (swkern's own once-per-process warning is dropped) + warnings = {name: [r.getMessage() for r in h.records if r.levelno == logging.WARNING] + for name, h in handlers.items()} + self.assertEqual(warnings["foldmatch.search.swkern"], []) + self.assertEqual(warnings["foldmatch.search.code_alignment"], + [f"esm-codes Stage 2: 1 candidate pair(s) skipped because their similarity matrix (query " + f"length x target length) exceeds max_pair_cells={limit} cells (the largest: " + f"{len(p['q16'])} x {len(p['codes'])}); they get no score and no hit"]) + self.assertNotIn(ca._DROP_SKIP_WARNING, logging.getLogger("foldmatch.search.swkern").filters) + with self.assertRaisesRegex(ValueError, "too long for the panel pass"): + world.align({p["qid"]: ([short["tid"]], [0.8])}, max_evalue=1.0, max_pair_cells=limit) + + def test_refusals(self): + world = self.f1_world() + p = _f1()[0] + pre = {p["qid"]: ([p["tid"]], [0.5])} + with self.assertRaisesRegex(ValueError, "no query encoding"): + world.align({"nope": ([p["tid"]], [0.5])}) + with self.assertRaisesRegex(ValueError, "seq_sha1 differs"): + world.align(pre, query_sequences={p["qid"]: "W" * len(p["qseq"])}) + with self.assertRaisesRegex(ValueError, "residues, its encoding"): + world.align(pre, query_sequences={p["qid"]: p["qseq"] + "A"}) + with self.assertRaisesRegex(ValueError, "no sequence"): + world.align(pre, query_sequences={}) + with self.assertRaisesRegex(ValueError, "non-empty database"): + world.align(pre, db_residues=0) + with self.assertRaises(ResidueCodeStoreError): + world.align({p["qid"]: (["not-in-the-store"], [0.5])}) + with self.assertRaisesRegex(ValueError, "no sequence in the sequence store"): + ca.align_candidates_esm_codes(world.query_index, pre, world.store, lambda ids: {}, _prepared(), N_F1, + None, query_sequences=world.query_sequences, output_needs=ALL_FIELDS, + max_evalue=None) + with self.assertRaisesRegex(ValueError, "another length"): + ca.align_candidates_esm_codes(world.query_index, pre, world.store, lambda ids: {i: "ACD" for i in ids}, + _prepared(), N_F1, None, query_sequences=world.query_sequences, + output_needs=ALL_FIELDS, max_evalue=None) + for bad in (dict(max_evalue=-1.0), dict(max_evalue=float("nan")), dict(min_seq_identity=1.5), + dict(task_cells=0), dict(query_batch=0), dict(db_residues=-1)): + with self.subTest(**bad), self.assertRaises(ValueError): + world.align(pre, **bad) + with self.assertRaises(TypeError): + world.align(pre, output_needs="evalue") + with self.assertRaises(TypeError): + world.align(pre, pool="four") + + def test_max_evalue_zero_keeps_nothing(self): + res, stats = self.f1_world().align(_f1_own_target(), max_evalue=0.0) + self.assertEqual((stats["survivors"], stats["hits"]), (0, 0)) + self.assertTrue(all(h == [] for h in res.values())) + + +# ===================================================================================================================== +# task planning, pool and worker isolation +# ===================================================================================================================== +class TaskPlanningTest(unittest.TestCase): + + def test_task_size_adapts_to_the_workers(self): + self.assertEqual(ca.target_task_cells(10 ** 6, 8, 1 << 24), ca.TASK_CELLS_FLOOR) # floor 2^20 + self.assertEqual(ca.target_task_cells(10 ** 12, 8, 1 << 24), 1 << 24) # cap task_cells + self.assertEqual(ca.target_task_cells(1 << 30, 8, 1 << 26), (1 << 30) // 32) # 4 per worker + self.assertEqual(ca.target_task_cells(10 ** 9, 8, 5000), 5000) # a smaller cap wins + + def test_one_interactive_query_uses_every_core(self): + """A 300-aa query's panel pass alone (107 M cells): at least 4 tasks per worker on 8 workers; on 64 the + 2^20-cell floor allows about 100 tasks, still more than one per worker.""" + lengths = np.diff(_panel_offsets()).tolist() + for workers in (8, 64): + total = 300 * sum(lengths) + chunks = ca.plan_chunks(300, lengths, ca.target_task_cells(total, workers, ca.DEFAULT_TASK_CELLS)) + self.assertGreaterEqual(len(chunks), min(4 * workers, total // ca.TASK_CELLS_FLOOR)) + self.assertGreater(len(chunks), workers) + self.assertEqual(chunks[0][0], 0) + self.assertEqual(chunks[-1][1], len(lengths)) + + def test_chunks_respect_the_target_and_keep_oversize_pairs_alone(self): + rng = np.random.default_rng(1) + lengths = rng.integers(1, 3000, 500).tolist() + for target in (1, 1000, 50_000, 10 ** 9): + chunks = ca.plan_chunks(10, lengths, target) + self.assertEqual([k for a, b in chunks for k in range(a, b)], list(range(len(lengths)))) + for a, b in chunks: + self.assertTrue(b - a == 1 or 10 * sum(lengths[a:b]) <= target) + self.assertEqual(ca.plan_chunks(5, [], 100), []) + + def test_lpt_order_is_longest_first_and_stable(self): + self.assertEqual(ca._lpt(["a", "b", "c", "d"], [3, 7, 3, 9]), ["d", "b", "a", "c"]) + + +class _Runtime: + """One OpenMP runtime's thread count: the count FAISS, the faiss-cpu wheel's OpenBLAS and torch (loaded first) + share on Linux.""" + + def __init__(self, threads): + self.threads = threads + + +def _fake_threadpoolctl(runtime: _Runtime, blas_reports: int): + """threadpoolctl over ``runtime`` with an OpenMP BLAS, as OpenBLAS 0.3.15's OpenMP build behaves: the BLAS keeps + no count of its own, so its limit and threadpoolctl's restore set the runtime's count, the restore to what the BLAS + reported at entry (``blas_reports``: its count of its last BLAS call, which lags behind FAISS's).""" + + class OpenMP: + @property + def num_threads(self): + return runtime.threads + + def set_num_threads(self, threads): + runtime.threads = threads + + class Controller: + def select(self, user_api=None): + assert user_api == "openmp", user_api + return mock.Mock(lib_controllers=[OpenMP()]) + + class Limiter: + def __init__(self, limits, user_api): + assert user_api == "blas", user_api + runtime.threads = limits # openblas_set_num_threads sets the OpenMP count + + def restore_original_limits(self): + runtime.threads = blas_reports + + return mock.Mock(ThreadpoolController=Controller, threadpool_limits=Limiter) + + +class InProcessBlasLimitTest(unittest.TestCase): + """P9: an in-process Stage 2 limits BLAS to one thread (``_blas_one_thread_here``) and gives every OpenMP runtime + its thread count again afterwards. With the faiss-cpu 1.15.1 wheel's OpenMP OpenBLAS, threadpoolctl's restore alone + set FAISS's (and torch's) count to OpenBLAS's stale one (NERSC job 60321459: FAISS at 7 came back at 128).""" + + def setUp(self): + noted = mock.patch.object(ca, "_note_uncontrolled_blas") + noted.start() + self.addCleanup(noted.stop) + + def test_the_openmp_count_comes_back(self): + runtime = _Runtime(8) + with mock.patch.dict(sys.modules, {"threadpoolctl": _fake_threadpoolctl(runtime, blas_reports=128)}): + with ca._blas_one_thread_here(): + self.assertEqual(runtime.threads, 1) # the limit reaches the OpenMP runtime + self.assertEqual(runtime.threads, 8) + limiter = sys.modules["threadpoolctl"].threadpool_limits(limits=1, user_api="blas") + limiter.restore_original_limits() + self.assertEqual(runtime.threads, 128) # threadpoolctl's restore alone (the P9 finding) + + def test_the_count_comes_back_after_an_error(self): + runtime = _Runtime(5) + with mock.patch.dict(sys.modules, {"threadpoolctl": _fake_threadpoolctl(runtime, blas_reports=64)}): + with self.assertRaises(KeyError), ca._blas_one_thread_here(): + raise KeyError("in the block") + self.assertEqual(runtime.threads, 5) + + def test_without_threadpoolctl_nothing_is_limited(self): + with mock.patch.dict(sys.modules, {"threadpoolctl": None}): # import threadpoolctl raises ImportError + self.assertEqual(ca._openmp_thread_counts(), []) + with ca._blas_one_thread_here(): + pass + ca._note_uncontrolled_blas.assert_not_called() + + def test_the_real_pools_are_as_before(self): + """Unmocked, on this platform: every pool threadpoolctl lists has its thread count of before after the block + (on Linux with the faiss-cpu wheel, FaissWheelTest checks FAISS's count set apart from OpenBLAS's).""" + try: + import threadpoolctl + except ImportError: + self.skipTest("threadpoolctl is not installed") + + def counts(): + return [(p.get("filepath"), p.get("num_threads")) for p in threadpoolctl.threadpool_info()] + before = counts() + with ca._blas_one_thread_here(): + pass + self.assertEqual(counts(), before) + + +@functools.lru_cache(maxsize=None) +def _panel_offsets(): + return _prepared().for_worker().load_panel()[1] + + +@pytest.mark.usefixtures("fmsw_libs") +class PoolAndWorkerTest(_WorldCase): + + def _init(self, world, **kwargs): + m = world.store.manifest() + return ca._WorkerInit(isa=swkern.load().isa, spec=_prepared().for_worker(), + codes=(str(world.store.bin_path), m["total_residues"], m["code_bytes"]), + queries=world.query_index.memmap_args(), block_floats=kwargs.get("block_floats", 1 << 17), + max_pair_cells=kwargs.get("max_pair_cells", 1 << 28)) + + def _tasks(self, world): + f1 = _f1() + p = f1[2] + entries = world.store.lookup([x["tid"] for x in f1[:6]]) + pairs = np.array([(entries[x["tid"]].offset, entries[x["tid"]].length) for x in f1[:6]], np.int64) + qe = world.query_index.entries[p["qid"]] + panel = np.column_stack((_panel_offsets()[:-1], np.diff(_panel_offsets())))[:40] + t_seqs = [x["tseq"] for x in f1[:6]] + return ((0, ca._CODES, qe.offset, qe.length, pairs), (1, ca._PANEL, qe.offset, qe.length, panel), + (2, qe.offset, qe.length, pairs, p["qseq"], t_seqs, (True, True, True, True))) + + def test_workers_import_neither_torch_nor_biotite(self): + """A subprocess with torch and biotite made unimportable initialises a worker and runs a round-1 task + (candidates and panel) and a round-2 task; the results equal the in-process ones bit for bit.""" + world = self.f1_world() + init = self._init(world) + t1, tp, t2 = self._tasks(world) + ca._init_worker(init, raise_errors=True) + try: + with ca._blas_one_thread_here(): # as in a worker: BLAS on one thread + want = (ca._round1_task(t1), ca._round1_task(tp), ca._round2_task(t2)) + finally: + ca._reset_worker_state() + src = str(Path(foldmatch.__file__).resolve().parents[1]) + with tempfile.TemporaryDirectory() as d: + inp, out = Path(d) / "in.pkl", Path(d) / "out.pkl" + inp.write_bytes(pickle.dumps((init, t1, tp, t2))) + code = ( + "import pickle, sys\n" + "sys.modules['torch'] = None\n" + "sys.modules['biotite'] = None\n" + "from foldmatch.search import code_alignment as ca\n" + f"init, t1, tp, t2 = pickle.loads(open({str(inp)!r}, 'rb').read())\n" + "ca._init_worker(init)\n" + "res = (ca._round1_task(t1), ca._round1_task(tp), ca._round2_task(t2))\n" + "heavy = sorted(m for m in sys.modules if m.split('.')[0] in ('torch', 'biotite')\n" + " and sys.modules[m] is not None)\n" + f"open({str(out)!r}, 'wb').write(pickle.dumps((res, heavy)))\n") + env = dict(__import__("os").environ, PYTHONPATH=src) + proc = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True) + self.assertEqual(proc.returncode, 0, proc.stderr) + got, heavy = pickle.loads(out.read_bytes()) + self.assertEqual(heavy, []) + for g, w in zip(got, want): + self.assertEqual(len(g), len(w)) + for a, b in zip(g, w): + if isinstance(a, np.ndarray): + self.assertEqual(a.dtype, b.dtype) + self.assertEqual(a.tobytes(), b.tobytes()) + else: + self.assertEqual(a, b) + self.assertEqual(got[2][3].shape, (6, len(ca._COLUMNS))) # round 2 returned the S12 column tallies + self.assertTrue(all(x for x in got[2][8][0])) # and the CIGARs it was asked for + + def test_worker_initialisation_errors_are_raised_by_tasks(self): + world = self.f1_world() + init = self._init(world) + bad = dataclasses.replace(init, spec=dataclasses.replace(init.spec, scoring_sha256="0" * 64)) + ca._init_worker(bad) + self.assertNotIn(ca._DROP_SKIP_WARNING, logging.getLogger("foldmatch.search.swkern").filters) # pool only + try: + with self.assertRaisesRegex(ss.BundleMismatchError, "sha256"): + ca._round1_task(self._tasks(world)[0]) + finally: + ca._reset_worker_state() + with self.assertRaises(InternalError): + ca._round1_task(self._tasks(world)[0]) + with self.assertRaises(ss.BundleMismatchError): # in-process: raised at once + ca._init_worker(bad, raise_errors=True) + ca._reset_worker_state() + + _mixed = _f1_mixed(range(0, 50, 5), per_query=6) + + def test_pool_arguments(self): + self.assertTrue(ca.CodeAlignmentPool(0).serial) + self.assertTrue(ca.CodeAlignmentPool(1).serial) + self.assertFalse(ca.CodeAlignmentPool().serial) + self.assertEqual(ca.CodeAlignmentPool(1).requested_workers(), 1) + self.assertEqual(ca.CodeAlignmentPool(5).requested_workers(), 5) + for bad in (-1, 2.0, True): + with self.assertRaises(ValueError): + ca.CodeAlignmentPool(bad) + with self.assertRaises(ValueError): + ca.CodeAlignmentPool(2, start_method="nope") + self.assertIn("num_workers=3", repr(ca.CodeAlignmentPool(3))) + + def test_a_worker_that_dies_stops_the_run(self): + """A worker killed in its task (SIGKILL, as the out-of-memory killer sends) raises WorkerPoolError at once, + instead of a wait that never ends; the pool is stopped, and the next batch starts a new one.""" + world = self.f1_world() + init = self._init(world) + with ca.CodeAlignmentPool(2) as pool: + t0 = time.perf_counter() + with _deadline(60), self.assertRaises(ca.WorkerPoolError) as ctx: + list(pool.run(signal.raise_signal, [int(signal.SIGKILL)] * 3, init)) + self.assertLess(time.perf_counter() - t0, 30) + self.assertIsInstance(ctx.exception.__cause__, BrokenProcessPool) + for text in ("out-of-memory killer", "if __name__ == \"__main__\":", "The worker pool was stopped"): + self.assertIn(text, str(ctx.exception)) + self.assertEqual((pool.workers, pool._pool, pool.starts), (0, None, 1)) + t1, tp, _ = self._tasks(world) + with _deadline(60): + got = list(pool.run(ca._round1_task, [t1, tp], init)) + self.assertEqual(sorted(r[0] for r in got), [0, 1]) + self.assertEqual((pool.starts, pool.workers), (2, 2)) + + def test_a_worker_that_dies_stops_a_session(self): + """The aligner's session (one task at a time, completion order) raises WorkerPoolError for a killed worker + too, and leaves no task behind.""" + world = self.f1_world() + init = self._init(world) + with ca.CodeAlignmentPool(2) as pool: + with _deadline(60), self.assertRaises(ca.WorkerPoolError): + with pool.session(init, 10 ** 9, 10) as session: + self.assertFalse(session.in_process) + self.assertEqual(session.window, 4) + for k in range(3): + session.submit(signal.raise_signal, int(signal.SIGKILL), ("kill", k)) + while not session.idle: + session.next() + self.assertEqual((pool.workers, pool._pool), (0, None)) + + def test_a_script_without_a_main_guard_fails_instead_of_hanging(self): + """Spawn workers re-import the main script: without an ``if __name__ == "__main__":`` guard they cannot + start. The run fails with WorkerPoolError (a multiprocessing.Pool respawned them forever); with the guard the + same script runs.""" + body = """ + from foldmatch.search import code_alignment as ca + init = ca._WorkerInit(isa="x", spec=None, codes=("", 0, 8), queries=("", 0, 64), block_floats=1, + max_pair_cells=1) + {guard} + with ca.CodeAlignmentPool(2) as pool: + print("RESULTS", sorted(pool.run(abs, [-1, -2, -3], init))) + """ + with tempfile.TemporaryDirectory() as d: + rc, out = _run_script(body.format(guard="if True:"), Path(d)) + self.assertIsNotNone(rc, f"the script without a main guard hung:\n{out[-2000:]}") + self.assertNotEqual(rc, 0, out[-2000:]) + self.assertIn("WorkerPoolError", out) + self.assertIn("if __name__ == \"__main__\":", out) + rc, out = _run_script(body.format(guard="if __name__ == '__main__':"), Path(d)) + self.assertEqual(rc, 0, out[-2000:]) + self.assertIn("RESULTS [1, 2, 3]", out) + + def test_oversize_pairs_are_reported_once_with_a_pool(self): + """swkern warns once per process about pairs above max_pair_cells; in pool workers that bypassed the + parent's logging (logging.lastResort), once per worker. Workers drop it and the parent warns once.""" + source = """ + import logging, sys, tempfile + from pathlib import Path + if __name__ == "__main__": + logging.basicConfig(level=logging.WARNING, format="PARENT %(name)s %(levelname)s: %(message)s") + import test_code_alignment as T + from foldmatch.search import code_alignment as ca + f1 = T._f1() + p = max(f1, key=lambda x: len(x["q16"])) + targets = [(f"t{k}", x["tseq"], x["codes"]) for k, x in enumerate(f1)] + with tempfile.TemporaryDirectory() as d: + world = T.World(Path(d), targets, [(p["qid"], p["qseq"], p["q16"])], name="skip") + with ca.CodeAlignmentPool(3) as pool: + _, stats = world.align({p["qid"]: ([t[0] for t in targets], [0.5] * len(targets))}, pool, + max_evalue=None, panel_policy="never", output_needs=["target"], + max_pair_cells=len(p["q16"]) * 60, task_cells=len(p["q16"]) * 200) + print("SKIPPED", stats["skipped_pairs"], "TASKS", stats["round1_tasks"], "WORKERS", stats["workers"]) + """ + with tempfile.TemporaryDirectory() as d: + rc, out = _run_script(source, Path(d)) + self.assertEqual(rc, 0, out[-3000:]) + skipped = sum(len(x["codes"]) > 60 for x in _f1()) + self.assertIn(f"SKIPPED {skipped} TASKS", out) + self.assertGreater(int(out.split("TASKS ")[1].split()[0]), 3) # every worker got tasks + self.assertIn("WORKERS 3", out) + self.assertNotIn(ca._SWKERN_SKIP_WARNING, out) + lines = [line for line in out.splitlines() if "skipped because their similarity matrix" in line] + self.assertEqual(len(lines), 1, out[-3000:]) + self.assertTrue(lines[0].startswith(f"PARENT foldmatch.search.code_alignment WARNING: esm-codes Stage 2: " + f"{skipped} candidate pair(s) skipped"), lines[0]) + + def test_single_task_batches_stay_out_of_process_without_threadpoolctl(self): + """A pool asked for workers runs a one-task batch in-process only when BLAS can be limited to one thread + there; without threadpoolctl it starts one worker instead (same bits as any other batch).""" + world = self.f1_world() + pre = {_f1()[0]["qid"]: ([_f1()[1]["tid"]], [0.5])} + with ca.CodeAlignmentPool(4) as pool: + ref, st = world.align(pre, pool, max_evalue=None, panel_policy="never") + self.assertEqual(st["round1_tasks"], 1) + self.assertEqual(pool.starts, 0 if ca._blas_limit_available() else 1) + with mock.patch.object(ca, "_blas_limit_available", return_value=False), ca.CodeAlignmentPool(4) as pool: + got, st = world.align(pre, pool, max_evalue=None, panel_policy="never") + self.assertEqual((pool.starts, pool.processes, st["workers"]), (1, 1, 1)) + self.assertEqual(pool.workers, 4) # the pool's size; one task started one worker + self.assertEqual(_signature(ref), _signature(got)) + + def test_a_small_first_batch_does_not_cap_the_pool(self): + """N workers: a first batch of 2 tasks starts 2 processes, a later batch of many tasks the rest of the N, + in the same pool (P7.1 found a warm-up call capping a persistent pool at its first batch's tasks).""" + world = self.f1_world() + f1 = _f1() + small = {f1[0]["qid"]: ([f1[1]["tid"], f1[2]["tid"]], [0.5, 0.4])} + with ca.CodeAlignmentPool(3) as pool: + _, st = world.align(small, pool, max_evalue=None, panel_policy="never", task_cells=1) + self.assertEqual((st["round1_tasks"], pool.starts, pool.workers, pool.processes), (2, 1, 3, 2)) + got, st = world.align(self.__class__._mixed, pool, max_evalue=None, panel_policy="never", + task_cells=1 << 14) + self.assertEqual((pool.starts, pool.workers, pool.processes, st["workers"]), (1, 3, 3, 3)) + ref, _ = world.align(self.__class__._mixed, None, max_evalue=None, panel_policy="never") + self.assertEqual(_signature(ref), _signature(got)) + + def test_the_default_budget_is_one_worker_per_physical_core(self): + """SMT siblings count once (a 2-socket EPYC 7763 node: 256 CPUs, 128 cores); without a readable topology every + CPU counts.""" + smt = {c: f"{c % 128},{c % 128 + 128}" for c in range(256)} + self.assertEqual(ca._physical_cpus(range(256), smt.__getitem__), 128) + self.assertEqual(ca._physical_cpus([0, 128, 1], smt.__getitem__), 2) + self.assertEqual(ca._physical_cpus(range(64), smt.__getitem__), 64) + ranges = {c: "0-3" if c < 4 else "4-7" for c in range(8)} + self.assertEqual(ca._physical_cpus(range(8), ranges.__getitem__), 2) + self.assertEqual(ca._physical_cpus(range(10), str), 10) # no SMT: every CPU its own core + + def unreadable(cpu): + raise OSError("no sysfs") + self.assertEqual(ca._physical_cpus(range(6), unreadable), 6) + self.assertGreaterEqual(ca._physical_cpus(), 1) + with mock.patch.object(ca, "_physical_cpus", return_value=7), mock.patch.object(ca, "_expand_cpu_affinity"): + self.assertEqual(ca.CodeAlignmentPool().requested_workers(), 7) + self.assertEqual(ca.CodeAlignmentPool(12).requested_workers(), 12) + + def test_the_default_pool_starts_workers_by_the_work(self): + """num_workers=None: one worker per min_worker_cells cells of the batch (at most the budget); a batch with + less work runs in-process; a later, larger batch restarts the pool with more workers; the bits stay the + same.""" + self.assertEqual(ca.CodeAlignmentPool(0).plan_workers(10 ** 12), 1) + self.assertEqual(ca.CodeAlignmentPool(5).plan_workers(1), 5) + p = ca.CodeAlignmentPool(None, min_worker_cells=100) + budget = p.requested_workers() + self.assertEqual(p.plan_workers(0), 1) + self.assertEqual(p.plan_workers(100), 1) + self.assertEqual(p.plan_workers(101), min(budget, 2)) + self.assertEqual(p.plan_workers(10 ** 9), budget) + with self.assertRaises(ValueError): + ca.CodeAlignmentPool(None, min_worker_cells=0) + world = self.f1_world() + f1 = _f1() + one = {f1[0]["qid"]: ([f1[1]["tid"]], [0.5])} + cells = len(f1[0]["q16"]) * len(f1[1]["codes"]) + mixed = self.__class__._mixed + mixed_cells = sum(len(next(x for x in f1 if x["qid"] == q)["q16"]) + * sum(len(next(x for x in f1 if x["tid"] == t)["codes"]) for t in tids) + for q, (tids, _) in mixed.items()) + ref, _ = world.align(mixed, None, max_evalue=None, panel_policy="never") + if not ca._blas_limit_available(): + self.skipTest("in-process batches need threadpoolctl") + with ca.CodeAlignmentPool(None, min_worker_cells=cells) as pool: + _, st = world.align(one, pool, max_evalue=None, panel_policy="never") + self.assertEqual((pool.starts, st["workers"]), (0, 0)) # one worker's share: in-process + pool.min_worker_cells = -(-mixed_cells // 2) + want = min(budget, 2) + got, st = world.align(mixed, pool, max_evalue=None, panel_policy="never", task_cells=1 << 14) + self.assertEqual((pool.starts, pool.workers), (1 if want > 1 else 0, want if want > 1 else 0)) + if budget >= 3: + pool.min_worker_cells = -(-mixed_cells // 3) + again, st = world.align(mixed, pool, max_evalue=None, panel_policy="never", task_cells=1 << 14) + self.assertEqual((pool.starts, pool.workers), (2, 3)) # grown: restarted with more + self.assertEqual(_signature(ref), _signature(again)) + self.assertEqual(_signature(ref), _signature(got)) + + def test_a_later_batch_grows_the_pool_within_one_call(self): + """num_workers=None, one call (as fm-search makes): a first batch below one worker's share runs in-process, + and a later batch with more work moves the call to a pool; a pool planned for 2 workers grows (by restart, + at least doubling) for a later batch that plans more. The bits stay those of an in-process run (review of + P7.2e: the first batch used to decide for the whole call).""" + if not ca._blas_limit_available(): + self.skipTest("in-process batches need threadpoolctl") + world = self.f1_world() + f1 = _f1() + + def cells(pre): + return sum(len(next(x for x in f1 if x["qid"] == q)["q16"]) + * sum(len(next(x for x in f1 if x["tid"] == t)["codes"]) for t in tids) + for q, (tids, _) in pre.items()) + + tiny = {f1[0]["qid"]: ([min(f1, key=lambda x: len(x["codes"]))["tid"]], [0.9])} + later = _f1_mixed(range(1, 13), per_query=6) # memmap order: f1[0] first, then these + per_batch = max(cells({q: v}) for q, v in later.items()) + mwc = max(cells(tiny), -(-per_batch // 2)) # the tiny batch: one share; later ones: 2 + ref, _ = world.align({**tiny, **later}, None, max_evalue=None, panel_policy="never") + with ca.CodeAlignmentPool(None, min_worker_cells=mwc) as pool: + budget = pool.requested_workers() + got, st = world.align({**tiny, **later}, pool, max_evalue=None, panel_policy="never", query_batch=1, + task_cells=1 << 14) + self.assertEqual(st["batches"], 13) + want = min(budget, 2) + if want > 1: + self.assertEqual((pool.starts, pool.workers), (1, want)) + self.assertGreaterEqual(st["workers"], 1) + else: + self.assertEqual((pool.starts, st["workers"]), (0, 0)) + self.assertEqual(_signature(ref), _signature(got)) + if budget < 4: + return + # a pool planned for 2 workers, then batches that plan more: restarts, each at least doubling the pool + small = _f1_mixed([1], per_query=2) + big = _f1_mixed(range(2, 14), per_query=12) + mwc = -(-cells(small) // 2) # the small first batch plans 2 workers + plans = [ca.CodeAlignmentPool(None, min_worker_cells=mwc).plan_workers(cells({q: v})) for q, v in big.items()] + with ca.CodeAlignmentPool(None, min_worker_cells=mwc) as pool: + got, st = world.align({**small, **big}, pool, max_evalue=None, panel_policy="never", query_batch=1, + task_cells=1 << 14) + self.assertLessEqual(pool.workers, budget) + if max(plans) > 2: + self.assertGreaterEqual(pool.starts, 2) + self.assertGreaterEqual(pool.workers, min(budget, max(max(plans), 4))) + self.assertLessEqual(pool.starts, 1 + math.ceil(math.log2(budget))) + ref, _ = world.align({**small, **big}, None, max_evalue=None, panel_policy="never") + self.assertEqual(_signature(ref), _signature(got)) + + def test_a_worker_releases_its_buffers_after_a_long_pair(self): + """After a task whose kernel buffers grew past WORKSPACE_KEEP_BYTES the worker drops them: here, with the bound + at 1 KiB, after every round-1 and round-2 task (checked in-process after each one); the hits do not + change. With the default bound the F1 buffers are kept.""" + world = self.f1_world() + pre = self.__class__._mixed + with ca.CodeAlignmentPool(0) as pool: + ref, _ = world.align(pre, pool, max_evalue=None, panel_policy="never") + kept = ca._STATE.workspace.nbytes() + self.assertGreater(kept, 0) + self.assertLessEqual(kept, ca.WORKSPACE_KEEP_BYTES) + seen = [] + + def after(fn): + def run(task): + out = fn(task) + seen.append((fn.__name__, ca._STATE.workspace.nbytes())) + return out + return run + + with mock.patch.object(ca, "WORKSPACE_KEEP_BYTES", 1024), \ + mock.patch.object(ca, "_round1_task", after(ca._round1_task)), \ + mock.patch.object(ca, "_round2_task", after(ca._round2_task)), ca.CodeAlignmentPool(0) as pool: + got, _ = world.align(pre, pool, max_evalue=None, panel_policy="never") + self.assertEqual({name for name, _ in seen}, {"_round1_task", "_round2_task"}) + self.assertEqual({nbytes for _, nbytes in seen}, {0}, seen) + self.assertEqual(_signature(ref), _signature(got)) + + def test_serial_pool_releases_its_in_process_state(self): + world = self.f1_world() + pool = ca.CodeAlignmentPool(0) + world.align(_f1_own_target(), pool, max_evalue=None, panel_policy="never") + self.assertIsNotNone(ca._STATE) # kept between calls of a persistent pool + pool.close() + self.assertIsNone(ca._STATE) + world.align(_f1_own_target(), None, max_evalue=None, panel_policy="never") + self.assertIsNone(ca._STATE) # a pool owned by the call is closed after it + + + +# ===================================================================================================================== +# P6.5: output fields +# ===================================================================================================================== +def _metrics(**kw): + base = dict(identity_aln=0.5, identity_shorter=0.5, query_coverage=1.0, subject_coverage=1.0, aln_len=10, + score=12.25, query_len=10, subject_len=10, n_ident=5, mismatch=5) + base.update(kw) + return alignment.AlignmentMetrics(**base) + + +def _row(m, fields): + return output.format_row("q", alignment.Hit("t", 0.5, m), fields).split(output.DELIMITER) + + +class OutputRenderingTest(unittest.TestCase): + + def test_field_registry(self): + self.assertEqual(output.DEFAULT_OUTPUT_FIELDS, + ("query", "target", "embscore", "fident", "alnlen", "mismatch", "gapopen", "qstart", "qend", + "tstart", "tend", "evalue", "bits")) + self.assertEqual(output.ESM_CODES_OUTPUT_FIELDS, ("rawrev", "q90", "rawadj", "ecal")) + self.assertEqual(output.SUPPORTED_OUTPUT_FIELDS[-4:], output.ESM_CODES_OUTPUT_FIELDS) + self.assertLessEqual(ca.PANEL_FIELDS, set(output.SUPPORTED_OUTPUT_FIELDS)) # the panel gate's fields exist + self.assertEqual(output.parse_format_output("query,rawrev,q90,rawadj,ecal"), + ["query", "rawrev", "q90", "rawadj", "ecal"]) + + def test_bits_round_half_up_including_negative_scores(self): + """esm-codes (a float score): floor(bits + 0.5) (D9, S11); MMseqs2's int(bits + 0.5) would print -1 for + -1.7 and 0 for -0.6.""" + for bits, want in ((-2.5, "-2"), (-0.4, "0"), (-0.5, "0"), (-0.6, "-1"), (-1.7, "-2"), (0.5, "1"), + (2.4999, "2"), (41.5, "42")): + self.assertEqual(_row(_metrics(bit_score=bits), ["bits"]), [want], bits) + self.assertEqual(_row(_metrics(), ["bits"]), [""]) + + @pytest.mark.usefixtures("fmsw_libs") + def test_blosum62_bits_keep_mmseqs2_rounding(self): + """BLOSUM62 (an integer score) keeps int(bits + 0.5), negative bit scores included, so its output is + byte-identical to before P6 (principle 5).""" + for bits, want in ((-7.6, "-7"), (-2.5, "-2"), (-0.6, "0"), (-1.7, "-1"), (-0.4, "0"), (0.5, "1"), + (2.4999, "2"), (41.5, "42")): + self.assertEqual(_row(_metrics(score=37, bit_score=bits), ["bits"]), [want], bits) + # --significance-mode sampled with low gap costs gives negative bit scores (here -3.2 and -7.5) + q = ("MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVQAPILSRVGDGTQDNLSGAEKAVQVKVKALPDAQFEVVHSLAKWKRQTLGQHDFSAGEGLYTHMKA" + "LRPDEDRLSPLHSVYVDQWDWERVMGDGERQFSTLKSTVEAIWAGIKATEAAVSEEFGLAPFLPDQIHFVHSQELLSRYPDLDAKGRERAIAKDLGAVF" + "LVGIGGKLSDGHRHDVRAPDYDDWUAIGLLKEDGDLTQQALGEAKLDD") + subjects = {"s_close": q[5:200], "s_far": "GSHMLEDPVDAFQKAIGLLQEHGKEVVHVVGHSMGGRWAKKLSEEGVPVAALVLFAPSKHGLTEPEA", + "s_rand": "PLWQERTYIPASDFGHKLCVNM" * 6} + res = alignment.align_candidates( + query_sequences={"q1": q}, + prefilter_results={"q1": (list(subjects), [0.9, 0.8, 0.7])}, + fetch_subject_sequences=lambda ids: {i: subjects[i] for i in ids}, + min_seq_identity=0.0, min_coverage=0.0, max_evalue=None, gap_open=3, gap_extend=1, num_workers=0, + subject_db_size=10_000_000, compute_significance=True, + significance_mode=alignment.SignificanceMode.sampled, output_fields=["bits"]) + negative = 0 + for h in res["q1"]: + self.assertIsInstance(h.metrics.score, int) + self.assertEqual(output.format_row("q1", h, ["bits"]), str(int(h.metrics.bit_score + 0.5))) + if h.metrics.bit_score < 0 and int(h.metrics.bit_score + 0.5) != math.floor(h.metrics.bit_score + 0.5): + negative += 1 + self.assertGreater(negative, 0, [h.metrics.bit_score for h in res["q1"]]) + + def test_evalue_from_ln_e_never_underflows(self): + self.assertEqual(_row(_metrics(log_evalue=-1000.0, evalue=0.0), ["evalue"]), ["5.076E-435"]) + self.assertEqual(_row(_metrics(log_evalue=0.0, evalue=1.0), ["evalue"]), ["1.000E+00"]) + self.assertEqual(_row(_metrics(log_evalue=math.log(2.5e-7), evalue=2.5e-7), ["evalue"]), ["2.500E-07"]) + self.assertEqual(_row(_metrics(evalue=3.2e-5), ["evalue"]), ["3.200E-05"]) # BLOSUM62, as before + self.assertEqual(_row(_metrics(), ["evalue"]), [""]) + + def test_esm_codes_scores_and_ecal(self): + m = _metrics(score=171.25, score_rev=12.5, q90=20.0625, score_adj=138.6875, ecal=True) + self.assertEqual(_row(m, ["raw", "rawrev", "q90", "rawadj", "ecal"]), + ["171.250", "12.500", "20.062", "138.688", "1"]) + self.assertEqual(_row(_metrics(score=0.5, score_adj=-1e-9, q90=-0.0004, ecal=False), + ["raw", "rawadj", "q90", "ecal"]), ["0.500", "0.000", "0.000", "0"]) + self.assertEqual(_row(_metrics(score_adj=-2.25), ["rawadj"]), ["-2.250"]) + + @pytest.mark.usefixtures("fmsw_libs") + def test_blosum62_metrics_keep_their_columns_and_leave_the_new_ones_empty(self): + m = _metrics(score=37, bit_score=19.78, evalue=1.5e-3) + self.assertEqual(_row(m, ["raw", "bits", "evalue", "rawrev", "q90", "rawadj", "ecal"]), + ["37", "20", "1.500E-03", "", "", "", ""]) + res = alignment.align_candidates( + query_sequences={"q1": "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVQ"}, + prefilter_results={"q1": (["s1"], [0.9])}, + fetch_subject_sequences=lambda ids: {"s1": "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVA"}, + min_seq_identity=0.0, min_coverage=0.0, max_evalue=None, num_workers=1, subject_db_size=1000, + output_fields=ALL_FIELDS) + h = res["q1"][0] + row = dict(zip(ALL_FIELDS, output.format_row("q1", h, ALL_FIELDS).split(output.DELIMITER))) + self.assertEqual(row["raw"], str(h.metrics.score)) + self.assertIsInstance(h.metrics.score, int) + self.assertEqual(row["bits"], str(int(h.metrics.bit_score + 0.5))) # bits >= 0: unchanged + self.assertEqual(row["evalue"], f"{h.metrics.evalue:.3E}") + self.assertEqual([row[f] for f in output.ESM_CODES_OUTPUT_FIELDS], ["", "", "", ""]) + + def test_tsv_header_names_the_new_fields(self): + fields = ["query", "target", "raw", "rawrev", "q90", "rawadj", "ecal"] + with tempfile.TemporaryDirectory() as d: + out = Path(d) / "hits.tsv" + m = _metrics(score=3.0, score_rev=1.0, q90=0.5, score_adj=1.5, ecal=True) + output.write_aligned_results({"q": [alignment.Hit("t", 0.5, m)]}, fields, str(out), + output.FormatMode.tsv) + lines = out.read_text().splitlines() + self.assertEqual(lines, ["\t".join(fields), "q\tt\t3.000\t1.000\t0.500\t1.500\t1"]) + + +@pytest.mark.usefixtures("fmsw_libs") +class EsmCodesOutputTest(_WorldCase): + + def test_columns_of_f1_hits(self): + res, _ = self.f1_world().align(_f1_own_target(), max_evalue=None) + fields = ["raw", "rawrev", "q90", "rawadj", "ecal", "evalue", "bits"] + for hits in res.values(): + m = hits[0].metrics + row = _row(m, fields) + self.assertEqual(row[:4], [f"{v:.3f}" for v in (m.score, m.score_rev, m.q90, m.score_adj)]) + self.assertEqual(row[4], "1") + self.assertEqual(row[5], f"{math.exp(m.log_evalue):.3E}") + self.assertEqual(row[6], str(math.floor(m.bit_score + 0.5))) + + def test_every_field_materialises_when_requested_alone(self): + """Under esm-codes every supported column renders, also when requested on its own (the E-value columns + then turn the panel on).""" + world = self.f1_world() + p = _f1()[4] + for field in ALL_FIELDS: + with self.subTest(field=field): + res, stats = world.align({p["qid"]: ([p["tid"]], [0.5])}, max_evalue=None, output_needs=[field]) + self.assertEqual(stats["panel"], field in ca.PANEL_FIELDS) + self.assertNotEqual(output.format_row(p["qid"], res[p["qid"]][0], [field]), "", field) + + def test_skipped_panel_leaves_the_e_value_columns_empty(self): + fields = ["query", "target", "raw", "rawrev", "ecal", "fident"] + res, stats = self.f1_world().align(_f1_mixed(range(0, 50, 10), per_query=4), max_evalue=None, + output_needs=fields) + self.assertFalse(stats["panel"]) + for qid, hits in res.items(): + for h in hits: + self.assertEqual(output.format_row(qid, h, ["evalue", "bits", "q90", "rawadj"]), "\t\t\t") + self.assertTrue(all(output.format_row(qid, h, fields).split("\t"))) + + def test_degenerate_hits_render_ecal_0(self): + z = np.load(_RES / "degenerate_v1.npz") + ro = z["real_offsets"] + names = list(z["real_ids"].astype(str)) + a, b = names.index("allX30"), names.index("polyA50") + folder = self.tmp / "ecal0" + folder.mkdir() + world = World(folder, [("t", str(z["real_sequences"][b]), z["real_codes"][ro[b]:ro[b + 1]])], + [("q", str(z["real_sequences"][a]), z["real_q"][ro[a]:ro[a + 1]])], name="ecal0") + res, _ = world.align({"q": (["t"], [0.5])}, max_evalue=None) + self.assertEqual(output.format_row("q", res["q"][0], ["ecal"]), "0") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_code_statistics.py b/tests/test_code_statistics.py new file mode 100644 index 0000000..6bec60b --- /dev/null +++ b/tests/test_code_statistics.py @@ -0,0 +1,307 @@ +"""P4: esm-codes E-value statistics (S10, S11; ``foldmatch.search.code_statistics``). + +F4 (``evalue_grid_v1.json``) holds lambda, lnK, lnE, E, bits and the printed bits from the research table +``ship_final.json`` (md5 c51735f9...) on a grid of query lengths (below the first centre, at each centre, between +centres, above the last), database sizes and S' values; the plan's tolerance is rel <= 1e-12. +""" +import json +import math +import unittest +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pytest + +from foldmatch.search import code_statistics as cs + +_RES = Path(__file__).parent / "resources" / "esm_codes" +F4 = json.loads((_RES / "evalue_grid_v1.json").read_text()) +REL = 1e-12 + + +def f4_table(**rules): + t = F4["table"] + return SimpleNamespace(bin_centres=tuple(t["center_len"]), lam=tuple(t["lam"]), lnK=tuple(t["lnK"]), + low_m_rule=rules.get("low_m_rule", "clamp"), high_m_rule=rules.get("high_m_rule", "clamp")) + + +def _rel(got, want): + return abs(got - want) / abs(want) + + +class F4GoldenTest(unittest.TestCase): + """Every F4 row at rel <= 1e-12: lambda, lnK, lnE, E, bits; the printed bits exactly.""" + + def test_rows(self): + table = f4_table() + self.assertEqual(len(F4["rows"]), 208) + worst = 0.0 + for r in F4["rows"]: + with self.subTest(m=r["m"], N=r["N"], S_prime=r["S_prime"]): + la, lk = cs.lam_lnk(r["m"], table) + lne = cs.log_evalue(r["S_prime"], r["m"], r["N"], table) + bits = cs.bit_score(r["S_prime"], r["m"], table) + for got, want in ((la, r["lam"]), (lk, r["lnK"]), (lne, r["lnE"]), (bits, r["bits"]), + (math.exp(lne), r["E"])): + self.assertLessEqual(_rel(got, want), REL) + worst = max(worst, _rel(got, want)) + self.assertEqual(cs.render_bits(bits), r["render_bits"]) + self.assertLessEqual(worst, REL) + + def test_exact_at_the_centres(self): + table = f4_table() + rows = [r for r in F4["rows"] if r["at_centre"] is not None] + self.assertEqual(sorted({r["at_centre"] for r in rows}), list(range(6))) + for r in rows: + k = r["at_centre"] + self.assertEqual(r["m"], table.bin_centres[k]) + self.assertEqual(cs.lam_lnk(r["m"], table), (table.lam[k], table.lnK[k])) + + def test_low_clamp(self): + table = f4_table() + rows = [r for r in F4["rows"] if r["region"] == "below_first_centre"] + self.assertEqual(sorted({r["m"] for r in rows}), [10.0, 32.58]) + for r in rows: + self.assertEqual(cs.lam_lnk(r["m"], table), (table.lam[0], table.lnK[0])) + self.assertEqual(cs.lam_lnk(1, table), (table.lam[0], table.lnK[0])) + + def test_high_m_clamp(self): + """D15: above the last centre lambda and lnK keep the last centre's values; F4's uncapped ln-m + extrapolation (the rejected alternative) is not used.""" + table = f4_table() + rows = [r for r in F4["rows"] if r["region"] == "above_last_centre"] + self.assertEqual(sorted({r["m"] for r in rows}), [594.92, 1000.0, 5000.0]) + for r in rows: + la, lk = cs.lam_lnk(r["m"], table) + self.assertEqual((la, lk), (table.lam[-1], table.lnK[-1])) + self.assertNotEqual(la, r["extrapolated"]["lam"]) + self.assertNotEqual(cs.log_evalue(r["S_prime"], r["m"], r["N"], table), r["extrapolated"]["lnE"]) + self.assertEqual(cs.lam_lnk(30000, table), (table.lam[-1], table.lnK[-1])) + + def test_inside_is_linear_in_ln_m(self): + table = f4_table() + c, lam = table.bin_centres, table.lam + m = math.sqrt(c[2] * c[3]) # half way in ln m + la, _ = cs.lam_lnk(m, table) + self.assertLessEqual(_rel(la, 0.5 * (lam[2] + lam[3])), 1e-12) + + def test_e_equals_m_n_2_to_minus_bits(self): + table = f4_table() + for m in (10.0, 32.58437573116228, 71.0, 300.0, 594.9175724688861, 966.0, 5000.0): + for n in (1.0, 303.0, 3.6e5, 8557236.0, 1e9, 1e11): + for sp in (-5.0, 0.0, 12.3, 80.0, 250.0): + with self.subTest(m=m, N=n, S_prime=sp): + e = math.exp(cs.log_evalue(sp, m, n, table)) + self.assertLessEqual(_rel(e, m * n * 2.0 ** -cs.bit_score(sp, m, table)), REL) + + def test_vectorised_equals_scalar(self): + table = f4_table() + m = np.array([10.0, 50.0, 100.0, 1000.0]) + sp = np.array([-5.0, 0.0, 12.3, 80.0]) + lne = cs.log_evalue(sp, m, 1e9, table) + bits = cs.bit_score(sp, m, table) + for i in range(4): + self.assertEqual(lne[i], cs.log_evalue(sp[i], m[i], 1e9, table)) + self.assertEqual(bits[i], cs.bit_score(sp[i], m[i], table)) + np.testing.assert_array_equal(cs.render_bits(bits), [cs.render_bits(b) for b in bits]) + lne_q = cs.log_evalue(sp, 300, 1e9, table) # one query, many targets + self.assertEqual(lne_q.shape, (4,)) + self.assertIsInstance(cs.log_evalue(1.0, 300, 1e9, table), float) + + +class RefusalTest(unittest.TestCase): + + def test_unimplemented_rules(self): + for rules in ({"high_m_rule": "extrapolate"}, {"low_m_rule": "extrapolate"}, {"high_m_rule": "Clamp"}): + with self.subTest(**rules), self.assertRaisesRegex(ValueError, "not implemented"): + cs.lam_lnk(100, f4_table(**rules)) + + def test_bad_inputs(self): + table = f4_table() + for m in (0, -3, float("nan"), float("inf")): + with self.subTest(m=m), self.assertRaises(ValueError): + cs.lam_lnk(m, table) + for n in (0, -1.0, float("nan")): + with self.subTest(N=n), self.assertRaisesRegex(ValueError, "database residues"): + cs.log_evalue(1.0, 100, n, table) + with self.assertRaises(ValueError): + cs.log_evalue(float("nan"), 100, 1e9, table) + with self.assertRaises(ValueError): + cs.bit_score(float("inf"), 100, table) + + def test_bad_table(self): + bad = f4_table() + bad.bin_centres = bad.bin_centres[::-1] + with self.assertRaises(ValueError): + cs.lam_lnk(100, bad) + short = f4_table() + short.lam = short.lam[:5] + with self.assertRaises(ValueError): + cs.lam_lnk(100, short) + + +class RenderBitsTest(unittest.TestCase): + + def test_floor_half_up(self): + cases = {-2.5: -2, -0.4: 0, -0.5: 0, -0.6: -1, -1.7: -2, 0.0: 0, 0.49: 0, 0.5: 1, 2.5: 3, 18.01: 18, + 22.698: 23} + for x, want in cases.items(): + with self.subTest(bits=x): + got = cs.render_bits(x) + self.assertEqual(got, want) + self.assertIsInstance(got, int) + self.assertNotEqual(cs.render_bits(-1.7), int(-1.7 + 0.5)) # the BLOSUM path's rounding differs here + + def test_array_and_refusal(self): + np.testing.assert_array_equal(cs.render_bits([-2.5, -0.4, 1.5]), [-2, 0, 2]) + self.assertEqual(cs.render_bits(np.array([1.5])).dtype, np.int64) + with self.assertRaises(ValueError): + cs.render_bits(float("nan")) + + def test_range_is_the_same_for_scalars_and_arrays(self): + """Beyond 2^53 neither path renders (the array path's int64 would saturate at 2^63; P4 review).""" + self.assertEqual(cs.render_bits(2.0 ** 53), 2 ** 53) + np.testing.assert_array_equal(cs.render_bits(np.array([-(2.0 ** 53), 2.0 ** 53])), [-(2 ** 53), 2 ** 53]) + for bad in (np.nextafter(2.0 ** 53, np.inf), 1e19, -1e300): + with self.subTest(bits=bad): + with self.assertRaisesRegex(ValueError, "2\\^53"): + cs.render_bits(bad) + with self.assertRaisesRegex(ValueError, "2\\^53"): + cs.render_bits(np.array([1.0, bad])) + + +class FormatEvalueTest(unittest.TestCase): + + def test_underflow(self): + self.assertEqual(cs.format_evalue_from_log(-1000), "5.076E-435") + self.assertTrue(cs.format_evalue_from_log(-1000).endswith("E-435")) + self.assertEqual(cs.format_evalue_from_log(-745), "2.822E-324") # subnormal exp(): formatted from ln E + self.assertEqual(cs.format_evalue_from_log(-1e6), "3.297E-434295") # 10^-434294.4819... = 3.29683E-434295 + + def test_against_decimal_arithmetic(self): + from decimal import Decimal, getcontext + getcontext().prec = 40 + for x in (-709.5, -800.25, -1000.0, -12345.678, -1e6, 750.0, 3000.5): + t = Decimal(x) / Decimal(10).ln() + e = math.floor(t) + want = f"{float(Decimal(10) ** (t - e)):.3f}E{'-' if e < 0 else '+'}{abs(e):02d}" + self.assertEqual(cs.format_evalue_from_log(x), want, x) + + def test_overflow(self): + self.assertEqual(cs.format_evalue_from_log(1000), "1.970E+434") + self.assertEqual(cs.format_evalue_from_log(710), "2.234E+308") + + def test_in_range_equals_the_blosum_format(self): + rng = np.random.default_rng(20261014) + for x in [0.0, math.log(1e-3), math.log(12.7), -708.0, 709.0] + rng.uniform(-708, 709, 500).tolist(): + with self.subTest(lnE=x): + self.assertEqual(cs.format_evalue_from_log(x), f"{math.exp(x):.3E}") + + def test_log10_path_agrees_with_direct_formatting(self): + rng = np.random.default_rng(7) + for x in rng.uniform(-700, 700, 2000).tolist(): + self.assertEqual(cs._format_from_log10(x / cs.LN10), f"{math.exp(x):.3E}", x) + + def test_carry_into_the_next_decade(self): + x = math.log(9.9996) - 500 * math.log(10) + self.assertEqual(cs.format_evalue_from_log(x), "1.000E-499") + x = math.log(9.9994) - 500 * math.log(10) + self.assertEqual(cs.format_evalue_from_log(x), "9.999E-500") + + def test_special_values(self): + self.assertEqual(cs.format_evalue_from_log(-math.inf), "0.000E+00") + self.assertEqual(cs.format_evalue_from_log(math.inf), "INF") + with self.assertRaises(ValueError): + cs.format_evalue_from_log(float("nan")) + + +class Q90Test(unittest.TestCase): + + def test_linear_quantile(self): + self.assertEqual(cs.q90(np.arange(11.0)), 9.0) + self.assertAlmostEqual(cs.q90([1.0, 2.0, 3.0, 4.0]), 3.7, places=15) + rng = np.random.default_rng(1) + s = rng.normal(size=1000) + self.assertEqual(cs.q90(s), float(np.quantile(s, 0.9))) + self.assertEqual(cs.q90(s.astype(np.float32)), float(np.quantile(s.astype(np.float32).astype(np.float64), + 0.9))) + self.assertEqual(cs.q90(s, quantile=0.5), float(np.median(s))) + self.assertIsInstance(cs.q90(s), float) + + def test_refusals(self): + for bad in ([], [[1.0, 2.0]], [1.0, float("nan")]): + with self.subTest(scores=bad), self.assertRaises(ValueError): + cs.q90(bad) + with self.assertRaises(ValueError): + cs.q90([1.0, 2.0], method="nearest") + with self.assertRaises(ValueError): + cs.q90([1.0, 2.0], quantile=1.0) + + + +# ===================================================================================================================== +# with the registry entry and its bundle +# ===================================================================================================================== +class RegistryTableTest(unittest.TestCase): + + def test_registry_entry_gives_the_f4_values(self): + from foldmatch.search import scoring_system as ss + system, table = ss.get(ss.DEFAULT_ESM_CODES), f4_table() + for r in F4["rows"]: + self.assertEqual(cs.lam_lnk(r["m"], system), cs.lam_lnk(r["m"], table)) + self.assertEqual(cs.log_evalue(r["S_prime"], r["m"], r["N"], system), + cs.log_evalue(r["S_prime"], r["m"], r["N"], table)) + + +@pytest.mark.usefixtures("fmsw_libs") +class BundlePanelTest(unittest.TestCase): + """S10 / S11 through the production path with the registry bundle's panel: per-pair BLAS similarity, fmsw S_f and + S_r, Q90 = np.quantile(S, 0.9), then lnE at N = 1e9. F1 (50 CATH pairs) within S16 of the research values (Q90 + and lnE within 1e-4); F3 (20 benchmark queries, 21-966 aa) Q90 within 1e-3.""" + + @classmethod + def setUpClass(cls): + import esm_codes_bundle + from foldmatch.search import scoring_system as ss + esm_codes_bundle.registry_files() # unittest.SkipTest only when offline and not cached + cls.spec = ss.prepare(ss.get(ss.DEFAULT_ESM_CODES)).for_worker() + cls.codes, cls.offsets = cls.spec.load_panel() + + def _scores(self, q16, codes, offsets): + from foldmatch.search import swkern + r = swkern.score_fwd_rev(swkern.query_prep(q16), codes, offsets, self.spec.dp_params()) + self.assertFalse(np.any(r.skipped)) + return r.s_fwd.astype(np.float64) - r.s_rev.astype(np.float64) + + def _q90(self, q16): + return cs.q90(self._scores(q16, self.codes, self.offsets), self.spec.panel_quantile, self.spec.quantile_method) + + def test_f1_q90_and_lne(self): + z = np.load(_RES / "pairs_v1.npz") + qo, to = z["q_offsets"], z["t_offsets"] + q90, lne = [], [] + for p in range(len(z["pair_q"])): + a, b = int(z["pair_q"][p]), int(z["pair_t"][p]) + q16 = z["q"][qo[a]:qo[a + 1]] + q = self._q90(q16) + S = float(self._scores(q16, z["codes"][to[b]:to[b + 1]], np.array([0, to[b + 1] - to[b]]))[0]) + bound = 5e-5 * (max(abs(z["S_f"][p]), 1.0) + max(abs(z["S_r"][p]), 1.0)) # S16 on S_f and S_r + self.assertLessEqual(abs(S - z["S"][p]), bound) + q90.append(q) + lne.append(cs.log_evalue(S - q, len(q16), 1e9, self.spec)) + q90, lne = np.array(q90), np.array(lne) + self.assertLessEqual(float(np.max(np.abs(q90 - z["q90"]))), 1e-4) + self.assertLessEqual(float(np.max(np.abs(lne - z["lnE_N1e9"]))), 1e-4) + self.assertEqual(int(np.sum(lne <= math.log(1e-3))), 3) + + def test_f3_q90(self): + z = np.load(_RES / "q90_queries_v1.npz") + o = z["offsets"] + got = np.array([self._q90(z["q"][o[k]:o[k + 1]]) for k in range(len(o) - 1)]) + self.assertEqual(len(got), 20) + self.assertLessEqual(float(np.max(np.abs(got - z["q90"]))), 1e-3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_csv_embeddings.py b/tests/test_csv_embeddings.py new file mode 100644 index 0000000..64b8286 --- /dev/null +++ b/tests/test_csv_embeddings.py @@ -0,0 +1,330 @@ +"""P9.B1, P9.B3: a .pt or .csv embedding file holds one vector. + +P9.B1: CsvBatchWriter writes a chain or assembly vector as one column, one value per line. stream_embeddings averaged +any .csv of more than one row over its rows, as residue embeddings, so each vector became its mean: 'fm-search build +embeddings' built a database of dimension 1 from such files (every score +1 or -1), and 'fm-search query embedding' +with them failed on the dimension. + +P9.B3: residue embeddings (one row per residue: a 2-D tensor, or a .csv of one line of D values per residue) were still +averaged over their rows, and the mean was indexed or queried as a chain embedding, which it is not. A .pt or .csv +file must now hold one vector ([D], [1, D], [D, 1]; a .csv of one column or one line): a table is refused, naming the +file and its shape, and so are a file that cannot be read, a file of no values and NaN or infinite values.""" +import io +import logging +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import faiss +import numpy as np +import pandas as pd +import torch + +from foldmatch.cli import search +from foldmatch.errors import RefusalError +from foldmatch.utils.data import stream_embeddings +from foldmatch.writer.batch_writer import CsvBatchWriter, TensorBatchWriter + +_IDS = ["1abc-1", "2xyz-1", "3def-1"] +_RESIDUES = Path(__file__).parent / "resources" / "embeddings" + + +def _vectors() -> dict: + generator = torch.Generator().manual_seed(0) + return {dom_id: torch.randn(1536, generator=generator) for dom_id in _IDS} + + +def _residues() -> dict: + """Residue embeddings: the ESM3 ones of chain 1acb.B (65 x 1536, its BOS and EOS rows included) and a 64-wide + table, since no width is assumed.""" + return {"1acb.B": torch.load(_RESIDUES / "1acb.B.pt", map_location="cpu", weights_only=True), + "r5": torch.randn(5, 64, generator=torch.Generator().manual_seed(1))} + + +def _read(folder: Path, extension: str = None) -> dict: + vectors = {} + for ids, batch in stream_embeddings(folder, file_extension=extension): + vectors.update(zip(ids, batch)) + return vectors + + +class CsvEmbeddingsTest(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + + def write(self, writer, embeddings: dict, name: str) -> Path: + """The files fm-embedding and inference write, through their writer.""" + folder = self.tmp / name + folder.mkdir() + files = writer(str(folder)) + for dom_id, embedding in embeddings.items(): + files._write_embedding(embedding, dom_id) + return folder + + def refusal(self, path: Path) -> str: + """The message of the RefusalError with which stream_embeddings refuses the file or folder ``path``.""" + with self.assertRaises(RefusalError) as refused: + _read(path) + return str(refused.exception) + + def test_a_vector_is_one_vector(self): + vectors = _vectors() + csv_folder = self.write(CsvBatchWriter, vectors, "csv") + self.assertEqual(len((csv_folder / "1abc-1.csv").read_text().splitlines()), 1536) # one value per line + from_csv = _read(csv_folder, ".csv") + from_pt = _read(self.write(TensorBatchWriter, vectors, "pt"), ".pt") + self.assertEqual(sorted(from_csv), _IDS) + for dom_id, vector in vectors.items(): + with self.subTest(item=dom_id): + self.assertEqual(from_csv[dom_id].shape, (1536,)) + np.testing.assert_array_equal(from_pt[dom_id], vector.numpy()) + np.testing.assert_allclose(from_csv[dom_id], vector.numpy(), rtol=1e-6, atol=0) + + def test_residue_embeddings_are_refused(self): + """Residue embeddings as fm-embedding and inference write them ([L, D], one row per residue) are refused, not + averaged: the file and its shape are named, with the ways to chain embeddings for its format.""" + for writer, ext in ((CsvBatchWriter, "csv"), (TensorBatchWriter, "pt")): + folder = self.write(writer, _residues(), ext) + for dom_id, table in _residues().items(): + with self.subTest(file=f"{dom_id}.{ext}"): + message = self.refusal(folder / f"{dom_id}.{ext}") + rows, columns = table.shape + self.assertIn(f"{folder / dom_id}.{ext} holds a table of {rows} x {columns} values, not one " + f"embedding vector.", message) + self.assertIn("compute chain embeddings with 'fm-embedding from-structures chain' or " + "'fm-embedding from-sequences chain'", message) + self.assertIn(f"'inference chain-embedding --src-file LIST --res-embedding-location DIR " + f"--res-embedding-format {ext} --output-folder OUT', LIST holding one line " + f"NAME,,CHAIN per file.", message) + self.assertNotIn("header", message) # the hints of pandas' layouts are for other tables + with self.subTest(folder=ext): + self.assertIn(f"{folder / '1acb.B'}.{ext} holds a table", self.refusal(folder)) + + def test_one_row_or_one_column_is_a_vector(self): + """A [1, D] or [D, 1] tensor and a one-line CSV hold one vector, read bit for bit ([D, 1] was read as the mean + of its D values); a bfloat16 tensor, which numpy cannot take, is read as its float32 values.""" + vector = torch.randn(1536, generator=torch.Generator().manual_seed(2)) + for name, writer, ext, shaped, expected in ( + ("row", CsvBatchWriter, "csv", vector.reshape(1, -1), vector), + ("row", TensorBatchWriter, "pt", vector.reshape(1, -1), vector), + ("column", TensorBatchWriter, "pt", vector.reshape(-1, 1), vector), + ("bfloat16", TensorBatchWriter, "pt", vector.bfloat16(), vector.bfloat16().float())): + with self.subTest(file=f"{name}.{ext}", shape=tuple(shaped.shape)): + folder = self.write(writer, {name: shaped}, f"{name}-{ext}") + if ext == "csv": + self.assertEqual(len((folder / "row.csv").read_text().splitlines()), 1) + read = _read(folder / f"{name}.{ext}") + np.testing.assert_array_equal(read[name], expected.numpy()) + + def test_nan_and_infinity_are_refused(self): + """A NaN or infinite value is refused, in a .csv (with what makes one in a CSV) as in a .pt: such an embedding + is never found, and as a query it finds nothing. A .pt table with a NaN is refused as a table: in a .pt the + shape is checked first.""" + vectors = _vectors() + vectors["2xyz-1"][100] = float("nan") + vectors["3def-1"][7] = float("inf") + for writer, ext in ((CsvBatchWriter, "csv"), (TensorBatchWriter, "pt")): + folder = self.write(writer, vectors, ext) + for dom_id in ("2xyz-1", "3def-1"): + with self.subTest(file=f"{dom_id}.{ext}"): + message = self.refusal(folder / f"{dom_id}.{ext}") + self.assertIn(f"{folder / dom_id}.{ext} holds 1 NaN or infinite value(s) among its 1536: such an " + f"embedding is never found, and as a query it finds nothing.", message) + self.assertEqual("In a CSV an empty field reads as NaN" in message, ext == "csv") + with self.subTest(folder=ext): + self.assertIn(f"{folder / '2xyz-1'}.{ext} holds 1 NaN", self.refusal(folder)) + table = _residues()["r5"] + table[2, 3] = float("nan") + folder = self.write(TensorBatchWriter, {"r5": table}, "nan-table") + with self.subTest(file="r5.pt"): + message = self.refusal(folder / "r5.pt") + self.assertIn(f"{folder / 'r5.pt'} holds a table of 5 x 64 values, not one embedding vector.", message) + self.assertNotIn("NaN", message) + + def test_empty_files_are_refused(self): + """An empty tensor holds no values; an empty .csv (CsvBatchWriter's file for an empty tensor) and an empty .pt + file (a run killed before it wrote one) cannot be read. Each is named.""" + folder = self.tmp / "empty" + folder.mkdir() + TensorBatchWriter(str(folder))._write_embedding(torch.zeros(0, 1536), "tensor") + CsvBatchWriter(str(folder))._write_embedding(torch.zeros(0, 1536), "table") + (folder / "file.pt").write_bytes(b"") + expected = {"tensor.pt": "holds no values.", + "table.csv": "is not a CSV of numbers (No columns to parse from file). A .csv embedding holds one " + "vector", + "file.pt": "could not be read as a tensor (EOFError). A .pt embedding holds one vector"} + for name, text in expected.items(): + with self.subTest(file=name): + self.assertIn(f"{folder / name} {text}", self.refusal(folder / name)) + + def test_a_pt_that_is_not_a_tensor_is_named(self): + """A .pt that torch.load cannot read (cut short, a pickled object, a text file) or that holds something other + than a dense tensor is refused with its name, not with a traceback. The cause is not torch's advice for a pickle + (load it with weights_only=False, which the command cannot do), nor a bare number.""" + folder = self.tmp / "not-tensors" + folder.mkdir() + torch.save(_vectors()["1abc-1"], folder / "whole.pt") + data = (folder / "whole.pt").read_bytes() + (folder / "cut.pt").write_bytes(data[:len(data) // 2]) + torch.save({"embedding": _vectors()["1abc-1"]}, folder / "dict.pt") + torch.save({"embedding": Path("x.pt")}, folder / "pickled.pt") # a class weights_only does not load + (folder / "text.pt").write_text("hello world\n") + torch.save(_vectors()["1abc-1"].to_sparse(), folder / "sparse.pt") + expected = {"cut.pt": "could not be read as a tensor (PytorchStreamReader failed", + "pickled.pt": "could not be read as a tensor (it holds pickled objects other than tensors, such as " + "a numpy array, or is not a torch file). A .pt embedding holds one vector", + "text.pt": "could not be read as a tensor (KeyError: 101). A .pt embedding holds one vector", + "dict.pt": "holds a dict, not a tensor. A .pt embedding holds one vector", + "sparse.pt": "holds a tensor that is not D dense values (can't convert Sparse layout tensor"} + for name, text in expected.items(): + with self.subTest(file=name): + message = self.refusal(folder / name) + self.assertIn(f"{folder / name} {text}", message) + self.assertLess(len(message), len(str(folder)) + 400) + + def test_a_csv_that_does_not_parse_is_named(self): + """A .csv that is not a table of numbers (another separator, a text header, UTF-16, a line of more values, a + number too large for a float) is refused with its name and the cause cut short: the error quoted the whole row + (11 KB for a tab-separated vector), without the file.""" + values = [repr(float(x)) for x in _vectors()["1abc-1"]] + folder = self.tmp / "unparsed" + folder.mkdir() + (folder / "tabs.csv").write_text("\t".join(values) + "\n") + (folder / "named.csv").write_text("embedding\n" + "\n".join(values) + "\n") + (folder / "utf16.csv").write_bytes("\n".join(values).encode("utf-16")) + (folder / "ragged.csv").write_text("1,2,3\n4,5,6\n7,8,9,10\n") + (folder / "huge.csv").write_text("1\n" + "9" * 400 + "\n") # read as a Python int + causes = {"tabs": f"could not convert string to float: '{values[0]}\\t{values[1]}", # the cell's repr + "named": "could not convert string to float: 'embedding')", + "utf16": "'utf-8' codec can't decode byte 0xff", + "ragged": "Error tokenizing data. C error: Expected 3 fields in line 3, saw 4)", + "huge": "int too large to convert to float)"} + for name, cause in causes.items(): + with self.subTest(file=name): + message = self.refusal(folder / f"{name}.csv") + self.assertIn(f"{folder / name}.csv is not a CSV of numbers ({cause}", message) + self.assertIn("A .csv embedding holds one vector: numbers separated by commas, one per line or all on " + "one line, with no header line or index column.", message) + self.assertLess(len(message), len(str(folder)) + 400) + + def test_a_csv_that_cannot_be_read_is_named(self): + """A folder named NAME.csv in an embedding folder, which the glob of .csv files matches, is refused with its + name, as a .pt that cannot be read is, not with a traceback.""" + folder = self.tmp / "folder" + (folder / "x.A.csv").mkdir(parents=True) + message = self.refusal(folder) + self.assertTrue(message.startswith(f"{folder / 'x.A.csv'} could not be read ("), message) + self.assertIn("Is a directory", message) + + def test_pandas_header_and_index_layouts_are_refused(self): + """Files written with pandas' default header line (0,1,...,D-1) or index column are tables, refused with what + makes them so: a vector on one line under a header (it was averaged with 0,1,...,1535), a vector in one + column with an index (it was averaged into 2 values) and residue embeddings under a header (with the residue + advice too: 'inference chain-embedding' reads the header as one more residue). With both, as to_csv writes + by default, the empty field above the index reads as NaN: a vector written from a DataFrame or a Series, and + residue embeddings, get the NaN message, which names that layout instead of a line cut short.""" + folder = self.tmp / "pandas" + folder.mkdir() + vector = _vectors()["1abc-1"].numpy() + pd.DataFrame([vector]).to_csv(folder / "header.csv", index=False) + pd.DataFrame(vector).to_csv(folder / "index.csv", header=False) + pd.DataFrame(_residues()["1acb.B"].numpy()).to_csv(folder / "residues.csv", index=False) + pd.DataFrame(vector).to_csv(folder / "default.csv") + pd.Series(vector).to_csv(folder / "series.csv") + pd.DataFrame(_residues()["1acb.B"].numpy()).to_csv(folder / "residues-default.csv") + header = "Its first line is 0,1,...,1535: a column header such as pandas writes unless header=False; remove it." + layout = "A header line, an index column or a decimal comma makes a vector a table." + both = ("Its first line is a column header over an index column, as pandas' to_csv writes them unless " + "header=False and index=False: the empty field above the index reads as NaN.") + for name, shape, hints in (("header", "2 x 1536", (header, layout)), ("index", "1536 x 2", (layout,)), + ("residues", "66 x 1536", (header,))): + with self.subTest(file=name): + message = self.refusal(folder / f"{name}.csv") + self.assertIn(f"{folder / name}.csv holds a table of {shape} values, not one embedding vector.", + message) + self.assertIn("--res-embedding-format csv --output-folder OUT", message) + for hint in (header, layout): + self.assertEqual(hint in message, hint in hints, hint) + for name, size in (("default", 1537 * 2), ("series", 1537 * 2), ("residues-default", 66 * 1537)): + with self.subTest(file=name): + path = folder / f"{name}.csv" + self.assertEqual(self.refusal(path), f"{path} holds 1 NaN or infinite value(s) among its {size}: such " + f"an embedding is never found, and as a query it finds nothing. {both}") + + def test_a_csv_line_cut_short_is_reported_as_such(self): + """A residue CSV cut mid-line (a killed run, a full disk) pads its last line with NaN: it gets the NaN message + and its likely causes, not the residue advice ('inference chain-embedding' made an all-NaN chain embedding of + it). So does a vector with a comma at the end of each line.""" + folder = self.write(CsvBatchWriter, {"1acb.B": _residues()["1acb.B"]}, "cut") + text = (folder / "1acb.B.csv").read_text() + kept = text[:text.index(",", text.index("\n", len(text) // 2) + 100) + 1] # cut after a field, mid-line + (folder / "1acb.B.csv").write_text(kept) + lines = kept.split("\n") + missing = 1536 - len([v for v in lines[-1].split(",") if v]) + (folder / "comma.csv").write_text("".join(f"{x!r},\n" for x in map(float, _vectors()["1abc-1"]))) + hint = ("In a CSV an empty field reads as NaN: a line cut short by an interrupted write, or a comma at the end " + "of each line.") + for name, bad, size in (("1acb.B", missing, len(lines) * 1536), ("comma", 1536, 3072)): + with self.subTest(file=name): + message = self.refusal(folder / f"{name}.csv") + self.assertIn(f"{folder / name}.csv holds {bad} NaN or infinite value(s) among its {size}: ", message) + self.assertIn(hint, message) + self.assertNotIn("inference chain-embedding", message) + + def test_the_command_refuses_residue_embeddings(self): + """'fm-search build embeddings' on a folder of residue CSVs (the default output of 'fm-embedding + from-structures residue'): exit status 1, one 'Error:' line that names the file, no database file.""" + folder = self.write(CsvBatchWriter, _residues(), "residues") + db = self.tmp / "db" + db.mkdir() + argv = ["fm-search", "build", "embeddings", "--embedding-folder", str(folder), "--output-db", + str(db / "residues")] + stderr = self.enterContext(mock.patch("sys.stderr", new_callable=io.StringIO)) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + with mock.patch.object(sys, "argv", argv), self.assertRaises(SystemExit) as ctx: + search.main() + finally: + root.handlers[:] = handlers + root.setLevel(level) + self.assertEqual(ctx.exception.code, 1) + errors = [line for line in stderr.getvalue().splitlines() if line.startswith("Error: ")] + self.assertEqual(len(errors), 1, stderr.getvalue()) + resolved = folder.resolve() # --embedding-folder is resolved (/var -> /private/var on macOS) + self.assertIn(f"Error: {resolved / '1acb.B.csv'} holds a table of 65 x 1536 values, not one embedding vector.", + errors[0]) + self.assertEqual(list(db.iterdir()), []) + + def test_build_and_query_from_csv(self): + """'fm-search build embeddings' and 'fm-search query embedding' on CSV vectors: a database of dimension 1536, + in which each vector finds itself. Logging is not under test: the commands leave it as it is.""" + self.enterContext(mock.patch.object(search, "set_log_level")) + folder = self.write(CsvBatchWriter, _vectors(), "csv") + db = self.tmp / "db" / "vectors" + db.parent.mkdir() + search.build_database_from_embeddings(embedding_folder=folder, output_db=str(db), file_extension=".csv", + use_gpu_index=False) + index = faiss.read_index(str(db.parent / "vectors.index")) + self.assertEqual((index.d, index.ntotal), (1536, 3)) + hits = self.tmp / "hits.tsv" + search.query_database_from_embedding(db_path=str(db), embedding_folder=folder, output_file=str(hits), + file_extension=".csv", top_k=3, threshold=None, use_gpu_index=False) + scores = {} + for line in hits.read_text().splitlines(): + query, hit, score = line.split("\t")[:3] + scores[query, hit] = float(score) + self.assertEqual(len(scores), 9, hits.read_text()) + for (query, hit), score in scores.items(): + with self.subTest(query=query, hit=hit): + if query == hit: + self.assertGreater(score, 0.9999) + else: + self.assertLess(abs(score), 0.2) # random vectors: not +1 or -1 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_download_models.py b/tests/test_download_models.py new file mode 100644 index 0000000..feb60c8 --- /dev/null +++ b/tests/test_download_models.py @@ -0,0 +1,556 @@ +"""P6.0: ``download-models`` downloads and verifies only (plan section 8, P6.0). + +One shared command (``fm-search``, ``fm-embedding`` and ``inference download-models``) fetches the pinned ESM3 +snapshot, the aggregator weights and the esm-codes bundles into the Hugging Face cache and checks their sha256s. It +builds no model, loads no weights, never imports torch and makes no CUDA call, so it is safe on a shared GPU login +node. + +ESM3 (2.8 GB) and the aggregator (690 MB) are synthetic in most tests: a fake Hub (``_FakeHub``) serves small files in +their place in the Hugging Face cache layout, and the pins are patched to their sha256s. The esm-codes bundle is the +published one (``tests/esm_codes_bundle.py``: skipped only when offline and not cached). ``RealCacheTest`` runs the +command on the real cached files, offline and without torch; ``RealBundleDownloadTest`` downloads the published bundle +into an empty cache (it needs the Hub, so it is skipped with ``HF_HUB_OFFLINE=1``). +""" +import functools +import hashlib +import importlib +import inspect +import io +import json +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from contextlib import ExitStack, contextmanager, redirect_stderr +from pathlib import Path +from unittest import mock + +import huggingface_hub +import typer +from huggingface_hub import constants as hf_constants +from typer.testing import CliRunner + +import foldmatch +from foldmatch.cli import args_utils +from foldmatch.search import scoring_system as ss +from foldmatch.utils import model +from foldmatch.utils.esm import loaders + +CLIS = {"fm-search": "search", "fm-embedding": "embedding", "inference": "inference"} + +ESM3_FILES = { + loaders.ESM_WEIGHTS_FILE: b"synthetic ESM3 weights", + "data/weights/esm3_structure_encoder_v0.pth": b"synthetic structure encoder", + "README.md": b"synthetic readme", +} +AGGREGATOR = b"synthetic aggregator weights" +_REAL_SNAPSHOT_DOWNLOAD = huggingface_hub.snapshot_download +_REAL_HF_HUB_DOWNLOAD = huggingface_hub.hf_hub_download + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +SYNTHETIC_PINS = {"esm3": _sha(ESM3_FILES[loaders.ESM_WEIGHTS_FILE]), "aggregator": _sha(AGGREGATOR)} + + +def _cli(label): + return importlib.import_module("foldmatch.cli." + CLIS[label]) + + +def _listing(root): + """Every path under ``root`` (symlinked directories followed) with its mtime, or "link" for a symlink.""" + root = Path(root) + out = [] + for folder, dirs, files in os.walk(root, followlinks=True): + for name in dirs + files: + p = Path(folder) / name + out.append((str(p.relative_to(root)), "link" if p.is_symlink() else p.stat().st_mtime_ns)) + return sorted(out) + + +@functools.lru_cache(maxsize=None) +def _published_bundle_bytes(): + """The published esm-codes-v1 bundle, ``{file name: bytes}`` (``unittest.SkipTest`` only offline and not + cached).""" + import esm_codes_bundle + return {name: Path(path).read_bytes() for name, path in esm_codes_bundle.registry_files().items()} + + +def _repo_dir(cache: Path, repo_id: str) -> Path: + return cache / ("models--" + repo_id.replace("/", "--")) + + +class _FakeHub: + """The Hugging Face Hub behind ``snapshot_download`` and ``hf_hub_download``, patched where ``loaders`` and + ``model`` call them. Online it writes a requested file that is not cached (or every file, on a + ``force_download``) into ``cache`` in the Hub cache layout (``blobs/``, ``snapshots//`` -> + blob) and records the download. Offline (``offline = True``) it hands the call to the real huggingface_hub + function with ``local_files_only=True``, which serves the cache or raises, as with ``HF_HUB_OFFLINE=1``. + ``served[(repo, file)]`` is what the Hub serves when it differs from the pinned content (a bad upload).""" + + def __init__(self, cache: Path, bundle): + system = ss.get(ss.DEFAULT_ESM_CODES) + self.cache = cache + self.repos = {loaders.ESM_REPO_ID: (loaders.ESM_REVISION, dict(ESM3_FILES)), + model.REPO_ID: (model.REVISION, {model.FILE_NAME: AGGREGATOR}), + system.bundle_repo: (system.bundle_revision, dict(bundle))} + self.served = {} + self.offline = False + self.calls, self.downloads = [], [] + + def all_files(self): + return sorted((repo, name) for repo, (_rev, files) in self.repos.items() for name in files) + + def pointer(self, repo_id, filename): + return _repo_dir(self.cache, repo_id) / "snapshots" / self.repos[repo_id][0] / filename + + def write(self, repo_id, filename, data, *, blob_name=None): + """Put ``data`` in the cache as ``repo_id``'s ``filename`` (``blob_name`` defaults to its sha256; a test plants + a corrupt file by giving the pinned name to other bytes, as a truncated download does).""" + blob = _repo_dir(self.cache, repo_id) / "blobs" / (blob_name or _sha(data)) + blob.parent.mkdir(parents=True, exist_ok=True) + tmp = blob.with_name(blob.name + ".incomplete") + tmp.write_bytes(data) + os.replace(tmp, blob) + link = self.pointer(repo_id, filename) + link.parent.mkdir(parents=True, exist_ok=True) + if link.is_symlink() or link.exists(): + link.unlink() + link.symlink_to(os.path.relpath(blob, link.parent)) + return link + + def _download(self, repo_id, filename): + self.downloads.append((repo_id, filename)) + return self.write(repo_id, filename, self.served.get((repo_id, filename), self.repos[repo_id][1][filename])) + + def hf_hub_download(self, *, repo_id, filename, revision, cache_dir=None, local_files_only=False, + force_download=False, **kwargs): + self.calls.append(("hf_hub_download", repo_id, filename, revision, force_download)) + assert cache_dir is None and revision == self.repos[repo_id][0], (cache_dir, revision) + if self.offline or local_files_only: + return _REAL_HF_HUB_DOWNLOAD(repo_id=repo_id, filename=filename, revision=revision, + cache_dir=str(self.cache), local_files_only=True, + force_download=force_download) + link = self.pointer(repo_id, filename) + if link.exists() and not force_download: + return str(link) + return str(self._download(repo_id, filename)) + + def snapshot_download(self, *, repo_id, revision, cache_dir=None, **kwargs): + self.calls.append(("snapshot_download", repo_id, revision)) + assert cache_dir is None and revision == self.repos[repo_id][0], (cache_dir, revision) + if self.offline: + return _REAL_SNAPSHOT_DOWNLOAD(repo_id=repo_id, revision=revision, cache_dir=str(self.cache), + local_files_only=True) + for filename in self.repos[repo_id][1]: + if not self.pointer(repo_id, filename).exists(): + self._download(repo_id, filename) + return str(_repo_dir(self.cache, repo_id) / "snapshots" / revision) + + def patches(self): + return [mock.patch.object(loaders, "snapshot_download", self.snapshot_download), + mock.patch.object(loaders, "hf_hub_download", self.hf_hub_download), + mock.patch.object(model, "hf_hub_download", self.hf_hub_download)] + + +def _synthetic_pins(): + return [mock.patch.object(loaders, "ESM_WEIGHTS_SHA256", SYNTHETIC_PINS["esm3"]), + mock.patch.object(model, "AGGREGATOR_SHA256", SYNTHETIC_PINS["aggregator"])] + + +@contextmanager +def _no_models_no_weights_no_cuda(): + """Every way download-models could build a model (any ``torch.nn.Module``, ESM3, the aggregator, the model + getters), load weights (``torch.load``), move anything to a device or call CUDA (every function in + ``torch.cuda``) raises and is recorded in the yielded list.""" + import torch + from esm.models.esm3 import ESM3 + from foldmatch.model.residue_embedding_aggregator import ResidueEmbeddingAggregator + calls = [] + + def refuse(name): + def refused(*args, **kwargs): + calls.append(name) + raise AssertionError(f"download-models called {name}") + return refused + + targets = [(torch.nn.Module, "__init__"), (ESM3, "__init__"), (ResidueEmbeddingAggregator, "__init__"), + (torch.nn.Module, "to"), (torch.nn.Module, "cuda"), (torch.Tensor, "to"), (torch.Tensor, "cuda"), + (torch, "load"), (model, "get_residue_model"), (model, "get_aggregator_model"), + (loaders, "esm_open"), (loaders, "structure_encoder"), (loaders, "data_root")] + cuda = sorted(name for name, obj in vars(torch.cuda).items() + if not name.startswith("__") and (inspect.isfunction(obj) or inspect.isbuiltin(obj))) + assert {"is_available", "device_count", "init", "_lazy_init", "set_device", "mem_get_info"} <= set(cuda), cuda + targets += [(torch.cuda, name) for name in cuda] + with ExitStack() as stack: + for owner, name in targets: + label = f"{getattr(owner, '__name__', type(owner).__name__)}.{name}" + stack.enter_context(mock.patch.object(owner, name, refuse(label))) + yield calls + + +class _HubCase(unittest.TestCase): + """A temporary cache served by a fake Hub, with the synthetic ESM3 and aggregator pins.""" + + @classmethod + def setUpClass(cls): + cls.bundle = _published_bundle_bytes() + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + loaders._hashed.clear() + self.addCleanup(loaders._hashed.clear) + + def hub(self, name="hub") -> _FakeHub: + cache = Path(self._tmp.name) / name + cache.mkdir() + hub = _FakeHub(cache, self.bundle) + stack = ExitStack() + for patch in hub.patches() + _synthetic_pins(): + stack.enter_context(patch) + self.addCleanup(stack.close) + return hub + + def invoke(self, label, *args): + return CliRunner().invoke(_cli(label).app, ["download-models", *args]) + + def assertOk(self, r): + self.assertEqual(r.exit_code, 0, f"{r.output}\n{r.exception!r}") + + +# ===================================================================================================================== +# download only: the three CLIs, no model, no weights loaded, no CUDA; a second run is a cache hit +# ===================================================================================================================== +class DownloadOnlyTest(_HubCase): + + def test_downloads_and_verifies_without_models_or_cuda_then_hits_the_cache(self): + system = ss.get(ss.DEFAULT_ESM_CODES) + for label in CLIS: + with self.subTest(cli=label): + loaders._hashed.clear() + hub = self.hub(label) + with _no_models_no_weights_no_cuda() as refused: + first = self.invoke(label) + self.assertOk(first) + self.assertEqual(sorted(hub.downloads), hub.all_files()) # the first run downloads all + after_first = _listing(hub.cache) + hub.downloads.clear() + second = self.invoke(label) # online again: a cache hit + self.assertOk(second) + hub.offline = True # and offline (real hf, cache only) + third = self.invoke(label) + self.assertOk(third) + self.assertEqual(refused, []) + self.assertEqual(hub.downloads, []) + self.assertEqual(_listing(hub.cache), after_first) # no new or rewritten file + lines = first.output.strip().splitlines() + self.assertEqual(len(lines), 3, first.output) + self.assertTrue(lines[0].startswith( + f"ESM3: {loaders.ESM_REPO_ID}@{loaders.ESM_REVISION}, weights sha256 verified, in "), lines[0]) + self.assertTrue(lines[1].startswith( + f"aggregator: {model.REPO_ID}@{model.REVISION}, sha256 verified, in "), lines[1]) + self.assertTrue(lines[2].startswith( + f"esm-codes bundle {system.name}: {system.bundle_repo}@{system.bundle_revision}, " + f"{len(system.bundle_files)} files, sha256 verified, in "), lines[2]) + self.assertEqual(second.output, first.output) + self.assertEqual(third.output, first.output) + + def test_every_file_is_hashed(self): + """The full sha256 of every file is checked (not only the blob names): ESM3 once per process.""" + hub = self.hub() + spy_esm = mock.Mock(wraps=loaders._file_sha256) + spy_model = mock.Mock(wraps=model._file_sha256) + with mock.patch.object(loaders, "_file_sha256", spy_esm), mock.patch.object(model, "_file_sha256", spy_model): + self.assertOk(self.invoke("fm-search")) + self.assertOk(self.invoke("fm-search")) + hashed = {Path(os.path.realpath(c.args[0])).name for c in spy_model.call_args_list} + self.assertEqual(spy_esm.call_count, 1) + self.assertIn(SYNTHETIC_PINS["aggregator"], hashed) + self.assertTrue({sha for _n, sha in ss.get(ss.DEFAULT_ESM_CODES).bundle_files} <= hashed) + + def test_unknown_system_is_refused_before_any_download(self): + for label in CLIS: + with self.subTest(cli=label): + hub = self.hub(label) + with _no_models_no_weights_no_cuda() as refused: + r = self.invoke(label, "--system", ss.DEFAULT_ESM_CODES, "--system", "nope") + self.assertEqual(r.exit_code, 2, r.output) + text = " ".join(r.output.replace("\u2502", " ").split()) # rich wraps the message in a box + self.assertIn("unknown esm-codes scoring system 'nope'; this FoldMatch knows: " + f"{', '.join(ss.known_names())}", text) + self.assertEqual((hub.calls, refused, _listing(hub.cache)), ([], [], [])) + + def test_system_option_fetches_that_bundle(self): + hub = self.hub() + r = self.invoke("fm-search", "--system", ss.DEFAULT_ESM_CODES) + self.assertOk(r) + system = ss.get(ss.DEFAULT_ESM_CODES) + self.assertEqual(sorted(f for repo, f in hub.downloads if repo == system.bundle_repo), + sorted(n for n, _ in system.bundle_files)) + + +# ===================================================================================================================== +# corrupt, missing and mismatching cached files +# ===================================================================================================================== +class CacheProblemsTest(_HubCase): + """A cached file whose content differs from the pin is downloaded once more; a Hub file that differs, a failed + re-download and a file that is neither cached nor downloadable are refused with one clear error.""" + + def plant(self, hub, *, esm3=None, aggregator=None): + """Cache every file, with ``esm3`` / ``aggregator`` as the weights' content under the pinned blob name.""" + for repo, name in hub.all_files(): + hub.write(repo, name, hub.repos[repo][1][name]) + if esm3 is not None: + hub.write(loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_FILE, esm3, blob_name=SYNTHETIC_PINS["esm3"]) + if aggregator is not None: + hub.write(model.REPO_ID, model.FILE_NAME, aggregator, blob_name=SYNTHETIC_PINS["aggregator"]) + + def test_corrupt_cached_files_are_downloaded_again(self): + hub = self.hub() + self.plant(hub, esm3=b"truncated", aggregator=b"truncated") + snapshot = loaders.download_esm3_snapshot() + aggregator = model.download_aggregator_weights() + self.assertEqual(hub.downloads, [(loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_FILE), + (model.REPO_ID, model.FILE_NAME)]) + self.assertEqual((snapshot / loaders.ESM_WEIGHTS_FILE).read_bytes(), ESM3_FILES[loaders.ESM_WEIGHTS_FILE]) + self.assertEqual(aggregator.read_bytes(), AGGREGATOR) + forced = [c for c in hub.calls if c[0] == "hf_hub_download" and c[-1]] + self.assertEqual(len(forced), 2) + + def test_hub_file_that_differs_from_the_pin_is_refused(self): + hub = self.hub() + self.plant(hub, esm3=b"truncated", aggregator=b"truncated") + hub.served = {(loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_FILE): b"another upload", + (model.REPO_ID, model.FILE_NAME): b"another upload"} + with self.assertRaisesRegex(loaders.EsmWeightsMismatchError, "after a fresh download"): + loaders.download_esm3_snapshot() + with self.assertRaisesRegex(model.AggregatorWeightsMismatchError, "after a fresh download"): + model.download_aggregator_weights() + + def test_offline_corrupt_file_is_refused_with_the_reason(self): + hub = self.hub() + self.plant(hub, esm3=b"truncated", aggregator=b"truncated") + hub.offline = True + with self.assertRaisesRegex(loaders.EsmWeightsMismatchError, r"is corrupt.*Downloading it again failed"): + loaders.download_esm3_snapshot() + with self.assertRaisesRegex(model.AggregatorWeightsMismatchError, + r"cached copy is corrupt.*Downloading it again failed"): + model.download_aggregator_weights() + self.assertEqual(hub.downloads, []) + + def test_offline_and_not_cached(self): + hub = self.hub() + hub.offline = True + with self.assertRaisesRegex(FileNotFoundError, rf"ESM3 snapshot {loaders.ESM_REPO_ID}@{loaders.ESM_REVISION} " + "is not in the Hugging Face cache"): + loaders.download_esm3_snapshot() + with self.assertRaisesRegex(FileNotFoundError, rf"aggregator weights {model.FILE_NAME} \({model.REPO_ID}@" + rf"{model.REVISION}\) are not in the Hugging Face cache"): + model.download_aggregator_weights() + self.assertEqual(_listing(hub.cache), []) + + def test_weights_missing_from_a_cached_snapshot(self): + hub = self.hub() + hub.write(loaders.ESM_REPO_ID, "README.md", ESM3_FILES["README.md"]) + hub.offline = True + with self.assertRaisesRegex(FileNotFoundError, "missing from the cached snapshot"): + loaders.download_esm3_snapshot() + + def test_the_clis_report_a_refusal_as_one_line(self): + hub = self.hub() + self.plant(hub, esm3=b"truncated") + hub.offline = True + for label in CLIS: + with self.subTest(cli=label), mock.patch.object(sys, "argv", [label, "download-models"]): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as ctx: + _cli(label).main() + self.assertEqual(ctx.exception.code, 1) + self.assertIn("Error: ESM3 weights", stderr.getvalue()) + self.assertIn("Downloading it again failed", stderr.getvalue()) + self.assertNotIn("Traceback", stderr.getvalue()) + + +# ===================================================================================================================== +# real huggingface_hub in a subprocess: HF_HUB_OFFLINE=1, an unreachable endpoint, torch cannot be imported +# ===================================================================================================================== +_SCRIPT = textwrap.dedent(""" + import importlib, json, sys + from pathlib import Path + sys.modules["torch"] = None # any torch import fails + cli, pins, stub = sys.argv[1], json.loads(sys.argv[2]), sys.argv[3] == "stub" + from foldmatch.utils import model + from foldmatch.utils.esm import loaders + if pins: # the test's synthetic ESM3 and aggregator files + loaders.ESM_WEIGHTS_SHA256, model.AGGREGATOR_SHA256 = pins["esm3"], pins["aggregator"] + if stub: # the bundle download test does not fetch 3.5 GB of weights + loaders.download_esm3_snapshot = lambda **kw: Path("/stub/esm3") + model.download_aggregator_weights = lambda **kw: Path("/stub/aggregator/weights.pt") + sys.argv = [cli, "download-models", *sys.argv[4:]] + code = "returned" + try: + importlib.import_module("foldmatch.cli." + cli).main() + except SystemExit as e: + code = e.code + print(json.dumps({"exit": code, "esm": "esm" in sys.modules, "lightning": "lightning" in sys.modules})) +""") + + +def _env(cache, *, offline=True): + env = dict(os.environ) + src = str(Path(foldmatch.__file__).resolve().parents[1]) + env["PYTHONPATH"] = src + os.pathsep + env.get("PYTHONPATH", "") + env.update(HF_HUB_CACHE=str(cache), HF_HUB_DISABLE_TELEMETRY="1", HF_HUB_ETAG_TIMEOUT="10", + HF_HUB_DOWNLOAD_TIMEOUT="30") + env.pop("HF_HUB_OFFLINE", None) + if offline: + env.update(HF_HUB_OFFLINE="1", HF_ENDPOINT="http://127.0.0.1:9") # any network use fails + return env + + +def _run(label, cache, *args, pins=None, stub=False, offline=True): + """``label download-models args`` in a fresh interpreter without torch; returns (exit code, stdout lines + before the result, the result dict, stderr).""" + r = subprocess.run([sys.executable, "-c", _SCRIPT, CLIS[label], json.dumps(pins or {}), + "stub" if stub else "-", *args], + env=_env(cache, offline=offline), capture_output=True, text=True, timeout=600) + lines = r.stdout.strip().splitlines() + if not lines: + raise AssertionError(f"no output (exit {r.returncode}): {r.stderr[-3000:]}") + result = json.loads(lines[-1]) + return result, lines[:-1], r.stderr + + +class OfflineSubprocessTest(_HubCase): + + def test_offline_after_the_download_without_torch(self): + hub = self.hub() + self.assertOk(self.invoke("fm-search")) # the download (fake Hub) + before = _listing(hub.cache) + for label in CLIS: + with self.subTest(cli=label): + result, lines, stderr = _run(label, hub.cache, pins=SYNTHETIC_PINS) + self.assertEqual(result["exit"], 0, stderr[-3000:]) + self.assertFalse(result["esm"] or result["lightning"], result) + self.assertEqual([line.split(",")[0] for line in lines], + [f"ESM3: {loaders.ESM_REPO_ID}@{loaders.ESM_REVISION}", + f"aggregator: {model.REPO_ID}@{model.REVISION}", + f"esm-codes bundle {ss.DEFAULT_ESM_CODES}: " + f"{ss.get(ss.DEFAULT_ESM_CODES).bundle_repo}@" + f"{ss.get(ss.DEFAULT_ESM_CODES).bundle_revision}"]) + self.assertEqual(_listing(hub.cache), before) + + def test_offline_before_any_download_is_a_one_line_error(self): + cache = Path(self._tmp.name) / "empty" + cache.mkdir() + result, lines, stderr = _run("fm-search", cache, pins=SYNTHETIC_PINS) + self.assertEqual(result["exit"], 1, stderr[-3000:]) + self.assertIn(f"Error: ESM3 snapshot {loaders.ESM_REPO_ID}@{loaders.ESM_REVISION} is not in the Hugging Face " + "cache", stderr) + self.assertNotIn("Traceback", stderr) + self.assertEqual(_listing(cache), []) + + +# ===================================================================================================================== +# the same command on the three CLIs +# ===================================================================================================================== +class SameCommandTest(unittest.TestCase): + + def test_same_help_and_option(self): + helps = {} + for label in CLIS: + command = typer.main.get_command(_cli(label).app).commands["download-models"] + with self.subTest(cli=label): + self.assertEqual(command.help, args_utils.DOWNLOAD_MODELS_HELP) + [param] = command.params + self.assertEqual((param.name, param.opts, param.multiple, param.default, param.help), + ("system", ["--system"], True, None, args_utils.DOWNLOAD_MODELS_SYSTEM_HELP)) + r = CliRunner().invoke(_cli(label).app, ["download-models", "--help"]) + self.assertEqual(r.exit_code, 0, r.output) + helps[label] = r.output + self.assertEqual(len(set(helps.values())), 1, helps) + self.assertIn("Download only", helps["fm-search"]) + self.assertIn("--system", helps["fm-search"]) + top = CliRunner().invoke(_cli("fm-search").app, ["--help"]) + self.assertIn("download-models", top.output) + + def test_every_cli_calls_the_shared_function(self): + for label in CLIS: + self.assertIs(_cli(label).download_models_command, args_utils.download_models_command) + with self.subTest(cli=label), mock.patch.object(_cli(label), "download_models_command") as shared: + r = CliRunner().invoke(_cli(label).app, ["download-models", "--system", "a", "--system", "b"]) + self.assertEqual(r.exit_code, 0, r.output) + shared.assert_called_once_with(["a", "b"]) + + def test_download_path_imports_without_torch(self): + code = ("import sys; sys.modules['torch'] = None\n" + "import foldmatch.utils.esm.loaders, foldmatch.utils.model, foldmatch.cli.args_utils\n" + "import foldmatch.cli.search, foldmatch.cli.embedding, foldmatch.cli.inference\n" + "print(sorted(m for m in ('esm', 'biotite', 'lightning') if m in sys.modules))\n") + r = subprocess.run([sys.executable, "-c", code], env=_env(Path(tempfile.gettempdir())), capture_output=True, + text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + self.assertEqual(r.stdout.strip(), "[]") + + +# ===================================================================================================================== +# the real files +# ===================================================================================================================== +class RealCacheTest(unittest.TestCase): + """The command on the real pinned files in this machine's Hugging Face cache, offline, without torch, with an + unreachable endpoint: the pins (ESM3, aggregator, bundle) match the files, and nothing is written.""" + + def test_real_files_offline(self): + system = ss.get(ss.DEFAULT_ESM_CODES) + _published_bundle_bytes() # skipped only offline and not cached + missing = [f"{repo}/{name}" for repo, name, rev in ( + (loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_FILE, loaders.ESM_REVISION), + (model.REPO_ID, model.FILE_NAME, model.REVISION)) + if not isinstance(huggingface_hub.try_to_load_from_cache(repo, name, revision=rev), str)] + if missing: + self.skipTest(f"not in the Hugging Face cache {hf_constants.HF_HUB_CACHE}: {missing} (run download-models)") + cache = Path(hf_constants.HF_HUB_CACHE) + repos = [_repo_dir(cache, r) for r in (loaders.ESM_REPO_ID, model.REPO_ID, system.bundle_repo)] + before = [_listing(r) for r in repos] + result, lines, stderr = _run("fm-search", cache) + self.assertEqual(result["exit"], 0, stderr[-3000:]) + self.assertFalse(result["esm"] or result["lightning"], result) + self.assertEqual(len(lines), 3, lines) + self.assertIn(f"{loaders.ESM_REPO_ID}@{loaders.ESM_REVISION}, weights sha256 verified", lines[0]) + self.assertIn(f"{model.REPO_ID}@{model.REVISION}, sha256 verified", lines[1]) + self.assertIn(f"{system.bundle_repo}@{system.bundle_revision}, 5 files, sha256 verified", lines[2]) + self.assertEqual([_listing(r) for r in repos], before) + + +class RealBundleDownloadTest(unittest.TestCase): + """``fm-search download-models`` downloads the published bundle into an empty cache (ESM3 and the aggregator + stubbed: 3.5 GB), then serves it offline with no new file. Needs the Hub: skipped with HF_HUB_OFFLINE=1.""" + + def test_fresh_download_then_offline(self): + if hf_constants.HF_HUB_OFFLINE: + self.skipTest("needs the Hugging Face Hub (HF_HUB_OFFLINE=1)") + system = ss.get(ss.DEFAULT_ESM_CODES) + with tempfile.TemporaryDirectory() as tmp: + cache = Path(tmp) / "hub" + cache.mkdir() + result, lines, stderr = _run("fm-search", cache, "--system", system.name, stub=True, offline=False) + self.assertEqual(result["exit"], 0, stderr[-3000:]) + snapshot = _repo_dir(cache, system.bundle_repo) / "snapshots" / system.bundle_revision + self.assertEqual(lines[-1], f"esm-codes bundle {system.name}: {system.bundle_repo}@" + f"{system.bundle_revision}, 5 files, sha256 verified, in {snapshot}") + for name, sha in system.bundle_files: + self.assertEqual(model._file_sha256(snapshot / name), sha, name) + before = _listing(cache) + result, lines2, stderr = _run("fm-search", cache, "--system", system.name, stub=True, offline=True) + self.assertEqual(result["exit"], 0, stderr[-3000:]) + self.assertEqual(lines2, lines) + self.assertEqual(_listing(cache), before) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_embedding_computer.py b/tests/test_embedding_computer.py new file mode 100644 index 0000000..e7c9f44 --- /dev/null +++ b/tests/test_embedding_computer.py @@ -0,0 +1,665 @@ +"""Shared run id and shard handling for multi-rank embedding passes (G7).""" +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import unittest +import zlib +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +import unittest.mock +from unittest.mock import patch + +import numpy as np +import torch + +import foldmatch +from foldmatch.types.api_types import Accelerator +from foldmatch.utils.dist import RUN_ID_ENV, command_digest, exported_run_id, launch_rank, run_id + +_TEST_PATH = Path(__file__).parent +_FIXTURE_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_SRC = str(Path(foldmatch.__file__).resolve().parent.parent) + +# Everything that decides the run id or the launch rank. +_LAUNCH_ENV = (RUN_ID_ENV, "SLURM_JOB_ID", "SLURM_STEP_ID", "SLURM_RESTART_COUNT", + "SLURM_PROCID", "RANK", "LOCAL_RANK", "NODE_RANK") + + +@contextmanager +def launch_env(**values): + """Run with exactly ``values`` set among the launch variables (the rest unset). + + Only those variables are saved and restored. Restoring the whole environment + would also undo what libraries set on first import (sklearn sets + KMP_DUPLICATE_LIB_OK, without which a second OpenMP runtime aborts the process). + """ + saved = {name: os.environ.get(name) for name in _LAUNCH_ENV} + try: + for name in _LAUNCH_ENV: + os.environ.pop(name, None) + os.environ.update({k: str(v) for k, v in values.items()}) + yield + finally: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +# Rendezvous state a parent process may hold from its own (single-rank) DDP pass; +# a child that inherited MASTER_PORT would try to join that dead process group. +_DDP_ENV_PREFIXES = ("MASTER_", "PL_", "TORCHELASTIC_") +_DDP_ENV = ("WORLD_SIZE", "LOCAL_WORLD_SIZE", "GROUP_RANK", "ROLE_RANK") + + +def _child_env(**values): + env = {k: v for k, v in os.environ.items() + if k not in _LAUNCH_ENV and k not in _DDP_ENV and not k.startswith(_DDP_ENV_PREFIXES)} + env.update({k: str(v) for k, v in values.items()}) + env["PYTHONPATH"] = _SRC + os.pathsep + env.get("PYTHONPATH", "") + return env + + +def _embedding(name, dim=8): + vec = np.random.default_rng(zlib.crc32(name.encode())).normal(size=dim).astype(np.float32) + return torch.from_numpy(vec / np.linalg.norm(vec)) + + +def _write_shard(folder, out_name, rank, names, keys=None): + """Write one rank's shard exactly as the Lightning writer callback does; each row's embedding is fixed by its + entry of ``keys`` (default: its id).""" + from foldmatch.writer.batch_writer import ParquetBatchWriter + writer = ParquetBatchWriter(folder, out_name) + trainer = SimpleNamespace(global_rank=rank) + writer.write_on_batch_end(trainer, None, ([_embedding(k) for k in (keys or names)], tuple(names)), + None, None, 0, 0) + writer.on_predict_end(trainer, None) + + +def fake_chain_predict(world_size=1, drop=(), by_sequence=False): + """Stand-in for chain_complete_inference.predict on FASTA input. + + Splits the entries over ``world_size`` simulated ranks the way the + distributed sampler does and writes one shard per rank; ``drop`` loses + entries, like a rank whose shard never reaches rank 0. With + ``by_sequence`` a row's embedding is fixed by the sequence the dataset + gives (as with ESM3) instead of by its id, so two records with one id + have different embeddings. + """ + def predict(**kwargs): + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + dataset = EsmProtFromFasta(kwargs["src_stream"], min_res_n=kwargs["min_res_n"]) + items = [dataset[i] for i in range(len(dataset))] + items = [(prot, n) for prot, n in items if n not in drop] + for rank in range(world_size): + dealt = items[rank::world_size] + _write_shard(kwargs["out_folder"], kwargs["out_name"], rank, [n for _, n in dealt], + keys=[prot.sequence for prot, _ in dealt] if by_sequence else None) + return predict + + +def _ids(computer): + return sorted(i for ids, _ in computer.get_embedding_batches() for i in ids) + + +def _digest(argv): + """What command_digest() gives for the command line ``argv``.""" + return hashlib.sha1("\0".join(argv[1:]).encode()).hexdigest()[:8] + + +# Two command lines, as two commands of one srun step would have them. +_BUILD_ARGV = ["fm-search", "build", "sequences", "--fasta-file", "a.fasta", "--output-db", "db"] +_QUERY_ARGV = ["fm-search", "query", "sequences", "--db-path", "db", "--fasta-file", "q.fasta"] + + +class RunIdTest(unittest.TestCase): + + def test_explicit_id_wins(self): + with launch_env(FOLDMATCH_RUN_ID="my-run.1", SLURM_JOB_ID=7, LOCAL_RANK=1): + self.assertEqual(run_id(), "my-run.1") + + def test_srun_step_id_includes_step_restart_count_and_command(self): + digest = _digest(_BUILD_ARGV) + with patch.object(sys, "argv", _BUILD_ARGV): + self.assertEqual(command_digest(), digest) + with launch_env(SLURM_JOB_ID=123, SLURM_STEP_ID=4, SLURM_RESTART_COUNT=2, SLURM_PROCID=3): + self.assertEqual(run_id(), f"123_4_2_{digest}") + with launch_env(SLURM_JOB_ID=123, SLURM_STEP_ID=4): + self.assertEqual(run_id(), f"123_4_0_{digest}") + # The program path is not part of it: every task runs the same arguments. + with patch.object(sys, "argv", ["/other/path/search.py"] + _BUILD_ARGV[1:]): + self.assertEqual(command_digest(), digest) + with patch.object(sys, "argv", _QUERY_ARGV): + self.assertNotEqual(command_digest(), digest) + + def test_sbatch_batch_step_draws_a_fresh_id_per_command(self): + # The batch step has no SLURM_STEP_ID, or one of SLURM's special step values. + for step in ({}, {"SLURM_STEP_ID": 4294967294}, {"SLURM_STEP_ID": 0xFFFFFFFB}): + with self.subTest(step=step), patch.object(sys, "argv", _BUILD_ARGV): + with launch_env(SLURM_JOB_ID=123, SLURM_PROCID=0, **step): + first, second = run_id(), run_id() + self.assertRegex(first, r"^[0-9a-f]{16}$") + self.assertNotEqual(first, second) + # Only a Lightning child (which inherits the id) can be rank > 0 there. + with launch_env(SLURM_JOB_ID=123, SLURM_PROCID=0, LOCAL_RANK=1, **step): + with self.assertRaises(ValueError): + run_id() + + def test_rank_zero_draws_a_fresh_id(self): + with launch_env(): + first, second = run_id(), run_id() + self.assertRegex(first, r"^[0-9a-f]{16}$") + self.assertNotEqual(first, second) + + def test_rank_above_zero_without_an_id_refuses(self): + for name in ("LOCAL_RANK", "SLURM_PROCID", "NODE_RANK", "RANK"): + with self.subTest(name=name), launch_env(**{name: 1}): + with self.assertRaises(ValueError) as ctx: + run_id() + self.assertIn(RUN_ID_ENV, str(ctx.exception)) + + def test_invalid_explicit_id_is_refused(self): + for bad in ("../x", "a/b", "a*", "-lead", "a b"): + with self.subTest(bad=bad), launch_env(FOLDMATCH_RUN_ID=bad): + with self.assertRaises(ValueError): + run_id() + + def test_launch_rank(self): + cases = [({}, 0), ({"LOCAL_RANK": 0, "NODE_RANK": 0}, 0), ({"LOCAL_RANK": 2}, 2), + ({"SLURM_PROCID": 5}, 5), ({"NODE_RANK": 1, "LOCAL_RANK": 0}, 1), + ({"RANK": 0, "LOCAL_RANK": 0, "SLURM_PROCID": 0}, 0), ({"RANK": 6, "LOCAL_RANK": 2}, 6), + # A Lightning child of a single srun task inherits SLURM_PROCID=0. + ({"SLURM_PROCID": 0, "LOCAL_RANK": 1}, 1)] + for values, expected in cases: + with self.subTest(values=values), launch_env(**values): + self.assertEqual(launch_rank(), expected) + + def test_exported_run_id_restores_the_environment(self): + with launch_env(): + with exported_run_id("abc"): + self.assertEqual(os.environ[RUN_ID_ENV], "abc") + self.assertNotIn(RUN_ID_ENV, os.environ) + with launch_env(FOLDMATCH_RUN_ID="user"): + with exported_run_id("user"): + self.assertEqual(os.environ[RUN_ID_ENV], "user") + self.assertEqual(os.environ[RUN_ID_ENV], "user") + with launch_env(): + with self.assertRaises(RuntimeError): + with exported_run_id("abc"): + raise RuntimeError("pass failed") + self.assertNotIn(RUN_ID_ENV, os.environ) + + +class ShardIsolationTest(unittest.TestCase): + """Runs sharing one folder only ever read their own shards.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def _computer(self, **env): + from foldmatch.search.embedding_computer import EmbeddingComputer + with launch_env(**env): + return EmbeddingComputer(self.tmp) + + def test_same_slurm_job_different_steps(self): + with patch.object(sys, "argv", _BUILD_ARGV): + step0 = self._computer(SLURM_JOB_ID=99, SLURM_STEP_ID=0) + step1 = self._computer(SLURM_JOB_ID=99, SLURM_STEP_ID=1) + requeued = self._computer(SLURM_JOB_ID=99, SLURM_STEP_ID=0, SLURM_RESTART_COUNT=1) + self.assertEqual(len({step0.out_name, step1.out_name, requeued.out_name}), 3) + _write_shard(self.tmp, step0.out_name, 0, ["s0_a"]) + _write_shard(self.tmp, step0.out_name, 1, ["s0_b"]) + _write_shard(self.tmp, step1.out_name, 0, ["s1_a"]) + _write_shard(self.tmp, requeued.out_name, 0, ["r_a"]) + self.assertEqual(_ids(step0), ["s0_a", "s0_b"]) + self.assertEqual(_ids(step1), ["s1_a"]) + self.assertEqual(_ids(requeued), ["r_a"]) + digest = _digest(_BUILD_ARGV) + self.assertEqual([p.name for p in step0.shard_files()], + [f"emb_99_0_0_{digest}-0.parquet", f"emb_99_0_0_{digest}-1.parquet"]) + + def test_different_commands_in_one_srun_step(self): + with patch.object(sys, "argv", _BUILD_ARGV): + build = self._computer(SLURM_JOB_ID=99, SLURM_STEP_ID=2, SLURM_PROCID=0) + build_rank1 = self._computer(SLURM_JOB_ID=99, SLURM_STEP_ID=2, SLURM_PROCID=1) + with patch.object(sys, "argv", _QUERY_ARGV): + query = self._computer(SLURM_JOB_ID=99, SLURM_STEP_ID=2, SLURM_PROCID=0) + self.assertEqual(build.out_name, build_rank1.out_name) + self.assertNotEqual(build.out_name, query.out_name) + _write_shard(self.tmp, build.out_name, 0, ["b0"]) + _write_shard(self.tmp, build_rank1.out_name, 1, ["b1"]) + _write_shard(self.tmp, query.out_name, 0, ["q0"]) + self.assertEqual(_ids(build), ["b0", "b1"]) + self.assertEqual(_ids(query), ["q0"]) + query.remove_shards() + self.assertEqual(_ids(build), ["b0", "b1"]) + + def test_commands_in_one_sbatch_batch_step(self): + with patch.object(sys, "argv", _BUILD_ARGV): + first = self._computer(SLURM_JOB_ID=99) + second = self._computer(SLURM_JOB_ID=99) + self.assertNotEqual(first.out_name, second.out_name) + + def test_existing_shards_with_this_id_are_refused_before_the_pass(self): + computer = self._computer(FOLDMATCH_RUN_ID="reused") + _write_shard(self.tmp, computer.out_name, 1, ["stale"]) + fasta = self.tmp / "one.fasta" + fasta.write_text(">q0\nACDEFGHIKL\n") + structures = self.tmp / "structures" + structures.mkdir() + (structures / "x.cif").write_text("") + predict = unittest.mock.Mock() + for method, path in ((computer.compute_from_fasta, fasta), (computer.compute_from_structures, structures)): + with self.subTest(method=method.__name__), \ + patch("foldmatch.search.embedding_computer.chain_predict", predict), \ + patch("foldmatch.search.embedding_computer.assembly_predict", predict): + with self.assertRaises(FileExistsError) as ctx: + method(path, accelerator=Accelerator.cpu) + self.assertIn("same run id (reused)", str(ctx.exception)) + self.assertIn(RUN_ID_ENV, str(ctx.exception)) + predict.assert_not_called() + # Another id in the same folder is not affected. + other = self._computer(FOLDMATCH_RUN_ID="fresh") + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + other.compute_from_fasta(fasta, accelerator=Accelerator.cpu) + self.assertEqual(_ids(other), ["q0"]) + + def test_id_prefix_does_not_leak(self): + short = self._computer(FOLDMATCH_RUN_ID="abc") + longer = self._computer(FOLDMATCH_RUN_ID="abc-1") + _write_shard(self.tmp, short.out_name, 0, ["short"]) + _write_shard(self.tmp, longer.out_name, 0, ["longer"]) + self.assertEqual(_ids(short), ["short"]) + self.assertEqual(_ids(longer), ["longer"]) + + def test_count_check(self): + computer = self._computer(FOLDMATCH_RUN_ID="run") + _write_shard(self.tmp, computer.out_name, 0, ["a", "b"]) + computer.expected_count = 2 + self.assertEqual(_ids(computer), ["a", "b"]) + # A stale shard from an earlier run with the same id is detected, not merged. + _write_shard(self.tmp, computer.out_name, 1, ["stale"]) + with self.assertRaises(ValueError) as ctx: + _ids(computer) + self.assertIn("Found 3 embeddings", str(ctx.exception)) + self.assertIn("2 entries", str(ctx.exception)) + # So is a missing rank. + computer.expected_count = 4 + with self.assertRaises(ValueError): + _ids(computer) + + def test_remove_shards_only_removes_this_run(self): + mine = self._computer(FOLDMATCH_RUN_ID="mine") + other = self._computer(FOLDMATCH_RUN_ID="mine-2") + _write_shard(self.tmp, mine.out_name, 0, ["a"]) + _write_shard(self.tmp, mine.out_name, 1, ["b"]) + _write_shard(self.tmp, other.out_name, 0, ["c"]) + mine.remove_shards() + self.assertEqual(mine.shard_files(), []) + self.assertEqual(_ids(other), ["c"]) + + +class InputOrderTest(unittest.TestCase): + """P7.0a: :meth:`EmbeddingComputer.in_input_order` puts a multi-rank run's search results (rank 0 reads the shards + rank by rank) back in input order, with the values of a 1-rank run, a repeated id included; values are never + changed. The fake pass fixes each row's embedding by its sequence, as ESM3 does, so two records with one id + differ.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(launch_env(FOLDMATCH_RUN_ID="order")) + + def _pass(self, records, world_size, min_res_n=0): + """A FASTA pass over ``records`` with the fake pass on ``world_size`` ranks; returns the computer and + ``{row: embedding}`` as rank 0 reads it from the shards (the dict a search builds with ``rows=True``).""" + from foldmatch.search.embedding_computer import EmbeddingComputer + fasta = self.tmp / "q.fasta" + fasta.write_text("".join(f">{name} some description\n{seq}\n\n" for name, seq in records)) + computer = EmbeddingComputer(self.tmp) + with patch("foldmatch.search.embedding_computer.chain_predict", + fake_chain_predict(world_size=world_size, by_sequence=True)): + computer.compute_from_fasta(fasta, min_res_n=min_res_n, accelerator=Accelerator.cpu) + results = {} + for rows, embeddings in computer.get_embedding_batches(rows=True): + for row, embedding in zip(rows, embeddings): + results[row] = embedding + return computer, results + + def _finish(self, computer): + computer.remove_shards() + + @staticmethod + def _one_rank(records, min_res_n=0): + """What a 1-rank search's dict holds: ``{id: embedding of its last kept record}``, each id at its first.""" + out = {} + for name, seq in records: + if len(seq) >= min_res_n: + out[name] = _embedding(seq).numpy() + return out + + def assertSameResults(self, got, want): + self.assertEqual(list(got), list(want)) + for name in want: + np.testing.assert_array_equal(got[name], want[name], err_msg=name) + + def test_two_ranks_are_put_back_in_input_order(self): + records = [(f"q{k}", "ACDEFGHIKL" * (k + 1)) for k in range(7)] + names = [name for name, _ in records] + for world_size in (2, 3): + with self.subTest(world_size=world_size): + computer, results = self._pass(records, world_size) + read = [i for ids, _ in computer.get_embedding_batches() for i in ids] + self.assertNotEqual(read, names) # rank order: rank 0's queries first + ordered = computer.in_input_order(results) + self.assertSameResults(ordered, self._one_rank(records)) + values = {id(v) for v in results.values()} + self.assertTrue(all(id(v) in values for v in ordered.values())) # the values, not copies + self.assertEqual(computer.input_ids(), names) + self._finish(computer) + + def test_one_rank_keeps_the_input_order(self): + records = [(f"q{k}", "ACDEFGHIKL" * k) for k in (3, 1, 2)] + computer, results = self._pass(records, 1) + ordered = computer.in_input_order(results) + self.assertEqual(list(ordered), ["q3", "q1", "q2"]) + for row, name in enumerate(["q3", "q1", "q2"]): + self.assertIs(ordered[name], results[row]) + + def test_only_the_kept_records_count(self): + """``min_res_n`` drops records before the pass; the order is that of the kept ones, and a subset of the + results (a filtered dict) keeps that order.""" + records = [("short", "ACD"), ("b", "ACDEFGHIKL"), ("a", "ACDEFGHIKLM"), ("c", "ACDEFGHIKLMN")] + computer, results = self._pass(records, 2, min_res_n=5) + self.assertEqual(computer.input_ids(), ["b", "a", "c"]) + self.assertEqual([i for ids, _ in computer.get_embedding_batches() for i in ids], ["b", "c", "a"]) + self.assertSameResults(computer.in_input_order(results), self._one_rank(records, 5)) + self.assertEqual(list(computer.in_input_order({1: "c", 0: "b"})), ["b", "c"]) + self.assertEqual(computer.input_sequences(), {"b": "ACDEFGHIKL", "a": "ACDEFGHIKLM", "c": "ACDEFGHIKLMN"}) + + def test_a_repeated_id_takes_its_last_kept_record(self): + """Duplicate ids (accepted by a plain pass): one key, at its first kept record, with the embedding of its + last kept record, as in a 1-rank run, whatever the ranks; and the sequence Stage 2 aligns is that record's + too. Two ranks deal x(1), z | y, x(2): rank 0's shard, read first, holds the earlier x.""" + x1, x2 = "ACDEFGHIKL", "KLMNPQRSTVWY" + records = [("x", x1), ("y", "ACDEFGHIKLM"), ("z", "ACDEFGHIKLMN"), ("x", x2)] + for world_size in (1, 2, 3): + with self.subTest(world_size=world_size): + computer, results = self._pass(records, world_size) + self.assertEqual(computer.input_ids(), ["x", "y", "z", "x"]) + ordered = computer.in_input_order(results) + self.assertSameResults(ordered, self._one_rank(records)) + np.testing.assert_array_equal(ordered["x"], _embedding(x2).numpy()) + self.assertEqual(computer.input_sequences()["x"], x2) + self._finish(computer) + # the last record is too short for the pass: the earlier one is the query, for Stage 1 and Stage 2 alike + short = [("x", x1 * 2), ("y", "ACDEFGHIKLM"), ("x", "ACD")] + for world_size in (1, 2): + with self.subTest(world_size=world_size, min_res_n=5): + computer, results = self._pass(short, world_size, min_res_n=5) + ordered = computer.in_input_order(results) + self.assertSameResults(ordered, self._one_rank(short, 5)) + self.assertEqual(computer.input_sequences(), {"x": x1 * 2, "y": "ACDEFGHIKLM"}) + self._finish(computer) + + def test_shards_not_dealt_round_robin(self): + """Shards that are not the sampler's round robin (never from Lightning's predict): an id that occurs once is + still placed exactly; a repeated id takes its positions in read order, with a warning.""" + from foldmatch.search.embedding_computer import EmbeddingComputer + for records, warned in (([("a", "AC"), ("b", "DE"), ("c", "FG"), ("d", "HI")], False), + ([("x", "AC"), ("b", "DE"), ("x", "FG"), ("d", "HI")], True)): + with self.subTest(warned=warned): + fasta = self.tmp / "q.fasta" + fasta.write_text("".join(f">{name}\n{seq}\n" for name, seq in records)) + + def contiguous(**kwargs): + half = [records[:2], records[2:]] + for rank, part in enumerate(half): + _write_shard(kwargs["out_folder"], kwargs["out_name"], rank, [n for n, _ in part], + keys=[seq for _, seq in part]) + + computer = EmbeddingComputer(self.tmp) + with patch("foldmatch.search.embedding_computer.chain_predict", contiguous): + computer.compute_from_fasta(fasta, accelerator=Accelerator.cpu) + results = {row: e for rows, embs in computer.get_embedding_batches(rows=True) + for row, e in zip(rows, embs)} + with self.assertLogs("foldmatch.search.embedding_computer", level="WARNING") if warned else \ + _no_warning(self, "foldmatch.search.embedding_computer"): + ordered = computer.in_input_order(results) + self.assertSameResults(ordered, self._one_rank(records)) + self._finish(computer) + + def test_an_id_not_in_the_input_is_an_error(self): + computer, results = self._pass([("a", "ACDEFGHIKL"), ("b", "ACDEFGHIKLM")], 2) + _write_shard(self.tmp, computer.out_name, 2, ["stale"]) + with self.assertRaises(ValueError) as ctx: + computer.in_input_order(results) + self.assertIn("'stale'", str(ctx.exception)) + self.assertIn("shards of another run", str(ctx.exception)) + (self.tmp / f"{computer.out_name}-2.parquet").unlink() + self.assertEqual(list(computer.in_input_order(results)), ["a", "b"]) + with self.assertRaises(ValueError) as ctx: + computer.in_input_order({99: None}) + self.assertIn("not a row", str(ctx.exception)) + + def test_only_after_a_fasta_pass(self): + from foldmatch.search.embedding_computer import EmbeddingComputer + for method in ("in_input_order", "input_sequences"): + with self.subTest(method=method), self.assertRaises(ValueError): + getattr(EmbeddingComputer(self.tmp), method)(*(({},) if method == "in_input_order" else ())) + + +@contextmanager +def _no_warning(case, logger_name): + """``assertNoLogs`` at WARNING.""" + with case.assertNoLogs(logger_name, level="WARNING"): + yield + + +# A simulated rank > 0: it re-runs the command, so it derives its own run id, +# then writes its shard the way the Lightning writer callback does. +_CHILD = """ +import json +import sys +from types import SimpleNamespace +import torch +from foldmatch.search.embedding_computer import EmbeddingComputer +from foldmatch.writer.batch_writer import ParquetBatchWriter +folder, rank, name = sys.argv[1], int(sys.argv[2]), sys.argv[3] +if len(sys.argv) > 4: + sys.argv = json.loads(sys.argv[4]) # the command line srun gives every task +computer = EmbeddingComputer(folder) +writer = ParquetBatchWriter(folder, computer.out_name) +trainer = SimpleNamespace(global_rank=rank) +writer.write_on_batch_end(trainer, None, ([torch.ones(8)], (name,)), None, None, 0, 0) +writer.on_predict_end(trainer, None) +""" + + +class RunIdSharingTest(unittest.TestCase): + """Simulated ranks in separate processes: rank 0 reads every rank's shard.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + self.fasta = self.tmp / "two.fasta" + self.fasta.write_text(">q0\nACDEFGHIKL\n>q1\nMNPQRSTVWY\n") + + def tearDown(self): + self._tmp.cleanup() + + def _child(self, rank, name, argv=None, **env): + return subprocess.run( + [sys.executable, "-c", _CHILD, str(self.tmp), str(rank), name] + + ([json.dumps(argv)] if argv is not None else []), + env=_child_env(**env), capture_output=True, text=True, + ) + + def _run(self, child_env, parent_env=None, inherit_run_id=True, same_argv=False): + """Rank 0 runs the pass; its predict launches rank 1 with ``child_env`` + (and, with ``same_argv``, rank 0's command line, as srun gives every task).""" + from foldmatch.search.embedding_computer import EmbeddingComputer + children = [] + + def predict(**kwargs): + _write_shard(kwargs["out_folder"], kwargs["out_name"], 0, ["q0"]) + env = dict(child_env) + if inherit_run_id and RUN_ID_ENV in os.environ: + env[RUN_ID_ENV] = os.environ[RUN_ID_ENV] # what a Lightning child inherits + children.append(self._child(1, "q1", argv=list(sys.argv) if same_argv else None, **env)) + + with launch_env(**(parent_env or {})): + computer = EmbeddingComputer(self.tmp) + with patch("foldmatch.search.embedding_computer.chain_predict", predict): + computer.compute_from_fasta(self.fasta, accelerator=Accelerator.cpu) + self.assertNotIn(RUN_ID_ENV, os.environ) # cleared after the pass + return computer, children[0] + + def test_lightning_child_with_local_rank(self): + computer, child = self._run({"LOCAL_RANK": 1}) + self.assertEqual(child.returncode, 0, child.stderr) + self.assertEqual(_ids(computer), ["q0", "q1"]) + + def test_child_with_slurm_procid(self): + computer, child = self._run({"SLURM_PROCID": 1}) + self.assertEqual(child.returncode, 0, child.stderr) + self.assertEqual(_ids(computer), ["q0", "q1"]) + + def test_srun_tasks_share_the_slurm_id(self): + slurm = {"SLURM_JOB_ID": 555, "SLURM_STEP_ID": 2} + with patch.object(sys, "argv", _BUILD_ARGV): + computer, child = self._run(dict(slurm, SLURM_PROCID=1), parent_env=dict(slurm, SLURM_PROCID=0), + inherit_run_id=False, same_argv=True) + self.assertEqual(child.returncode, 0, child.stderr) + self.assertEqual(computer.run_id, f"555_2_0_{_digest(_BUILD_ARGV)}") + self.assertEqual(_ids(computer), ["q0", "q1"]) + + def test_child_without_an_id_refuses(self): + for rank_env in ({"LOCAL_RANK": 1}, {"SLURM_PROCID": 1}): + with self.subTest(rank_env=rank_env): + for path in self.tmp.glob("emb_*"): + path.unlink() + computer, child = self._run(rank_env, inherit_run_id=False) + self.assertNotEqual(child.returncode, 0) + self.assertIn(RUN_ID_ENV, child.stderr) + # Only rank 0's shard exists, and rank 0 says so instead of building from half the data. + with self.assertRaises(ValueError): + _ids(computer) + + +class CliShardLifecycleTest(unittest.TestCase): + """Build shards are kept unless --cleanup-embeddings; query shards are deleted; + a lost rank fails the build before the index is written.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(launch_env()) + + def _build(self, name, predict, **kwargs): + from foldmatch.cli.search import build_database_from_fasta + with patch("foldmatch.search.embedding_computer.chain_predict", predict): + build_database_from_fasta(fasta_file=_FIXTURE_FASTA, output_db=str(self.tmp / name), + tmp_embedding_folder=self.tmp, accelerator=Accelerator.cpu, + **kwargs) + + def _shards(self): + return sorted(p.name for p in self.tmp.glob("emb_*.parquet")) + + def test_two_rank_build_keeps_shards_and_query_deletes_its_own(self): + from foldmatch.cli.search import query_database_from_fasta + from foldmatch.search.faiss_database import FaissEmbeddingDatabase + self._build("db", fake_chain_predict(world_size=2)) + db = FaissEmbeddingDatabase(self.tmp, "db") + db.load_database() + self.assertEqual(sorted(db.chain_ids), ["1acb_E", "2uzi_A"]) + build_shards = self._shards() + self.assertEqual(len(build_shards), 2) + self.assertEqual(len({name.rsplit("-", 1)[0] for name in build_shards}), 1) + + output = self.tmp / "hits.tsv" + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(world_size=2)): + query_database_from_fasta(db_path=str(self.tmp / "db"), fasta_file=_FIXTURE_FASTA, + tmp_embedding_folder=self.tmp, output_file=str(output), + threshold=None, seq_identity=False, accelerator=Accelerator.cpu) + self.assertEqual({line.split("\t")[0] for line in output.read_text().splitlines()}, + {"1acb_E", "2uzi_A"}) + self.assertEqual(self._shards(), build_shards) + + def test_build_then_query_in_one_sbatch_batch_step(self): + """Each command of a batch step gets its own id, so the query leaves the build's shards alone.""" + from foldmatch.cli.search import query_database_from_fasta + with launch_env(SLURM_JOB_ID=4242, SLURM_PROCID=0): + self._build("db", fake_chain_predict(world_size=2)) + build_shards = self._shards() + self.assertEqual(len(build_shards), 2) + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(world_size=1)): + query_database_from_fasta(db_path=str(self.tmp / "db"), fasta_file=_FIXTURE_FASTA, + tmp_embedding_folder=self.tmp, output_file=str(self.tmp / "hits.tsv"), + threshold=None, seq_identity=False, accelerator=Accelerator.cpu) + self._build("db2", fake_chain_predict(world_size=1)) + shards = self._shards() + self.assertEqual(len(shards), 3) + self.assertTrue(set(build_shards) <= set(shards)) + + def test_same_command_twice_in_one_srun_step_is_refused_before_esm3(self): + with launch_env(SLURM_JOB_ID=4242, SLURM_STEP_ID=3, SLURM_PROCID=0), patch.object(sys, "argv", _BUILD_ARGV): + self._build("db", fake_chain_predict(world_size=2)) + build_shards = self._shards() + for suffix in (".index", ".metadata", ".sequences"): + (self.tmp / f"db{suffix}").unlink() + predict = unittest.mock.Mock(side_effect=fake_chain_predict(world_size=1)) + with self.assertRaises(FileExistsError) as ctx: + self._build("db", predict) + predict.assert_not_called() + self.assertIn("same run id", str(ctx.exception)) + self.assertEqual(self._shards(), build_shards) + + def test_cleanup_embeddings(self): + self._build("db", fake_chain_predict(world_size=2), cleanup_embeddings=True) + self.assertTrue((self.tmp / "db.index").exists()) + self.assertEqual(self._shards(), []) + + def test_lost_rank_fails_before_the_index_is_written(self): + with self.assertRaises(ValueError) as ctx: + self._build("db", fake_chain_predict(world_size=2, drop=("2uzi_A",))) + self.assertIn("Found 1 embeddings", str(ctx.exception)) + self.assertFalse((self.tmp / "db.index").exists()) + self.assertFalse((self.tmp / "db.sequences").exists()) + + +@unittest.skipUnless(os.environ.get("FOLDMATCH_SLOW_TESTS") == "1", + "slow: two ESM3 ranks on CPU; set FOLDMATCH_SLOW_TESTS=1 to run") +class TwoRankDDPSmoke(unittest.TestCase): + """A real two-process DDP build (CPU, gloo) keeps both ranks' embeddings.""" + + def test_two_rank_cpu_build_indexes_every_entry(self): + from foldmatch.search.faiss_database import FaissEmbeddingDatabase + from foldmatch.search.sequence_store import SequenceStore + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + result = subprocess.run( + [sys.executable, "-m", "foldmatch.cli.search", "build", "sequences", + "--fasta-file", str(_FIXTURE_FASTA), "--output-db", str(tmp / "db"), + "--tmp-embedding-folder", str(tmp), "--accelerator", "cpu", "--devices", "2"], + env=_child_env(), capture_output=True, text=True, timeout=1800, + ) + self.assertEqual(result.returncode, 0, result.stderr[-4000:]) + db = FaissEmbeddingDatabase(tmp, "db") + db.load_database() + self.assertEqual(sorted(db.chain_ids), ["1acb_E", "2uzi_A"]) + self.assertEqual(SequenceStore(tmp, "db").count(), 2) + shards = sorted(p.name for p in tmp.glob("emb_*.parquet")) + self.assertEqual(len(shards), 2, shards) + self.assertEqual(shards[0].rsplit("-", 1)[0], shards[1].rsplit("-", 1)[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_esm_codes_build_channel.py b/tests/test_esm_codes_build_channel.py new file mode 100644 index 0000000..75d848c --- /dev/null +++ b/tests/test_esm_codes_build_channel.py @@ -0,0 +1,1286 @@ +"""P5.1-P5.4: the residue-code channel of the ESM3 pass (plan §8 P5). + +Most tests need no ESM3: a fake ``per_residue_embedding`` hands the module real ESM3 rows from the fixtures (F2c, the +FoldMatch CPU fp32 rows of ``test_sequences.fasta``; F2b, structure-derived rows used for arithmetic only), so the +module hook, the plumbing, the writer and the readers run with the real esm-codes-v1 encoders on known inputs. The +F2c acceptance test runs real ESM3 on the CPU. + +The real encoders come from the scoring-system registry bundle (``esm_codes_bundle``): skipped only when offline and +not cached. +""" +import dataclasses +import functools +import gc +import hashlib +import json +import os +import subprocess +import sys +import tempfile +import unittest +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import torch + +import esm_codes_bundle +import foldmatch +from foldmatch.model.residue_embedding_aggregator import ResidueEmbeddingAggregator +from foldmatch.modules import chain_complete_module +from foldmatch.modules.chain_complete_module import ChainCompleteModule +from foldmatch.search import scoring_system as ss +from foldmatch.search import residue_shards as rs +from foldmatch.types.api_types import Accelerator, OutFormat, SrcEsmFrom +from foldmatch.utils.data import collate_seq_embeddings +from foldmatch.utils.model import get_aggregator_model, release_mps_cache + +_TEST_PATH = Path(__file__).parent +_FIXTURES = _TEST_PATH / "resources" / "esm_codes" +_EMBEDDINGS = _TEST_PATH / "resources" / "embeddings" +_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_SRC = str(Path(foldmatch.__file__).resolve().parent.parent) +_LAUNCH_ENV = ("FOLDMATCH_RUN_ID", "SLURM_JOB_ID", "SLURM_STEP_ID", "SLURM_RESTART_COUNT", "SLURM_PROCID", + "SLURM_NTASKS", "SLURM_STEP_NUM_TASKS", "RANK", "LOCAL_RANK", "NODE_RANK", "WORLD_SIZE") + +SYSTEM = ss.get(ss.DEFAULT_ESM_CODES) +# Another encoding (a different target encoder): what a stale shard of another system holds. +OTHER_ENCODING = dataclasses.replace(SYSTEM, name="esm-codes-test-other", target_encoder_sha256="0" * 64) +# The same encoding, other statistics: may reuse the residue codes, never the query encodings. +OTHER_STATISTICS = dataclasses.replace(SYSTEM, name="esm-codes-test-stats", calibration_version="test") + +#: A4_THRESHOLDS.json (sha256 7eabf36e...), gate A4-xcpu: CPU Mac vs CPU Linux code bits, also the F2c threshold. +A4_XCPU_BITS = 0.999 +#: A4_THRESHOLDS.json: code bits on every device class. +A4_DEVICE_BITS = 0.990 +_TRAINER = SimpleNamespace(global_rank=0) + + +@contextmanager +def launch_env(**values): + """Exactly ``values`` among the launch variables; the rest unset (only those variables are restored).""" + saved = {name: os.environ.get(name) for name in _LAUNCH_ENV} + try: + for name in _LAUNCH_ENV: + os.environ.pop(name, None) + os.environ.update({k: str(v) for k, v in values.items()}) + yield + finally: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +# ===================================================================================================================== +# fixtures +# ===================================================================================================================== +@functools.lru_cache(maxsize=None) +def _f2c(): + """F2c: ``[(id, sequence, ESM3 rows [L + 2, 1536] fp32)]``, reference codes ``[303, 8]``, q ``[303, 64]``, + offsets. The rows are stored fp16; S2 starts with fp16(x), so as fp32 they encode to the same codes.""" + z = np.load(_FIXTURES / "fasta_rows_codes_v1.npz", allow_pickle=False) + ro = z["rows_offsets"] + items = [(str(z["ids"][k]), str(z["sequences"][k]), torch.from_numpy(z["rows"][ro[k]:ro[k + 1]].astype(np.float32))) + for k in range(len(ro) - 1)] + return items, z["codes"], z["q"], z["offsets"] + + +@functools.lru_cache(maxsize=None) +def _f2b(): + """F2b (arithmetic only): structure-derived ESM3 rows ``[R, 1536]`` fp32 of two chains, standing in for an ESM3 + output with BOS and EOS, so the hook encodes rows ``1..R-2``. The reference encoded every row, and an inner slice + encodes to the same values (FIXTURES.json ``inner_slice``).""" + z = np.load(_FIXTURES / "ood_rows_codes_v1.npz", allow_pickle=False) + names = [str(n) for n in z["ids"]] + rows = [torch.load(_EMBEDDINGS / f"{name}.pt", map_location="cpu", weights_only=True) for name in names] + # Any sequence of the right length: the codes depend on the rows only. + items = [(f"f2b_{name}", "A" * (r.shape[0] - 2), r) for name, r in zip(names, rows)] + return items, z["codes"], z["q"], z["offsets"] + + +@functools.lru_cache(maxsize=None) +def _prepared(): + esm_codes_bundle.registry_files() # unittest.SkipTest only when offline and not cached + return ss.prepare(SYSTEM) + + +@functools.lru_cache(maxsize=None) +def _head(): + return _prepared().load_head() + + +@functools.lru_cache(maxsize=None) +def _aggregator(): + return get_aggregator_model(torch.device("cpu")).eval() + + +def _bits_equal(a, b): + return int((np.unpackbits(a, axis=1) == np.unpackbits(b, axis=1)).sum()), int(a.size * 8) + + +def _row_ulps(q, ref): + """max |q - ref| in fp16 ulps of each reference row's largest |value| (the P2 measure).""" + q32, r32 = q.astype(np.float32), ref.astype(np.float32) + scale = np.spacing(np.abs(ref).max(axis=1).astype(np.float16)).astype(np.float32)[:, None] + return float((np.abs(q32 - r32) / scale).max()) + + +def _bitwise(t: torch.Tensor) -> np.ndarray: + return t.detach().cpu().contiguous().numpy().view(np.uint8) + + +def _sha1(sequence: str) -> bytes: + return hashlib.sha1(sequence.encode("ascii")).digest() + + +def _fake_esm3(items): + """A per_residue_embedding stand-in: the rows of the item whose sequence the protein has.""" + rows = {sequence: r for _, sequence, r in items} + + def per_residue_embedding(model, esm_prot): + return rows[esm_prot.sequence].clone() + return per_residue_embedding + + +def _batch(items): + return [(SimpleNamespace(sequence=sequence, coordinates=None), name) for name, sequence, _ in items] + + +def _legacy_predict_step(module, prot_batch, batch_idx): + # ChainCompleteModule.predict_step before P5.1 (e8b389c), verbatim but for the module-level name lookup, so the + # same fake per_residue_embedding serves both. + prot_embeddings = [] + prot_names = [] + for esm_prot, name in prot_batch: + embeddings = chain_complete_module.per_residue_embedding(module.esm3, esm_prot) + prot_embeddings.append(embeddings) + prot_names.append(name) + res_batch_embedding, res_batch_mask = collate_seq_embeddings(prot_embeddings) + chain_embeddings = module.aggregator(res_batch_embedding, res_batch_mask) + release_mps_cache(module.device) + return chain_embeddings, tuple(prot_names) + + +class _RealHead(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.head = _head() + cls.aggregator = _aggregator() + + +# ===================================================================================================================== +# P5.1 module hook +# ===================================================================================================================== +class ModuleHookTest(_RealHead): + """The hook: aggregator input untouched, payloads from rows 1..L, no raise in predict_step.""" + + def _module(self, residue_output): + return ChainCompleteModule(res_model=torch.nn.Identity(), aggregator_model=self.aggregator, + residue_head=self.head if residue_output else None, residue_output=residue_output) + + def _run(self, residue_output, items, batch_size=2, esm3=None): + """``(outputs per batch, aggregator inputs per batch)``.""" + module = self._module(residue_output) + seen = [] + forward = ResidueEmbeddingAggregator.forward + + def spy(aggregator, x, x_mask=None): + seen.append((x.clone(), None if x_mask is None else x_mask.clone())) + return forward(aggregator, x, x_mask) + + outputs = [] + with mock.patch.object(chain_complete_module, "per_residue_embedding", esm3 or _fake_esm3(items)), \ + mock.patch.object(ResidueEmbeddingAggregator, "forward", spy), torch.inference_mode(): + for start in range(0, len(items), batch_size): + outputs.append(module.predict_step(_batch(items[start:start + batch_size]), start // batch_size)) + return outputs, seen, module + + def test_a_module_without_a_head_returns_the_legacy_2_tuple(self): + items = _f2c()[0] + _f2b()[0] + outputs, _, module = self._run(None, items) + with mock.patch.object(chain_complete_module, "per_residue_embedding", _fake_esm3(items)), \ + torch.inference_mode(): + legacy = [_legacy_predict_step(module, _batch(items[s:s + 2]), s // 2) for s in range(0, len(items), 2)] + for got, want in zip(outputs, legacy): + self.assertEqual(len(got), 2) + self.assertEqual(got[1], want[1]) + np.testing.assert_array_equal(_bitwise(got[0]), _bitwise(want[0])) + + def test_aggregator_input_and_chain_embeddings_are_identical_with_and_without_codes(self): + # Batches of two proteins of different lengths, so the aggregator input is padded and masked. + items = _f2c()[0] + _f2b()[0] + plain, plain_seen, _ = self._run(None, items) + self.assertTrue(plain_seen[0][1].any()) + for residue_output in rs.RESIDUE_OUTPUTS: + with self.subTest(residue_output=residue_output): + outputs, seen, _ = self._run(residue_output, items) + self.assertEqual(len(seen), len(plain_seen)) + for (x, m), (x0, m0) in zip(seen, plain_seen): + np.testing.assert_array_equal(_bitwise(x), _bitwise(x0)) + np.testing.assert_array_equal(m.numpy(), m0.numpy()) + for got, want in zip(outputs, plain): + self.assertEqual(len(got), 4) + self.assertEqual(got[1], want[1]) + np.testing.assert_array_equal(_bitwise(got[0]), _bitwise(want[0])) + + def test_f2c_rows_give_the_reference_codes(self): + items, codes_ref, q_ref, offsets = _f2c() + codes = b"".join(p for out in self._run(rs.RESIDUE_CODES, items)[0] for p in out[2]) + np.testing.assert_array_equal(np.frombuffer(codes, dtype=np.uint8).reshape(-1, 8), codes_ref) + outputs = self._run(rs.QUERY_ENCODING, items)[0] + q = np.frombuffer(b"".join(p for out in outputs for p in out[2]), dtype="a\nACDE\n>a\nACDE\n", "repeat"), (">a\nACDE\n", "'<'"), + (">a\nACÉ\n", "non-ASCII")): + with self.subTest(what=what): + path = Path(tmp) / "bad.fasta" + path.write_text(text, encoding="utf-8") + self.assertIn(what, self._refused(ValueError, src_stream=path, residue_output=rs.RESIDUE_CODES)) + prepare.return_value.load_head.assert_not_called() + + def test_a_plain_pass_is_not_strict(self): + """Without a residue output the pre-flight problems are only warnings, as before (the spy is reached).""" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "dup.fasta" + path.write_text(">a\nACDE\n>a\nACDE\n", encoding="utf-8") + from foldmatch.inference.chain_complete_inference import predict + with _esm3_spy() as esm3, self.assertRaisesRegex(AssertionError, "ESM3 was loaded"): + predict(src_stream=path, src_from=SrcEsmFrom.fasta, accelerator=Accelerator.cpu, devices=1) + esm3.assert_called_once() + + +class BundleFetchRuleTest(unittest.TestCase): + """prepare(system, local_files_only=True) on every rank but the one process that starts the run (P4 outcome).""" + + def test_rule(self): + from foldmatch.inference.chain_complete_inference import bundle_fetch_allowed + cases = [ + ({}, True), + ({"LOCAL_RANK": 1}, False), # a child Lightning started + ({"RANK": 0, "WORLD_SIZE": 1}, True), + ({"RANK": 0, "WORLD_SIZE": 2}, False), # torchrun: all ranks start together + ({"SLURM_JOB_ID": 5, "SLURM_STEP_ID": 0, "SLURM_STEP_NUM_TASKS": 2, "SLURM_PROCID": 0}, False), + ({"SLURM_JOB_ID": 5, "SLURM_STEP_ID": 0, "SLURM_STEP_NUM_TASKS": 1, "SLURM_NTASKS": 4}, True), + ({"SLURM_JOB_ID": 5, "SLURM_NTASKS": 4}, True), # the batch step is one process + ] + for env, allowed in cases: + with self.subTest(env=env), launch_env(**env): + self.assertIs(bundle_fetch_allowed(), allowed) + self.assertFalse(bundle_fetch_allowed(num_nodes=2)) # the other nodes start on their own + + def test_prepare_scoring_system(self): + from foldmatch.inference.chain_complete_inference import prepare_scoring_system + with mock.patch.object(ss, "prepare") as prepare: + with launch_env(): + prepare_scoring_system() + prepare.assert_called_with(SYSTEM, local_files_only=False) + with launch_env(LOCAL_RANK=1): + prepare_scoring_system(SYSTEM.name) + prepare.assert_called_with(SYSTEM, local_files_only=True) + prepare_scoring_system(SYSTEM, local_files_only=True) + prepare.assert_called_with(SYSTEM, local_files_only=True) + prepared = ss.PreparedSystem(system=SYSTEM, files={}, bundle={}, encoders_file="e", scoring_file="s") + with mock.patch.object(ss, "prepare") as prepare: + self.assertIs(prepare_scoring_system(prepared, residue_output=rs.QUERY_ENCODING), prepared) + prepare.assert_not_called() + foreign = ss.PreparedSystem(system=OTHER_STATISTICS, files={}, bundle={}, encoders_file="e", scoring_file="s") + with self.assertRaises(ss.UnknownScoringSystemError): + prepare_scoring_system(foreign) + + +class _FakeEsm3Pass(_RealHead): + """predict() through Lightning on the CPU with ESM3 replaced by the F2c rows of the fixture FASTA.""" + + @contextmanager + def fake_models(self): + with mock.patch("foldmatch.inference.chain_complete_inference.get_residue_model", + return_value=torch.nn.Identity()), \ + mock.patch("foldmatch.inference.chain_complete_inference.get_aggregator_model", + return_value=self.aggregator), \ + mock.patch.object(chain_complete_module, "per_residue_embedding", _fake_esm3(_f2c()[0])): + yield + + def predict(self, **kwargs): + from foldmatch.inference.chain_complete_inference import predict + with self.fake_models(), launch_env(): + return predict(src_stream=_FASTA, src_from=SrcEsmFrom.fasta, accelerator=Accelerator.cpu, devices=1, + **kwargs) + + +class InferencePlumbingTest(_FakeEsm3Pass): + + def test_callers_without_the_keyword_get_the_legacy_module_and_outputs(self): + made = [] + real = ChainCompleteModule + + def construct(*args, **kwargs): + made.append(kwargs) + return real(*args, **kwargs) + + with mock.patch("foldmatch.inference.chain_complete_inference.ChainCompleteModule", side_effect=construct): + predictions = self.predict(batch_size=2) + self.assertEqual(made, [{"res_model": mock.ANY, "aggregator_model": self.aggregator}]) + module = real(torch.nn.Identity(), self.aggregator) + items = _f2c()[0] + with mock.patch.object(chain_complete_module, "per_residue_embedding", _fake_esm3(items)), \ + torch.inference_mode(): + legacy = _legacy_predict_step(module, _batch(items), 0) + (embeddings, names), = predictions + self.assertEqual(names, legacy[1]) + np.testing.assert_array_equal(_bitwise(embeddings), _bitwise(legacy[0])) + + def test_residue_predictions(self): + items, codes_ref, q_ref, offsets = _f2c() + plain = self.predict(batch_size=2) + for residue_output in rs.RESIDUE_OUTPUTS: + with self.subTest(residue_output=residue_output): + kwargs = {"scoring_system": SYSTEM} if residue_output == rs.QUERY_ENCODING else {} + (embeddings, names, payloads, digests), = self.predict(batch_size=2, residue_output=residue_output, + **kwargs) + np.testing.assert_array_equal(_bitwise(embeddings), _bitwise(plain[0][0])) + self.assertEqual(names, plain[0][1]) + self.assertEqual(list(digests), [_sha1(s) for _, s, _ in items]) + data = b"".join(payloads) + if residue_output == rs.RESIDUE_CODES: + np.testing.assert_array_equal(np.frombuffer(data, dtype=np.uint8).reshape(-1, 8), codes_ref) + else: + q = np.frombuffer(data, dtype="= self.buffer_size: + self._flush(trainer.global_rank) + + def _flush(self, rank): + if not self._id_buffer: + return + table = pa.table( + { + 'id': self._id_buffer, + 'embedding': pa.array(self._emb_buffer, type=pa.list_(pa.float32())), + }, + schema=self._schema, + ) + self._get_writer(rank).write_table(table) + self._id_buffer.clear() + self._emb_buffer.clear() + + def on_predict_end(self, trainer, pl_module): + self._flush(trainer.global_rank) + if self._writer is not None: + self._writer.close() + + +def _rows(n, seed=0, residues=(3, 0, 5), dim=4): + """``n`` synthetic rows: ``(id, embedding, codes payload, query payload, seq_sha1)``; the second has an empty + payload (a length mismatch).""" + rng = np.random.default_rng(seed) + out = [] + for k in range(n): + length = residues[k % len(residues)] + emb = torch.from_numpy(rng.normal(size=dim).astype(np.float32)) + codes = rng.integers(0, 256, size=(length, 8), dtype=np.uint8).tobytes() + q = rng.normal(size=(length, 64)).astype("= 0).numpy(), axis=1), hq.to(torch.float16).numpy() + + def test_outputs_match_the_written_out_spec(self): + codes = self.head.encode_target(self.x) + q = self.head.encode_query(self.x) + self.assertEqual((codes.dtype, tuple(codes.shape)), (torch.uint8, (23, 8))) + self.assertEqual((q.dtype, tuple(q.shape)), (torch.float16, (23, 64))) + ref_codes, ref_q = self._reference(self.x) + np.testing.assert_array_equal(codes.numpy(), ref_codes) + np.testing.assert_array_equal(q.numpy().view(np.uint16), ref_q.view(np.uint16)) + + def test_transform_output_is_fp16_exact(self): + xw = self.head.transform(self.x) + self.assertEqual(xw.dtype, torch.float32) + self.assertTrue(torch.equal(xw, xw.to(torch.float16).to(torch.float32))) + + def test_input_is_cast_to_fp16_first(self): + x16 = self.x.to(torch.float16) + self.assertTrue(torch.equal(self.head.encode_target(self.x), self.head.encode_target(x16))) + self.assertTrue(torch.equal(self.head.encode_query(self.x), self.head.encode_query(x16.double()))) + + def test_empty_and_single_residue(self): + self.assertEqual(tuple(self.head.encode_target(torch.zeros(0, 16)).shape), (0, 8)) + self.assertEqual(tuple(self.head.encode_query(torch.zeros(0, 16)).shape), (0, 64)) + self.assertTrue(torch.equal(self.head.encode_target(self.x[4:5]), self.head.encode_target(self.x)[4:5])) + + def test_wrong_width_refused(self): + with self.assertRaises(ValueError): + self.head.encode_target(torch.zeros(4, 15)) + with self.assertRaises(ValueError): + self.head.encode_query(torch.zeros(16)) + + def test_autocast_is_disabled(self): + q = self.head.encode_query(self.x) + with torch.autocast("cpu", dtype=torch.bfloat16): + self.assertTrue(torch.is_autocast_enabled("cpu")) + q_ac = self.head.encode_query(self.x) + c_ac = self.head.encode_target(self.x) + self.assertTrue(torch.is_autocast_enabled("cpu")) # restored after each call + self.assertTrue(torch.equal(q, q_ac)) + self.assertTrue(torch.equal(self.head.encode_target(self.x), c_ac)) + + def test_reduced_precision_weights_refused(self): + self.head.half() + with self.assertRaisesRegex(RuntimeError, "float32"): + self.head.encode_target(self.x) + + def test_sign_rule_zero_is_a_one_bit(self): + """S4: bit_k = (h_k >= 0), so an output of exactly +0.0 or -0.0 is a 1 bit (and the smallest negative + subnormal a 0 bit). Real weights never produce an exact zero on the fixtures, so the rule is pinned here.""" + # (a) through the real code path: f_t outputs 0, 9 and 63 have all-zero weights and biases -> h is exactly 0 + zeroed = (0, 9, 63) + st = self.bundle["f_t"] + for key in ("lin.weight", "lin.bias", "fc2.weight", "fc2.bias"): + st[key][list(zeroed)] = 0.0 + head = head_from_bundle(self.bundle) + with torch.no_grad(): + h = head.f_t(head.transform(self.x)) + self.assertTrue(bool((h[:, list(zeroed)] == 0).all())) + self.assertTrue(bool((h[:, [k for k in range(64) if k not in zeroed]] != 0).all())) + codes = head.encode_target(self.x).numpy() + np.testing.assert_array_equal(codes, np.packbits((h >= 0).numpy(), axis=1)) + self.assertTrue(np.unpackbits(codes, axis=1)[:, list(zeroed)].all(), "h = 0 must give bit 1") + + # (b) signed zeros and subnormals, through encode_target with f_t replaced by fixed outputs + tiny = float(np.nextafter(np.float32(0), np.float32(1))) + row = torch.tensor([0.0, -0.0, tiny, -tiny, 1.0, -1.0, 3.5e38, -3.5e38] * 8, dtype=torch.float32) + fixed = torch.stack([row, -row, row.roll(1), torch.zeros(64), -torch.zeros(64)]) + + class _Fixed(torch.nn.Module): + def forward(self, xw): + return fixed[: xw.shape[0]] + head.f_t = _Fixed() + codes = head.encode_target(self.x[:5]).numpy() + bits = np.unpackbits(codes, axis=1) + np.testing.assert_array_equal(codes, np.packbits((fixed >= 0).numpy(), axis=1)) + np.testing.assert_array_equal(bits[0, :8], [1, 1, 1, 0, 1, 0, 1, 0]) + np.testing.assert_array_equal(bits[1, :8], [1, 1, 0, 1, 0, 1, 0, 1]) + self.assertTrue(bits[3:].all(), "+0.0 and -0.0 rows must be all 1 bits") + + def test_content_digests(self): + d = self.head.content_digests() + self.assertEqual(d["transform_sha256"], tensor_digest(self.bundle["transform"])) + self.assertEqual(d["query_encoder_sha256"], tensor_digest(self.bundle["f_q"])) + self.assertEqual(d["target_encoder_sha256"], tensor_digest(self.bundle["f_t"])) + # numpy and torch containers give the same digest; any change of a value, dtype or name changes it + t = self.bundle["transform"] + self.assertEqual(tensor_digest({k: v.numpy() for k, v in t.items()}), d["transform_sha256"]) + bumped = dict(t, M=t["M"].clone()) + bumped["M"][0, 0] = torch.nextafter(bumped["M"][0, 0], torch.tensor(1e9)) + self.assertNotEqual(tensor_digest(bumped), d["transform_sha256"]) + self.assertNotEqual(tensor_digest(dict(t, M=t["M"].double())), d["transform_sha256"]) + self.assertNotEqual(tensor_digest({"m": t["M"], "dims": t["dims"], "offset": t["offset"]}), + d["transform_sha256"]) + + +_CONTEXT_SCRIPT = textwrap.dedent(""" + import json, sys, warnings + warnings.simplefilter("ignore") + import torch + from foldmatch.model import residue_code_encoder as rce + + def snapshot(): + out = {} + for path in ("", "cudnn", "mkldnn", "cuda.matmul", "cudnn.conv", "cudnn.rnn", "mkldnn.matmul", "mkldnn.conv", + "mkldnn.rnn"): + try: + out["new:" + path] = rce._backend(path).fp32_precision + except (AttributeError, RuntimeError) as e: + out["new:" + path] = "unavailable" if isinstance(e, AttributeError) else "refused" + for name, get in (("matmul_precision", torch.get_float32_matmul_precision), + ("cublas_tf32", torch._C._get_cublas_allow_tf32), + ("cudnn_tf32", torch._C._get_cudnn_allow_tf32), + ("autocast_cpu", lambda: torch.is_autocast_enabled("cpu"))): + try: + out[name] = get() + except RuntimeError: + out[name] = "refused" + return out + + scenario = sys.argv[1] + if scenario == "legacy_path": + rce._fp32_precision_handles = lambda paths: [] # torch < 2.9 code path + torch.backends.cuda.matmul.allow_tf32 = True + torch.backends.cudnn.allow_tf32 = True + elif scenario == "legacy_high": + torch.set_float32_matmul_precision("high") + elif scenario == "legacy_medium": + torch.set_float32_matmul_precision("medium") + elif scenario == "new_tf32": + torch.backends.cuda.matmul.fp32_precision = "tf32" + torch.backends.cudnn.conv.fp32_precision = "tf32" + elif scenario == "new_generic_tf32": + torch.backends.fp32_precision = "tf32" + elif scenario == "mkldnn_bf16": + torch.backends.mkldnn.matmul.fp32_precision = "bf16" + before = snapshot() + ac = torch.autocast("cpu", dtype=torch.bfloat16) if scenario == "autocast" else None + if ac is not None: + ac.__enter__() + before = snapshot() + with rce.strict_fp32("cpu"): + inside = snapshot() + after = snapshot() + try: + with rce.strict_fp32(): + raise KeyError("boom") + except KeyError: + pass + after_error = snapshot() + if ac is not None: + ac.__exit__(None, None, None) + print(json.dumps(dict(before=before, inside=inside, after=after, after_error=after_error))) +""") + + +class Fp32ContextTest(unittest.TestCase): + """strict_fp32: inside, full fp32 matmuls and no autocast; on exit (also after an exception) every flag is back, + whichever TF32 API the caller used. Each scenario runs in its own interpreter: the flags are process-global.""" + + SCENARIOS = ("default", "legacy_high", "legacy_medium", "new_tf32", "new_generic_tf32", "mkldnn_bf16", "autocast", + "legacy_path") + + def _run(self, scenario): + env = dict(os.environ, PYTHONPATH=os.pathsep.join([str(_SRC), os.environ.get("PYTHONPATH", "")])) + r = subprocess.run([sys.executable, "-c", _CONTEXT_SCRIPT, scenario], capture_output=True, text=True, + env=env, timeout=300) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + return json.loads(r.stdout.strip().splitlines()[-1]) + + def test_flags_inside_and_restored(self): + for scenario in self.SCENARIOS: + with self.subTest(scenario=scenario): + s = self._run(scenario) + if scenario == "legacy_path": + # torch < 2.9 has only the legacy flags; on this torch the simulation leaves the per-backend + # settings unmanaged, so only the legacy flags are compared + s = {k: {f: v for f, v in snap.items() if not f.startswith("new:")} for k, snap in s.items()} + self.assertEqual(s["after"], s["before"]) + self.assertEqual(s["after_error"], s["before"]) + inside = s["inside"] + self.assertIs(inside["autocast_cpu"], False) + self.assertIs(inside["cublas_tf32"], False) + self.assertEqual(inside["matmul_precision"], "highest") + if scenario == "legacy_path": + self.assertIs(inside["cudnn_tf32"], False) + else: + for path in ("cuda.matmul", "cudnn.conv", "cudnn.rnn", "mkldnn.matmul", "mkldnn.conv", + "mkldnn.rnn"): + if inside["new:" + path] != "unavailable": + self.assertEqual(inside["new:" + path], "ieee", path) + + +# ===================================================================================================================== +# Tests with the real esm-codes-v1 bundle +# ===================================================================================================================== +class _RealBundle(unittest.TestCase): + @classmethod + def setUpClass(cls): + d = esm_codes_bundle.bundle_dir() # unittest.SkipTest only when offline and not cached + cls.dir = d + for f in (ENCODERS_FILE, SCORING_FILE, "bundle.json"): + if not (d / f).is_file(): + raise AssertionError(f"the esm-codes bundle directory {d} has no {f}") + cls.bundle_json = json.loads((d / "bundle.json").read_text()) + cls.head = load_esm_codes_encoders(d / ENCODERS_FILE, + sha256=cls.bundle_json["files"][ENCODERS_FILE]["sha256"]) + + +class BundleFilesTest(_RealBundle): + """The published files themselves: weights_only, sha256s, content digests, torch-free scoring npz.""" + + def test_encoders_file_is_weights_only(self): + raw = torch.load(self.dir / ENCODERS_FILE, map_location="cpu", weights_only=True) + self.assertEqual(raw["format"], ENCODERS_FORMAT) + self.assertEqual(raw["system"], self.bundle_json["system"]) + self.assertEqual(raw["architecture"], ARCHITECTURES["esm-codes-mlp-v1"]) + self.assertEqual(self.head.meta["architecture"]["name"], "esm-codes-mlp-v1") + self.assertEqual(sum(p.numel() for p in self.head.parameters()), 3_471_360) + + def test_sha256s_match_bundle_json(self): + for f in (ENCODERS_FILE, SCORING_FILE): + self.assertEqual(_sha256(self.dir / f), self.bundle_json["files"][f]["sha256"], f) + with self.assertRaisesRegex(EsmCodesBundleError, "sha256"): + load_esm_codes_encoders(self.dir / ENCODERS_FILE, sha256="0" * 64) + + def test_content_digests_match_bundle_json(self): + want = self.bundle_json["content_digests"] + self.assertEqual(want["method"], rce.DIGEST_METHOD) + for k, v in self.head.content_digests().items(): + self.assertEqual(v, want[k], k) + + def test_scoring_npz_loads_without_torch(self): + code = textwrap.dedent(f""" + import json, sys + sys.modules["torch"] = None + import numpy as np + with np.load({str(self.dir / SCORING_FILE)!r}, allow_pickle=False) as z: + a = {{k: z[k] for k in z.files}} + meta = json.loads(str(a["meta_json"])) + print(json.dumps(dict(files=sorted(a), system=meta["system"], codes=[list(a["panel_codes"].shape), + str(a["panel_codes"].dtype)], members=len(a["panel_offsets"]) - 1, + residues=int(a["panel_offsets"][-1]), lam=a["lam"].tolist(), dtype_lam=str(a["lam"].dtype), + torch_loaded=sys.modules.get("torch") is not None))) + """) + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + out = json.loads(r.stdout) + self.assertEqual(out["files"], ["bin_centres", "lam", "lnK", "meta_json", "panel_codes", "panel_ids", + "panel_lengths", "panel_offsets"]) + self.assertEqual(out["system"], "esm-codes-v1") + self.assertEqual(out["codes"], [[355626, 8], "uint8"]) + self.assertEqual((out["members"], out["residues"]), (1000, 355626)) + self.assertEqual((len(out["lam"]), out["dtype_lam"]), (6, "float64")) + self.assertEqual(out["lam"], self.bundle_json["statistics"]["lam"]) + self.assertFalse(out["torch_loaded"]) + + +def _f2_items(): + z = np.load(_FIXTURES / "encoder_rows_v1.npz", allow_pickle=False) + o = z["offsets"] + return [torch.from_numpy(z["rows"][o[k]:o[k + 1]].copy()) for k in range(len(o) - 1)], z["codes"], z["q"] + + +def _f2c_items(): + z = np.load(_FIXTURES / "fasta_rows_codes_v1.npz", allow_pickle=False) + ro = z["rows_offsets"] + rows = [torch.from_numpy(z["rows"][ro[k] + 1:ro[k + 1] - 1].copy()) for k in range(len(ro) - 1)] # S1: no BOS/EOS + return rows, z["codes"], z["q"] + + +def _f2b_items(): + z = np.load(_FIXTURES / "ood_rows_codes_v1.npz", allow_pickle=False) + rows = [torch.load(_EMBEDDINGS / f"{name}.pt", map_location="cpu", weights_only=True) + for name in z["ids"].astype(str)] + return rows, z["codes"], z["q"] + + +def _encode(head, items, device=None): + if device is not None: + head = head.to(device) + codes = np.concatenate([head.encode_target(x).cpu().numpy() for x in items]) + q = np.concatenate([head.encode_query(x).cpu().numpy() for x in items]) + return codes, q + + +def _bits_equal(a, b): + return int((np.unpackbits(a, axis=1) == np.unpackbits(b, axis=1)).sum()), int(a.size * 8) + + +class EncoderParityTest(_RealBundle): + """Acceptance (plan §6): F2 bits >= 0.99999 and q within 1 fp16 ulp; F2b and F2c codes identical to the reference + encoder on the same rows. + + "Within 1 fp16 ulp" is measured against each row's largest |q| (``_row_ulps``), on every platform: the fixtures + were made on an Apple silicon CPU, and other CPUs round the fp32 GEMMs differently. On Linux x86-64 (OpenBLAS, + torch 2.9 and 2.11) F2 has 19,631 of 20,416 q values bit-equal, with per-element distances up to 927 ulps on + near-zero values but at most 1 ulp of the row maximum; the codes are identical.""" + + def test_f2(self): + items, codes_ref, q_ref = _f2_items() + codes, q = _encode(self.head, items) + equal, n = _bits_equal(codes, codes_ref) + self.assertEqual(n, 20_416) + self.assertGreaterEqual(equal / n, 0.99999, f"F2 code bits {equal}/{n}") + self.assertLessEqual(_row_ulps(q, q_ref), 1.0, "F2 q beyond 1 fp16 ulp of the row maximum") + + def test_f2c(self): + items, codes_ref, q_ref = _f2c_items() + codes, q = _encode(self.head, items) + np.testing.assert_array_equal(codes, codes_ref) + self.assertLessEqual(_row_ulps(q, q_ref), 1.0) + # the same rows as fp32 (what ESM3 hands over) give the same encodings: S2 starts with fp16(x) + codes32, q32 = _encode(self.head, [x.float() for x in items]) + np.testing.assert_array_equal(codes32, codes) + np.testing.assert_array_equal(q32.view(np.uint16), q.view(np.uint16)) + + def test_f2b(self): + items, codes_ref, q_ref = _f2b_items() + self.assertEqual([x.dtype for x in items], [torch.float32, torch.float32]) + codes, q = _encode(self.head, items) + np.testing.assert_array_equal(codes, codes_ref) + self.assertLessEqual(_row_ulps(q, q_ref), 1.0) + + @unittest.skipUnless(torch.backends.mps.is_available() or torch.cuda.is_available(), "no GPU (MPS or CUDA)") + def test_gpu_matches_cpu(self): + """D18: the encoders on the GPU (on-device packing, strict fp32, autocast off) agree with the CPU at least as + well as the pre-registered Mac-vs-Linux CPU gate (A4-xcpu, 0.999).""" + device = "cuda" if torch.cuda.is_available() else "mps" + items, codes_ref, q_ref = _f2c_items() + head = load_esm_codes_encoders(self.dir / ENCODERS_FILE, device=device) + codes, q = _encode(head, items) + equal, n = _bits_equal(codes, codes_ref) + self.assertGreaterEqual(equal / n, 0.999, f"{device} code bits {equal}/{n}") + self.assertLessEqual(_row_ulps(q, q_ref), 4.0) + + +class NotRecoverableFromWeightsTest(_RealBundle): + """The weights alone do not define the codes: GELU-tanh, dropping an fp16 cast or folding the whitener into the + first layer each changes the reference codes and the query encodings. + + Each variant is compared with the module's own output on the same machine (the arithmetic differs only by the + variant), and its codes with the fixture codes (which the module reproduces on every platform). Comparing variant + q with the fixture q instead would mix in the platform's own last-bit differences (785 of 20,416 F2 values on + Linux x86-64).""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.items, cls.codes_ref, cls.q_ref = _f2_items() + cls.codes, cls.q = _encode(cls.head, cls.items) + + def _variant(self, xw_fn, approximate="none", items=None): + h = self.head + + def f(m, xw): + return m.lin(xw) + m.fc2(F.gelu(m.fc1(xw), approximate=approximate)) + codes, q = [], [] + with strict_fp32("cpu"), torch.no_grad(): + for x in (items if items is not None else self.items): + xw = xw_fn(x) + codes.append(packbits_rows(f(h.f_t, xw) >= 0).numpy()) + q.append(f(h.f_q, xw).to(torch.float16).numpy()) + return np.concatenate(codes), np.concatenate(q) + + def _assert_changes(self, codes, q, codes_ref, q_module): + self.assertFalse(np.array_equal(codes, codes_ref), "codes unchanged") + changed = int((q.view(np.uint16) != q_module.view(np.uint16)).sum()) + self.assertGreater(changed, q.size // 100, f"only {changed} of {q.size} q values changed") + + def test_module_reproduces_the_reference(self): + np.testing.assert_array_equal(self.codes, self.codes_ref) + + def test_variant_harness_reproduces_the_module(self): + """Control: the harness with the module's own arithmetic gives the module's codes and q bit for bit, so every + difference below comes from the variant.""" + codes, q = self._variant(self.head.transform) + np.testing.assert_array_equal(codes, self.codes) + np.testing.assert_array_equal(q.view(np.uint16), self.q.view(np.uint16)) + + def test_gelu_tanh(self): + codes, q = self._variant(self.head.transform, approximate="tanh") + self._assert_changes(codes, q, self.codes_ref, self.q) + + def test_without_the_xw_fp16_cast(self): + t = self.head.transform + + def xw(x): + return (x.to(torch.float16).index_select(1, t.dims).float() - t.offset) @ t.M + codes, q = self._variant(xw) + self._assert_changes(codes, q, self.codes_ref, self.q) + + def test_whitener_folded_into_the_first_layer(self): + t, h = self.head.transform, self.head + + def run(m, xs): + return (xs @ (t.M @ m.lin.weight.T) + m.lin.bias) + m.fc2(F.gelu(xs @ (t.M @ m.fc1.weight.T) + m.fc1.bias)) + codes, q = [], [] + with strict_fp32("cpu"), torch.no_grad(): + for x in self.items: + xs = x.to(torch.float16).index_select(1, t.dims).float() - t.offset + codes.append(packbits_rows(run(h.f_t, xs) >= 0).numpy()) + q.append(run(h.f_q, xs).to(torch.float16).numpy()) + self._assert_changes(np.concatenate(codes), np.concatenate(q), self.codes_ref, self.q) + + def test_without_the_input_fp16_cast(self): + items, codes_ref, _q_ref = _f2b_items() # fp32 rows: the input cast only matters for them + codes_module, q_module = _encode(self.head, items) + np.testing.assert_array_equal(codes_module, codes_ref) + t = self.head.transform + + def xw(x): + return ((x.float().index_select(1, t.dims) - t.offset) @ t.M).half().float() + codes, q = self._variant(xw, items=items) + self._assert_changes(codes, q, codes_ref, q_module) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_esm_geom_attn.py b/tests/test_esm_geom_attn.py new file mode 100644 index 0000000..6fe9a64 --- /dev/null +++ b/tests/test_esm_geom_attn.py @@ -0,0 +1,201 @@ +import os +import threading +import unittest +from types import SimpleNamespace +from unittest import mock + +import torch +from esm.sdk.api import ESMProtein, SamplingConfig + +from foldmatch import FoldMatch +from foldmatch.dataset.esm_prot_from_fasta import parse_fasta +from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure +from foldmatch.modules.chain_complete_module import ChainCompleteModule +from foldmatch.modules.esm_module import EsmModule +from foldmatch.types.api_types import SrcLocation +from foldmatch.utils.data import collate_seq_embeddings +from foldmatch.utils.esm import loaders +from foldmatch.utils.model import get_aggregator_model, get_residue_model + + +class GeomAttnOffTest(unittest.TestCase): + """G13: sequence-only proteins run ESM3 with geometric attention off; embeddings are bit-identical.""" + + __test_path = os.path.dirname(__file__) + + @classmethod + def setUpClass(cls): + cls.model = get_residue_model(torch.device("cpu")) + cls.proteins = [ + (name, ESMProtein(sequence=sequence)) + for name, sequence in parse_fasta(f"{cls.__test_path}/resources/fasta/test_sequences.fasta") + ] + + def _geom_blocks(self): + return [i for i, block in enumerate(self.model.transformer.blocks) if block.use_geom_attn] + + def _geom_attn_spy(self): + geom_attn = self.model.transformer.blocks[0].geom_attn + return mock.patch.object(geom_attn, "forward", wraps=geom_attn.forward) + + def _with_geom_attn(self, esm_prot): + # The pre-G13 call: geometric attention left on. + return self.model.forward_and_sample( + self.model.encode(esm_prot), SamplingConfig(return_per_residue_embeddings=True) + ).per_residue_embedding + + def test_only_block_0_has_geometric_attention(self): + self.assertEqual(self._geom_blocks(), [0]) + + def test_sequence_only_embeddings_are_bit_identical(self): + for name, esm_prot in self.proteins: + with torch.inference_mode(): + with self._geom_attn_spy() as on_spy: + on = self._with_geom_attn(esm_prot) + self.assertEqual(on_spy.call_count, 1) + with self._geom_attn_spy() as off_spy: + off = loaders.per_residue_embedding(self.model, esm_prot) + self.assertEqual(off_spy.call_count, 0) + self.assertEqual(tuple(off.shape), (len(esm_prot) + 2, 1536)) + self.assertTrue(torch.equal(on, off), name) + self.assertEqual(self._geom_blocks(), [0]) + + def test_structure_input_keeps_geometric_attention(self): + esm_prot, _name = next(iter(EsmProtFromStructure( + src_stream=[("2uzi", f"{self.__test_path}/resources/pdb/2uzi.cif", "2uzi")], + src_location=SrcLocation.stream + ))) + self.assertIsNotNone(esm_prot.coordinates) + with torch.inference_mode(), self._geom_attn_spy() as spy: + expected = self._with_geom_attn(esm_prot) + embedding = loaders.per_residue_embedding(self.model, esm_prot) + self.assertEqual(spy.call_count, 2) + self.assertTrue(torch.equal(expected, embedding)) + self.assertEqual(self._geom_blocks(), [0]) + + def test_flag_restored_when_esm3_raises(self): + with mock.patch.object(self.model, "forward_and_sample", side_effect=RuntimeError("out of memory")): + with self.assertRaisesRegex(RuntimeError, "out of memory"): + loaders.per_residue_embedding(self.model, self.proteins[0][1]) + self.assertEqual(self._geom_blocks(), [0]) + + def test_modules_run_sequence_only_proteins_without_geometric_attention(self): + batch = [(esm_prot, name) for name, esm_prot in self.proteins] + aggregator = get_aggregator_model(torch.device("cpu")).eval() + chain_module = ChainCompleteModule(self.model, aggregator) + with torch.inference_mode(): + with self._geom_attn_spy() as spy: + chain_embeddings, chain_names = chain_module.predict_step(batch, 0) + residue_embeddings, names = EsmModule(self.model).predict_step(batch, 0) + self.assertEqual(spy.call_count, 0) + expected = [self._with_geom_attn(esm_prot) for _name, esm_prot in self.proteins] + expected_chain = aggregator(*collate_seq_embeddings(expected)) + self.assertEqual(names, tuple(name for name, _ in self.proteins)) + self.assertEqual(chain_names, names) + for embedding, expected_embedding in zip(residue_embeddings, expected): + self.assertTrue(torch.equal(expected_embedding, embedding)) + self.assertTrue(torch.equal(expected_chain, chain_embeddings)) + self.assertEqual(self._geom_blocks(), [0]) + + def test_foldmatch_sequence_embedding_runs_without_geometric_attention(self): + # FoldMatch.sequence_embedding, the Python API's sequence-only path, uses the same call. + with mock.patch("foldmatch.foldmatch._load_res_model", return_value=self.model): + api = FoldMatch() + for name, esm_prot in self.proteins: + with torch.inference_mode(): + with self._geom_attn_spy() as spy: + embedding = api.sequence_embedding(esm_prot.sequence) + self.assertEqual(spy.call_count, 0) + expected = self._with_geom_attn(esm_prot) + self.assertTrue(torch.equal(expected, embedding), name) + self.assertEqual(self._geom_blocks(), [0]) + + +class _FakeEsm3: + """Stands in for ESM3 without weights: records the geometric-attention flag each forward sees. + + A forward of a protein named in ``hold`` signals ``inside`` and waits for ``release``. + """ + + def __init__(self, hold=()): + self.block = SimpleNamespace(use_geom_attn=True) + self.transformer = SimpleNamespace(blocks=[self.block]) + self.hold = hold + self.inside, self.release = threading.Event(), threading.Event() + self.seen = {} + + def encode(self, protein): + return protein + + def forward_and_sample(self, protein, config): + self.seen[protein.name] = self.block.use_geom_attn + if protein.name in self.hold: + self.inside.set() + self.release.wait(10) + return SimpleNamespace(per_residue_embedding=torch.zeros(3, 4)) + + +class GeomAttnThreadSafetyTest(unittest.TestCase): + """G13 switches a flag on the shared model: no other forward on it may run meanwhile. No weights.""" + + SEQUENCE = SimpleNamespace(name="sequence", coordinates=None) + STRUCTURE = SimpleNamespace(name="structure", coordinates=object()) + + def _in_thread(self, model, protein): + thread = threading.Thread(target=loaders.per_residue_embedding, args=(model, protein)) + thread.start() + return thread + + def test_structure_forward_waits_for_a_sequence_forward_on_the_same_model(self): + model = _FakeEsm3(hold=("sequence",)) + sequence = self._in_thread(model, self.SEQUENCE) + self.assertTrue(model.inside.wait(10)) + structure = self._in_thread(model, self.STRUCTURE) + structure.join(1.0) + # Still waiting for the model while the sequence forward runs with the flag off. + self.assertTrue(structure.is_alive()) + self.assertNotIn("structure", model.seen) + model.release.set() + sequence.join(10) + structure.join(10) + self.assertEqual(model.seen, {"sequence": False, "structure": True}) + self.assertTrue(model.block.use_geom_attn) + + def test_other_models_do_not_wait(self): + held, other = _FakeEsm3(hold=("sequence",)), _FakeEsm3() + sequence = self._in_thread(held, self.SEQUENCE) + self.assertTrue(held.inside.wait(10)) + try: + loaders.per_residue_embedding(other, self.STRUCTURE) + loaders.per_residue_embedding(other, self.SEQUENCE) + self.assertEqual(other.seen, {"structure": True, "sequence": False}) + finally: + held.release.set() + sequence.join(10) + + def test_foldmatch_structure_methods_take_the_same_path(self): + """FoldMatch's structure methods go through per_residue_embedding, so they take the lock too.""" + test_path = os.path.dirname(__file__) + cif = f"{test_path}/resources/pdb/2uzi.cif" + fake = _FakeEsm3() + fake.forward_and_sample = lambda protein, config: SimpleNamespace( + per_residue_embedding=torch.zeros(len(protein) + 2, 4)) + spy = mock.Mock(wraps=loaders.per_residue_embedding) + with mock.patch("foldmatch.foldmatch._load_res_model", return_value=fake), \ + mock.patch("foldmatch.foldmatch._load_model", return_value=lambda x: x), \ + mock.patch("foldmatch.foldmatch.per_residue_embedding", spy): + api = FoldMatch() + calls = {} + for method in (api.residue_embedding, api.residue_embedding_by_chain, + api.residue_embedding_by_assembly): + spy.reset_mock() + method(cif) + calls[method.__name__] = spy.call_count + for call in spy.call_args_list: + self.assertIs(call.args[0], fake) + self.assertIsNotNone(call.args[1].coordinates) + self.assertTrue(all(count > 0 for count in calls.values()), calls) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_faiss_database.py b/tests/test_faiss_database.py new file mode 100644 index 0000000..a3c7a82 --- /dev/null +++ b/tests/test_faiss_database.py @@ -0,0 +1,88 @@ +"""FaissEmbeddingDatabase search of an ivf_pq database: FAISS 1.14.2 to 1.15.1 lose coarse centroids in a batch.""" +import pickle +import sys +import tempfile +import unittest +from pathlib import Path + +# Lightning first, as set_log_level: torch and FAISS each bring a libomp on macOS; scikit-learn sets KMP_DUPLICATE_LIB_OK. +import lightning # noqa: F401 +import faiss +import numpy as np + +from foldmatch.search.faiss_database import FaissEmbeddingDatabase +from foldmatch.types.api_types import IndexConfig, IndexType + + +def _write_one_vector_per_cell_database(folder: Path, name: str, nlist: int) -> np.ndarray: + """An IVF database with one vector per cell, saved as FoldMatch saves an ivf_pq one, as a query's top hit is the id of + its nearest cell only if the coarse search looks at that cell. Training a real OPQ + PQ on 39 * nlist vectors is not + needed here, and the defect is in the coarse stage. Returns the vectors.""" + d = 8 + vectors = np.random.default_rng(0).standard_normal((nlist, d)).astype(np.float32) + faiss.normalize_L2(vectors) + quantizer = faiss.IndexFlatIP(d) + quantizer.add(vectors) + index = faiss.IndexIVFFlat(quantizer, d, nlist, faiss.METRIC_INNER_PRODUCT) + index.is_trained = True + index.add(vectors) + faiss.write_index(index, str(folder / f"{name}.index")) + with open(folder / f"{name}.metadata", "wb") as f: + pickle.dump({"embedding_ids": [f"chain{i}" for i in range(nlist)], "dimension": d, + "index_type": IndexType.ivf_pq, "index_config": IndexConfig(nlist=nlist, nprobe=4)}, f) + return vectors + + +@unittest.skipIf(sys.platform == "darwin", "foldmatch pins OMP_NUM_THREADS=1 on macOS: FAISS deadlocks with more threads") +class IvfBatchSearchTest(unittest.TestCase): + """Every query of a batch finds its own cell, whatever the batch size and the thread count.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.threads = faiss.omp_get_max_threads() + self.addCleanup(faiss.omp_set_num_threads, self.threads) + self.nlist = 16384 + self.vectors = _write_one_vector_per_cell_database(Path(self.tmp.name), "db", self.nlist) + self.db = FaissEmbeddingDatabase(Path(self.tmp.name), "db") + self.db.load_database() + + def test_batches_find_the_nearest_cell_at_every_thread_count(self): + # The cells at the end of the table are the ones the defect never looks at. + for threads in (1, 2, 3, 4, 8, 16, 17, 32): + faiss.omp_set_num_threads(threads) + for nq in (1, 2, 3, threads - 1, threads, threads + 1, threads * threads - 1, threads * threads): + if nq < 1: + continue + with self.subTest(threads=threads, nq=nq): + cells = [self.nlist - 1 - j for j in range(nq)] + results = self.db.search_batch(self.vectors[cells], top_k=1) + self.assertEqual([ids[0] for ids, _ in results], [f"chain{c}" for c in cells]) + + def test_batch_equals_one_query_at_a_time(self): + faiss.omp_set_num_threads(8) + cells = list(range(self.nlist - 40, self.nlist)) + batch = self.db.search_batch(self.vectors[cells], top_k=4) + single = [self.db.search_batch(self.vectors[c], top_k=4)[0] for c in cells] + self.assertEqual([ids for ids, _ in batch], [ids for ids, _ in single]) + + +class IvfParallelModeTest(unittest.TestCase): + """parallel_mode is not stored in the index file: the database sets it wherever an ivf_pq index enters.""" + + def test_load_and_build_set_parallel_mode_3(self): + with tempfile.TemporaryDirectory() as tmp: + folder = Path(tmp) + d = 16 + vectors = np.random.default_rng(1).standard_normal((1024, d)).astype(np.float32) + built = FaissEmbeddingDatabase(folder, "built") + built.create_database(iter([([f"c{i}" for i in range(len(vectors))], vectors)]), index_type=IndexType.ivf_pq, + index_config=IndexConfig(nlist=8, m=4, nbits=4, nprobe=2, opq_train_size=1024)) + self.assertEqual(faiss.extract_index_ivf(built.index).parallel_mode, 3) + loaded = FaissEmbeddingDatabase(folder, "built") + loaded.load_database() + self.assertEqual(faiss.extract_index_ivf(loaded.index).parallel_mode, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inference.py b/tests/test_inference.py index b0865b7..d2a560b 100644 --- a/tests/test_inference.py +++ b/tests/test_inference.py @@ -1,6 +1,11 @@ import os +import tempfile import unittest from pathlib import Path +from types import SimpleNamespace + +import pyarrow.parquet as pq +import torch from foldmatch.types.api_types import Accelerator, SrcProteinFrom, SrcLocation, \ StructureFormat, SrcAssemblyFrom, SrcTensorFrom @@ -71,6 +76,48 @@ def test_chain_inference_from_tensor_files(self): self.assertEqual(tuple(chain_embeddings[3][0][0].shape), (1536,)) self.assertEqual(tuple(chain_embeddings[4][0][0].shape), (1536,)) + def test_chain_inference_from_parquet(self): + from foldmatch.inference.chain_inference import predict + from foldmatch.writer.batch_writer import ParquetBatchWriter + tensor_files = [ + (f"{self.__test_path}/resources/embeddings/{name}.pt", name) + for name in ("1acb.A", "1acb.B", "2uzi.A", "2uzi.B", "2uzi.C") + ] + with tempfile.TemporaryDirectory() as tmp: + # The same residue embeddings as the residue commands write them to Parquet: two chains per row group. + writer = ParquetBatchWriter(tmp, "inference", residue_embeddings=True, buffer_size=2) + trainer = SimpleNamespace(global_rank=0) + for file, name in tensor_files: + residues = torch.load(file, map_location="cpu", weights_only=True) + writer.write_on_batch_end(trainer, None, ([residues], (name,)), None, None, 0, 0) + writer.on_predict_end(trainer, None) + self.assertEqual(pq.ParquetFile(f"{tmp}/inference-0.parquet").metadata.num_row_groups, 3) + parquet_embeddings = predict( + src_stream=f"{tmp}/inference-0.parquet", + src_from=SrcTensorFrom.parquet, + accelerator=Accelerator.cpu + ) + chain_embeddings = predict( + src_stream=tensor_files, + src_location=SrcLocation.stream, + accelerator=Accelerator.cpu + ) + + self.assertEqual(len(parquet_embeddings), 5) + self.assertEqual([ids for _, ids in parquet_embeddings], [ids for _, ids in chain_embeddings]) + for (parquet_embedding, ids), (chain_embedding, _) in zip(parquet_embeddings, chain_embeddings): + self.assertEqual(tuple(parquet_embedding[0].shape), (1536,)) + self.assertTrue(torch.equal(parquet_embedding, chain_embedding), ids) + + def test_chain_inference_takes_no_embedding_dim(self): + """embedding_dim, predict's 18th parameter, is gone: a call that passes it fails before anything runs, by name + or by position (return_predictions, which took its place, is keyword-only, so D cannot land in it).""" + from foldmatch.inference.chain_inference import predict + with self.assertRaisesRegex(TypeError, "unexpected keyword argument 'embedding_dim'"): + predict(src_stream=[], src_from=SrcTensorFrom.parquet, embedding_dim=1536) + with self.assertRaisesRegex(TypeError, "takes from 1 to 17 positional arguments but 18 were given"): + predict(*([None] * 17), 1536) + def test_chain_inference_from_structure(self): from foldmatch.inference.chain_inference import predict chain_embeddings = predict( diff --git a/tests/test_inference_follow_ups.py b/tests/test_inference_follow_ups.py new file mode 100644 index 0000000..68184f2 --- /dev/null +++ b/tests/test_inference_follow_ups.py @@ -0,0 +1,815 @@ +"""P9.A4: the four follow-ups of P9.A3. + +F1. FoldMatch tests max_res against the protein chains a call is about to embed, counted as min_res counts them +(polymer amino-acid residues, summed over the pieces kept), all of them together: the chain given by chain_id (in an +assembly, each of its copies), else every protein chain of the structure or assembly. It counted every amino acid of the +structure, or of the atoms with the chain ID, free amino acids and chains under min_res included. +F2. inference residue-embedding, structure-embedding and chain-embedding no longer take --min-res-n, which selected +nothing there; complete-embedding passes it to its assembly step only; the library refuses a min_res_n it would ignore. +F3. One reader for the chain lists of inference and of EsmProtFromChain, whose embedding name defaults to +.; chain-embedding reads and writes by the embedding name; complete-embedding refuses other +names before any model is built; and a row of more than 4 fields is refused, by every command and before any model. + +The fixtures are archive 1acb (chains A, 241 residues, and B, 63) and 1acb with author fields (E and I), edited and +written as PDB text by biotite (with REMARK 350 for an assembly), as in test_chain_pieces.py. ESM3 and the aggregator +are never loaded: FoldMatch embeds through a stub, and the inference commands run with stub models.""" +import io +import logging +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import biotite.structure as struc +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +import torch +from biotite.structure.io.pdb import PDBFile +from biotite.structure.io.pdbx import CIFFile, get_structure +from esm.sdk.api import ESMProtein +from esm.utils.structure.protein_chain import ProteinChain +from typer.testing import CliRunner + +from foldmatch import FoldMatch +from foldmatch.cli import inference +from foldmatch.dataset.esm_prot_from_chain import EsmProtFromChain +from foldmatch.errors import RefusalError +# Imported before any test patches chain_inference.predict, which assembly_inferece binds at its import +from foldmatch.inference import assembly_inferece, chain_inference, esm_inference, structure_inference +from foldmatch.search.embedding_computer import compute_from_structure_file +from foldmatch.types.api_types import Accelerator, Granularity, SrcLocation, SrcProteinFrom, SrcTensorFrom, \ + StructureFormat +from foldmatch.utils.structure_parser import filter_residues, get_protein_chains, get_structure_from_src, \ + protein_chain_iter, rename_atom_attr +from click_messages import no_such_option + +_RESOURCES = Path(__file__).parent / "resources" +_CIF = _RESOURCES / "pdb" / "1acb.cif" +_PDB, _MMCIF = StructureFormat.pdb, StructureFormat.mmcif +_CPU = ("--accelerator", "cpu", "--devices", "1") + +_ATOMS = get_structure(CIFFile.read(str(_CIF)), model=1, use_author_fields=True) +_E = _ATOMS[(_ATOMS.chain_id == "E") & ~_ATOMS.hetero] +_I = _ATOMS[(_ATOMS.chain_id == "I") & ~_ATOMS.hetero] +_WATER = _ATOMS[_ATOMS.hetero] + + +def _residues(atoms) -> list: + starts = list(struc.get_residue_starts(atoms)) + [len(atoms)] + return [atoms[a:b] for a, b in zip(starts[:-1], starts[1:])] + + +def _free_glu(): + """A GLU of E as a free amino acid with chain ID I: a HETATM record, residue 500, 40 Å away.""" + glu = next(residue for residue in _residues(_E) if residue.res_name[0] == "GLU").copy() + glu.chain_id[:] = "I" + glu.hetero[:] = True + glu.res_id[:] = 500 + glu.coord += 40.0 + return glu + + +def _atoms(): + """A free GLU with chain ID I listed first, which the rule leaves out of I; E (241 residues) and I (63); S, I's + first 6 residues as a chain of its own, under min_res 10; the waters. The old count: 311 amino acids in all, 64 + with chain ID I.""" + short = struc.concatenate(_residues(_I)[:6]).copy() + short.chain_id[:] = "S" + short.coord += 80.0 + return _free_glu() + _E + _I + short + _WATER + + +def _write_pdb(atoms, path: Path, operators: int = 0) -> Path: + """The atoms as a PDB file; with operators, REMARK 350 of an assembly 1 that applies the first ones of the + identity and a 2-fold about z then 40 Å along x to chains E and I.""" + pdb = PDBFile() + pdb.set_structure(atoms) + pdb.write(str(path)) + if operators: + remarks = ["REMARK 300", "REMARK 300 BIOMOLECULE: 1", "REMARK 350", "REMARK 350 BIOMOLECULE: 1", + "REMARK 350 APPLY THE FOLLOWING TO CHAINS: E, I"] + motions = ((np.eye(3), np.zeros(3)), (np.diag([-1.0, -1.0, 1.0]), np.array([40.0, 0.0, 0.0]))) + for k, (rotation, translation) in enumerate(motions[:operators]): + for i in range(3): + row = "".join(f"{value:10.6f}" for value in rotation[i]) + remarks.append(f"REMARK 350 BIOMT{i + 1} {k + 1:3d}{row}{translation[i]:15.5f}") + path.write_text("".join(f"{line}\n" for line in remarks) + path.read_text()) + return path + + +def _invoke(*args): + """inference in-process (exit codes as in a shell, the exception kept), the logging setup restored after.""" + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(inference.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + +def _flat(output: str) -> str: + """The output on one line, typer's error panel borders dropped.""" + return " ".join(output.translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +class _Esm3: + """Stands in for ESM3 in FoldMatch: keeps the ESMProteins it is given, and returns L + 2 rows (BOS and EOS + included), as per_residue_embedding does.""" + + def __init__(self): + self.proteins = [] + + def __call__(self, model, protein): + self.proteins.append(protein) + return torch.zeros(len(protein) + 2, 4) + + +class MaxResTest(unittest.TestCase): + """F1: max_res counts the residues of the protein chains a call embeds, as min_res counts them, all together.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.esm3 = _Esm3() + self.enterContext(mock.patch("foldmatch.foldmatch._load_res_model", return_value="ESM3")) + self.enterContext(mock.patch("foldmatch.foldmatch._load_model", return_value=lambda x: x.mean(dim=0))) + self.enterContext(mock.patch("foldmatch.foldmatch.per_residue_embedding", self.esm3)) + self.path = _write_pdb(_atoms(), self.tmp / "f1.pdb") + + def embeds(self, call, *args) -> list: + """The lengths of the chains the call embeds, one ESM3 pass each, in order.""" + self.esm3.proteins.clear() + call(*args) + return [len(protein) for protein in self.esm3.proteins] + + def refuses(self, call, *args) -> str: + """The message of the call's ValueError, raised before any ESM3 pass.""" + self.esm3.proteins.clear() + with self.assertRaises(ValueError) as refused: + call(*args) + self.assertEqual(self.esm3.proteins, [], "ESM3 passes") + return str(refused.exception) + + def test_the_sizes_are_the_min_res_count(self): + """protein_chains_with_sizes gives the chains of protein_chain_iter with the number that min_res_n is tested + against: E 241, I 63 (its free GLU left out); S, 6, is not a chain at 10. Each chain is kept at a threshold of + its size, not at one more.""" + from foldmatch.utils.structure_parser import protein_chains_with_sizes + structure = get_structure_from_src(str(self.path), _PDB) + sized = list(protein_chains_with_sizes(structure, 10)) + self.assertEqual([(str(chain.chain_id[0]), size) for chain, size in sized], [("E", 241), ("I", 63)]) + self.assertEqual([size for _, size in protein_chains_with_sizes(structure, 0, "S")], [6]) + chains = list(protein_chain_iter(structure, 10)) + self.assertEqual(len(chains), len(sized)) + for (chain, size), same in zip(sized, chains): + self.assertTrue(np.array_equal(chain.coord, same.coord)) + self.assertIn(chain.chain_id[0], get_protein_chains(structure, size)) + self.assertNotIn(chain.chain_id[0], get_protein_chains(structure, size + 1)) + + def test_a_chain_of_exactly_max_res(self): + """A chain of max_res residues, counted as min_res counts them, is embedded, and refused at one fewer: E at + 241, I at 63 (the old count, with I's free GLU, refused I at 63), by each method that takes a chain ID.""" + path = str(self.path) + for c, size in (("E", 241), ("I", 63)): + with self.subTest(chain=c): + within = FoldMatch(max_res=size) + self.assertEqual(self.embeds(within.residue_embedding_by_chain, path, _PDB, c), [size]) + self.assertEqual(self.embeds(within.residue_embedding, path, _PDB, c), [size]) + self.assertEqual(self.embeds(within.structure_embedding, path, _PDB, c), [size]) + over = FoldMatch(max_res=size - 1) + message = (f"Protein too long for embedding: protein chain {c} has {size} residues " + f"(max {size - 1} residues)") + for call in (over.residue_embedding_by_chain, over.residue_embedding, over.structure_embedding): + self.assertEqual(self.refuses(call, path, _PDB, c), message, call.__name__) + + def test_residues_embedded_but_not_counted(self): + """max_res counts as min_res counts, not the residues ESM3 is given: the free GLU right after I, in I's piece, + or I's 11th residue alone between two numbering gaps, is embedded with I but is not a polymer residue, so it is + not counted (63 and 62 counted, 64 and 63 embedded). A max_res of the count embeds I, one fewer refuses it, as + min_res keeps I at the count and not at one more.""" + from foldmatch.utils.structure_parser import protein_chains_with_sizes + lone = [residue.copy() for residue in _residues(_I)] + lone[10].res_id[:] += 3 + for residue in lone[11:]: + residue.res_id[:] += 6 + for case, atoms, size in (("free_glu", _E + _I + _free_glu() + _WATER, 63), + ("lone_residue", _E + struc.concatenate(lone), 62)): + with self.subTest(case=case): + path = str(_write_pdb(atoms, self.tmp / f"{case}.pdb")) + structure = get_structure_from_src(path, _PDB) + self.assertEqual([n for _, n in protein_chains_with_sizes(structure, 10, "I")], [size]) + self.assertIn("I", get_protein_chains(structure, size)) + self.assertNotIn("I", get_protein_chains(structure, size + 1)) + self.assertEqual(self.embeds(FoldMatch(max_res=size).residue_embedding_by_chain, path, _PDB, "I"), + [size + 1]) + self.assertEqual(self.refuses(FoldMatch(max_res=size - 1).residue_embedding_by_chain, path, _PDB, "I"), + f"Protein too long for embedding: protein chain I has {size} residues " + f"(max {size - 1} residues)") + + def test_a_chain_within_the_limit_in_a_structure_above_it(self): + """The reported case: at max_res 100, between I's 63 residues and E's 241, I alone is embedded, by the query + too (compute_from_structure_file with a chain ID); E alone is refused, and so are E and I together (304); the + old count, the whole structure's 311 amino acids, refused I too.""" + path = str(self.path) + api = FoldMatch(max_res=100) + self.assertEqual(self.embeds(api.residue_embedding_by_chain, path, _PDB, "I"), [63]) + self.assertEqual(self.embeds(api.residue_embedding, path, _PDB, "I"), [63]) + query = dict(min_res=10, max_res=100, accelerator=Accelerator.cpu) + ids, vectors = next(compute_from_structure_file(self.path, _PDB, Granularity.chain, chain_id="I", **query)) + self.assertEqual((ids, vectors.shape), (["f1.I"], (1, 4))) + message = "Protein too long for embedding: protein chain E has 241 residues (max 100 residues)" + self.assertEqual(self.refuses(api.residue_embedding_by_chain, path, _PDB, "E"), message) + self.assertEqual(self.refuses(api.residue_embedding, path, _PDB, "E"), message) + self.assertEqual(self.refuses(lambda: list(compute_from_structure_file( + self.path, _PDB, Granularity.chain, chain_id="E", **query))), message) + message = ("Protein too long for embedding: the 2 protein chains of the structure have 304 residues " + "(max 100 residues)") + for call in (api.residue_embedding_by_chain, api.residue_embedding, api.residue_embedding_by_assembly): + self.assertEqual(self.refuses(call, path, _PDB), message, call.__name__) + + def test_every_chain_together(self): + """Without a chain ID, the chains embedded, together: 304 residues (S, under min_res, and the free GLU are not + counted), which max_res 304 lets through (the old count, 311, refused it) and 303 refuses.""" + path = str(self.path) + within, over = FoldMatch(max_res=304), FoldMatch(max_res=303) + message = ("Protein too long for embedding: the 2 protein chains of the structure have 304 residues " + "(max 303 residues)") + for method in ("residue_embedding_by_chain", "residue_embedding", "residue_embedding_by_assembly"): + with self.subTest(method=method): + self.assertEqual(self.embeds(getattr(within, method), path, _PDB), [241, 63]) + self.assertEqual(self.refuses(getattr(over, method), path, _PDB), message) + + def test_every_copy_in_an_assembly(self): + """Assembly 1 with two copies of E and I: residue_embedding with chain ID I counts both copies of I (126), + residue_embedding_by_assembly and residue_embedding without a chain ID every chain of every copy (608), the + assembly query too; the old count added each copy's free GLU (128 and 610).""" + path = _write_pdb(_atoms(), self.tmp / "f1_assembly.pdb", operators=2) + self.assertEqual(self.embeds(FoldMatch(max_res=126).residue_embedding, str(path), _PDB, "I", "1"), [63, 63]) + self.assertEqual(self.refuses(FoldMatch(max_res=125).residue_embedding, str(path), _PDB, "I", "1"), + "Protein too long for embedding: the 2 copies of protein chain I in assembly 1 have 126 " + "residues (max 125 residues)") + message = ("Protein too long for embedding: the 4 protein chains of assembly 1 have 608 residues " + "(max 607 residues)") + for method, args in (("residue_embedding_by_assembly", ("1",)), ("residue_embedding", (None, "1"))): + with self.subTest(method=method): + self.assertEqual(self.embeds(getattr(FoldMatch(max_res=608), method), str(path), _PDB, *args), + [241, 63, 241, 63]) + self.assertEqual(self.refuses(getattr(FoldMatch(max_res=607), method), str(path), _PDB, *args), + message) + query = dict(assembly_id="1", min_res=10, accelerator=Accelerator.cpu) + ids, _ = next(compute_from_structure_file(path, _PDB, Granularity.assembly, max_res=608, **query)) + self.assertEqual(ids, ["f1_assembly-1"]) + self.assertEqual(self.refuses(lambda: list(compute_from_structure_file( + path, _PDB, Granularity.assembly, max_res=607, **query))), message) + + def test_the_message_names_what_was_counted(self): + """One copy, one chain: in the singular.""" + path = _write_pdb(_atoms(), self.tmp / "one_copy.pdb", operators=1) + self.assertEqual(self.refuses(FoldMatch(max_res=62).residue_embedding, str(path), _PDB, "I", "1"), + "Protein too long for embedding: the 1 copy of protein chain I in assembly 1 has 63 residues " + "(max 62 residues)") + self.assertEqual(self.refuses(FoldMatch(min_res=64, max_res=240).residue_embedding_by_chain, str(_CIF)), + "Protein too long for embedding: the 1 protein chain of the structure has 241 residues " + "(max 240 residues)") + + def test_nothing_to_embed(self): + """No chain of at least min_res residues (242; E has 241): residue_embedding raises its own ValueError and the + other two methods give an empty dict, at any max_res, a negative one included: nothing to embed is never too + long (the old count refused the structure as too long).""" + path = str(self.path) + for max_res in (10, -1): + with self.subTest(max_res=max_res): + api = FoldMatch(min_res=242, max_res=max_res) + self.assertEqual(self.refuses(api.residue_embedding, path, _PDB), + "No protein chain of at least 242 residues found in the structure") + self.assertEqual(self.refuses(api.residue_embedding, path, _PDB, "E"), + "No protein chain E of at least 242 residues found in the structure") + self.assertEqual(api.residue_embedding_by_chain(path, _PDB), {}) + self.assertEqual(api.residue_embedding_by_chain(path, _PDB, "E"), {}) + self.assertEqual(api.residue_embedding_by_assembly(path, _PDB), {}) + self.assertEqual(self.esm3.proteins, []) + + +class MinResNTest(unittest.TestCase): + """F2: --min-res-n only where it selects chains.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.chains = self.tmp / "chains.csv" + self.chains.write_text(f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n") + self.assemblies = self.tmp / "assemblies.csv" + self.assemblies.write_text(f"1acb,{_CIF},1,1acb-1\n") + self.out = self.tmp / "out" + self.out.mkdir() + + def test_the_chain_list_commands_refuse_it(self): + """residue-embedding, structure-embedding and chain-embedding: Typer's usage error (exit status 2), before + anything runs (before, it was taken and did nothing).""" + never = mock.Mock(side_effect=AssertionError("predict was called")) + with mock.patch.object(esm_inference, "predict", never), \ + mock.patch.object(structure_inference, "predict", never), \ + mock.patch.object(chain_inference, "predict", never): + for command, extra in (("residue-embedding", ()), ("structure-embedding", ()), + ("chain-embedding", ("--res-embedding-location", self.out))): + with self.subTest(command=command): + result = _invoke(command, "--src-file", self.chains, "--output-folder", self.out, *extra, + "--min-res-n", 10) + self.assertEqual(result.exit_code, 2, result.output) + self.assertRegex(_flat(result.output), no_such_option("--min-res-n")) + never.assert_not_called() + + def test_assembly_and_complete_embedding_take_it(self): + """assembly-embedding passes it to its predict, and complete-embedding to its assembly step only (before, to + its residue and chain steps too, where it did nothing).""" + with mock.patch.object(esm_inference, "predict") as residue, \ + mock.patch.object(chain_inference, "predict") as chain, \ + mock.patch.object(assembly_inferece, "predict") as assembly: + result = _invoke("assembly-embedding", "--src-file", self.assemblies, "--res-embedding-location", self.out, + "--output-folder", self.out, "--min-res-n", 7) + self.assertEqual(result.exit_code, 0, result.output) + self.assertEqual(assembly.call_args.kwargs["min_res_n"], 7) + assembly.reset_mock() + result = _invoke("complete-embedding", "--src-chain-file", self.chains, + "--src-assembly-file", self.assemblies, "--output-res-path", self.out, + "--output-chain-path", self.out, "--output-assembly-path", self.out, "--min-res-n", 7) + self.assertEqual(result.exit_code, 0, result.output) + for step in (residue, chain, assembly): + step.assert_called_once() + self.assertNotIn("min_res_n", residue.call_args.kwargs) + self.assertNotIn("min_res_n", chain.call_args.kwargs) + self.assertEqual(assembly.call_args.kwargs["min_res_n"], 7) + + def test_the_library_refuses_a_min_res_n_it_would_ignore(self): + """esm_inference and structure_inference given chains, chain_inference given residue files, Parquet or an + inference_set: a RefusalError before the input is read (it does not exist) or any model is built. min_res_n 0, + and min_res_n with structures read whole, go on to read the input, as before.""" + absent = self.tmp / "absent.csv" + lead = ("min_res_n=10 selects the protein chains of structures read whole (src_from structure), but the " + "chains of ") + end = " are embedded as listed: leave min_res_n at 0, or list only the chains to embed." + chain_list = lead + "a chain list (src_from chain)" + end + listed = lead + "an inference_set or of residue embedding files or Parquet (src_from file or parquet)" + end + never = mock.Mock(side_effect=AssertionError("a model was built")) + with mock.patch.object(esm_inference, "get_residue_model", never), \ + mock.patch.object(structure_inference, "get_residue_model", never), \ + mock.patch.object(structure_inference, "get_aggregator_model", never), \ + mock.patch.object(chain_inference, "get_aggregator_model", never): + for name, call, message in ( + ("esm_inference", lambda n: esm_inference.predict( + absent, src_from=SrcProteinFrom.chain, min_res_n=n), chain_list), + ("structure_inference", lambda n: structure_inference.predict( + absent, src_from=SrcProteinFrom.chain, min_res_n=n), chain_list), + ("chain_inference, files", lambda n: chain_inference.predict( + absent, src_from=SrcTensorFrom.file, min_res_n=n), listed), + ("chain_inference, Parquet", lambda n: chain_inference.predict( + absent, src_from=SrcTensorFrom.parquet, min_res_n=n), listed), + ("chain_inference, inference_set", lambda n: chain_inference.predict( + absent, src_from=SrcTensorFrom.structure, min_res_n=n, inference_set=[]), listed)): + with self.subTest(call=name): + with self.assertRaises(RefusalError) as refused: + call(10) + self.assertEqual(str(refused.exception), message) + if name != "chain_inference, inference_set": + with self.assertRaises((FileNotFoundError, RefusalError)) as read: + call(0) + self.assertIn(str(absent), str(read.exception)) + with self.assertRaises(FileNotFoundError): + esm_inference.predict(absent, src_from=SrcProteinFrom.structure, min_res_n=10) + with self.assertRaises(FileNotFoundError): + structure_inference.predict(absent, src_from=SrcProteinFrom.structure, min_res_n=10) + with self.assertRaisesRegex(FileNotFoundError, "does not exist"): + chain_inference.predict(absent, res_embedding_location=self.tmp / "absent", + src_from=SrcTensorFrom.structure, min_res_n=10) + never.assert_not_called() + + def test_the_help_says_which_chains_it_selects(self): + """assembly-embedding and complete-embedding: the chains of each assembly (complete-embedding said "When using + all chains in a structure"); the help of the three chain-list commands no longer names the option.""" + for command, selects in (("assembly-embedding", True), ("complete-embedding", True), + ("residue-embedding", False), ("structure-embedding", False), + ("chain-embedding", False)): + with self.subTest(command=command): + result = CliRunner().invoke(inference.app, [command, "--help"], env={"COLUMNS": "400"}) + self.assertEqual(result.exit_code, 0, result.output) + if selects: + self.assertIn("--min-res-n INTEGER Consider only assembly chains with at least " + "residues.", _flat(result.output)) + else: + self.assertNotIn("min-res", _flat(result.output)) + + +def _esm3(model, protein): + """Stands in for ESM3 in the inference modules: L + 2 rows (BOS and EOS, zeros), each residue's row its letter, its + position and the chain's length, so that each chain has a tensor of its own (exact sums in float32).""" + rows = [[float(ord(aa)), float(k), float(len(protein))] for k, aa in enumerate(protein.sequence)] + return torch.tensor([[0.0] * 3] + rows + [[0.0] * 3]) + + +def _aggregate(x, x_mask): + """Stands in for the aggregator: the sum of each item's rows.""" + return x.masked_fill(x_mask.unsqueeze(-1), 0.0).sum(dim=1) + + +def _csv(residues) -> str: + """The chain embedding of these residue embeddings as CsvBatchWriter writes it: _aggregate's vector, one value per + line.""" + return pd.DataFrame(residues.sum(dim=0).numpy()).to_csv(index=False, header=False) + + +def _protein(chain_id: str) -> ESMProtein: + """Archive 1acb's protein chain, read as every path reads it.""" + atoms = next(protein_chain_iter(get_structure_from_src(str(_CIF), _MMCIF), chain_id=chain_id)) + return ESMProtein.from_protein_chain(ProteinChain.from_atomarray(rename_atom_attr(filter_residues(atoms)))) + + +class ChainListTest(unittest.TestCase): + """F3: the inference commands on chain lists, with ESM3 and the aggregator stubbed (_esm3, _aggregate).""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + for module in ("esm_inference", "structure_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_residue_model", return_value="ESM3")) + for module in ("structure_inference", "chain_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_aggregator_model", return_value=_aggregate)) + for module in ("esm_module", "chain_complete_module"): + self.enterContext(mock.patch(f"foldmatch.modules.{module}.per_residue_embedding", _esm3)) + # The residue embeddings of archive 1acb's chains, as the stub gives them + self.residues_of = {c: _esm3(None, _protein(c)) for c in "AB"} + + def folder(self, name: str) -> Path: + return Path(tempfile.mkdtemp(prefix=f"{name}-", dir=self.tmp)) + + def chain_list(self, text: str) -> Path: + path = self.tmp / f"list-{len(list(self.tmp.glob('list-*.csv')))}.csv" + path.write_text(text) + return path + + def residues(self, chains: Path): + """inference residue-embedding: its folder, and the tensors it wrote there, by file name.""" + out = self.folder("residues") + result = _invoke("residue-embedding", "--src-file", chains, "--output-folder", out, *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + return out, {f.name: torch.load(f, weights_only=True) for f in out.iterdir()} + + def chains(self, chains: Path, residues: Path, res_format: str = "pt"): + """inference chain-embedding (CSV): its result, and the files it wrote, by name.""" + out = self.folder("chains") + result = _invoke("chain-embedding", "--src-file", chains, "--res-embedding-location", residues, + "--res-embedding-format", res_format, "--output-folder", out, *_CPU) + return result, {f.name: f.read_text() for f in out.iterdir()} + + def structures(self, chains: Path) -> dict: + """inference structure-embedding: its chain embeddings, by id.""" + out = self.folder("structures") + result = _invoke("structure-embedding", "--src-file", chains, "--output-folder", out, *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + table = pq.read_table(out / "inference-0.parquet").to_pydict() + return dict(zip(table["id"], table["embedding"])) + + def assert_residues(self, written: dict, expected: dict): + """The residue files written, by name, hold the residue embeddings of these chains.""" + self.assertEqual(sorted(written), sorted(expected)) + for name, chain in expected.items(): + self.assertTrue(torch.equal(written[name], self.residues_of[chain]), name) + + def assert_chains(self, result, written: dict, expected: dict): + """chain-embedding ran, and wrote the chain embeddings of these chains, by name.""" + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + self.assertEqual(written, {name: _csv(self.residues_of[chain]) for name, chain in expected.items()}) + + def vector(self, chain: str) -> list: + return self.residues_of[chain].sum(dim=0).tolist() + + def test_custom_names_flow_from_residue_to_chain_embeddings(self): + """residue-embedding writes each row under its embedding name, and chain-embedding reads that file and writes + under that name (before, it read 1acb.A.pt and 1acb.B.pt, which were not there).""" + chains = self.chain_list(f"1acb,{_CIF},A,first\n1acb,{_CIF},B,second\n") + residues, written = self.residues(chains) + self.assert_residues(written, {"first.pt": "A", "second.pt": "B"}) + self.assert_chains(*self.chains(chains, residues), {"first.csv": "A", "second.csv": "B"}) + self.assertEqual(self.structures(chains), {"first": self.vector("A"), "second": self.vector("B")}) + + def test_a_missing_or_empty_name_is_structure_dot_chain(self): + """A row of three fields, or with an empty 4th: 1acb.A and 1acb.B, in residue-embedding (before, the name '' + made torch.save refuse the file name '.pt'), chain-embedding and structure-embedding (before, empty ids).""" + chains = self.chain_list(f"1acb,{_CIF},A\n1acb,{_CIF},B,\n") + residues, written = self.residues(chains) + self.assert_residues(written, {"1acb.A.pt": "A", "1acb.B.pt": "B"}) + self.assert_chains(*self.chains(chains, residues), {"1acb.A.csv": "A", "1acb.B.csv": "B"}) + self.assertEqual(self.structures(chains), {"1acb.A": self.vector("A"), "1acb.B": self.vector("B")}) + + def test_conventional_lists_as_before(self): + """Embedding names .: the names and contents that the stubs give through the + unchanged readers and writers (EsmProtFromChain, TensorBatchWriter, CsvBatchWriter, ParquetBatchWriter), + recorded here, so that this test passes before and after this change.""" + chains = self.chain_list(f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n") + residues, written = self.residues(chains) + self.assert_residues(written, {"1acb.A.pt": "A", "1acb.B.pt": "B"}) + self.assert_chains(*self.chains(chains, residues), {"1acb.A.csv": "A", "1acb.B.csv": "B"}) + self.assertEqual(self.structures(chains), {"1acb.A": self.vector("A"), "1acb.B": self.vector("B")}) + + def test_the_recipe_of_residue_files(self): + """The CHANGELOG's recipe for residue files NAME.CHAIN.pt or NAME.CHAIN.csv, a list of NAME,,CHAIN lines: + chain embeddings NAME.CHAIN.csv, as before. The file column is not read, so neither an empty one nor two + files under one name is refused.""" + residues = self.folder("recipe") + for chain, tensor in self.residues_of.items(): + torch.save(tensor, residues / f"x.{chain}.pt") + pd.DataFrame(self.residues_of["A"].numpy()).to_csv(residues / "x.A.csv", index=False, header=False) + self.assert_chains(*self.chains(self.chain_list("x,,A\nx,,B\n"), residues), {"x.A.csv": "A", "x.B.csv": "B"}) + self.assert_chains(*self.chains(self.chain_list("x,,A\n"), residues, "csv"), {"x.A.csv": "A"}) + self.assert_chains(*self.chains(self.chain_list(f"x,{_CIF},A\nx,{self.tmp / 'other.cif'},B\n"), residues), + {"x.A.csv": "A", "x.B.csv": "B"}) + + def test_a_quoted_field_is_read_alike(self): + """Quoted fields: residue-embedding and chain-embedding read the same names, 1acb.A (the quotes of "1acb" and + "A" dropped) and b,1 (a comma within quotes); before, chain-embedding split the lines on commas, and looked for + "1acb"."A".pt.""" + chains = self.chain_list(f'"1acb",{_CIF},"A",\n1acb,{_CIF},B,"b,1"\n') + residues, written = self.residues(chains) + self.assert_residues(written, {"1acb.A.pt": "A", "b,1.pt": "B"}) + self.assert_chains(*self.chains(chains, residues), {"1acb.A.csv": "A", "b,1.csv": "B"}) + + def test_a_missing_residue_file_is_named(self): + """chain-embedding names the file it did not find, with the row's embedding name, and says what to do; a line + of one or two fields gets an empty chain ID, and so a missing file (before, it was skipped without a word).""" + residues = self.folder("empty") + for text, name in ((f"1acb,{_CIF},A,first\n", "first"), (f"1acb,{_CIF}\n", "1acb."), ("1acb\n", "1acb.")): + with self.subTest(line=text.strip()): + result, written = self.chains(self.chain_list(text), residues) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, FileNotFoundError) + self.assertEqual(str(result.exception), + f"No residue embedding file {residues.resolve() / f'{name}.pt'} for {name}: compute " + f"it first, or check the folder and the format of the residue embeddings.") + self.assertEqual(written, {}) + + def test_complete_embedding_with_the_default_names(self): + """complete-embedding on a list without the 4th column: the residue files, chain embeddings and assembly + embedding under . (before, the residue step stopped on the file name '.pt').""" + chains = self.chain_list(f"1acb,{_CIF},A\n1acb,{_CIF},B\n") + assemblies = self.chain_list(f"1acb,{_CIF},1,1acb-1\n") + residues, chain_out, assembly_out = self.folder("residues"), self.folder("chains"), self.folder("assemblies") + result = _invoke("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", assemblies, + "--output-res-path", residues, "--output-chain-path", chain_out, + "--output-assembly-path", assembly_out, *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + self.assert_residues({f.name: torch.load(f, weights_only=True) for f in residues.iterdir()}, + {"1acb.A.pt": "A", "1acb.B.pt": "B"}) + self.assertEqual({f.name: f.read_text() for f in chain_out.iterdir()}, + {f"1acb.{c}.csv": _csv(self.residues_of[c]) for c in "AB"}) + self.assertEqual({f.name: f.read_text() for f in assembly_out.iterdir()}, + {"1acb-1.csv": _csv(torch.cat([self.residues_of["A"], self.residues_of["B"]]))}) + + def test_assembly_embedding_with_min_res_n(self): + """F2 through assembly_inferece and chain_inference, neither mocked: assembly-embedding --min-res-n 100 embeds + assembly 1 of 1acb from A alone (B has 63 residues). chain_inference refuses a min_res_n given an + inference_set, so assembly_inferece must not hand it on (complete-embedding's assembly step is this command).""" + residues, _ = self.residues(self.chain_list(f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n")) + out = self.folder("assemblies") + result = _invoke("assembly-embedding", "--src-file", self.chain_list(f"1acb,{_CIF},1,1acb-1\n"), + "--res-embedding-location", residues, "--output-folder", out, "--min-res-n", 100, *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + self.assertEqual({f.name: f.read_text() for f in out.iterdir()}, {"1acb-1.csv": _csv(self.residues_of["A"])}) + + def test_complete_embedding_refuses_other_names_before_any_model(self): + """A chain list with other embedding names: complete-embedding refuses it (RefusalError, exit status 1) before + any step runs, naming up to 5 such rows (5 of the first list's 7) and how many, each field quoted, so that + whitespace shows (the second list's first structure name starts with a space). Before, it ran the residue step, + and failed at the chain step on the first list; it ran the second as if the space were not there.""" + names = [("A", "a1"), ("A", "a2"), ("B", "b1"), ("A", "a3"), ("B", "b2"), ("B", "b3"), ("A", "a4")] + assemblies = self.chain_list(f"1acb,{_CIF},1,1acb-1\n") + for case, text, found in ( + ("7 rows named otherwise", + f"1acb,{_CIF},A,1acb.A\n" + "".join(f"1acb,{_CIF},{c},{n}\n" for c, n in names), + "7 row(s) with an embedding name other than .: " + + "; ".join(f"'1acb' chain '{c}' named '{n}'" for c, n in names[:5]) + "; and 2 more rows"), + ("a structure name that starts with a space", f" 1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n", + "1 row(s) with an embedding name other than .: " + "' 1acb' chain 'A' named '1acb.A'")): + with self.subTest(case=case): + chains = self.chain_list(text) + out = self.folder("out") + never = mock.Mock(side_effect=AssertionError("a step ran")) + with mock.patch.object(esm_inference, "predict", never), \ + mock.patch.object(chain_inference, "predict", never), \ + mock.patch.object(assembly_inferece, "predict", never): + result = _invoke("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", + assemblies, "--output-res-path", out, "--output-chain-path", out, + "--output-assembly-path", out, *_CPU) + never.assert_not_called() + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError) + self.assertEqual(str(result.exception), + f"{chains.resolve()}: {found}. The assembly step of complete-embedding reads each " + f"chain's residue embedding as ., as assembly-embedding " + f"does: name those rows so, or leave their 4th column out.") + self.assertEqual(list(out.iterdir()), []) + + def test_the_reader(self): + """read_chain_list, of a file and of a stream: the four columns as strings, in input order (not sorted), each + field as written, spaces included; a missing or empty embedding name, .; quoted + fields read as CSV; nothing refused, a blank chain ID included.""" + from foldmatch.dataset.utils import read_chain_list + text = 'g\nb,f.cif,B\na,f.cif,A,named\nc,f.cif,C,\n"d,1",,"D",\ne,f.cif,,\nf,f.cif\n h , f.cif, H ,\n' + expected = [["g", "", "", "g."], ["b", "f.cif", "B", "b.B"], ["a", "f.cif", "A", "named"], + ["c", "f.cif", "C", "c.C"], ["d,1", "", "D", "d,1.D"], ["e", "f.cif", "", "e."], + ["f", "f.cif", "", "f."], [" h ", " f.cif", " H ", " h . H "]] + data = read_chain_list(self.chain_list(text)) + self.assertEqual(list(data.columns), ["stream_name", "stream", "chain_id", "item_name"]) + self.assertEqual(data.values.tolist(), expected) + stream = [("h", "f.cif", "H", None), ("a", "f.cif", "A", "named"), ("b", "f.cif", "B"), ("c", "f.cif", "C", "")] + self.assertEqual(read_chain_list(stream, SrcLocation.stream).values.tolist(), + [["h", "f.cif", "H", "h.H"], expected[2], expected[1], expected[3]]) + + def test_a_blank_chain_id_is_a_chain_id(self): + """A PDB file whose chain I has a blank chain ID, which biotite reads as '': EsmProtFromChain embeds it for a + stream row of three fields, named x. (before, a stream row of three fields was refused by pandas).""" + atoms = _E + _I + atoms.chain_id[atoms.chain_id == "I"] = "" + path = _write_pdb(atoms, self.tmp / "blank.pdb") + protein, name = EsmProtFromChain(src_stream=[("x", str(path), "")], src_location=SrcLocation.stream, + structure_format=_PDB)[0] + self.assertEqual((name, len(protein)), ("x.", 63)) + + +# What a refusal of a chain list row of more than 4 fields says after what it found +_AT_MOST_4_FIELDS = ("A chain list row holds at most 4 fields (structure name, structure file or URL, chain ID, " + "embedding name): ") +_REMOVE_THE_EXTRA_FIELDS = "remove the extra fields (a comma after the embedding name makes a 5th)." + + +class LongerRowTest(unittest.TestCase): + """F3: a chain list row of more than 4 fields is refused (RefusalError) by read_chain_list, for a file and for a + stream, naming the file or the stream and the row, and so by each command that reads a chain list and by + EsmProtFromChain, before any model is built (here the model getters fail if called). Before, pandas read the leading + fields of a first such row as the index, which shifted the columns of every row (by two for 6 fields), and stopped + at a later one with a ParserError that named no file; chain-embedding ignored the fields after the 3rd. Lists of at + most 4 fields per row are read as before.""" + + # (case, chain list, what the refusal found, after the file's path: a regular expression) + LISTS = ( + ("first row, a trailing comma", f"1acb,{_CIF},A,1acb.A,\n1acb,{_CIF},B,1acb.B\n", + re.escape("its first row has 5 fields")), + ("first row, a 5th field of text", f"1acb,{_CIF},A,1acb.A,extra\n1acb,{_CIF},B,1acb.B\n", + re.escape("its first row has 5 fields")), + ("first row of 6 fields", f"1acb,{_CIF},A,1acb.A,,extra\n1acb,{_CIF},B,1acb.B\n", + re.escape("its first row has 6 fields")), + ("a later row of 5 fields", f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B,\n", + r"a row has more than 4 fields \(pandas: [^()]*Expected 4 fields in line 2, saw 5\)"), + ("a first row of 5, a later row of 6", f"1acb,{_CIF},A,1acb.A,\n1acb,{_CIF},B,1acb.B,,\n", + r"a row has more than 4 fields \(pandas: [^()]*Expected 5 fields in line 2, saw 6\)"), + ) + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.never = mock.Mock(side_effect=AssertionError("a model was built")) + for getter in ("esm_inference.get_residue_model", "structure_inference.get_residue_model", + "structure_inference.get_aggregator_model", "chain_inference.get_aggregator_model"): + self.enterContext(mock.patch(f"foldmatch.inference.{getter}", self.never)) + + def chain_list(self, case: str, text: str) -> Path: + path = self.tmp / f"{case.replace(',', '').replace(' ', '_')}.csv" + path.write_text(text) + return path + + def assert_refused(self, refusal: BaseException, source: str, found: str): + """The refusal names the source and what it found, says that a row holds at most 4 fields, and what to do.""" + self.assertIsInstance(refusal, RefusalError) + self.assertRegex(str(refusal), f"^{re.escape(source)}: {found}\\. " + f"{re.escape(_AT_MOST_4_FIELDS + _REMOVE_THE_EXTRA_FIELDS)}$") + + def assert_frame(self, data: pd.DataFrame, expected: list): + """The four columns, the rows as expected, every value a string, the index pandas gives (0, 1, ...).""" + self.assertEqual(list(data.columns), ["stream_name", "stream", "chain_id", "item_name"]) + self.assertEqual(data.values.tolist(), expected) + self.assertTrue(all(isinstance(value, str) for value in data.values.ravel())) + self.assertIsInstance(data.index, pd.RangeIndex) + self.assertEqual(list(data.index), list(range(len(expected)))) + + def test_the_reader_refuses_rows_of_more_than_4_fields(self): + """A first row with a trailing comma, with a 5th field of text or with 6 fields (pandas took its first one or + two fields as the index), and a later row of 5, or of 6 after a first row of 5 (pandas' ParserError, which + names the line, quoted and kept as the cause): from the path, from an open file (named by its path) and from a + text buffer.""" + from foldmatch.dataset.utils import read_chain_list + for case, text, found in self.LISTS: + with self.subTest(case=case): + path = self.chain_list(case, text) + with self.assertRaises(RefusalError) as refused: + read_chain_list(path) + self.assert_refused(refused.exception, str(path), found) + if "later row" in case: + self.assertIsInstance(refused.exception.__cause__, pd.errors.ParserError) + with open(path) as opened, self.assertRaises(RefusalError) as refused: + read_chain_list(opened) + self.assert_refused(refused.exception, str(path), found) + with self.assertRaises(RefusalError) as refused: + read_chain_list(io.StringIO(text)) + self.assert_refused(refused.exception, "The chain list", found) + + def test_a_stream_row_of_more_than_4_elements(self): + """A stream: how many rows have more than 4 elements, and up to 5 of them, counted from 1 (before, pandas' + ValueError named no row).""" + from foldmatch.dataset.utils import read_chain_list + rule = ("A chain list row holds at most 4 fields (structure name, structure file or URL, chain ID, embedding " + "name): remove the extra elements.") + for rows, found in ( + ([("x", "f", "A", "x.A"), ("y", "f", "B", "y.B", "")], "1 row(s) of more than 4 elements (row(s) 2"), + ([("w", "f", "W")] + [("x", "f", "A", "x.A", "z")] * 5 + [("y",) * 6] * 2, + "7 row(s) of more than 4 elements (row(s) 2, 3, 4, 5, 6 and 2 more")): + with self.subTest(rows=len(rows)): + with self.assertRaises(RefusalError) as refused: + read_chain_list(rows, SrcLocation.stream) + self.assertEqual(str(refused.exception), f"The chain list stream: {found}, counted from 1). {rule}") + + def test_another_parser_error_is_not_claimed_as_extra_fields(self): + """Another pandas ParserError (an unclosed quote) is refused too, naming the file and quoting pandas, without + saying that a row has more than 4 fields.""" + from foldmatch.dataset.utils import read_chain_list + path = self.chain_list("unclosed quote", f'1acb,"{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n') + with self.assertRaises(RefusalError) as refused: + read_chain_list(path) + self.assertIsInstance(refused.exception.__cause__, pd.errors.ParserError) + self.assertRegex(str(refused.exception), + f"^{re.escape(str(path))}: pandas could not read it as CSV \\([^()]*EOF inside string" + f"[^()]*\\)\\. " + re.escape("A chain list is CSV without a header, of at most 4 fields per " + "row (structure name, structure file or URL, chain ID, embedding " + "name): correct it where pandas' message points.") + "$") + self.assertNotIn("more than 4", str(refused.exception)) + + def test_each_command_refuses_it_before_any_model(self): + """inference residue-embedding, structure-embedding, chain-embedding and complete-embedding: exit status 1 with + the RefusalError, nothing written and no model built. Before, residue-embedding and structure-embedding read the + shifted columns or stopped on pandas' ParserError, chain-embedding read each line's 1st and 3rd fields and went + on, and complete-embedding's residue step did as residue-embedding did.""" + assemblies = self.chain_list("assemblies", f"1acb,{_CIF},1,1acb-1\n") + for case, text, found in self.LISTS: + path = self.chain_list(case, text) + for command in ("residue-embedding", "structure-embedding", "chain-embedding", "complete-embedding"): + with self.subTest(case=case, command=command): + out = Path(tempfile.mkdtemp(dir=self.tmp)) + if command == "complete-embedding": + args = ("--src-chain-file", path, "--src-assembly-file", assemblies, "--output-res-path", out, + "--output-chain-path", out, "--output-assembly-path", out) + elif command == "chain-embedding": + args = ("--src-file", path, "--res-embedding-location", out, "--output-folder", out) + else: + args = ("--src-file", path, "--output-folder", out) + result = _invoke(command, *args, *_CPU) + self.assertEqual(result.exit_code, 1, result.output) + self.assert_refused(result.exception, str(path.resolve()), found) + self.assertEqual(list(out.iterdir()), []) + self.never.assert_not_called() + + def test_esm_prot_from_chain_refuses_it(self): + """The library's EsmProtFromChain, given a file or a stream (before: the shifted columns or pandas' ParserError + for a file, pandas' ValueError for a stream).""" + for case, text, found in self.LISTS: + with self.subTest(case=case): + path = self.chain_list(case, text) + with self.assertRaises(RefusalError) as refused: + EsmProtFromChain(src_stream=path) + self.assert_refused(refused.exception, str(path), found) + with self.assertRaises(RefusalError) as refused: + EsmProtFromChain(src_stream=[("1acb", str(_CIF), "A", "1acb.A"), ("1acb", str(_CIF), "B", "1acb.B", "x")], + src_location=SrcLocation.stream) + self.assertTrue(str(refused.exception).startswith( + "The chain list stream: 1 row(s) of more than 4 elements (row(s) 2, counted from 1)."), refused.exception) + + def test_lists_of_at_most_4_fields_read_as_before(self): + """The frames that the reader gave before it refused longer rows (pandas' parse is unchanged), recorded here: + rows of 1 to 4 fields, an empty 4th, a byte-order mark and CRLF, quoted fields with commas and newlines, blank + and whitespace-only lines, NA, nan and None kept as text, an empty file and a file of blank lines, from a path + and from a text buffer; and streams of 3 and 4 elements.""" + from foldmatch.dataset.utils import read_chain_list + for case, text, expected in ( + ("4 fields", "x,f.cif,A,x.A\ny,f.cif,B,named\n", + [["x", "f.cif", "A", "x.A"], ["y", "f.cif", "B", "named"]]), + ("3, 2 and 1 fields and an empty 4th", "x,f.cif,A\ny,f.cif\nz\nw,f.cif,W,\n", + [["x", "f.cif", "A", "x.A"], ["y", "f.cif", "", "y."], ["z", "", "", "z."], + ["w", "f.cif", "W", "w.W"]]), + ("a byte-order mark and CRLF", "\ufeffx,f.cif,A,x.A\r\ny,f.cif,B\r\n", + [["x", "f.cif", "A", "x.A"], ["y", "f.cif", "B", "y.B"]]), + ("quoted fields with commas and newlines", 'x,"a,b",A,"n,1"\n"y\nz",f.cif,"B",\n', + [["x", "a,b", "A", "n,1"], ["y\nz", "f.cif", "B", "y\nz.B"]]), + ("blank and whitespace-only lines", "\nx,,A\n\n \n\t\ny,,B\n\n", + [["x", "", "A", "x.A"], ["y", "", "B", "y.B"]]), + ("NA, nan and None", "NA,nan,None\nN/A,NaN,,null\n", + [["NA", "nan", "None", "NA.None"], ["N/A", "NaN", "", "null"]]), + ("an empty file", "", []), + ("a file of blank lines", "\n\n\n", [])): + path = self.chain_list(case, text) + for source in ("path", "text buffer"): + with self.subTest(case=case, source=source): + self.assert_frame(read_chain_list(path if source == "path" else io.StringIO(text)), expected) + for rows in ([("x", "f.cif", "A"), ("y", "f.cif", "B")], [("x", "f.cif", "A", "x.A"), ("y", "f.cif", "B", "")]): + with self.subTest(stream=len(rows[0])): + self.assert_frame(read_chain_list(rows, SrcLocation.stream), + [["x", "f.cif", "A", "x.A"], ["y", "f.cif", "B", "y.B"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inference_outputs.py b/tests/test_inference_outputs.py new file mode 100644 index 0000000..64ddebc --- /dev/null +++ b/tests/test_inference_outputs.py @@ -0,0 +1,241 @@ +"""P9.H: what the inference commands write, as their help says, and what --structure-format and the default names of +complete-embedding do. ESM3 and the aggregator are never loaded: the commands run with stub models, as in +test_inference_follow_ups.py.""" +import gzip +import json +import logging +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import pyarrow.parquet as pq +import torch +from typer.testing import CliRunner + +from foldmatch.cli import inference +# Imported before any test patches chain_inference.predict, which assembly_inferece binds at its import +from foldmatch.inference import assembly_inferece, chain_inference, esm_inference +from foldmatch.types.api_types import StructureFormat +from click_messages import no_such_option + +_CIF = Path(__file__).parent / "resources" / "pdb" / "1acb.cif" +_CPU = ("--accelerator", "cpu", "--devices", "1") +_CHAINS = ["1acb.A", "1acb.B"] +_ASSEMBLIES = ["1acb-1"] + + +def _esm3(model, protein): + """Stands in for ESM3: L + 2 rows (BOS and EOS, zeros), each residue's row its letter and its position.""" + rows = [[float(ord(aa)), float(k)] for k, aa in enumerate(protein.sequence)] + return torch.tensor([[0.0] * 2] + rows + [[0.0] * 2]) + + +def _aggregate(x, x_mask): + """Stands in for the aggregator: the sum of each item's rows.""" + return x.masked_fill(x_mask.unsqueeze(-1), 0.0).sum(dim=1) + + +def _invoke(*args): + """inference in-process, the logging setup restored after.""" + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(inference.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + +def _flat(output: str) -> str: + """The output on one line, typer's error panel borders dropped, and its colours (on with GITHUB_ACTIONS, FORCE_COLOR + or PY_COLORS set).""" + return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", output).translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +def _ids(path: Path) -> list: + """The embedding ids of a Parquet or JSON file.""" + if path.suffix == ".parquet": + return pq.read_table(path).column("id").to_pylist() + with gzip.open(path, "rt") as f: + return [row["id"] for row in json.load(f)] + + +class _Lists(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.chains = self.tmp / "chains.csv" + self.chains.write_text(f"1acb,{_CIF},A,1acb.A\n1acb,{_CIF},B,1acb.B\n") + self.assemblies = self.tmp / "assemblies.csv" + self.assemblies.write_text(f"1acb,{_CIF},1,1acb-1\n") + + def folder(self) -> Path: + return Path(tempfile.mkdtemp(dir=self.tmp)) + + def run_ok(self, *args): + result = _invoke(*args, *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + + +class _StubModels(_Lists): + + def setUp(self): + super().setUp() + for module in ("esm_inference", "structure_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_residue_model", return_value="ESM3")) + for module in ("structure_inference", "chain_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_aggregator_model", return_value=_aggregate)) + for module in ("esm_module", "chain_complete_module"): + self.enterContext(mock.patch(f"foldmatch.modules.{module}.per_residue_embedding", _esm3)) + + def residues(self) -> Path: + out = self.folder() + self.run_ok("residue-embedding", "--src-file", self.chains, "--output-folder", out) + return out + + +def _files(names: list, output_format: str, output_name: str) -> list: + """The files that the help says a run writes: one per item for csv and pt, one per process for parquet and json.""" + return {"csv": sorted(f"{n}.csv" for n in names), "pt": sorted(f"{n}.pt" for n in names), + "parquet": [f"{output_name}-0.parquet"], "json": [f"{output_name}-0.json.gz"]}[output_format] + + +class OutputFilesTest(_StubModels): + """Each command writes what its help says, in each format it takes.""" + + def assert_files(self, folder: Path, names: list, output_format: str, output_name: str): + self.assertEqual(sorted(f.name for f in folder.iterdir()), _files(names, output_format, output_name)) + if output_format in ("parquet", "json"): + self.assertCountEqual(_ids(next(folder.iterdir())), names) + + def test_residue_embedding_is_pt_by_default(self): + self.assert_files(self.residues(), _CHAINS, "pt", "inference") + + def test_residue_embedding_in_each_format(self): + for output_format in ("csv", "pt", "parquet", "json"): + with self.subTest(output_format=output_format): + out = self.folder() + self.run_ok("residue-embedding", "--src-file", self.chains, "--output-folder", out, + "--output-format", output_format, "--output-name", "r") + self.assert_files(out, _CHAINS, output_format, "r") + + def test_structure_embedding_is_parquet(self): + """Its help said a pandas DataFrame file, and a JSON file.""" + out = self.folder() + self.run_ok("structure-embedding", "--src-file", self.chains, "--output-folder", out, "--output-name", "s") + self.assert_files(out, _CHAINS, "parquet", "s") + + def test_chain_and_assembly_embedding_are_csv_by_default(self): + residues = self.residues() + for command, src, names in (("chain-embedding", self.chains, _CHAINS), + ("assembly-embedding", self.assemblies, _ASSEMBLIES)): + with self.subTest(command=command): + out = self.folder() + self.run_ok(command, "--src-file", src, "--res-embedding-location", residues, "--output-folder", out) + self.assert_files(out, names, "csv", "inference") + + def test_chain_and_assembly_embedding_in_each_format(self): + residues = self.residues() + for command, src, names in (("chain-embedding", self.chains, _CHAINS), + ("assembly-embedding", self.assemblies, _ASSEMBLIES)): + for output_format in ("csv", "pt", "parquet", "json"): + with self.subTest(command=command, output_format=output_format): + out = self.folder() + self.run_ok(command, "--src-file", src, "--res-embedding-location", residues, + "--output-folder", out, "--output-format", output_format, "--output-name", "x") + self.assert_files(out, names, output_format, "x") + + def test_complete_embedding_in_each_format(self): + """Residue embeddings as torch tensor files, chain and assembly embeddings in the format of --output-format; + its help said a single JSON file, whatever the format.""" + for output_format in ("csv", "pt", "parquet", "json"): + with self.subTest(output_format=output_format): + res, chain, assembly = self.folder(), self.folder(), self.folder() + self.run_ok("complete-embedding", "--src-chain-file", self.chains, + "--src-assembly-file", self.assemblies, "--output-res-path", res, + "--output-chain-path", chain, "--output-assembly-path", assembly, + "--output-format", output_format, "--output-chain-name", "c", + "--output-assembly-name", "a") + self.assert_files(res, _CHAINS, "pt", "inference") + self.assert_files(chain, _CHAINS, output_format, "c") + self.assert_files(assembly, _ASSEMBLIES, output_format, "a") + + +class StructureFormatTest(_Lists): + """chain-embedding reads residue embeddings and no structure, so it takes no --structure-format.""" + + def test_chain_embedding_refuses_it(self): + """Typer's usage error (exit status 2), before anything runs (before, it was taken and did nothing).""" + with mock.patch.object(chain_inference, "predict") as chain: + out = self.folder() + result = _invoke("chain-embedding", "--src-file", self.chains, "--res-embedding-location", out, + "--output-folder", out, "--structure-format", "pdb") + self.assertEqual(result.exit_code, 2, result.output) + self.assertRegex(_flat(result.output), no_such_option("--structure-format")) + chain.assert_not_called() + + def test_complete_embedding_keeps_it_for_the_steps_that_read_structures(self): + """complete-embedding passes it to its residue and assembly steps, which read structures; its chain step does + not get it.""" + out = self.folder() + with mock.patch.object(esm_inference, "predict") as residue, \ + mock.patch.object(chain_inference, "predict") as chain, \ + mock.patch.object(assembly_inferece, "predict") as assembly: + self.run_ok("complete-embedding", "--src-chain-file", self.chains, "--src-assembly-file", self.assemblies, + "--output-res-path", out, "--output-chain-path", out, "--output-assembly-path", out, + "--structure-format", "pdb") + for step in (residue, chain, assembly): + step.assert_called_once() + self.assertEqual(residue.call_args.kwargs["structure_format"], StructureFormat.pdb) + self.assertEqual(assembly.call_args.kwargs["structure_format"], StructureFormat.pdb) + self.assertNotIn("structure_format", chain.call_args.kwargs) + + def test_chain_embedding_does_not_read_the_structure_file(self): + """The list's structure files are not read: a list naming no file gives the chain embeddings of a list naming + the real one, as the option's removal assumes.""" + with mock.patch("foldmatch.inference.chain_inference.get_aggregator_model", return_value=_aggregate): + residues = self.folder() + for name in _CHAINS: + torch.save(torch.arange(6.0).reshape(3, 2) + ord(name[-1]), residues / f"{name}.pt") + nowhere = self.tmp / "nowhere.csv" + nowhere.write_text("1acb,/no/such/file.bcif,A,1acb.A\n1acb,,B,1acb.B\n") + real, other = self.folder(), self.folder() + self.run_ok("chain-embedding", "--src-file", self.chains, "--res-embedding-location", residues, + "--output-folder", real) + self.run_ok("chain-embedding", "--src-file", nowhere, "--res-embedding-location", residues, + "--output-folder", other) + for name in _CHAINS: + self.assertEqual((real / f"{name}.csv").read_bytes(), (other / f"{name}.csv").read_bytes()) + + +class CompleteEmbeddingNamesTest(_StubModels): + """The default file names of complete-embedding's chain and assembly files differ: --output-assembly-name was + chain-inference, the default of --output-chain-name, so that in one folder the assembly file replaced the chain + file.""" + + def run_complete(self, output_format: str, *names) -> Path: + out = self.folder() + self.run_ok("complete-embedding", "--src-chain-file", self.chains, "--src-assembly-file", self.assemblies, + "--output-res-path", self.folder(), "--output-chain-path", out, "--output-assembly-path", out, + "--output-format", output_format, *names) + return out + + def test_one_folder_keeps_both_files(self): + for output_format, postfix in (("parquet", "parquet"), ("json", "json.gz")): + with self.subTest(output_format=output_format): + out = self.run_complete(output_format) + self.assertEqual(sorted(f.name for f in out.iterdir()), + [f"assembly-inference-0.{postfix}", f"chain-inference-0.{postfix}"]) + self.assertCountEqual(_ids(out / f"chain-inference-0.{postfix}"), _CHAINS) + self.assertCountEqual(_ids(out / f"assembly-inference-0.{postfix}"), _ASSEMBLIES) + + def test_the_names_given_are_used(self): + out = self.run_complete("parquet", "--output-chain-name", "c", "--output-assembly-name", "a") + self.assertEqual(sorted(f.name for f in out.iterdir()), ["a-0.parquet", "c-0.parquet"]) + + def test_the_help_shows_the_default(self): + result = CliRunner().invoke(inference.app, ["complete-embedding", "--help"], env={"COLUMNS": "400"}) + self.assertEqual(result.exit_code, 0, result.output) + self.assertRegex(_flat(result.output), r"--output-assembly-name TEXT .*\[default: assembly-inference\]") diff --git a/tests/test_mps_support.py b/tests/test_mps_support.py new file mode 100644 index 0000000..ba76c89 --- /dev/null +++ b/tests/test_mps_support.py @@ -0,0 +1,216 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import torch + +import foldmatch +from foldmatch.model.residue_embedding_aggregator import ResidueEmbeddingAggregator +from foldmatch.modules import assembly_complete_module, chain_complete_module, esm_module +from foldmatch.modules.assembly_complete_module import AssemblyCompleteModule +from foldmatch.modules.chain_complete_module import ChainCompleteModule +from foldmatch.modules.esm_module import EsmModule +from foldmatch.types.api_types import SrcEsmFrom, Accelerator +from foldmatch.utils.data import collate_seq_embeddings +from foldmatch.utils.model import get_aggregator_model, release_mps_cache + +_TEST_PATH = os.path.dirname(__file__) +_FASTA = f"{_TEST_PATH}/resources/fasta/test_sequences.fasta" + + +def _legacy_forward(aggregator, x, x_mask=None): + # ResidueEmbeddingAggregator.forward before G14, verbatim. + return aggregator.embedding(aggregator.transformer(x, src_key_padding_mask=x_mask).sum(dim=x.dim() - 2)) + + +def _max_rel(a, b): + return float((a - b).abs().max() / b.abs().max()) + + +class AggregatorPaddingTest(unittest.TestCase): + """G14: the aggregator gives the same chain embeddings with and without PyTorch's nested-tensor fast path.""" + + @classmethod + def setUpClass(cls): + cls.aggregator = get_aggregator_model(torch.device("cpu")).eval() + # Real ESM3 residue embeddings of different lengths (243 and 116 rows), so one item is padded. + cls.residues = [ + torch.load(f"{_TEST_PATH}/resources/embeddings/{name}.pt", map_location="cpu", weights_only=True).float() + for name in ("1acb.A", "2uzi.A") + ] + + def test_fast_path_disabled_matches_fast_path_and_batch_size_1(self): + x, x_mask = collate_seq_embeddings(self.residues) + self.assertTrue(x_mask[1].any()) + transformer = self.aggregator.transformer + with torch.inference_mode(): + fast = self.aggregator(x, x_mask) + single = [self.aggregator(*collate_seq_embeddings([r]))[0] for r in self.residues] + transformer.use_nested_tensor = False + try: + slow = self.aggregator(x, x_mask) + legacy_slow = _legacy_forward(self.aggregator, x, x_mask) + finally: + transformer.use_nested_tensor = True + for i in range(len(self.residues)): + self.assertLessEqual(_max_rel(slow[i], fast[i]), 1e-5) + self.assertLessEqual(_max_rel(slow[i], single[i]), 1e-5) + self.assertLessEqual(_max_rel(fast[i], single[i]), 1e-5) + # Before G14 the slow path (MPS with PYTORCH_ENABLE_MPS_FALLBACK=1) summed the padded rows. + self.assertGreater(_max_rel(legacy_slow[1], single[1]), 1e-2) + + def test_cpu_output_byte_identical_to_legacy(self): + x, x_mask = collate_seq_embeddings(self.residues) + with torch.inference_mode(): + cases = [ + (x, x_mask), + collate_seq_embeddings(self.residues[:1]), + collate_seq_embeddings(self.residues[1:]), + (self.residues[1], None), + ] + for xs, mask in cases: + self.assertTrue(torch.equal(self.aggregator(xs, mask), _legacy_forward(self.aggregator, xs, mask))) + + def test_device_without_nested_tensor_kernels(self): + # The meta device lacks aten::_nested_tensor_from_mask_left_aligned exactly as MPS does. + with torch.device("meta"): + aggregator = ResidueEmbeddingAggregator().eval() + x = torch.zeros(2, 7, 1536, device="meta") + x_mask = torch.zeros(2, 7, dtype=torch.bool, device="meta") + x_mask[1, 4:] = True + seen = [] + layer = aggregator.transformer.layers[0] + with torch.inference_mode(): + with self.assertRaises(NotImplementedError): + _legacy_forward(aggregator, x, x_mask) + with mock.patch.object( + layer, "forward", + side_effect=lambda *a, _f=layer.forward, **k: seen.append(aggregator.transformer.use_nested_tensor) or _f(*a, **k) + ): + out = aggregator(x, x_mask) + self.assertEqual(tuple(out.shape), (2, 1536)) + self.assertEqual(seen, [False]) + self.assertTrue(aggregator.transformer.use_nested_tensor) + + def test_fixture_fasta_cpu_byte_identical_at_batch_sizes_1_and_2(self): + from foldmatch.inference.chain_complete_inference import predict + calls = [] + forward = ResidueEmbeddingAggregator.forward + + def spy(aggregator, x, x_mask=None): + out = forward(aggregator, x, x_mask) + calls.append(torch.equal(out, _legacy_forward(aggregator, x, x_mask))) + return out + + for batch_size in (1, 2): + with mock.patch.object(ResidueEmbeddingAggregator, "forward", spy): + predict(src_stream=_FASTA, src_from=SrcEsmFrom.fasta, batch_size=batch_size, + accelerator=Accelerator.cpu, devices=1) + self.assertEqual(calls, [True, True, True]) + + +class ReleaseMpsCacheTest(unittest.TestCase): + """G14: every module that runs ESM3 releases the MPS cache after each batch, and only on MPS.""" + + def test_release_only_on_mps(self): + events = [] + with mock.patch.object(torch.mps, "synchronize", side_effect=lambda: events.append("synchronize")), \ + mock.patch.object(torch.mps, "empty_cache", side_effect=lambda: events.append("empty_cache")): + for device in ("cpu", torch.device("cpu"), "cuda", "meta"): + release_mps_cache(device) + self.assertEqual(events, []) + release_mps_cache(torch.device("mps")) + release_mps_cache("mps:0") + self.assertEqual(events, ["synchronize", "empty_cache"] * 2) + + def test_esm3_modules_release_once_per_batch(self): + def embedding(_model, esm_prot): + return torch.ones(len(esm_prot) + 2, 4) + + def aggregator(x, x_mask): + return x.sum(dim=1) + + chains = [("AAAA", "a"), ("CC", "c")] + cases = [ + (chain_complete_module, ChainCompleteModule(None, aggregator), chains), + (esm_module, EsmModule(None), chains), + (assembly_complete_module, AssemblyCompleteModule(None, aggregator), [(["AAAA", "CC"], "x"), (["CC"], "y")]), + ] + for module, lightning_module, batch in cases: + with mock.patch.object(module, "per_residue_embedding", side_effect=embedding), \ + mock.patch.object(module, "release_mps_cache") as release: + lightning_module.predict_step(batch, 0) + lightning_module.predict_step(batch, 1) + self.assertEqual(release.call_args_list, [mock.call(lightning_module.device)] * 2, module.__name__) + + +@unittest.skipUnless(torch.backends.mps.is_available(), "Apple GPU (MPS) not available") +class MpsSmokeTest(unittest.TestCase): + """G14: predict() on MPS runs without PYTORCH_ENABLE_MPS_FALLBACK and matches CPU.""" + + _CHILD = r""" +import json, sys +import numpy as np, torch +from unittest import mock +from foldmatch.inference.chain_complete_inference import predict +from foldmatch.modules.chain_complete_module import ChainCompleteModule +from foldmatch.types.api_types import SrcEsmFrom, Accelerator + +fasta, out = sys.argv[1], sys.argv[2] +events = [] +empty_cache, synchronize, predict_step = torch.mps.empty_cache, torch.mps.synchronize, ChainCompleteModule.predict_step + +def step(module, batch, batch_idx): + events.append("start") + result = predict_step(module, batch, batch_idx) + events.append("end") + return result + +result = {} +with mock.patch.object(torch.mps, "empty_cache", side_effect=lambda: events.append("empty_cache") or empty_cache()), \ + mock.patch.object(torch.mps, "synchronize", side_effect=lambda: events.append("synchronize") or synchronize()), \ + mock.patch.object(ChainCompleteModule, "predict_step", step): + for accelerator, batch_size in (("cpu", 2), ("gpu", 1), ("gpu", 2)): + events.clear() + predictions = predict(src_stream=fasta, src_from=SrcEsmFrom.fasta, batch_size=batch_size, + accelerator=Accelerator(accelerator), devices=1) + key = f"{accelerator}_b{batch_size}" + result[key] = list(events) + np.savez(f"{out}/{key}.npz", **{ + name: embedding.float().cpu().numpy() + for embeddings, names in predictions for embedding, name in zip(embeddings, names) + }) +json.dump(result, open(f"{out}/events.json", "w")) +""" + + def test_predict_on_mps(self): + env = {k: v for k, v in os.environ.items() if k != "PYTORCH_ENABLE_MPS_FALLBACK"} + env["PYTHONPATH"] = os.pathsep.join( + [str(Path(foldmatch.__file__).parents[1])] + ([env["PYTHONPATH"]] if env.get("PYTHONPATH") else []) + ) + with tempfile.TemporaryDirectory() as out: + run = subprocess.run( + [sys.executable, "-c", self._CHILD, _FASTA, out], env=env, capture_output=True, text=True + ) + self.assertEqual(run.returncode, 0, run.stderr[-3000:]) + events = json.load(open(f"{out}/events.json")) + cpu = np.load(f"{out}/cpu_b2.npz") + mps = {key: np.load(f"{out}/{key}.npz") for key in ("gpu_b1", "gpu_b2")} + for name in cpu.files: + for key in mps: + rel = np.abs(mps[key][name] - cpu[name]).max() / np.abs(cpu[name]).max() + self.assertLessEqual(rel, 1e-5, f"{key} {name}") + per_batch = ["start", "synchronize", "empty_cache", "end"] + self.assertEqual(events["gpu_b1"], per_batch * 2) + self.assertEqual(events["gpu_b2"], per_batch) + self.assertEqual(events["cpu_b2"], ["start", "end"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_name_conflicts.py b/tests/test_name_conflicts.py new file mode 100644 index 0000000..a305f33 --- /dev/null +++ b/tests/test_name_conflicts.py @@ -0,0 +1,1082 @@ +"""P9.N1: an inference run refuses names that conflict in its files. + +One run never writes two of its outputs to one file, never writes an output onto one of its own inputs, and never +writes outside its output folder: such a run is refused (RefusalError, exit status 1) before any model is built. Each +refusal is checked before any model (the model getters fail if called), with its message, next to the same command +without the conflict, which still runs: reruns into the same folders, the default names of complete-embedding in one +folder, assembly names such as 1acb-1 beside residue files, Parquet ids with '/'. Lists give every row a name (D5). + +ESM3 and the aggregator are never loaded: the commands run with stub models, as in tests/test_inference_outputs.py.""" +import contextlib +import gzip +import io +import json +import logging +import os +import re +import shutil +import sys +import tempfile +import time +import unicodedata +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import torch +from typer.testing import CliRunner + +from foldmatch.cli import inference +from foldmatch.errors import RefusalError +# Imported before any test patches chain_inference.predict, which assembly_inferece binds at its import +from foldmatch.inference import assembly_inferece, chain_inference, esm_inference # noqa: F401 +from foldmatch.types.api_types import Accelerator, OutFormat, SrcLocation, SrcProteinFrom, SrcTensorFrom + +_PDB = Path(__file__).parent / "resources" / "pdb" +_CIF, _CIF2 = _PDB / "1acb.cif", _PDB / "2uzi.cif" +_CPU = ("--accelerator", "cpu", "--devices", "1") +_LIB_CPU = {"accelerator": Accelerator.cpu, "devices": 1} +_FORMATS = ("csv", "pt", "parquet", "json") + + +def _esm3(model, protein): + """Stands in for ESM3: L + 2 rows (BOS and EOS, zeros), each residue's row its letter and its position.""" + rows = [[float(ord(aa)), float(k)] for k, aa in enumerate(protein.sequence)] + return torch.tensor([[0.0] * 2] + rows + [[0.0] * 2]) + + +def _aggregate(x, x_mask): + """Stands in for the aggregator: the sum of each item's rows.""" + return x.masked_fill(x_mask.unsqueeze(-1), 0.0).sum(dim=1) + + +def _invoke(*args): + """inference in-process, the logging setup restored after.""" + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(inference.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + +def _flat(text: str) -> str: + """Text on one line, typer's error panel borders and colour codes dropped.""" + return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", text).translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +def _ids(path: Path) -> list: + if path.suffix == ".parquet": + return pq.read_table(path).column("id").to_pylist() + with gzip.open(path, "rt") as f: + return [row["id"] for row in json.load(f)] + + +def _tree(folder: Path) -> dict: + """Every file under a folder, by relative path, with its bytes.""" + return {str(p.relative_to(folder)): p.read_bytes() for p in sorted(folder.rglob("*")) if p.is_file()} + + +def _case_insensitive() -> bool: + """Whether the temporary folder takes 'A' and 'a' for one name (macOS APFS by default), found out here without + FoldMatch, so that this module imports on trees without foldmatch.utils.output_files.""" + with tempfile.NamedTemporaryFile(prefix="CaseProbe-") as f: + head, name = os.path.split(f.name) + return os.path.exists(os.path.join(head, name.swapcase())) + + +def _output_files(): + """foldmatch.utils.output_files (P9.N), imported when a test needs it.""" + from foldmatch.utils import output_files + return output_files + + +_CASE_INSENSITIVE = _case_insensitive() + + +class _Case(unittest.TestCase): + + def setUp(self): + # Resolved, as typer resolves the paths the commands are given (/var is /private/var on macOS) + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())).resolve() + for module in ("esm_inference", "structure_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_residue_model", self.residue_model)) + for module in ("structure_inference", "chain_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_aggregator_model", self.aggregator)) + for module in ("esm_module", "chain_complete_module"): + self.enterContext(mock.patch(f"foldmatch.modules.{module}.per_residue_embedding", _esm3)) + self.models = 0 + self.refusing = False + + def residue_model(self, **_): + self.models += 1 + if self.refusing: + raise AssertionError("a model was built") + return "ESM3" + + def aggregator(self, **_): + self.models += 1 + if self.refusing: + raise AssertionError("a model was built") + return _aggregate + + def folder(self, name: str = "out") -> Path: + return Path(tempfile.mkdtemp(prefix=f"{name}-", dir=self.tmp)) + + def write(self, name: str, text: str) -> Path: + path = self.tmp / name + path.write_text(text) + return path + + def run_ok(self, *args): + self.refusing = False + result = _invoke(*args, *_CPU) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + return result + + def refused(self, *args, written=()) -> str: + """The command is refused (exit status 1, a RefusalError) before any model, and writes nothing in the folders + given: its message, on one line.""" + before = {folder: _tree(folder) for folder in written} + self.refusing = True + try: + result = _invoke(*args, *_CPU) + finally: + self.refusing = False + self.assertEqual(result.exit_code, 1, f"{result.output}\n{result.exception!r}") + self.assertIsInstance(result.exception, RefusalError, repr(result.exception)) + for folder, tree in before.items(): + self.assertEqual(_tree(folder), tree, folder) + return _flat(str(result.exception)) + + def residues(self, chains: Path, out_format: str = "pt") -> Path: + out = self.folder("residues") + self.run_ok("residue-embedding", "--src-file", chains, "--output-folder", out, "--output-format", out_format) + return out + + def chain_list(self, *rows) -> Path: + """A chain list of rows (structure name, chain ID, embedding name) of 1acb or 2uzi.""" + files = {"1acb": _CIF, "2uzi": _CIF2} + return self.write(f"chains-{len(list(self.tmp.iterdir()))}.csv", + "".join(f"{s},{files[s]},{c},{n}\n" for s, c, n in rows)) + + +class FileNameTest(_Case): + """D1: with csv and pt, an embedding name is a file name of the output folder; the output name is one in every + format.""" + + def test_a_name_with_a_slash_is_refused(self): + """'../victim' wrote outside the output folder (sweep case 10); 'Q9/1-100' stopped the run with a traceback. + In Parquet and JSON an id is a string in a file, so both keep working there.""" + victim = self.write("victim.pt", "precious") + chains = self.chain_list(("1acb", "A", "../victim"), ("1acb", "B", "Q9/1-100")) + for out_format in ("csv", "pt"): + with self.subTest(out_format=out_format): + out = self.folder() + message = self.refused("residue-embedding", "--src-file", chains, "--output-folder", out, + "--output-format", out_format, written=[self.tmp]) + self.assertIn(f"2 embedding name(s) that cannot name a file .{out_format} in the output folder: " + f"'../victim' (row 1) holds '/'; 'Q9/1-100' (row 2) holds '/'.", message) + self.assertEqual(victim.read_text(), "precious") + for out_format in ("parquet", "json"): + with self.subTest(out_format=out_format): + out = self.folder() + self.run_ok("residue-embedding", "--src-file", chains, "--output-folder", out, + "--output-format", out_format) + self.assertCountEqual(_ids(next(out.iterdir())), ["../victim", "Q9/1-100"]) + + def test_dot_names_and_long_names(self): + """'.', '..' and a name of more than 255 bytes with its suffix are refused; 251 bytes and '.csv' fit.""" + longest = "é" * 125 + "x" # 251 bytes in UTF-8 + for name, problem in ((".", "is '.'"), ("..", "is '..'"), (longest + "y", "makes a file name of 256 bytes")): + with self.subTest(name=name[:5]): + message = self.refused("residue-embedding", "--src-file", self.chain_list(("1acb", "A", name)), + "--output-folder", self.folder(), "--output-format", "csv") + self.assertIn(f"{name!r} (row 1) {problem}", message) + self.assertIn("of at most 251 bytes in UTF-8", message) + out = self.folder() + self.run_ok("residue-embedding", "--src-file", self.chain_list(("1acb", "A", longest)), "--output-folder", out, + "--output-format", "csv") + self.assertEqual([f.name for f in out.iterdir()], [f"{longest}.csv"]) + + def test_the_output_name_in_every_format(self): + """--output-name '../up' wrote up-0.parquet outside the folder; it is refused in every format, as it names + -.parquet and .json.gz; 'inference' and a name of 240 bytes run.""" + chains = self.chain_list(("1acb", "A", "1acb.A")) + for out_format in _FORMATS: + with self.subTest(out_format=out_format): + message = self.refused("residue-embedding", "--src-file", chains, "--output-folder", self.folder(), + "--output-format", out_format, "--output-name", "../up", written=[self.tmp]) + self.assertEqual(message, "The output name '../up' holds '/': it names the files " + "-.parquet or -.json.gz of the output folder. Give a " + "name that is not empty, without '/' or NUL, other than '.' and '..', of at " + "most 240 bytes in UTF-8.") + message = self.refused("structure-embedding", "--src-file", chains, "--output-folder", self.folder(), + "--output-name", "x" * 241) + self.assertIn(f"The output name {'x' * 241!r} makes a file name of 256 bytes once a rank and a suffix are " + f"added: it names the files", message) + out = self.folder() + self.run_ok("structure-embedding", "--src-file", chains, "--output-folder", out, "--output-name", "x" * 240) + self.assertEqual([f.name for f in out.iterdir()], ["x" * 240 + "-0.parquet"]) + + def test_an_output_name_that_is_empty_or_dots(self): + """'', '.' and '..' are refused as what they are, not as what they would be with a rank and a suffix (the file + would be '.-000000.parquet'), and the advice names the rule that refused ''.""" + chains = self.chain_list(("1acb", "A", "1acb.A")) + for name, problem in (("", "is empty"), (".", "is '.'"), ("..", "is '..'")): + with self.subTest(name=name): + message = self.refused("residue-embedding", "--src-file", chains, "--output-folder", self.folder(), + "--output-format", "parquet", "--output-name", name, written=[self.tmp]) + self.assertEqual(message, f"The output name {name!r} {problem}: it names the files " + f"-.parquet or -.json.gz of the output folder. Give " + f"a name that is not empty, without '/' or NUL, other than '.' and '..', of " + f"at most 240 bytes in UTF-8.") + + def test_integer_ids_name_their_files(self): + """chain_inference.predict on residue embeddings in Parquet whose id column is int64 writes 101.csv and 101.pt + as it did; an integer id is not empty.""" + ints = self.tmp / "ints.parquet" + pq.write_table(pa.table({"id": [101, 102], "embedding": [[[1.0, 2.0]], [[3.0, 4.0]]]}, + schema=pa.schema([("id", pa.int64()), + ("embedding", pa.list_(pa.list_(pa.float32(), 2)))])), ints) + for out_format in ("csv", "pt"): + with self.subTest(out_format=out_format): + out = self.folder() + chain_inference.predict(src_stream=ints, src_from=SrcTensorFrom.parquet, out_folder=out, + out_format=out_format, return_predictions=False, **_LIB_CPU) + self.assertEqual(sorted(f.name for f in out.iterdir()), [f"101.{out_format}", f"102.{out_format}"]) + + def test_the_rules(self): + """name_problem, NUL included, and its 255-byte limit.""" + name_problem = _output_files().name_problem + self.assertIsNone(name_problem("1acb.A", ".csv")) + self.assertIsNone(name_problem("a" * 251, ".csv")) + for name, problem in (("", "is empty"), (None, "is empty"), (float("nan"), "is empty"), (".", "is '.'"), + ("..", "is '..'"), ("a/b", "holds '/'"), ("a\0b", "holds a NUL character"), + ("a" * 252, "makes a file name of 256 bytes"), (pd.NA, "is empty"), + (np.float32("nan"), "is empty")): + with self.subTest(name=repr(name)[:8]): + self.assertEqual(name_problem(name, ".csv"), problem) + for name in (101, np.int64(102), 0): + with self.subTest(name=repr(name)): + self.assertIsNone(name_problem(name, ".csv")) + + +class RepeatedNameTest(_Case): + """D3: an embedding name given to more than one row is refused, in every format; with csv and pt, names that the + output folder takes for one file too.""" + + def test_residue_embedding(self): + """Two rows named x kept the last one's embedding (sweep case 4); in Parquet and JSON two rows of id x.""" + chains = self.chain_list(("1acb", "A", "x"), ("1acb", "B", "x")) + for out_format in _FORMATS: + with self.subTest(out_format=out_format): + message = self.refused("residue-embedding", "--src-file", chains, "--output-folder", self.folder(), + "--output-format", out_format) + self.assertEqual(message, f"{chains}: 1 embedding name(s) given to more than one row: 'x' (rows 1 and " + f"2). Each row's embedding is written under its name (the file .csv " + f"or .pt, or the id of its row in Parquet and JSON), so a repeated " + f"name keeps one of them: give each row its own name (a missing or empty " + f"name is .).") + out = self.folder() + self.run_ok("residue-embedding", "--src-file", self.chain_list(("1acb", "A", "x"), ("1acb", "B", "y")), + "--output-folder", out) + self.assertEqual(sorted(f.name for f in out.iterdir()), ["x.pt", "y.pt"]) + + def test_every_list_command(self): + """structure-embedding, chain-embedding and assembly-embedding, and complete-embedding's two lists; a default + name repeated, an explicit one (rows counted from 1, 5 shown).""" + residues = self.residues(self.chain_list(("1acb", "A", ""), ("1acb", "B", ""), ("2uzi", "A", ""), + ("2uzi", "B", ""), ("2uzi", "C", ""))) + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", ""), ("1acb", "A", "1acb.A")) + assemblies = self.write("assemblies.csv", f"1acb,{_CIF},1,\n2uzi,{_CIF2},1,1acb-1\n") + found = "1 embedding name(s) given to more than one row: '1acb.A' (rows 1 and 3)" + message = self.refused("structure-embedding", "--src-file", chains, "--output-folder", self.folder()) + self.assertIn(f"{chains}: {found}", message) + message = self.refused("chain-embedding", "--src-file", chains, "--res-embedding-location", residues, + "--output-folder", self.folder(), "--output-format", "parquet") + self.assertIn(f"{chains}: {found}", message) + message = self.refused("assembly-embedding", "--src-file", assemblies, "--res-embedding-location", residues, + "--output-folder", self.folder(), "--output-format", "json") + self.assertIn(f"{assemblies}: 1 embedding name(s) given to more than one row: '1acb-1' (rows 1 and 2)", message) + self.assertIn("(a missing or empty name is -)", message) + folders = [self.folder(name) for name in ("r", "c", "a")] + for chain_list, assembly_list, source in ((chains, self.write("one.csv", f"1acb,{_CIF},1,\n"), chains), + (self.chain_list(("1acb", "A", "")), assemblies, assemblies)): + with self.subTest(source=source.name): + message = self.refused("complete-embedding", "--src-chain-file", chain_list, "--src-assembly-file", + assembly_list, "--output-res-path", folders[0], "--output-chain-path", + folders[1], "--output-assembly-path", folders[2], written=folders) + self.assertIn(f"{source}: 1 embedding name(s) given to more than one row", message) + many = self.chain_list(*[("1acb", "A", f"n{k // 2}") for k in range(14)]) + message = self.refused("structure-embedding", "--src-file", many, "--output-folder", self.folder()) + self.assertIn("7 embedding name(s) given to more than one row: 'n0' (rows 1 and 2); 'n1' (rows 3 and 4); 'n2' " + "(rows 5 and 6); 'n3' (rows 7 and 8); 'n4' (rows 9 and 10); and 2 more names.", message) + self.run_ok("assembly-embedding", "--src-file", self.write("ok.csv", f"1acb,{_CIF},1,\n2uzi,{_CIF2},1,\n"), + "--res-embedding-location", residues, "--output-folder", self.folder()) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_names_that_differ_in_case_on_this_mac(self): + """x and X give one file x.csv on a case-insensitive folder (the second replaced the first); in Parquet and + JSON they are two ids, as on any file system; so are the NFC and NFD spellings of é.""" + nfc, nfd = "café", unicodedata.normalize("NFD", "café") + for first, second, apart in (("x", "X", "upper and lower case"), + (nfc, nfd, "Unicode spellings (NFC and NFD)")): + chains = self.chain_list(("1acb", "A", first), ("1acb", "B", second)) + for out_format in ("csv", "pt"): + with self.subTest(first=first, out_format=out_format): + message = self.refused("residue-embedding", "--src-file", chains, "--output-folder", + self.folder(), "--output-format", out_format) + self.assertEqual(f"{chains}: 1 file(s) of the output folder would each be written for more than " + f"one embedding name (the output folder does not tell {apart} apart): " + f"{first!r} (row 1) and {second!r} (row 2) give {first}.{out_format}. Each one " + f"would replace the one before, as the folder takes those names for one: write " + f"them to a folder of a file system that tells {apart} apart or as parquet or " + f"json (which keep each embedding name as given), or give those rows embedding " + f"names that differ otherwise.", message) + for out_format in ("parquet", "json"): + with self.subTest(first=first, out_format=out_format): + out = self.folder() + self.run_ok("residue-embedding", "--src-file", chains, "--output-folder", out, + "--output-format", out_format) + self.assertCountEqual(_ids(next(out.iterdir())), [first, second]) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_names_equal_under_full_case_folding_on_this_mac(self): + """APFS folds case fully: straße and strasse, ff1 and ff1 are one file there (chain A's embedding was replaced + by chain B's, exit status 0); refused with csv and pt. Parquet keeps both ids.""" + for first, second in (("straße", "strasse"), ("ff1", "ff1")): + chains = self.chain_list(("1acb", "A", first), ("1acb", "B", second)) + for out_format in ("csv", "pt"): + with self.subTest(first=first, out_format=out_format): + out = self.folder() + message = self.refused("residue-embedding", "--src-file", chains, "--output-folder", out, + "--output-format", out_format, written=[out]) + self.assertIn(f"{first!r} (row 1) and {second!r} (row 2) give {first}.{out_format}.", message) + out = self.folder() + self.run_ok("residue-embedding", "--src-file", chains, "--output-folder", out, "--output-format", "parquet") + self.assertCountEqual(_ids(next(out.iterdir())), [first, second]) + + def test_full_and_simple_case_folding(self): + """A folder that folds case fully (APFS) takes straße and strasse for one name; one that folds simply, one + character for one, keeps them (the probe's result faked); İ and i stay two names under both.""" + of = _output_files() + folder = self.folder() + for full in (False, True): + with self.subTest(full=full): + folding = of.Folding(case=True, unicode=True, full=full) + names = ["straße", "strasse", "ff", "FF", "İ", "i"] + repeats = of.OutputFiles(folder, folding).add_files(names, ".csv") + self.assertEqual([file for file, _ in repeats], ["straße.csv", "ff.csv"] if full else []) + with mock.patch("foldmatch.utils.output_files.probe_folding", return_value=folding): + if full: + with self.assertRaises(RefusalError): + of.check_outputs(folder, OutFormat.pt, None, names=["straße", "STRASSE"]) + else: + of.check_outputs(folder, OutFormat.pt, None, names=["straße", "STRASSE"]) + + def test_the_comparison_on_a_case_sensitive_folder(self): + """A folder that tells case apart (Linux; the probe's result faked here) keeps 'A' and 'a': lowercase chain IDs + are real. One that does not refuses them; NFC and NFD only where the folder takes them for one name.""" + of = _output_files() + EXACT, Folding, OutputFiles, name_key = of.EXACT, of.Folding, of.OutputFiles, of.name_key + folder = self.folder() + self.assertEqual(OutputFiles(folder, EXACT).add_files(["1abc.A", "1abc.a"], ".csv"), []) + with mock.patch("foldmatch.utils.output_files.probe_folding", return_value=EXACT): + outputs = of.check_outputs(folder, OutFormat.csv, None, names=["1abc.A", "1abc.a"]) + self.assertEqual(sorted(outputs.files), ["1abc.A.csv", "1abc.a.csv"]) + repeats = OutputFiles(folder, Folding(case=True)).add_files(["1abc.A", "1abc.a"], ".csv") + self.assertEqual(repeats, [("1abc.A.csv", [("1abc.A", None), ("1abc.a", None)])]) + nfd = unicodedata.normalize("NFD", "é") + self.assertEqual(OutputFiles(folder, Folding(case=True)).add_files(["é", nfd], ".pt"), []) + self.assertEqual(len(OutputFiles(folder, Folding(unicode=True)).add_files(["é", nfd], ".pt")), 1) + self.assertEqual(name_key("É", Folding(True, True)), "é") + with mock.patch("foldmatch.utils.output_files.probe_folding", return_value=Folding(case=True)): + with self.assertRaises(RefusalError): + of.check_outputs(folder, OutFormat.csv, None, names=["1abc.A", "1abc.a"]) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_the_probe_of_this_mac(self): + """The probe finds this folder case- and normalisation-insensitive, and leaves nothing behind.""" + of = _output_files() + folder = self.folder() + self.assertEqual(of.probe_folding(folder), of.Folding(case=True, unicode=True, full=True)) + self.assertEqual(list(folder.iterdir()), []) + self.assertEqual(of.probe_folding(self.tmp / "missing"), of.EXACT) + + def test_the_library(self): + """esm_inference.predict on a stream, chain_inference.predict on Parquet input with an id twice.""" + out = self.folder() + self.refusing = True + with self.assertRaises(RefusalError) as refused: + esm_inference.predict(src_stream=[("1acb", str(_CIF), "A", "x"), ("1acb", str(_CIF), "B", "x")], + src_location=SrcLocation.stream, out_folder=out, return_predictions=False, + **_LIB_CPU) + self.assertIn("The chain list stream: 1 embedding name(s) given to more than one row: 'x' (rows 1 and 2)", + str(refused.exception)) + twice = self.tmp / "twice.parquet" + pq.write_table(pa.table({"id": ["x", "y", "x"], "embedding": [[[1.0, 2.0]]] * 3}, + schema=pa.schema([("id", pa.string()), + ("embedding", pa.list_(pa.list_(pa.float32(), 2)))])), twice) + for out_format in _FORMATS: + with self.subTest(out_format=out_format): + with self.assertRaises(RefusalError) as refused: + chain_inference.predict(src_stream=twice, src_from=SrcTensorFrom.parquet, out_folder=out, + out_format=out_format, return_predictions=False, **_LIB_CPU) + self.assertIn("1 id(s) held by more than one row: 'x'.", str(refused.exception)) + self.assertEqual(self.models, 0) + self.assertEqual(list(out.iterdir()), []) + self.refusing = False + esm_inference.predict(src_stream=[("1acb", str(_CIF), "A", "x"), ("1acb", str(_CIF), "B", "y")], + src_location=SrcLocation.stream, out_folder=out, return_predictions=False, **_LIB_CPU) + self.assertEqual(sorted(f.name for f in out.iterdir()), ["x.csv", "y.csv"]) + + +class DefaultNameTest(_Case): + """D5: every list gives every row a name; NA is a name.""" + + def test_assembly_lists_without_names(self): + """Two assemblies without a 4th column were both written to nan.csv (sweep case 5), and refused by pyarrow in + Parquet (5b); they are now 1acb-1 and 2uzi-1, as fm-embedding names them. NA names NA.csv.""" + residues = self.residues(self.chain_list(("1acb", "A", ""), ("1acb", "B", ""), ("2uzi", "A", ""), + ("2uzi", "B", ""), ("2uzi", "C", ""))) + unnamed = self.write("unnamed.csv", f"1acb,{_CIF},1\n2uzi,{_CIF2},1,\n") + for out_format in _FORMATS: + with self.subTest(out_format=out_format): + out = self.folder() + self.run_ok("assembly-embedding", "--src-file", unnamed, "--res-embedding-location", residues, + "--output-folder", out, "--output-format", out_format) + files = sorted(f.name for f in out.iterdir()) + if out_format in ("csv", "pt"): + self.assertEqual(files, [f"1acb-1.{out_format}", f"2uzi-1.{out_format}"]) + else: + self.assertCountEqual(_ids(out / files[0]), ["1acb-1", "2uzi-1"]) + out = self.folder() + self.run_ok("assembly-embedding", "--src-file", self.write("na.csv", f"1acb,{_CIF},1,NA\n"), + "--res-embedding-location", residues, "--output-folder", out) + self.assertEqual([f.name for f in out.iterdir()], ["NA.csv"]) + message = self.refused("assembly-embedding", "--src-file", self.write("twice.csv", f"1acb,{_CIF},1\n1acb," + f"{_CIF},1,\n"), + "--res-embedding-location", residues, "--output-folder", self.folder()) + self.assertIn("'1acb-1' (rows 1 and 2)", message) + + def test_structure_lists_without_names(self): + """A structure list without its 3rd column gave every structure the hidden files .A.csv, .B.csv (the last + structure's); each now takes the structure name.""" + out = self.folder() + esm_inference.predict(src_stream=self.write("structures.csv", f"1acb,{_CIF}\n2uzi,{_CIF2},\n"), + src_from=SrcProteinFrom.structure, out_folder=out, return_predictions=False, + **_LIB_CPU) + self.assertEqual(sorted(f.name for f in out.iterdir()), + ["1acb.A.csv", "1acb.B.csv", "2uzi.A.csv", "2uzi.B.csv", "2uzi.C.csv"]) + self.refusing = True + with self.assertRaises(RefusalError) as refused: + esm_inference.predict(src_stream=self.write("same.csv", f"1acb,{_CIF},x\n2uzi,{_CIF2},x\n"), + src_from=SrcProteinFrom.structure, out_folder=self.folder(), + return_predictions=False, **_LIB_CPU) + self.assertIn("1 embedding name(s) given to more than one row: 'x' (rows 1 and 2)", str(refused.exception)) + self.assertIn("(a missing or empty name is the structure name)", str(refused.exception)) + + +class InputTest(_Case): + """D2: an output never replaces an input of its run.""" + + def test_chain_embedding_into_its_residue_folder(self): + """In the residue format, every residue embedding became a chain vector (sweep case 6): refused, naming both + options, also through a symbolic link to the folder; in another format the folder is shared as before.""" + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", "")) + for res_format, other in (("pt", "csv"), ("csv", "pt")): + residues = self.residues(chains, res_format) + link = self.tmp / f"link-{res_format}" + link.symlink_to(residues) + for folder in (residues, link): + with self.subTest(res_format=res_format, folder=folder.name): + message = self.refused("chain-embedding", "--src-file", chains, "--res-embedding-location", + residues, "--res-embedding-format", res_format, "--output-folder", folder, + "--output-format", res_format, written=[residues]) + self.assertEqual(message, f"--output-folder and --res-embedding-location are one folder " + f"({folder.resolve()}), and --output-format is --res-embedding-format " + f"({res_format}): each chain embedding .{res_format} " + f"would replace the residue embedding it is computed from. Give another " + f"--output-folder, or --output-format {other}, parquet or json.") + self.run_ok("chain-embedding", "--src-file", chains, "--res-embedding-location", residues, + "--res-embedding-format", res_format, "--output-folder", residues, "--output-format", other) + self.assertEqual(sorted(f.name for f in residues.iterdir()), + sorted(f"1acb.{c}.{fmt}" for c in "AB" for fmt in (res_format, other))) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_a_folder_spelled_in_another_case(self): + """On macOS, SHARED and shared are one folder: compared as files, not as strings.""" + shared = self.tmp / "shared" + shared.mkdir() + chains = self.chain_list(("1acb", "A", "")) + self.run_ok("residue-embedding", "--src-file", chains, "--output-folder", shared) + message = self.refused("chain-embedding", "--src-file", chains, "--res-embedding-location", shared, + "--output-folder", self.tmp / "SHARED", "--output-format", "pt") + self.assertIn("--output-folder and --res-embedding-location are one folder", message) + + def test_the_list_and_the_structure_files(self): + """A chain list in the output folder named like an output (x.csv, a row named x), and a structure file named + like one: refused; a list named otherwise there runs.""" + out = self.folder() + listed = out / "x.csv" + listed.write_text(f"1acb,{_CIF},A,x\n") + message = self.refused("residue-embedding", "--src-file", listed, "--output-folder", out, "--output-format", + "csv", written=[out]) + self.assertEqual(message, f"1 input(s) of this run would be replaced by its output, in the output folder " + f"{out}: the chain list {listed} by the output 'x'. An output never replaces an " + f"input of its run: write the output to another folder or under other names, or " + f"move the input.") + structure = Path(shutil.copyfile(_CIF, out / "s.A.pt")) + message = self.refused("residue-embedding", "--src-file", self.write("s.csv", f"s,{structure},A,s.A\n"), + "--output-folder", out, "--output-format", "pt", written=[out]) + self.assertIn(f"the structure file of row 1 {structure} by the output 's.A'", message) + message = self.refused("chain-embedding", "--src-file", listed, "--res-embedding-location", self.folder(), + "--output-folder", out, written=[out]) + self.assertIn(f"the chain list {listed} by the output 'x'", message) + listed.rename(out / "list.csv") + self.run_ok("residue-embedding", "--src-file", out / "list.csv", "--output-folder", out, "--output-format", + "csv") + self.assertEqual(sorted(f.name for f in out.iterdir()), ["list.csv", "s.A.pt", "x.csv"]) + + def test_assembly_names_beside_residue_files(self): + """An assembly named .<...>, written as pt to the pt residue folder, could replace a residue + embedding: refused. 1acb-1 there runs, and so does 1acb.x in csv.""" + residues = self.residues(self.chain_list(("1acb", "A", ""), ("1acb", "B", ""))) + dotted = self.write("dotted.csv", f"1acb,{_CIF},1,1acb.A\n") + message = self.refused("assembly-embedding", "--src-file", dotted, "--res-embedding-location", residues, + "--output-folder", residues, "--output-format", "pt", written=[residues]) + self.assertIn(f"{dotted}: 1 assembly embedding name(s) of the form .<...>, written as pt files " + f"to the folder of the residue embeddings ({residues}): '1acb.A' (row 1).", message) + self.run_ok("assembly-embedding", "--src-file", dotted, "--res-embedding-location", residues, + "--output-folder", residues, "--output-format", "csv") + self.run_ok("assembly-embedding", "--src-file", self.write("plain.csv", f"1acb,{_CIF},1,\n"), + "--res-embedding-location", residues, "--output-folder", residues, "--output-format", "pt") + self.assertEqual(sorted(f.name for f in residues.iterdir()), + ["1acb-1.pt", "1acb.A.csv", "1acb.A.pt", "1acb.B.pt"]) + + def test_parquet_input_in_the_output_folder(self): + """chain_inference.predict(src_from=parquet) on /inference-0.parquet into F, default name: its output would + replace its input; another name runs.""" + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", "")) + folder = self.residues(chains, "parquet") + before = _tree(folder) + self.refusing = True + with self.assertRaises(RefusalError) as refused: + chain_inference.predict(src_stream=folder / "inference-0.parquet", src_from=SrcTensorFrom.parquet, + out_folder=folder, out_format=OutFormat.parquet, return_predictions=False, + **_LIB_CPU) + self.assertIn(f"the residue embedding Parquet file {folder / 'inference-0.parquet'} by a file " + f"-.parquet of the output name 'inference'", str(refused.exception)) + self.assertEqual(_tree(folder), before) + self.refusing = False + chain_inference.predict(src_stream=folder / "inference-0.parquet", src_from=SrcTensorFrom.parquet, + out_folder=folder, out_format=OutFormat.parquet, out_name="chains", + return_predictions=False, **_LIB_CPU) + self.assertCountEqual(_ids(folder / "chains-0.parquet"), ["1acb.A", "1acb.B"]) + + +class CompleteEmbeddingTest(_Case): + """complete-embedding checks its three steps before its residue step, and refuses two that would write one file.""" + + def complete(self, folders, out_format: str, *names, refused: bool = False): + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", "")) + assemblies = self.write("assemblies.csv", f"1acb,{_CIF},1,\n") + args = ("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", assemblies, + "--output-res-path", folders[0], "--output-chain-path", folders[1], "--output-assembly-path", + folders[2], "--output-format", out_format, *names) + return self.refused(*args, written=set(folders)) if refused else self.run_ok(*args) + + def test_residues_and_chains_in_one_folder_as_pt(self): + """The chain step replaced the residue files, and the assembly step failed (sweep case 3): refused before the + residue step. In csv the folder is shared as before, and a rerun into the same folders runs.""" + shared = self.folder("shared") + message = self.complete((shared, shared, self.folder("a")), "pt", refused=True) + self.assertEqual(message, f"--output-res-path and --output-chain-path are one folder ({shared}), in which " + f"complete-embedding's residue and chain embeddings would share 2 file(s): " + f"1acb.A.pt; 1acb.B.pt. Each step would replace the other's files: give the steps " + f"different folders, or an --output-format other than pt (residue embeddings are pt " + f"files).") + for _ in range(2): + self.complete((shared, shared, shared), "csv") + self.assertEqual(sorted(f.name for f in shared.iterdir()), + ["1acb-1.csv", "1acb.A.csv", "1acb.A.pt", "1acb.B.csv", "1acb.B.pt"]) + + def test_one_name_for_chains_and_assemblies(self): + """The recipe of P9.H2's CHANGELOG, --output-assembly-name chain-inference with one folder for chains and + assemblies, lost the chain file (sweep case 1), and so did any one name (case 2): refused, in parquet and json. + The default names in one folder run, in every format, and so do all three steps in one folder but in pt.""" + for out_format in ("parquet", "json"): + for names in (("--output-assembly-name", "chain-inference"), + ("--output-chain-name", "X", "--output-assembly-name", "X")): + with self.subTest(out_format=out_format, names=names): + shared = self.folder("shared") + message = self.complete((self.folder("r"), shared, shared), out_format, *names, refused=True) + self.assertIn(f"--output-chain-path and --output-assembly-path are one folder ({shared}), in which " + f"complete-embedding's chain and assembly embeddings would share 1 file(s): " + f"{names[-1]}-", message) + for out_format in _FORMATS: + with self.subTest(out_format=out_format, defaults=True): + shared = self.folder("shared") + for _ in range(2): + self.complete((self.folder("r"), shared, shared), out_format) + self.assertEqual(len(list(shared.iterdir())), 3 if out_format in ("csv", "pt") else 2) + if out_format != "pt": + everything = self.folder("everything") + self.complete((everything, everything, everything), out_format) + + def test_each_step_is_checked_first(self): + """An assembly named 1acb.A beside pt residues, and an --output-chain-name that cannot name a file: refused + before the residue step, naming the option.""" + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", "")) + shared = self.folder("shared") + message = self.refused("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", + self.write("dotted.csv", f"1acb,{_CIF},1,1acb.A\n"), "--output-res-path", shared, + "--output-chain-path", self.folder("c"), "--output-assembly-path", shared, + "--output-format", "pt", written=[shared]) + self.assertIn("1 assembly embedding name(s) of the form .<...>", message) + message = self.complete((self.folder("r"), self.folder("c"), self.folder("a")), "parquet", + "--output-chain-name", "a/b", refused=True) + self.assertTrue(message.startswith("--output-chain-name 'a/b' holds '/'"), message) + + +class CompleteEmbeddingInputTest(_Case): + """complete-embedding: no step's output replaces an input of the run, whichever step reads it (both lists and the + structure files of both), and an assembly list that can be read once (/dev/fd/N, a pipe) gets every check.""" + + def complete(self, chains: Path, assemblies, folders, out_format: str, *names, refused: bool = False, + kept=()): + before = {path: path.read_bytes() for path in kept} + args = ("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", assemblies, + "--output-res-path", folders[0], "--output-chain-path", folders[1], "--output-assembly-path", + folders[2], "--output-format", out_format, *names) + result = self.refused(*args) if refused else self.run_ok(*args) + for path, content in before.items(): + self.assertEqual(path.read_bytes(), content, path) + return result + + def chains(self, path: Path = None) -> Path: + path = path or self.tmp / "chains.csv" + path.write_text(f"1acb,{_CIF},A,\n1acb,{_CIF},B,\n") + return path + + def test_a_step_over_the_other_list(self): + """The assembly embedding x.csv over the chain list ASM/x.csv (exit status 0, the list lost); the chain + embedding 1acb.A.csv over the assembly list CHAIN/1acb.A.csv, and the residue embedding 1acb.A.pt over the + assembly list RES/1acb.A.pt (each read later by the assembly step, which failed with a traceback): refused + before any model, the lists unchanged. The same lists under other names run.""" + folders = [self.folder(name) for name in ("r", "c", "a")] + chains = self.chains(folders[2] / "x.csv") + named_x = self.write("x-assemblies.csv", f"1acb,{_CIF},1,x\n") + message = self.complete(chains, named_x, folders, "csv", refused=True, kept=[chains]) + self.assertIn(f"the chain list {chains} by the output 'x'", message) + for k, suffix in ((1, "csv"), (0, "pt")): + assemblies = folders[k] / f"1acb.A.{suffix}" + assemblies.write_text(f"1acb,{_CIF},1,\n") + with self.subTest(step=k): + message = self.complete(self.chains(), assemblies, folders, "csv", refused=True, kept=[assemblies]) + self.assertIn(f"the assembly list {assemblies} by the output '1acb.A'", message) + assemblies.rename(folders[k] / "assemblies.txt") + self.complete(self.chains(), folders[k] / "assemblies.txt", folders, "csv") + self.assertEqual(sorted(f.name for f in folders[2].iterdir()), ["1acb-1.csv", "x.csv"]) + + def test_a_structure_file_of_the_chain_list_under_an_assembly_name(self): + """A structure file of the chain list, read by the residue step, in the assembly folder under the name of the + assembly embedding: refused, naming the list.""" + folders = [self.folder(name) for name in ("r", "c", "a")] + structure = Path(shutil.copyfile(_CIF, folders[2] / "1acb-1.csv")) + chains = self.write("chains-structure.csv", f"1acb,{structure},A,\n1acb,{structure},B,\n") + message = self.complete(chains, self.write("assemblies.csv", f"1acb,{_CIF},1,\n"), folders, "csv", + refused=True, kept=[structure]) + self.assertIn(f"the structure file of row 1 of the chain list {structure} by the output '1acb-1'", message) + + @unittest.skipUnless(os.path.isdir("/dev/fd"), "/dev/fd/N needs /dev/fd") + def test_an_assembly_list_as_dev_fd(self): + """An assembly list given as /dev/fd/N, with one folder for chains and assemblies: one name for both in + Parquet or JSON, and an assembly named 1acb.A in csv or pt, replaced the chain embeddings with exit status 0. + Refused as with the list in a regular file; the default names run, and the assembly is written.""" + cases = [(fmt, ("--output-chain-name", "X", "--output-assembly-name", "X"), "1acb-1", "X-") + for fmt in ("parquet", "json")] + cases += [(fmt, (), "1acb.A", f"1acb.A.{fmt}") for fmt in ("csv", "pt")] + for out_format, names, assembly, shared_file in cases: + with self.subTest(out_format=out_format): + listed = self.write(f"assemblies-{out_format}.csv", f"1acb,{_CIF},1,{assembly}\n") + folders = [self.folder("r")] + [self.folder("shared")] * 2 + fd = os.open(listed, os.O_RDONLY) + try: + message = self.complete(self.chains(), f"/dev/fd/{fd}", folders, out_format, *names, refused=True) + finally: + os.close(fd) + self.assertIn(f"--output-chain-path and --output-assembly-path are one folder ({folders[1]}), in " + f"which complete-embedding's chain and assembly embeddings would share 1 file(s): " + f"{shared_file}", message) + self.assertEqual(list(folders[0].iterdir()), []) + listed = self.write("assemblies-ok.csv", f"1acb,{_CIF},1,\n") + folders = [self.folder("r")] + [self.folder("shared")] * 2 + fd = os.open(listed, os.O_RDONLY) + try: + self.complete(self.chains(), f"/dev/fd/{fd}", folders, "csv") + finally: + os.close(fd) + self.assertEqual(sorted(f.name for f in folders[1].iterdir()), ["1acb-1.csv", "1acb.A.csv", "1acb.B.csv"]) + + @unittest.skipUnless(os.path.isdir("/dev/fd"), "/dev/fd/N needs /dev/fd") + def test_a_long_row_in_an_assembly_list_as_dev_fd(self): + """A row of more than 4 fields in an assembly list given as /dev/fd/N is refused before the residue step too, + naming the path given, also where os.path.realpath resolves /dev/fd/N to the file the descriptor is open on + (Linux; emulated elsewhere).""" + listed = self.write("long.csv", f"1acb,{_CIF},1,1acb-1,extra\n") + realpath = os.path.realpath + for linux in (False, True): + with self.subTest(linux=linux): + folders = [self.folder(name) for name in ("r", "c", "a")] + fd = os.open(listed, os.O_RDONLY) + given = f"/dev/fd/{fd}" + as_linux = (lambda path, *args, **kwargs: os.fspath(listed) if os.fspath(path) == given + else realpath(path, *args, **kwargs)) + try: + with mock.patch("os.path.realpath", as_linux) if linux else contextlib.nullcontext(): + message = self.complete(self.chains(), given, folders, "csv", refused=True) + finally: + os.close(fd) + self.assertTrue(message.startswith(f"{given}: "), message) + self.assertEqual(list(folders[0].iterdir()), []) + + +class PrefixTest(_Case): + """The files . and - of a structure list, whose IDs are + known only once the structure is read: looked up per name, and prefixes that the folder takes for one refused.""" + + def test_structures_in_the_output_folder_scale(self): + """N structure files in the output folder were checked against N prefixes each, N^2 steps (213 s for 50,000 + names); a lookup per separator now: 20,000 names in well under a second.""" + of = _output_files() + folder = self.folder() + names = [f"s{k}" for k in range(20000)] + inputs = [("the structure file", folder / f"{name}.cif") for name in names] + start = time.perf_counter() + outputs = of.check_outputs(folder, OutFormat.pt, None, prefixes=names, inputs=inputs, + folding=of.Folding(True, True, True)) + self.assertLess(time.perf_counter() - start, 5.0) + self.assertEqual(outputs.holds("S7.A.pt"), "a file s7..pt of the output 's7'") + self.assertEqual(outputs.holds("s7.b.c.pt"), "a file s7..pt of the output 's7'") + for name in ("s7.pt", "s7..pt", ".A.pt", "s7.A.csv", "x7.A.pt", "s20000.A.pt"): + self.assertIsNone(outputs.holds(name), name) + nested = of.OutputFiles(folder, of.EXACT) + nested.add_prefixes(["1abc.v2", "1abc"], "-", ".pt") + self.assertIn("output '1abc'", nested.holds("1abc-1.pt")) + self.assertIn("output '1abc.v2'", nested.holds("1abc.v2-1.pt")) + + def test_prefixes_that_differ_in_case(self): + """Structures abc and ABC both wrote abc.A.pt on APFS, after the models, stopped by the writer only within one + process: refused before work where the folder takes them for one; a case-sensitive folder keeps both.""" + of = _output_files() + folder = self.folder() + with self.assertRaises(RefusalError) as refused: + of.check_outputs(folder, OutFormat.pt, None, prefixes=["abc", "x", "ABC"], separator="-", + labels=["row 1", "row 2", "row 3"], folding=of.Folding(case=True), source="list.csv") + self.assertEqual(str(refused.exception), + "list.csv: 1 file(s) of the output folder would each be written for more than one embedding " + "name (the output folder does not tell upper and lower case apart): 'abc' (row 1) and 'ABC' " + "(row 3) give abc-.pt. Each one would replace the one before, as the folder takes those " + "names for one: write them to a folder of a file system that tells upper and lower case apart " + "or as parquet or json (which keep each embedding name as given), or give those rows " + "embedding names that differ otherwise.") + of.check_outputs(folder, OutFormat.pt, None, prefixes=["abc", "ABC"], folding=of.EXACT) + of.check_outputs(folder, OutFormat.parquet, None, prefixes=["abc", "ABC"], folding=of.Folding(case=True)) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_structure_names_that_differ_in_case_on_this_mac(self): + """esm_inference.predict on a structure list of abc and ABC, as pt: refused before any model.""" + rows = [("abc", str(_CIF), "abc"), ("ABC", str(_CIF2), "ABC")] + out = self.folder() + self.refusing = True + with self.assertRaises(RefusalError) as refused: + esm_inference.predict(src_stream=rows, src_location=SrcLocation.stream, src_from=SrcProteinFrom.structure, + out_folder=out, out_format=OutFormat.pt, return_predictions=False, **_LIB_CPU) + self.assertIn("'abc' (row 1) and 'ABC' (row 2) give abc..pt", str(refused.exception)) + self.assertEqual((self.models, list(out.iterdir())), (0, [])) + self.refusing = False + esm_inference.predict(src_stream=rows, src_location=SrcLocation.stream, src_from=SrcProteinFrom.structure, + out_folder=out, out_format=OutFormat.parquet, return_predictions=False, **_LIB_CPU) + + def test_ranks_in_ascii_digits(self): + """The writers spell the rank in ASCII digits: emb-٣.parquet (Arabic-Indic 3) is no output's file.""" + of = _output_files() + outputs = of.OutputFiles(self.folder(), of.Folding(True, True, True)) + outputs.add_stem("emb", ".parquet") + self.assertIsNotNone(outputs.holds("emb-3.parquet")) + for digit in ("٣", "3", "𝟑", "๓"): + self.assertIsNone(outputs.holds(f"emb-{digit}.parquet"), digit) + + +class FoldedNameTest(_Case): + """Names that the folder takes for one: compared as the folder compares them, and the refusal says what keeps + them apart.""" + + def test_canonical_caseless_keys(self): + """APFS takes 'ᾷ' and 'ᾼ' with U+0342 for one name; case folding before decomposing missed it.""" + of = _output_files() + folding = of.Folding(True, True, True) + for first, second in (("xᾷ", "xᾼ͂"), ("xῇ", "xῌ͂"), ("xῷ", "xῼ͂")): + with self.subTest(first=first): + self.assertEqual(of.name_key(first, folding), of.name_key(second, folding)) + self.assertNotEqual(of.name_key(first, of.EXACT), of.name_key(second, of.EXACT)) + with self.assertRaises(RefusalError): + of.check_outputs(self.folder(), OutFormat.pt, None, names=[first, second], folding=folding) + self.assertEqual(of.name_key("STRAẞE", folding), of.name_key("strasse", folding)) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_complete_embedding_chains_a_and_a_on_this_mac(self): + """complete-embedding names each chain ., so its chains A and a of one structure + cannot be renamed, and its residue step writes pt: the refusal names the folder as what keeps them apart.""" + chains = self.chain_list(("1acb", "A", ""), ("1acb", "a", "")) + folders = [self.folder(name) for name in ("r", "c", "a")] + message = self.refused("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", + self.write("asm.csv", f"1acb,{_CIF},1\n"), "--output-res-path", folders[0], + "--output-chain-path", folders[1], "--output-assembly-path", folders[2], + "--output-format", "parquet", written=folders) + self.assertEqual(message, f"{chains}: 1 file(s) of the output folder would each be written for more than one " + f"embedding name (the output folder does not tell upper and lower case apart): " + f"'1acb.A' (row 1) and '1acb.a' (row 2) give 1acb.A.pt. Each one would replace the " + f"one before, as the folder takes those names for one: give --output-res-path a " + f"folder of a file system that tells upper and lower case apart (complete-embedding " + f"names each chain ., and its residue step writes pt " + f"files).") + + def test_complete_embedding_chain_step(self): + """The residue folder tells case apart, the chain folder does not (the probe's results faked): the chain step + is refused with csv, naming its folder and Parquet or JSON.""" + of = _output_files() + chains = self.chain_list(("1acb", "A", ""), ("1acb", "a", "")) + folders = [self.folder(name) for name in ("r", "c", "a")] + + def probe(folder): + return of.EXACT if of.same_folder(folder, folders[0]) else of.Folding(case=True) + with mock.patch("foldmatch.utils.output_files.probe_folding", side_effect=probe): + message = self.refused("complete-embedding", "--src-chain-file", chains, "--src-assembly-file", + self.write("asm.csv", f"1acb,{_CIF},1\n"), "--output-res-path", folders[0], + "--output-chain-path", folders[1], "--output-assembly-path", folders[2], + "--output-format", "csv", written=folders) + self.assertTrue(message.endswith("as the folder takes those names for one: give --output-chain-path a folder " + "of a file system that tells upper and lower case apart, or use " + "--output-format parquet or json (complete-embedding names each chain " + ".)."), message) + self.assertIn("'1acb.A' (row 1) and '1acb.a' (row 2) give 1acb.A.csv", message) + + +class LinkedInputTest(_Case): + """D2: an input reached through a symbolic or a hard link from another folder is compared as a file.""" + + def stage(self, residues: Path) -> Path: + """A folder of links to the residue embeddings: 1acb.A.pt a symbolic link, 1acb.B.pt a hard link.""" + stage = self.folder("stage") + (stage / "1acb.A.pt").symlink_to(residues / "1acb.A.pt") + os.link(residues / "1acb.B.pt", stage / "1acb.B.pt") + return stage + + def test_chain_embedding_over_linked_residues(self): + """chain-embedding on links to residue embeddings in its output folder replaced each by its chain vector: + refused, naming the file linked; into another folder, or in another format, it runs.""" + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", "")) + residues = self.residues(chains) + stage = self.stage(residues) + message = self.refused("chain-embedding", "--src-file", chains, "--res-embedding-location", stage, + "--output-folder", residues, "--output-format", "pt", written=[residues]) + self.assertIn(f"2 input(s) of this run would be replaced by its output, in the output folder {residues}: the " + f"residue embedding file of row 1 {stage}/1acb.A.pt (which is {residues}/1acb.A.pt) by the " + f"output '1acb.A'; the residue embedding file of row 2 {stage}/1acb.B.pt (which is " + f"{residues}/1acb.B.pt) by the output '1acb.B'.", message) + for out_folder, out_format in ((self.folder(), "pt"), (residues, "csv")): + self.run_ok("chain-embedding", "--src-file", chains, "--res-embedding-location", stage, + "--output-folder", out_folder, "--output-format", out_format) + self.assertEqual(torch.load(residues / "1acb.A.pt").dim(), 2) + + def test_assembly_embedding_over_linked_residues(self): + """An assembly named 1acb.A, written as pt to the folder whose residue embeddings are linked from the residue + folder given: refused; 1acb-1 runs.""" + residues = self.residues(self.chain_list(("1acb", "A", ""), ("1acb", "B", ""))) + stage = self.stage(residues) + dotted = self.write("dotted.csv", f"1acb,{_CIF},1,1acb.A\n") + message = self.refused("assembly-embedding", "--src-file", dotted, "--res-embedding-location", stage, + "--output-folder", residues, "--output-format", "pt", written=[residues]) + self.assertIn(f"the residue embedding file {stage}/1acb.A.pt (which is {residues}/1acb.A.pt) by the output " + f"'1acb.A'", message) + self.run_ok("assembly-embedding", "--src-file", self.write("plain.csv", f"1acb,{_CIF},1,\n"), + "--res-embedding-location", stage, "--output-folder", residues, "--output-format", "pt") + + def test_the_check(self): + """check_outputs with a symbolic link, a hard link, and a file of another name linked.""" + of = _output_files() + out, stage = self.folder(), self.folder("stage") + (out / "y.pt").write_bytes(b"y") + (stage / "x.pt").symlink_to(out / "y.pt") + os.link(out / "y.pt", stage / "hard.pt") + (stage / "other.pt").write_bytes(b"o") + os.link(stage / "other.pt", stage / "other-too.pt") + for name in ("x.pt", "hard.pt"): + with self.subTest(name=name): + with self.assertRaises(RefusalError) as refused: + of.check_outputs(out, OutFormat.pt, None, names=["y"], inputs=[("the input", stage / name)]) + self.assertIn(f"the input {stage / name} (which is {out / 'y.pt'}) by the output 'y'", + str(refused.exception)) + of.check_outputs(out, OutFormat.pt, None, names=["z"], inputs=[("the input", stage / "x.pt")]) + of.check_outputs(out, OutFormat.pt, None, names=["y"], inputs=[("the input", stage / "other.pt")]) + of.check_outputs(self.folder(), OutFormat.pt, None, names=["y"], inputs=[("the input", stage / "x.pt")]) + + def test_an_output_file_linked_to_an_input(self): + """An output's file that exists as a symbolic link to an input of another folder (out/x.pt to in/x.pt) is that + input, which the writer, following the link, replaced: refused; a link to a file that is no input is not.""" + of = _output_files() + out, inputs, other = self.folder(), self.folder("in"), self.folder("other") + (inputs / "x.pt").write_bytes(b"x") + (out / "x.pt").symlink_to(inputs / "x.pt") + with self.assertRaises(RefusalError) as refused: + of.check_outputs(out, OutFormat.pt, None, names=["x"], inputs=[("the input", inputs / "x.pt")]) + self.assertIn(f"the input {inputs / 'x.pt'} (which is {out / 'x.pt'}) by the output 'x'", + str(refused.exception)) + (other / "y.pt").write_bytes(b"y") + unrelated = self.folder() + (unrelated / "y.pt").symlink_to(other / "y.pt") + of.check_outputs(unrelated, OutFormat.pt, None, names=["y"], inputs=[("the input", inputs / "x.pt")]) + + def test_no_link_among_the_output_files(self): + """Where no output's file that exists is a link, an input that is not a link is not looked up beyond its + lstat: one lstat per output file and per input, and a stat per folder.""" + of = _output_files() + out, inputs, n = self.folder(), self.folder("in"), 200 + for k in range(n): + (out / f"s{k}.pt").write_bytes(b"o") + (inputs / f"s{k}.cif").write_bytes(b"i") + folding = of.probe_folding(out) + with mock.patch.object(of.os, "stat", wraps=os.stat) as stat, \ + mock.patch.object(of.os, "lstat", wraps=os.lstat) as lstat: + of.check_outputs(out, OutFormat.pt, None, names=[f"s{k}" for k in range(n)], folding=folding, + inputs=[("the structure file", inputs / f"s{k}.cif") for k in range(n)]) + self.assertLessEqual(stat.call_count + lstat.call_count, 2 * n + 4) + + def test_chain_embedding_into_links_to_its_residues(self): + """chain-embedding into a folder whose 1acb.A.pt is a symbolic link to the residue embedding it reads + replaced that residue embedding by a chain vector (exit status 0): refused, naming the link; a link to a file + that is no input of the run is replaced through the link, as before.""" + chains = self.chain_list(("1acb", "A", ""), ("1acb", "B", "")) + residues = self.residues(chains) + out = self.folder() + (out / "1acb.A.pt").symlink_to(residues / "1acb.A.pt") + message = self.refused("chain-embedding", "--src-file", chains, "--res-embedding-location", residues, + "--output-folder", out, "--output-format", "pt", written=[residues, out]) + self.assertIn(f"1 input(s) of this run would be replaced by its output, in the output folder {out}: the " + f"residue embedding file of row 1 {residues}/1acb.A.pt (which is {out}/1acb.A.pt) by the " + f"output '1acb.A'.", message) + other, out = self.folder("other"), self.folder() + torch.save(torch.zeros(3), other / "1acb.A.pt") + (out / "1acb.A.pt").symlink_to(other / "1acb.A.pt") + before = _tree(residues) + self.run_ok("chain-embedding", "--src-file", chains, "--res-embedding-location", residues, + "--output-folder", out, "--output-format", "pt") + fresh = self.folder() + self.run_ok("chain-embedding", "--src-file", chains, "--res-embedding-location", residues, + "--output-folder", fresh, "--output-format", "pt") + self.assertTrue((out / "1acb.A.pt").is_symlink()) + self.assertEqual(torch.load(other / "1acb.A.pt").tolist(), torch.load(fresh / "1acb.A.pt").tolist()) + self.assertEqual(_tree(residues), before) + + +class ReadOnlyFolderTest(_Case): + """A folder in which no file can be created, though its files can be written, is probed on the names it holds.""" + + def read_only(self, *files) -> Path: + folder = self.folder() + for name in files: + (folder / name).write_text("old\n") + folder.chmod(0o555) + self.addCleanup(folder.chmod, 0o755) + if os.access(folder, os.W_OK): + self.skipTest("the folder stays writable (root)") + return folder + + def test_the_probe(self): + """As the probe of a folder of the same file system that can be written.""" + of = _output_files() + self.assertEqual(of.probe_folding(self.read_only("x.csv")), of.probe_folding(self.folder())) + + def test_a_sibling_that_links_to_the_folder(self): + """On a case-sensitive file system, a sibling OUT that links to the folder out is not out in another case: the + folder compares exactly, its entries probed first, and A and a are two files. Simulated here: the parent lists + the sibling, and only the sibling names the folder.""" + of = _output_files() + listdir = os.listdir + for files in (("x.csv",), ()): + folder = self.read_only(*files) + alias = str(folder.parent / of._swap_case(folder.name)) + + def names_it(spelling, path, folder=folder, alias=alias): + return os.fspath(spelling) == os.fspath(path) or (os.fspath(spelling) == alias and + os.fspath(path) == str(folder)) + + def listed(path=".", folder=folder, alias=alias): + entries = listdir(path) + return entries + [os.path.basename(alias)] if os.fspath(path) == str(folder.parent) else entries + + with self.subTest(files=files), mock.patch.object(of, "_names_it", names_it), \ + mock.patch.object(of.os, "listdir", listed): + self.assertEqual(of.probe_folding(folder), of.EXACT) + of.check_outputs(folder, OutFormat.csv, None, names=["A", "a"]) + + @unittest.skipIf(_CASE_INSENSITIVE, "a case-sensitive file system") + def test_a_sibling_that_links_to_the_folder_here(self): + """As test_a_sibling_that_links_to_the_folder, on this case-sensitive file system.""" + of = _output_files() + for files in (("x.csv",), ()): + folder = self.read_only(*files) + alias = folder.parent / of._swap_case(folder.name) + alias.symlink_to(folder) + with self.subTest(files=files): + self.assertEqual(of.probe_folding(folder), of.EXACT) + of.check_outputs(folder, OutFormat.csv, None, names=["A", "a"]) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_names_that_differ_in_case_on_this_mac(self): + """X and x both went to the earlier run's x.csv, the second kept (exit status 0): refused. A rerun of x + runs.""" + out = self.read_only("x.csv") + message = self.refused("residue-embedding", "--src-file", self.chain_list(("1acb", "A", "X"), + ("1acb", "B", "x")), + "--output-folder", out, "--output-format", "csv", written=[out]) + self.assertIn("'X' (row 1) and 'x' (row 2) give X.csv", message) + self.run_ok("residue-embedding", "--src-file", self.chain_list(("1acb", "B", "x")), "--output-folder", out, + "--output-format", "csv") + self.assertEqual([f.name for f in out.iterdir()], ["x.csv"]) + + +class ErrorLineTest(_Case): + """The refusal reaches the shell as one 'Error:' line, exit status 1, no traceback.""" + + def test_main(self): + chains = self.chain_list(("1acb", "A", "x"), ("1acb", "B", "x")) + stderr = io.StringIO() + self.refusing = True + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + with mock.patch.object(sys, "argv", ["inference", "residue-embedding", "--src-file", str(chains), + "--output-folder", str(self.folder())]), \ + contextlib.redirect_stderr(stderr), self.assertRaises(SystemExit) as exit_status: + inference.main() + finally: + root.handlers[:] = handlers + root.setLevel(level) + self.assertEqual(exit_status.exception.code, 1) + lines = re.sub(r"\x1b\[[0-9;]*m", "", stderr.getvalue()).splitlines() + self.assertEqual(len(lines), 1, lines) + self.assertTrue(lines[0].startswith(f"Error: {chains}: 1 embedding name(s) given to more than one row")) + self.assertEqual(self.models, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_name_conflicts_embedding.py b/tests/test_name_conflicts_embedding.py new file mode 100644 index 0000000..769c0cc --- /dev/null +++ b/tests/test_name_conflicts_embedding.py @@ -0,0 +1,334 @@ +"""P9.N2: fm-embedding refuses names that conflict in its files. + +from-sequences refuses, before ESM3, a FASTA id repeated (every format), and with csv and pt an id that cannot name a +file of the output folder or that the folder takes for another one, and a FASTA file among its own output files; the +output name follows the file name rules in every format (from-structures too). CsvBatchWriter and TensorBatchWriter +refuse a second write of one file in a run (D4), the backstop for the chains of structures read whole, whose names are +known only once a structure is read (chains A and a are one file on a case-insensitive folder). + +ESM3 and the aggregator are never loaded: the commands run with stub models (as in tests/test_structure_lists.py).""" +import logging +import os +import re +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pyarrow.parquet as pq +import torch +from biotite.structure.io.pdb import PDBFile +from biotite.structure.io.pdbx import CIFFile, get_structure +from typer.testing import CliRunner + +from foldmatch.cli import embedding +from foldmatch.errors import RefusalError +# Imported before any test patches their model getters +from foldmatch.inference import assembly_complete_inference, chain_complete_inference, esm_inference, \ + sequence_inference # noqa: F401 +from foldmatch.writer.batch_writer import CsvBatchWriter, TensorBatchWriter + +_CIF = Path(__file__).parent / "resources" / "pdb" / "1acb.cif" +_CPU = ("--accelerator", "cpu", "--devices", "1") +_FORMATS = ("csv", "pt", "parquet", "json") + + +def _esm3(model, protein): + """Stands in for ESM3: L + 2 rows (BOS and EOS, zeros), each residue's row its letter and its position.""" + rows = [[float(ord(aa)), float(k)] for k, aa in enumerate(protein.sequence)] + return torch.tensor([[0.0] * 2] + rows + [[0.0] * 2]) + + +def _aggregate(x, x_mask): + """Stands in for the aggregator: the sum of each item's rows.""" + return x.masked_fill(x_mask.unsqueeze(-1), 0.0).sum(dim=1) + + +def _flat(text: str) -> str: + """Text on one line, typer's error panel borders and colour codes dropped.""" + return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", text).translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +def _ids(path: Path) -> list: + return pq.read_table(path).column("id").to_pylist() + + +def _case_insensitive() -> bool: + """Whether the temporary folder takes 'A' and 'a' for one name (macOS APFS by default), found out without + FoldMatch.""" + with tempfile.NamedTemporaryFile(prefix="CaseProbe-") as f: + head, name = os.path.split(f.name) + return os.path.exists(os.path.join(head, name.swapcase())) + + +_CASE_INSENSITIVE = _case_insensitive() + + +class _Case(unittest.TestCase): + + def setUp(self): + # Resolved, as typer resolves the paths the commands are given (/var is /private/var on macOS) + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())).resolve() + for module in ("esm_inference", "sequence_inference", "chain_complete_inference", + "assembly_complete_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_residue_model", self.model("ESM3"))) + for module in ("chain_complete_inference", "assembly_complete_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_aggregator_model", + self.model(_aggregate))) + for module in ("esm_module", "chain_complete_module", "assembly_complete_module"): + self.enterContext(mock.patch(f"foldmatch.modules.{module}.per_residue_embedding", _esm3)) + self.refusing = False + self.models = 0 + + def model(self, model): + def get(**_): + self.models += 1 + if self.refusing: + raise AssertionError("a model was built") + return model + return get + + def folder(self, name: str = "out") -> Path: + return Path(tempfile.mkdtemp(prefix=f"{name}-", dir=self.tmp)) + + def fasta(self, *records, name: str = None) -> Path: + path = Path(name) if name else self.tmp / f"q{len(list(self.tmp.iterdir()))}.fasta" + path.write_text("".join(f">{header}\n{sequence}\n" for header, sequence in records)) + return path + + def invoke(self, *args): + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(embedding.app, [str(a) for a in (*args, *_CPU)]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + def run_ok(self, *args): + self.refusing = False + result = self.invoke(*args) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + + def refused(self, *args, written=()) -> str: + before = {folder: sorted(p.name for p in folder.iterdir()) for folder in written} + self.refusing = True + try: + result = self.invoke(*args) + finally: + self.refusing = False + self.assertEqual(result.exit_code, 1, f"{result.output}\n{result.exception!r}") + self.assertIsInstance(result.exception, RefusalError, repr(result.exception)) + for folder, names in before.items(): + self.assertEqual(sorted(p.name for p in folder.iterdir()), names, folder) + return _flat(str(result.exception)) + + +class FastaIdTest(_Case): + + def test_a_repeated_id_in_every_format(self): + """A repeated id kept the last record's embedding in csv and pt, and two rows of one id in Parquet and JSON: + refused before ESM3, as the residue-code pass refuses it; a record under min_res_n is not written, so its id + may repeat a kept one's. A rerun into the same folder runs.""" + repeated = self.fasta(("a first", "MKV"), ("b", "MKVL"), ("a second", "MKVLA")) + for command in ("residue", "chain"): + for out_format in _FORMATS: + with self.subTest(command=command, out_format=out_format): + message = self.refused("from-sequences", command, "--fasta-file", repeated, "--output-folder", + self.folder(), "--output-format", out_format) + self.assertEqual(message, f"{repeated}: 1 record(s) repeat an earlier id (e.g. a). fm-embedding " + f"writes each record's embedding under its id, the first word of its " + f"header (the file .csv or .pt, or the id of its row in Parquet " + f"and JSON), so a repeated id keeps one of them: give each record its " + f"own id.") + out = self.folder() + for _ in range(2): + self.run_ok("from-sequences", command, "--fasta-file", repeated, "--output-folder", out, + "--min-res-n", "4") + self.assertEqual(sorted(f.name for f in out.iterdir()), ["a.csv", "b.csv"]) + + def test_an_id_that_cannot_name_a_file(self): + """'../precious' replaced a file outside the output folder, 'Q9/1-100' stopped the run with a traceback: refused + in csv and pt, before ESM3, naming the records; Parquet keeps both ids.""" + precious = self.tmp / "precious.csv" + precious.write_text("keep") + fasta = self.fasta(("../precious", "MKV"), ("Q9/1-100", "MKVL")) + for out_format in ("csv", "pt"): + with self.subTest(out_format=out_format): + message = self.refused("from-sequences", "chain", "--fasta-file", fasta, "--output-folder", + self.folder(), "--output-format", out_format) + self.assertIn(f"{fasta}: 2 FASTA id(s) that cannot name a file .{out_format} in the output " + f"folder: '../precious' (record 1) holds '/'; 'Q9/1-100' (record 2) holds '/'. Each " + f"FASTA id names its own file there: give those records a FASTA id that is not empty, " + f"without '/'", message) + self.assertEqual(precious.read_text(), "keep") + out = self.folder() + self.run_ok("from-sequences", "chain", "--fasta-file", fasta, "--output-folder", out, "--output-format", + "parquet") + self.assertEqual(_ids(out / "inference-0.parquet"), ["../precious", "Q9/1-100"]) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_ids_that_differ_in_case_on_this_mac(self): + """abc and ABC were one file, which held the second record's embedding: refused in csv and pt; Parquet keeps + both.""" + fasta = self.fasta(("abc", "MKV"), ("ABC", "MKVL")) + message = self.refused("from-sequences", "residue", "--fasta-file", fasta, "--output-folder", self.folder(), + "--output-format", "pt") + self.assertIn("1 file(s) of the output folder would each be written for more than one FASTA id (the output " + "folder does not tell upper and lower case apart): 'abc' (record 1) and 'ABC' (record 2) give " + "abc.pt. Each one would replace the one before, as the folder takes those names for one: write " + "them to a folder of a file system that tells upper and lower case apart or as parquet or json " + "(which keep each FASTA id as given), or give those records FASTA ids that differ otherwise.", + message) + out = self.folder() + self.run_ok("from-sequences", "residue", "--fasta-file", fasta, "--output-folder", out, "--output-format", + "parquet") + self.assertEqual(_ids(out / "inference-0.parquet"), ["abc", "ABC"]) + + def test_the_fasta_file_among_the_outputs(self): + """--fasta-file OUT/x.csv with a record x: the first embedding replaced the FASTA file, and the next record + failed. Refused; under another name in that folder it runs.""" + out = self.folder() + fasta = self.fasta(("x", "MKV"), ("y", "MKVL"), name=out / "x.csv") + message = self.refused("from-sequences", "chain", "--fasta-file", fasta, "--output-folder", out, + written=[out]) + self.assertIn(f"the FASTA file {fasta} by the output 'x'", message) + renamed = fasta.rename(out / "queries.fasta") + self.run_ok("from-sequences", "chain", "--fasta-file", renamed, "--output-folder", out) + self.assertEqual(sorted(f.name for f in out.iterdir()), ["queries.fasta", "x.csv", "y.csv"]) + + def test_the_output_name(self): + """--output-name '../up' wrote up-0.parquet outside the folder: refused, from sequences and from structures.""" + fasta = self.fasta(("x", "MKV")) + structures = self.folder("structures") + os.symlink(_CIF, structures / "1acb.cif") + for args in (("from-sequences", "chain", "--fasta-file", fasta), + ("from-sequences", "residue", "--fasta-file", fasta), + ("from-structures", "chain", "--src-folder", structures), + ("from-structures", "assembly", "--src-folder", structures)): + with self.subTest(command=args[:2]): + message = self.refused(*args, "--output-folder", self.folder(), "--output-format", "parquet", + "--output-name", "../up", written=[self.tmp]) + self.assertTrue(message.startswith("The output name '../up' holds '/'"), message) + out = self.folder() + self.run_ok("from-sequences", "chain", "--fasta-file", fasta, "--output-folder", out, "--output-format", + "parquet", "--output-name", "up") + self.assertEqual([f.name for f in out.iterdir()], ["up-0.parquet"]) + + +def _chains_a_and_a(path: Path) -> Path: + """1acb's two protein chains (author fields E and I) renamed A and a, written as PDB.""" + atoms = get_structure(CIFFile.read(str(_CIF)), model=1, use_author_fields=True) + atoms = atoms[~atoms.hetero] + atoms.chain_id[atoms.chain_id == "E"] = "A" + atoms.chain_id[atoms.chain_id == "I"] = "a" + pdb = PDBFile() + pdb.set_structure(atoms) + pdb.write(str(path)) + return path + + +class WriterBackstopTest(_Case): + """D4: a writer refuses a second write of one file in a run, naming the file; the first file is kept, the second + item not written. Per process: not across ranks.""" + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_chains_a_and_a_on_this_mac(self): + """from-structures chain on a structure with chains A and a: cc.A.csv held chain a's data, and chain A was gone + (sweep). Now the run stops at the second; Parquet keeps both.""" + structures = self.folder("structures") + _chains_a_and_a(structures / "cc.pdb") + for command in ("residue", "chain"): + for out_format in ("csv", "pt"): + with self.subTest(command=command, out_format=out_format): + out = self.folder() + result = self.invoke("from-structures", command, "--src-folder", structures, "--structure-format", + "pdb", "--output-folder", out, "--output-format", out_format) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError) + written = [f.name for f in out.iterdir()] + self.assertEqual(len(written), 1, written) + self.assertRegex(_flat(str(result.exception)), + f"^{re.escape(str(out))}/cc\\.(A|a)\\.{out_format} would be written twice by this " + f"run, for 'cc\\.(A|a)' and then for 'cc\\.(A|a)' \\(the folder does not tell " + f"upper and lower case or Unicode spellings \\(NFC and NFD\\) apart\\): the first " + f"is kept, 'cc\\.(A|a)' is not written, and the run stops\\.") + out = self.folder() + self.run_ok("from-structures", "chain", "--src-folder", structures, "--structure-format", "pdb", + "--output-folder", out, "--output-format", "parquet") + self.assertCountEqual(_ids(out / "inference-0.parquet"), ["cc.A", "cc.a"]) + + def test_the_writers(self): + """The same name twice is refused on any folder; A and a only where the folder takes them for one name (the + probe's result faked: a case-sensitive folder keeps both).""" + trainer = SimpleNamespace(global_rank=0) + vectors = [torch.ones(3), torch.zeros(3)] + for writer, postfix in ((CsvBatchWriter, "csv"), (TensorBatchWriter, "pt")): + with self.subTest(writer=writer.__name__): + out = self.folder() + files = writer(str(out)) + with self.assertRaises(RefusalError) as refused: + files.write_on_batch_end(trainer, None, (vectors, ["x", "x"]), None, None, 0, 0) + self.assertTrue(str(refused.exception).startswith(f"{out}/x.{postfix} would be written twice by this " + f"run, for 'x' and then for 'x': the first is kept")) + self.assertEqual([f.name for f in out.iterdir()], [f"x.{postfix}"]) + with self.assertRaises(RefusalError): + files._write_embedding(vectors[0], "../x") + from foldmatch.utils.output_files import EXACT, Folding + for folding in (EXACT, Folding(case=True)): + out = self.folder() + with mock.patch("foldmatch.writer.batch_writer.probe_folding", return_value=folding): + files = writer(str(out)) + files._write_embedding(vectors[0], "A") + if folding.case: + with self.assertRaises(RefusalError): + files._write_embedding(vectors[1], "a") + else: + # Both written (on this Mac's case-insensitive folder, the file system makes them one) + files._write_embedding(vectors[1], "a") + self.assertEqual(len(list(out.iterdir())), 1 if _CASE_INSENSITIVE else 2) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_the_writers_fold_case_fully_on_this_mac(self): + """straße and strasse, ff1 and ff1 are one file on APFS (the second replaced the first): the writers refuse the + second; a folder that folds case simply (the probe's result faked) keeps both names.""" + trainer = SimpleNamespace(global_rank=0) + vectors = [torch.ones(3), torch.zeros(3)] + from foldmatch.utils.output_files import Folding + for writer, postfix in ((CsvBatchWriter, "csv"), (TensorBatchWriter, "pt")): + for first, second in (("straße", "strasse"), ("ff1", "ff1")): + with self.subTest(writer=writer.__name__, first=first): + out = self.folder() + with self.assertRaises(RefusalError) as refused: + writer(str(out)).write_on_batch_end(trainer, None, (vectors, [first, second]), None, None, 0, + 0) + self.assertTrue(str(refused.exception).startswith( + f"{out}/{first}.{postfix} would be written twice by this run, for {first!r} and then for " + f"{second!r} (the folder does not tell upper and lower case"), str(refused.exception)) + self.assertEqual([f.name for f in out.iterdir()], [f"{first}.{postfix}"]) + with mock.patch("foldmatch.writer.batch_writer.probe_folding", + return_value=Folding(case=True, unicode=True)): + files = writer(str(self.folder())) + files._write_embedding(vectors[0], first) + files._write_embedding(vectors[1], second) + + def test_runs_into_one_folder_still_run(self): + """Not a conflict within a run: residue then chain embeddings of the same structures into one csv folder (the + chain files replace the residue files, as before: an earlier run's files are not checked), and a rerun. A + structure list given twice the same structure, which the writer would write twice, is refused by its reader.""" + structures = self.folder("structures") + os.symlink(_CIF, structures / "1acb.cif") + out = self.folder() + for command in ("residue", "chain", "chain"): + self.run_ok("from-structures", command, "--src-folder", structures, "--output-folder", out) + self.assertEqual(sorted(f.name for f in out.iterdir()), ["1acb.A.csv", "1acb.B.csv"]) + self.refusing = True + with self.assertRaises(RefusalError): + chain_complete_inference.predict(src_stream=[("1acb", str(_CIF), "x"), ("2acb", str(_CIF), "x")], + src_location="stream", out_folder=self.folder(), accelerator="cpu", + devices=1, return_predictions=False) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_name_conflicts_search.py b/tests/test_name_conflicts_search.py new file mode 100644 index 0000000..c172940 --- /dev/null +++ b/tests/test_name_conflicts_search.py @@ -0,0 +1,245 @@ +"""P9.N3: fm-search refuses an --output-file that is one of its inputs, and query embeddings that share an id. + +Every command that takes --output-file (query structure, embedding, sequences, db, cluster, similarity-graph) refuses, +before any work, an output file that is a file of an input database (.index, .metadata, .invlists, +.sequences, .rescodes.*), the query FASTA or structure file, or a query embedding file it reads (D6): the results +replaced it with exit status 0. Compared as files (a symbolic link, and on macOS another case, name the same file). An +output file of an earlier command is not an input, even one named .hits.tsv beside the database: a rerun with +the same --output-file replaces it, as before. query embedding refuses two vectors of one id (D7): one of +the queries disappeared from the output. + +No model is loaded: the databases are built from vectors (build embeddings), query structure's embedding is a stub, +and query sequences is refused or stopped before ESM3.""" +import logging +import os +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import torch +from typer.testing import CliRunner + +from foldmatch.cli import search +from foldmatch.errors import RefusalError + +_CIF = Path(__file__).parent / "resources" / "pdb" / "1acb.cif" + + +def _flat(text: str) -> str: + """Text on one line, typer's error panel borders and colour codes dropped.""" + return " ".join(re.sub(r"\x1b\[[0-9;]*m", "", text).translate({ord(c): " " for c in "│╭╮╰╯─"}).split()) + + +def _case_insensitive() -> bool: + """Whether the temporary folder takes 'A' and 'a' for one name (macOS APFS by default), found out without + FoldMatch.""" + with tempfile.NamedTemporaryFile(prefix="CaseProbe-") as f: + head, name = os.path.split(f.name) + return os.path.exists(os.path.join(head, name.swapcase())) + + +_CASE_INSENSITIVE = _case_insensitive() + + +def _vectors(folder: Path, ids, offset: float = 0.0) -> Path: + folder.mkdir(exist_ok=True) + for k, name in enumerate(ids): + torch.save(torch.arange(8, dtype=torch.float32) * (k + 1) + offset, folder / f"{name}.pt") + return folder + + +def _parquet(path: Path, ids) -> Path: + pq.write_table(pa.table({"id": list(ids), "embedding": [[float(k + 1)] * 8 for k in range(len(ids))]}, + schema=pa.schema([("id", pa.string()), ("embedding", pa.list_(pa.float32()))])), path) + return path + + +class _Case(unittest.TestCase): + + def setUp(self): + # Resolved, as typer resolves the paths the commands are given (/var is /private/var on macOS) + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())).resolve() + self.db = self.build("db", ["s1", "s2", "s3"]) + + def invoke(self, *args): + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + return CliRunner().invoke(search.app, [str(a) for a in args]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + + def run_ok(self, *args): + result = self.invoke(*args) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + return result + + def build(self, name: str, ids) -> Path: + db = self.tmp / name + self.run_ok("build", "embeddings", "--embedding-folder", _vectors(self.tmp / f"{name}-vectors", ids), + "--output-db", db) + return db + + def refused(self, *args, kept: Path) -> str: + """Refused (exit status 1, a RefusalError) and the file ``kept`` unchanged: the message, on one line.""" + before = kept.read_bytes() + result = self.invoke(*args) + self.assertEqual(result.exit_code, 1, f"{result.output}\n{result.exception!r}") + self.assertIsInstance(result.exception, RefusalError, repr(result.exception)) + self.assertEqual(kept.read_bytes(), before) + return _flat(str(result.exception)) + + +class DatabaseFileTest(_Case): + """D6: an output file that is a file of an input database.""" + + def test_query_embedding(self): + """--output-file .index replaced the index, and every later command failed (sweep case 11).""" + queries = _vectors(self.tmp / "queries", ["q1", "q2"]) + db_files = sorted(f.name for f in self.tmp.iterdir() if f.name.startswith("db.")) + self.assertEqual(db_files, ["db.index", "db.metadata"]) + for name in db_files: + with self.subTest(name=name): + message = self.refused("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, + "--output-file", self.tmp / name, kept=self.tmp / name) + self.assertEqual(message, f"--output-file {self.tmp / name} is the file {self.tmp / name} of the " + f"database {self.db}: the results would replace an input of this command. " + f"Write them to another file.") + link = self.tmp / "results-link.tsv" + link.symlink_to(self.tmp / "db.metadata") + self.refused("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, "--output-file", link, + kept=self.tmp / "db.metadata") + message = self.refused("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, + "--output-file", queries / "q1.pt", kept=queries / "q1.pt") + self.assertIn(f"is the query embedding file {queries / 'q1.pt'}", message) + results = self.tmp / "results.tsv" + for _ in range(2): + self.run_ok("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, "--output-file", + results, "--threshold", "0") + self.assertEqual(sorted({line.split("\t")[0] for line in results.read_text().splitlines()}), ["q1", "q2"]) + self.run_ok("stats", "--db-path", self.db) + + @unittest.skipUnless(_CASE_INSENSITIVE, "a case-insensitive file system (macOS APFS)") + def test_another_case_on_this_mac(self): + """DB.INDEX is db.index on macOS: refused.""" + queries = _vectors(self.tmp / "queries", ["q1"]) + message = self.refused("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, + "--output-file", self.tmp / "DB.INDEX", kept=self.tmp / "db.index") + self.assertIn(f"is the file {self.tmp / 'db.index'} of the database {self.db}", message) + + def test_query_db_cluster_and_graph(self): + """query db (either database), cluster and similarity-graph; another file runs, twice.""" + other = self.build("other", ["o1", "o2"]) + cases = (("query", "db", "--query-db-path", other, "--subject-db-path", self.db), + ("cluster", "--db-path", self.db, "--threshold", "0"), + ("similarity-graph", "--db-path", self.db, "--threshold", "0")) + for args in cases: + targets = [self.tmp / "db.index"] + ([self.tmp / "other.metadata"] if args[0] == "query" else []) + for target in targets: + with self.subTest(command=args[:2], target=target.name): + message = self.refused(*args, "--output-file", target, kept=target) + self.assertIn(f"--output-file {target} is the file {target} of the ", message) + out = self.tmp / f"{args[0]}-out.txt" + for _ in range(2): + self.run_ok(*args, "--output-file", out) + self.assertTrue(out.exists()) + + + def test_results_named_after_the_database_rerun(self): + """Results named . beside the database are not a file of it: query embedding, cluster and + similarity-graph run twice into them (the second run was refused), and one command into another's results. + A file of the residue-code store, .rescodes.*, is one.""" + queries = _vectors(self.tmp / "queries", ["q1"]) + cases = (("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, "--threshold", "0"), + ("cluster", "--db-path", self.db, "--threshold", "0"), + ("similarity-graph", "--db-path", self.db, "--threshold", "0")) + for args in cases: + with self.subTest(command=args[0]): + out = self.tmp / f"db.{args[0]}.tsv" + for _ in range(2): + self.run_ok(*args, "--output-file", out) + self.assertTrue(out.exists()) + self.run_ok(*cases[0], "--output-file", self.tmp / "db.cluster.tsv") + store = self.tmp / "db.rescodes.bin" + store.write_bytes(b"codes") + message = self.refused(*cases[1], "--output-file", store, kept=store) + self.assertIn(f"is the file {store} of the database {self.db}", message) + + +class QueryInputTest(_Case): + """D6: the query FASTA and the query structure file.""" + + def test_query_sequences(self): + """--output-file q.fasta replaced the queries: refused before the ESM3 pass; another file gets past the check + to the pass (stopped there, no model is loaded).""" + fasta = self.tmp / "q.fasta" + fasta.write_text(">q1\nMKV\n") + tmp = self.tmp / "embeddings" + tmp.mkdir() + never = mock.Mock(side_effect=AssertionError("the ESM3 pass ran")) + with mock.patch("foldmatch.search.embedding_computer.chain_predict", never): + message = self.refused("query", "sequences", "--db-path", self.db, "--fasta-file", fasta, + "--tmp-embedding-folder", tmp, "--output-file", fasta, kept=fasta) + self.assertEqual(message, f"--output-file {fasta} is the FASTA file {fasta}: the results would replace an " + f"input of this command. Write them to another file.") + never.assert_not_called() + stop = mock.Mock(side_effect=RuntimeError("the ESM3 pass")) + with mock.patch("foldmatch.search.embedding_computer.chain_predict", stop): + result = self.invoke("query", "sequences", "--db-path", self.db, "--fasta-file", fasta, + "--tmp-embedding-folder", tmp, "--output-file", self.tmp / "results.tsv") + stop.assert_called_once() + self.assertIsInstance(result.exception, RuntimeError) + + def test_query_structure(self): + """--output-file q.cif replaced the query structure: refused before its embedding; another file runs.""" + structure = Path(self.tmp / "q.cif") + structure.write_bytes(_CIF.read_bytes()) + never = mock.Mock(side_effect=AssertionError("the query was embedded")) + with mock.patch("foldmatch.search.embedding_computer.compute_from_structure_file", never): + message = self.refused("query", "structure", "--db-path", self.db, "--query-structure", structure, + "--output-file", structure, kept=structure) + self.assertIn(f"is the query structure file {structure}", message) + never.assert_not_called() + stub = mock.Mock(return_value=[(["q.A"], np.arange(8, dtype=np.float32)[None, :])]) + with mock.patch("foldmatch.search.embedding_computer.compute_from_structure_file", stub): + self.run_ok("query", "structure", "--db-path", self.db, "--query-structure", structure, + "--output-file", self.tmp / "results.tsv", "--threshold", "0") + self.assertTrue((self.tmp / "results.tsv").read_text().startswith("q.A\t")) + + +class QueryIdTest(_Case): + """D7: query embedding refuses two vectors of one id (one query disappeared from the output).""" + + def test_repeated_ids(self): + queries = self.tmp / "queries" + queries.mkdir() + _parquet(queries / "a.parquet", ["x", "y", "x"]) + _parquet(queries / "b.parquet", ["z"]) + _vectors(queries, ["z"]) + results = self.tmp / "results.tsv" + result = self.invoke("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, + "--output-file", results) + self.assertEqual(result.exit_code, 1, result.output) + self.assertIsInstance(result.exception, RefusalError) + self.assertEqual(_flat(str(result.exception)), + f"{queries}: 2 id(s) held by more than one query embedding: 'x' in 2 rows of a.parquet; 'z' " + f"in b.parquet and z.pt. The results are written by query id, so a repeated id would keep the " + f"hits of one of its embeddings and drop the others': give each embedding its own id, or " + f"query its files apart (--file-extension, or one folder per kind).") + self.assertFalse(results.exists()) + self.run_ok("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, "--file-extension", + ".pt", "--output-file", results, "--threshold", "0") + (queries / "a.parquet").unlink() + self.run_ok("query", "embedding", "--db-path", self.db, "--embedding-folder", queries, "--file-extension", + ".parquet", "--output-file", results, "--threshold", "0") + self.assertEqual({line.split("\t")[0] for line in results.read_text().splitlines()}, {"z"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..dc184c0 --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,537 @@ +"""Packaging of the native Smith-Waterman kernel (PLAN section 7, P3.2; test matrix: CI tier ``packaging``). + +What is checked: +- wheel contents: every ``_fmsw_*`` library expected for the wheel's platform, built for that architecture, embedding + the source sha256 of the shipped kernel source (= this tree), a non-pure ``py3-none-`` tag; +- sdist contents: the default hatchling set (no include list) with the hook, the C source and the test fixtures, + and no compiled library; +- stale libraries: the in-place libraries of this tree embed its source sha256; +- ``selftest()`` from the installed wheel in a clean venv, and the refusal of a pure wheel; +- the build hook: pure-wheel fallback without a C compiler and after a partial compile failure, + ``FOLDMATCH_REQUIRE_NATIVE=1`` makes both fatal, editable builds compile in place, the platform tag. + +Inputs (environment): +- ``FOLDMATCH_DIST_DIR``: directory holding the sdist and the wheel(s) to check (CI: ``python -m build``, or the + cibuildwheel wheelhouse). When unset, the sdist and a wheel built from it are made here with the build python + (``python -m build --no-isolation``) if it has ``build`` and ``hatchling``; otherwise those tests skip. +- ``FOLDMATCH_BUILD_PYTHON``: interpreter with ``build``, ``hatchling`` and ``editables`` for the hook tests (default: + the interpreter running the tests). + +Clean venvs are created from the interpreter running the tests with ``--system-site-packages`` (so they see numpy and +importlib-metadata) and get the wheel with ``pip install --no-deps --no-index``; they run outside the tree without +``PYTHONPATH``, and every check asserts that ``foldmatch`` was imported from the venv. +""" +from __future__ import annotations + +import importlib.util +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tarfile +import tomllib +import zipfile +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +NATIVE_SRC = ROOT / "src" / "foldmatch" / "_native" +DIST_ENV = "FOLDMATCH_DIST_DIR" +BUILD_PYTHON_ENV = "FOLDMATCH_BUILD_PYTHON" +REQUIRE_ENV = "FOLDMATCH_REQUIRE_NATIVE" +NO_CC = "/nonexistent/foldmatch-no-cc" + + +def _load_by_path(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# the tree's library-discovery module, loaded by path: these tests never import the foldmatch package itself +TREE = _load_by_path("_packaging_test_native", NATIVE_SRC / "__init__.py") +TREE_SHA = TREE.source_sha256() +DP_GOLDEN = re.search(r'^DP_GOLDEN_SYNTH_SHA256 = "([0-9a-f]{64})"', + (ROOT / "src" / "foldmatch" / "search" / "swkern.py").read_text(), re.M).group(1) +FMSWI_GOLDEN = re.search(r'^GOLDEN_SHA256 = "([0-9a-f]{64})"', + (ROOT / "src" / "foldmatch" / "search" / "swint.py").read_text(), re.M).group(1) + + +# ===================================================================================================================== +# helpers +# ===================================================================================================================== +def _run(cmd, *, env=None, cwd=None, timeout=600) -> subprocess.CompletedProcess: + return subprocess.run([str(c) for c in cmd], env=env, cwd=cwd, capture_output=True, text=True, timeout=timeout) + + +def _build_python() -> str: + return os.environ.get(BUILD_PYTHON_ENV) or sys.executable + + +def _has_modules(python: str, *modules: str) -> bool: + return _run([python, "-c", "import " + ", ".join(modules)]).returncode == 0 + + +_BUILD_ENV_DROP = ("PYTHONPATH", REQUIRE_ENV, "CC", "ARCHFLAGS", "MACOSX_DEPLOYMENT_TARGET", "_PYTHON_HOST_PLATFORM") + + +def _build_env(**overrides: Optional[str]) -> Dict[str, str]: + """Environment of a build: this one without PYTHONPATH, the compiler choice, the native requirement and the macOS + cross-build variables, plus ``overrides``.""" + env = {k: v for k, v in os.environ.items() if k not in _BUILD_ENV_DROP} + for k, v in overrides.items(): + if v is not None: + env[k] = v + return env + + +def _clean_env() -> Dict[str, str]: + """Environment of a clean-venv process: no PYTHONPATH, no user site, no ISA override.""" + env = {k: v for k, v in os.environ.items() if k not in ("PYTHONPATH", "FOLDMATCH_SW_ISA", "PYTHONHOME")} + env["PYTHONNOUSERSITE"] = "1" + return env + + +def _parse_wheel_name(path: Path) -> Tuple[str, str, str, str, str]: + """(name, version, python tag, abi tag, platform tag) of a wheel file name (no build tag in ours).""" + parts = path.name[:-len(".whl")].split("-") + assert len(parts) == 5, f"unexpected wheel file name {path.name}" + return tuple(parts) + + +def _wheel_arch(plat: str) -> str: + """``x86_64`` or ``arm64`` from a wheel platform tag (all compressed tags must agree).""" + archs = set() + for p in plat.split("."): + if p.endswith("x86_64"): + archs.add("x86_64") + elif p.endswith("aarch64") or p.endswith("arm64"): + archs.add("arm64") + else: + archs.add(p) + assert len(archs) == 1 and archs <= {"x86_64", "arm64"}, f"unsupported wheel platform {plat!r}" + return archs.pop() + + +_ELF_MACHINE = {62: "x86_64", 183: "arm64"} +_MACHO_CPU = {0x01000007: "x86_64", 0x0100000C: "arm64"} + + +def _binary_kind(data: bytes) -> Tuple[str, str]: + """(format, architecture) of a shared library: ('elf', arch) for ELF64 ET_DYN, ('macho', arch) for a 64-bit + Mach-O MH_DYLIB; anything else fails.""" + if data[:4] == b"\x7fELF": + assert data[4] == 2 and data[5] == 1, "not a 64-bit little-endian ELF" + e_type, e_machine = int.from_bytes(data[16:18], "little"), int.from_bytes(data[18:20], "little") + assert e_type == 3, f"ELF type {e_type} is not a shared object" + return "elf", _ELF_MACHINE.get(e_machine, f"elf-machine-{e_machine}") + if data[:4] == b"\xcf\xfa\xed\xfe": + cpu, filetype = int.from_bytes(data[4:8], "little"), int.from_bytes(data[12:16], "little") + assert filetype == 6, f"Mach-O file type {filetype} is not a dylib" + return "macho", _MACHO_CPU.get(cpu, f"macho-cpu-{cpu:#x}") + raise AssertionError(f"not an ELF64 / Mach-O 64 shared library (magic {data[:4]!r})") + + +def _installable_here(path: Path) -> bool: + try: + from packaging.tags import sys_tags + from packaging.utils import parse_wheel_filename + except ImportError: + _n, _v, _py, _abi, plat = _parse_wheel_name(path) + if plat == "any": + return True + machine = TREE.normalized_machine() + mac = plat.startswith("macosx") + return _wheel_arch(plat) == machine and mac == (sys.platform == "darwin") + supported = set(sys_tags()) + return bool(set(parse_wheel_filename(path.name)[3]) & supported) + + +def _source_files_sha(files: Dict[str, bytes], tmp: Path) -> Optional[str]: + """The kernel source sha256 (TREE.source_sha256 algorithm) of the kernel source files (TREE.SOURCE_FILES) given + as bytes.""" + d = tmp / "fmsw_src" + d.mkdir(parents=True, exist_ok=True) + for name, data in files.items(): + (d / name).write_bytes(data) + return TREE.source_sha256(d) + + +def _make_venv(where: Path) -> Path: + for mod in ("numpy", "importlib_metadata"): + if importlib.util.find_spec(mod) is None: + pytest.skip(f"{mod} is not importable here; the clean venv borrows it through --system-site-packages") + res = _run([sys.executable, "-m", "venv", "--system-site-packages", where]) + assert res.returncode == 0, res.stderr + py = where / "bin" / "python" + assert py.is_file() + return py + + +def _install(py: Path, wheel: Path): + # --ignore-installed: the venv sees the test interpreter's site-packages, which may hold a foldmatch of the same + # version (then a plain install is a no-op and the probe would import that one) + res = _run([py, "-m", "pip", "install", "--no-deps", "--no-index", "--ignore-installed", "--no-cache-dir", + "--disable-pip-version-check", "--quiet", wheel], env=_clean_env()) + assert res.returncode == 0, res.stdout + res.stderr + + +_PROBE = r""" +import json, sys +import foldmatch +from foldmatch import _native +from foldmatch.search import swkern +out = {"file": foldmatch.__file__, "in_source_tree": _native.in_source_tree(), "source_sha256": _native.source_sha256(), + "libraries": sorted(p.name for p in _native.NATIVE_DIR.glob("_fmsw_*"))} +try: + out["selftest"] = swkern.selftest() +except swkern.KernelError as e: + out["error"] = str(e) +print(json.dumps(out, default=str)) +""" + + +def _probe(py: Path, cwd: Path, isa: Optional[str] = None) -> dict: + env = _clean_env() + if isa is not None: + env["FOLDMATCH_SW_ISA"] = isa + res = _run([py, "-c", _PROBE], env=env, cwd=cwd) + assert res.returncode == 0, res.stdout + res.stderr + out = json.loads(res.stdout.strip().splitlines()[-1]) + venv = py.parent.parent.resolve() + assert Path(out["file"]).resolve().is_relative_to(venv), f"foldmatch imported from {out['file']}, not {venv}" + assert out["in_source_tree"] is False + return out + + +# ===================================================================================================================== +# the distribution under test +# ===================================================================================================================== +@pytest.fixture(scope="module") +def dist_dir(tmp_path_factory) -> Path: + given = os.environ.get(DIST_ENV) + if given: + d = Path(given).resolve() + assert d.is_dir(), f"{DIST_ENV}={given} is not a directory" + return d + py = _build_python() + if not _has_modules(py, "build", "hatchling"): + pytest.skip(f"set {DIST_ENV} to a built dist, or {BUILD_PYTHON_ENV} to a python with build and hatchling") + out = tmp_path_factory.mktemp("dist") + res = _run([py, "-m", "build", "--no-isolation", "--outdir", out, ROOT], env=_build_env(**{REQUIRE_ENV: "1"})) + assert res.returncode == 0, res.stdout + res.stderr + return out + + +@pytest.fixture(scope="module") +def wheels(dist_dir) -> List[Path]: + found = sorted(dist_dir.glob("foldmatch-*.whl")) + if not found: + pytest.skip(f"no foldmatch wheel in {dist_dir}") + return found + + +@pytest.fixture(scope="module") +def sdist(dist_dir) -> Path: + found = sorted(dist_dir.glob("foldmatch-*.tar.gz")) + if not found: + pytest.skip(f"no foldmatch sdist in {dist_dir}") + assert len(found) == 1, f"several sdists in {dist_dir}: {found}" + return found[0] + + +# ===================================================================================================================== +# wheel contents +# ===================================================================================================================== +def test_wheel_contents(wheels, tmp_path): + for wheel in wheels: + _name, version, py_tag, abi_tag, plat = _parse_wheel_name(wheel) + assert (py_tag, abi_tag) == ("py3", "none"), wheel.name + assert plat != "any", f"{wheel.name} is a pure wheel: the native kernel was not built" + arch = _wheel_arch(plat) + macos = plat.startswith("macosx") + suffix = ".dylib" if macos else ".so" + isas = TREE.machine_isas(arch) + with zipfile.ZipFile(wheel) as z: + names = z.namelist() + assert len(names) == len(set(names)), f"duplicate entries in {wheel.name}" + info = f"foldmatch-{version}.dist-info" + meta = z.read(f"{info}/WHEEL").decode() + assert "Root-Is-Purelib: false" in meta, meta + assert sorted(re.findall(r"^Tag: (\S+)$", meta, re.M)) == \ + sorted(f"py3-none-{p}" for p in plat.split(".")), meta + pkg = z.read(f"{info}/METADATA").decode() + assert "Classifier: Operating System :: POSIX :: Linux" in pkg + assert "Classifier: Operating System :: MacOS" in pkg + assert "OS Independent" not in pkg + + libs = sorted(n for n in names if Path(n).name.startswith(("_fmsw_", "._fmsw_"))) + expected = sorted(f"foldmatch/_native/_fmsw_{isa}{suffix}" for isa in isas) + assert libs == expected, f"{wheel.name}: libraries {libs}, expected {expected}" + # auditwheel / delocate vendor external libraries here; the kernel needs none beyond libc / libm + assert not [n for n in names if n.startswith(("foldmatch.libs/", "foldmatch/.dylibs/"))], wheel.name + assert not [n for n in names if "__pycache__" in n or n.endswith((".pyc", ".tmp"))], wheel.name + + shipped = {n: z.read(f"foldmatch/_native/fmsw/{n}") for n in TREE.SOURCE_FILES} + for n, data in shipped.items(): + assert data == (NATIVE_SRC / "fmsw" / n).read_bytes(), f"{wheel.name}: fmsw/{n} differs from the tree" + for n in ("foldmatch/_native/__init__.py", "foldmatch/_native/build.py", "foldmatch/search/swkern.py"): + assert z.read(n) == (ROOT / "src" / n).read_bytes(), f"{wheel.name}: {n} differs from the tree" + sha = _source_files_sha(shipped, tmp_path / wheel.name) + assert sha == TREE_SHA + for lib in libs: + data = z.read(lib) + fmt, lib_arch = _binary_kind(data) + assert fmt == ("macho" if macos else "elf") and lib_arch == arch, f"{lib}: {fmt} {lib_arch}" + # stale-library check on the shipped binaries: compiled from exactly the shipped source + embeds_sha, records_tmp = sha.encode("ascii") in data, b"/foldmatch-fmsw-" in data + assert embeds_sha, f"{wheel.name}: {lib} does not embed source sha256 {sha}" + assert not records_tmp, f"{wheel.name}: {lib} records its temporary build directory" + + +# ===================================================================================================================== +# sdist contents +# ===================================================================================================================== +def test_sdist_contents(sdist): + config = tomllib.loads((ROOT / "pyproject.toml").read_text()) + build = config.get("tool", {}).get("hatch", {}).get("build", {}) + for section in (build, build.get("targets", {}).get("sdist", {})): + assert not {"include", "only-include"} & set(section), "the sdist must use hatchling's default file set" + assert build["targets"]["wheel"]["hooks"]["custom"]["path"] == "hatch_build.py" + + with tarfile.open(sdist) as t: + members = [m for m in t.getmembers() if m.isfile()] + top = {m.name.split("/", 1)[0] for m in members} + assert len(top) == 1, top + prefix = top.pop() + "/" + files = {m.name[len(prefix):]: m for m in members} + + def read(rel): + return t.extractfile(files[rel]).read() + + kernel_sources = [f"src/foldmatch/_native/fmsw/{name}" for name in TREE.SOURCE_FILES] + required = ["pyproject.toml", "hatch_build.py", "README.md", "LICENSE.md", "PKG-INFO", + "src/foldmatch/_native/__init__.py", "src/foldmatch/_native/build.py", *kernel_sources, + "src/foldmatch/search/swkern.py", "src/foldmatch/search/swint.py", "tests/conftest.py", + "tests/test_swkern.py", "tests/esm_codes_reference.py", "tests/test_swint.py", + "tests/blosum62_reference.py", "tests/test_packaging.py"] + required += [p.relative_to(ROOT).as_posix() for p in sorted((ROOT / "tests/resources/esm_codes").rglob("*")) + if p.is_file()] + missing = [r for r in required if r not in files] + assert not missing, f"missing from {sdist.name}: {missing}" + for rel in ("pyproject.toml", "hatch_build.py", *kernel_sources, "src/foldmatch/_native/build.py"): + assert read(rel) == (ROOT / rel).read_bytes(), f"{sdist.name}: {rel} differs from the tree" + compiled = [r for r in files if Path(r).name.startswith(("_fmsw_", "._fmsw_")) + or r.endswith((".so", ".dylib", ".pyc", ".o")) or "__pycache__" in r] + assert not compiled, f"compiled files in {sdist.name}: {compiled}" + pkg = read("PKG-INFO").decode() + assert "Classifier: Operating System :: POSIX :: Linux" in pkg and "OS Independent" not in pkg + + +# ===================================================================================================================== +# stale libraries (in place) +# ===================================================================================================================== +@pytest.fixture +def tree_libs(request): + """The in-place libraries of the tree (conftest ``fmsw_libs``), when foldmatch is importable (PYTHONPATH=src or an + editable install).""" + pytest.importorskip("foldmatch._native.build") + return request.getfixturevalue("fmsw_libs") + + +@pytest.mark.usefixtures("tree_libs") +def test_in_place_libraries_match_the_tree(): + from foldmatch import _native + from foldmatch._native import build as native_build + from foldmatch.search import swkern + if not _native.NATIVE_DIR.samefile(NATIVE_SRC): + pytest.skip(f"foldmatch is imported from {_native.NATIVE_DIR}, not from this tree") + assert _native.source_sha256() == TREE_SHA + for isa in _native.machine_isas(): + path = _native.find_library(isa) + assert path is not None, f"{isa} library missing after the developer build" + assert native_build.library_is_current(path), f"{path} is stale" + assert swkern.load().source_sha256 == TREE_SHA + + +# ===================================================================================================================== +# selftest() from the installed wheel in a clean venv +# ===================================================================================================================== +def test_wheel_selftest_in_clean_venv(wheels, tmp_path): + here = [w for w in wheels if _installable_here(w)] + if not here: + pytest.skip(f"none of {[w.name for w in wheels]} installs on this machine") + for wheel in here: + py = _make_venv(tmp_path / f"venv-{wheel.stem}") + _install(py, wheel) + out = _probe(py, tmp_path) + assert "error" not in out, out.get("error") + report = out["selftest"] + assert report["dp_golden_sha256"] == DP_GOLDEN + assert report["fmswi"]["golden_sha256"] == FMSWI_GOLDEN + assert report["s16_max_score_ratio"] <= 1.0 + kernel = report["kernel"] + assert kernel["abi"] == 1 and kernel["source_sha256"] == TREE_SHA == out["source_sha256"] + assert Path(kernel["path"]).resolve().is_relative_to(py.parent.parent.resolve()) + machine = TREE.normalized_machine() + if machine == "arm64": + assert kernel["isa"] == "neon" + elif machine == "x86_64": + assert kernel["isa"] in ("sse2", "avx2") + # every other library of the wheel, through the validated override (avx2 may be refused on this CPU) + for isa in TREE.machine_isas(machine): + if isa == kernel["isa"]: + continue + o = _probe(py, tmp_path, isa=isa) + if isa == "avx2" and "error" in o and "avx2 refused" in o["error"]: + continue + assert "error" not in o, o.get("error") + assert (o["selftest"]["kernel"]["isa"], o["selftest"]["dp_golden_sha256"]) == (isa, DP_GOLDEN) + assert o["selftest"]["fmswi"]["golden_sha256"] == FMSWI_GOLDEN + + +# ===================================================================================================================== +# the build hook +# ===================================================================================================================== +@pytest.fixture(scope="module") +def build_python() -> str: + py = _build_python() + if not _has_modules(py, "build", "hatchling"): + pytest.skip(f"{BUILD_PYTHON_ENV} (default: this python) needs build and hatchling for the build-hook tests") + return py + + +def _build_wheel(py: str, out: Path, **env: Optional[str]) -> subprocess.CompletedProcess: + return _run([py, "-m", "build", "--wheel", "--no-isolation", "--outdir", out, ROOT], env=_build_env(**env)) + + +def test_pure_wheel_without_a_compiler_and_its_refusal(build_python, tmp_path): + out = tmp_path / "dist" + res = _build_wheel(build_python, out, CC=NO_CC) + assert res.returncode == 0, res.stdout + res.stderr + log = res.stdout + res.stderr + assert "FoldMatch native kernel not built" in log and TREE.REINSTALL_COMMAND in log, log + (wheel,) = out.glob("*.whl") + assert wheel.name.endswith("-py3-none-any.whl"), wheel.name + with zipfile.ZipFile(wheel) as z: + assert not [n for n in z.namelist() if "_fmsw_" in n] + assert "Root-Is-Purelib: true" in z.read(next(n for n in z.namelist() if n.endswith(".dist-info/WHEEL"))).decode() + py = _make_venv(tmp_path / "venv-pure") + _install(py, wheel) + o = _probe(py, tmp_path) + assert o["libraries"] == [] + assert "not found" in o["error"] and "without its native kernel" in o["error"], o["error"] + assert TREE.REINSTALL_COMMAND in o["error"], o["error"] + + +def test_require_native_makes_a_compile_failure_fatal(build_python, tmp_path): + out = tmp_path / "dist" + res = _build_wheel(build_python, out, CC=NO_CC, **{REQUIRE_ENV: "1"}) + assert res.returncode != 0 + assert f"{REQUIRE_ENV}=1" in res.stdout + res.stderr + assert not list(out.glob("*.whl")) + + +def _cc_failing_for_generic(tmp_path: Path) -> str: + """A C compiler wrapper that fails for the generic library only, which every machine with an ISA library builds + last: the other libraries compile, then the build fails part-way.""" + real = next((shutil.which(c) for c in ("cc", "clang", "gcc") if shutil.which(c)), None) + if real is None: + pytest.skip("no C compiler") + wrapper = tmp_path / "cc-failing-for-generic" + wrapper.write_text("#!/bin/sh\n" + 'for a in "$@"; do [ "$a" = "-DFMSW_GENERIC" ] && ' + '{ echo "simulated compile failure (generic)" >&2; exit 1; }; done\n' + f'exec "{real}" "$@"\n') + wrapper.chmod(0o755) + return str(wrapper) + + +def test_partial_compile_failure(build_python, tmp_path): + """One ISA fails after others compiled: without FOLDMATCH_REQUIRE_NATIVE the pure wheel and the warning (no + library of the partial build), with it a clear build error.""" + cc = _cc_failing_for_generic(tmp_path) + out = tmp_path / "dist" + res = _build_wheel(build_python, out, CC=cc) + log = res.stdout + res.stderr + assert res.returncode == 0, log + assert "FoldMatch native kernel not built" in log and "simulated compile failure" in log, log + (wheel,) = out.glob("*.whl") + assert wheel.name.endswith("-py3-none-any.whl"), wheel.name + with zipfile.ZipFile(wheel) as z: + assert not [n for n in z.namelist() if "_fmsw_" in n] + wheel_meta = z.read(next(n for n in z.namelist() if n.endswith(".dist-info/WHEEL"))).decode() + assert "Root-Is-Purelib: true" in wheel_meta + out = tmp_path / "dist-required" + res = _build_wheel(build_python, out, CC=cc, **{REQUIRE_ENV: "1"}) + log = res.stdout + res.stderr + assert res.returncode != 0 + assert f"FoldMatch native kernel build failed and {REQUIRE_ENV}=1" in log, log + assert "simulated compile failure" in log, log + assert not list(out.glob("*.whl")) + + +def test_editable_build_compiles_in_place(build_python, tmp_path): + if not _has_modules(build_python, "editables"): + pytest.skip("the build python has no editables (hatchling's editable-build requirement)") + proj = tmp_path / "proj" + proj.mkdir() + for name in ("pyproject.toml", "README.md", "LICENSE.md", "hatch_build.py"): + shutil.copy(ROOT / name, proj / name) + shutil.copytree(ROOT / "src", proj / "src", ignore=shutil.ignore_patterns("_fmsw_*", "._fmsw_*", "__pycache__")) + out = tmp_path / "dist" + res = _run([build_python, "-m", "hatchling", "build", "-t", "wheel:editable", "-d", out], cwd=proj, + env=_build_env(**{REQUIRE_ENV: "1"})) + assert res.returncode == 0, res.stdout + res.stderr + (wheel,) = out.glob("*.whl") + with zipfile.ZipFile(wheel) as z: + assert not [n for n in z.namelist() if "_fmsw_" in n], "the editable wheel must not carry libraries" + native = proj / "src" / "foldmatch" / "_native" + built = sorted(p.name for p in native.glob("_fmsw_*")) + assert built == sorted(TREE.library_filename(isa) for isa in TREE.machine_isas()) + for name in built: + current = TREE_SHA.encode("ascii") in (native / name).read_bytes() + assert current, f"{name} does not embed source sha256 {TREE_SHA}" + assert not list(native.glob("._fmsw_*")), "temporary build files left behind" + + +_TAG_PROBE = "import json, hatch_build; print(json.dumps(hatch_build.target_platform()))" + + +@pytest.mark.parametrize("env, expect", [ + ({}, "native"), + ({"MACOSX_DEPLOYMENT_TARGET": "12.3"}, "macos-12"), + ({"MACOSX_DEPLOYMENT_TARGET": "10.9"}, "macos-10.9"), + ({"ARCHFLAGS": "-arch x86_64"}, "macos-x86_64"), + ({"ARCHFLAGS": "-arch arm64 -arch x86_64"}, "macos-universal2"), +]) +def test_hook_platform_tag(build_python, env, expect): + if expect.startswith("macos") and sys.platform != "darwin": + pytest.skip("macOS-only case") + res = _run([build_python, "-c", _TAG_PROBE], cwd=ROOT, env=_build_env(**env)) + if expect == "macos-universal2": + assert res.returncode != 0 and "universal2" in res.stderr, res.stderr + return + assert res.returncode == 0, res.stderr + tag, machine, flags = json.loads(res.stdout) + host = platform.machine() + if sys.platform != "darwin": + import sysconfig + assert tag == re.sub(r"[^A-Za-z0-9_]", "_", sysconfig.get_platform()) and flags == [] + assert TREE.normalized_machine(machine) == TREE.normalized_machine() + return + if expect == "native": + assert (tag, machine, flags) == (f"macosx_11_0_{host}", host, ["-mmacosx-version-min=11.0"]) + elif expect == "macos-12": + assert (tag, flags) == (f"macosx_12_0_{host}", ["-mmacosx-version-min=12.3"]) + elif expect == "macos-10.9": + assert tag == ("macosx_11_0_arm64" if host == "arm64" else "macosx_10_9_x86_64") + elif expect == "macos-x86_64": + assert tag == "macosx_11_0_x86_64" and machine == "x86_64" + assert flags == (["-arch", "x86_64"] if host != "x86_64" else []) + ["-mmacosx-version-min=11.0"] diff --git a/tests/test_parquet_embeddings.py b/tests/test_parquet_embeddings.py new file mode 100644 index 0000000..9e0aa07 --- /dev/null +++ b/tests/test_parquet_embeddings.py @@ -0,0 +1,382 @@ +"""P9.B4: a Parquet row holds one vector, and the embeddings read have one dimension. + +stream_embeddings, the reader of 'fm-search build embeddings' and 'fm-search query embedding', reshaped each batch of a +Parquet file's 'embedding' column into one row per id. Residue embeddings flattened one table per row (rows of +different lengths) were cut into equal slices that mixed chains, or stopped on a numpy error that named no file, and a +null row was sliced the same way. Every row of a file must now hold as many values as its first row, across batches; a +row with no values, NaN or infinite values and an 'embedding' column that is not a list of floats are refused, naming +the file and the row or the type. A file that cannot be read as Parquet, or without the column id or embedding, is +refused, naming the file: it stopped on pyarrow's message, which named no file, or on a traceback. The embeddings read +have one dimension, the first one's at build and the database's at query: two dimensions stopped a build on numpy's +concatenate error, and a query on an empty FAISS AssertionError. A .pt and a .csv of one stem, two embeddings under one +id, are refused.""" +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import torch + +from foldmatch.cli import search +from foldmatch.errors import RefusalError +from foldmatch.search.faiss_database import FaissEmbeddingDatabase +from foldmatch.utils.data import stream_embeddings +from foldmatch.writer.batch_writer import CsvBatchWriter, ParquetBatchWriter, TensorBatchWriter + +_RESOURCES = Path(__file__).parent / "resources" +_ROW_ADVICE = ("A row holds one embedding vector; rows of different lengths are usually residue embeddings flattened " + "by an earlier FoldMatch: compute chain embeddings with 'fm-embedding from-structures chain' or " + "'fm-embedding from-sequences chain'.") +_ONE_DIMENSION = "all embeddings of a database have one dimension." +_DATABASE_DIMENSION = "query it with embeddings of the kind it was built from." + + +def _vectors(dim: int, prefix: str = "c", n: int = 5, seed: int = 0) -> dict: + generator = torch.Generator().manual_seed(seed) + return {f"{prefix}{i}.A": torch.randn(dim, generator=generator) for i in range(n)} + + +def _load(path: Path) -> torch.Tensor: + return torch.load(path, map_location="cpu", weights_only=True) + + +def _residues() -> dict: + """The ESM3 residue embeddings of chains 1acb.A (243 x 1536) and 1acb.B (65 x 1536), BOS and EOS rows included.""" + return {name: _load(_RESOURCES / "embeddings" / f"{name}.pt") for name in ("1acb.A", "1acb.B")} + + +def _residues_64() -> dict: + """Residue embeddings of 64 values per residue of chains r7 (7 residues, 448 values) and r5 (5 residues, 320).""" + generator = torch.Generator().manual_seed(1) + return {"r7": torch.randn(7, 64, generator=generator), "r5": torch.randn(5, 64, generator=generator)} + + +def _chains() -> dict: + """The FoldMatch chain embeddings of the five test chains, 1-D [1536], in file name order.""" + return {path.stem: _load(path) for path in sorted((_RESOURCES / "chain_embeddings").glob("*.pt"))} + + +def _parquet(path: Path, rows: list, ids: list, kind=pa.list_(pa.float32())) -> Path: + """A Parquet file of the columns id and embedding written with pyarrow, for rows that FoldMatch does not write.""" + pq.write_table(pa.table({"id": ids, "embedding": pa.array(rows, type=kind)}), path) + return path + + +def _flat_residues(path: Path, tables: dict) -> Path: + """Residue embeddings in Parquet as FoldMatch wrote them: one row per chain, its [L, D] table flattened into a row + of list (no shape stored).""" + return _parquet(path, [table.numpy().ravel() for table in tables.values()], list(tables)) + + +def _read(path: Path, **kwargs) -> tuple: + """The ids (in order), the [N, D] vectors and the batch shapes that stream_embeddings reads from ``path``.""" + ids, batches = [], [] + for batch_ids, batch in stream_embeddings(path, **kwargs): + ids.extend(batch_ids) + batches.append(batch) + return ids, np.concatenate(batches), [batch.shape for batch in batches] + + +class ParquetEmbeddingsTest(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + + def folder(self, name: str) -> Path: + folder = self.tmp / name + folder.mkdir() + return folder + + def shard(self, folder: Path, embeddings: dict, df_id: str = "inference", rank: int = 0) -> Path: + """The shard {df_id}-{rank}.parquet that fm-embedding and inference write to Parquet, through their writer.""" + writer = ParquetBatchWriter(str(folder), df_id) + trainer = SimpleNamespace(global_rank=rank) + writer.write_on_batch_end(trainer, None, (list(embeddings.values()), list(embeddings)), None, None, 0, 0) + writer.on_predict_end(trainer, None) + return folder / f"{df_id}-{rank}.parquet" + + def files(self, writer, folder: Path, embeddings: dict) -> Path: + """The .pt or .csv files that fm-embedding and inference write, through their writer.""" + files = writer(str(folder)) + for dom_id, embedding in embeddings.items(): + files._write_embedding(embedding, dom_id) + return folder + + def refusal(self, path: Path, **kwargs) -> str: + """The message of the RefusalError with which stream_embeddings refuses the file or folder ``path``.""" + with self.assertRaises(RefusalError) as refused: + _read(path, **kwargs) + return str(refused.exception) + + def test_chain_parquet_is_read_bit_for_bit(self): + """Chain or assembly embeddings as ParquetBatchWriter writes them (D 1536 and 64) read bit for bit and in + order, at the default batch size and at 2 and 1, which split the row group.""" + for dim in (1536, 64): + vectors = _vectors(dim) + folder = self.folder(f"d{dim}") + self.shard(folder, vectors) + for batch_size, rows in ((32768, [5]), (2, [2, 2, 1]), (1, [1] * 5)): + with self.subTest(dim=dim, batch_size=batch_size): + ids, read, shapes = _read(folder, batch_size=batch_size) + self.assertEqual(ids, list(vectors)) + self.assertEqual(shapes, [(n, dim) for n in rows]) + self.assertEqual(read.dtype, np.float32) + np.testing.assert_array_equal(read, np.stack([v.numpy() for v in vectors.values()])) + + def test_other_lists_of_floats_are_read(self): + """An 'embedding' column of large lists, of fixed-size lists or of doubles holds one vector per row too, read + as float32 as before.""" + vectors = _vectors(64) + rows = [v.numpy() for v in vectors.values()] + folder = self.folder("kinds") + for name, kind in (("large", pa.large_list(pa.float32())), ("fixed", pa.list_(pa.float32(), 64)), + ("double", pa.list_(pa.float64()))): + with self.subTest(kind=str(kind)): + ids, read, _ = _read(_parquet(folder / f"{name}.parquet", rows, list(vectors), kind)) + self.assertEqual(ids, list(vectors)) + np.testing.assert_array_equal(read, np.stack(rows)) + + def test_residue_rows_are_refused(self): + """Residue embeddings flattened one table per row are refused, naming the file and two rows of different + lengths, instead of cut into equal slices (r7 and r5 made 2 vectors of 384 values): also with batch size 1, + since the file's first row sets the length of all its batches. When each file holds one chain, the files + differ in dimension. After a chain shard, the row lengths of the file's first batch are checked before its + dimension, so the residue advice is given, unless that batch holds one row (batch size 1).""" + tables = {"esm3": _residues(), "width-64": _residues_64()} + for name, residues in tables.items(): + path = _flat_residues(self.folder(name) / "inference-0.parquet", residues) + (first, a), (second, b) = residues.items() + expected = f"{path}: row {first} has {a.numel()} values, but row {second} has {b.numel()}. {_ROW_ADVICE}" + for batch_size in (32768, 1): + with self.subTest(tables=name, batch_size=batch_size): + self.assertEqual(self.refusal(path.parent, batch_size=batch_size), expected) + folder = self.folder("one-chain-per-file") + files = [_flat_residues(folder / f"inference-{rank}.parquet", {name: table}) + for rank, (name, table) in enumerate(_residues().items())] + with self.subTest(tables="one chain per file"): + self.assertEqual(self.refusal(folder), f"{files[1]} (row 1acb.B) has 99840 values, but {files[0]} (row " + f"1acb.A) has 373248: {_ONE_DIMENSION}") + folder = self.folder("after-a-chain-shard") + self.shard(folder, _vectors(64), df_id="a") + path = _flat_residues(folder / "inference-0.parquet", tables["width-64"]) + with self.subTest(tables="after a chain shard"): + self.assertEqual(self.refusal(folder), f"{path}: row r7 has 448 values, but row r5 has 320. {_ROW_ADVICE}") + # at batch size 1 the first batch holds one row, whose length is refused as a dimension + self.assertEqual(self.refusal(folder, batch_size=1), f"{path} (row r7) has 448 values, but " + f"{folder / 'a-0.parquet'} (row c0.A) has 64: " + f"{_ONE_DIMENSION}") + + def test_rows_without_values_or_with_nan_are_refused(self): + """A row with no embedding (null, or an empty list) and a row with NaN or infinite values (a NaN chain + embedding as ParquetBatchWriter writes it, infinity, a null inside the list) are refused, naming the file + and the row: the batch was reshaped around a null row, and a NaN embedding is never found.""" + vectors = _vectors(64, n=3) + rows = [v.numpy() for v in vectors.values()] + folder = self.folder("rows") + infinite = rows[1].copy() + infinite[[3, 5]] = np.inf + hole = rows[1].tolist() + hole[7] = None + never = "such an embedding is never found, and as a query it finds nothing." + cases = {"null": (_parquet(folder / "null.parquet", [rows[0], None, rows[2]], list(vectors)), + "row c1.A has no embedding."), + "empty": (_parquet(folder / "empty.parquet", [rows[0], [], rows[2]], list(vectors)), + "row c1.A has no embedding."), + "nan": (self.shard(self.folder("nan"), dict(vectors, **{"c1.A": torch.full((64,), float("nan"))})), + f"row c1.A holds 64 NaN or infinite value(s) among its 64: {never}"), + "infinity": (_parquet(folder / "infinity.parquet", [rows[0], infinite, rows[2]], list(vectors)), + f"row c1.A holds 2 NaN or infinite value(s) among its 64: {never}"), + "null value": (_parquet(folder / "hole.parquet", [rows[0], hole, rows[2]], list(vectors)), + f"row c1.A holds 1 NaN or infinite value(s) among its 64: {never}")} + for name, (path, text) in cases.items(): + with self.subTest(row=name): + self.assertEqual(self.refusal(path), f"{path}: {text}") + + def test_a_column_of_another_type_is_refused(self): + """An 'embedding' column that is not a list of floating-point numbers is refused, naming the file and the + type.""" + folder = self.folder("types") + for name, rows, kind, stored in (("text", ["0.1,0.2,0.3"], pa.string(), "string"), + ("integers", [[1, 2, 3]], pa.list_(pa.int64()), "list")): + with self.subTest(kind=stored): + path = _parquet(folder / f"{name}.parquet", rows, ["c0.A"], kind) + self.assertEqual(self.refusal(path), f"{path}: its 'embedding' column is of type {stored}, not a list " + f"of floating-point numbers: a row holds one embedding vector.") + + def test_a_file_that_cannot_be_read_or_lacks_a_column_is_refused(self): + """A Parquet file that cannot be read and one without the column id or embedding are refused, naming the + file: an empty file after a chain shard, a shard cut short (its first half, as an interrupted run leaves it: + the footer is written last), a shard with a corrupted page (its first page header zeroed) and a shard's rows + under the columns name and vector, or without id. They stopped on pyarrow's message, which names no file, or + on a traceback: an OSError, or a KeyError (iter_batches drops a missing column without a word). A folder named + *.parquet, such as the dataset folder that Spark writes, is refused as a folder, and a broken link with the + system's reason, which pyarrow's message gives after the whole path.""" + shard = self.shard(self.folder("shard"), _vectors(64)) + data, table = shard.read_bytes(), pq.read_table(shard) + empty = self.folder("empty") + self.shard(empty, _vectors(64), df_id="a") + (empty / "inference-0.parquet").write_bytes(b"") + folder = self.folder("files") + cut, corrupted = folder / "cut.parquet", folder / "corrupted.parquet" + cut.write_bytes(data[:len(data) // 2]) + corrupted.write_bytes(data[:4] + bytes(16) + data[20:]) # the first page header follows the magic bytes PAR1 + renamed, no_id = folder / "renamed.parquet", folder / "no-id.parquet" + pq.write_table(table.rename_columns(["name", "vector"]), renamed) + pq.write_table(table.drop_columns(["id"]), no_id) + dataset = self.folder("dataset") + (dataset / "embeddings.parquet").mkdir() + broken = self.folder("broken") + (broken / "inference-0.parquet").symlink_to(broken / "moved.parquet") + again = "Write it again: an interrupted run or a damaged copy leaves an unreadable file." + columns = "a Parquet embedding file holds the columns id and embedding." + cases = {"empty": (empty, f"{empty / 'inference-0.parquet'} could not be read as Parquet (Parquet file size is " + f"0 bytes). {again}"), + "cut short": (cut, f"{cut} could not be read as Parquet (Parquet magic bytes not found in footer. " + f"Either the file is corrupted or this is not a parquet file.). {again}"), + "corrupted page": (corrupted, f"{corrupted} could not be read as Parquet (Couldn't deserialize " + f"thrift: TProtocolException: Invalid data). {again}"), + "name and vector": (renamed, f"{renamed} has no 'id' or 'embedding' column: {columns}"), + "no id": (no_id, f"{no_id} has no 'id' column: {columns}"), + "folder": (dataset, f"{dataset / 'embeddings.parquet'} is a folder, not a Parquet file."), + "broken link": (broken, f"{broken / 'inference-0.parquet'} could not be opened (No such file or " + f"directory).")} + for name, (path, text) in cases.items(): + with self.subTest(file=name): + self.assertEqual(self.refusal(path), text) + + def test_two_dimensions_are_refused(self): + """Embeddings of two dimensions are refused, naming both, a Parquet file with its first row (Parquet files + are read first, then .pt, then .csv), and a build writes no database: it stopped on numpy's concatenate + error, which names no file.""" + two_shards = self.folder("two-shards") + a = self.shard(two_shards, _vectors(1536, "a"), df_id="a") + b = self.shard(two_shards, _vectors(64, "b"), df_id="b") + shard_and_pt = self.folder("shard-and-pt") + c = self.shard(shard_and_pt, _vectors(1536, "c"), df_id="c") + self.files(TensorBatchWriter, shard_and_pt, {"x.A": _vectors(64)["c0.A"]}) + pt_and_csv = self.folder("pt-and-csv") + self.files(TensorBatchWriter, pt_and_csv, {"y.A": _vectors(1536)["c0.A"]}) + self.files(CsvBatchWriter, pt_and_csv, {"z.A": _vectors(64)["c0.A"]}) + cases = {"two shards": (two_shards, f"{b} (row b0.A) has 64 values, but {a} (row a0.A) has 1536"), + "shard and .pt": (shard_and_pt, f"{shard_and_pt / 'x.A.pt'} has 64 values, but {c} (row c0.A) has " + f"1536"), + ".pt and .csv": (pt_and_csv, f"{pt_and_csv / 'z.A.csv'} has 64 values, but {pt_and_csv / 'y.A.pt'} " + f"has 1536")} + for name, (folder, text) in cases.items(): + with self.subTest(files=name): + self.assertEqual(self.refusal(folder), f"{text}: {_ONE_DIMENSION}") + self.enterContext(mock.patch.object(search, "set_log_level")) + db = self.folder("db") / "mixed" + with self.assertRaises(RefusalError): + search.build_database_from_embeddings(embedding_folder=two_shards, output_db=str(db), file_extension=None, + use_gpu_index=False) + self.assertEqual(list(db.parent.iterdir()), []) + + def test_a_query_of_another_dimension_is_refused_before_faiss(self): + """'fm-search query embedding' refuses query embeddings of another dimension than the database's, naming the + file (or Parquet file and row), before FAISS (whose AssertionError had no message), and writes no output: 64 + values against 1536, 1536 against 64, and residue embeddings in Parquet whose rows all have one length (here + one chain), which a build cannot tell from vectors. Residue rows of two lengths get the residue advice: the + row lengths are checked before the database's dimension.""" + self.enterContext(mock.patch.object(search, "set_log_level")) + dbs = {1536: self.folder("db1536") / "chains", 64: self.folder("db64") / "vectors"} + sources = {1536: _RESOURCES / "chain_embeddings", + 64: self.files(TensorBatchWriter, self.folder("v64"), _vectors(64))} + for dim, db in dbs.items(): + search.build_database_from_embeddings(embedding_folder=sources[dim], output_db=str(db), file_extension=None, + use_gpu_index=False) + csv = self.files(CsvBatchWriter, self.folder("q64"), _vectors(64, "q", n=2)) / "q0.A.csv" + shard = self.shard(self.folder("q1536"), _vectors(1536, "q", n=2)) + residue = _flat_residues(self.folder("residue") / "inference-0.parquet", {"1acb.B": _residues()["1acb.B"]}) + residues = _flat_residues(self.folder("residues") / "inference-0.parquet", _residues_64()) + cases = {"64 against 1536": (1536, csv.parent, f"{csv} has 64 values, but the database has dimension 1536: " + f"{_DATABASE_DIMENSION}"), + "1536 against 64": (64, shard.parent, f"{shard} (row q0.A) has 1536 values, but the database has " + f"dimension 64: {_DATABASE_DIMENSION}"), + "residue rows": (1536, residue.parent, f"{residue} (row 1acb.B) has 99840 values, but the database " + f"has dimension 1536: {_DATABASE_DIMENSION}"), + "residue rows of two lengths": (64, residues.parent, f"{residues}: row r7 has 448 values, but row r5 " + f"has 320. {_ROW_ADVICE}")} + hits = self.tmp / "hits.tsv" + for name, (dim, queries, text) in cases.items(): + with self.subTest(query=name): + with mock.patch.object(FaissEmbeddingDatabase, "search_batch") as search_batch: + with self.assertRaises(RefusalError) as refused: + search.query_database_from_embedding(db_path=str(dbs[dim]), embedding_folder=queries, + output_file=str(hits), file_extension=None, top_k=5, + threshold=None, use_gpu_index=False) + self.assertEqual(str(refused.exception), text) + search_batch.assert_not_called() + self.assertFalse(hits.exists()) + + def test_build_and_query_from_chain_parquet(self): + """Chain embeddings in Parquet build the database that the same vectors in .pt files build, and query it as + before: the five chain fixtures, written by ParquetBatchWriter in the shards of two ranks, each find + themselves first.""" + self.enterContext(mock.patch.object(search, "set_log_level")) + chains = _chains() + names = list(chains) + shards = self.folder("shards") + self.shard(shards, {name: chains[name] for name in names[:3]}, rank=0) + self.shard(shards, {name: chains[name] for name in names[3:]}, rank=1) + databases = {} + for source, folder in (("parquet", shards), ("pt", _RESOURCES / "chain_embeddings")): + db = self.folder(source) / "chains" + search.build_database_from_embeddings(embedding_folder=folder, output_db=str(db), file_extension=None, + use_gpu_index=False) + databases[source] = db + built = FaissEmbeddingDatabase(databases["parquet"].parent, "chains") + built.load_database() + self.assertEqual((built.dimension, built.chain_ids), (1536, names)) + for suffix in (".index", ".metadata"): + with self.subTest(file=suffix): + self.assertEqual((databases["parquet"].parent / f"chains{suffix}").read_bytes(), + (databases["pt"].parent / f"chains{suffix}").read_bytes()) + hits = self.tmp / "hits.tsv" + search.query_database_from_embedding(db_path=str(databases["parquet"]), embedding_folder=shards, + output_file=str(hits), file_extension=None, top_k=5, threshold=None, + use_gpu_index=False) + top_hits = {} + for line in hits.read_text().splitlines(): + query, hit, score = line.split("\t")[:3] + top_hits.setdefault(query, (hit, float(score))) + self.assertEqual(sorted(top_hits), names) + for query, (hit, score) in top_hits.items(): + with self.subTest(query=query): + self.assertEqual(hit, query) + self.assertGreater(score, 0.9999) + + def test_a_pt_and_a_csv_of_one_id_are_refused(self): + """A .pt and a .csv of one stem give two embeddings one id: refused, naming one pair and the number of others, + and --file-extension reads one kind. Residue and chain runs that wrote to one folder leave such pairs: the + pair is refused before the residue table is read.""" + vectors = _vectors(64, n=3) + both = self.folder("both") + for writer in (TensorBatchWriter, CsvBatchWriter): + self.files(writer, both, vectors) + advice =("the id of a .pt or .csv embedding is its file name without the extension. Read one kind of file " + "with --file-extension .pt or --file-extension .csv (runs that write residue and chain embeddings " + "to one folder leave such pairs).") + self.assertEqual(self.refusal(both), f"{both / 'c0.A.pt'} and {both / 'c0.A.csv'} give the same id, c0.A, as " + f"do 2 more pair(s) of .pt and .csv files: {advice}") + for extension in (".pt", ".csv"): + with self.subTest(file_extension=extension): + ids, read, _ = _read(both, file_extension=extension) + self.assertEqual(ids, list(vectors)) + self.assertEqual(read.shape, (3, 64)) + mixed = self.folder("residue-and-chain") + self.files(CsvBatchWriter, mixed, {"1acb.B": _residues()["1acb.B"]}) + self.files(TensorBatchWriter, mixed, {"1acb.B": _chains()["1acb.B"]}) + self.assertEqual(self.refusal(mixed), f"{mixed / '1acb.B.pt'} and {mixed / '1acb.B.csv'} give the same id, " + f"1acb.B: {advice}") + ids, read, _ = _read(mixed, file_extension=".pt") + self.assertEqual(ids, ["1acb.B"]) + np.testing.assert_array_equal(read[0], _chains()["1acb.B"].numpy()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parquet_residue_embeddings.py b/tests/test_parquet_residue_embeddings.py new file mode 100644 index 0000000..e2cf65f --- /dev/null +++ b/tests/test_parquet_residue_embeddings.py @@ -0,0 +1,327 @@ +"""P9.B5: residue embeddings in Parquet keep their shape. + +ParquetBatchWriter wrote the residue embeddings of 'fm-embedding ... residue' and 'inference residue-embedding' one row +per chain, its [L, D] table flattened into list, which stores no shape: ResidueEmbeddingFromParquet and +ResidueAssemblyEmbeddingFromParquet were given D (embedding_dim) to reshape each row, read as a Python list. With +residue_embeddings=True the writer stores one list of D values per residue, list>, in row +groups of about 256 MB, so D is in the type: the readers take it from there and refuse a flat file and one that cannot +be read, and stream_embeddings refuses the new files as residue embeddings. Without the flag the writer writes what it +wrote before, but refuses a table. The CHANGELOG's conversion of a flat file is run as it is written there.""" +import re +import tempfile +import textwrap +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq +import torch + +from foldmatch.cli import search +from foldmatch.dataset.residue_assembly_embedding_from_parquet import ResidueAssemblyEmbeddingFromParquet +from foldmatch.dataset.residue_embedding_from_parquet import ResidueEmbeddingFromParquet +from foldmatch.errors import RefusalError +from foldmatch.search import scoring_system +from foldmatch.types.api_types import SrcLocation +from foldmatch.utils.data import stream_embeddings +from foldmatch.writer.batch_writer import DEFAULT_BYTE_BUDGET, ParquetBatchWriter + +_ROOT = Path(__file__).resolve().parents[1] +_RESOURCES = _ROOT / "tests" / "resources" +_TRAINER = SimpleNamespace(global_rank=0) +_FLAT = ("holds flat 'embedding' rows (one list of numbers per row): chain or assembly embeddings, or residue " + "embeddings as earlier versions wrote them, each table flattened. Residue embeddings in Parquet now hold one " + "list of D values per residue: recompute them, or convert an earlier file as the CHANGELOG shows.") +_RESIDUES = ("holds residue embeddings (one list of values per residue), not one embedding vector per row: compute chain " + "embeddings with 'fm-embedding from-structures chain' or 'fm-embedding from-sequences chain', or in Python " + "with foldmatch.inference.chain_inference.predict(src_stream=FILE, src_from=SrcTensorFrom.parquet, " + "out_folder=OUT), which writes one chain embedding per .csv file to the existing folder OUT.") + + +def _esm3() -> dict: + """The ESM3 residue embeddings of the five test chains, [L + 2, 1536]: the first and last rows are BOS and EOS.""" + return {path.stem: torch.load(path, map_location="cpu", weights_only=True) + for path in sorted((_RESOURCES / "embeddings").glob("*.pt"))} + + +def _tables(dim: int, lengths=(9, 1, 5), prefix: str = "t", seed: int = 0) -> dict: + generator = torch.Generator().manual_seed(seed) + return {f"{prefix}{i}.A": torch.randn(n, dim, generator=generator) for i, n in enumerate(lengths)} + + +def _write(folder: Path, tables: dict, df_id: str = "inference", batch: int = 2, **kwargs) -> Path: + """The shard {df_id}-0.parquet that the residue commands write to Parquet, through their writer, ``batch`` tables + per prediction.""" + writer = ParquetBatchWriter(str(folder), df_id, residue_embeddings=True, **kwargs) + names = list(tables) + for start in range(0, len(names), batch): + chunk = names[start:start + batch] + writer.write_on_batch_end(_TRAINER, None, ([tables[n] for n in chunk], tuple(chunk)), None, None, 0, 0) + writer.on_predict_end(_TRAINER, None) + return folder / f"{df_id}-0.parquet" + + +def _flat(path: Path, tables: dict, row_group_size=None) -> Path: + """Residue embeddings as earlier versions wrote them: one row per chain, its table flattened into list.""" + rows = pa.array([table.numpy().ravel() for table in tables.values()], type=pa.list_(pa.float32())) + pq.write_table(pa.table({"id": list(tables), "embedding": rows}), path, row_group_size=row_group_size) + return path + + +def _changelog_nest(): + """The function nest of the CHANGELOG's entry on this change, defined by its code as written there.""" + text = (_ROOT / "CHANGELOG.md").read_text() + entry = text[text.index("**BREAKING — residue embeddings in Parquet keep their shape.**"):] + namespace = {} + exec(textwrap.dedent(re.search(r"```python\n(.*?)\n *```", entry, re.S).group(1)), namespace) + return namespace["nest"] + + +class ResidueParquetTest(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + + def folder(self, name: str) -> Path: + folder = self.tmp / name + folder.mkdir() + return folder + + def assert_tables(self, dataset, tables: dict): + """The items of ``dataset`` (ResidueEmbeddingFromParquet) are ``tables``, in order and bit for bit.""" + items = [dataset[i] for i in range(len(dataset))] + self.assertEqual([name for _, name in items], list(tables)) + for (tensor, name), expected in zip(items, tables.values()): + with self.subTest(item=name): + self.assertEqual(tensor.dtype, torch.float32) + self.assertTrue(torch.equal(tensor, expected), (name, tuple(tensor.shape))) + + def refusal(self, read) -> str: + with self.assertRaises(RefusalError) as refused: + read() + return str(refused.exception) + + def test_residue_embeddings_keep_their_shape(self): + """One row per chain, one list of D values per residue: list>, D in the type, no + metadata, for the ESM3 fixtures (D 1536, BOS and EOS rows included) and for D 64. The row groups after the + first keep its schema, and the readers give back every table bit for bit: an assembly, its chains' tables + concatenated; a list of two shards (of two df_ids, both written at rank 0), their tables in order.""" + esm3 = _esm3() + for dim, tables in ((1536, esm3), (64, _tables(64))): + with self.subTest(dim=dim): + path = _write(self.folder(f"d{dim}"), tables, buffer_size=2) + schema = pq.read_schema(path) + self.assertEqual(schema.names, ["id", "embedding"]) + self.assertEqual(schema.field("embedding").type, pa.list_(pa.list_(pa.float32(), dim))) + self.assertEqual(schema.field("embedding").type.value_type.list_size, dim) + self.assertIsNone(schema.metadata) + self.assertEqual(pq.ParquetFile(path).metadata.num_row_groups, (len(tables) + 1) // 2) + self.assert_tables(ResidueEmbeddingFromParquet(path), tables) + assembly = ResidueAssemblyEmbeddingFromParquet( + src_stream=[("1acb", str(_RESOURCES / "pdb" / "1acb.cif"), "1", "1acb-1")], + parquet_path=self.tmp / "d1536" / "inference-0.parquet", src_location=SrcLocation.stream) + tensor, name = assembly[0] + self.assertEqual(name, "1acb-1") + self.assertTrue(torch.equal(tensor, torch.cat([esm3["1acb.A"], esm3["1acb.B"]]))) + shards = self.folder("shards") + first, second = _tables(64, prefix="a"), _tables(64, (4, 2), prefix="b", seed=1) + self.assert_tables(ResidueEmbeddingFromParquet([_write(shards, first, df_id="a"), + _write(shards, second, df_id="b")]), {**first, **second}) + + def test_row_groups_of_about_256_mb(self): + """Residue tables are written in row groups of about 256 MB, the default byte budget (as with a residue column), + not of 16,384 chains (buffer_size): at 6 KiB per residue (D 1536) these would buffer tens of GB, and a flush + takes several times its row group. A table counts its bytes, so a given budget cuts the row groups. Without + residue_embeddings the writer has no budget, which keeps its bytes: tests/test_esm_codes_build_channel.py.""" + writer = ParquetBatchWriter(str(self.tmp), "x", residue_embeddings=True) + self.assertEqual(writer.byte_budget, DEFAULT_BYTE_BUDGET) + tables = _tables(64) # 2,304, 256 and 1,280 bytes, and 4 bytes of id each: the first crosses 2,000 bytes + path = _write(self.folder("budget"), tables, batch=1, byte_budget=2000) + meta = pq.ParquetFile(path).metadata + self.assertEqual([meta.row_group(i).num_rows for i in range(meta.num_row_groups)], [1, 2]) + self.assert_tables(ResidueEmbeddingFromParquet(path), tables) + + def test_the_writer_refuses_what_it_cannot_store(self): + """With residue_embeddings: an embedding that is not a 2-D [L, D] table of D > 0 (one vector, a stack of + tables, no values per residue) and a second D, in the batch of the first or in a later one, refused before any + of the batch is buffered; residue embeddings and a residue output together. Without it, a table: this writer + stores one vector per row.""" + stores = "this Parquet writer stores residue embeddings" + cases = {"one vector": ([torch.zeros(64)], f"{stores}, one [L, D] table per row, but x.A has shape [64]"), + "a stack": ([torch.zeros(1, 5, 64)], f"{stores}, one [L, D] table per row, but x.A has shape [1, 5, 64]"), + "no values": ([torch.zeros(5, 0)], f"{stores}, one [L, D] table per row, but x.A has shape [5, 0]"), + "two dimensions": ([torch.zeros(5, 64), torch.zeros(3, 32)], + f"{stores} of one dimension, but y.A has 32 values per residue and x.A has 64")} + for name, (embeddings, message) in cases.items(): + with self.subTest(case=name): + folder = self.folder(name.replace(" ", "-")) + writer = ParquetBatchWriter(str(folder), "inference", residue_embeddings=True) + prediction = (embeddings, ("x.A", "y.A")[:len(embeddings)]) + self.assertEqual(self.refusal(lambda: writer.write_on_batch_end(_TRAINER, None, prediction, None, None, + 0, 0)), message) + writer.on_predict_end(_TRAINER, None) + self.assertEqual(list(folder.iterdir()), []) + later = self.folder("later") + tables = _tables(64, (5,), prefix="x") + writer = ParquetBatchWriter(str(later), "inference", residue_embeddings=True) + writer.write_on_batch_end(_TRAINER, None, (list(tables.values()), tuple(tables)), None, None, 0, 0) + self.assertEqual(self.refusal(lambda: writer.write_on_batch_end( + _TRAINER, None, ([torch.zeros(2, 64), torch.zeros(3, 32)], ("y.A", "z.A")), None, None, 0, 0)), + f"{stores} of one dimension, but z.A has 32 values per residue and x0.A has 64") + writer.on_predict_end(_TRAINER, None) + self.assert_tables(ResidueEmbeddingFromParquet(later / "inference-0.parquet"), tables) + system = scoring_system.get(scoring_system.DEFAULT_ESM_CODES) + with self.assertRaises(ValueError) as refused: + ParquetBatchWriter(str(self.tmp), "x", residue_output="residue_codes", system=system, + residue_embeddings=True) + self.assertEqual(str(refused.exception), "a residue_codes writer stores chain embeddings, not residue embeddings") + plain = self.folder("plain") + writer = ParquetBatchWriter(str(plain), "inference") + prediction = ([torch.zeros(64), torch.zeros(5, 64)], ("v.A", "r.A")) + self.assertEqual(self.refusal(lambda: writer.write_on_batch_end(_TRAINER, None, prediction, None, None, 0, 0)), + "this Parquet writer stores one embedding vector per row, but r.A has shape [5, 64]: residue " + "embeddings are written with residue_embeddings=True") + writer.on_predict_end(_TRAINER, None) + self.assertEqual(list(plain.iterdir()), []) + + def test_the_plain_writer_is_unchanged(self): + """Without residue_embeddings, embedding vectors are written as before: list, no metadata, one vector + of D values per row, [1, D] and [D, 1] tensors included (one axis longer than 1). Byte for byte against the + writer before P5.3: tests/test_esm_codes_build_channel.py.""" + generator = torch.Generator().manual_seed(2) + vectors = {"a.A": torch.randn(64, generator=generator), "b.A": torch.randn(1, 64, generator=generator), + "c.A": torch.randn(64, 1, generator=generator)} + folder = self.folder("plain") + writer = ParquetBatchWriter(str(folder), "inference") + self.assertEqual(writer.schema, pa.schema([("id", pa.string()), ("embedding", pa.list_(pa.float32()))])) + writer.write_on_batch_end(_TRAINER, None, (list(vectors.values()), tuple(vectors)), None, None, 0, 0) + writer.on_predict_end(_TRAINER, None) + path = folder / "inference-0.parquet" + schema = pq.read_schema(path) + self.assertEqual(schema.field("embedding").type, pa.list_(pa.float32())) + self.assertIsNone(schema.metadata) + table = pq.read_table(path) + self.assertEqual(table.column("id").to_pylist(), list(vectors)) + np.testing.assert_array_equal(np.stack(table.column("embedding").to_numpy(zero_copy_only=False)), + np.stack([v.numpy().ravel() for v in vectors.values()])) + + def test_the_readers_refuse_a_flat_file(self): + """A flat 'embedding' column, each table flattened into one row as earlier versions wrote residue embeddings, + is refused by both readers, naming the file: they reshaped it with the embedding_dim they were given.""" + flat = _flat(self.tmp / "inference-0.parquet", _esm3()) + assemblies = [("1acb", str(_RESOURCES / "pdb" / "1acb.cif"), "1", "1acb-1")] + readers = {"ResidueEmbeddingFromParquet": lambda: ResidueEmbeddingFromParquet(flat), + "ResidueAssemblyEmbeddingFromParquet": lambda: ResidueAssemblyEmbeddingFromParquet( + src_stream=assemblies, parquet_path=flat, src_location=SrcLocation.stream)} + for name, read in readers.items(): + with self.subTest(reader=name): + self.assertEqual(self.refusal(read), f"{flat} {_FLAT}") + + def test_the_readers_refuse_other_columns_and_nulls(self): + """An 'embedding' column of another type is refused, naming its type and the type of residue embeddings; a + column of lists of D values per residue (list>, as pandas writes [L, D] lists) is read once cast + as the message says. A null row or value is refused too, which Arrow's flatten would skip, shifting the tables + after it. pyarrow 24 refuses to write a null residue in the middle of the column, and writes one at its start + or end but cannot read it back.""" + folder = self.folder("other") + doubles = pa.array([[[0.5, 0.25], [1.0, 2.0]]]) + types = {"text": (pa.array(["0.1,0.2"]), "string"), + "lists of any length": (pa.array([[[0.1, 0.2], [0.3]]], type=pa.list_(pa.list_(pa.float32()))), + "list>"), + "lists of D doubles": (doubles, "list>"), + "integers": (pa.array([[[1, 2]]], type=pa.list_(pa.list_(pa.int64(), 2))), + "list[2]>")} + for name, (column, kind) in types.items(): + with self.subTest(column=name): + path = folder / f"{name.replace(' ', '-')}.parquet" + pq.write_table(pa.table({"id": ["x.A"], "embedding": column}), path) + self.assertEqual(self.refusal(lambda: ResidueEmbeddingFromParquet(path)), + f"{path}: its 'embedding' column is of type {kind}, not residue embeddings, which are " + f"of type list> (D in the type), as " + f"ParquetBatchWriter(residue_embeddings=True) writes them. A column of lists of D " + f"values per residue converts with .cast(pa.list_(pa.list_(pa.float32(), D))).") + path = folder / "cast.parquet" + pq.write_table(pa.table({"id": ["x.A"], "embedding": doubles.cast(pa.list_(pa.list_(pa.float32(), 2)))}), path) + self.assert_tables(ResidueEmbeddingFromParquet(path), {"x.A": torch.tensor([[0.5, 0.25], [1.0, 2.0]])}) + kind = pa.list_(pa.list_(pa.float32(), 2)) + nulls = {"row": [[[0.1, 0.2]], None, [[0.5, 0.6]]], + "value": [[[0.1, 0.2]], [[None, 0.4]], [[0.5, 0.6]]]} + for name, rows in nulls.items(): + with self.subTest(null=name): + path = folder / f"null-{name}.parquet" + pq.write_table(pa.table({"id": ["a.A", "b.A", "c.A"], "embedding": pa.array(rows, type=kind)}), path) + self.assertEqual(self.refusal(lambda: ResidueEmbeddingFromParquet(path)), + f"{path}: its 'embedding' column holds a null row, residue or value.") + + def test_a_file_that_cannot_be_read_is_refused(self): + """A file that cannot be read as Parquet is refused, naming it, instead of stopping on pyarrow's message, which + names no file: an empty file and a shard whose first page is corrupted, which fails when read, not when opened. + So is a folder, which the readers used to read through pq.read_table: they read one file, or a list of files + (ResidueEmbeddingFromParquet only), such as the shards of a multi-device run that + sorted(Path(OUT).glob("*.parquet")) lists. A missing path is refused with the system's reason.""" + shards = self.folder("shards") + tables = _tables(64) + data = _write(shards, tables).read_bytes() + empty, corrupted = self.tmp / "empty.parquet", self.tmp / "corrupted.parquet" + empty.write_bytes(b"") + corrupted.write_bytes(data[:4] + bytes(16) + data[20:]) # the first page header follows the magic bytes PAR1 + again = "Write it again: an interrupted run or a damaged copy leaves an unreadable file." + causes = {empty: "Parquet file size is 0 bytes", + corrupted: "Couldn't deserialize thrift: TProtocolException: Invalid data"} + for path, cause in causes.items(): + with self.subTest(file=path.name): + self.assertEqual(self.refusal(lambda: ResidueEmbeddingFromParquet(path)), + f"{path} could not be read as Parquet ({cause}). {again}") + with self.subTest(file="folder"): + self.assertEqual(self.refusal(lambda: ResidueEmbeddingFromParquet(shards)), + f"{shards} is a folder, not a Parquet file.") + self.assert_tables(ResidueEmbeddingFromParquet(sorted(shards.glob("*.parquet"))), tables) + with self.subTest(file="missing"): + missing = self.tmp / "missing.parquet" + self.assertEqual(self.refusal(lambda: ResidueEmbeddingFromParquet(missing)), + f"{missing} could not be opened (No such file or directory).") + + def test_build_and_query_refuse_residue_embeddings(self): + """stream_embeddings, the reader of 'fm-search build embeddings' and 'fm-search query embedding', refuses + residue embeddings in Parquet, naming the file, whatever their row lengths: also one chain, whose flattened + row it could not tell from a vector. A build writes no database.""" + esm3 = _esm3() + cases = {"five-chains": esm3, "one-chain": {"1acb.B": esm3["1acb.B"]}, "width-64": _tables(64)} + for name, tables in cases.items(): + with self.subTest(tables=name): + path = _write(self.folder(name), tables) + for batch_size in (32768, 1): + self.assertEqual(self.refusal(lambda: list(stream_embeddings(path.parent, batch_size=batch_size))), + f"{path} {_RESIDUES}") + self.enterContext(mock.patch.object(search, "set_log_level")) + db = self.folder("db") / "residues" + with self.assertRaises(RefusalError): + search.build_database_from_embeddings(embedding_folder=self.tmp / "five-chains", output_db=str(db), + file_extension=None, use_gpu_index=False) + self.assertEqual(list(db.parent.iterdir()), []) + + def test_the_changelog_conversion(self): + """The CHANGELOG's nest, defined as written there, turns a flat file (in row groups of two chains; D 1536 and + 64) into one that the readers read bit for bit and stream_embeddings refuses as residue embeddings. It + refuses a D that does not divide every row, and writes nothing then.""" + nest = _changelog_nest() + for dim, tables in ((1536, _esm3()), (64, _tables(64))): + with self.subTest(dim=dim): + flat = _flat(self.folder(f"d{dim}") / "inference-0.parquet", tables, row_group_size=2) + nested = self.folder(f"nested{dim}") / "inference-0.parquet" + nest(str(flat), str(nested), dim) + self.assertEqual(pq.read_schema(nested).field("embedding").type, pa.list_(pa.list_(pa.float32(), dim))) + self.assert_tables(ResidueEmbeddingFromParquet(nested), tables) + self.assertEqual(self.refusal(lambda: list(stream_embeddings(nested))), f"{nested} {_RESIDUES}") + flat, wrong = self.tmp / "d64" / "inference-0.parquet", self.tmp / "wrong.parquet" + with self.assertRaises(ValueError) as refused: + nest(str(flat), str(wrong), 7) + self.assertEqual(str(refused.exception), f"{flat}: a row is not a multiple of 7 values") + self.assertFalse(wrong.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_preflight.py b/tests/test_preflight.py new file mode 100644 index 0000000..f706731 --- /dev/null +++ b/tests/test_preflight.py @@ -0,0 +1,404 @@ +"""Read-only FASTA pre-flight on every rank before ESM3 (G9).""" +import functools +import json +import os +import subprocess +import sys +import tempfile +import unittest +import zlib +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import numpy as np +import pytest +import torch + +import foldmatch +from foldmatch.types.api_types import Accelerator, StructureFormat +from foldmatch.utils.sequence import FastaPreflight, preflight_fasta, scan_fasta + +_TEST_PATH = Path(__file__).parent +_FIXTURE_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_TOKENIZER_CASES = _TEST_PATH / "resources" / "esm_codes" / "tokenizer_cases_v1.json" +_SRC = str(Path(foldmatch.__file__).resolve().parent.parent) +_LAUNCH_ENV = ("FOLDMATCH_RUN_ID", "SLURM_JOB_ID", "SLURM_STEP_ID", "SLURM_RESTART_COUNT", + "SLURM_PROCID", "RANK", "LOCAL_RANK", "NODE_RANK") + +# Records, in order: a(4) b(3, '<') a(5, dup) c(4, non-ASCII) d(10) a(2, dup) [empty e] b(2, dup) +_PROBLEM_FASTA = (">a x\nACDE\n>b\nmk<\n>a\nWWWWW\n>c\nACDÉ\n>d\nAAAAA AAAAA\n" + ">a\nCC\n>e\n\n>b\nZZ\n") + + +@contextmanager +def launch_env(**values): + """Run with exactly ``values`` set among the launch variables (the rest unset). + + Only those variables are saved and restored. Restoring the whole environment + would also undo what libraries set on first import (sklearn sets + KMP_DUPLICATE_LIB_OK, without which a second OpenMP runtime aborts the process). + """ + saved = {name: os.environ.get(name) for name in _LAUNCH_ENV} + try: + for name in _LAUNCH_ENV: + os.environ.pop(name, None) + os.environ.update({k: str(v) for k, v in values.items()}) + yield + finally: + for name, value in saved.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + + +def _write(path: Path, text: str) -> Path: + path.write_text(text, encoding="utf-8") + return path + + +def fake_chain_predict(on_pass=None): + """Stand-in for chain_complete_inference.predict: one shard, built from the + same dataset the real pass uses.""" + def predict(**kwargs): + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + from foldmatch.writer.batch_writer import ParquetBatchWriter + dataset = EsmProtFromFasta(kwargs["src_stream"], min_res_n=kwargs["min_res_n"]) + names = [dataset[i][1] for i in range(len(dataset))] + embeddings = [] + for name in names: + vec = np.random.default_rng(zlib.crc32(name.encode())).normal(size=8).astype(np.float32) + embeddings.append(torch.from_numpy(vec)) + writer = ParquetBatchWriter(kwargs["out_folder"], kwargs["out_name"]) + trainer = SimpleNamespace(global_rank=0) + writer.write_on_batch_end(trainer, None, (embeddings, tuple(names)), None, None, 0, 0) + writer.on_predict_end(trainer, None) + if on_pass is not None: + on_pass() + return predict + + +def _esm3_spy(): + """Patch the ESM3 loader the FASTA pass uses; any call means ESM3 was reached.""" + return patch("foldmatch.inference.chain_complete_inference.get_residue_model", + Mock(side_effect=AssertionError("ESM3 was loaded"))) + + +class PreflightCountsTest(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.fasta = _write(self.tmp / "problems.fasta", _PROBLEM_FASTA) + + def test_counts(self): + pf = preflight_fasta(self.fasta) + self.assertEqual( + (pf.entries, pf.entries_kept, pf.residues, pf.duplicates, pf.unique_entries, + pf.unique_residues, pf.max_length), + (7, 7, 30, 3, 4, 21, 10)) + self.assertEqual((pf.nonstandard_entries, pf.nonstandard_residues), (3, 4)) + self.assertEqual((pf.non_ascii_entries, pf.angle_bracket_entries), (1, 1)) + self.assertEqual(pf.duplicate_ids, ("a", "b")) + self.assertEqual((pf.non_ascii_ids, pf.angle_bracket_ids), (("c",), ("b",))) + self.assertEqual(pf.offsets.tolist(), [offset for offset, _, _ in scan_fasta(self.fasta)]) + + def test_min_res_n_applies_to_every_count_but_entries(self): + pf = preflight_fasta(self.fasta, min_res_n=4) + self.assertEqual( + (pf.entries, pf.entries_kept, pf.residues, pf.duplicates, pf.unique_residues, + pf.angle_bracket_entries, pf.non_ascii_entries, pf.nonstandard_entries), + (7, 4, 23, 1, 18, 0, 1, 1)) + kept = [offset for offset, _, length in scan_fasta(self.fasta) if length >= 4] + self.assertEqual(pf.offsets.tolist(), kept) + + def test_fixture_and_empty_file(self): + pf = preflight_fasta(_FIXTURE_FASTA) + self.assertEqual((pf.entries_kept, pf.residues, pf.duplicates, pf.max_length), (2, 303, 0, 245)) + self.assertEqual(pf.problems(), []) + empty = preflight_fasta(_write(self.tmp / "empty.fasta", "")) + self.assertEqual((empty.entries, empty.entries_kept, empty.residues, len(empty.offsets)), (0, 0, 0, 0)) + + def test_hash_collisions_are_resolved_exactly(self): + # Every id hashing to the same value must still count only true repeats. + with patch("foldmatch.utils.sequence._id_hash", return_value=42): + pf = preflight_fasta(self.fasta) + self.assertEqual((pf.duplicates, pf.unique_residues, pf.duplicate_ids), (3, 21, ("a", "b"))) + many = _write(self.tmp / "many.fasta", "".join(f">id{i}\nACDE\n" for i in range(200)) + ">id7\nAC\n") + with patch("foldmatch.utils.sequence._id_hash", side_effect=lambda name: len(name)): + pf = preflight_fasta(many) + self.assertEqual((pf.duplicates, pf.unique_residues, pf.duplicate_ids), (1, 800, ("id7",))) + + def test_strict_refuses_each_problem_and_plain_warns(self): + cases = { + "duplicate": ">a\nACDE\n>a\nACDE\n", + "angle": ">a\nACDE\n", + "non_ascii": ">a\nACDÉ\n", + } + for label, text in cases.items(): + with self.subTest(label=label): + pf = preflight_fasta(_write(self.tmp / f"{label}.fasta", text)) + self.assertEqual(len(pf.problems()), 1) + with self.assertRaises(ValueError) as ctx: + pf.check(strict=True) + self.assertIn(str(self.tmp / f"{label}.fasta"), str(ctx.exception)) + with self.assertLogs("foldmatch.utils.sequence", level="WARNING"): + pf.check(strict=False) + clean = preflight_fasta(_FIXTURE_FASTA) + clean.check(strict=True) + + def test_tokenizer_cases(self): + """F-tok: the pre-flight's per-record figures agree with the fixture.""" + with open(_TOKENIZER_CASES) as f: + cases = json.load(f)["cases"] + for case in cases: + with self.subTest(case=case["id"]): + pf = preflight_fasta(_write(self.tmp / "case.fasta", f">{case['id']}\n{case['raw']}\n")) + self.assertEqual(pf.entries, 1) + expected = case["strict_refusal"] + self.assertEqual(pf.angle_bracket_entries, int(expected == "angle_bracket")) + self.assertEqual(pf.non_ascii_entries, int(expected == "non_ascii")) + if expected is not None: + with self.assertRaises(ValueError): + pf.check(strict=True) + if not case["normalize_refuses"]: + self.assertEqual(pf.residues, case["len_normalized"]) + self.assertEqual(pf.nonstandard_residues, case["n_nonstandard"]) + + def test_dataset_takes_the_preflight_offsets(self): + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + pf = preflight_fasta(self.fasta, min_res_n=4) + with patch("foldmatch.utils.sequence._scan_records") as scan: + dataset = EsmProtFromFasta(pf, min_res_n=4) + scan.assert_not_called() + by_path = EsmProtFromFasta(self.fasta, min_res_n=4) + self.assertEqual(len(dataset), 4) + self.assertEqual([dataset[i][1] for i in range(4)], [by_path[i][1] for i in range(4)]) + self.assertEqual([dataset[i][0].sequence for i in range(4)], ["ACDE", "WWWWW", "ACDÉ", "A" * 10]) + with self.assertRaises(ValueError): + EsmProtFromFasta(pf, min_res_n=0) + + +_PREFLIGHT_CHILD = """ +import sys +from foldmatch.cli.search import _fasta_preflight +try: + _fasta_preflight(sys.argv[1], 0, strict=True) +except ValueError as exc: + print(exc) + sys.exit(3) +""" + + +class PreflightTest(unittest.TestCase): + """Every rank refuses the same way, before ESM3 is ever loaded.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.enterContext(launch_env()) + + def test_every_rank_raises_the_same_refusal(self): + fasta = _write(self.tmp / "problems.fasta", _PROBLEM_FASTA) + outputs = set() + for rank_env in ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}, + {"SLURM_JOB_ID": 5, "SLURM_PROCID": 3}): + env = {k: v for k, v in os.environ.items() if k not in _LAUNCH_ENV} + env.update({k: str(v) for k, v in rank_env.items()}) + env["PYTHONPATH"] = _SRC + env["PYTHONHASHSEED"] = str(len(outputs) + 1) # ids must not depend on hash() + child = subprocess.run([sys.executable, "-c", _PREFLIGHT_CHILD, str(fasta)], + env=env, capture_output=True, text=True) + self.assertEqual(child.returncode, 3, child.stderr) + outputs.add(child.stdout) + self.assertEqual(len(outputs), 1) + self.assertIn("3 record(s) repeat an earlier id", outputs.pop()) + + def _build(self, output_db, fasta=_FIXTURE_FASTA, **kwargs): + from foldmatch.cli.search import build_database_from_fasta + build_database_from_fasta(fasta_file=fasta, output_db=str(output_db), + tmp_embedding_folder=self.tmp, accelerator=Accelerator.cpu, **kwargs) + + def test_existing_output_is_refused_before_esm3(self): + for suffix in (".index", ".metadata", ".invlists", ".sequences", ".rescodes.json", ".rescodes.bin"): + for rank_env in ({}, {"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}): + with self.subTest(suffix=suffix, rank_env=rank_env), launch_env(**rank_env), \ + _esm3_spy() as esm3: + existing = self.tmp / f"db{suffix}" + existing.touch() + try: + with self.assertRaises(FileExistsError) as ctx: + self._build(self.tmp / "db") + finally: + existing.unlink() + self.assertIn(str(existing), str(ctx.exception)) + esm3.assert_not_called() + + def test_existing_index_is_refused_before_esm3_for_structures(self): + from foldmatch.cli.search import build_database_from_structures + (self.tmp / "sdb.index").touch() + with _esm3_spy() as esm3: + with self.assertRaises(FileExistsError): + build_database_from_structures( + structure_folder=_TEST_PATH / "resources" / "pdb", output_db=str(self.tmp / "sdb"), + tmp_embedding_folder=self.tmp, structure_format=StructureFormat.mmcif, + file_extension=".cif", accelerator=Accelerator.cpu) + esm3.assert_not_called() + + def test_strict_refusal_comes_before_esm3(self): + from foldmatch.cli import search + fasta = _write(self.tmp / "dups.fasta", ">a\nACDE\n>a\nACDE\n") + strict = functools.partial(search._fasta_preflight, strict=True) + with patch("foldmatch.cli.search._fasta_preflight", strict), _esm3_spy() as esm3: + with self.assertRaises(ValueError) as ctx: + self._build(self.tmp / "db", fasta=fasta) + self.assertIn("repeat an earlier id", str(ctx.exception)) + esm3.assert_not_called() + self.assertEqual(list(self.tmp.glob("db*")), []) + + def test_plain_build_warns_and_keeps_todays_behaviour(self): + from foldmatch.search.faiss_database import FaissEmbeddingDatabase + from foldmatch.search.sequence_store import SequenceStore + fasta = _write(self.tmp / "problems.fasta", _PROBLEM_FASTA) + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()), \ + self.assertLogs("foldmatch.utils.sequence", level="WARNING") as logs: + self._build(self.tmp / "db", fasta=fasta) + self.assertEqual(len([r for r in logs.records if r.levelname == "WARNING"]), 3) + pf = preflight_fasta(fasta) + db = FaissEmbeddingDatabase(self.tmp, "db") + db.load_database() + self.assertEqual(len(db.chain_ids), pf.entries_kept) + store = SequenceStore(self.tmp, "db") + self.assertEqual(store.meta(), {"entry_count": pf.unique_entries, + "total_residues": pf.unique_residues, + "duplicates_ignored": pf.duplicates}) + # First occurrence kept, normalised; '<' and non-ASCII kept as they are. + self.assertEqual(store.fetch(["a", "b", "c", "d"]), + {"a": "ACDE", "b": "MK<", "c": "ACDÉ", "d": "A" * 10}) + + def test_one_scan_feeds_the_preflight_and_the_dataset(self): + import foldmatch.utils.sequence as sequence + scans = [] + real_scan = sequence._scan_records + + def counting_scan(path): + scans.append(path) + return real_scan(path) + + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()), \ + patch("foldmatch.utils.sequence._scan_records", counting_scan): + self._build(self.tmp / "db") + self.assertEqual(len(scans), 1) + + def test_store_meta_must_equal_the_preflight(self): + fasta = _write(self.tmp / "grows.fasta", ">a\nACDE\n>b\nFGHIK\n") + + def append_record(): + # The FASTA changes after the pre-flight: the store would no longer match the index. + with open(fasta, "a") as f: + f.write(">c\nLMNPQ\n") + + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict(append_record)): + with self.assertRaises(ValueError) as ctx: + self._build(self.tmp / "db", fasta=fasta) + self.assertIn("pre-flight", str(ctx.exception)) + # The whole database goes, not just the store, so the same build can run again. + self.assertEqual(sorted(p.name for p in self.tmp.glob("db.*")), []) + self.assertIn("db.index, db.metadata, db.sequences", str(ctx.exception)) + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + self._build(self.tmp / "db", fasta=fasta, cleanup_embeddings=True) + self.assertTrue((self.tmp / "db.index").exists()) + + def test_nothing_to_build_is_refused_before_esm3(self): + for text, min_res_n in (("", 0), (">a\nACD\n>b\nEF GH\n", 5)): + with self.subTest(text=text, min_res_n=min_res_n), _esm3_spy() as esm3: + fasta = _write(self.tmp / "nothing.fasta", text) + with self.assertRaisesRegex(ValueError, "nothing to build"): + self._build(self.tmp / "db", fasta=fasta, min_res_n=min_res_n) + esm3.assert_not_called() + self.assertEqual(list(self.tmp.glob("db*")), []) + + def _store_problem(self, problem): + """Build a database from the fixture, then give its sequence store ``problem``.""" + import sqlite3 + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + self._build(self.tmp / "db", cleanup_embeddings=True) + store = self.tmp / "db.sequences" + if problem == "missing": + store.unlink() + return + conn = sqlite3.connect(store) + try: + if problem == "empty": + conn.execute("DELETE FROM sequences") + conn.execute("UPDATE meta SET value = 0 WHERE key IN ('entry_count', 'total_residues')") + elif problem == "incomplete": + conn.execute("DELETE FROM meta") + conn.commit() + finally: + conn.close() + + @pytest.mark.usefixtures("fmsw_libs") # the empty store with --max-evalue inf runs Stage 2 + def test_stage2_store_problems_are_refused_before_esm3(self): + from foldmatch.cli.search import query_database_from_fasta + cases = [ # (store problem, query options, expected message) + ("empty", {}, "non-empty database"), + ("incomplete", {}, "is incomplete"), + ("incomplete", {"max_evalue": float("inf")}, "is incomplete"), + ("missing", {"seq_identity": True}, "has no sequence store"), + ] + for problem, options, message in cases: + with self.subTest(problem=problem, options=options): + for path in self.tmp.glob("db*"): + path.unlink() + self._store_problem(problem) + with _esm3_spy() as esm3: + with self.assertRaisesRegex(ValueError, message): + query_database_from_fasta( + db_path=str(self.tmp / "db"), fasta_file=_FIXTURE_FASTA, + tmp_embedding_folder=self.tmp, output_file=str(self.tmp / "hits.tsv"), + accelerator=Accelerator.cpu, **options) + esm3.assert_not_called() + # Without Stage 2, or with an E-value filter switched off, an empty store is fine. + for options in ({"seq_identity": False}, {"max_evalue": float("inf")}): + with self.subTest(problem="empty", options=options): + for path in self.tmp.glob("db*"): + path.unlink() + self._store_problem("empty") + with patch("foldmatch.search.embedding_computer.chain_predict", fake_chain_predict()): + query_database_from_fasta( + db_path=str(self.tmp / "db"), fasta_file=_FIXTURE_FASTA, + tmp_embedding_folder=self.tmp, output_file=str(self.tmp / "hits.tsv"), + threshold=None, accelerator=Accelerator.cpu, **options) + + def test_stage2_store_problems_are_refused_before_the_db_search(self): + from foldmatch.cli.search import query_database_from_database + for problem, message in (("empty", "non-empty database"), ("incomplete", "is incomplete")): + with self.subTest(problem=problem): + for path in self.tmp.glob("db*"): + path.unlink() + self._store_problem(problem) + with patch("foldmatch.search.embedding_database.EmbeddingDatabase.load", + side_effect=AssertionError("database loaded")): + with self.assertRaisesRegex(ValueError, message): + query_database_from_database( + query_db_path=str(self.tmp / "db"), subject_db_path=str(self.tmp / "db"), + output_file=str(self.tmp / "hits.tsv")) + + def test_output_check_shares_the_database_path_rules_without_faiss(self): + from foldmatch.search import db_path, embedding_database + self.assertIs(embedding_database._parse_db_path, db_path.parse_db_path) + child = subprocess.run( + [sys.executable, "-c", + "import sys\n" + "from foldmatch.cli import search\n" + "search._fasta_preflight(sys.argv[1], 0, output_db=sys.argv[2])\n" + "print(sorted(m for m in sys.modules if m.startswith(('faiss', 'torch', 'esm'))))", + str(_FIXTURE_FASTA), str(self.tmp / "db")], + env=dict(os.environ, PYTHONPATH=_SRC), capture_output=True, text=True) + self.assertEqual(child.returncode, 0, child.stderr) + self.assertEqual(child.stdout.strip(), "[]") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_pins.py b/tests/test_release_pins.py new file mode 100644 index 0000000..abc798a --- /dev/null +++ b/tests/test_release_pins.py @@ -0,0 +1,399 @@ +import ast +import hashlib +import os +import pickle +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import torch + +import foldmatch +from foldmatch.utils import model +from foldmatch.utils.data import load_residue_embedding +from foldmatch.utils.esm import loaders + +_PACKAGE = Path(foldmatch.__file__).parent + + +def _listing(root): + return sorted(str(p.relative_to(root)) for p in Path(root).rglob("*")) + + +def _package_modules(): + for path in sorted(_PACKAGE.rglob("*.py")): + yield path.relative_to(_PACKAGE.parent), ast.parse(path.read_text(), filename=str(path)) + + +def _call_name(call): + func = call.func + if isinstance(func, ast.Attribute): + prefix = func.value.id + "." if isinstance(func.value, ast.Name) else "" + return prefix + func.attr + return func.id if isinstance(func, ast.Name) else None + + +class EsmWeightsPinTest(unittest.TestCase): + """G1: the ESM3 snapshot is pinned and its weights are verified without writing anything.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + loaders._hashed.clear() + + def tearDown(self): + loaders._hashed.clear() + self._tmp.cleanup() + + def _cache_layout(self, blob_name, content=b"weights"): + """A Hugging Face cache entry: snapshots// -> ../../blobs/.""" + blob = self.root / "blobs" / blob_name + blob.parent.mkdir(parents=True) + blob.write_bytes(content) + link = self.root / "snapshots" / loaders.ESM_REVISION / loaders.ESM_WEIGHTS_FILE + link.parent.mkdir(parents=True) + link.symlink_to(os.path.relpath(blob, link.parent)) + return link + + def _copied_layout(self, content): + path = self.root / "local" / loaders.ESM_WEIGHTS_FILE + path.parent.mkdir(parents=True) + path.write_bytes(content) + return path + + def test_pins(self): + self.assertEqual(loaders.ESM_REPO_ID, "rcsb/rcsb-esm") + self.assertEqual(loaders.ESM_REVISION, "636acb420daf674df37ab0220f56061a5de2f169") + self.assertEqual( + loaders.ESM_WEIGHTS_SHA256, "5ead5a135c658068db6a4f1b933e72d6110992c4668822e1c0e2dcc53e38acd9" + ) + self.assertEqual(loaders.ESM_WEIGHTS_FILE, "data/weights/esm3_sm_open_v1.pth") + + def test_blob_name_verifies_without_hashing(self): + link = self._cache_layout(loaders.ESM_WEIGHTS_SHA256) + before = _listing(self.root) + with mock.patch.object(loaders, "_file_sha256", side_effect=AssertionError("hashed")): + self.assertEqual(loaders.verify_esm_weights(link), "blob-name") + self.assertEqual(_listing(self.root), before) + + def test_wrong_blob_name_is_refused(self): + link = self._cache_layout("0" * 64) + with self.assertRaisesRegex(loaders.EsmWeightsMismatchError, "expected " + loaders.ESM_WEIGHTS_SHA256): + loaders.verify_esm_weights(link) + + def test_hidden_layout_is_hashed_once_per_process(self): + content = b"copied weights" + path = self._copied_layout(content) + before = _listing(self.root) + sha = hashlib.sha256(content).hexdigest() + spy = mock.Mock(wraps=loaders._file_sha256) + with mock.patch.object(loaders, "ESM_WEIGHTS_SHA256", sha), mock.patch.object(loaders, "_file_sha256", spy): + self.assertEqual(loaders.verify_esm_weights(path), "sha256") + self.assertEqual(loaders.verify_esm_weights(path), "sha256") + self.assertEqual(spy.call_count, 1) + self.assertEqual(_listing(self.root), before) + + def test_hidden_layout_with_wrong_content_is_refused(self): + path = self._copied_layout(b"not the weights") + with self.assertRaisesRegex(loaders.EsmWeightsMismatchError, "sha256 " + hashlib.sha256(b"not the weights").hexdigest()): + loaders.verify_esm_weights(path) + + def test_full_hash_reads_the_content_behind_a_blob_name(self): + link = self._cache_layout(loaders.ESM_WEIGHTS_SHA256, content=b"truncated download") + with self.assertRaisesRegex(loaders.EsmWeightsMismatchError, "is corrupt"): + loaders.verify_esm_weights(link, full_hash=True) + + def _data_root(self, cached): + """Resolve DataRoot against a fake cache; returns (path, snapshot_download mock, verify mock).""" + snapshot = self.root / "snapshots" / loaders.ESM_REVISION + snapshot.mkdir(parents=True) + cached_value = str(snapshot / loaders.ESM_WEIGHTS_FILE) if cached else None + saved = loaders.DataRoot._instance + loaders.DataRoot._instance = None + try: + with mock.patch.object(loaders, "try_to_load_from_cache", return_value=cached_value) as lookup, \ + mock.patch.object(loaders, "snapshot_download", return_value=str(snapshot)) as download, \ + mock.patch.object(loaders, "verify_esm_weights") as verify: + path = loaders.data_root() + self.assertIs(loaders.DataRoot(), loaders.DataRoot()) + self.assertEqual(verify.call_count, 1) + finally: + loaders.DataRoot._instance = saved + lookup.assert_called_once_with(loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_FILE, revision=loaders.ESM_REVISION) + download.assert_called_once_with(repo_id=loaders.ESM_REPO_ID, revision=loaders.ESM_REVISION) + self.assertEqual(path, snapshot) + return verify + + def test_data_root_checks_a_cached_snapshot_by_blob_name(self): + verify = self._data_root(cached=True) + verify.assert_called_once_with( + self.root / "snapshots" / loaders.ESM_REVISION / loaders.ESM_WEIGHTS_FILE, full_hash=False + ) + + def test_data_root_hashes_a_fresh_download_once(self): + verify = self._data_root(cached=False) + verify.assert_called_once_with( + self.root / "snapshots" / loaders.ESM_REVISION / loaders.ESM_WEIGHTS_FILE, full_hash=True + ) + + def test_failed_verification_leaves_no_data_root(self): + saved = loaders.DataRoot._instance + loaders.DataRoot._instance = None + try: + with mock.patch.object(loaders, "try_to_load_from_cache", return_value=None), \ + mock.patch.object(loaders, "snapshot_download", return_value=str(self.root)), \ + mock.patch.object(loaders, "verify_esm_weights", side_effect=loaders.EsmWeightsMismatchError("mismatch")): + with self.assertRaisesRegex(loaders.EsmWeightsMismatchError, "mismatch"): + loaders.data_root() + self.assertIsNone(loaders.DataRoot._instance) + finally: + loaders.DataRoot._instance = saved + + def test_mismatch_is_reported_as_a_clean_cli_error(self): + """The CLIs render ValueError as one 'Error: ...' line (exit 1), not a traceback.""" + self.assertTrue(issubclass(loaders.EsmWeightsMismatchError, ValueError)) + link = self._cache_layout("0" * 64) + import importlib + import io + from contextlib import redirect_stderr + for name in ("foldmatch.cli.search", "foldmatch.cli.embedding", "foldmatch.cli.inference"): + cli = importlib.import_module(name) + with self.subTest(cli=name), \ + mock.patch.object(cli, "app", side_effect=lambda: loaders.verify_esm_weights(link)): + stderr = io.StringIO() + with redirect_stderr(stderr), self.assertRaises(SystemExit) as ctx: + cli.main() + self.assertEqual(ctx.exception.code, 1) + self.assertIn("Error: ESM3 weights", stderr.getvalue()) + self.assertIn("run download-models again", stderr.getvalue()) + self.assertNotIn("Traceback", stderr.getvalue()) + + def test_local_cache_holds_the_pinned_blob(self): + cached = loaders.try_to_load_from_cache( + loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_FILE, revision=loaders.ESM_REVISION + ) + if not isinstance(cached, str): + self.skipTest(f"{loaders.ESM_REPO_ID}@{loaders.ESM_REVISION} is not in the local Hugging Face cache") + if not re.fullmatch(r"[0-9a-f]{64}", Path(os.path.realpath(cached)).name): + self.skipTest("the local Hugging Face cache does not use blob symlinks") + with mock.patch.object(loaders, "_file_sha256", side_effect=AssertionError("hashed")): + self.assertEqual(loaders.verify_esm_weights(cached), "blob-name") + + +class DependencyPinsTest(unittest.TestCase): + """The runtime dependencies in pyproject.toml are exact pins; threadpoolctl is one of them (P7.0d: the esm-codes + Stage 2 checks and limits its BLAS threads with it, S6/S16).""" + + def setUp(self): + path = _PACKAGE.parent.parent / "pyproject.toml" + if not path.is_file(): + self.skipTest(f"no pyproject.toml next to the package ({path}): not a source tree") + import tomllib + self.dependencies = tomllib.loads(path.read_text())["project"]["dependencies"] + + def test_every_dependency_is_an_exact_pin(self): + self.assertGreaterEqual(len(self.dependencies), 12) + for requirement in self.dependencies: + with self.subTest(requirement=requirement): + self.assertRegex(requirement, r"^[A-Za-z0-9][A-Za-z0-9._-]*==[0-9][0-9A-Za-z.+!]*$") + + def test_threadpoolctl_is_declared(self): + self.assertIn("threadpoolctl==3.6.0", self.dependencies) + + def test_faiss_cpu_is_the_threaded_wheel(self): + """faiss-cpu 1.15.1 (P9): its Linux wheels bundle an OpenMP OpenBLAS, where 1.13.2's was single-threaded.""" + self.assertIn("faiss-cpu==1.15.1", self.dependencies) + self.assertFalse([r for r in self.dependencies if r.startswith("faiss") and r != "faiss-cpu==1.15.1"]) + + def test_the_index_phase_names_the_pinned_faiss_cpu(self): + """The index phase's line about a single-threaded FAISS BLAS tells to install the pinned faiss-cpu.""" + from foldmatch.cli import search + self.assertIn(f"faiss-cpu=={search._FAISS_CPU_PIN}", self.dependencies) + + +class _NotAWeight: + pass + + +class TorchLoadWeightsOnlyTest(unittest.TestCase): + """G5: every torch.load in the package refuses pickled code (weights_only=True).""" + + def test_every_torch_load_passes_weights_only(self): + calls = [] + for module, tree in _package_modules(): + for node in ast.walk(tree): + if isinstance(node, ast.Call) and _call_name(node) == "torch.load": + weights_only = [k.value for k in node.keywords if k.arg == "weights_only"] + calls.append((f"{module}:{node.lineno}", weights_only)) + self.assertGreaterEqual(len(calls), 4) + for where, weights_only in calls: + self.assertEqual(len(weights_only), 1, where) + self.assertIsInstance(weights_only[0], ast.Constant, where) + self.assertIs(weights_only[0].value, True, where) + + def test_residue_embedding_loader_refuses_pickled_objects(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "x.pt" + torch.save(torch.arange(6.0).reshape(2, 3), path) + self.assertTrue(torch.equal(load_residue_embedding(path), torch.arange(6.0).reshape(2, 3))) + torch.save({"embedding": _NotAWeight()}, path) + with self.assertRaises(pickle.UnpicklingError): + load_residue_embedding(path) + + +# Hugging Face calls that take a revision; esm's own from_pretrained fetches EvolutionaryScale +# repositories at whatever revision main points to, so the package must not call it. +_HF_FETCHES = {"snapshot_download", "hf_hub_download", "try_to_load_from_cache", "hf_hub_url"} +_UNPINNED_FETCHES = {"from_pretrained"} +_COMMIT = re.compile(r"[0-9a-f]{40}") +_UNRESOLVED = "" +# ``revision=.bundle_revision``: the esm-codes bundle loader takes its revision from a scoring-system registry +# entry, the only place a bundle revision exists (plan P4). Such a call is pinned when every registry entry's +# bundle_revision is a 40-hex commit (EsmCodesSystem refuses anything else; checked again here). +_REGISTRY_PIN = "" + + +def _registry_is_pinned(): + from foldmatch.search import scoring_system + return all(isinstance(s.bundle_revision, str) and _COMMIT.fullmatch(s.bundle_revision) is not None + for s in scoring_system.REGISTRY.values()) + + +def _is_pinned(revision): + if revision == _REGISTRY_PIN: + return _registry_is_pinned() + return isinstance(revision, str) and _COMMIT.fullmatch(revision) is not None + + +def _resolve(node, scope, seen=()): + if isinstance(node, ast.Constant): + return node.value + if isinstance(node, ast.Attribute) and node.attr == "bundle_revision": + return _REGISTRY_PIN + if isinstance(node, ast.Name) and node.id in scope and node.id not in seen: + return _resolve(scope[node.id], scope, seen + (node.id,)) + return _UNRESOLVED + + +def _is_main_guard(node): + test = node.test if isinstance(node, ast.If) else None + return ( + isinstance(test, ast.Compare) and isinstance(test.left, ast.Name) and test.left.id == "__name__" + and any(isinstance(c, ast.Constant) and c.value == "__main__" for c in test.comparators) + ) + + +def _revisions_in(tree, module): + """Every Hugging Face revision a module declares or passes, as (where, value). + + A *REVISION* constant or a ``revision=`` argument resolves through literals, module + constants and parameter defaults; anything else, or a missing ``revision=``, is not a pin. + """ + constants = { + target.id: node.value + for node in tree.body if isinstance(node, (ast.Assign, ast.AnnAssign)) + for target in (node.targets if isinstance(node, ast.Assign) else [node.target]) + if isinstance(target, ast.Name) and node.value is not None + } + found = [ + (f"{module}:{name}", _resolve(value, constants)) + for name, value in constants.items() if "REVISION" in name.upper() + ] + + def visit(node, scope): + if _is_main_guard(node): + return + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + args = node.args + positional = args.posonlyargs + args.args + defaults = dict(zip([a.arg for a in positional[len(positional) - len(args.defaults):]], args.defaults)) + defaults.update({a.arg: d for a, d in zip(args.kwonlyargs, args.kw_defaults) if d is not None}) + scope = {**scope, **{a.arg: defaults.get(a.arg, ast.Name(id="")) for a in + positional + args.kwonlyargs}} + if isinstance(node, ast.Call): + name = (_call_name(node) or "").split(".")[-1] + where = f"{module}:{node.lineno} {name}()" + if name in _HF_FETCHES: + revision = [k.value for k in node.keywords if k.arg == "revision"] + found.append((where, _resolve(revision[0], scope) if revision else None)) + elif name in _UNPINNED_FETCHES: + found.append((where, None)) + for child in ast.iter_child_nodes(node): + visit(child, scope) + + visit(tree, constants) + return found + + +class ReleasePinTest(unittest.TestCase): + """G11: every Hugging Face revision in the package is a 40-hex commit; None (unpinned) is refused.""" + + @classmethod + def setUpClass(cls): + cls.revisions = [found for module, tree in _package_modules() for found in _revisions_in(tree, module)] + + def test_every_revision_is_a_commit(self): + unpinned = [(where, value) for where, value in self.revisions if not _is_pinned(value)] + self.assertEqual(unpinned, []) + + def test_known_pins_are_found(self): + found = dict(self.revisions) + self.assertEqual(found["foldmatch/utils/esm/loaders.py:ESM_REVISION"], loaders.ESM_REVISION) + self.assertEqual(found["foldmatch/utils/model.py:REVISION"], model.REVISION) + calls = [where.split()[-1] for where, _ in self.revisions if where.endswith("()")] + for fetch in ("snapshot_download()", "hf_hub_download()", "try_to_load_from_cache()"): + self.assertIn(fetch, calls) + bundle_calls = [where for where, value in self.revisions if value == _REGISTRY_PIN] + self.assertEqual(len(bundle_calls), 1, bundle_calls) + self.assertTrue(bundle_calls[0].startswith("foldmatch/utils/model.py:"), bundle_calls) + self.assertTrue(bundle_calls[0].endswith("hf_hub_download()"), bundle_calls) + + def test_runtime_pins(self): + for revision in (loaders.ESM_REVISION, model.REVISION): + self.assertTrue(_is_pinned(revision), revision) + + def test_unpinned_revisions_are_refused(self): + sha = "636acb420daf674df37ab0220f56061a5de2f169" + cases = { + f'REVISION = "{sha}"': sha, + "REVISION = None": None, + 'REVISION = "main"': "main", + f'REVISION = "{sha[:12]}"': sha[:12], + f'REVISION = "{sha.upper()}"': sha.upper(), + 'snapshot_download(repo_id="r")': None, + 'hf_hub_download(repo_id="r", filename="f", revision=None)': None, + 'hf_hub_download(repo_id="r", filename="f", revision=REV)': _UNRESOLVED, + f'R = "{sha}"\nsnapshot_download(repo_id="r", revision=R)': sha, + 'def get(revision=None):\n return hf_hub_download(repo_id="r", filename="f", revision=revision)': None, + f'def get(revision="{sha}"):\n return hf_hub_download(repo_id="r", filename="f", revision=revision)': sha, + 'def get(revision):\n return hf_hub_download(repo_id="r", filename="f", revision=revision)': _UNRESOLVED, + 'ESM3.from_pretrained("esm3_sm_open_v1")': None, + 'if __name__ == "__main__":\n ESM3.from_pretrained("esm3_sm_open_v1")': "", + } + for source, expected in cases.items(): + values = [value for _where, value in _revisions_in(ast.parse(source), "m.py")] + if expected == "": + self.assertEqual(values, [], source) + continue + self.assertEqual(values[-1], expected, source) + self.assertEqual(_is_pinned(values[-1]), expected == sha, source) + + def test_registry_revisions(self): + """revision=.bundle_revision resolves to the registry, pinned exactly when every entry is a commit; + any other attribute is not a pin.""" + source = 'def get(system):\n return hf_hub_download(repo_id=system.bundle_repo, filename="f", ' \ + 'revision=system.bundle_revision)' + self.assertEqual([v for _w, v in _revisions_in(ast.parse(source), "m.py")], [_REGISTRY_PIN]) + source = 'def get(system):\n return hf_hub_download(repo_id="r", filename="f", revision=system.revision)' + self.assertEqual([v for _w, v in _revisions_in(ast.parse(source), "m.py")], [_UNRESOLVED]) + self.assertTrue(_is_pinned(_REGISTRY_PIN)) + from foldmatch.search import scoring_system + loose = mock.Mock(bundle_revision="main") + with mock.patch.object(scoring_system, "REGISTRY", {"x": loose}): + self.assertFalse(_is_pinned(_REGISTRY_PIN)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_residue_assembly_embedding_from_parquet.py b/tests/test_residue_assembly_embedding_from_parquet.py index b89015e..3cddd40 100644 --- a/tests/test_residue_assembly_embedding_from_parquet.py +++ b/tests/test_residue_assembly_embedding_from_parquet.py @@ -25,11 +25,8 @@ def setUp(self): os.unlink(fp) def _build_parquet_from_tensor_files(self): - """Write the test .pt embeddings into a single Parquet file.""" - schema = pa.schema([ - ('id', pa.string()), - ('embedding', pa.list_(pa.float32())) - ]) + """Write the test .pt embeddings into a single Parquet file: one row per chain, one list of D values per + residue.""" ids = [] embeddings = [] for pt_file in sorted(os.listdir(self.__embeddings_path)): @@ -40,8 +37,12 @@ def _build_parquet_from_tensor_files(self): ) item_id = pt_file[:-3] # strip .pt ids.append(item_id) - embeddings.append(tensor.numpy().flatten().tolist()) + embeddings.append(tensor.numpy().tolist()) + schema = pa.schema([ + ('id', pa.string()), + ('embedding', pa.list_(pa.list_(pa.float32(), len(embeddings[0][0])))) + ]) table = pa.table({'id': ids, 'embedding': embeddings}, schema=schema) parquet_path = os.path.join(self.__tmp_path, "residue-embeddings.parquet") pq.write_table(table, parquet_path) @@ -62,11 +63,9 @@ def test_assembly_from_parquet_matches_tensor_files(self): structure_format=StructureFormat.mmcif ) - embedding_dim = 1536 parquet_dataset = ResidueAssemblyEmbeddingFromParquet( src_stream=src_stream, parquet_path=parquet_path, - embedding_dim=embedding_dim, src_location=SrcLocation.stream, structure_format=StructureFormat.mmcif ) diff --git a/tests/test_residue_code_store.py b/tests/test_residue_code_store.py new file mode 100644 index 0000000..6e15027 --- /dev/null +++ b/tests/test_residue_code_store.py @@ -0,0 +1,1095 @@ +"""P5.5: the residue-code store ``P.rescodes.{bin,idx,json}`` (plan section 8 P5.5, Appendix B). + +No ESM3: the codes are the F2c reference codes of ``test_sequences.fasta`` or random bytes. Most tests pass the ESM3 +revision and weights sha256 explicitly, so only the tests of the ``_INSTALLED`` defaults import +``foldmatch.utils.esm.loaders`` (torch, never the weights). +""" +import dataclasses +import hashlib +import json +import os +import random +import sqlite3 +import struct +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import numpy as np +import pyarrow as pa + +from foldmatch.search import residue_code_store as rcs +from foldmatch.search import scoring_system as ss +from foldmatch.search.residue_code_store import ( + MANIFEST_KEYS, CodeStoreValidationError, IncompatibleCodeStoreError, ResidueCodeStore, ResidueCodeStoreError, + entries_digest, +) +from foldmatch.search.sequence_store import SequenceStore +from foldmatch.utils.sequence import STANDARD_AMINO_ACIDS, preflight_fasta, seq_sha1 + +_TEST_PATH = Path(__file__).parent +_SRC = str(_TEST_PATH.parent / "src") +_F2C = _TEST_PATH / "resources" / "esm_codes" / "fasta_rows_codes_v1.npz" +_FIXTURE_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" + +SYSTEM = ss.get(ss.DEFAULT_ESM_CODES) +ESM_SHA = SYSTEM.esm_weights_sha256 +ESM_REV = "636acb420daf674df37ab0220f56061a5de2f169" # passed explicitly: these tests never need torch +CODE_BYTES = SYSTEM.code_bytes + + +def _write_fasta(path: Path, records): + with open(path, "w") as f: + for name, sequence in records: + f.write(f">{name}\n{sequence}\n") + return path + + +def _records(n, rng, min_len=3, max_len=40, prefix="P"): + return [(f"{prefix}{i:05d}", + "".join(rng.choice(STANDARD_AMINO_ACIDS) for _ in range(rng.randint(min_len, max_len)))) + for i in range(n)] + + +def _entries(records, seed=0): + """``[(id, codes uint8 [L, 8], seq_sha1)]`` with random codes.""" + gen = np.random.default_rng(seed) + return [(name, gen.integers(0, 256, size=(len(seq), CODE_BYTES), dtype=np.uint8), seq_sha1(seq)) + for name, seq in records] + + +def _batches(entries, size=None): + size = size or max(len(entries), 1) + out = [] + for start in range(0, len(entries), size): + chunk = entries[start:start + size] + out.append(([e[0] for e in chunk], [e[1] for e in chunk], [e[2] for e in chunk])) + return out + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _store_files(store): + return {p.name: _sha256(p) for p in (store.bin_path, store.idx_path, store.manifest_path) if p.exists()} + + +class _StoreCase(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def _database(self, records, name="db"): + """A FASTA, its sequence store and its pre-flight (what a build has before the code store).""" + fasta = _write_fasta(self.tmp / f"{name}.fasta", records) + seq = SequenceStore(self.tmp, name) + seq.create(fasta) + return seq, preflight_fasta(fasta) + + def _create(self, entries, name="db", size=None, **kwargs): + store = ResidueCodeStore(self.tmp, name) + kwargs.setdefault("esm_revision", ESM_REV) + summary = store.create(_batches(entries, size), kwargs.pop("system", SYSTEM), **kwargs) + return store, summary + + def _build(self, records, entries=None, name="db", size=None, faiss=None, **kwargs): + seq, preflight = self._database(records, name) + entries = _entries(records) if entries is None else entries + store, _ = self._create(entries, name, size, **kwargs) + manifest = store.validate(seq, len(records) if faiss is None else faiss, preflight=preflight) + return store, seq, manifest + + def _check(self, store, **kwargs): + kwargs.setdefault("sequence_store", store.sequence_store()) + kwargs.setdefault("esm_weights_sha256", ESM_SHA) + return store.check_compatible(kwargs.pop("system", SYSTEM), **kwargs) + + def _edit_manifest(self, store, edit): + manifest = json.loads(store.manifest_path.read_text()) + edit(manifest) + store.manifest_path.write_text(json.dumps(manifest)) + + def assertNoStoreFiles(self, store): + left = sorted(p.name for p in self.tmp.glob(f"{store.index_name}.rescodes*")) + self.assertEqual(left, []) + self.assertFalse(store.exists()) + self.assertFalse(store.is_partial()) + self.assertFalse(store.has_pending_build()) + + def assertStoreUnchanged(self, store, before): + """The complete store is exactly as it was (``before`` = :func:`_store_files`), usable, with no pending + files next to it.""" + self.assertEqual(_store_files(store), before) + self.assertEqual(sorted(p.name for p in self.tmp.glob(f"{store.index_name}.rescodes*")), + [f"{store.index_name}.rescodes.{x}" for x in ("bin", "idx", "json")]) + self.assertFalse(store.has_pending_build()) + self._check(store) + + +# ===================================================================================================================== +class RoundTripTest(_StoreCase): + def test_f2c_codes_round_trip(self): + """The F2c reference codes of test_sequences.fasta: manifest entries 2 / total_residues 303 (Appendix B), + the memmap equals the codes, lookup gives each entry's slice.""" + f2c = np.load(_F2C, allow_pickle=False) + ids, offsets, codes = [str(i) for i in f2c["ids"]], f2c["offsets"], f2c["codes"] + seq, preflight = SequenceStore(self.tmp, "db"), preflight_fasta(_FIXTURE_FASTA) + seq.create(_FIXTURE_FASTA) + sequences = seq.fetch(ids) + self.assertEqual([sequences[i] for i in ids], [str(s) for s in f2c["sequences"]]) + store = ResidueCodeStore(self.tmp, "db") + batches = [([i], [codes[offsets[k]:offsets[k + 1]]], [seq_sha1(sequences[i])]) for k, i in enumerate(ids)] + build = {"run_id": "run-1", "accelerator": "cpu", "devices": 1, "esm_precision": "fp32", + "encoder_precision": "fp32-no-tf32"} + summary = store.create(iter(batches), SYSTEM, build=build, esm_revision=ESM_REV) + self.assertEqual((summary["entries"], summary["total_residues"]), (2, 303)) + manifest = store.validate(seq, 2, preflight=preflight) + + self.assertEqual(tuple(sorted(manifest)), tuple(sorted(MANIFEST_KEYS))) + self.assertEqual(json.loads(store.manifest_path.read_text()), manifest) + self.assertEqual((manifest["manifest_version"], manifest["kind"]), (1, "foldmatch-residue-codes")) + self.assertEqual((manifest["entries"], manifest["total_residues"], manifest["min_res_n"]), (2, 303, 0)) + self.assertEqual(manifest["system_name"], "esm-codes-v1") + self.assertEqual(manifest["system"], SYSTEM.to_dict()) + self.assertEqual(ss.EsmCodesSystem.from_dict(manifest["system"]), SYSTEM) + self.assertEqual((manifest["system_digest"], manifest["encoding_digest"]), + (SYSTEM.digest(), SYSTEM.encoding_digest())) + self.assertEqual((manifest["code_bytes"], manifest["bitorder"]), (8, "big")) + self.assertEqual(manifest["esm"], {"repo": "rcsb/rcsb-esm", "revision": ESM_REV, "weights_sha256": ESM_SHA}) + self.assertEqual(manifest["bundle"], {"repo": SYSTEM.bundle_repo, "revision": SYSTEM.bundle_revision, + "files": dict(SYSTEM.bundle_files)}) + self.assertEqual(manifest["bin_sha256"], hashlib.sha256(codes.tobytes()).hexdigest()) + self.assertEqual(manifest["entries_digest"], entries_digest([(i, seq_sha1(sequences[i])) for i in ids])) + self.assertEqual((manifest["nonstandard_entries"], manifest["input_normalisation"]), + (0, "ascii_upper_nows_v1")) + self.assertEqual(manifest["cross_checks"], { + "preflight_entries": 2, "preflight_residues": 303, "sequence_store_entries": 2, + "sequence_store_residues": 303, "faiss_entries": 2, "seq_sha1_mismatches": 0, "length_mismatches": 0}) + for key, value in build.items(): + self.assertEqual(manifest["build"][key], value) + self.assertEqual(manifest["build"]["platform"].count("-"), 1) + + self.assertTrue(store.exists()) + self.assertFalse(store.is_partial()) + self.assertEqual(store.bin_path.stat().st_size, 8 * 303) + memmap = store.open_memmap() + self.assertIsInstance(memmap, np.memmap) + self.assertEqual((memmap.shape, memmap.dtype), ((303, 8), np.uint8)) + np.testing.assert_array_equal(memmap, codes) + self.assertFalse(memmap.flags.writeable) + found = store.lookup(ids) + for k, i in enumerate(ids): + entry = found[i] + self.assertEqual((entry.offset, entry.length), (offsets[k], offsets[k + 1] - offsets[k])) + self.assertEqual(entry.seq_sha1, seq_sha1(sequences[i])) + np.testing.assert_array_equal(memmap[entry.offset:entry.offset + entry.length], + codes[offsets[k]:offsets[k + 1]]) + self.assertEqual(self._check(store), manifest) + store.check_faiss_count(2) + with self.assertRaisesRegex(IncompatibleCodeStoreError, "2 entries, the FAISS index 3"): + store.check_faiss_count(3) + + def test_batch_forms_and_sizes_give_the_same_store(self): + """Tuples of arrays or bytes, pyarrow batches and tables, any batch size and transactions of any size (here + 5 rows) write the same files.""" + records = _records(23, random.Random(1)) + entries = _entries(records) + reference = None + + def arrow(chunk, table=False): + data = {"id": pa.array([e[0] for e in chunk], pa.string()), + "residue_codes": pa.array([e[1].tobytes() for e in chunk], pa.large_binary()), + "seq_sha1": pa.array([e[2] for e in chunk], pa.binary(20))} + return pa.table(data) if table else pa.record_batch(data) + + forms = { + "arrays": lambda chunk: ([e[0] for e in chunk], [e[1] for e in chunk], [e[2] for e in chunk]), + "bytes": lambda chunk: ([e[0] for e in chunk], [e[1].tobytes() for e in chunk], + [bytearray(e[2]) for e in chunk]), + "flat-arrays": lambda chunk: ([e[0] for e in chunk], [e[1].reshape(-1) for e in chunk], + [memoryview(e[2]) for e in chunk]), + "record-batch": arrow, + "table": lambda chunk: arrow(chunk, table=True), + } + for (form, make), size in [(f, s) for f in forms.items() for s in (1, 7, 23)]: + with self.subTest(form=form, size=size), mock.patch.object(rcs, "_INSERT_BATCH", 5): + name = f"db_{form}_{size}" + seq, preflight = self._database(records, name) + store = ResidueCodeStore(self.tmp, name) + batches = [make(entries[s:s + size]) for s in range(0, len(entries), size)] + store.create(batches, SYSTEM, esm_revision=ESM_REV) + manifest = store.validate(seq, len(records), preflight=preflight) + conn = sqlite3.connect(store.idx_path) + rows = conn.execute("SELECT id, offset, length, seq_sha1 FROM entries ORDER BY id").fetchall() + conn.close() + got = (manifest["bin_sha256"], manifest["entries_digest"], manifest["total_residues"], rows) + if reference is None: + reference = got + self.assertEqual(got, reference) + + def test_entry_order_changes_bin_sha256_not_entries_digest(self): + records = _records(30, random.Random(2)) + entries = _entries(records) + store, _, manifest = self._build(records, entries, name="a") + shuffled = list(entries) + random.Random(3).shuffle(shuffled) + other, _, manifest2 = self._build(records, shuffled, name="b") + self.assertEqual(manifest2["entries_digest"], manifest["entries_digest"]) + self.assertNotEqual(manifest2["bin_sha256"], manifest["bin_sha256"]) + memmap, memmap2 = store.open_memmap(), other.open_memmap() + found, found2 = store.lookup([e[0] for e in entries]), other.lookup([e[0] for e in entries]) + for name, codes, _ in entries: + a, b = found[name], found2[name] + np.testing.assert_array_equal(memmap[a.offset:a.offset + a.length], codes) + np.testing.assert_array_equal(memmap2[b.offset:b.offset + b.length], codes) + + +# ===================================================================================================================== +class DigestTest(_StoreCase): + def test_bin_sha256_is_the_file_hash_computed_while_writing(self): + records = _records(40, random.Random(4)) + entries = _entries(records) + with mock.patch.object(rcs, "_INSERT_BATCH", 7): + store, summary = self._create(entries, size=6) + self.assertEqual(summary["bin_sha256"], _sha256(store.tmp_bin_path)) + self.assertEqual(summary["bin_sha256"], + hashlib.sha256(b"".join(e[1].tobytes() for e in entries)).hexdigest()) + + def test_entries_digest_definition(self): + """sha256(tag, then per entry in UTF-8 byte order of the ids: uint32 LE id length, id, 20-byte seq_sha1). + SQLite's ORDER BY id (BINARY) is that order, ASCII case and non-ASCII ids included.""" + names = ["b", "a", "Z", "a_1", "été", "A", "a1", "中"] + records = [(n, "ACDE"[: 1 + k % 4]) for k, n in enumerate(names)] + entries = _entries(records) + store, summary = self._create(entries) + h = hashlib.sha256(b"foldmatch-residue-codes-entries/1\n") + for name, _, sha1 in sorted(entries, key=lambda e: e[0].encode("utf-8")): + h.update(struct.pack("= 0)", + "query_encoder_sha256": "4" * 64, + "query_dtype": "float16", + "query_prep": "fp64_seq_k_asc", + "query_prep_eps": 1e-12, + "similarity": "pm1_fp32_blas_gemm_per_pair", + "determinism": "tolerance_s16_v1", + "normalisation": {"name": "rowcol_z", "std": "population", "eps": 1e-08, "degenerate_std_lt": 1e-06}, + "offset": 1.0, + "gap_open": 5.0, + "gap_extend": 0.5, + "dp": "fmsw/1", + "score_form": "fwd_minus_rev_view", + "score_diff": "float64", + "panel_sha256": "5" * 64, + "panel_size": 3, + "panel_residues": 40, + "panel_quantile": 0.9, + "quantile_method": "linear", + "bin_centres": [30.0, 50.0, 80.0, 150.0, 280.0, 600.0], + "lam": [0.19, 0.12, 0.1, 0.08, 0.05, 0.04], + "lnK": [-13.4, -14.0, -13.7, -13.5, -14.5, -15.9], + "low_m_rule": "clamp", + "high_m_rule": "clamp", + "calibrated_domain": {"query_min_length": 21, "query_max_length": 966, "target_max_length": 3000, + "max_nonstandard_fraction": 0.01, "target_min_distinct_codes": 2}, + "degenerate_query_rule": "no_hits", + "bundle_repo": "owner/bundle", + "bundle_revision": "a" * 40, + "bundle_files": [["bundle.json", "6" * 64], ["encoders.pt", "7" * 64], ["scoring.npz", "8" * 64]], + } + d.update(overrides) + return d + + +def _synthetic(**overrides): + return EsmCodesSystem.from_dict(_synthetic_dict(**overrides)) + + +def _flip_hex(h): + return h[:-1] + ("0" if h[-1] != "0" else "1") + + +def _up(x): + return math.nextafter(x, math.inf) + + +def _mutations(d): + """(label, mutated dict) for every leaf of the system's JSON form; each mutation keeps the system valid.""" + out = [] + for key, value in d.items(): + if key in ("normalisation", "calibrated_domain"): + for sub, v in value.items(): + m = copy.deepcopy(d) + m[key][sub] = (v + "x") if isinstance(v, str) else (v + 1 if isinstance(v, int) else _up(v)) + out.append((f"{key}.{sub}", m)) + continue + m = copy.deepcopy(d) + if key in ("code_bits", "code_bytes"): + m["code_bits"], m["code_bytes"] = 128, 16 + elif key == "bundle_files": + m[key][1][1] = _flip_hex(m[key][1][1]) + elif key in ("bin_centres", "lam", "lnK"): + m[key][-1] = _up(m[key][-1]) + elif key.endswith("sha256") or key == "bundle_revision": + m[key] = _flip_hex(value) + elif isinstance(value, str): + m[key] = value + "x" + elif isinstance(value, int): + m[key] = value + 1 + else: + m[key] = _up(value) + out.append((key, m)) + return out + + +class RoundTripTest(unittest.TestCase): + + def test_json_round_trip(self): + s = _synthetic() + text = s.to_json() + self.assertEqual(EsmCodesSystem.from_json(text), s) + self.assertEqual(EsmCodesSystem.from_json(json.loads(text)), s) + self.assertEqual(EsmCodesSystem.from_json(text).digest(), s.digest()) + self.assertEqual(EsmCodesSystem.from_json(text).encoding_digest(), s.encoding_digest()) + self.assertEqual(text, json.dumps(json.loads(text), sort_keys=True, separators=(",", ":"))) + self.assertNotIn(" ", text.replace("bit_k = (h_k >= 0)", "")) + + def test_floats_survive_the_text_exactly(self): + lam = [0.18775162858061797, 0.1227859873586807, 0.10488523441851651, 0.08222582327223689, + 0.04784353519669357, 0.041619283794126286] + s = _synthetic(lam=lam) + self.assertEqual(EsmCodesSystem.from_json(s.to_json()).lam, tuple(lam)) + + def test_digests_are_16_hex(self): + s = _synthetic() + for d in (s.digest(), s.encoding_digest()): + self.assertRegex(d, r"^[0-9a-f]{16}$") + self.assertNotEqual(s.digest(), s.encoding_digest()) + + def test_digest_is_hashable_and_frozen(self): + s = _synthetic() + self.assertEqual(hash(s), hash(_synthetic())) + with self.assertRaises(dataclasses.FrozenInstanceError): + s.lam = (1.0,) * 6 + with self.assertRaises(dataclasses.FrozenInstanceError): + s.normalisation.eps = 1.0 + + +class DigestSensitivityTest(unittest.TestCase): + """Every field changes digest(); exactly the ENCODING_FIELDS change encoding_digest() (so statistics, panel, + scoring and bundle changes never flip it).""" + + def test_every_leaf(self): + base = _synthetic() + d = base.to_dict() + seen = set() + for label, m in _mutations(d): + with self.subTest(field=label): + s = EsmCodesSystem.from_dict(m) + self.assertNotEqual(s, base) + self.assertNotEqual(s.digest(), base.digest()) + top = label.split(".")[0] + seen.add(top) + if top in ENCODING_FIELDS: + self.assertNotEqual(s.encoding_digest(), base.encoding_digest()) + else: + self.assertEqual(s.encoding_digest(), base.encoding_digest()) + self.assertEqual(seen, {f.name for f in dataclasses.fields(EsmCodesSystem)}) + + def test_architecture_and_precision_flip_the_encoding_digest(self): + base = _synthetic() + for change in (dict(encoder_architecture="esm-codes-mlp-v2"), dict(encoder_dtype="bfloat16")): + with self.subTest(**change): + self.assertNotEqual(dataclasses.replace(base, **change).encoding_digest(), base.encoding_digest()) + + def test_statistics_fields_do_not_flip_the_encoding_digest(self): + base = _synthetic() + s = dataclasses.replace(base, lam=tuple(v * 1.01 for v in base.lam), lnK=tuple(v - 1 for v in base.lnK), + panel_quantile=0.95, high_m_rule="extrapolate", calibration_version="other", + bundle_revision="b" * 40, name="esm-codes-test2", offset=0.5) + self.assertEqual(s.encoding_digest(), base.encoding_digest()) + self.assertNotEqual(s.digest(), base.digest()) + + +class NormalisationTest(unittest.TestCase): + + def test_lists_become_tuples(self): + s = _synthetic() + for name in ("bin_centres", "lam", "lnK", "bundle_files"): + self.assertIsInstance(getattr(s, name), tuple, name) + self.assertTrue(all(isinstance(p, tuple) for p in s.bundle_files)) + self.assertIsInstance(s.normalisation, Normalisation) + self.assertIsInstance(s.calibrated_domain, CalibratedDomain) + r = dataclasses.replace(s, lam=list(s.lam)) + self.assertIsInstance(r.lam, tuple) + self.assertEqual(r, s) + + def test_integral_floats_and_file_order_do_not_change_the_digest(self): + s = _synthetic() + r = _synthetic(offset=1, gap_open=5, bin_centres=[30, 50, 80, 150, 280, 600], + bundle_files={"scoring.npz": "8" * 64, "bundle.json": "6" * 64, "encoders.pt": "7" * 64}) + self.assertEqual(r, s) + self.assertEqual(r.digest(), s.digest()) + self.assertIsInstance(r.offset, float) + self.assertEqual([n for n, _ in r.bundle_files], ["bundle.json", "encoders.pt", "scoring.npz"]) + + def test_bundle_sha256(self): + s = _synthetic() + self.assertEqual(s.bundle_sha256("encoders.pt"), "7" * 64) + with self.assertRaises(KeyError): + s.bundle_sha256("README.md") + + +class UnknownFieldsTest(unittest.TestCase): + + def test_unknown_top_level_field(self): + with self.assertRaisesRegex(ValueError, r"unknown field\(s\) \['colour'\]"): + EsmCodesSystem.from_json(json.dumps(_synthetic_dict(colour="blue"))) + + def test_unknown_nested_fields(self): + for key in ("normalisation", "calibrated_domain"): + d = _synthetic_dict() + d[key] = dict(d[key], extra=1) + with self.subTest(key=key), self.assertRaisesRegex(ValueError, rf"{key}: unknown field\(s\) \['extra'\]"): + EsmCodesSystem.from_dict(d) + + def test_missing_fields(self): + d = _synthetic_dict() + del d["high_m_rule"] + with self.assertRaisesRegex(ValueError, r"missing field\(s\) \['high_m_rule'\]"): + EsmCodesSystem.from_dict(d) + d = _synthetic_dict() + del d["calibrated_domain"]["target_min_distinct_codes"] + with self.assertRaisesRegex(ValueError, "missing field"): + EsmCodesSystem.from_dict(d) + + def test_not_an_object(self): + with self.assertRaises(ValueError): + EsmCodesSystem.from_json("[1, 2]") + + +class InconsistentValuesTest(unittest.TestCase): + + CASES = { + "five centres": dict(bin_centres=[30, 50, 80, 150, 280]), + "seven lambdas": dict(lam=[0.1] * 7), + "centres not increasing": dict(bin_centres=[30, 50, 50, 150, 280, 600]), + "centres decreasing": dict(bin_centres=[600, 280, 150, 80, 50, 30]), + "centre not positive": dict(bin_centres=[0, 50, 80, 150, 280, 600]), + "lambda zero": dict(lam=[0.19, 0.12, 0.0, 0.08, 0.05, 0.04]), + "lambda negative": dict(lam=[0.19, 0.12, -0.1, 0.08, 0.05, 0.04]), + "lnK NaN": dict(lnK=[-13.4, float("nan"), -13.7, -13.5, -14.5, -15.9]), + "lambda infinite": dict(lam=[0.19, float("inf"), 0.1, 0.08, 0.05, 0.04]), + "gap_extend above gap_open": dict(gap_open=0.5, gap_extend=5.0), + "negative gap_extend": dict(gap_extend=-0.5), + "quantile 0": dict(panel_quantile=0.0), + "quantile 1": dict(panel_quantile=1.0), + "quantile 1.2": dict(panel_quantile=1.2), + "revision None": dict(bundle_revision=None), + "revision main": dict(bundle_revision="main"), + "revision short": dict(bundle_revision="a" * 12), + "revision upper": dict(bundle_revision="A" * 40), + "sha short": dict(transform_sha256="2" * 63), + "sha upper": dict(panel_sha256="F" * 64), + "sha None": dict(esm_weights_sha256=None), + "code bits vs bytes": dict(code_bits=63), + "code bytes bool": dict(code_bytes=True), + "panel size float": dict(panel_size=3.0), + "panel residues below size": dict(panel_residues=2), + "offset string": dict(offset="1.0"), + "offset bool": dict(offset=True), + "empty dp": dict(dp=""), + "bad name": dict(name="ESM codes"), + "bad repo": dict(bundle_repo="no-slash"), + "no bundle.json": dict(bundle_files=[["encoders.pt", "7" * 64]]), + "file twice": dict(bundle_files=[["bundle.json", "6" * 64], ["bundle.json", "7" * 64]]), + "file outside the repo": dict(bundle_files=[["bundle.json", "6" * 64], ["../x", "7" * 64]]), + "absolute file": dict(bundle_files=[["bundle.json", "6" * 64], ["/x", "7" * 64]]), + "bad file sha": dict(bundle_files=[["bundle.json", "6" * 60]]), + "query lengths reversed": dict(calibrated_domain={"query_min_length": 967, "query_max_length": 966, + "target_max_length": 3000, + "max_nonstandard_fraction": 0.01, + "target_min_distinct_codes": 2}), + "nonstandard fraction above 1": dict(calibrated_domain={"query_min_length": 21, "query_max_length": 966, + "target_max_length": 3000, + "max_nonstandard_fraction": 1.5, + "target_min_distinct_codes": 2}), + "eps zero": dict(normalisation={"name": "rowcol_z", "std": "population", "eps": 0.0, + "degenerate_std_lt": 1e-06}), + "query_prep_eps negative": dict(query_prep_eps=-1e-12), + } + + def test_each_case_raises(self): + for label, overrides in self.CASES.items(): + with self.subTest(case=label), self.assertRaises(ValueError): + _synthetic(**overrides) + + def test_replace_validates_too(self): + with self.assertRaises(ValueError): + dataclasses.replace(_synthetic(), gap_extend=6.0) + + +class RegistryTest(unittest.TestCase): + + def test_get_unknown_lists_the_known_names(self): + a, b = _synthetic(name="esm-codes-a"), _synthetic(name="esm-codes-b") + with mock.patch.object(ss, "REGISTRY", ss._registry(b, a)): + self.assertIs(ss.get("esm-codes-a"), a) + self.assertEqual(ss.known_names(), ("esm-codes-a", "esm-codes-b")) + with self.assertRaisesRegex(UnknownScoringSystemError, "esm-codes-a, esm-codes-b") as ctx: + ss.get("esm-codes-v9") + self.assertIn("'esm-codes-v9'", str(ctx.exception)) + self.assertIn("newer FoldMatch", str(ctx.exception)) + with self.assertRaises(UnknownScoringSystemError): + ss.get(None) + self.assertTrue(issubclass(UnknownScoringSystemError, ValueError)) + + def test_registry_is_read_only(self): + self.assertIsInstance(ss.REGISTRY, MappingProxyType) + with self.assertRaises(TypeError): + ss.REGISTRY["x"] = _synthetic() + with self.assertRaises(AssertionError): + ss._registry(_synthetic(), _synthetic()) + + def test_default_name(self): + self.assertEqual(ss.DEFAULT_ESM_CODES, "esm-codes-v1") + + +class FieldGroupsTest(unittest.TestCase): + + def test_groups_partition_the_fields(self): + groups = IDENTITY_FIELDS + ENCODING_FIELDS + SCORING_FIELDS + STATISTICS_FIELDS + BUNDLE_FIELDS + self.assertEqual(sorted(groups), sorted(f.name for f in dataclasses.fields(EsmCodesSystem))) + self.assertEqual(len(set(groups)), len(groups)) + + def test_encoding_fields_are_the_plans(self): + """The plan's encoding fields, plus the encoder architecture and precision (P4 review): the content digests + cover only the weight tensors, and the same weights give other codes with GELU-tanh or another precision.""" + self.assertEqual(set(ENCODING_FIELDS), { + "esm_repo", "esm_weights_sha256", "embedding_tap", "input_normalisation", "input_cast", + "transform_sha256", "transform_output_cast", "target_encoder_sha256", "code_bits", "code_bytes", + "bitorder", "sign_rule"} | {"encoder_architecture", "encoder_dtype"}) + self.assertEqual(BUNDLE_FIELDS, ("bundle_repo", "bundle_revision", "bundle_files")) + + def test_every_field_is_required(self): + for f in dataclasses.fields(EsmCodesSystem): + self.assertIs(f.default, dataclasses.MISSING, f.name) + self.assertIs(f.default_factory, dataclasses.MISSING, f.name) + + +class ScoringChoiceTest(unittest.TestCase): + + def test_values_and_default(self): + self.assertEqual([c.value for c in ScoringChoice], ["blosum62", "esm-codes"]) + self.assertIs(DEFAULT_SCORING, ScoringChoice.blosum62) + self.assertIs(ScoringChoice("esm-codes"), ScoringChoice.esm_codes) + + + +# ===================================================================================================================== +# a synthetic bundle in a Hugging Face cache layout +# ===================================================================================================================== +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _npz_bytes(arrays) -> bytes: + buf = io.BytesIO() + np.savez(buf, **arrays) + return buf.getvalue() + + +class _SyntheticBundle: + """A bundle laid out like rcsb/foldmatch-esm-codes (bundle.json, encoders, scoring npz) for ``self.system``, + written as ``/models--owner--bundle/{blobs/, snapshots// -> blob}``. + + ``bundle_json``, ``meta`` and ``arrays`` may be edited by the caller before :meth:`write`, which pins the + resulting files' sha256s in the system (so a content check, not the sha256 check, must catch an edit).""" + + ENCODERS = "esm_codes_test_encoders.pt" + SCORING = "esm_codes_test_scoring.npz" + + def __init__(self, **overrides): + d = _synthetic_dict(**overrides) + rng = np.random.default_rng(3) + lengths = np.array([10, 13, 17], np.int64) + self.codes = rng.integers(0, 256, (int(lengths.sum()), 8), dtype=np.uint8) + self.offsets = np.concatenate([[0], np.cumsum(lengths)]).astype(np.int64) + d["panel_size"], d["panel_residues"] = len(lengths), int(lengths.sum()) + d["panel_sha256"] = tensor_digest({"codes": self.codes, "offsets": self.offsets}) + self.d = d + table = {k: np.array(d[k], np.float64) for k in ("bin_centres", "lam", "lnK")} + scoring = {k: d[k] for k in ("dp", "offset", "gap_open", "gap_extend", "score_form", "score_diff", + "query_prep", "query_prep_eps", "similarity", "determinism")} + scoring["normalisation"] = dict(d["normalisation"], formula="z = ...") + esm3 = {"model": "esm3_sm_open_v1", "repo": d["esm_repo"], "weights_sha256": d["esm_weights_sha256"], + "embedding_tap": d["embedding_tap"], "input_normalisation": d["input_normalisation"]} + statistics = {k: d[k] for k in ("bin_centres", "lam", "lnK", "low_m_rule", "high_m_rule")} + statistics["fit_db_residues"] = 8557236.0 + panel = {"size": d["panel_size"], "residues": d["panel_residues"], "quantile": d["panel_quantile"], + "quantile_method": d["quantile_method"], "panel_sha256": d["panel_sha256"]} + self.meta = { + "format": ss.SCORING_FORMAT, "system": d["name"], "calibration_version": d["calibration_version"], + "code": {"bits": d["code_bits"], "bytes": d["code_bytes"], "bitorder": d["bitorder"], + "sign_rule": d["sign_rule"]}, + "scoring": copy.deepcopy(scoring), "statistics": copy.deepcopy(statistics), "panel": dict(panel), + "calibrated_domain": dict(d["calibrated_domain"]), "degenerate_query_rule": d["degenerate_query_rule"], + "esm3": dict(esm3), + } + self.arrays = {"panel_codes": self.codes, "panel_offsets": self.offsets, "panel_lengths": lengths, + "panel_ids": np.array(["UniRef50_A", "UniRef50_B", "UniRef50_C"]), **table} + self.encoders = b"synthetic encoders (prepare() does not read them)" + self.bundle_json = { + "bundle_format": ss.BUNDLE_FORMAT, "system": d["name"], "calibration_version": d["calibration_version"], + "hf_repo": d["bundle_repo"], + "files": {self.ENCODERS: {"sha256": None, "format": ss.ENCODERS_FORMAT}, + self.SCORING: {"sha256": None, "format": ss.SCORING_FORMAT}}, + "content_digests": {"method": "foldmatch-tensor-digest/1", "transform_sha256": d["transform_sha256"], + "query_encoder_sha256": d["query_encoder_sha256"], + "target_encoder_sha256": d["target_encoder_sha256"], + "panel_sha256": d["panel_sha256"], "table_sha256": tensor_digest(table)}, + "esm3": esm3, + "encoding": {**{k: d[k] for k in ("input_cast", "transform_output_cast", "code_bits", "code_bytes", + "bitorder", "sign_rule", "query_dtype")}, + "architecture": {"name": d["encoder_architecture"], "activation": "gelu_exact"}, + "encoder": {"dtype": d["encoder_dtype"], "tf32": False, "autocast": False}}, + "scoring": scoring, "statistics": statistics, "panel": panel, + "calibrated_domain": dict(d["calibrated_domain"]), "degenerate_query_rule": d["degenerate_query_rule"], + "licence": {"spdx": "MIT"}, + } + + def write(self, cache: Path) -> EsmCodesSystem: + npz = _npz_bytes(dict(self.arrays, meta_json=np.array(json.dumps(self.meta)))) + files = {self.ENCODERS: self.encoders, self.SCORING: npz} + bj = copy.deepcopy(self.bundle_json) + for name in (self.ENCODERS, self.SCORING): + if name in bj["files"] and bj["files"][name]["sha256"] is None: + bj["files"][name]["sha256"] = _sha(files[name]) + files[ss.BUNDLE_JSON] = json.dumps(bj, indent=1).encode() + d = dict(self.d, bundle_files=[[n, _sha(b)] for n, b in files.items()]) + system = EsmCodesSystem.from_dict(d) + repo = cache / ("models--" + system.bundle_repo.replace("/", "--")) + for name, data in files.items(): + blob = repo / "blobs" / _sha(data) + blob.parent.mkdir(parents=True, exist_ok=True) + blob.write_bytes(data) + link = repo / "snapshots" / system.bundle_revision / name + link.parent.mkdir(parents=True, exist_ok=True) + if link.is_symlink() or link.exists(): + link.unlink() + link.symlink_to(os.path.relpath(blob, link.parent)) + self.files = files + return system + + +def _listing(root: Path): + return sorted((str(p.relative_to(root)), p.is_symlink() or p.stat().st_mtime_ns) for p in root.rglob("*")) + + +def _subprocess_env(**extra): + env = dict(os.environ) + src = str(Path(foldmatch.__file__).resolve().parents[1]) + env["PYTHONPATH"] = src + os.pathsep + env.get("PYTHONPATH", "") + env.update(extra) + return env + + +def _registered(*systems): + """The registry holding exactly ``systems`` (the bundle loader serves registry entries only).""" + return mock.patch.object(ss, "REGISTRY", ss._registry(*systems)) + + +class _CacheCase(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.cache = Path(self._tmp.name) / "hub" + self.cache.mkdir() + + def register(self, *systems): + """Make ``systems`` the registry for the rest of the test; returns the first.""" + patch = _registered(*systems) + patch.start() + self.addCleanup(patch.stop) + return systems[0] + + def tearDown(self): + for p in [self.cache, *self.cache.rglob("*")]: # undo read-only modes so the cleanup can delete + if p.is_dir() and not p.is_symlink(): + p.chmod(stat.S_IRWXU) + self._tmp.cleanup() + + +# ===================================================================================================================== +# get_esm_codes_bundle +# ===================================================================================================================== +class _FakeHub: + """Stands in for hf_hub_download: serves ``files[(repo, revision, filename)]`` from a temp dir (``forced[key]`` + instead on a ``force_download``), records the calls and their ``local_files_only`` / ``force_download``.""" + + def __init__(self, root: Path, files, forced=None): + self.root, self.files, self.forced, self.calls, self.flags = root, files, dict(forced or {}), [], [] + + def __call__(self, *, repo_id, filename, revision, cache_dir=None, local_files_only=False, force_download=False): + self.calls.append((repo_id, revision, filename, cache_dir)) + self.flags.append((filename, local_files_only, force_download)) + key = (repo_id, revision, filename) + if key not in self.files: + raise LocalEntryNotFoundError(f"{key} not cached") + path = self.root / repo_id.replace("/", "--") / revision / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(self.forced.get(key, self.files[key]) if force_download else self.files[key]) + return str(path) + + +def _hub_system(name, revision, contents): + return _synthetic(name=name, bundle_revision=revision, + bundle_files=[[f, _sha(b)] for f, b in contents.items()]) + + +class BundleLoaderTest(_CacheCase): + + CONTENTS = {"bundle.json": b"{}", "encoders.pt": b"weights", "scoring.npz": b"panel"} + + def _hub(self, *systems): + files = {(s.bundle_repo, s.bundle_revision, f): self.CONTENTS[f] for s in systems for f, _ in s.bundle_files} + return _FakeHub(self.cache, files) + + def test_returns_verified_paths(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + hub = self._hub(s) + with mock.patch.object(model, "hf_hub_download", hub): + files = model.get_esm_codes_bundle(s) + self.assertEqual(sorted(files), sorted(self.CONTENTS)) + for name, path in files.items(): + self.assertIsInstance(path, Path) + self.assertEqual(path.read_bytes(), self.CONTENTS[name]) + self.assertEqual({c[:2] for c in hub.calls}, {(s.bundle_repo, s.bundle_revision)}) + self.assertEqual({f[1:] for f in hub.flags}, {(False, False)}) + + def test_every_call_hashes_every_file_and_writes_nothing(self): + system = self.register(_SyntheticBundle().write(self.cache)) + before = _listing(self.cache) + spy = mock.Mock(wraps=model._file_sha256) + with mock.patch.object(model, "_file_sha256", spy): + first = model.get_esm_codes_bundle(system, cache_dir=self.cache) + second = model.get_esm_codes_bundle(system, cache_dir=self.cache) + self.assertEqual(first, second) + self.assertEqual(spy.call_count, 2 * len(system.bundle_files)) + self.assertEqual(_listing(self.cache), before) + + def test_sha256_mismatch_is_refused(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, dict(self.CONTENTS, **{"scoring.npz": b"other panel"}))) + with mock.patch.object(model, "hf_hub_download", self._hub(s)): + with self.assertRaisesRegex(BundleMismatchError, r"scoring\.npz.*expected " + _sha(b"other panel")): + model.get_esm_codes_bundle(s) + + def test_corrupt_cache_file_is_refused(self): + """The message names the blob to delete: deleting the snapshot link alone re-links the same corrupt blob.""" + bundle = _SyntheticBundle() + system = self.register(bundle.write(self.cache)) + blob = self.cache / "models--owner--bundle" / "blobs" / system.bundle_sha256(bundle.SCORING) + blob.write_bytes(b"truncated") + with self.assertRaisesRegex(BundleMismatchError, "corrupt") as ctx: + model.get_esm_codes_bundle(system, cache_dir=self.cache) + self.assertIn(f"delete {blob.resolve()} (the cache blob ", str(ctx.exception)) + self.assertIn("download-models", str(ctx.exception)) + + def test_download_models_redownloads_a_corrupt_file(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + key = (s.bundle_repo, s.bundle_revision, "scoring.npz") + files = {(s.bundle_repo, s.bundle_revision, f): b for f, b in self.CONTENTS.items()} + files[key] = b"truncated" + hub = _FakeHub(self.cache, files, forced={key: self.CONTENTS["scoring.npz"]}) + with mock.patch.object(model, "hf_hub_download", hub): + with self.assertRaisesRegex(BundleMismatchError, "corrupt"): # an ordinary call never forces + model.get_esm_codes_bundle(s) + self.assertFalse(any(f[2] for f in hub.flags)) + hub.flags.clear() + got = model.download_esm_codes_bundles(["esm-codes-a"])["esm-codes-a"] + self.assertEqual(got["scoring.npz"].read_bytes(), self.CONTENTS["scoring.npz"]) + self.assertEqual([f for f in hub.flags if f[2]], [("scoring.npz", False, True)]) # forced once, that file + + def test_redownload_that_still_differs_or_fails(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + key = (s.bundle_repo, s.bundle_revision, "scoring.npz") + files = {(s.bundle_repo, s.bundle_revision, f): b for f, b in self.CONTENTS.items()} + files[key] = b"truncated" + with mock.patch.object(model, "hf_hub_download", _FakeHub(self.cache, files, forced={key: b"other"})): + with self.assertRaisesRegex(BundleMismatchError, "after a fresh download"): + model.get_esm_codes_bundle(s, redownload_corrupt=True) + hub = _FakeHub(self.cache, files) + + def failing(**kw): + if kw.get("force_download"): + raise ValueError("Force download failed due to the above error.") + return hub(**kw) + with mock.patch.object(model, "hf_hub_download", failing): + with self.assertRaisesRegex(BundleUnavailableError, "could not be downloaded again"): + model.get_esm_codes_bundle(s, redownload_corrupt=True) + with self.assertRaisesRegex(BundleMismatchError, "corrupt"): # cached-only never forces + model.get_esm_codes_bundle(s, redownload_corrupt=True, local_files_only=True) + + def test_unpinned_revisions_are_refused_before_any_download(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + hub = self._hub(s) + for bad in (None, "main", "a" * 12, "A" * 40): + loose = copy.copy(s) + object.__setattr__(loose, "bundle_revision", bad) + with self.subTest(revision=bad), mock.patch.object(model, "hf_hub_download", hub): + with self.assertRaisesRegex(ValueError, "unpinned"): + model.get_esm_codes_bundle(loose) + self.assertEqual(hub.calls, []) + + def test_only_registry_systems(self): + for bad in (None, {"bundle_revision": "a" * 40}, SimpleNamespace(**_synthetic().__dict__)): + with self.subTest(system=type(bad).__name__), self.assertRaises(TypeError): + model.get_esm_codes_bundle(bad) + + def test_only_the_registered_entry(self): + """A system parsed from elsewhere (e.g. a database manifest) is fetched only if it equals the registry entry + of its name; a foreign repo or revision under the registered name, or an unknown name, is refused before + any Hub call (P4 review).""" + entry = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + d = json.loads(entry.to_json()) + d.update(bundle_repo="someone/else", bundle_revision="d" * 40) + cases = {"foreign repo and revision": (EsmCodesSystem.from_json(json.dumps(d)), + r"in \['bundle_repo', 'bundle_revision'\]"), + "other name": (dataclasses.replace(entry, name="esm-codes-z"), "knows: esm-codes-a"), + "one ulp": (dataclasses.replace(entry, lam=[_up(entry.lam[0])] + list(entry.lam[1:])), + r"in \['lam'\]")} + hub = self._hub(entry) + for label, (system, pattern) in cases.items(): + for fetch in (model.get_esm_codes_bundle, ss.prepare): + with self.subTest(case=label, call=fetch.__name__), \ + mock.patch.object(model, "hf_hub_download", hub): + with self.assertRaisesRegex(UnknownScoringSystemError, pattern): + fetch(system) + self.assertEqual(hub.calls, []) + with mock.patch.object(model, "hf_hub_download", hub): + model.get_esm_codes_bundle(EsmCodesSystem.from_json(entry.to_json())) # an equal copy is the entry + self.assertEqual({c[:2] for c in hub.calls}, {(entry.bundle_repo, entry.bundle_revision)}) + self.assertIs(ss.registered(entry), entry) + + def test_not_cached_and_offline(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + with mock.patch.object(model, "hf_hub_download", side_effect=LocalEntryNotFoundError("offline")): + with self.assertRaisesRegex(BundleUnavailableError, "download-models") as ctx: + model.get_esm_codes_bundle(s) + self.assertIn(f"{s.bundle_repo}@{s.bundle_revision}", str(ctx.exception)) + + def test_cached_only(self): + """local_files_only (the every-rank pre-flight after one fetch): passed to every call, never downloads.""" + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + hub = self._hub(s) + with mock.patch.object(model, "hf_hub_download", hub): + model.get_esm_codes_bundle(s, local_files_only=True) + self.assertEqual(hub.flags, [(f, True, False) for f, _ in s.bundle_files]) + bundle = _SyntheticBundle() + system = self.register(bundle.write(self.cache)) + before = _listing(self.cache) + ss.prepare(system, cache_dir=self.cache, local_files_only=True) # the real hf_hub_download + missing = self.register(dataclasses.replace(system, bundle_revision="c" * 40)) + with self.assertRaisesRegex(BundleUnavailableError, r"cached-only use was requested \(local_files_only"): + ss.prepare(missing, cache_dir=self.cache, local_files_only=True) + self.assertEqual(_listing(self.cache), before) + + def test_read_only_cache_without_the_bundle(self): + """A reachable Hub but a cache that cannot be written: a bundle refusal naming download-models, not a raw + PermissionError (P4 review). Other OS / HTTP errors are refusals too.""" + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + from huggingface_hub.errors import HfHubHTTPError + errors = {"read-only": (PermissionError(13, "Permission denied", "/ro/hub/models--owner--bundle"), + "the cache is not writable"), + "disk full": (OSError(28, "No space left on device"), "could not be fetched"), + "server error": (HfHubHTTPError("500 Server Error"), "HfHubHTTPError")} + for label, (error, pattern) in errors.items(): + with self.subTest(case=label), mock.patch.object(model, "hf_hub_download", side_effect=error): + with self.assertRaisesRegex(BundleUnavailableError, pattern): + model.get_esm_codes_bundle(s) + + def test_missing_at_the_pinned_revision(self): + s = self.register(_hub_system("esm-codes-a", "a" * 40, self.CONTENTS)) + with mock.patch.object(model, "hf_hub_download", side_effect=EntryNotFoundError("404")): + with self.assertRaisesRegex(BundleMismatchError, "does not match the Hub"): + model.get_esm_codes_bundle(s) + + def test_the_bundle_comes_only_from_the_registry(self): + """A second registered system with another revision downloads that revision, and only that one.""" + a = _hub_system("esm-codes-a", "a" * 40, self.CONTENTS) + b = _hub_system("esm-codes-b", "b" * 40, self.CONTENTS) + hub = self._hub(a, b) + with _registered(a, b), mock.patch.object(model, "hf_hub_download", hub): + model.get_esm_codes_bundle(ss.get("esm-codes-b")) + self.assertEqual({c[1] for c in hub.calls}, {"b" * 40}) + hub.calls.clear() + model.download_esm_codes_bundles(["esm-codes-b"], cache_dir="/some/cache") + self.assertEqual({(c[1], c[3]) for c in hub.calls}, {("b" * 40, "/some/cache")}) + hub.calls.clear() + got = model.download_esm_codes_bundles() + self.assertEqual(list(got), ["esm-codes-a", "esm-codes-b"]) + self.assertEqual(sorted({c[1] for c in hub.calls}), ["a" * 40, "b" * 40]) + self.assertEqual(len(hub.calls), 2 * len(self.CONTENTS)) + hub.calls.clear() + with self.assertRaisesRegex(UnknownScoringSystemError, "esm-codes-a, esm-codes-b"): + model.download_esm_codes_bundles(["esm-codes-b", "esm-codes-x"]) + self.assertEqual(hub.calls, []) + + +# ===================================================================================================================== +# prepare() +# ===================================================================================================================== +class PrepareTest(_CacheCase): + + def test_prepare_and_worker_view(self): + bundle = _SyntheticBundle() + system = self.register(bundle.write(self.cache)) + prepared = ss.prepare(system, cache_dir=self.cache) + self.assertIs(prepared.system, system) + self.assertEqual((prepared.encoders_file, prepared.scoring_file), (bundle.ENCODERS, bundle.SCORING)) + self.assertEqual(prepared.bundle["system"], system.name) + spec = prepared.for_worker() + clone = pickle.loads(pickle.dumps(spec)) + self.assertEqual(clone, spec) + for f in dataclasses.fields(spec): + value = getattr(spec, f.name) + leaves = value if isinstance(value, tuple) else ( + dataclasses.astuple(value) if dataclasses.is_dataclass(value) else (value,)) + self.assertTrue(all(isinstance(v, (str, int, float)) for v in leaves), f.name) + self.assertEqual(spec.system_digest, system.digest()) + self.assertEqual(spec.dp_params(), swkern.SETTING_D) + codes, offsets = spec.load_panel() + np.testing.assert_array_equal(codes, bundle.codes) + np.testing.assert_array_equal(offsets, bundle.offsets) + self.assertEqual(cs.lam_lnk(100, spec), cs.lam_lnk(100, system)) # a WorkerSpec is a stats table + + def test_worker_rechecks_the_npz(self): + bundle = _SyntheticBundle() + system = self.register(bundle.write(self.cache)) + spec = ss.prepare(system, cache_dir=self.cache).for_worker() + Path(spec.scoring_npz).resolve().write_bytes(b"replaced") + with self.assertRaisesRegex(BundleMismatchError, "sha256"): + spec.load_panel() + + def _refused(self, bundle, pattern, system=None): + written = bundle.write(self.cache) + candidate = self.register(system(written) if system else written) + with self.assertRaisesRegex(BundleMismatchError, pattern) as ctx: + ss.prepare(candidate, cache_dir=self.cache) + return str(ctx.exception) + + def test_table_off_by_one_ulp_is_refused(self): + for key, i in (("lam", 0), ("lnK", 5), ("bin_centres", 2)): + with self.subTest(field=key): + def ulp(s, key=key, i=i): + v = list(getattr(s, key)) + v[i] = math.nextafter(v[i], math.inf) + return dataclasses.replace(s, **{key: v}) + self._refused(_SyntheticBundle(), rf"bundle\.json statistics\.{key} = .*the system has {key}", ulp) + + def test_document_values_are_compared_exactly(self): + cases = { + "bundle.json gap_open one ulp": (lambda b: b.bundle_json["scoring"].update(gap_open=_up(5.0)), + r"bundle\.json scoring\.gap_open"), + "meta_json lam one ulp": (lambda b: b.meta["statistics"]["lam"].__setitem__(1, _up(0.12)), + r"meta_json statistics\.lam"), + "npz lnK array one ulp": (lambda b: b.arrays.__setitem__("lnK", np.nextafter(b.arrays["lnK"], 0)), + r"scoring npz lnK = \[-13\.39999.* differs from the system's \[-13\.4, " + r"-14\.0, .*\] at index 0 \(exact float64"), + "encoder architecture": (lambda b: b.bundle_json["encoding"]["architecture"].update(name="other-mlp"), + r"bundle\.json encoding\.architecture\.name = 'other-mlp'.*encoder_architecture"), + "encoder dtype": (lambda b: b.bundle_json["encoding"]["encoder"].update(dtype="bfloat16"), + r"bundle\.json encoding\.encoder\.dtype = 'bfloat16'"), + "no architecture": (lambda b: b.bundle_json["encoding"].pop("architecture"), + r"bundle\.json has no encoding\.architecture\.name"), + "a number as text": (lambda b: b.bundle_json["scoring"].update(offset="1.0"), + r"scoring\.offset = '1\.0'"), + "missing section": (lambda b: b.bundle_json.pop("calibrated_domain"), + r"bundle\.json has no calibrated_domain\.query_min_length"), + "calibrated domain": (lambda b: b.meta["calibrated_domain"].update(target_max_length=1000), + r"meta_json calibrated_domain\.target_max_length = 1000"), + "high-m rule": (lambda b: b.bundle_json["statistics"].update(high_m_rule="extrapolate"), + r"statistics\.high_m_rule"), + "esm weights": (lambda b: b.meta["esm3"].update(weights_sha256="0" * 64), r"meta_json esm3\.weights"), + "hf repo": (lambda b: b.bundle_json.update(hf_repo="other/repo"), r"hf_repo"), + "table digest": (lambda b: b.bundle_json["content_digests"].update(table_sha256="0" * 64), + r"table_sha256"), + "digest method": (lambda b: b.bundle_json["content_digests"].update(method="other/1"), + r"content_digests\.method"), + "meta format": (lambda b: b.meta.update(format="other/1"), r"scoring npz format"), + "panel byte": (lambda b: b.arrays.__setitem__("panel_codes", b.codes ^ np.eye(40, 8, dtype=np.uint8)), + r"panel content digest"), + "panel lengths": (lambda b: b.arrays.__setitem__("panel_lengths", b.arrays["panel_lengths"][::-1]), + r"panel_lengths"), + "unpinned file": (lambda b: b.bundle_json["files"].update({"extra.bin": {"sha256": "0" * 64}}), + r"lists extra\.bin, which the system does not pin"), + "file sha in bundle.json": (lambda b: b.bundle_json["files"][b.ENCODERS].update(sha256="0" * 64), + r"gives esm_codes_test_encoders\.pt sha256 0+"), + "no scoring file": (lambda b: b.bundle_json["files"][b.SCORING].update(format="other/1"), + r"lists no foldmatch-esm-codes-scoring/1 file"), + } + for label, (edit, pattern) in cases.items(): + with self.subTest(case=label): + b = _SyntheticBundle() + edit(b) + self.assertNotIn("np.float64(", self._refused(b, pattern)) + + def test_integral_json_numbers_equal_floats(self): + b = _SyntheticBundle() + b.bundle_json["scoring"].update(gap_open=5, offset=1) + system = self.register(b.write(self.cache)) + ss.prepare(system, cache_dir=self.cache) + + def test_unknown_bundle_format(self): + b = _SyntheticBundle() + b.bundle_json["bundle_format"] = "foldmatch-esm-codes-bundle/2" + system = self.register(b.write(self.cache)) + with self.assertRaisesRegex(BundleMismatchError, "bundle.json format"): + ss.prepare(system, cache_dir=self.cache) + + def test_unsupported_systems_are_refused_before_any_download(self): + cases = {"dp": dict(dp="fmsw/2"), "setting": dict(gap_open=6.0), "high-m rule": dict(high_m_rule="extra"), + "similarity": dict(similarity="pm1_fp32_fixed_order"), "eps": dict( + normalisation={"name": "rowcol_z", "std": "population", "eps": 1e-7, "degenerate_std_lt": 1e-6}), + "code layout": dict(code_bits=128, code_bytes=16), "degenerate rule": dict(degenerate_query_rule="x"), + "architecture": dict(encoder_architecture="esm-codes-mlp-v2"), + "encoder dtype": dict(encoder_dtype="bfloat16")} + for label, overrides in cases.items(): + with self.subTest(case=label), mock.patch.object(model, "hf_hub_download") as hub: + with self.assertRaises(UnsupportedSystemError): + ss.prepare(_synthetic(**overrides)) + hub.assert_not_called() + ss.check_implemented(_synthetic()) + + def test_dp_values_come_from_the_kernel(self): + s = _synthetic() + self.assertEqual((s.offset, s.gap_open, s.gap_extend), (swkern.SETTING_D.offset, swkern.SETTING_D.gap_open, + swkern.SETTING_D.gap_extend)) + with mock.patch.object(swkern, "DP_ID", "fmsw/9"), self.assertRaisesRegex(UnsupportedSystemError, "fmsw/9"): + ss.check_implemented(s) + with mock.patch.object(swkern, "SETTING_D", swkern.DPParams(1.0, 5.0, 1.0)), \ + self.assertRaisesRegex(UnsupportedSystemError, "setting D"): + ss.check_implemented(s) + + def test_format_tags_match_the_encoder_module(self): + from foldmatch.model import residue_code_encoder as rce + self.assertEqual(ss.ENCODERS_FORMAT, rce.ENCODERS_FORMAT) + self.assertEqual(rce.DIGEST_METHOD, "foldmatch-tensor-digest/1") + self.assertIn(ss.IMPLEMENTED["encoder_architecture"], rce.ARCHITECTURES) + self.assertEqual(ss.IMPLEMENTED["encoder_dtype"], rce.IMPLEMENTED_CONVENTIONS[("encoder", "dtype")]) + + +class _OfflineBase(_CacheCase): + """The bundle loader and prepare() in a subprocess with HF_HUB_OFFLINE=1, an unreachable endpoint (any network + use would fail) and, optionally, a read-only cache; without torch (Stage 2 never imports it).""" + + CODE = textwrap.dedent(""" + import json, sys + sys.modules["torch"] = None + from pathlib import Path + from foldmatch.search import code_statistics as cs, scoring_system as ss + from foldmatch.utils import model + system = ss.EsmCodesSystem.from_json(Path(sys.argv[1]).read_text()) + if sys.argv[2] == "register": # a synthetic system; the published one is registered + ss.REGISTRY = ss._registry(system) + files = model.get_esm_codes_bundle(system) + spec = ss.prepare(system).for_worker() + codes, offsets = spec.load_panel() + print(json.dumps(dict(files=sorted(files), residues=int(offsets[-1]), lnE=cs.log_evalue(3.0, 100, 1e9, spec), + torch=sys.modules.get("torch") is not None))) + """) + + REGISTER = True + + def _run(self, system, *, offline=True): + spec = Path(self._tmp.name) / "system.json" + spec.write_text(system.to_json()) + env = _subprocess_env(HF_HUB_CACHE=str(self.cache), HF_ENDPOINT="http://127.0.0.1:9", + HF_HUB_DISABLE_TELEMETRY="1", HF_HUB_ETAG_TIMEOUT="2", HF_HUB_DOWNLOAD_TIMEOUT="2") + env.pop("HF_HUB_OFFLINE", None) + if offline: + env["HF_HUB_OFFLINE"] = "1" + return subprocess.run([sys.executable, "-c", self.CODE, str(spec), "register" if self.REGISTER else "-"], + env=env, capture_output=True, text=True, timeout=300) + + def _check(self, system, **kw): + before = _listing(self.cache) + r = self._run(system, **kw) + self.assertEqual(r.returncode, 0, r.stderr[-3000:]) + out = json.loads(r.stdout.strip().splitlines()[-1]) + self.assertEqual(out["files"], sorted(n for n, _ in system.bundle_files)) + self.assertEqual(out["residues"], system.panel_residues) + self.assertFalse(out["torch"]) + self.assertEqual(_listing(self.cache), before) + return out + + def _require_read_only(self): + """Skip when the permissions do not make the cache read-only (e.g. a CI container running as root).""" + probe = self.cache / ".write-probe" + try: + probe.write_text("x") + except OSError: + return + probe.unlink() + self.skipTest("the cache stays writable after chmod (running as root?): read-only use cannot be tested here") + + def _read_only(self): + for p in [self.cache, *self.cache.rglob("*")]: + if p.is_dir() and not p.is_symlink(): + p.chmod(stat.S_IRUSR | stat.S_IXUSR) + elif p.is_file() and not p.is_symlink(): + p.chmod(stat.S_IRUSR) + + +class OfflineReadOnlyTest(_OfflineBase): + + def test_offline(self): + self._check(_SyntheticBundle().write(self.cache)) + + def test_read_only_cache_offline_and_online(self): + system = _SyntheticBundle().write(self.cache) + self._read_only() + self._require_read_only() + self._check(system, offline=True) + self._check(system, offline=False) # a cached file at a commit revision: no network, no write + + def test_offline_and_not_cached(self): + system = _SyntheticBundle().write(self.cache) + missing = dataclasses.replace(system, bundle_revision="c" * 40) + r = self._run(missing) + self.assertNotEqual(r.returncode, 0) + self.assertIn("BundleUnavailableError", r.stderr) + self.assertIn("offline mode is on", r.stderr) + + +# ===================================================================================================================== +# download-models +# ===================================================================================================================== +class DownloadModelsTest(unittest.TestCase): + + def setUp(self): + self.a, self.b = _synthetic(name="esm-codes-a"), _synthetic(name="esm-codes-b", bundle_revision="b" * 40) + self.fetched = [] + + def fake_bundle(system, cache_dir=None, **kw): + self.fetched.append(system.name) + return {n: Path("/cache/snap") / n for n, _ in system.bundle_files} + from foldmatch.utils.esm import loaders + patches = [mock.patch.object(ss, "REGISTRY", ss._registry(self.a, self.b)), + mock.patch.object(model, "get_esm_codes_bundle", side_effect=fake_bundle), + mock.patch.object(loaders, "download_esm3_snapshot", return_value=Path("/cache/esm3")), + mock.patch.object(model, "download_aggregator_weights", return_value=Path("/cache/agg/a.pt"))] + self.mocks = [p.start() for p in patches] + for p in patches: + self.addCleanup(p.stop) + + def _apps(self): + from foldmatch.cli import embedding, inference, search + return {"fm-search": search.app, "fm-embedding": embedding.app, "inference": inference.app} + + def test_every_registered_system_by_default(self): + for label, app in self._apps().items(): + with self.subTest(cli=label): + self.fetched.clear() + r = CliRunner().invoke(app, ["download-models"]) + self.assertEqual(r.exit_code, 0, r.output) + self.assertEqual(self.fetched, ["esm-codes-a", "esm-codes-b"]) + self.assertIn(f"esm-codes-b: owner/bundle@{'b' * 40}, 3 files, sha256 verified", r.output) + + def test_system_restricts(self): + for label, app in self._apps().items(): + with self.subTest(cli=label): + self.fetched.clear() + r = CliRunner().invoke(app, ["download-models", "--system", "esm-codes-b"]) + self.assertEqual(r.exit_code, 0, r.output) + self.assertEqual(self.fetched, ["esm-codes-b"]) + + def test_unknown_system_is_refused_before_any_download(self): + esm3, aggregator = self.mocks[2], self.mocks[3] + for label, app in self._apps().items(): + with self.subTest(cli=label): + r = CliRunner().invoke(app, ["download-models", "--system", "esm-codes-a", "--system", "nope"]) + self.assertEqual(r.exit_code, 2, r.output) + self.assertIn("unknown esm-codes scoring system 'nope'", r.output) + self.assertIn("esm-codes-a, esm-codes-b", r.output) + self.assertNotIn("database", r.output) + self.assertEqual(self.fetched, []) + esm3.assert_not_called() + aggregator.assert_not_called() + + def test_help_names_the_option(self): + for label, app in self._apps().items(): + with self.subTest(cli=label): + r = CliRunner().invoke(app, ["download-models", "--help"]) + self.assertEqual(r.exit_code, 0) + self.assertIn("--system", r.output) + self.assertIn("esm-codes", r.output) + + +# ===================================================================================================================== +# the frozen registry (P4.4) and the published bundle +# ===================================================================================================================== +# The golden digests of the registry freeze live in tests/test_scoring_system_golden.py (standard library only, so the +# CI kernel jobs on Linux and macOS run them too). +_FIXTURES = Path(__file__).parent / "resources" / "esm_codes" + + +class RegistryEntryTest(unittest.TestCase): + + def test_default_is_registered(self): + self.assertEqual(ss.known_names(), ("esm-codes-v1",)) + self.assertIs(ss.get(ss.DEFAULT_ESM_CODES), ss.REGISTRY["esm-codes-v1"]) + + def test_every_entry_is_implemented(self): + for s in ss.REGISTRY.values(): + ss.check_implemented(s) + self.assertEqual(s.dp, swkern.DP_ID) + self.assertEqual(swkern.DPParams(s.offset, s.gap_open, s.gap_extend), swkern.SETTING_D) + + def test_esm3_pin_is_the_loaders(self): + from foldmatch.utils.esm import loaders + for s in ss.REGISTRY.values(): + self.assertEqual((s.esm_repo, s.esm_weights_sha256), (loaders.ESM_REPO_ID, loaders.ESM_WEIGHTS_SHA256)) + + def test_v1_matches_the_frozen_fixtures(self): + """The v1 entry against the P0.8 fixtures (made from the research artefacts, independently of the bundle): + the E-value table of ship_final.json bit for bit, the setting and the identity strings.""" + s = ss.get("esm-codes-v1") + f4 = json.loads((_FIXTURES / "evalue_grid_v1.json").read_text()) + self.assertEqual(s.bin_centres, tuple(f4["table"]["center_len"])) + self.assertEqual(s.lam, tuple(f4["table"]["lam"])) + self.assertEqual(s.lnK, tuple(f4["table"]["lnK"])) + fixtures = json.loads((_FIXTURES / "FIXTURES.json").read_text()) + self.assertEqual((s.name, s.calibration_version), (fixtures["system"], fixtures["calibration_version"])) + setting = fixtures["setting"] + self.assertEqual((s.offset, s.gap_open, s.gap_extend), (setting["offset"], setting["gap_open"], + setting["gap_extend"])) + self.assertEqual((s.normalisation.degenerate_std_lt, s.query_prep_eps), (setting["s7_eps"], + setting["qprep_eps"])) + self.assertEqual(s.esm_weights_sha256, fixtures["esm3"]["weights_blob_sha256"]) + self.assertEqual(f4["calibration_version"], s.calibration_version) + + def test_v1_encoder_is_the_encoder_modules(self): + from foldmatch.model import residue_code_encoder as rce + s = ss.get("esm-codes-v1") + self.assertEqual((s.encoder_architecture, s.encoder_dtype), ("esm-codes-mlp-v1", "float32")) + self.assertEqual(rce.ARCHITECTURES[s.encoder_architecture]["activation"], "gelu_exact") + + def test_v1_calibrated_domain_and_rules(self): + """S13 / S13b / D14 / D15 as decided 2026-10-12.""" + s = ss.get("esm-codes-v1") + self.assertEqual(s.calibrated_domain, CalibratedDomain(21, 966, 3000, 0.01, 2)) + self.assertEqual((s.low_m_rule, s.high_m_rule, s.degenerate_query_rule), ("clamp", "clamp", "no_hits")) + self.assertEqual((s.similarity, s.determinism), ("pm1_fp32_blas_gemm_per_pair", "tolerance_s16_v1")) + self.assertEqual((s.panel_size, s.panel_residues, s.panel_quantile, s.quantile_method), + (1000, 355626, 0.9, "linear")) + + def test_bundle_json_states_every_value_but_the_pins(self): + stated = {entry[0][0] for entry in ss.DOCUMENT_FIELDS} + self.assertEqual(stated, {f.name for f in dataclasses.fields(EsmCodesSystem)} - {"bundle_revision", + "bundle_files"}) + + +class SingleSourceOfTruthTest(unittest.TestCase): + """A bundle revision is written down exactly once: in its registry entry (plan principle 2, R5).""" + + SKIP_DIRS = {".git", "build", "dist", "__pycache__", ".venv", "venv", ".idea", ".pytest_cache"} + SKIP_SUFFIXES = {".so", ".dylib", ".pyc", ".o"} + + def test_bundle_revision_exists_only_in_the_registry_entry(self): + root = Path(__file__).resolve().parents[1] + if not (root / "src" / "foldmatch").is_dir(): + self.skipTest("not a source checkout") + revisions = {s.bundle_revision.encode() for s in ss.REGISTRY.values()} + hits = {} + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in self.SKIP_DIRS and not d.endswith(".egg-info") + and (not d.startswith(".") or d == ".github")] + for f in filenames: + path = Path(dirpath) / f + if path.suffix in self.SKIP_SUFFIXES or path.is_symlink(): + continue + data = path.read_bytes() + for rev in revisions: + if rev in data: + hits[(path.relative_to(root).as_posix(), rev.decode())] = data.count(rev) + self.assertEqual(hits, {("src/foldmatch/search/scoring_system.py", r.decode()): 1 for r in revisions}) + + +class RealBundleTest(unittest.TestCase): + """prepare() and the head on the published bundle at the registry's revision (downloaded once, sha256-verified).""" + + @classmethod + def setUpClass(cls): + import esm_codes_bundle + cls.files = esm_codes_bundle.registry_files() + cls.system = ss.get(ss.DEFAULT_ESM_CODES) + cls.prepared = ss.prepare(cls.system) + + def test_files_are_the_pinned_ones(self): + self.assertEqual(sorted(self.files), [n for n, _ in self.system.bundle_files]) + for name, sha in self.system.bundle_files: + self.assertEqual(model._file_sha256(self.files[name]), sha, name) + self.assertEqual(Path(self.files[name]).parent.name, self.system.bundle_revision) + + def test_prepare_accepts_the_published_bundle(self): + p = self.prepared + self.assertEqual((p.encoders_file, p.scoring_file), ("esm_codes_v1_encoders.pt", "esm_codes_v1_scoring.npz")) + self.assertEqual(p.bundle["licence"]["spdx"], "MIT") + spec = p.for_worker() + self.assertEqual(pickle.loads(pickle.dumps(spec)), spec) + codes, offsets = spec.load_panel() + self.assertEqual((codes.shape, codes.dtype, offsets.shape), ((355626, 8), np.uint8, (1001,))) + self.assertEqual(tensor_digest({"codes": codes, "offsets": offsets}), self.system.panel_sha256) + + def test_table_off_by_one_ulp_is_refused(self): + """A registry entry one ulp off the bundle (as if mistyped and registered) is refused by prepare(); an + unregistered variant never reaches the bundle (test_unregistered_variants_are_refused).""" + for key, i, direction in (("lam", 3, math.inf), ("lnK", 0, -math.inf), ("bin_centres", 5, math.inf)): + v = list(getattr(self.system, key)) + v[i] = math.nextafter(v[i], direction) + variant = dataclasses.replace(self.system, **{key: v}) + with self.subTest(field=key), _registered(variant), \ + self.assertRaisesRegex(BundleMismatchError, rf"statistics\.{key}"): + ss.prepare(variant) + + def test_other_differences_are_refused(self): + cases = {"panel_quantile": 0.95, "calibration_version": "foldmatch-evalue-other", + "calibrated_domain": CalibratedDomain(21, 966, 1000, 0.01, 2), "panel_residues": 355627, + "target_encoder_sha256": "0" * 64, "name": "esm-codes-v2"} + for key, value in cases.items(): + variant = dataclasses.replace(self.system, **{key: value}) + with self.subTest(field=key), _registered(variant), self.assertRaises(BundleMismatchError): + ss.prepare(variant) + + def test_unregistered_variants_are_refused(self): + d = json.loads(self.system.to_json()) + d["bundle_revision"] = "d" * 40 + variants = {"foreign revision": EsmCodesSystem.from_json(json.dumps(d)), + "other quantile": dataclasses.replace(self.system, panel_quantile=0.95)} + for label, variant in variants.items(): + with self.subTest(case=label), mock.patch.object(model, "hf_hub_download") as hub: + with self.assertRaises(UnknownScoringSystemError): + ss.prepare(variant) + hub.assert_not_called() + self.assertEqual(ss.registered(EsmCodesSystem.from_json(self.system.to_json())), self.system) + + def test_load_head(self): + head = self.prepared.load_head() + self.assertEqual(head.content_digests(), {k: getattr(self.system, k) for k in ( + "transform_sha256", "query_encoder_sha256", "target_encoder_sha256")}) + self.assertEqual(head.meta["system"], self.system.name) + other = dataclasses.replace(self.prepared, system=dataclasses.replace(self.system, + query_encoder_sha256="0" * 64)) + with self.assertRaisesRegex(BundleMismatchError, "query_encoder_sha256"): + other.load_head() + foreign = dataclasses.replace(self.prepared, system=dataclasses.replace(self.system, + esm_weights_sha256="0" * 64)) + with self.assertRaisesRegex(UnsupportedSystemError, "ESM3 weights"): + foreign.load_head() + for change, pattern in ((dict(encoder_architecture="esm-codes-mlp-v2"), "architecture is 'esm-codes-mlp-v1'"), + (dict(encoder_dtype="bfloat16"), "encoder dtype is 'float32'")): + other = dataclasses.replace(self.prepared, system=dataclasses.replace(self.system, **change)) + with self.subTest(**change), self.assertRaisesRegex(BundleMismatchError, pattern): + other.load_head() + + +class RealBundleOfflineTest(_OfflineBase): + """The published bundle, copied from the cache it was downloaded into, used offline and read-only, without + torch and with an unreachable endpoint.""" + + REGISTER = False # the published system is the real registry entry + + def setUp(self): + super().setUp() + import shutil + import esm_codes_bundle + self.system = ss.get(ss.DEFAULT_ESM_CODES) + repo = Path(esm_codes_bundle.registry_files()[ss.BUNDLE_JSON]).parents[2] + shutil.copytree(repo, self.cache / repo.name, symlinks=True) + + def test_offline(self): + out = self._check(self.system) + self.assertEqual(out["lnE"], cs.log_evalue(3.0, 100, 1e9, self.system)) + + def test_read_only_cache_offline_and_online(self): + self._read_only() + self._require_read_only() + self._check(self.system, offline=True) + self._check(self.system, offline=False) + + +class BundleHelperSkipRuleTest(unittest.TestCase): + """tests/esm_codes_bundle.py skips the real-bundle tests only when offline mode is on and the bundle is not + cached; an unreachable Hub with offline mode off, a sha256 mismatch or any other failure is a test error, so CI + never turns green without the acceptance checks (P4 review).""" + + def setUp(self): + import esm_codes_bundle + self.helper = esm_codes_bundle + esm_codes_bundle._fetch.cache_clear() + self.addCleanup(esm_codes_bundle._fetch.cache_clear) + + def _classify(self, *, offline, error=None): + from huggingface_hub import constants as hf_constants + self.helper._fetch.cache_clear() + files = {"bundle.json": Path("/cache/snap/bundle.json")} + with mock.patch.object(model, "get_esm_codes_bundle", side_effect=error, return_value=files), \ + mock.patch.object(hf_constants, "HF_HUB_OFFLINE", offline): + try: + got = self.helper.registry_files() + except unittest.SkipTest: + return "skip" + except RuntimeError as e: + self.assertIsInstance(e.__cause__, type(error)) + return "error" + self.assertEqual(got, files) + return "ok" + + def test_skip_only_offline_and_not_cached(self): + unavailable = BundleUnavailableError("bundle.json is not in the Hugging Face cache and offline mode is on") + unreachable = BundleUnavailableError("bundle.json is not in the cache and the Hugging Face Hub is unreachable") + cases = { + "offline, not cached": (dict(offline=True, error=unavailable), "skip"), + "Hub unreachable, offline mode off": (dict(offline=False, error=unreachable), "error"), + "sha256 mismatch offline": (dict(offline=True, error=BundleMismatchError("sha256 differs")), "error"), + "sha256 mismatch online": (dict(offline=False, error=BundleMismatchError("sha256 differs")), "error"), + "not the registry entry": (dict(offline=True, error=UnknownScoringSystemError("x")), "error"), + "a bug": (dict(offline=True, error=TypeError("x")), "error"), + "fetched": (dict(offline=False), "ok"), + "fetched offline": (dict(offline=True), "ok"), + } + for label, (kw, want) in cases.items(): + with self.subTest(case=label): + self.assertEqual(self._classify(**kw), want) + + def test_one_fetch_per_session(self): + from huggingface_hub import constants as hf_constants + with mock.patch.object(model, "get_esm_codes_bundle", side_effect=BundleUnavailableError("down")) as fetch, \ + mock.patch.object(hf_constants, "HF_HUB_OFFLINE", False): + for _ in range(3): + with self.assertRaises(RuntimeError): + self.helper.registry_files() + self.assertEqual(fetch.call_count, 1) + + +class NoTorchImportTest(unittest.TestCase): + + def test_bundle_loader_and_statistics_import_without_torch(self): + code = ("import sys; sys.modules['torch'] = None\n" + "import foldmatch.search.scoring_system, foldmatch.search.code_statistics, foldmatch.utils.model\n" + "import foldmatch.utils.tensor_digest\n" + "print('ok')\n") + r = subprocess.run([sys.executable, "-c", code], env=_subprocess_env(), capture_output=True, text=True, + timeout=300) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + self.assertEqual(r.stdout.strip(), "ok") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scoring_system_golden.py b/tests/test_scoring_system_golden.py new file mode 100644 index 0000000..58f5a7a --- /dev/null +++ b/tests/test_scoring_system_golden.py @@ -0,0 +1,84 @@ +"""P4: golden digests of every registered esm-codes scoring system (plan section 8 P4, acceptance). + +The registry froze on 2026-10-14, after P0.6, P0.7 and the bundle upload. From here on any change of a registered +system fails this test: a different value is a new system with a new name and its own golden line (D17), never an +edit of an existing entry. ``digest``: every field; ``encoding_digest``: ``ENCODING_FIELDS``; ``canonical_sha256``: +sha256 of ``to_json()``. + +The module needs only the standard library and ``foldmatch.search.scoring_system`` (which imports neither numpy nor +torch), so CI runs it in the kernel jobs too, on Linux x86_64, Linux arm64 and macOS arm64: the plan's "digests +identical on Linux and macOS CI". +""" +import hashlib +import os +import subprocess +import sys +import unittest +from pathlib import Path + +from foldmatch.search import scoring_system as ss + +GOLDEN = { + "esm-codes-v1": { + "digest": "81de063d13a7ef66", + "encoding_digest": "92220d870638cb57", + "canonical_sha256": "3c81bbceac34688135262daa92d06544b87d4a20bbdddba3fb354b14e44cfad3", + }, +} + +#: Run in a fresh interpreter with every non-stdlib dependency of FoldMatch blocked: the digests need none of them. +_FRESH = """ +import sys +for blocked in ("numpy", "torch", "huggingface_hub", "typer", "esm", "faiss", "biotite", "lightning"): + sys.modules[blocked] = None +import hashlib +from foldmatch.search import scoring_system as ss +for name, s in sorted(ss.REGISTRY.items()): + print(name, s.digest(), s.encoding_digest(), hashlib.sha256(s.to_json().encode()).hexdigest()) +""" + + +def _env(**extra): + env = dict(os.environ) + src = str(Path(ss.__file__).resolve().parents[2]) + env["PYTHONPATH"] = src + os.pathsep + env.get("PYTHONPATH", "") + env.update(extra) + return env + + +class RegistryGoldenTest(unittest.TestCase): + + def test_golden_digests(self): + self.assertEqual(sorted(ss.REGISTRY), sorted(GOLDEN)) + for name, want in GOLDEN.items(): + s = ss.get(name) + with self.subTest(system=name): + self.assertEqual(s.digest(), want["digest"]) + self.assertEqual(s.encoding_digest(), want["encoding_digest"]) + self.assertEqual(hashlib.sha256(s.to_json().encode()).hexdigest(), want["canonical_sha256"]) + self.assertEqual(ss.EsmCodesSystem.from_json(s.to_json()), s) + + def test_digests_are_their_definition(self): + """16 hex of sha256(tag + newline + canonical JSON): the definition, independent of the methods.""" + for name, want in GOLDEN.items(): + s = ss.get(name) + d = s.to_dict() + for tag, obj, key in ((ss.SYSTEM_DIGEST_TAG, d, "digest"), + (ss.ENCODING_DIGEST_TAG, {k: d[k] for k in ss.ENCODING_FIELDS}, "encoding_digest")): + text = ss.canonical_json(obj) + self.assertTrue(text.isascii()) + got = hashlib.sha256(tag.encode() + b"\n" + text.encode()).hexdigest()[:16] + self.assertEqual(got, want[key], f"{name} {key}") + + def test_digests_in_fresh_processes_without_numpy_or_torch(self): + want = [f"{n} {g['digest']} {g['encoding_digest']} {g['canonical_sha256']}" for n, g in sorted(GOLDEN.items())] + for seed in ("0", "12345"): + with self.subTest(PYTHONHASHSEED=seed): + r = subprocess.run([sys.executable, "-c", _FRESH], env=_env(PYTHONHASHSEED=seed), capture_output=True, + text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stderr[-2000:]) + self.assertEqual(r.stdout.strip().splitlines(), want) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sequence_inference.py b/tests/test_sequence_inference.py index 19c46c2..5952e6f 100644 --- a/tests/test_sequence_inference.py +++ b/tests/test_sequence_inference.py @@ -113,6 +113,9 @@ def test_sequence_chain_inference(self): def test_sequence_chain_inference_from_parquet(self): _remove_files_in_directory(self.__tmp_path) + import pyarrow as pa + import pyarrow.compute as pc + import pyarrow.parquet as pq from foldmatch.inference.sequence_inference import predict as sequence_predict from foldmatch.inference.chain_inference import predict as chain_predict @@ -127,6 +130,10 @@ def test_sequence_chain_inference_from_parquet(self): out_name="residue-embeddings" ) self.assertTrue(os.path.exists(parquet_path)) + # One row per sequence, one list of 1536 values per residue, BOS and EOS included (245 + 2 and 58 + 2). + embedding = pq.read_table(parquet_path).column("embedding") + self.assertEqual(embedding.type, pa.list_(pa.list_(pa.float32(), 1536))) + self.assertEqual(pc.list_value_length(embedding).to_pylist(), [247, 60]) chain_embeddings = chain_predict( src_stream=parquet_path, diff --git a/tests/test_sequence_normalisation.py b/tests/test_sequence_normalisation.py new file mode 100644 index 0000000..5d02628 --- /dev/null +++ b/tests/test_sequence_normalisation.py @@ -0,0 +1,252 @@ +"""Input normalisation shared by every FASTA path (G8).""" +import hashlib +import json +import string +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from foldmatch.utils.sequence import STANDARD_AMINO_ACIDS, normalize_sequence, scan_fasta, seq_sha1 + +_TEST_PATH = Path(__file__).parent +_FIXTURE_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_TOKENIZER_CASES = _TEST_PATH / "resources" / "esm_codes" / "tokenizer_cases_v1.json" + + +def _fixture_records(): + from foldmatch.utils.fasta import parse_fasta + return parse_fasta(_FIXTURE_FASTA) + + +def _messy_fasta(path: Path): + """The fixture FASTA in lowercase, with whitespace inside the sequence lines.""" + with open(path, "w") as f: + for name, sequence in _fixture_records(): + messy = sequence.lower() + lines = [messy[i:i + 30] for i in range(0, len(messy), 30)] + # Spaces and tabs inside lines, and a CRLF line ending. + lines = [" ".join([line[:10], line[10:20], line[20:]]).rstrip() + "\t" for line in lines] + f.write(f">{name} some description\r\n") + f.write("\n".join(lines) + "\n\n") + + +class NormalizeSequenceTest(unittest.TestCase): + + def test_uppercases_and_removes_ascii_whitespace(self): + self.assertEqual(normalize_sequence("acDe"), "ACDE") + self.assertEqual(normalize_sequence(" A\tC\nD\rE\x0bF\x0c "), "ACDEF") + self.assertEqual(normalize_sequence(""), "") + + def test_every_other_ascii_character_is_kept_one_for_one(self): + for char in map(chr, range(128)): + with self.subTest(char=repr(char)): + if char in string.whitespace: + expected = "" + elif char in string.ascii_lowercase: + expected = char.upper() + else: + expected = char + self.assertEqual(normalize_sequence(f"{char}"), expected) + self.assertEqual(normalize_sequence(f"A{char}C"), f"A{expected}C") + + def test_non_ascii_is_refused(self): + for raw in ("ACDÉ", "AC DE", "ßA"): + with self.subTest(raw=raw): + with self.assertRaises(ValueError): + normalize_sequence(raw) + + def test_non_ascii_kept_when_allowed(self): + # Only ASCII letters change: 'ß'.upper() would be 'SS' and shift every later residue. + self.assertEqual(normalize_sequence("aß cé", allow_non_ascii=True), "AßCé") + self.assertEqual(normalize_sequence("ac de", allow_non_ascii=True), "ACDE") + + def test_idempotent(self): + for raw in ("ac de", "XBZUO", "-.|_*", "ab"): + once = normalize_sequence(raw) + self.assertEqual(normalize_sequence(once), once) + + def test_seq_sha1(self): + self.assertEqual(seq_sha1("ACDE"), hashlib.sha1(b"ACDE").digest()) + self.assertEqual(seq_sha1("ac de\n"), seq_sha1("ACDE")) + self.assertEqual(len(seq_sha1("A")), 20) + with self.assertRaises(ValueError): + seq_sha1("ACÉ") + + def test_scan_counts_normalised_residues(self): + with tempfile.TemporaryDirectory() as tmp: + fasta = Path(tmp) / "x.fasta" + fasta.write_text(">a desc\nAC DE\n FG\t\n\n>empty\n\n>b\nmk\n>c\n \n") + records = list(scan_fasta(fasta)) + self.assertEqual([(name, length) for _offset, name, length in records], [("a", 6), ("b", 2)]) + self.assertEqual(records[0][0], 0) + + +class TokenizerInvariantTest(unittest.TestCase): + """F-tok: what ESM3's tokenizer makes of raw and normalised input (no weights needed).""" + + @classmethod + def setUpClass(cls): + from esm.tokenization import EsmSequenceTokenizer + from esm.utils.encoding import tokenize_sequence + # The tokenizer foldmatch.utils.esm.loaders gives ESM3 (ESM3.encode -> tokenize_sequence). + tokenizer = EsmSequenceTokenizer() + cls.tokenize = staticmethod( + lambda s: tokenize_sequence(s, tokenizer, add_special_tokens=True).tolist() + ) + with open(_TOKENIZER_CASES) as f: + cls.cases = json.load(f)["cases"] + + def test_cases(self): + self.assertGreaterEqual(len(self.cases), 20) + for case in self.cases: + with self.subTest(case=case["id"]): + raw = case["raw"] + self.assertEqual(self.tokenize(raw), case["tokens_raw"]) + if case["normalize_refuses"]: + with self.assertRaises(ValueError): + normalize_sequence(raw) + continue + normalized = normalize_sequence(raw) + self.assertEqual(normalized, case["normalized"]) + tokens = self.tokenize(normalized) + self.assertEqual(tokens, case["tokens_normalized"]) + # One token per residue of the normalised string, plus BOS and EOS. + self.assertEqual(len(tokens) - 2, len(normalized)) + self.assertTrue(case["length_preserved"]) + self.assertEqual(sum(c not in STANDARD_AMINO_ACIDS for c in normalized), + case["n_nonstandard"]) + + def test_lowercase_and_whitespace_no_longer_tokenise_as_unk(self): + unk = 3 + raw = "acde fghik" + self.assertIn(unk, self.tokenize(raw)) + self.assertNotIn(unk, self.tokenize(normalize_sequence(raw))) + + +class FoldMatchApiNormalisationTest(unittest.TestCase): + """FoldMatch.sequence_embedding (Python API) embeds the normalised sequence too.""" + + class FakeEsm3: + """Records the sequence ESM3 is asked to encode; no weights.""" + def __init__(self): + from types import SimpleNamespace + self.transformer = SimpleNamespace(blocks=[]) + self.encoded = [] + + def encode(self, protein): + self.encoded.append(protein.sequence) + return protein + + def forward_and_sample(self, tensor, config): + import torch + from types import SimpleNamespace + return SimpleNamespace(per_residue_embedding=torch.zeros(len(tensor.sequence) + 2, 4)) + + def _embed(self, sequence, **kwargs): + from foldmatch.foldmatch import FoldMatch + fake = self.FakeEsm3() + with patch("foldmatch.foldmatch._load_res_model", return_value=fake): + embedding = FoldMatch(**kwargs).sequence_embedding(sequence) + return fake.encoded, embedding + + def test_lowercase_and_whitespace_reach_esm3_normalised(self): + encoded, embedding = self._embed("acde fghik\tLM\r\n", min_res=1) + self.assertEqual(encoded, ["ACDEFGHIKLM"]) + self.assertEqual(tuple(embedding.shape), (13, 4)) + encoded, _ = self._embed(">q some header\nacde fghik\nlm\n", min_res=1) + self.assertEqual(encoded, ["ACDEFGHIKLM"]) + + def test_length_limits_count_normalised_residues(self): + with self.assertRaisesRegex(ValueError, "too short"): + self._embed("ACDE FGHIK", min_res=10) # 10 characters, 9 residues + encoded, _ = self._embed("ACDE FGHIK LM", max_res=11, min_res=1) # 13 characters, 11 residues + self.assertEqual(encoded, ["ACDEFGHIKLM"]) + + +class LowercaseFastaTest(unittest.TestCase): + """A lowercase FASTA with whitespace inside lines is read as the uppercase fixture.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.messy = self.tmp / "messy.fasta" + _messy_fasta(self.messy) + + def test_dataset_yields_the_normalised_sequences(self): + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + clean = EsmProtFromFasta(_FIXTURE_FASTA) + messy = EsmProtFromFasta(self.messy) + self.assertEqual(len(messy), 2) + for i in range(2): + (clean_prot, clean_name), (messy_prot, messy_name) = clean[i], messy[i] + self.assertEqual(messy_name, clean_name) + self.assertEqual(messy_prot.sequence, clean_prot.sequence) + self.assertEqual([len(messy[i][0]) for i in range(2)], [245, 58]) + + def test_min_res_n_counts_normalised_residues(self): + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + fasta = self.tmp / "spaced.fasta" + # 4 residues but 8 characters on the line. + fasta.write_text(">short\nA C D E\n>long\nACDEFGHIK\n") + dataset = EsmProtFromFasta(fasta, min_res_n=5) + self.assertEqual([dataset[i][1] for i in range(len(dataset))], ["long"]) + + def test_sequence_store_keeps_the_normalised_sequences(self): + from foldmatch.search.sequence_store import SequenceStore + store = SequenceStore(self.tmp, "messy") + store.create(self.messy) + self.assertEqual(store.fetch(["1acb_E", "2uzi_A"]), dict(_fixture_records())) + self.assertEqual(store.meta()["total_residues"], 303) + + def test_query_path_aligns_the_normalised_sequences(self): + from foldmatch.cli import search + captured = {} + + def stage2(**kwargs): + captured.update(kwargs["query_sequences"]) + + with patch("foldmatch.search.embedding_computer.chain_predict"), \ + patch("foldmatch.search.embedding_computer.EmbeddingComputer.get_embedding_batches", + return_value=iter(())), \ + patch("foldmatch.search.embedding_database.EmbeddingDatabase.load"), \ + patch("foldmatch.search.embedding_database.EmbeddingDatabase.search_by_embeddings", + return_value={}), \ + patch("foldmatch.cli.search._resolve_stage2", return_value=True), \ + patch("foldmatch.cli.search._check_stage2_stores"), \ + patch("foldmatch.cli.search._check_blosum62_kernel"), \ + patch("foldmatch.cli.search._stage2_align_and_report", side_effect=stage2): + search.query_database_from_fasta( + db_path=str(self.tmp / "db"), fasta_file=self.messy, tmp_embedding_folder=self.tmp, + output_file=str(self.tmp / "out.tsv"), + ) + self.assertEqual(captured, dict(_fixture_records())) + + def test_tokens_match_the_uppercase_fixture(self): + from esm.tokenization import EsmSequenceTokenizer + from esm.utils.encoding import tokenize_sequence + from foldmatch.dataset.esm_prot_from_fasta import EsmProtFromFasta + tokenizer = EsmSequenceTokenizer() + clean, messy = EsmProtFromFasta(_FIXTURE_FASTA), EsmProtFromFasta(self.messy) + for i in range(2): + self.assertTrue(tokenize_sequence(messy[i][0].sequence, tokenizer).equal( + tokenize_sequence(clean[i][0].sequence, tokenizer))) + + def test_chain_embeddings_match_the_uppercase_fixture(self): + """Before normalisation the lowercase records were all and embedded as garbage.""" + import torch + from foldmatch.inference.chain_complete_inference import predict + from foldmatch.types.api_types import Accelerator, SrcEsmFrom + + def embed(fasta): + batches = predict(src_stream=fasta, src_from=SrcEsmFrom.fasta, accelerator=Accelerator.cpu, + devices=1, batch_size=2) + return {name: emb for embs, names in batches for emb, name in zip(embs, names)} + + clean, messy = embed(_FIXTURE_FASTA), embed(self.messy) + self.assertEqual(sorted(messy), ["1acb_E", "2uzi_A"]) + for name in clean: + self.assertTrue(torch.equal(messy[name], clean[name]), name) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sequence_store.py b/tests/test_sequence_store.py new file mode 100644 index 0000000..d76ee3b --- /dev/null +++ b/tests/test_sequence_store.py @@ -0,0 +1,153 @@ +import os +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from foldmatch.search.sequence_store import META_KEYS, SequenceStore + +_TEST_PATH = os.path.dirname(__file__) + + +def _write_fasta(path: Path, records): + with open(path, "w") as f: + for name, sequence in records: + f.write(f">{name}\n{sequence}\n") + + +def _scan(store: SequenceStore): + conn = sqlite3.connect(str(store.path)) + try: + return conn.execute("SELECT COUNT(*), COALESCE(SUM(length), 0) FROM sequences").fetchone() + finally: + conn.close() + + +class SequenceStoreMetaTest(unittest.TestCase): + """The store records entry_count, total_residues and duplicates_ignored at build + time, and old stores without the meta table still answer from the rows.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def _store(self, records, min_res_n=0, name="db"): + fasta = self.tmp / f"{name}.fasta" + _write_fasta(fasta, records) + store = SequenceStore(self.tmp, name) + stored = store.create(fasta, min_res_n=min_res_n) + return store, stored + + def test_meta_counts_stored_entries_residues_and_duplicates(self): + records = [ + ("a", "ACDEFGHIK"), # 9 + ("b", "LMNPQ"), # 5 + ("a", "WWWWWWWWWWWW"), # duplicate id: the first occurrence is kept + ("c", "RST"), # 3, below min_res_n + ("d", "VWYACDEFGH"), # 10 + ("b", "KKKKKK"), # duplicate id + ] + store, stored = self._store(records, min_res_n=4) + meta = store.meta() + self.assertEqual(meta, {"entry_count": 3, "total_residues": 24, "duplicates_ignored": 2}) + self.assertEqual(stored, 3) + self.assertEqual(tuple(meta), META_KEYS) + for key, value in meta.items(): + self.assertIsInstance(value, int, key) + # The recorded totals equal a scan over the stored rows. + self.assertEqual(_scan(store), (meta["entry_count"], meta["total_residues"])) + self.assertEqual(store.count(), 3) + self.assertEqual(store.total_residues(), 24) + self.assertEqual(store.fetch(["a", "b"]), {"a": "ACDEFGHIK", "b": "LMNPQ"}) + + def test_fixture_fasta(self): + store = SequenceStore(self.tmp, "fixture") + store.create(Path(_TEST_PATH) / "resources" / "fasta" / "test_sequences.fasta") + self.assertEqual(store.meta(), {"entry_count": 2, "total_residues": 303, "duplicates_ignored": 0}) + + def test_empty_store(self): + store, stored = self._store([("short", "AC")], min_res_n=10) + self.assertEqual(stored, 0) + self.assertEqual(store.meta(), {"entry_count": 0, "total_residues": 0, "duplicates_ignored": 0}) + self.assertEqual(store.total_residues(), 0) + + def test_readers_use_the_meta_table(self): + store, _ = self._store([("a", "ACDE"), ("b", "FGHIK")]) + conn = sqlite3.connect(str(store.path)) + try: + conn.execute("UPDATE meta SET value = 1000 WHERE key = 'total_residues'") + conn.execute("UPDATE meta SET value = 7 WHERE key = 'entry_count'") + conn.commit() + finally: + conn.close() + self.assertEqual(store.total_residues(), 1000) + self.assertEqual(store.count(), 7) + + def test_old_store_without_meta_falls_back_to_the_rows(self): + # The layout written before the meta table existed. + store = SequenceStore(self.tmp, "old") + conn = sqlite3.connect(str(store.path)) + try: + conn.execute( + "CREATE TABLE sequences (" + "id TEXT PRIMARY KEY, sequence TEXT NOT NULL, length INTEGER NOT NULL)" + ) + conn.executemany( + "INSERT INTO sequences (id, sequence, length) VALUES (?, ?, ?)", + [("a", "ACDE", 4), ("b", "FGHIKLM", 7)], + ) + conn.commit() + finally: + conn.close() + self.assertEqual(store.meta(), {"entry_count": 2, "total_residues": 11, "duplicates_ignored": None}) + self.assertEqual(store.count(), 2) + self.assertEqual(store.total_residues(), 11) + + def test_force_rebuild_rewrites_the_meta(self): + store, _ = self._store([("a", "ACDE"), ("a", "ACDE")]) + self.assertEqual(store.meta()["duplicates_ignored"], 1) + fasta = self.tmp / "rebuild.fasta" + _write_fasta(fasta, [("x", "ACDEFG"), ("y", "HIK")]) + store.create(fasta, force=True) + self.assertEqual(store.meta(), {"entry_count": 2, "total_residues": 9, "duplicates_ignored": 0}) + + def test_interrupted_build_is_refused(self): + """A build that stopped after some batches leaves rows but no totals; that is not a store.""" + from foldmatch.search import sequence_store + fasta = self.tmp / "ten.fasta" + _write_fasta(fasta, [(f"s{i}", "ACDEFGHIKL") for i in range(10)]) + real_iter_fasta = sequence_store.iter_fasta + + def killed_after_six(path): + for i, record in enumerate(real_iter_fasta(path)): + if i == 6: + raise KeyboardInterrupt("build killed") + yield record + + store = SequenceStore(self.tmp, "interrupted") + with mock.patch.object(sequence_store, "_INSERT_BATCH", 3), \ + mock.patch.object(sequence_store, "iter_fasta", killed_after_six): + with self.assertRaises(KeyboardInterrupt): + store.create(fasta) + # Two batches of three were committed before the kill. + self.assertTrue(store.exists()) + self.assertEqual(_scan(store), (6, 60)) + for method in (store.meta, store.count, store.total_residues): + with self.subTest(method=method.__name__): + with self.assertRaisesRegex(ValueError, "is incomplete"): + method() + + def test_missing_store_raises(self): + store = SequenceStore(self.tmp, "absent") + for method in (store.meta, store.count, store.total_residues): + with self.subTest(method=method.__name__): + with self.assertRaises(FileNotFoundError): + method() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_split_run.py b/tests/test_split_run.py new file mode 100644 index 0000000..a50f9c8 --- /dev/null +++ b/tests/test_split_run.py @@ -0,0 +1,814 @@ +"""P8b.2: the run record of a split sequence build (``foldmatch.search.split_run``) and ``multi_rank_launch``. + +The shards are written by the fake ESM3 pass of ``test_cli_residue_codes`` (the pass's dataset and writer), plain and +with residue codes, on one or more ranks; the records are assembled here as step 1 assembles them. +""" +import builtins +import errno +import hashlib +import json +import os +import shutil +import socket +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import pyarrow.parquet as pq + +import foldmatch +from foldmatch.errors import RefusalError +from foldmatch.inference.chain_complete_inference import bundle_fetch_allowed +from foldmatch.search import split_run +from foldmatch.search.residue_shards import META_PASS, pass_record, run_shard_files +from foldmatch.search.sequence_store import SequenceStore +from foldmatch.utils.dist import multi_rank_launch +from foldmatch.utils.sequence import preflight_fasta +from test_cli_residue_codes import OTHER_ENCODING, SYSTEM, _write, fake_chain_predict, launch_env + +_TEST_PATH = Path(__file__).parent +_FASTA = _TEST_PATH / "resources" / "fasta" / "test_sequences.fasta" +_FIVE = ">e0\nACDEFGHIKL\n>e1 second\nKLMNPQRSTV\n>e2\nVWYACDEFGH\n>e3\nIKLMNPQRST\n>e4\nACDEFGHIKLMN\n" + + +def _system_block(system): + return {"name": system.name, "digest": system.digest(), "encoding_digest": system.encoding_digest()} + + +class _Run(unittest.TestCase): + """A temporary folder, the launch variables cleared, and helpers that write a run's shards and its record.""" + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.shards = self.tmp / "shards" + self.shards.mkdir() + self.enterContext(launch_env()) + + def write_shards(self, run_id="r", fasta=_FASTA, *, ranks=1, codes=False, min_res_n=0, folder=None, **fake): + """The shards the pass writes for ``fasta`` on ``ranks`` ranks; returns ``(preflight, shard paths)``.""" + folder = folder or self.shards + preflight = preflight_fasta(fasta, min_res_n) + kwargs = {"src_stream": preflight, "min_res_n": min_res_n, "out_folder": folder, "out_name": f"emb_{run_id}"} + if codes: + kwargs.update(residue_output="residue_codes", scoring_system=SimpleNamespace(system=SYSTEM)) + fake_chain_predict(ranks=ranks, **fake)(**kwargs) + return preflight, run_shard_files(folder, run_id) + + def make_record(self, run_id="r", fasta=_FASTA, *, ranks=1, codes=False, min_res_n=0, folder=None, **fake): + """Shards plus the record step 1 would write for them; returns ``(record, preflight, shard paths)``.""" + folder = folder or self.shards + preflight, paths = self.write_shards(run_id, fasta, ranks=ranks, codes=codes, min_res_n=min_res_n, + folder=folder, **fake) + sha256, _ = split_run.fasta_sha256(fasta) + stat = split_run.fasta_stat(fasta) + if codes: + run_pass = split_run.check_codes_footers(paths, system=SYSTEM, min_res_n=min_res_n, world_size=ranks) + else: + run_pass = pass_record(min_res_n=min_res_n, esm_revision="test-revision", world_size=ranks) + shards = split_run.shard_entries(paths) + record = { + "kind": split_run.RUN_RECORD_KIND, "record_version": split_run.RUN_RECORD_VERSION, "run_id": run_id, + "command": split_run.EMBED_COMMAND, "sealed": False, "created_utc": "2026-10-22T12:00:00Z", + "foldmatch_version": foldmatch.__version__, + "fasta": {"path": str(Path(fasta).resolve()), "bytes": stat.bytes, "mtime_ns": stat.mtime_ns, + "sha256": sha256}, + "min_res_n": min_res_n, "strict": codes, "preflight": split_run.preflight_counts(preflight), + "output_db": str(self.tmp / "db"), "residue_output": "residue_codes" if codes else None, + "system": _system_block(SYSTEM) if codes else None, "world_size": ranks, "pass": run_pass, + "options": {"accelerator": "cpu", "devices": ["1"], "num_nodes": 1, "strategy": "auto", "batch_size": 1, + "num_workers": 0}, + "shards": shards, "rows": sum(shard["rows"] for shard in shards), + } + return record, preflight, paths + + def refused(self, call, *texts): + """``call()`` raised a RefusalError containing every one of ``texts``; returns its message.""" + with self.assertRaises(RefusalError) as ctx: + call() + message = str(ctx.exception) + for text in texts: + self.assertIn(text, message) + return message + + def names(self, folder=None): + return sorted(p.name for p in (folder or self.shards).iterdir()) + + +# ===================================================================================================================== +# record I/O +# ===================================================================================================================== +class RunRecordIOTest(_Run): + + def test_round_trip(self): + record, _, _ = self.make_record(codes=True) + path = split_run.write_run_record(self.shards, record) + self.assertEqual(path, self.shards / "emb_r.run.json") + self.assertEqual(path, split_run.run_record_path(self.shards, "r")) + self.assertEqual(path.read_text(), json.dumps(record, indent=2, sort_keys=True) + "\n") + self.assertEqual(split_run.read_run_record(path), record) + self.assertEqual(self.names(), ["emb_r-0.parquet", "emb_r.run.json"]) + self.assertEqual(split_run.record_run_ids(self.shards), ["r"]) + self.assertEqual(split_run.tmp_record_run_ids(self.shards), []) + + def test_a_record_is_no_shard(self): + """The record is not taken for a shard of any run, nor for an embedding file of 'build embeddings'.""" + from foldmatch.search.residue_shards import shard_run_ids + record, _, _ = self.make_record() + split_run.write_run_record(self.shards, record) + self.assertEqual(shard_run_ids(self.shards), ["r"]) + self.assertEqual(run_shard_files(self.shards, "r"), [self.shards / "emb_r-0.parquet"]) + self.assertFalse(split_run.RUN_RECORD_SUFFIX.endswith((".pt", ".csv", ".parquet"))) + + def test_a_failure_leaves_no_file(self): + """The dump, the write, the fsync or the link raising: neither the record nor the .tmp is left, and the + refusal names the file.""" + record, _, _ = self.make_record() + before = self.names() + failures = { + "the dump": ({**record, "rows_nan": float("nan")}, None, "cannot be written as JSON"), + "the write": (record, mock.patch.object(split_run, "_write_all", + side_effect=OSError(errno.ENOSPC, "No space left on device")), + "emb_r.run.json"), + "the fsync": (record, mock.patch.object(split_run.os, "fsync", side_effect=OSError(errno.EIO, "I/O")), + "emb_r.run.json"), + "the link": (record, mock.patch.object(split_run.os, "link", side_effect=OSError(errno.EIO, "I/O")), + "emb_r.run.json"), + } + for case, (value, patch, text) in failures.items(): + with self.subTest(case=case): + with patch or nullcontext(): + self.refused(lambda: split_run.write_run_record(self.shards, value), text) + self.assertEqual(self.names(), before) + + def test_the_staged_record_is_checked_before_it_is_committed(self): + record, _, _ = self.make_record() + record["shards"][0]["bytes"] += 1 + self.refused(lambda: split_run.write_run_record(self.shards, record), "emb_r.run.json.tmp failed its check", + "changed since the run") + self.assertEqual(self.names(), ["emb_r-0.parquet"]) + record, _, _ = self.make_record("s") + record["rows"] += 1 + self.refused(lambda: split_run.write_run_record(self.shards, record), "failed its check", "rows is 3") + self.assertEqual(self.names(), ["emb_r-0.parquet", "emb_s-0.parquet"]) + + def test_an_existing_record_or_tmp_is_refused(self): + record, _, _ = self.make_record() + tmp = split_run.run_record_tmp_path(self.shards, "r") + tmp.write_text("another writer's") + self.refused(lambda: split_run.write_run_record(self.shards, record), str(tmp)) + self.assertEqual(tmp.read_text(), "another writer's") # not ours: left alone + self.refused(lambda: split_run.refuse_existing_record(self.shards, "r"), str(tmp), "same run id (r)", + "Delete the .tmp", "FOLDMATCH_RUN_ID") + message = self.refused(lambda: split_run.refuse_existing_record(self.shards, "r", + command=split_run.SEAL_COMMAND), + str(tmp), "Delete the .tmp, then run 'build sequences-seal' again.") + self.assertNotIn("FOLDMATCH_RUN_ID", message) + tmp.unlink() + final = split_run.write_run_record(self.shards, record) + text = final.read_text() + self.refused(lambda: split_run.write_run_record(self.shards, record), str(final)) + self.assertEqual((final.read_text(), self.names()), (text, ["emb_r-0.parquet", "emb_r.run.json"])) + self.refused(lambda: split_run.refuse_existing_record(self.shards, "r"), str(final), "same run id (r)", + "Remove it and its shards", "another --shards-folder", "FOLDMATCH_RUN_ID") + message = self.refused(lambda: split_run.refuse_existing_record(self.shards, "r", + command=split_run.SEAL_COMMAND), + str(final), "has its run record already, so there is nothing to seal", + f"Use it: 'build sequences-index --from-shards {self.shards} --run-id r'") + self.assertNotIn("Remove it", message) + split_run.refuse_existing_record(self.shards, "other") + split_run.refuse_existing_record(self.shards, "other", command=split_run.SEAL_COMMAND) + + def test_without_hard_links(self): + """``os.link`` unsupported (ENOTSUP): the .tmp is renamed into place (INFO), unless a record exists.""" + record, _, _ = self.make_record() + with mock.patch.object(split_run.os, "link", side_effect=OSError(errno.ENOTSUP, "Operation not supported")), \ + self.assertLogs("foldmatch.search.split_run", "INFO") as logs: + path = split_run.write_run_record(self.shards, record) + self.assertIn("does not support hard links", logs.output[0]) + self.assertEqual(split_run.read_run_record(path), record) + self.assertEqual(self.names(), ["emb_r-0.parquet", "emb_r.run.json"]) + + other, _, _ = self.make_record("s") + final = split_run.run_record_path(self.shards, "s") + + def appears(src, dst): # another writer commits between the check of the .tmp and the link + final.write_text("theirs") + raise OSError(errno.EOPNOTSUPP, "Operation not supported") + + with mock.patch.object(split_run.os, "link", side_effect=appears): + self.refused(lambda: split_run.write_run_record(self.shards, other), str(final)) + self.assertEqual(final.read_text(), "theirs") + self.assertFalse(split_run.run_record_tmp_path(self.shards, "s").exists()) + + def test_each_errno_of_a_missing_hard_link(self): + """EPERM (link(2) on a filesystem without hard links, as on Linux), ENOTSUP and EOPNOTSUPP (one errno on Linux, + two on macOS): each renames the .tmp into place, and the folder holds only the shard and the record.""" + for code in sorted({errno.EPERM, errno.ENOTSUP, errno.EOPNOTSUPP}): + folder = self.tmp / f"errno-{code}" + folder.mkdir() + with self.subTest(errno=errno.errorcode[code]): + record, _, _ = self.make_record(folder=folder) + with mock.patch.object(split_run.os, "link", side_effect=OSError(code, os.strerror(code))), \ + self.assertLogs("foldmatch.search.split_run", "INFO") as logs: + path = split_run.write_run_record(folder, record) + self.assertIn("does not support hard links", logs.output[0]) + self.assertEqual(split_run.read_run_record(path), record) + self.assertEqual(self.names(folder), ["emb_r-0.parquet", "emb_r.run.json"]) + + def test_discovery_ignores_names_that_are_no_run_id(self): + for name in ("emb_a.run.json", "emb_b.run.json.tmp", "emb_copy[1].run.json", "emb_-x.run.json", + "emb_c.run.json.bak", "emb_.run.json", "x_d.run.json"): + (self.shards / name).touch() + self.assertEqual(split_run.record_run_ids(self.shards), ["a"]) + self.assertEqual(split_run.tmp_record_run_ids(self.shards), ["b"]) + self.assertEqual(split_run.run_record_path(self.shards, "a.b"), self.shards / "emb_a.b.run.json") + self.refused(lambda: split_run.run_record_path(self.shards, "a b"), "is not a run id") + + +class ReadRunRecordTest(_Run): + """Every I7 refusal of ``read_run_record``, each naming the file and the key.""" + + def setUp(self): + super().setUp() + self.record, _, _ = self.make_record(ranks=2, codes=True) + self.path = self.shards / "emb_r.run.json" + + def read(self, value, path=None): + path = path or self.path + path.write_text(json.dumps(value) if not isinstance(value, str) else value) + return split_run.read_run_record(path) + + def bad(self, change, *texts): + record = json.loads(json.dumps(self.record)) + change(record) + return self.refused(lambda: self.read(record), str(self.path), *texts) + + def test_the_record_reads(self): + self.assertEqual(self.read(self.record), self.record) + self.assertEqual(self.read({**self.record, "written_by_a_later_version": [1]})["run_id"], "r") # ignored + + def test_unreadable_or_not_an_object(self): + self.refused(lambda: split_run.read_run_record(self.shards / "emb_none.run.json"), "cannot be read") + self.path.write_bytes(b"\xff\xfe") + self.refused(lambda: split_run.read_run_record(self.path), "cannot be read") + self.refused(lambda: self.read("{not json"), "not JSON") + self.refused(lambda: self.read([self.record]), "not a JSON object") + + def test_kind_and_version(self): + self.bad(lambda r: r.update(kind="foldmatch-other"), "kind is 'foldmatch-other'") + message = self.bad(lambda r: r.update(record_version=2), "newer FoldMatch", "record_version 2") + self.assertIn("upgrade FoldMatch", message) + for version in (0, "1", 1.0, True, None): + with self.subTest(version=version): + self.bad(lambda r: r.update(record_version=version), "record_version is") + + def test_every_key_is_required(self): + for key in split_run._RECORD_TYPES: + if key in ("kind", "record_version"): # test_kind_and_version + continue + with self.subTest(key=key): + self.bad(lambda r: r.pop(key), f"the key {key} is missing") + for block, keys in (("fasta", split_run.FASTA_KEYS), ("preflight", split_run.PREFLIGHT_KEYS), + ("system", split_run.SYSTEM_KEYS), ("pass", tuple(self.record["pass"]))): + for key in keys: + with self.subTest(block=block, key=key): + self.bad(lambda r: r[block].pop(key), f"the key {block}.{key} is missing") + for key in ("name", "rank", "rows", "bytes"): + with self.subTest(shard_key=key): + self.bad(lambda r: r["shards"][1].pop(key), f"the key shards[1].{key} is missing") + + def test_wrong_types(self): + cases = {"rows": "2", "sealed": 0, "world_size": True, "output_db": 3, "residue_output": 1, "system": [], + "options": None, "shards": {}, "min_res_n": 1.5, "created_utc": 0, "pass": "cpu"} + for key, value in cases.items(): + with self.subTest(key=key): + self.bad(lambda r: r.update({key: value}), f"{key} is {value!r}") + self.bad(lambda r: r["fasta"].update(bytes="303"), "fasta.bytes is '303', expected integer") + self.bad(lambda r: r["preflight"].update(entries=None), "preflight.entries is None") + self.bad(lambda r: r["pass"].update(world_size="2"), "pass.world_size is '2'") + self.bad(lambda r: r["pass"].update(accelerator=1), "pass.accelerator is 1, expected string or null") + self.bad(lambda r: r["shards"].__setitem__(0, "emb_r-0.parquet"), "shards[0] is 'emb_r-0.parquet'") + self.bad(lambda r: r["shards"][0].update(rows=False), "shards[0].rows is False") + + def test_negative_counts(self): + self.bad(lambda r: r["preflight"].update(residues=-1), "preflight.residues is negative") + self.bad(lambda r: r.update(min_res_n=-1), "min_res_n is negative") + self.bad(lambda r: r["fasta"].update(bytes=-5), "fasta.bytes is negative") + self.bad(lambda r: r["shards"][0].update(bytes=-1), "shards[0].bytes is negative") + self.bad(lambda r: r.update(world_size=0), "world_size is 0") + + def test_inconsistent_fields(self): + self.refused(lambda: self.read(self.record, self.shards / "emb_s.run.json"), "run_id is 'r'", + "named for run 's'") + self.refused(lambda: self.read(self.record, self.shards / "record.json"), "named for no run") + self.bad(lambda r: r.update(run_id="a b"), "run_id 'a b' is not a run id") + self.bad(lambda r: r.update(command="build sequences"), "command is 'build sequences'") + self.bad(lambda r: r.update(sealed=True), "sealed is True for the command 'build sequences-embed'") + self.bad(lambda r: r.update(command=split_run.SEAL_COMMAND), "sealed is False") + self.bad(lambda r: r.update(created_utc="2026-10-22 12:00:00"), "created_utc") + self.bad(lambda r: r["fasta"].update(sha256="ABC"), "fasta.sha256 'ABC' is not a sha256") + self.bad(lambda r: r.update(residue_output="query_encoding"), "residue_output is 'query_encoding'") + self.bad(lambda r: r.update(strict=False), "strict is False, but residue_output is 'residue_codes'") + self.bad(lambda r: r.update(system=None), "system is null") + self.bad(lambda r: r.update(residue_output=None, strict=False), "system is set, but residue_output is None") + self.bad(lambda r: r["pass"].update(min_res_n=100), "pass.min_res_n is 100, but min_res_n is 0") + self.bad(lambda r: r["pass"].update(world_size=1), "pass.world_size is 1, but world_size is 2") + self.bad(lambda r: r["preflight"].update(unique_entries=1), "preflight.unique_entries is 1") + self.bad(lambda r: r["preflight"].update(entries=1), "entries_kept (2) exceeds preflight.entries (1)") + + def test_rows_and_the_deal(self): + self.bad(lambda r: r.update(rows=3), "rows is 3, but the shards hold 2") + self.bad(lambda r: (r.update(rows=3), r["shards"][0].update(rows=2)), "rows is 3, but preflight") + self.bad(lambda r: r["shards"][1].update(name="emb_r-2.parquet"), "shards[1] is named 'emb_r-2.parquet'") + self.bad(lambda r: r["shards"].reverse(), "in file-name order") + self.bad(lambda r: r["shards"].__setitem__(1, dict(r["shards"][0])), "listed once each") + # Two rows on rank 0 and none on rank 1: the right total, not the deal of 2 over 2. + message = self.bad(lambda r: (r["shards"].pop(), r["shards"][0].update(rows=2)), "not the deal of 2 entries " + "over 2 rank(s)") + self.assertIn("rank 0: 2 row(s)", message) + self.assertIn("rank 1: 1 row(s)", message) + # A world of 3 for 2 entries: ranks 0 and 1, one row each (rank 2 writes no shard). + record = json.loads(json.dumps(self.record)) + record.update(world_size=3) + record["pass"].update(world_size=3) + self.assertEqual(self.read(record)["world_size"], 3) + record.update(world_size=1) + record["pass"].update(world_size=1) + self.refused(lambda: self.read(record), "not the deal of 2 entries over 1 rank(s)") + + +# ===================================================================================================================== +# selection and checks +# ===================================================================================================================== +class FindRunTest(_Run): + + def commit(self, run_id, **kwargs): + record, _, _ = self.make_record(run_id, **kwargs) + return split_run.write_run_record(self.shards, record) + + def test_no_run(self): + self.refused(lambda: split_run.find_run(self.shards), "No run in", "--shards-folder") + self.refused(lambda: split_run.find_run(self.shards, "r"), "Run r has no run record", + "neither a run record nor embedding shards", "Runs with a record here: none") + + def test_the_only_run(self): + path = self.commit("r") + self.assertEqual(split_run.find_run(self.shards), ("r", path)) + self.assertEqual(split_run.find_run(self.shards, "r"), ("r", path)) + + def test_shards_without_a_record(self): + self.write_shards("r", ranks=2) + message = self.refused(lambda: split_run.find_run(self.shards), "Run r", "2 shard(s), no run record", + "did not finish", "kept by 'build sequences'", "build sequences-seal", + f"--from-shards {self.shards} --run-id r", "build sequences-embed' again") + self.assertNotIn(".tmp", message) + self.refused(lambda: split_run.find_run(self.shards, "r"), "Run r has no run record", "2 shard(s)", + "build sequences-seal", "Runs with a record here: none") + + def test_a_tmp_only(self): + split_run.run_record_tmp_path(self.shards, "r").write_text("{}") + self.refused(lambda: split_run.find_run(self.shards), "Run r", "unfinished run record (.tmp)", + f"delete {split_run.run_record_tmp_path(self.shards, 'r')} first") + self.refused(lambda: split_run.find_run(self.shards, "r"), "Delete", "emb_r.run.json.tmp") + + def test_a_complete_run_beside_an_unfinished_one(self): + """D33: two runs and no --run-id is refused, even when only one of them has a record.""" + path = self.commit("done") + self.write_shards("half") + self.refused(lambda: split_run.find_run(self.shards), "holds 2 runs", "done (run record, 1 shard(s))", + "half (1 shard(s), no run record)", "Pass --run-id") + self.assertEqual(split_run.find_run(self.shards, "done"), ("done", path)) + self.refused(lambda: split_run.find_run(self.shards, "half"), "Run half has no run record", + "build sequences-seal", "Runs with a record here: done") + + def test_several_runs(self): + paths = {rid: self.commit(rid) for rid in ("a", "b")} + split_run.run_record_tmp_path(self.shards, "c").write_text("{}") + self.refused(lambda: split_run.find_run(self.shards), "holds 3 runs", "a (run record, 1 shard(s))", + "b (run record, 1 shard(s))", "c (0 shard(s), an unfinished run record (.tmp))") + for rid, path in paths.items(): + self.assertEqual(split_run.find_run(self.shards, rid), (rid, path)) + self.refused(lambda: split_run.find_run(self.shards, "z"), "Run z has no run record", + "neither a run record nor embedding shards", "Runs with a record here: a, b") + + def test_names_that_are_no_run_id_are_ignored(self): + path = self.commit("r") + shutil.copyfile(path, self.shards / "emb_copy[1].run.json") + shutil.copyfile(self.shards / "emb_r-0.parquet", self.shards / "emb_copy[1]-0.parquet") + (self.shards / "notes.run.json").touch() + self.assertEqual(split_run.find_run(self.shards), ("r", path)) + + +class CheckRunShardsTest(_Run): + + def setUp(self): + super().setUp() + self.record, _, self.paths = self.make_record(ranks=2, codes=True) + + def test_the_recorded_shards(self): + self.assertEqual(split_run.check_run_shards(self.shards, self.record), self.paths) + copy = self.tmp / "copy" + shutil.copytree(self.shards, copy) + self.assertEqual(split_run.check_run_shards(copy, self.record), run_shard_files(copy, "r")) + + def test_a_missing_or_an_extra_shard(self): + self.paths[1].unlink() + self.refused(lambda: split_run.check_run_shards(self.shards, self.record), "missing emb_r-1.parquet", + "changed since the run") + shutil.copyfile(self.paths[0], self.paths[1]) + shutil.copyfile(self.paths[0], self.shards / "emb_r-2.parquet") + self.refused(lambda: split_run.check_run_shards(self.shards, self.record), + "not in the record emb_r-2.parquet") + + def test_a_changed_shard(self): + self.write_shards("x", _write(self.tmp / "other.fasta", ">o\nACDEFGHIKLMNPQRSTVWY\n"), codes=True, + folder=self.tmp) + shutil.copyfile(self.tmp / "emb_x-0.parquet", self.paths[1]) # same name, one row, other bytes + self.refused(lambda: split_run.check_run_shards(self.shards, self.record), "1 shard(s) of run r", + "changed since the run", "emb_r-1.parquet (") + self.record["shards"][1].update(rows=1, bytes=self.paths[1].stat().st_size) + self.assertEqual(split_run.check_run_shards(self.shards, self.record), self.paths) + self.record["shards"][1].update(rows=2) + self.refused(lambda: split_run.check_run_shards(self.shards, self.record), "1 rows; recorded", + "2 rows") + self.paths[0].write_bytes(b"x" * self.record["shards"][0]["bytes"]) # same size, no footer + self.refused(lambda: split_run.check_run_shards(self.shards, self.record), "emb_r-0.parquet", + "cannot be read") + + +class CheckDealtIdsTest(_Run): + + def rewrite(self, path, ids): + """``path`` with its rows' ids replaced by ``ids`` (rows and schema kept).""" + import pyarrow as pa + table = pq.read_table(path) + table = table.set_column(table.schema.get_field_index("id"), "id", pa.array(ids, pa.string())) + pq.write_table(table, path) + + def test_the_deal(self): + fasta = _write(self.tmp / "five.fasta", _FIVE) + for ranks in (1, 2, 3, 5, 7): + with self.subTest(ranks=ranks): + record, preflight, paths = self.make_record(f"w{ranks}", fasta, ranks=ranks) + self.assertEqual(len(paths), min(ranks, 5)) + self.assertIsInstance(split_run.check_dealt_ids(paths, record, preflight), float) + + def test_a_swapped_pair(self): + record, preflight, paths = self.make_record() + self.rewrite(paths[0], ["2uzi_A", "1acb_E"]) + message = self.refused(lambda: split_run.check_dealt_ids(paths, record, preflight), + "2 shard row(s) of run r", "emb_r-0.parquet row 0 holds '2uzi_A', entry 0 is '1acb_E'", + "emb_r-0.parquet row 1 holds '1acb_E', entry 1 is '2uzi_A'") + self.assertIn(str(_FASTA), message) + + def test_a_repeated_id_at_the_wrong_position(self): + """A plain FASTA may repeat an id: the multiset of ids is right, the positions are not.""" + fasta = _write(self.tmp / "dup.fasta", ">x\nACDEFGHIKL\n>y\nKLMNPQRSTV\n>x\nVWYACDEFGH\n>z\nIKLMNPQRST\n") + record, preflight, paths = self.make_record("d", fasta, ranks=2) + split_run.check_dealt_ids(paths, record, preflight) + # rank 0 holds entries 0 and 2 (x, x), rank 1 entries 1 and 3 (y, z); move one x to rank 1. + self.rewrite(paths[0], ["x", "y"]) + self.rewrite(paths[1], ["x", "z"]) + self.refused(lambda: split_run.check_dealt_ids(paths, record, preflight), "2 shard row(s)", + "emb_d-0.parquet row 1 holds 'y', entry 2 is 'x'", + "emb_d-1.parquet row 0 holds 'x', entry 1 is 'y'") + + def test_a_world_larger_than_the_input(self): + record, preflight, paths = self.make_record(ranks=3) + self.assertEqual([p.name for p in paths], ["emb_r-0.parquet", "emb_r-1.parquet"]) + split_run.check_dealt_ids(paths, record, preflight) + shutil.copyfile(paths[1], self.shards / "emb_r-2.parquet") + self.refused(lambda: split_run.check_dealt_ids(run_shard_files(self.shards, "r"), record, preflight), + "not the deal of the 2 entries", "rank 2: 1 row(s)") + self.refused(lambda: split_run.check_dealt_ids(paths[:1], record, preflight), "rank 0: 1 row(s)) are not") + + def test_the_sequences_of_codes_shards(self): + """``sequences``: every codes row's seq_sha1 against its entry's sequence, in the same pass. A FASTA with the + same ids, order and lengths but another residue is refused (the ids still pass); a mismatched id is reported + as such, before any seq_sha1; plain shards have no seq_sha1 to check.""" + fasta = _write(self.tmp / "five.fasta", _FIVE) + record, preflight, paths = self.make_record("c", fasta, ranks=2, codes=True) + self.assertIsInstance(split_run.check_dealt_ids(paths, record, preflight, sequences=True), float) + other = _write(self.tmp / "other.fasta", _FIVE.replace("VWYACDEFGH", "VWYACDEFGW").replace("IKLMNPQRST", + "IKLMNPQRSW")) + moved = preflight_fasta(other, 0) + split_run.check_dealt_ids(paths, record, moved) # the ids alone agree + message = self.refused(lambda: split_run.check_dealt_ids(paths, record, moved, sequences=True), + "2 shard row(s) of run c hold a seq_sha1 that is not the sha1 of the sequence of the " + f"entry of {other} the pass dealt them, though they hold its id: emb_c-0.parquet row 1 " + "('e2', entry 2); emb_c-1.parquet row 1 ('e3', entry 3). These shards were not computed " + "from this FASTA's sequences.") + self.assertNotIn("do not hold the id", message) + self.rewrite(paths[0], ["e0", "e1", "e4"]) + self.refused(lambda: split_run.check_dealt_ids(paths, record, preflight, sequences=True), + "1 shard row(s) of run c do not hold the id", "emb_c-0.parquet row 1 holds 'e1', entry 2 is 'e2'") + (self.tmp / "plain").mkdir() + plain, plain_preflight, plain_paths = self.make_record("p", fasta, folder=self.tmp / "plain") + self.refused(lambda: split_run.check_dealt_ids(plain_paths, plain, plain_preflight, sequences=True), + "the ids and seq_sha1 of a shard of run p cannot be read") + + def test_many_offsets(self): + """More entries than one offset chunk (the chunk patched down to 2).""" + fasta = _write(self.tmp / "five.fasta", _FIVE) + record, preflight, paths = self.make_record("m", fasta, ranks=2) + with mock.patch.object(split_run, "_OFFSET_CHUNK", 2): + split_run.check_dealt_ids(paths, record, preflight) + self.rewrite(paths[1], ["e1", "e4"]) + self.refused(lambda: split_run.check_dealt_ids(paths, record, preflight), + "emb_m-1.parquet row 1 holds 'e4', entry 3 is 'e3'") + + +class FooterTest(_Run): + + def test_plain_footers(self): + _, paths = self.write_shards(ranks=2) + split_run.check_plain_footers(paths) + _, codes = self.write_shards("c", codes=True) + message = self.refused(lambda: split_run.check_plain_footers(paths + codes), "1 of the 3 shard(s)", + "not plain embedding shards", "emb_c-0.parquet") + self.assertIn("foldmatch.residue_output", message) + table = pq.read_table(paths[0]) + pq.write_table(table.replace_schema_metadata({b"foldmatch.pass": b"{}"}), paths[0]) + self.refused(lambda: split_run.check_plain_footers(paths), "emb_r-0.parquet", "foldmatch.pass") + + def test_codes_footers_against_the_system(self): + five = _write(self.tmp / "five.fasta", _FIVE) + _, paths = self.write_shards(fasta=five, ranks=2, codes=True, min_res_n=10, device="cuda") + self.assertEqual(len(paths), 2) + run_pass = split_run.check_codes_footers(paths, system=SYSTEM, min_res_n=10, world_size=2) + self.assertEqual((run_pass["min_res_n"], run_pass["world_size"], run_pass["accelerator"]), (10, 2, "cuda")) + self.refused(lambda: split_run.check_codes_footers(paths, system=SYSTEM, min_res_n=0, world_size=2), + "2 of the 2 shard(s) do not record a pass with --min-res-n 0 on 2 rank(s)", + "emb_r-0.parquet (min_res_n 10, world_size 2)") + self.refused(lambda: split_run.check_codes_footers(paths, system=SYSTEM, min_res_n=10, world_size=3), + "on 3 rank(s)") + _, plain = self.write_shards("p") + self.refused(lambda: split_run.check_codes_footers(paths + plain, system=SYSTEM, min_res_n=10, + world_size=2), "mixed embedding shards") + _, stale = self.write_shards("o", codes=True, system=OTHER_ENCODING) + self.refused(lambda: split_run.check_codes_footers(stale, system=SYSTEM, min_res_n=0, world_size=1), + "stale embedding shards") + table = pq.read_table(paths[1]) + meta = {k: v for k, v in table.schema.metadata.items() if k != META_PASS} + pq.write_table(table.replace_schema_metadata(meta), paths[1]) + self.refused(lambda: split_run.check_codes_footers(paths, system=SYSTEM, min_res_n=10, world_size=2), + "1 of the 2 shard(s)", "emb_r-1.parquet (no pass record)") + self.refused(lambda: split_run.check_codes_footers([], system=SYSTEM, min_res_n=0, world_size=1), + "no shards") + + def test_codes_footers_against_a_record(self): + """``expect`` (a record's system block): self-consistency, also for a system this FoldMatch does not know.""" + _, paths = self.write_shards(codes=True) + split_run.check_codes_footers(paths, expect=_system_block(SYSTEM), min_res_n=0, world_size=1) + _, other = self.write_shards("o", codes=True, system=OTHER_ENCODING) + run_pass = split_run.check_codes_footers(other, expect=_system_block(OTHER_ENCODING), min_res_n=0, + world_size=1) + self.assertEqual(run_pass["world_size"], 1) + message = self.refused(lambda: split_run.check_codes_footers(other, expect=_system_block(SYSTEM), min_res_n=0, + world_size=1), + f"residue codes of {OTHER_ENCODING.name}", f"not those of {SYSTEM.name}") + self.assertIn(SYSTEM.encoding_digest(), message) + self.assertIn(OTHER_ENCODING.encoding_digest(), message) + self.refused(lambda: split_run.check_codes_footers(paths + other, expect=_system_block(SYSTEM), min_res_n=0, + world_size=1), "Mixed embedding shards") + _, plain = self.write_shards("p") + self.refused(lambda: split_run.check_codes_footers(plain, expect=_system_block(SYSTEM), min_res_n=0, + world_size=1), "no residue column", + f"not the residue codes of {SYSTEM.name}") + with self.assertRaises(ValueError): + split_run.check_codes_footers(paths, min_res_n=0, world_size=1) + with self.assertRaises(ValueError): + split_run.check_codes_footers(paths, system=SYSTEM, expect=_system_block(SYSTEM), min_res_n=0, + world_size=1) + + +# ===================================================================================================================== +# FASTA and shard facts +# ===================================================================================================================== +class FactsTest(_Run): + + def test_fasta_sha256(self): + reads = [] + real_open = builtins.open + + class Spy: + def __init__(self, f): + self.f = f + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.f.close() + + def read(self, size=-1): + reads.append(size) + return self.f.read(size) + + data = os.urandom(3 * 1000 + 7) + path = self.tmp / "data.fasta" + path.write_bytes(data) + with mock.patch.object(split_run, "open", create=True, side_effect=lambda *a, **k: Spy(real_open(*a, **k))): + digest, seconds = split_run.fasta_sha256(path) + self.assertEqual(digest, hashlib.sha256(data).hexdigest()) + self.assertGreaterEqual(seconds, 0.0) + self.assertEqual(reads, [16 << 20, 16 << 20]) # one read of the file, one at its end + reads.clear() + with mock.patch.object(split_run, "HASH_CHUNK", 1000), \ + mock.patch.object(split_run, "open", create=True, side_effect=lambda *a, **k: Spy(real_open(*a, **k))): + self.assertEqual(split_run.fasta_sha256(path)[0], hashlib.sha256(data).hexdigest()) + self.assertEqual(reads, [1000] * 5) + + def test_fasta_stat(self): + path = _write(self.tmp / "a.fasta", ">a\nACDE\n") + st = os.stat(path) + self.assertEqual(split_run.fasta_stat(path), split_run.FastaStat(st.st_size, st.st_mtime_ns, st.st_ino)) + self.assertEqual(split_run.fasta_stat(path).bytes, 8) + with open(path, "a") as f: + f.write(">b\nKLMN\n") + self.assertEqual(split_run.fasta_stat(path).bytes, 16) + with self.assertRaises(OSError): + split_run.fasta_stat(self.tmp / "missing.fasta") + + def test_preflight_counts(self): + preflight = preflight_fasta(_write(self.tmp / "d.fasta", ">a\nACDX\n>a\nACDE\n>b\nKL\n"), 3) + counts = split_run.preflight_counts(preflight) + self.assertEqual(tuple(counts), split_run.PREFLIGHT_KEYS) + self.assertEqual(counts, {"entries": 3, "entries_kept": 2, "residues": 8, "duplicates": 1, + "unique_entries": 1, "unique_residues": 4, "nonstandard_entries": 1, + "nonstandard_residues": 1, "non_ascii_entries": 0, "angle_bracket_entries": 0, + "max_length": 4}) + + def test_dealt_rows(self): + self.assertEqual(split_run.dealt_rows(5, 2), {0: 3, 1: 2}) + self.assertEqual(split_run.dealt_rows(2, 3), {0: 1, 1: 1}) + self.assertEqual(split_run.dealt_rows(4, 4), {0: 1, 1: 1, 2: 1, 3: 1}) + self.assertEqual(split_run.dealt_rows(0, 4), {}) + self.assertEqual(split_run.dealt_rows(7, 1), {0: 7}) + + def test_shard_entries(self): + fasta = _write(self.tmp / "five.fasta", _FIVE) + _, paths = self.write_shards("r-1", fasta, ranks=3) + self.assertEqual(split_run.shard_entries(paths), [ + {"name": f"emb_r-1-{rank}.parquet", "rank": rank, "rows": rows, "bytes": paths[rank].stat().st_size} + for rank, rows in ((0, 2), (1, 2), (2, 1))]) + bad = self.tmp / "emb_r-x.parquet" + bad.touch() + self.refused(lambda: split_run.shard_entries([bad]), "is not an embedding shard") + (self.tmp / "emb_r-0.parquet").write_bytes(b"") + self.refused(lambda: split_run.shard_entries([self.tmp / "emb_r-0.parquet"]), "not a readable Parquet") + + +# ===================================================================================================================== +# the build lock +# ===================================================================================================================== +class BuildLockTest(_Run): + + def setUp(self): + super().setUp() + self.db = str(self.tmp / "db") + self.record = {"run_id": "r", "residue_output": "residue_codes"} + + def enter(self, record=None, **kwargs): + """Take the lock (a refusal is raised here; a lock taken is released at cleanup).""" + self.enterContext(split_run.build_lock(self.db, record or self.record, **kwargs)) + + def test_content_and_removal(self): + lock = Path(f"{self.db}.build.lock") + with launch_env(SLURM_JOB_ID=4242), split_run.build_lock(self.db, self.record) as held: + self.assertEqual(held, lock) + content = json.loads(lock.read_text()) + self.assertEqual(set(content), {"run_id", "host", "pid", "slurm_job_id", "started_utc", "lock_id"}) + self.assertRegex(content["lock_id"], r"^[0-9a-f]{32}$") + self.assertEqual((content["run_id"], content["host"], content["pid"], content["slurm_job_id"]), + ("r", socket.gethostname(), os.getpid(), "4242")) + self.assertRegex(content["started_utc"], r"^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$") + self.assertFalse(lock.exists()) + with split_run.build_lock(self.db, self.record): + self.assertIsNone(json.loads(lock.read_text())["slurm_job_id"]) + + def test_removed_on_an_exception(self): + lock = Path(f"{self.db}.build.lock") + for exc in (RefusalError("refused"), RuntimeError("bug"), KeyboardInterrupt()): + with self.subTest(exc=type(exc).__name__), self.assertRaises(type(exc)): + with split_run.build_lock(self.db, self.record): + self.assertTrue(lock.exists()) + raise exc + self.assertFalse(lock.exists()) + + def test_exclusive(self): + (self.tmp / "db.index").touch() + (self.tmp / "db.rescodes.bin.tmp").touch() + with split_run.build_lock(self.db, self.record): + message = self.refused(self.enter, + f"{self.db}.build.lock exists", '"run_id": "r"', f'"pid": {os.getpid()}', + "is running, or was killed", "files present: db.index, db.rescodes.bin.tmp", + "delete the lock and these files, then rerun") + self.assertNotIn("build residue-codes", message) + self.assertTrue(Path(f"{self.db}.build.lock").exists()) + self.assertFalse(Path(f"{self.db}.build.lock").exists()) + + def test_a_lock_that_is_no_longer_ours_stays(self): + """A lock deleted during the build, and maybe created again by another build, is not removed at the end. The + other build's lock can be a new file on the inode of the deleted one (Linux reuses it at once); "in place" is + that case on every system: another build's lock, another lock_id, on the same inode.""" + lock = Path(f"{self.db}.build.lock") + another = json.dumps({"run_id": "other", "lock_id": "0" * 32}) + for case in ("deleted", "deleted and created again", "in place", "in place, not JSON"): + with self.subTest(case=case): + with split_run.build_lock(self.db, self.record): + if case.startswith("deleted"): + lock.unlink() + if case == "deleted and created again": + lock.write_text(another) + else: + with open(lock, "r+") as f: + f.truncate() + f.write(another if case == "in place" else "another build's") + if case == "deleted": + self.assertFalse(lock.exists()) + else: + self.assertEqual(lock.read_text(), another if "JSON" not in case else "another build's") + lock.unlink() + + def test_the_hint_when_only_the_code_store_is_missing(self): + lock = Path(f"{self.db}.build.lock") + lock.write_text("{}") + for suffix in (".index", ".metadata"): + Path(f"{self.db}{suffix}").touch() + store = SequenceStore(self.tmp, "db") + store.create(_FASTA) + Path(f"{self.db}.rescodes.bin.tmp").touch() + message = self.refused(lambda: self.enter(shards_folder=self.shards), + "files present: db.index, db.metadata, db.sequences, db.rescodes.bin.tmp", + f"'fm-search build residue-codes {self.tmp} db --from-shards {self.shards} " + f"--run-id r'") + self.assertIn("complete but for its residue-code store", message) + self.assertNotIn("build residue-codes", + self.refused(lambda: self.enter(record={"run_id": "r", "residue_output": None}))) + Path(f"{self.db}.rescodes.json").touch() + self.assertNotIn("build residue-codes", self.refused(self.enter)) + Path(f"{self.db}.rescodes.json").unlink() + con = sqlite3.connect(store.path) + con.execute("DELETE FROM meta") # a store whose build stopped before its totals + con.commit() + con.close() + self.assertNotIn("build residue-codes", self.refused(self.enter)) + self.assertEqual(lock.read_text(), "{}") + + +class TorchFreeTest(unittest.TestCase): + + def test_importing_split_run_loads_no_torch_or_faiss(self): + """The module is pyarrow, numpy and the standard library only (plan P8b, Module API): a fresh interpreter that + imports it has loaded neither torch nor FAISS. (This test process has both loaded already.)""" + src = str(Path(split_run.__file__).resolve().parents[2]) + env = {**os.environ, "PYTHONPATH": os.pathsep.join([src] + [p for p in [os.environ.get("PYTHONPATH")] if p])} + code = ("import sys, foldmatch.search.split_run; " + "print(sorted(m for m in ('torch', 'faiss') if m in sys.modules))") + with tempfile.TemporaryDirectory() as cwd: + run = subprocess.run([sys.executable, "-c", code], env=env, cwd=cwd, capture_output=True, text=True, + timeout=300) + self.assertEqual(run.returncode, 0, run.stderr[-4000:]) + self.assertEqual(run.stdout.strip(), "[]", run.stderr[-4000:]) + + +# ===================================================================================================================== +# utils.dist.multi_rank_launch +# ===================================================================================================================== +class MultiRankLaunchTest(unittest.TestCase): + + def test_launch_environments(self): + cases = [ + ({}, None), + ({"SLURM_JOB_ID": 7, "SLURM_PROCID": 0, "SLURM_NTASKS": 32}, None), # the batch step of a 32-task job + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 4294967294, "SLURM_PROCID": 0, "SLURM_NTASKS": 32}, None), + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 0, "SLURM_PROCID": 0, "SLURM_STEP_NUM_TASKS": 2, "SLURM_NTASKS": 2}, + "SLURM_STEP_NUM_TASKS=2"), + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 0, "SLURM_PROCID": 1, "SLURM_STEP_NUM_TASKS": 2, "SLURM_NTASKS": 2}, + "SLURM_PROCID=1"), + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 3, "SLURM_PROCID": 0, "SLURM_STEP_NUM_TASKS": 1, + "SLURM_NTASKS": 32}, None), # srun -n 1 in a 32-task job + ({"SLURM_JOB_ID": 7, "SLURM_STEP_ID": 3, "SLURM_PROCID": 0, "SLURM_NTASKS": 4}, "SLURM_NTASKS=4"), + ({"WORLD_SIZE": 2}, "WORLD_SIZE=2"), + ({"WORLD_SIZE": 1, "RANK": 0, "LOCAL_RANK": 0}, None), + ({"RANK": 3, "WORLD_SIZE": 4}, "RANK=3"), + ({"LOCAL_RANK": 1, "FOLDMATCH_RUN_ID": "r"}, "LOCAL_RANK=1"), + ({"NODE_RANK": 2}, "NODE_RANK=2"), + ({"WORLD_SIZE": "x", "RANK": "", "SLURM_NTASKS": "2"}, None), # not whole numbers; no srun step + ] + for env, expected in cases: + with self.subTest(env=env), launch_env(**env): + self.assertEqual(multi_rank_launch(), expected) + # The rule of bundle_fetch_allowed: one process exactly where a single-node run may fetch. + self.assertEqual(bundle_fetch_allowed(1), expected is None) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_structure_lists.py b/tests/test_structure_lists.py new file mode 100644 index 0000000..d6c049d --- /dev/null +++ b/tests/test_structure_lists.py @@ -0,0 +1,862 @@ +"""P9.A6: a structure list row of more than 3 fields and a residue file list row of more than 2 are refused. + +A structure list is CSV without a header, of structure name, structure file or URL and embedding name, read by +read_structure_list for EsmProtFromStructure, EsmAssemblyFromStructure, ResidueEmbeddingFromStructure and +ResidueAssemblyDatasetFromStructure. A residue file list is CSV without a header, of residue embedding file and +embedding name, read by read_residue_file_list for ResidueEmbeddingFromTensorFile. These lists are given in a file +through the library only; the commands give the readers streams. A row of more fields is refused (RefusalError), from a +file and from a stream, naming the file or the stream and the row, as read_chain_list and read_assembly_list refuse one, +and so by every entry point before any model is built. Before, pandas read the leading fields of a first such row as the +index, which shifted the columns of every row, stopped at a later one with a ParserError that named no file, and refused +a stream row of one element too many with its ValueError; a list whose every row began with an index column worked. +Lists of at most that many fields per row are read as before, but for the residue file list's strings (D3): it was the +only list read without dtype=str, and without it pandas 3 reads an index column 0, 1, 2 as a RangeIndex, which no check +can tell from no index, and the names 007 and 1.50 as 7.0 and 1.5. + +ESM3 and the aggregator are never loaded: the entry points run with stub models (_esm3, _aggregate, as in +test_assembly_lists.py) or with model getters that fail if called.""" +import errno +import gzip +import io +import json +import logging +import os +import re +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +import torch +from esm.sdk.api import ESMProtein +from esm.utils.structure.protein_chain import ProteinChain +from typer.testing import CliRunner + +from foldmatch.cli import embedding +from foldmatch.dataset import resdiue_assembly_embedding_from_structure, residue_embedding_from_structure +from foldmatch.dataset.esm_assembly_from_structure import EsmAssemblyFromStructure +from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure +from foldmatch.dataset.resdiue_assembly_embedding_from_structure import ResidueAssemblyDatasetFromStructure +from foldmatch.dataset.residue_embedding_from_structure import ResidueEmbeddingFromStructure +from foldmatch.dataset.residue_embedding_from_tensor_file import ResidueEmbeddingFromTensorFile +from foldmatch.errors import RefusalError +# Imported before any test patches their model getters +from foldmatch.inference import assembly_complete_inference, assembly_inferece, chain_complete_inference, \ + chain_inference, esm_inference, structure_inference +from foldmatch.types.api_types import Accelerator, OutFormat, SrcAssemblyFrom, SrcEsmFrom, SrcLocation, \ + SrcProteinFrom, SrcTensorFrom, StructureFormat +from foldmatch.utils.data import collate_seq_embeddings, scan_structure_folder +from foldmatch.utils.structure_parser import filter_residues, get_assemblies, get_structure_from_src, \ + protein_chain_iter, rename_atom_attr + +_RESOURCES = Path(__file__).parent / "resources" +_PDB = _RESOURCES / "pdb" +_CIF, _CIF2 = _PDB / "1acb.cif", _PDB / "2uzi.cif" +_EMBEDDINGS = _RESOURCES / "embeddings" +_CPU = {"accelerator": Accelerator.cpu, "devices": 1} +_STRUCTURE_COLUMNS = ["stream_name", "stream", "item_name"] +_RESIDUE_FILE_COLUMNS = ["file", "item_name"] +_STRUCTURE_LIST, _RESIDUE_FILE_LIST = "structure list", "residue file list" + +# What a refusal of a row of too many fields says after what it found, per list +_RULE = { + _STRUCTURE_LIST: ("A structure list row holds at most 3 fields (structure name, structure file or URL, embedding " + "name)"), + _RESIDUE_FILE_LIST: "A residue file list row holds at most 2 fields (residue embedding file, embedding name)", +} +_REMOVE_THE_EXTRA_FIELDS = { + _STRUCTURE_LIST: "remove the extra fields (a comma after the embedding name makes a 4th).", + _RESIDUE_FILE_LIST: "remove the extra fields (a comma after the embedding name makes a 3rd).", +} + + +def _first_row(fields: int) -> str: + return re.escape(f"its first row has {fields} fields") + + +def _later_row(columns: int, expected: int, saw: int) -> str: + return rf"a row has more than {columns} fields \(pandas: [^()]*Expected {expected} fields in line 2, saw {saw}\)" + + +# (case, list, what the refusal found, after the file's path: a regular expression) +_LISTS = { + _STRUCTURE_LIST: ( + ("first row, a trailing comma", "a,f.cif,a-n,\nb,g.cif,b-n\n", _first_row(4)), + ("first row, a 4th field of text", "a,f.cif,a-n,extra\nb,g.cif,b-n\n", _first_row(4)), + ("first row of 5 fields", "a,f.cif,a-n,,extra\nb,g.cif,b-n\n", _first_row(5)), + ("an index column", "0,a,f.cif,a-n\n1,b,g.cif,b-n\n2,c,h.cif,c-n\n", _first_row(4)), + ("two index columns", "0,0,a,f.cif,a-n\n0,1,b,g.cif,b-n\n", _first_row(5)), + ("a later row of 4 fields", "a,f.cif,a-n\nb,g.cif,b-n,\n", _later_row(3, 3, 4)), + ("a first row of 4, a later row of 5", "a,f.cif,a-n,\nb,g.cif,b-n,,\n", _later_row(3, 4, 5)), + ("a chain list", "a,f.cif,A,a.A\nb,g.cif,B,b.B\n", _first_row(4)), + ), + _RESIDUE_FILE_LIST: ( + ("first row, a trailing comma", "a.pt,a-n,\nb.pt,b-n\n", _first_row(3)), + ("first row, a 3rd field of text", "a.pt,a-n,extra\nb.pt,b-n\n", _first_row(3)), + ("first row of 4 fields", "a.pt,a-n,,extra\nb.pt,b-n\n", _first_row(4)), + # Without dtype=str, pandas 3 reads these two as a RangeIndex, as it reads no index at all: the trap of D3 + ("an index column", "0,a.pt,a-n\n1,b.pt,b-n\n2,c.pt,c-n\n", _first_row(3)), + ("an index column of other integers", "5,a.pt,a-n\n7,b.pt,b-n\n", _first_row(3)), + ("two index columns", "0,0,a.pt,a-n\n0,1,b.pt,b-n\n", _first_row(4)), + ("a later row of 3 fields", "a.pt,a-n\nb.pt,b-n,\n", _later_row(2, 2, 3)), + ("a first row of 3, a later row of 4", "a.pt,a-n,\nb.pt,b-n,,\n", _later_row(2, 3, 4)), + ("a structure list", "a,f.cif,a-n\nb,g.cif,b-n\n", _first_row(3)), + ("a chain list", "a,f.cif,A,a.A\nb,g.cif,B,b.B\n", _first_row(4)), + ), +} +_READERS = { + _STRUCTURE_LIST: ("read_structure_list", "EsmProtFromStructure", "EsmAssemblyFromStructure", + "ResidueEmbeddingFromStructure", "ResidueAssemblyDatasetFromStructure"), + _RESIDUE_FILE_LIST: ("read_residue_file_list", "ResidueEmbeddingFromTensorFile"), +} +# A row of one element too many, and a list whose rows are longer still +_STREAMS = { + _STRUCTURE_LIST: ( + ([("a", "f.cif", "a-n"), ("b", "g.cif", "b-n", "")], "1 row(s) of more than 3 elements (row(s) 2"), + ([("a", "f.cif", "a-n"), ("b", "g.cif", "b-n", "", "x")], "1 row(s) of more than 3 elements (row(s) 2"), + ([("w", "f.cif", "w-n")] + [("x", "g.cif", "x-n", "z")] * 5 + [("y",) * 5] * 2, + "7 row(s) of more than 3 elements (row(s) 2, 3, 4, 5, 6 and 2 more"), + ), + _RESIDUE_FILE_LIST: ( + ([("a.pt", "a-n"), ("b.pt", "b-n", "")], "1 row(s) of more than 2 elements (row(s) 2"), + ([("a.pt", "a-n"), ("b.pt", "b-n", "", "x")], "1 row(s) of more than 2 elements (row(s) 2"), + ([("w.pt", "w-n")] + [("x.pt", "x-n", "z")] * 5 + [("y",) * 4] * 2, + "7 row(s) of more than 2 elements (row(s) 2, 3, 4, 5, 6 and 2 more"), + ), +} +_UNCLOSED_QUOTE = { + _STRUCTURE_LIST: 'a,"f.cif,a-n\nb,g.cif,b-n\n', + _RESIDUE_FILE_LIST: '"a.pt,a-n\nb.pt,b-n\n', +} +_CSV_RULE = { + _STRUCTURE_LIST: ("A structure list is CSV without a header, of at most 3 fields per row (structure name, " + "structure file or URL, embedding name): correct it where pandas' message points."), + _RESIDUE_FILE_LIST: ("A residue file list is CSV without a header, of at most 2 fields per row (residue embedding " + "file, embedding name): correct it where pandas' message points."), +} + + +def _read_structure_list(*args, **kwargs): + """foldmatch.dataset.utils.read_structure_list, imported when called.""" + from foldmatch.dataset.utils import read_structure_list + return read_structure_list(*args, **kwargs) + + +def _read_residue_file_list(*args): + """foldmatch.dataset.utils.read_residue_file_list, imported when called.""" + from foldmatch.dataset.utils import read_residue_file_list + return read_residue_file_list(*args) + + +def _esm3(model, protein): + """Stands in for ESM3 in the inference modules: L + 2 rows (BOS and EOS, zeros), each residue's row its letter, its + position and the chain's length (as in test_assembly_lists.py).""" + rows = [[float(ord(aa)), float(k), float(len(protein))] for k, aa in enumerate(protein.sequence)] + return torch.tensor([[0.0] * 3] + rows + [[0.0] * 3]) + + +def _aggregate(x, x_mask): + """Stands in for the aggregator: the sum of each item's rows.""" + return x.masked_fill(x_mask.unsqueeze(-1), 0.0).sum(dim=1) + + +def _chain_embedding(residues) -> np.ndarray: + """_aggregate's vector for these residue embeddings, through the padding of a batch, as the modules give it.""" + return _aggregate(*collate_seq_embeddings([residues]))[0].numpy() + + +def _csv(values) -> str: + """These values as CsvBatchWriter writes them.""" + return pd.DataFrame(values).to_csv(index=False, header=False) + + +def _proteins(cif: Path, assembly_id=None) -> dict: + """The protein chains of a structure, or of one of its assemblies, by chain ID: read as every path reads them.""" + structure = get_structure_from_src(str(cif), StructureFormat.mmcif, assembly_id=assembly_id) + return {atoms.chain_id[0]: ESMProtein.from_protein_chain( + ProteinChain.from_atomarray(rename_atom_attr(filter_residues(atoms)))) + for atoms in protein_chain_iter(structure)} + + +def _expected(rows: list, index: list, columns: list) -> pd.DataFrame: + """A frame of these columns, as strings, None read as NaN.""" + return pd.DataFrame(rows, index=pd.Index(index, dtype="int64"), columns=columns, dtype=str) + + +def _unsorted(rows: list, index: list) -> list: + """The rows of a sorted frame, in input order.""" + return [row for _, row in sorted(zip(index, rows), key=lambda item: item[0])] + + +def _write_once(fifo: Path, text: str, stop: threading.Event): + """The writer of the named pipe fifo, until stop is set: its first reader reads text, a later one an end of file at + once, as from a pipe whose writer has exited (rather than a wait for another writer). As in + test_assembly_lists.py.""" + while not stop.is_set(): + try: + # Opened only once a reader has opened it, so that stop always ends the loop + fd = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK) + except OSError as e: + if e.errno != errno.ENXIO: + raise + stop.wait(0.01) + continue + os.set_blocking(fd, True) + with os.fdopen(fd, "w", encoding="utf-8") as f: + f.write(text) + text = "" + stop.wait(0.01) + + +class _Frame(Exception): + """Carries the frame that a reader gave refuse_shared_names: ResidueEmbeddingFromStructure and + ResidueAssemblyDatasetFromStructure keep their list's own in no attribute, and read the structures right after.""" + + def __init__(self, data: pd.DataFrame): + super().__init__() + self.data = data + + +def _raise_frame(data, *_): + raise _Frame(data) + + +class _ListCase(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + + def list_file(self, case: str, text: str) -> Path: + path = self.tmp / f"{case.replace(',', '').replace(' ', '_')}.csv" + path.write_text(text, encoding="utf-8", newline="") + return path + + def sources(self, kind: str, path: Path, text: str) -> tuple: + """(source, what opens it, how a refusal names it): a path, an open file (named by its path), a text buffer.""" + return (("path", lambda: path, str(path)), + ("open file", lambda: open(path, encoding="utf-8", newline=""), str(path)), + ("text buffer", lambda: io.StringIO(text), f"The {kind}")) + + def readers(self, src, src_location: SrcLocation = SrcLocation.file) -> dict: + """The read functions and the 5 readers (_READERS), each a callable that reads src, by name.""" + return { + "read_structure_list": lambda: _read_structure_list(src, src_location), + "EsmProtFromStructure": lambda: EsmProtFromStructure(src_stream=src, src_location=src_location), + "EsmAssemblyFromStructure": lambda: EsmAssemblyFromStructure(src_stream=src, src_location=src_location), + "ResidueEmbeddingFromStructure": lambda: ResidueEmbeddingFromStructure( + src_stream=src, res_embedding_location=self.tmp, src_location=src_location), + "ResidueAssemblyDatasetFromStructure": lambda: ResidueAssemblyDatasetFromStructure( + src_stream=src, res_embedding_location=self.tmp, src_location=src_location), + "read_residue_file_list": lambda: _read_residue_file_list(src, src_location), + "ResidueEmbeddingFromTensorFile": lambda: ResidueEmbeddingFromTensorFile(src_stream=src, + src_location=src_location), + } + + def frame(self, reader: str, src, src_location: SrcLocation = SrcLocation.file) -> pd.DataFrame: + """The frame that this reader made of the list.""" + modules = {"ResidueEmbeddingFromStructure": residue_embedding_from_structure, + "ResidueAssemblyDatasetFromStructure": resdiue_assembly_embedding_from_structure} + read = self.readers(src, src_location)[reader] + if reader not in modules: + return read().data + with mock.patch.object(modules[reader], "refuse_shared_names", _raise_frame): + with self.assertRaises(_Frame) as frame: + read() + return frame.exception.data + + def assert_refused(self, refusal: BaseException, kind: str, source: str, found: str): + """The refusal names the source and what it found, says what a row holds, and what to do.""" + self.assertIsInstance(refusal, RefusalError) + self.assertRegex(str(refusal), f"^{re.escape(source)}: {found}\\. " + f"{re.escape(_RULE[kind] + ': ' + _REMOVE_THE_EXTRA_FIELDS[kind])}$") + + +class LongerRowTest(_ListCase): + """D1: the read functions and the 5 readers refuse a row of more fields than their columns, from a path, an open + file (named by its path) and a text buffer ("The structure list", "The residue file list"), and from a stream; any + other ParserError too.""" + + def test_a_file_row_of_too_many_fields(self): + """A first row with a trailing comma, with a field of text or with two more fields, every row with one or two + index columns (pandas took them as the index), a chain list or a structure list given by mistake, and a later + row longer than the first, or longer still after a longer first row (pandas' ParserError, which names the line, + quoted and kept as the cause).""" + for kind, lists in _LISTS.items(): + for case, text, found in lists: + path = self.list_file(f"{kind} {case}", text) + for source, make, named in self.sources(kind, path, text): + for reader in _READERS[kind]: + with self.subTest(kind=kind, case=case, source=source, reader=reader): + src = make() + try: + with self.assertRaises(RefusalError) as refused: + self.readers(src)[reader]() + finally: + if hasattr(src, "close"): + src.close() + self.assert_refused(refused.exception, kind, named, found) + if "later row" in case: + self.assertIsInstance(refused.exception.__cause__, pd.errors.ParserError) + + def test_the_index_column_of_to_csv(self): + """What pandas' to_csv(header=False) writes, an index column 0, 1, 2 before the columns, is refused, on every + pandas: read without dtype=str, pandas 3 makes a RangeIndex of it.""" + rows = {_STRUCTURE_LIST: [(f"{c}", f"{c}.cif", f"{c}-n") for c in "abcd"], + _RESIDUE_FILE_LIST: [(f"{c}.pt", f"{c}-n") for c in "abcd"]} + for kind, listed in rows.items(): + path = self.tmp / f"{kind.replace(' ', '_')}_to_csv.csv" + pd.DataFrame(listed).to_csv(path, header=False) + for reader in _READERS[kind]: + with self.subTest(kind=kind, reader=reader): + with self.assertRaises(RefusalError) as refused: + self.readers(path)[reader]() + self.assert_refused(refused.exception, kind, str(path), _first_row(len(listed[0]) + 1)) + + def test_a_stream_row_of_too_many_elements(self): + """A stream: how many rows have too many elements, and up to 5 of them, counted from 1 (before, pandas' + ValueError named no row), in a list and in a generator.""" + for kind, streams in _STREAMS.items(): + for rows, found in streams: + message = f"The {kind} stream: {found}, counted from 1). {_RULE[kind]}: remove the extra elements." + for container in (list, iter): + for reader in _READERS[kind]: + with self.subTest(kind=kind, rows=len(rows), longest=max(map(len, rows)), + container=container.__name__, reader=reader): + with self.assertRaises(RefusalError) as refused: + self.readers(container(rows), SrcLocation.stream)[reader]() + self.assertEqual(str(refused.exception), message) + + def test_another_parser_error_is_not_claimed_as_extra_fields(self): + """Another pandas ParserError (an unclosed quote) is refused too, naming the file and quoting pandas, without + saying that a row has too many fields (before, pandas' bare ParserError).""" + for kind, text in _UNCLOSED_QUOTE.items(): + path = self.list_file(f"{kind} unclosed quote", text) + for source, make, named in self.sources(kind, path, text): + for reader in _READERS[kind]: + with self.subTest(kind=kind, source=source, reader=reader): + src = make() + try: + with self.assertRaises(RefusalError) as refused: + self.readers(src)[reader]() + finally: + if hasattr(src, "close"): + src.close() + self.assertIsInstance(refused.exception.__cause__, pd.errors.ParserError) + self.assertRegex(str(refused.exception), + f"^{re.escape(named)}: pandas could not read it as CSV \\([^()]*EOF inside " + f"string[^()]*\\)\\. {re.escape(_CSV_RULE[kind])}$") + self.assertNotIn("more than", str(refused.exception)) + + +class EntryPointTest(_ListCase): + """D1 and D4: each entry point refuses such a list, in a file and as a stream, before any model is built (the model + getters fail if called), and writes nothing.""" + + def setUp(self): + super().setUp() + self.never = mock.Mock(side_effect=AssertionError("a model was built")) + for module in ("esm_inference", "structure_inference", "chain_complete_inference", + "assembly_complete_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_residue_model", self.never)) + for module in ("structure_inference", "chain_inference", "chain_complete_inference", + "assembly_complete_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_aggregator_model", self.never)) + self.out = self.tmp / "out" + self.out.mkdir() + self.residues = self.tmp / "residues" + self.residues.mkdir() + + def entry_points(self, src, src_location: SrcLocation, out_format: OutFormat = OutFormat.csv) -> dict: + """Each entry point of a structure list on src, a callable by name (out_format: the writer it would make).""" + common = {"src_stream": src, "src_location": src_location, "out_folder": self.out, **_CPU} + written = {**common, "out_format": out_format} + return { + "esm_inference": lambda: esm_inference.predict(src_from=SrcProteinFrom.structure, **written), + "structure_inference": lambda: structure_inference.predict(src_from=SrcProteinFrom.structure, **common), + "chain_inference": lambda: chain_inference.predict(src_from=SrcTensorFrom.structure, + res_embedding_location=self.residues, **written), + "assembly_inferece": lambda: assembly_inferece.predict(src_from=SrcAssemblyFrom.structure, + res_embedding_location=self.residues, **written), + "chain_complete_inference": lambda: chain_complete_inference.predict(src_from=SrcEsmFrom.structure, + **written), + "assembly_complete_inference": lambda: assembly_complete_inference.predict(**written), + } + + def assert_nothing_done(self): + self.never.assert_not_called() + self.assertEqual(list(self.out.iterdir()), []) + self.assertEqual(list(self.residues.iterdir()), []) + + def test_a_structure_list_in_a_file(self): + for case, text, found in _LISTS[_STRUCTURE_LIST]: + path = self.list_file(case, text) + for entry_point, predict in self.entry_points(path, SrcLocation.file).items(): + with self.subTest(case=case, entry_point=entry_point): + with self.assertRaises(RefusalError) as refused: + predict() + self.assert_refused(refused.exception, _STRUCTURE_LIST, str(path), found) + self.assert_nothing_done() + + def test_a_residue_file_list_in_a_file(self): + """chain_inference.predict from residue embedding files, its default.""" + for case, text, found in _LISTS[_RESIDUE_FILE_LIST]: + path = self.list_file(case, text) + with self.subTest(case=case): + with self.assertRaises(RefusalError) as refused: + chain_inference.predict(src_stream=path, out_folder=self.out, **_CPU) + self.assert_refused(refused.exception, _RESIDUE_FILE_LIST, str(path), found) + self.assert_nothing_done() + + def test_a_stream(self): + for rows, found in _STREAMS[_STRUCTURE_LIST]: + for entry_point, predict in self.entry_points(rows, SrcLocation.stream).items(): + with self.subTest(rows=len(rows), longest=max(map(len, rows)), entry_point=entry_point): + with self.assertRaises(RefusalError) as refused: + predict() + self.assertIn(f"The structure list stream: {found}", str(refused.exception)) + self.assert_nothing_done() + for rows, found in _STREAMS[_RESIDUE_FILE_LIST]: + with self.subTest(rows=len(rows), longest=max(map(len, rows)), entry_point="chain_inference, files"): + with self.assertRaises(RefusalError) as refused: + chain_inference.predict(src_stream=rows, src_location=SrcLocation.stream, out_folder=self.out, + **_CPU) + self.assertIn(f"The residue file list stream: {found}", str(refused.exception)) + self.assert_nothing_done() + + def test_every_writer(self): + """chain_complete_inference makes its writer before it reads its list: no writer leaves a file behind.""" + path = self.list_file("trailing comma", "a,f.cif,a-n,\n") + for out_format in OutFormat: + for entry_point, predict in self.entry_points(path, SrcLocation.file, out_format).items(): + with self.subTest(out_format=out_format.value, entry_point=entry_point): + with self.assertRaises(RefusalError): + predict() + self.assert_nothing_done() + + +class AsBeforeTest(_ListCase): + """Lists of at most that many fields per row read as before, but for P9.N's D5: the frames recorded from the readers + of 8ded017, and the streams of the commands. Since P9.N every reader reads NA, nan and null as text and a missing or + empty field as '' (before, ResidueEmbeddingFromStructure, ResidueAssemblyDatasetFromStructure and + ResidueEmbeddingFromTensorFile read them as NaN), and a missing or empty embedding name is the structure name, or + the residue file's name without its extension (before, '' or NaN).""" + + # (case, structure list, the rows of EsmProtFromStructure's and EsmAssemblyFromStructure's frame, in input order; + # the index of ResidueEmbeddingFromStructure's and ResidueAssemblyDatasetFromStructure's, sorted by structure name) + STRUCTURE_LISTS = ( + ("rows of 1, 2 and 3 fields", "c,f.cif,c-n\nb,f.cif\na\n", + [["c", "f.cif", "c-n"], ["b", "f.cif", "b"], ["a", "", "a"]], [2, 1, 0]), + ("an empty 3rd field and NA", "b,f.cif,\nNA,g.cif,a-n\na,NA,NA\n", + [["b", "f.cif", "b"], ["NA", "g.cif", "a-n"], ["a", "NA", "NA"]], [1, 2, 0]), + ("a byte-order mark and CRLF", "b,f.cif,b-n\r\na,g.cif,a-n\r\n", + [["b", "f.cif", "b-n"], ["a", "g.cif", "a-n"]], [1, 0]), + ("quoted fields", 'b,"f,1.cif","n,1"\n"a\nz",g.cif,\n', + [["b", "f,1.cif", "n,1"], ["a\nz", "g.cif", "a\nz"]], [1, 0]), + ("blank lines", "\nb,f.cif,b-n\n\n \n\t\na,g.cif,a-n\n\n", + [["b", "f.cif", "b-n"], ["a", "g.cif", "a-n"]], [1, 0]), + ("an empty file", "", [], []), + ) + # (case, residue file list, the rows of ResidueEmbeddingFromTensorFile's frame, sorted by file; its index) + RESIDUE_FILE_LISTS = ( + ("rows of 1 and 2 fields", "c.pt,c-n\nb.pt\na.pt,a-n\n", + [["a.pt", "a-n"], ["b.pt", "b"], ["c.pt", "c-n"]], [2, 1, 0]), + ("an empty 2nd field and NA", "b.pt,\nNA,a-n\na.pt,NA\n", + [["NA", "a-n"], ["a.pt", "NA"], ["b.pt", "b"]], [1, 2, 0]), + ("a byte-order mark and CRLF", "b.pt,b-n\r\na.pt,a-n\r\n", + [["a.pt", "a-n"], ["b.pt", "b-n"]], [1, 0]), + ("quoted fields", 'b.pt,"n,1"\n"a\nz.pt",\n', + [["a\nz.pt", "a\nz"], ["b.pt", "n,1"]], [1, 0]), + ("blank lines", "\nb.pt,b-n\n\n \n\t\na.pt,a-n\n\n", + [["a.pt", "a-n"], ["b.pt", "b-n"]], [1, 0]), + ("an empty file", "", [], []), + ) + + def test_the_readers_columns(self): + """D2: each reader keeps its COLUMNS, which are the read function's.""" + from foldmatch.dataset.utils.utils import RESIDUE_FILE_LIST_COLUMNS, STRUCTURE_LIST_COLUMNS + self.assertEqual(STRUCTURE_LIST_COLUMNS, _STRUCTURE_COLUMNS) + self.assertEqual(RESIDUE_FILE_LIST_COLUMNS, _RESIDUE_FILE_COLUMNS) + for reader in (EsmProtFromStructure, EsmAssemblyFromStructure, ResidueEmbeddingFromStructure, + ResidueAssemblyDatasetFromStructure): + self.assertEqual(reader.COLUMNS, STRUCTURE_LIST_COLUMNS, reader) + self.assertEqual(ResidueEmbeddingFromTensorFile.COLUMNS, RESIDUE_FILE_LIST_COLUMNS) + + def test_the_structure_readers_read_them_as_before(self): + """The 4 readers' frames, from a path, an open file and a text buffer, each with its sort.""" + for case, text, kept, index in self.STRUCTURE_LISTS: + path = self.list_file(case, text) + rows = [kept[k] for k in index] + for source, make, _ in self.sources(_STRUCTURE_LIST, path, text): + for reader, expected in ( + ("EsmProtFromStructure", _expected(kept, list(range(len(kept))), _STRUCTURE_COLUMNS)), + ("EsmAssemblyFromStructure", _expected(kept, list(range(len(kept))), _STRUCTURE_COLUMNS)), + ("ResidueEmbeddingFromStructure", _expected(rows, index, _STRUCTURE_COLUMNS)), + ("ResidueAssemblyDatasetFromStructure", _expected(rows, index, _STRUCTURE_COLUMNS))): + with self.subTest(case=case, source=source, reader=reader): + src = make() + try: + data = self.frame(reader, src) + finally: + if hasattr(src, "close"): + src.close() + pd.testing.assert_frame_equal(data, expected, check_exact=True) + + def test_the_residue_file_reader_reads_them_as_before(self): + """ResidueEmbeddingFromTensorFile's frame, from a path, an open file and a text buffer: its names as strings + where they are not numeric (D3 changes nothing there), NA as text and an empty name the file's name without its + extension (P9.N). An empty file's columns were of object type on pandas 3: NamesTest.""" + for case, text, rows, index in self.RESIDUE_FILE_LISTS: + path = self.list_file(case, text) + for source, make, _ in self.sources(_RESIDUE_FILE_LIST, path, text): + with self.subTest(case=case, source=source): + src = make() + try: + data = self.frame("ResidueEmbeddingFromTensorFile", src) + finally: + if hasattr(src, "close"): + src.close() + pd.testing.assert_frame_equal(data, _expected(rows, index, _RESIDUE_FILE_COLUMNS), + check_exact=True, check_dtype=bool(rows)) + + def test_the_read_functions(self): + """read_structure_list and read_residue_file_list: the frame of the readers' pandas call on 8ded017, in input + order, its index pandas' (0, 1, ...). A residue file list's empty file: columns of strings (NamesTest).""" + for case, text, kept, index in self.STRUCTURE_LISTS: + path = self.list_file(case, text) + for source, src in (("path", path), ("text buffer", None)): + with self.subTest(case=case, source=source): + data = _read_structure_list(src or io.StringIO(text)) + pd.testing.assert_frame_equal(data, _expected(kept, list(range(len(kept))), _STRUCTURE_COLUMNS), + check_exact=True) + self.assertIsInstance(data.index, pd.RangeIndex) + for case, text, rows, index in self.RESIDUE_FILE_LISTS: + path = self.list_file(case, text) + for source, src in (("path", path), ("text buffer", None)): + with self.subTest(case=case, source=source): + data = _read_residue_file_list(src or io.StringIO(text)) + pd.testing.assert_frame_equal( + data, _expected(_unsorted(rows, index), sorted(index), _RESIDUE_FILE_COLUMNS), + check_exact=True) + self.assertIsInstance(data.index, pd.RangeIndex) + + def assert_structure_stream(self, make, rows: list, index: list): + """The 4 readers' frames of the stream that make gives (rows: in input order; index: of the sorted frame).""" + unsorted_frame = _expected(rows, list(range(len(rows))), _STRUCTURE_COLUMNS) + for reader, expected in ( + ("EsmProtFromStructure", unsorted_frame), ("EsmAssemblyFromStructure", unsorted_frame), + ("ResidueEmbeddingFromStructure", unsorted_frame.loc[index]), + ("ResidueAssemblyDatasetFromStructure", unsorted_frame.loc[index])): + with self.subTest(reader=reader): + pd.testing.assert_frame_equal(self.frame(reader, make(), SrcLocation.stream), expected, + check_exact=True) + + def test_the_streams_of_the_commands(self): + """D5: fm-embedding from-structures gives scan_structure_folder's tuple of (name, path as a string, name), + fm-search build a list of (stem, Path, stem): read by pandas as before, and so are a generator and short + rows.""" + scanned = scan_structure_folder(_PDB, StructureFormat.mmcif) + self.assertEqual(scanned, (("1acb", str(_CIF), "1acb"), ("2uzi", str(_CIF2), "2uzi"))) + reversed_paths = [(Path(f).stem, Path(f), Path(f).stem) for _, f, _ in reversed(scanned)] + as_read = [[Path(f).stem, f, Path(f).stem] for _, f, _ in reversed(scanned)] + for case, make, rows, index in ( + ("scan_structure_folder", lambda: scanned, [list(row) for row in scanned], [0, 1]), + ("(stem, Path, stem)", lambda: reversed_paths, as_read, [1, 0]), + ("a generator", lambda: iter(reversed_paths), as_read, [1, 0]), + ("short rows and None", lambda: [("b", "g.cif"), ("a", None, "x"), ("c",)], + [["b", "g.cif", "b"], ["a", None, "x"], ["c", None, "c"]], [1, 0, 2]), + ("an empty list", lambda: [], [], [])): + with self.subTest(case=case): + self.assert_structure_stream(make, rows, index) + + def test_the_streams_of_the_residue_file_reader(self): + """inference chain-embedding gives a list of (file, name), ResidueEmbeddingFromStructure its parent a tuple of + them: read by pandas as before, numeric-looking names as they are.""" + rows = [("/x/b.pt", "007"), ("/x/a.pt", "1.50")] + expected = _expected([["/x/a.pt", "1.50"], ["/x/b.pt", "007"]], [1, 0], _RESIDUE_FILE_COLUMNS) + for case, make in (("a list", lambda: rows), ("a tuple", lambda: tuple(rows)), + ("a generator", lambda: iter(rows)), + ("Path elements", lambda: [(Path(f), n) for f, n in rows])): + with self.subTest(case=case): + pd.testing.assert_frame_equal(self.frame("ResidueEmbeddingFromTensorFile", make(), SrcLocation.stream), + expected, check_exact=True) + with self.subTest(case="short rows and None"): + pd.testing.assert_frame_equal( + self.frame("ResidueEmbeddingFromTensorFile", [("b.pt", "n"), ("a.pt",), ("c.pt", None)], + SrcLocation.stream), + _expected([["a.pt", "a"], ["b.pt", "n"], ["c.pt", "c"]], [1, 0, 2], _RESIDUE_FILE_COLUMNS), + check_exact=True) + + def test_to_csv_without_the_index(self): + """What the CHANGELOG tells them to write: to_csv(header=False, index=False) gives the list as it was meant.""" + rows = [("b", "g.cif", "b-n"), ("a", "f.cif", "a-n")] + path = self.tmp / "structures.csv" + pd.DataFrame(rows).to_csv(path, header=False, index=False) + for reader in ("EsmProtFromStructure", "EsmAssemblyFromStructure"): + with self.subTest(reader=reader): + pd.testing.assert_frame_equal(self.frame(reader, path), _expected(rows, [0, 1], _STRUCTURE_COLUMNS), + check_exact=True) + for reader in ("ResidueEmbeddingFromStructure", "ResidueAssemblyDatasetFromStructure"): + with self.subTest(reader=reader): + pd.testing.assert_frame_equal(self.frame(reader, path), + _expected(rows[::-1], [1, 0], _STRUCTURE_COLUMNS), check_exact=True) + rows = [("b.pt", "b-n"), ("a.pt", "a-n")] + path = self.tmp / "residues.csv" + pd.DataFrame(rows).to_csv(path, header=False, index=False) + with self.subTest(reader="ResidueEmbeddingFromTensorFile"): + pd.testing.assert_frame_equal(self.frame("ResidueEmbeddingFromTensorFile", path), + _expected(rows[::-1], [1, 0], _RESIDUE_FILE_COLUMNS), check_exact=True) + + +class _StubModels(_ListCase): + """The entry points with stub models: no ESM3, no aggregator.""" + + def setUp(self): + super().setUp() + for module in ("esm_inference", "structure_inference", "chain_complete_inference", + "assembly_complete_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_residue_model", return_value="ESM3")) + for module in ("structure_inference", "chain_inference", "chain_complete_inference", + "assembly_complete_inference"): + self.enterContext(mock.patch(f"foldmatch.inference.{module}.get_aggregator_model", + return_value=_aggregate)) + for module in ("esm_module", "chain_complete_module", "assembly_complete_module"): + self.enterContext(mock.patch(f"foldmatch.modules.{module}.per_residue_embedding", _esm3)) + self.residues_of = {(cif.stem, chain): _esm3(None, protein) for cif in (_CIF, _CIF2) + for chain, protein in _proteins(cif).items()} + + def folder(self, name: str) -> Path: + return Path(tempfile.mkdtemp(prefix=f"{name}-", dir=self.tmp)) + + def assert_files(self, out: Path, expected: dict): + self.assertEqual({f.name: f.read_text() for f in out.iterdir()}, expected) + + def stored(self, entry: str, chain: str) -> torch.Tensor: + """A residue embedding stored in tests/resources (1536 values per residue).""" + return torch.load(_EMBEDDINGS / f"{entry}.{chain}.pt", map_location="cpu", weights_only=True) + + +class ConventionalListTest(_StubModels): + """The files that a conventional list gives with stub models, through the unchanged writers, as before.""" + + def test_the_entry_points_on_a_list_in_a_file(self): + structures = self.list_file("structures", f"1acb,{_CIF},x\n") + residues = {c: self.residues_of[("1acb", c)] for c in "AB"} + out = self.folder("out") + esm_inference.predict(src_stream=structures, src_from=SrcProteinFrom.structure, out_folder=out, + return_predictions=False, **_CPU) + self.assert_files(out, {f"x.{c}.csv": _csv(residues[c].numpy()) for c in "AB"}) + + out = self.folder("out") + chain_complete_inference.predict(src_stream=structures, src_from=SrcEsmFrom.structure, out_folder=out, + return_predictions=False, **_CPU) + self.assert_files(out, {f"x.{c}.csv": _csv(_chain_embedding(residues[c])) for c in "AB"}) + + out = self.folder("out") + assembly_complete_inference.predict(src_stream=structures, out_folder=out, return_predictions=False, **_CPU) + self.assert_files(out, {"x-1.csv": _csv(_chain_embedding(torch.cat([residues["A"], residues["B"]])))}) + + out = self.folder("out") + structure_inference.predict(src_stream=structures, src_from=SrcProteinFrom.structure, out_folder=out, + return_predictions=False, **_CPU) + table = pq.read_table(out / "inference-0.parquet") + self.assertEqual(table["id"].to_pylist(), ["x.A", "x.B"]) + for c, row in zip("AB", table["embedding"].to_pylist()): + np.testing.assert_array_equal(np.array(row, dtype=np.float32), _chain_embedding(residues[c])) + + stored = {c: self.stored("1acb", c) for c in "AB"} + out = self.folder("out") + chain_inference.predict(src_stream=structures, src_from=SrcTensorFrom.structure, + res_embedding_location=_EMBEDDINGS, out_folder=out, return_predictions=False, **_CPU) + self.assert_files(out, {f"x.{c}.csv": _csv(_chain_embedding(stored[c])) for c in "AB"}) + + out = self.folder("out") + assembly_inferece.predict(src_stream=structures, src_from=SrcAssemblyFrom.structure, + res_embedding_location=_EMBEDDINGS, out_folder=out, return_predictions=False, **_CPU) + self.assert_files(out, {"x-1.csv": _csv(_chain_embedding(torch.cat([stored["A"], stored["B"]])))}) + + files = self.list_file("files", f"{_EMBEDDINGS}/1acb.B.pt,r\n{_EMBEDDINGS}/1acb.A.pt,s\n") + out = self.folder("out") + chain_inference.predict(src_stream=files, out_folder=out, return_predictions=False, **_CPU) + self.assert_files(out, {"r.csv": _csv(_chain_embedding(stored["B"])), + "s.csv": _csv(_chain_embedding(stored["A"]))}) + + def from_structures(self, command: str, out: Path): + """fm-embedding from-structures on tests/resources/pdb in-process, the logging setup restored after.""" + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + result = CliRunner().invoke(embedding.app, ["from-structures", command, "--src-folder", str(_PDB), + "--output-folder", str(out), "--accelerator", "cpu", + "--devices", "1"]) + finally: + root.handlers[:] = handlers + root.setLevel(level) + self.assertEqual(result.exit_code, 0, f"{result.output}\n{result.exception!r}") + + def test_from_structures_residue(self): + out = self.folder("out") + self.from_structures("residue", out) + self.assert_files(out, {f"{entry}.{c}.csv": _csv(residues.numpy()) + for (entry, c), residues in self.residues_of.items()}) + + def test_from_structures_chain(self): + out = self.folder("out") + self.from_structures("chain", out) + self.assert_files(out, {f"{entry}.{c}.csv": _csv(_chain_embedding(residues)) + for (entry, c), residues in self.residues_of.items()}) + + def test_from_structures_assembly(self): + out = self.folder("out") + self.from_structures("assembly", out) + expected = {} + for cif in (_CIF, _CIF2): + for assembly_id in get_assemblies(structure=str(cif), structure_format=StructureFormat.mmcif): + structure = get_structure_from_src(str(cif), StructureFormat.mmcif, assembly_id=assembly_id) + proteins = [ESMProtein.from_protein_chain(ProteinChain.from_atomarray( + rename_atom_attr(filter_residues(atoms)))) for atoms in protein_chain_iter(structure)] + residues = torch.cat([_esm3(None, protein) for protein in proteins]) + expected[f"{cif.stem}-{assembly_id}.csv"] = _csv(_chain_embedding(residues)) + self.assertTrue(expected) + self.assert_files(out, expected) + + @unittest.skipUnless(hasattr(os, "mkfifo"), "a named pipe needs os.mkfifo") + def test_a_list_in_a_pipe(self): + """D4: a list given as a pipe can be read once: each reader reads it once, where it did, and no entry point + reads it first, so a list in a named pipe reaches esm_inference.predict and chain_inference.predict, as on + 8ded017.""" + residues = self.residues_of[("1acb", "A")].numpy() + stored = self.stored("1acb", "A") + for case, text, predict, expected in ( + ("esm_inference", f"1acb,{_CIF},x\n", + lambda fifo, out: esm_inference.predict(src_stream=fifo, src_from=SrcProteinFrom.structure, + out_folder=out, return_predictions=False, **_CPU), + {"x.A.csv": _csv(residues), "x.B.csv": _csv(self.residues_of[("1acb", "B")].numpy())}), + ("chain_inference", f"{_EMBEDDINGS}/1acb.A.pt,r\n", + lambda fifo, out: chain_inference.predict(src_stream=fifo, out_folder=out, return_predictions=False, + **_CPU), + {"r.csv": _csv(_chain_embedding(stored))})): + with self.subTest(case=case): + fifo = self.tmp / f"{case}.fifo" + os.mkfifo(fifo) + stop = threading.Event() + writer = threading.Thread(target=_write_once, args=(fifo, text, stop), daemon=True) + writer.start() + out = self.folder("out") + try: + predict(fifo, out) + finally: + stop.set() + writer.join() + self.assert_files(out, expected) + + +class NamesTest(_StubModels): + """D3: the residue file list is read as strings, as every other list is, its names keeping their spelling. Since + P9.N (D5), NA is a name and an empty name is the file's name without its extension (before, NaN).""" + + def setUp(self): + super().setUp() + for name in "ab": + torch.save(torch.ones(3, 2) * (ord(name) - 96), self.tmp / f"{name}.pt") + self.a, self.b = _chain_embedding(torch.ones(3, 2)), _chain_embedding(torch.ones(3, 2) * 2) + + def names_list(self, *names) -> Path: + return self.list_file("names", "".join(f"{self.tmp}/{f}.pt,{name}\n" for f, name in zip("ab", names))) + + def predict(self, names: tuple, out_format: OutFormat) -> Path: + """chain_inference.predict on a list of a.pt and b.pt under these names, stub models: its output folder.""" + out = self.folder("out") + chain_inference.predict(src_stream=self.names_list(*names), out_format=out_format, out_folder=out, + return_predictions=False, **_CPU) + return out + + def test_the_names_as_read(self): + """From a path, an open file and a text buffer, as strings (before, 7.0 and 1.5, the integers 1 and 2, the + booleans True and False, or 1 and 2 without their space), and by each item of the dataset.""" + for names in (("007", "1.50"), ("1", "2"), ("007", "008"), ("1.0", "2.0"), ("TRUE", "false"), (" 1", "2 ")): + path = self.names_list(*names) + text = path.read_text() + for source, make, _ in self.sources(_RESIDUE_FILE_LIST, path, text): + with self.subTest(names=names, source=source): + src = make() + try: + dataset = ResidueEmbeddingFromTensorFile(src_stream=src) + finally: + if hasattr(src, "close"): + src.close() + self.assertEqual(list(dataset.data.item_name), list(names)) + self.assertEqual([dataset[k][1] for k in range(len(dataset))], list(names)) + self.assertIsInstance(dataset[0][1], str) + + def test_a_name_with_its_spelling_names_the_output(self): + """007 and 1.50 give 007.csv and 1.50.csv (before, 7.0.csv and 1.5.csv), the same in pt, and the ids '007' and + '1.50' in JSON (before, the numbers 7.0 and 1.5) and in Parquet (before, ArrowTypeError).""" + names = ("007", "1.50") + self.assert_files(self.predict(names, OutFormat.csv), {"007.csv": _csv(self.a), "1.50.csv": _csv(self.b)}) + self.assertEqual(sorted(f.name for f in self.predict(names, OutFormat.pt).iterdir()), ["007.pt", "1.50.pt"]) + self.assert_ids(names, names) + + def test_a_name_that_looks_like_a_boolean(self): + """TRUE and false give TRUE.csv and false.csv (before, True.csv and False.csv), the same in pt, and the ids + 'TRUE' and 'false' in JSON (before, true and false) and in Parquet (before, ArrowTypeError).""" + names = ("TRUE", "false") + self.assert_files(self.predict(names, OutFormat.csv), {"TRUE.csv": _csv(self.a), "false.csv": _csv(self.b)}) + self.assertEqual(sorted(f.name for f in self.predict(names, OutFormat.pt).iterdir()), ["TRUE.pt", "false.pt"]) + self.assert_ids(names, names) + + def assert_ids(self, names: tuple, ids: tuple): + """The ids in the Parquet and JSON output of these names.""" + with self.subTest(names=names, out_format="parquet"): + table = pq.read_table(self.predict(names, OutFormat.parquet) / "inference-0.parquet") + self.assertEqual(table["id"].to_pylist(), list(ids)) + with self.subTest(names=names, out_format="json"): + with gzip.open(self.predict(names, OutFormat.json) / "inference-0.json.gz", "rt") as f: + self.assertEqual([row["id"] for row in json.load(f)], list(ids)) + + def test_plain_integers_in_parquet_and_json(self): + """1 and 2 give the ids '1' and '2' in Parquet (before, ArrowTypeError, nothing written) and in JSON (before, + the numbers 1 and 2).""" + self.assert_ids(("1", "2"), ("1", "2")) + + def test_plain_integers_in_csv_and_pt(self): + """1 and 2 name the same files as before.""" + self.assert_files(self.predict(("1", "2"), OutFormat.csv), {"1.csv": _csv(self.a), "2.csv": _csv(self.b)}) + self.assertEqual(sorted(f.name for f in self.predict(("1", "2"), OutFormat.pt).iterdir()), ["1.pt", "2.pt"]) + + def test_an_empty_name_is_the_file_name(self): + """P9.N: an empty name is the file's name without its extension, b.csv (before, NaN, the file nan.csv), and NA + is a name, NA.csv (before, NaN too).""" + for names, files in ((("x", ""), ("x.csv", "b.csv")), (("x", "NA"), ("x.csv", "NA.csv"))): + with self.subTest(names=names): + dataset = ResidueEmbeddingFromTensorFile(src_stream=self.names_list(*names)) + self.assertEqual(list(dataset.data.item_name), [names[0], files[1][:-4]]) + self.assert_files(self.predict(names, OutFormat.csv), {files[0]: _csv(self.a), files[1]: _csv(self.b)}) + + def test_an_integer_next_to_an_empty_name(self): + """An integer keeps its spelling next to an empty name: 1.csv and b.csv (before, 1.0.csv, then nan.csv).""" + self.assert_files(self.predict(("1", ""), OutFormat.csv), {"1.csv": _csv(self.a), "b.csv": _csv(self.b)}) + + def test_the_empty_name_column_is_of_strings(self): + """Empty names only: the files' names, in a column of strings (before, NaN; before that, floats).""" + data = ResidueEmbeddingFromTensorFile(src_stream=self.names_list("", "")).data + pd.testing.assert_frame_equal(data, _expected([[f"{self.tmp}/a.pt", "a"], [f"{self.tmp}/b.pt", "b"]], [0, 1], + _RESIDUE_FILE_COLUMNS), check_exact=True) + + def test_an_empty_file_is_of_strings(self): + """An empty file: no rows, its columns of strings, as every other list's (before, object on pandas 3).""" + data = ResidueEmbeddingFromTensorFile(src_stream=self.list_file("empty", "")).data + pd.testing.assert_frame_equal(data, _expected([], [], _RESIDUE_FILE_COLUMNS), check_exact=True) + + def test_a_file_column_of_numbers_sorts_as_text(self): + """Contrived: a file column that holds only numbers sorts as text (before, as numbers).""" + data = ResidueEmbeddingFromTensorFile(src_stream=self.list_file("numbers", "2.5,x\n10,y\n")).data + pd.testing.assert_frame_equal(data, _expected([["10", "y"], ["2.5", "x"]], [1, 0], _RESIDUE_FILE_COLUMNS), + check_exact=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_structure_names.py b/tests/test_structure_names.py new file mode 100644 index 0000000..c773d9e --- /dev/null +++ b/tests/test_structure_names.py @@ -0,0 +1,153 @@ +"""P9.B2: a structure name names one source. + +The datasets read a structure once for all the rows that name it (one per chain or assembly): StructureProvider keeps +the last structure read under its name, whether the source is a path or a downloaded stream. Rows that gave one name +to two files or URLs got the first one's structure for both; they are now refused when the rows are loaded.""" +import gzip +import io +import logging +import os +import shutil +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +import torch + +from foldmatch.cli import embedding +from foldmatch.dataset import resdiue_assembly_embedding_from_structure, residue_embedding_from_structure +from foldmatch.dataset.esm_assembly_from_structure import EsmAssemblyFromStructure +from foldmatch.dataset.esm_prot_from_chain import EsmProtFromChain +from foldmatch.dataset.esm_prot_from_structure import EsmProtFromStructure +from foldmatch.dataset.resdiue_assembly_embedding_from_structure import ResidueAssemblyDatasetFromStructure +from foldmatch.dataset.residue_assembly_embedding_from_parquet import ResidueAssemblyEmbeddingFromParquet +from foldmatch.dataset.residue_assembly_embedding_from_tensor_file import ResidueAssemblyEmbeddingFromTensorFile +from foldmatch.dataset.residue_embedding_from_structure import ResidueEmbeddingFromStructure +from foldmatch.dataset.utils import refuse_shared_names +from foldmatch.errors import RefusalError +from foldmatch.inference import chain_complete_inference +from foldmatch.types.api_types import SrcLocation + +_RESOURCES = Path(__file__).parent / "resources" +_CIF = _RESOURCES / "pdb" / "1acb.cif" +_EMBEDDINGS = _RESOURCES / "embeddings" + + +class SharedNameTest(unittest.TestCase): + + def setUp(self): + self.tmp = Path(self.enterContext(tempfile.TemporaryDirectory())) + self.copy = Path(shutil.copyfile(_CIF, self.tmp / "1acb.cif")) + self.link = self.tmp / "link.cif" + os.symlink(_CIF, self.link) + self.parquet = self.tmp / "residues.parquet" + embeddings = [torch.load(_EMBEDDINGS / f"1acb.{c}.pt", map_location="cpu") for c in "AB"] + kind = pa.list_(pa.list_(pa.float32(), embeddings[0].shape[1])) # one list of D values per residue + pq.write_table(pa.table({"id": ["1acb.A", "1acb.B"], "embedding": [e.numpy().tolist() for e in embeddings]}, + schema=pa.schema([("id", pa.string()), ("embedding", kind)])), + self.parquet) + + def datasets(self, sources): + """Each dataset that reads structures, built on rows of the name '1acb' with these sources (two rows each).""" + three = [("1acb", source, f"1acb-{k}") for k, source in enumerate(sources)] + chains = [("1acb", source, chain, f"1acb-{k}.{chain}") for k, source in enumerate(sources) for chain in "AB"] + assemblies = [("1acb", source, "1", f"1acb-{k}-1") for k, source in enumerate(sources)] + stream = SrcLocation.stream + return { + "EsmAssemblyFromStructure": lambda: EsmAssemblyFromStructure(src_stream=three, src_location=stream), + "EsmProtFromStructure": lambda: EsmProtFromStructure(src_stream=three, src_location=stream), + "EsmProtFromChain": lambda: EsmProtFromChain(src_stream=chains, src_location=stream), + "ResidueAssemblyEmbeddingFromTensorFile": lambda: ResidueAssemblyEmbeddingFromTensorFile( + src_stream=assemblies, res_embedding_location=_EMBEDDINGS, src_location=stream), + "ResidueAssemblyEmbeddingFromParquet": lambda: ResidueAssemblyEmbeddingFromParquet( + src_stream=assemblies, parquet_path=self.parquet, src_location=stream), + "ResidueAssemblyDatasetFromStructure": lambda: ResidueAssemblyDatasetFromStructure( + src_stream=three, res_embedding_location=_EMBEDDINGS, src_location=stream), + "ResidueEmbeddingFromStructure": lambda: ResidueEmbeddingFromStructure( + src_stream=three, res_embedding_location=_EMBEDDINGS, src_location=stream), + } + + def test_two_files_under_one_name(self): + for dataset, build in self.datasets([str(_CIF), str(self.copy)]).items(): + with self.subTest(dataset=dataset): + with self.assertRaises(RefusalError) as ctx: + build() + message = str(ctx.exception) + self.assertIn(f"'1acb' names {min(str(_CIF), str(self.copy))} and {max(str(_CIF), str(self.copy))}", + message) + self.assertIn("give each file its own name", message) + + def test_one_file_under_one_name(self): + """The same file on several rows, under two spellings: a symbolic link to it, a hard link to it.""" + hard = self.tmp / "hard.cif" + os.link(self.copy, hard) + for spellings in ([str(_CIF), str(self.link)], [str(self.copy), str(hard)]): + for dataset, build in self.datasets(spellings).items(): + with self.subTest(spellings=spellings, dataset=dataset): + build() + + def test_what_is_compared(self): + """URLs differing only in the case of their scheme and host are one source, other URLs two; a row without a + name or a source is left to the dataset, as before.""" + def refused(rows): + try: + refuse_shared_names(pd.DataFrame(rows, columns=["stream_name", "stream"]), "stream_name", "stream") + except RefusalError: + return True + return False + self.assertFalse(refused([("x", "HTTPS://Example.ORG/a/1acb.cif"), ("x", "https://example.org/a/1acb.cif")])) + self.assertTrue(refused([("x", "https://example.org/a/1acb.cif"), ("x", "https://example.org/A/1acb.cif")])) + self.assertTrue(refused([("x", "https://example.org/1acb.cif"), ("x", str(_CIF))])) + self.assertFalse(refused([("x", str(_CIF)), ("x", None), (None, str(self.copy)), ("y", str(self.copy))])) + + def test_two_urls_under_one_name(self): + """Refused before anything is downloaded.""" + urls = ["https://example.org/a/1acb.cif", "https://example.org/b/1acb.cif"] + never = mock.Mock(side_effect=AssertionError("downloaded")) + with mock.patch.object(resdiue_assembly_embedding_from_structure, "stringio_from_url", never), \ + mock.patch.object(residue_embedding_from_structure, "stringio_from_url", never): + for dataset, build in self.datasets(urls).items(): + with self.subTest(dataset=dataset): + with self.assertRaises(RefusalError) as ctx: + build() + self.assertIn(f"'1acb' names {urls[0]} and {urls[1]}", str(ctx.exception)) + never.assert_not_called() + + def test_a_folder_with_two_files_of_one_entry(self): + """'fm-embedding from-structures chain' on a folder with 1acb.cif and 1acb.cif.gz, which both get the name + 1acb: exit status 1, one 'Error:' line, before ESM3 is loaded.""" + folder = self.tmp / "structures" + folder.mkdir() + shutil.copyfile(_CIF, folder / "1acb.cif") + with gzip.open(folder / "1acb.cif.gz", "wt") as f: + f.write(_CIF.read_text()) + out = self.tmp / "out" + out.mkdir() + argv = ["fm-embedding", "from-structures", "chain", "--src-folder", str(folder), "--output-folder", str(out)] + stderr = self.enterContext(mock.patch("sys.stderr", new_callable=io.StringIO)) + root = logging.getLogger() + handlers, level = root.handlers[:], root.level + try: + with mock.patch.object(sys, "argv", argv), \ + mock.patch.object(chain_complete_inference, "get_residue_model", + side_effect=AssertionError("ESM3 loaded")), \ + self.assertRaises(SystemExit) as ctx: + embedding.main() + finally: + root.handlers[:] = handlers + root.setLevel(level) + self.assertEqual(ctx.exception.code, 1) + errors = [line for line in stderr.getvalue().splitlines() if line.startswith("Error: ")] + self.assertEqual(len(errors), 1, stderr.getvalue()) + resolved = folder.resolve() # --src-folder is resolved (/var -> /private/var on macOS) + self.assertIn(f"'1acb' names {resolved / '1acb.cif'} and {resolved / '1acb.cif.gz'}", errors[0]) + self.assertEqual(list(out.iterdir()), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_swint.py b/tests/test_swint.py new file mode 100644 index 0000000..8127faa --- /dev/null +++ b/tests/test_swint.py @@ -0,0 +1,317 @@ +"""The BLOSUM62 Stage 2's native integer kernel (fmswi_align, :mod:`foldmatch.search.swint`) against biotite. + +The kernel must be output-equivalent to biotite 1.7.1 ``align_optimal(..., local=True, max_number=1)``: the same +score, end cell and path, so every output column of the Stage 2 is the one the biotite path (tests/blosum62_reference.py) +gave. biotite is the oracle here; the biotite-free golden digest of the kernel is in test_swkern.py. The libraries come +from the session fixture ``fmsw_libs``. +""" +import dataclasses +import random +import unittest + +import numpy as np +import pytest + +import blosum62_reference as ref +from foldmatch import _native +from foldmatch.search import alignment, karlin_altschul, swint, swkern + +AA = "ACDEFGHIKLMNPQRSTVWY" +ALL_NEEDS = alignment._FieldNeeds(cigar=True, qaln=True, taln=True, qseq=True, tseq=True) + + +def _trace_from_ops(ops: str, q_start: int, t_start: int) -> np.ndarray: + i, j, out = q_start - 1, t_start - 1, [] + for c in ops: + if c == "M": + out.append((i, j)); i += 1; j += 1 + elif c == "D": + out.append((-1, j)); j += 1 + else: + out.append((i, -1)); i += 1 + return np.array(out, dtype=np.int64).reshape(-1, 2) + + +def _kernels(): + """The integer kernel of every ISA library of this machine that can run here (avx2 only on an AVX2 CPU).""" + out = [] + for isa in _native.machine_isas(): + if _native.find_library(isa) is None: + continue + try: + out.append(swint.load(isa)) + except swkern.KernelError: + if isa != "avx2": + raise + return out + + +def _long_gap_pairs(rng): + """Pairs whose best alignment has one long gap (60 residues, across many stripe segments), in each direction.""" + core = "".join(rng.choice(AA) for _ in range(150)) + insert = "".join(rng.choice(AA) for _ in range(60)) + gapped = core[:75] + insert + core[75:] + return [(core, [gapped, gapped[:120] + gapped[150:]]), (gapped, [core])] + + +@pytest.mark.usefixtures("fmsw_libs") +class TestKernelAgainstBiotite(unittest.TestCase): + def setUp(self): + matrix = ref.blosum62() + self.assertEqual("".join(matrix.get_alphabet1().get_symbols()), swint.ALPHABET) + self.mat = matrix.score_matrix() + + def check(self, q, targets, gap=(11, 1), bias=None, kernel=None): + res = swint.align_strings(q, targets, self.mat, gap[0], gap[1], bias=bias, ops=True, kernel=kernel, + trace="full") + two = swint.align_strings(q, targets, self.mat, gap[0], gap[1], bias=bias, ops=True, kernel=kernel, + trace="prefix") + np.testing.assert_array_equal(two.out, res.out) # the traceback strategy changes nothing + self.assertEqual(two.ops, res.ops) + for idx, t in enumerate(targets): + want = ref.biotite_alignment(q, t, gap, bias) + o = res.out[idx] + msg = f"{q!r} vs {t!r} gap {gap}" + self.assertEqual(int(o[swint.SCORE]), int(want.score), msg) + self.assertEqual(int(o[swint.ALNLEN]), len(want.trace), msg) + if len(want.trace): + got = _trace_from_ops(res.path(idx), int(o[swint.QSTART]), int(o[swint.TSTART])) + np.testing.assert_array_equal(got, want.trace, err_msg=msg) + self.assertEqual(int(o[swint.QEND]), int(want.trace[:, 0].max()) + 1, msg) + self.assertEqual(int(o[swint.TEND]), int(want.trace[:, 1].max()) + 1, msg) + return res + + def test_constructed_tie_cases(self): + # gaps go as early as possible, prefixes summing to zero are cut, first maximal cell in row-major order + r = self.check("MWKAAAAKWM", ["MWKAAAKWM"]) + self.assertEqual(r.path(0), "MMMIMMMMMM") # MWKAAAAKWM / MWK-AAAKWM + r = self.check("APWW", ["AWWW"]) + self.assertEqual((int(r.out[0, swint.QSTART]), int(r.out[0, swint.TSTART])), (3, 2)) + self.check("AWA", ["AWAAWA", "W", "WAW"]) + core = "MKTAYIAKQRQISFVKSHFSRQ" + self.check(core + "W" * 12, [core + "D" * 12]) # no insertion next to a deletion (biotite) + self.check("WW", ["PP"]) # score 0: empty alignment + self.check("BZX*", ["BZX*", "XXXX", "**", "B", "Z"]) + + def test_random_pairs_every_gap_setting(self): + rng = random.Random(3) + alphabets = ["AG", "AGW", "ACDW", "WD", "AAAAGP", AA, AA + "BZX*"] + for gap in [(11, 1), (10, 1), (5, 2), (3, 3), (1, 1), (20, 5), (2, 0), (1, 4)]: + for _ in range(25): + q = "".join(rng.choice(rng.choice(alphabets)) for _ in range(rng.randint(1, 90))) + ts = ["".join(rng.choice(rng.choice(alphabets)) for _ in range(rng.randint(1, 90))) for _ in range(4)] + self.check(q, ts, gap) + + def test_long_gaps(self): + rng = random.Random(4) + for gap in [(11, 1), (20, 5), (3, 3)]: + for q, ts in _long_gap_pairs(rng): + r = self.check(q, ts, gap) + if gap == (11, 1): + self.assertIn("D" * 60 if len(q) == 150 else "I" * 60, r.path(0), gap) + + def test_position_specific_bias(self): + rng = random.Random(5) + for _ in range(60): + q = "".join(rng.choice(AA) for _ in range(rng.randint(1, 120))) + bias = np.array([rng.randint(-3, 3) for _ in q], dtype=np.int64) + ts = ["".join(rng.choice(AA) for _ in range(rng.randint(1, 120))) for _ in range(3)] + self.check(q, ts, (11, 1), bias=bias) + q = "Q" * 30 + "MKTAYIAKQRQISFVKSHFSRQ" + "GGGGS" * 8 # the MMseqs2 correction itself + self.check(q, [q, q[10:70], "MKTAYIAKQRQISFVKSHFSRQ"], bias=alignment.local_composition_bias(q)) + + def test_int32_redo(self): + q = "W" * 3100 # 34,100 > 32767 - 11 + r = self.check(q, [q, q[:3000] + "P" + q[3000:]]) + self.assertEqual(int(r.out[0, swint.SCORE]), 34100) + self.assertEqual(int(r.out[0, swint.WIDE]), 1) + small = swint.align_strings("WWWW", ["WWWW"], self.mat, 11, 1) + self.assertEqual(int(small.out[0, swint.WIDE]), 0) + + def test_every_isa_library_agrees(self): + """Every ISA library of the machine (not only the dispatched one) gives the same outputs and paths, in both + traceback strategies: random pairs, a 32-bit pair, a biased pair and long gaps, at two gap settings (the + dispatched library is checked against biotite above).""" + rng = random.Random(9) + q = "".join(rng.choice(AA) for _ in range(300)) + cases = [(q, ["".join(rng.choice(AA) for _ in range(rng.randint(1, 400))) for _ in range(20)] + [q], None)] + wide = "W" * 3100 + cases.append((wide, [wide, wide[:3000] + "P" + wide[3000:], "W" * 40], None)) + biased = "Q" * 30 + q[:120] + "GGGGS" * 8 + cases.append((biased, [biased, q, biased[25:140]], alignment.local_composition_bias(biased))) + cases += [(lq, lts, None) for lq, lts in _long_gap_pairs(rng)] + kernels = _kernels() + self.assertIn(swint.load().isa, [k.isa for k in kernels]) + for gap in [(11, 1), (20, 5)]: + for query, targets, bias in cases: + base = swint.align_strings(query, targets, self.mat, *gap, bias=bias, ops=True, trace="full") + if query is wide and gap == (11, 1): + self.assertEqual(base.out[:, swint.WIDE].tolist(), [1, 1, 0]) + for k in kernels: + for trace in swint.TRACE_MODES: + msg = f"{k.isa} {trace} gap {gap} query of {len(query)}" + other = swint.align_strings(query, targets, self.mat, *gap, bias=bias, ops=True, kernel=k, + trace=trace) + np.testing.assert_array_equal(other.out, base.out, err_msg=msg) + self.assertEqual(other.ops, base.ops, msg) + + def test_the_golden_set_against_biotite(self): + """The biotite-free golden digest (swint.GOLDEN_SHA256, checked on every build by test_swkern.py) pins + biotite's alignments: every pair of swint.golden_cases (synthetic profiles, offsets, long gaps, gap_extend > + gap_open, the 32-bit path, the 16-bit limit to the unit, the long lazy-F loop, empty alignments and targets) + aligns as biotite aligns it with the same position-specific scores, every output field and the path.""" + import biotite.sequence as seq + import biotite.sequence.align as align + wide = 0 + for case in swint.golden_cases(): + res = swint.run_golden(case, 1) + n = len(case.qcodes) + alphabet = seq.Alphabet(list(range(n))) + positional = seq.GeneralSequence(alphabet, list(range(n))) + matrix = align.SubstitutionMatrix(alphabet, seq.ProteinSequence.alphabet, case.qprof) + for k, t in enumerate(case.targets): + o = res.out[k] + msg = f"{case.kind} n={n} m={len(t)} gaps {case.gap_open}/{case.gap_extend}" + wide += int(o[swint.WIDE]) + if len(t) == 0: + self.assertFalse(o.any(), msg) + continue + target = seq.ProteinSequence() + target.code = t + want = align.align_optimal(positional, target, matrix, local=True, max_number=1, + gap_penalty=(-case.gap_open, -case.gap_extend))[0] + self.assertEqual(int(o[swint.SCORE]), int(want.score), msg) + self.assertEqual(int(o[swint.ALNLEN]), len(want.trace), msg) + if not len(want.trace): + continue + trace = want.trace + np.testing.assert_array_equal(_trace_from_ops(res.path(k), int(o[swint.QSTART]), + int(o[swint.TSTART])), trace, err_msg=msg) + gaps = trace < 0 # [L, 2]: 'D' (query side), 'I' (target side) + both = ~gaps.any(axis=1) + self.assertEqual((int(o[swint.QEND]), int(o[swint.TEND])), tuple(trace.max(axis=0) + 1), msg) + self.assertEqual(int(o[swint.NIDENT]), int((case.qcodes[trace[both, 0]] == t[trace[both, 1]]).sum()), + msg) + self.assertEqual(int(o[swint.GAPOPEN]), int(gaps[0].sum() + (gaps[1:] & ~gaps[:-1]).sum()), msg) + self.assertEqual((int(o[swint.NQGAP]), int(o[swint.NTGAP])), tuple(gaps.sum(axis=0)), msg) + self.assertEqual(wide, 10) + + def test_score_only_and_arguments(self): + res = swint.align_batch(swint.codes_of(b"MKV"), swint.query_profile(swint.codes_of(b"MKV"), self.mat), + swint.codes_of(b"MKVMKV"), [0, 3], [3, 3], 11, 1, traceback=False) + self.assertEqual(res.out[:, swint.SCORE].tolist(), [14, 14]) # M 5 + K 5 + V 4 + self.assertEqual(res.out[:, swint.ALNLEN].tolist(), [0, 0]) + empty = swint.align_strings("MKV", ["", "MKV"], self.mat, 11, 1) + self.assertEqual(empty.out[0].tolist(), [0] * swint.NOUT) + for go, ge in ((0, 1), (11, -1), (swint.MAX_GAP + 1, 1), (11, swint.MAX_GAP + 1)): + with self.subTest(gap=(go, ge)), self.assertRaises(ValueError): + swint.align_strings("MKV", ["MKV"], self.mat, go, ge) + swint.align_strings("MKV", ["MKV"], self.mat, swint.MAX_GAP, swint.MAX_GAP) + with self.assertRaises(ValueError): + swint.align_strings("MKV", ["MKV"], self.mat, 11, 1, trace="banded") + with self.assertRaises(ValueError): + swint.align_batch(np.array([30], np.uint8), np.zeros((1, 24), np.int32), np.zeros(1, np.uint8), [0], + [1], 11, 1) + self.assertEqual(swint.DEFAULT_TRACE, "prefix") + + def test_sanitize_fast_path(self): + chars = [chr(c) for c in range(128)] + ["ß", "é", "Ω", "fi", "µ"] + rng = random.Random(1) + for _ in range(500): + s = "".join(rng.choice(chars) for _ in range(rng.randint(0, 40))) + fast = alignment._sanitized_bytes(s) + self.assertEqual(fast.decode("ascii"), alignment.sanitize_sequence(s), repr(s)) + + +_STORE = { + "s1": "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVQ", + "s2": "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVA", + "s3": "MKTAYIAKQRQISFVKSHFSRQ", + "s4": "GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG", + "s5": "PLYISNDACEFHIKLMNPQRSTVWYACDEFGHIK", + "s6": "mktayiakqrqisfvkshfsrqleerlglieee", + "s7": "ACDEFGHIKLMNPQRSTVWYACDEFGHIKLMNPQUOJ*", + "s8": "", + "s9": "W", + "s10": " MKTAYIAKQRQ ISFVKSHFSRQ\n", +} +_QUERIES = {"q1": "MKTAYIAKQRQISFVKSHFSRQLEERLGLIEVQ", "q2": "ACDEFGHIKLMNPQRSTVWY", "q3": "GGGGWGGGG", + "q4": "", "q5": "BZX*MKTAYIAKQ", "q6": "mktayJakqrqisfvkshfsrqUO"} +_FIELDS = ["query", "target", "cigar", "qaln", "taln", "qseq", "tseq", "evalue", "bits"] + + +def _expected(gap=(11, 1), comp_bias_scale=None, alp=None, lam=None, k=None, db_residues=None): + """align_candidates' result built from the biotite reference, pair by pair: the empty alignments dropped, sorted + by bits (else raw score), then embedding score, descending (filters off).""" + out = {} + for q, qseq in _QUERIES.items(): + hits = [] + for i, (sid, sseq) in enumerate(_STORE.items()): + m = ref.reference_metrics(qseq, sseq, gap=gap, comp_bias_scale=comp_bias_scale, needs=ALL_NEEDS, + lam=lam, k=k, alp=alp, db_residues=db_residues) + if m is not None and m.aln_len > 0: + hits.append((sid, 1.0 - 0.01 * i, m)) + hits.sort(key=lambda h: (h[2].bit_score if h[2].bit_score is not None else h[2].score, h[1]), reverse=True) + out[q] = [(sid, emb, dataclasses.astuple(m)) for sid, emb, m in hits] + return out + + +def _run(**kw): + pre = {q: (list(_STORE), [1.0 - 0.01 * i for i in range(len(_STORE))]) for q in _QUERIES} + res = alignment.align_candidates( + _QUERIES, pre, lambda ids: {i: _STORE[i] for i in ids}, min_seq_identity=0.0, min_coverage=0.0, + max_evalue=None, subject_db_size=10_000_000, output_fields=_FIELDS, **kw) + return {q: [(h.subject_id, h.emb_score, dataclasses.astuple(h.metrics)) for h in hits] for q, hits in res.items()} + + +@pytest.mark.usefixtures("fmsw_libs") +class TestStage2AgainstBiotite(unittest.TestCase): + """The Stage 2 on the kernel against the biotite path it replaced (tests/blosum62_reference.py): every + AlignmentMetrics field, every heavy string, the significance columns, the order.""" + + def tearDown(self): + alignment._worker_init(11, 1, None, None, None) + + def test_align_candidates_equals_the_biotite_path(self): + alp = karlin_altschul.params_for(11, 1) + for workers in (1, 2): + with self.subTest(workers=workers): + self.assertEqual(_run(num_workers=workers), _expected(alp=alp, db_residues=10_000_000)) + self.assertEqual(_run(num_workers=workers, comp_bias_corr=True), + _expected(comp_bias_scale=1.0, alp=alp, db_residues=10_000_000)) + self.assertEqual(_run(num_workers=workers, comp_bias_corr=True, comp_bias_corr_scale=0.5, + compute_significance=False), + _expected(comp_bias_scale=0.5)) + self.assertEqual(_run(num_workers=workers, gap_open=-11, gap_extend=-1, compute_significance=False), + _expected()) + + def test_sampled_significance_equals_the_biotite_path(self): + kw = dict(gap_open=5, gap_extend=2, significance_mode="sampled", significance_sample_size=20, + significance_sample_length=50) + lam, k = alignment._estimate_lambda_k(5, 2, 20, 50, 0) + self.assertEqual(_run(num_workers=1, **kw), _expected(gap=(5, 2), lam=lam, k=k, db_residues=10_000_000)) + + def test_worker_metrics_equal_the_biotite_path(self): + """``_align_query`` (one kernel call for a task) against the biotite path pair by pair, on random and related + pairs, at several gap settings, with and without the composition bias and significance.""" + rng = random.Random(11) + query = "".join(rng.choice(AA) for _ in range(150)) + cands = [(f"t{i}", "".join(rng.choice(AA + "BZX*") for _ in range(rng.randint(1, 300))), 0.5) + for i in range(30)] + cands += [("self", query, 0.9), ("part", query[40:120], 0.8), ("mut", query[:60] + "W" * 7 + query[60:], 0.7), + ("empty", "", 0.1)] + alp = karlin_altschul.params_for(11, 1) + settings = [dict(gap=(11, 1), alp=alp, db_residues=10_169_347_728), + dict(gap=(11, 1), comp_bias_scale=1.0, alp=alp, db_residues=10_169_347_728), + dict(gap=(11, 1), comp_bias_scale=0.5), + dict(gap=(5, 2), lam=0.3, k=0.05, db_residues=1_000_000), + dict(gap=(1, 1)), dict(gap=(3, 3)), dict(gap=(20, 5)), dict(gap=(2, 0))] + for s in settings: + with self.subTest(**{k: v for k, v in s.items() if k != "alp"}): + alignment._worker_init(s["gap"][0], s["gap"][1], s.get("lam"), s.get("k"), s.get("db_residues"), + s.get("alp"), ALL_NEEDS, s.get("comp_bias_scale")) + qid, hits = alignment._align_query(("q", query, cands)) + want = [ref.reference_metrics(query, seq, needs=ALL_NEEDS, **s) for _, seq, _ in cands] + self.assertEqual(qid, "q") + self.assertEqual([h.subject_id for h in hits], [c[0] for c, w in zip(cands, want) if w is not None]) + self.assertEqual([h.metrics for h in hits], [w for w in want if w is not None]) diff --git a/tests/test_swkern.py b/tests/test_swkern.py new file mode 100644 index 0000000..f8ee9cb --- /dev/null +++ b/tests/test_swkern.py @@ -0,0 +1,1235 @@ +"""Native Smith-Waterman kernel (fmsw, ABI 1) and its ctypes wrapper (PLAN section 7, P3.5 tests 1-9). + +The libraries come from the session fixture ``fmsw_libs`` (conftest.py: developer build in place when missing or +stale). Tolerances are the S16 contract: S_f / S_r within 5e-5 max(|ref|, 1); paths identical unless the float64 +oracle's best-cell or traceback margin is below 1e-4 + 2e-6 |S_f|; Q90 and ln E within 1e-4. + +The same library's integer DP (``fmswi_align``, the BLOSUM62 Stage 2) is checked here by its golden digest, without +biotite, so the kernel jobs and the sanitizer job run it on every ISA (section 3b); tests/test_swint.py checks it +against biotite. +""" +import ctypes +import functools +import hashlib +import json +import os +import resource +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest + +import foldmatch +from foldmatch import _native +from foldmatch._native import build as native_build +from foldmatch.search import swint, swkern + +import esm_codes_reference as ref + +_RES = Path(__file__).parent / "resources" / "esm_codes" +D = swkern.SETTING_D + +# sha256 of the DP golden records (swkern.dp_digest_update) of F1 (50 pairs, pairs_v1.npz order) followed by +# swkern.golden_pairs(1000), from q fp16 through fmsw_sim_ref, setting D. Computed 2026-10-13 on an Apple M1 Max: +# Apple clang 21 neon / generic (arm64), sse2 / generic (x86_64 under Rosetta) and GCC 14 neon / generic all agree. +GOLDEN_F1_SYNTH_SHA256 = "30ec40c6d57f9b7d28c2502b7a40b5ec8e97d46b83ce69d97fbcd44b2fe281e3" +# sha256 of the golden_pairs(1000) input bytes (kind, q fp16, codes): the generator is integer-only and portable +GOLDEN_INPUTS_SHA256 = "6725eda9e247c975f722e85fdefd1d71dbd74c0d2b28ee4c016c2027b634955d" +# sha256 of swint.golden_cases()'s input records (swint.golden_digest_update in mode 0 with all-zero outputs) +FMSWI_GOLDEN_INPUTS_SHA256 = "3b9907272b7e9aad2363b870d4d365fce5a3d59aad84e77dbdf02246619f8f6e" + +_AA = np.frombuffer(b"ACDEFGHIKLMNPQRSTVWY", np.uint8) + + +# ===================================================================================================================== +# data +# ===================================================================================================================== +@functools.lru_cache(maxsize=None) +def _f1(): + z = np.load(_RES / "pairs_v1.npz") + qo, to = z["q_offsets"], z["t_offsets"] + pairs = [] + for p in range(len(z["pair_q"])): + a, b = int(z["pair_q"][p]), int(z["pair_t"][p]) + pairs.append(dict(q16=z["q"][qo[a]:qo[a + 1]], codes=z["codes"][to[b]:to[b + 1]], + qseq=str(z["q_sequences"][a]), tseq=str(z["t_sequences"][b]), + S_f=float(z["S_f"][p]), S_r=float(z["S_r"][p]), S_f_f64=float(z["S_f_f64"][p]), + S_r_f64=float(z["S_r_f64"][p]), end=tuple(int(x) for x in z["end_1based"][p]), + path=z["path_flat"][z["path_off"][p]:z["path_off"][p + 1]], + n_aligned=int(z["n_aligned"][p]), aln_cols=int(z["aln_cols"][p]), + n_ident=int(z["n_ident"][p]))) + return pairs + + +@functools.lru_cache(maxsize=None) +def _golden_pairs(): + return swkern.golden_pairs() + + +@functools.lru_cache(maxsize=None) +def _evalue_table(): + t = json.loads((_RES / "evalue_grid_v1.json").read_text())["table"] + return t["center_len"], t["lam"], t["lnK"] + + +def _random_seq(rng, n): + return _AA[rng.integers(0, len(_AA), n)].tobytes().decode("ascii") + + +def _related_codes(rng, q16): + """Target codes = sign bits of a noisy, gapped copy of a query segment with random flanks.""" + q = q16.astype(np.float32) + n = len(q) + a, b = sorted(int(x) for x in rng.integers(0, n, 2)) + seg = q[a:b + 1] + seg = seg[rng.random(len(seg)) > 0.1] + ins = rng.standard_normal((int(rng.integers(0, 10)), 64)).astype(np.float32) + pos = int(rng.integers(0, len(seg) + 1)) + t = np.concatenate([rng.standard_normal((int(rng.integers(0, 60)), 64)).astype(np.float32), seg[:pos], ins, + seg[pos:], rng.standard_normal((int(rng.integers(0, 60)), 64)).astype(np.float32)]) + if not len(t): + t = rng.standard_normal((1, 64)).astype(np.float32) + t = t + float(rng.uniform(0.3, 1.5)) * rng.standard_normal(t.shape).astype(np.float32) + return np.packbits(t >= 0, axis=1) + + +@functools.lru_cache(maxsize=None) +def _property_cases(seed=20261013, queries=50, per_query=10): + """500 seeded cases: 50 queries (n in [1, 400]) x 10 candidates (m in [1, 400] random, or related).""" + rng = np.random.default_rng(seed) + cases = [] + for _ in range(queries): + n = int(rng.integers(1, 401)) + q16 = rng.standard_normal((n, 64)).astype(np.float16) + cands = [] + for _ in range(per_query): + if rng.random() < 0.5: + codes = rng.integers(0, 256, (int(rng.integers(1, 401)), 8), dtype=np.uint8) + else: + codes = _related_codes(rng, q16) + cands.append(codes) + lens = [len(c) for c in cands] + offsets = np.concatenate([[0], np.cumsum(lens)]).astype(np.int64) + cases.append(dict(q16=q16, codes=np.concatenate(cands), offsets=offsets, qseq=_random_seq(rng, n), + tseqs=[_random_seq(rng, m) for m in lens])) + return cases + + +def _kernels(): + """Every kernel build of this machine that can run here (avx2 only on an AVX2 CPU).""" + out = [] + for isa in _native.machine_isas(): + if _native.find_library(isa) is None: + continue + try: + out.append(swkern.load(isa)) + except swkern.KernelError: + if isa != "avx2": + raise + return out + + +def _subprocess_env(): + env = dict(os.environ) + src = str(Path(foldmatch.__file__).resolve().parents[1]) + env["PYTHONPATH"] = src + os.pathsep + env.get("PYTHONPATH", "") + return env + + +# ===================================================================================================================== +# shared checks +# ===================================================================================================================== +def _assert_s16_score(tc, got, want, what=""): + got, want = np.asarray(got, np.float64), np.asarray(want, np.float64) + bound = swkern.s16_score_bound(want) + bad = np.flatnonzero(~(np.abs(got - want) <= bound)) + tc.assertEqual(len(bad), 0, f"{what}: outside S16 at {bad[:5]}: got {got[bad[:5]]} want {want[bad[:5]]}") + + +def _assert_path_invariants(tc, path, end, score, n, m): + path = np.asarray(path) + if len(path) == 0: + tc.assertEqual(score, 0.0) + tc.assertEqual(tuple(int(x) for x in end), (0, 0)) + return + tc.assertGreater(score, 0.0) + tc.assertTrue(np.all(path[0] >= 0) and np.all(path[-1] >= 0), "both path ends must be aligned pairs") + tc.assertEqual((int(path[-1, 0]) + 1, int(path[-1, 1]) + 1), tuple(int(x) for x in end)) + tc.assertFalse(np.any((path[:, 0] < 0) & (path[:, 1] < 0)), "double-gap column") + qi, tj = path[:, 0][path[:, 0] >= 0], path[:, 1][path[:, 1] >= 0] + tc.assertTrue(np.all(np.diff(qi) == 1) and np.all(np.diff(tj) == 1), "coordinates must be monotone, step 1") + tc.assertTrue(qi.max() < n and tj.max() < m) + + +def _assert_cigar_round_trip(tc, path, qseq, tseq): + pm = swkern.path_metrics(path, qseq, tseq, strings=True) + ops = swkern.path_ops(path).tobytes().decode("ascii") + tc.assertEqual(swkern.expand_cigar(pm.cigar), ops) + nM, nI, nD = ops.count("M"), ops.count("I"), ops.count("D") + tc.assertEqual(nM + nI + nD, pm.aln_len) + tc.assertEqual(pm.aln_len, len(path)) + naive_ident = sum(1 for i, j in np.asarray(path).tolist() if i >= 0 and j >= 0 and qseq[i] == tseq[j]) + tc.assertEqual(pm.n_ident, naive_ident) + tc.assertEqual(pm.mismatch, nM - naive_ident) + if not len(path): + tc.assertEqual((pm.cigar, pm.q_aln, pm.t_aln, pm.q_start, pm.q_end, pm.t_start, pm.t_end), + ("", "", "", 0, 0, 0, 0)) + return + tc.assertEqual(pm.q_aln.replace("-", ""), qseq[pm.q_start - 1:pm.q_end]) + tc.assertEqual(pm.t_aln.replace("-", ""), tseq[pm.t_start - 1:pm.t_end]) + tc.assertEqual(pm.q_end - pm.q_start + 1, nM + nI) + tc.assertEqual(pm.t_end - pm.t_start + 1, nM + nD) + tc.assertEqual((pm.q_aligned, pm.t_aligned), (nM + nI, nM + nD)) + runs = sum(1 for a, b in zip(" " + ops, ops) if b in "ID" and a != b) + tc.assertEqual(pm.gap_open, runs) + + +def _golden_runs(kernel, reference_sim): + """(F1 + synthetic) PairRun list in golden order.""" + pairs = [(p["q16"], p["codes"]) for p in _f1()] + [(q, c) for _, q, c in _golden_pairs()] + return [swkern.run_pair(q, c, reference_sim=reference_sim, kernel=kernel) for q, c in pairs], pairs + + +def _digests(runs, pairs): + hall, hsyn = swkern.dp_golden_new(), swkern.dp_golden_new() + for i, (r, (q, c)) in enumerate(zip(runs, pairs)): + swkern.dp_digest_update(hall, len(q), len(c), r.s_f, r.s_r, r.path) + if i >= len(_f1()): + swkern.dp_digest_update(hsyn, len(q), len(c), r.s_f, r.s_r, r.path) + return hall.hexdigest(), hsyn.hexdigest() + + +# ===================================================================================================================== +# 1. differential F1 (and the P3.6 oracle) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class DifferentialF1Test(unittest.TestCase): + + def test_production_path_matches_research_within_s16(self): + for k in _kernels(): + sf, sr, paths = [], [], 0 + for p in _f1(): + r = swkern.run_pair(p["q16"], p["codes"], reference_sim=False, kernel=k) + sf.append(r.s_f) + sr.append(r.s_r) + paths += int(np.array_equal(r.path, p["path"])) + self.assertEqual(r.end, p["end"]) + self.assertEqual(r.s_tb, r.s_f) + self.assertEqual(paths, 50, k.isa) + _assert_s16_score(self, sf, [p["S_f"] for p in _f1()], f"{k.isa} S_f vs research") + _assert_s16_score(self, sr, [p["S_r"] for p in _f1()], f"{k.isa} S_r vs research") + _assert_s16_score(self, sf, [p["S_f_f64"] for p in _f1()], f"{k.isa} S_f vs float64 oracle") + _assert_s16_score(self, sr, [p["S_r_f64"] for p in _f1()], f"{k.isa} S_r vs float64 oracle") + + def test_batched_f1_targets_equal_single_pairs(self): + """One query against all 50 F1 targets in one call: the pair (k, k) equals the single-pair run bit for bit.""" + f1 = _f1() + codes = np.concatenate([p["codes"] for p in f1]) + offsets = np.concatenate([[0], np.cumsum([len(p["codes"]) for p in f1])]).astype(np.int64) + for a in (0, 17, 49): + qp = swkern.query_prep(f1[a]["q16"]) + res = swkern.score_fwd_rev(qp, codes, offsets, D) + single = swkern.run_pair(f1[a]["q16"], f1[a]["codes"], reference_sim=False) + self.assertEqual(res.s_fwd[a], np.float32(single.s_f)) + self.assertEqual(res.s_rev[a], np.float32(single.s_r)) + + def test_cigar_and_metrics_match_fixture(self): + for p in _f1(): + r = swkern.run_pair(p["q16"], p["codes"], reference_sim=False) + pm = swkern.path_metrics(r.path, p["qseq"], p["tseq"], strings=True) + self.assertEqual((pm.aln_len, pm.n_ident, pm.mismatch + pm.n_ident), + (p["aln_cols"], p["n_ident"], p["n_aligned"])) + _assert_cigar_round_trip(self, r.path, p["qseq"], p["tseq"]) + + +@pytest.mark.usefixtures("fmsw_libs") +class OracleTest(unittest.TestCase): + + def test_verbatim_copy(self): + self.assertTrue(ref.check_verbatim()) + self.assertEqual(ref.ORACLE_SOURCE_MD5, "7ec19b7e5ff89dc7b44d2512aed3997e") + + def test_s7_wrapper_changes_nothing_on_f1(self): + for p in _f1(): + Q, T = ref.reference_qprep64(p["q16"]), ref.decode64(p["codes"]) + for Qd in (Q, Q[::-1]): + sim = Qd @ T.T + z7, nr, nc = ref.reference_rowcol_s7(sim) + self.assertEqual((nr, nc), (0, 0)) + self.assertTrue(np.array_equal(z7, ref.normalize_scores(sim, "rowcol"))) + + def test_oracle_reproduces_fixture_and_margin_helper_matches_oracle(self): + for p in _f1()[::7]: + r = ref.reference_pair64(p["q16"], p["codes"], margins=True) + self.assertAlmostEqual(r["S_f"], p["S_f_f64"], delta=1e-9 * max(abs(p["S_f_f64"]), 1.0)) + self.assertTrue(np.array_equal(r["path"], p["path"])) + mg = ref.sw_margins64(r["z"] - D.offset, D.gap_open, D.gap_extend) + self.assertEqual(mg["score"], r["S_f"]) + self.assertTrue(np.array_equal(mg["path"], r["path"])) + self.assertGreater(min(mg["best_margin"], mg["tb_margin"]), 0.0) + + def test_reference_evalue_reproduces_f4(self): + rows = json.loads((_RES / "evalue_grid_v1.json").read_text())["rows"] + centres, lam, lnK = _evalue_table() + for r in rows: + lnE, bits = ref.reference_evalue(r["S_prime"], r["m"], r["N"], centres, lam, lnK) + self.assertLessEqual(abs(lnE - r["lnE"]), 1e-12 * max(abs(r["lnE"]), 1.0)) + self.assertLessEqual(abs(bits - r["bits"]), 1e-12 * max(abs(r["bits"]), 1.0)) + + def test_path_score_helper_reproduces_the_oracle_score(self): + for p in _f1()[::5]: + r = ref.reference_pair64(p["q16"], p["codes"]) + self.assertAlmostEqual(swkern.path_score(r["z"], r["path"], D), r["S_f"], delta=1e-9 * abs(r["S_f"])) + + +# ===================================================================================================================== +# 2. properties (500 seeded cases) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class PropertyTest(unittest.TestCase): + + def test_scores_paths_invariants_and_round_trips(self): + near_ties = compared = 0 + for c in _property_cases(): + qp = swkern.query_prep(c["q16"]) + n = len(qp) + res = swkern.score_fwd_rev(qp, c["codes"], c["offsets"], D) + aln = swkern.align(qp, c["codes"], c["offsets"], D) + aln_score_only = swkern.align(qp, c["codes"], c["offsets"], D, traceback=False) + self.assertTrue(np.all(np.isfinite(res.s_fwd)) and np.all(np.isfinite(res.s_rev))) + self.assertTrue(np.all(res.s_fwd >= 0) and np.all(res.s_rev >= 0)) + self.assertTrue(np.array_equal(aln.scores.view(np.int32), res.s_fwd.view(np.int32))) + self.assertTrue(np.array_equal(aln_score_only.scores.view(np.int32), res.s_fwd.view(np.int32))) + self.assertTrue(np.array_equal(aln_score_only.ends, aln.ends)) + self.assertFalse(np.any(np.signbit(res.s_fwd)) or np.any(np.signbit(res.s_rev)), "-0 not canonicalised") + for k in range(len(c["offsets"]) - 1): + codes_k = c["codes"][c["offsets"][k]:c["offsets"][k + 1]] + path = aln.path_of(k) + _assert_path_invariants(self, path, aln.ends[k], float(aln.scores[k]), n, len(codes_k)) + _assert_cigar_round_trip(self, path, c["qseq"], c["tseqs"][k]) + o = ref.reference_pair64(c["q16"], codes_k, margins=True) + s_f = float(aln.scores[k]) + self.assertLessEqual(abs(swkern.path_score(o["z"], path, D) - s_f), 5e-5 * max(abs(s_f), 1.0)) + margin = min(o["best_margin"], o["tb_margin"]) + if margin > max(1e-3, swkern.s16_path_margin(o["S_f"])): + compared += 1 + self.assertTrue(np.array_equal(path, o["path"]), f"path differs at margin {margin}") + else: + near_ties += 1 + self.assertGreater(compared, 400) + + def test_batch_order_and_block_size_invariance(self): + rng = np.random.default_rng(7) + for c in _property_cases()[::5]: + qp = swkern.query_prep(c["q16"]) + base = swkern.score_fwd_rev(qp, c["codes"], c["offsets"], D) + base_aln = swkern.align(qp, c["codes"], c["offsets"], D) + for bf in (1 << 12, 1 << 17, 1 << 20): + r = swkern.score_fwd_rev(qp, c["codes"], c["offsets"], D, block_floats=bf) + a = swkern.align(qp, c["codes"], c["offsets"], D, block_floats=bf) + self.assertTrue(np.array_equal(r.s_fwd.view(np.int32), base.s_fwd.view(np.int32)), bf) + self.assertTrue(np.array_equal(r.s_rev.view(np.int32), base.s_rev.view(np.int32)), bf) + self.assertTrue(np.array_equal(a.path, base_aln.path) and np.array_equal(a.path_off, base_aln.path_off)) + K = len(c["offsets"]) - 1 + perm = rng.permutation(K) + starts, lens = c["offsets"][:-1][perm], np.diff(c["offsets"])[perm] + r = swkern.score_fwd_rev(qp, c["codes"], starts, D, lengths=lens) # non-contiguous candidate rows + a = swkern.align(qp, c["codes"], starts, D, lengths=lens) + self.assertTrue(np.array_equal(r.s_fwd.view(np.int32), base.s_fwd[perm].view(np.int32))) + self.assertTrue(np.array_equal(r.s_rev.view(np.int32), base.s_rev[perm].view(np.int32))) + for i, k in enumerate(perm): + self.assertTrue(np.array_equal(a.path_of(i), base_aln.path_of(k))) + + def test_reversed_view_equals_reversed_query(self): + for c in _property_cases()[::3]: + qp = swkern.query_prep(c["q16"]) + qr = np.ascontiguousarray(qp[::-1]) + ref_view = swkern.score_fwd_rev(qp, c["codes"], c["offsets"], D, reference_sim=True) + ref_qrev = swkern.score_fwd_rev(qr, c["codes"], c["offsets"], D, reference_sim=True) + self.assertTrue(np.array_equal(ref_view.s_rev.view(np.int32), ref_qrev.s_fwd.view(np.int32))) + blas_view = swkern.score_fwd_rev(qp, c["codes"], c["offsets"], D) + blas_qrev = swkern.score_fwd_rev(qr, c["codes"], c["offsets"], D) + _assert_s16_score(self, blas_view.s_rev, blas_qrev.s_fwd, "BLAS reversed view") + _assert_s16_score(self, blas_view.s_rev, ref_view.s_rev, "BLAS vs reference S_r") + + +# ===================================================================================================================== +# 3. DP golden hash +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class GoldenHashTest(unittest.TestCase): + + def test_golden_inputs_are_stable(self): + h = hashlib.sha256() + for kind, q, c in _golden_pairs(): + h.update(kind.encode() + b"\0") + h.update(np.ascontiguousarray(q, " 0)).sum()) + wide_gapped += int((w & (r.out[:, swint.GAPOPEN] > 0)).sum()) + empty += int((r.out[:, swint.ALNLEN] == 0).sum()) + for k in range(len(c.targets)): + path = r.path(k) + for op in runs: + runs[op] = max([runs[op]] + [len(x) for x in path.split("M") if x.startswith(op)]) + self.assertEqual(wide, 10) + self.assertGreater(narrow, 400) + self.assertGreaterEqual(wide_gapped, 3) + self.assertGreaterEqual(empty, 2) + self.assertGreaterEqual(min(runs.values()), 60) + self.assertEqual(sum(1 for c in cases if c.kind == "edge" and any(len(t) == 0 for t in c.targets)), 1) + # the limit: column 2,978's best is 32,757 = 32768 - 11, so the pair ends in 32 bits at 32,768 + limit = [c for c in cases if c.kind == "wide" and len(c.qcodes) == 2979] + self.assertEqual(len(limit), 1) + out = swint.run_golden(limit[0], 0).out[0] + self.assertEqual((out[swint.SCORE], out[swint.TEND], out[swint.WIDE]), (32768, 2979, 1)) + # the floor: one code at -40,000 (+25,536 if it wrapped to 16 bits) and no score above 11, so 32 bits from the + # start at a score far below the limit + floor = [c for c in cases if c.qprof.min() < -32768] + self.assertEqual([(c.kind, int(c.qprof.max())) for c in floor], [("wide", 11)]) + self.assertIn(int(floor[0].qprof.argmin() % floor[0].qprof.shape[1]), floor[0].targets[0]) + out = swint.run_golden(floor[0], 0).out[0] + self.assertEqual((out[swint.SCORE], out[swint.WIDE]), (750, 1)) + # the carry: 1000/1000, a 128-row lane with 8 lanes (n = 1024) and with 16 (n = 2048); 16 bits, 32,400 + carry = [c for c in cases if c.kind == "carry"] + self.assertEqual(sorted((len(c.qcodes), c.gap_open, c.gap_extend) for c in carry), + [(1024, 1000, 1000), (2048, 1000, 1000)]) + for c in carry: + out = swint.run_golden(c, 1).out[0] + self.assertEqual(tuple(out[[swint.SCORE, swint.QEND, swint.TEND, swint.WIDE]]), (32400, 384, 201, 0)) + + def test_golden_inputs_are_stable(self): + """The generator is integer-only (splitmix64), so the inputs are the same bytes everywhere: a changed digest + means changed outputs.""" + h = hashlib.sha256() + for c in swint.golden_cases(): + swint.golden_digest_update(h, c, 0, np.zeros((len(c.targets), swint.NOUT), np.int32), None, None) + self.assertEqual(h.hexdigest(), FMSWI_GOLDEN_INPUTS_SHA256) + + def test_a_changed_result_fails_the_selftest(self): + real = swint.run_golden + + def off_by_one(case, mode, kernel=None): + r = real(case, mode, kernel) + if case.kind == "wide" and mode: # the same in both traceback modes + r.out[0, swint.NIDENT] += 1 + return r + + def modes_disagree(case, mode, kernel=None): + r = real(case, mode, kernel) + if mode == 2 and case.kind == "related": + r.out[0, swint.QSTART] += 1 + return r + + for fake, message in ((off_by_one, "golden digest"), (modes_disagree, "traceback modes disagree")): + with self.subTest(message), mock.patch.object(swint, "run_golden", fake): + with self.assertRaisesRegex(swkern.KernelError, message): + swint.selftest() + + +@pytest.mark.usefixtures("fmsw_libs") +class FmswiArgumentsTest(unittest.TestCase): + """fmswi_align's refusals (status 2: bad arguments, or a score beyond int32; status 1: an ops buffer too small for + the next pair) and its empty inputs, on every ISA library, called directly (swint.align_batch checks nothing and + always sizes the ops buffer itself). No biotite, so the kernel jobs and the sanitizer job run them.""" + + A = 24 + + def setUp(self): + rng = np.random.default_rng(7) + self.q = rng.integers(0, self.A, 30).astype(np.uint8) + self.prof = rng.integers(-4, 12, (30, self.A)).astype(np.int32) + self.t = rng.integers(0, self.A, 70).astype(np.uint8) + self.kernels = [swint.IntKernel(k) for k in _kernels()] + + def call(self, ik, *, q=None, prof=None, t=None, off=(0, 40), lens=(40, 30), go=11, ge=1, mode=1, cap=None, + n=None, A=None, K=None, ops=True): + """One raw fmswi_align call; ``n``, ``A`` and ``K`` default to the arrays' sizes and ``cap`` to the bytes + align_batch would give. Returns (status, out, ops_off).""" + q = self.q if q is None else q + prof = self.prof if prof is None else prof + t = self.t if t is None else t + off, lens = np.asarray(off, np.int64), np.asarray(lens, np.int32) + cap = len(lens) * len(q) + int(lens.sum()) if cap is None else cap + buf = ctypes.create_string_buffer(max(cap, 1)) if ops else None + out = np.full((max(len(lens), 1), swint.NOUT), -7, np.int32) + ops_off = np.full(len(lens) + 1, -7, np.int64) + st = ik.lib.fmswi_align(q.ctypes.data, prof.ctypes.data, len(q) if n is None else n, + prof.shape[1] if A is None else A, t.ctypes.data, off.ctypes.data, lens.ctypes.data, + len(lens) if K is None else K, go, ge, mode, out.ctypes.data, buf, cap, + ops_off.ctypes.data) + return st, out, ops_off + + def test_bad_arguments_return_status_2(self): + big, bad_q, bad_t = self.prof.copy(), self.q.copy(), self.t.copy() + bad_q[5] = self.A + bad_t[50] = self.A # in the second target + cases = {"code >= A in the query": dict(q=bad_q), "code >= A in a target": dict(t=bad_t), + "A > 255": dict(prof=np.zeros((30, 256), np.int32)), "A < 1": dict(A=0), "n < 0": dict(n=-1), + "K < 0": dict(K=-1), "m < 0": dict(lens=(40, -1)), "t_off < 0": dict(off=(0, -1)), + "traceback 3": dict(mode=3), "traceback -1": dict(mode=-1)} + for score in (swint.MAX_SCORE + 1, -swint.MAX_SCORE - 1): + big[3, 4] = score + cases[f"score {score}"] = dict(prof=big.copy()) + for go, ge in ((0, 1), (-1, 1), (swint.MAX_GAP + 1, 1), (11, -1), (11, swint.MAX_GAP + 1)): + cases[f"gaps {go}/{ge}"] = dict(go=go, ge=ge) + # a score beyond int32: 2,100 x 2^20 on the diagonal (refused after the 32-bit pass, in every mode) + n = 2100 + cases["a score beyond int32"] = dict(q=np.zeros(n, np.uint8), prof=np.full((n, self.A), swint.MAX_SCORE, + np.int32), + t=np.zeros(n, np.uint8), off=(0,), lens=(n,), mode=0) + for ik in self.kernels: + self.assertEqual(self.call(ik)[0], swkern.STATUS_OK, ik) + edge = np.full((30, self.A), swint.MAX_SCORE, np.int32) + edge[:, 0] = -swint.MAX_SCORE + self.assertEqual(self.call(ik, prof=edge, go=swint.MAX_GAP, ge=swint.MAX_GAP)[0], swkern.STATUS_OK, ik) + self.assertEqual(self.call(ik, go=1, ge=0)[0], swkern.STATUS_OK, ik) + for what, kw in cases.items(): + with self.subTest(isa=ik.isa, case=what): + self.assertEqual(self.call(ik, **kw)[0], swkern.STATUS_ARGS) + + def test_short_ops_buffer_returns_status_1(self): + """Before each pair the kernel needs ops_off[k] + n + m[k] bytes of ops; without ops it needs none.""" + for ik in self.kernels: + for mode in (1, 2): + with self.subTest(isa=ik.isa, traceback=mode): + one = dict(off=(0,), lens=(70,), mode=mode) + self.assertEqual(self.call(ik, cap=30 + 70, **one)[0], swkern.STATUS_OK) + self.assertEqual(self.call(ik, cap=30 + 70 - 1, **one)[0], swkern.STATUS_WORKSPACE) + st, _, ops_off = self.call(ik, mode=mode) + self.assertEqual(st, swkern.STATUS_OK) + cap = int(ops_off[1]) + 30 + 30 - 1 # room for pair 0 and its path, not for pair 1 + self.assertGreaterEqual(cap, 30 + 40) + self.assertEqual(self.call(ik, cap=cap, mode=mode)[0], swkern.STATUS_WORKSPACE) + self.assertEqual(self.call(ik, cap=0, ops=False, mode=mode)[0], swkern.STATUS_OK) + + def test_empty_inputs(self): + for ik in self.kernels: + with self.subTest(isa=ik.isa): + st, out, ops_off = self.call(ik, lens=(0, 0)) # all-empty targets + self.assertEqual(st, swkern.STATUS_OK) + self.assertFalse(out.any()) + self.assertEqual(ops_off.tolist(), [0, 0, 0]) + st, out, ops_off = self.call(ik, n=0) # an empty query + self.assertEqual(st, swkern.STATUS_OK) + self.assertFalse(out.any()) + self.assertEqual(ops_off.tolist(), [0, 0, 0]) + st, out, _ = self.call(ik, K=0) # no target: nothing written + self.assertEqual(st, swkern.STATUS_OK) + self.assertTrue((out == -7).all()) + r = swint.align_batch(self.q, self.prof, self.t, [0, 0, 40], [40, 0, 30], 11, 1, ops=True, kernel=ik) + self.assertEqual(r.out[1].tolist(), [0] * swint.NOUT) # an empty target among others + self.assertEqual(r.path(1), "") + + +@pytest.mark.usefixtures("fmsw_libs") +class SelftestCommandTest(unittest.TestCase): + + def test_a_failed_fmswi_check_is_reported(self): + """``python -m foldmatch.search.swkern`` reports a failed fmswi golden check as a failed selftest (exit status + 1, no traceback): swint raises the KernelError of the package module, not that of ``__main__``.""" + code = ("import runpy\n" + "from foldmatch.search import swint\n" + "swint.GOLDEN_SHA256 = '0' * 64\n" + "runpy.run_module('foldmatch.search.swkern', run_name='__main__', alter_sys=True)\n") + out = subprocess.run([sys.executable, "-W", "ignore::RuntimeWarning", "-c", code], env=_subprocess_env(), + capture_output=True, text=True) + self.assertEqual(out.returncode, 1, out.stderr) + self.assertIn("fmsw selftest failed: fmswi selftest: golden digest", out.stderr) + self.assertNotIn("Traceback", out.stderr) + + +# ===================================================================================================================== +# 4. BLAS conformance (S16) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class BlasConformanceTest(unittest.TestCase): + + def test_production_vs_golden_on_the_golden_pairs(self): + k = swkern.load() + gold, pairs = _golden_runs(k, reference_sim=True) + prod, _ = _golden_runs(k, reference_sim=False) + _assert_s16_score(self, [r.s_f for r in prod], [r.s_f for r in gold], "S_f") + _assert_s16_score(self, [r.s_r for r in prod], [r.s_r for r in gold], "S_r") + for (q, c), g, b in zip(pairs, gold, prod): + if not np.array_equal(g.path, b.path): + z = swkern.z64_pair(q, c) + gap = abs(swkern.path_score(z, b.path, D) - swkern.path_score(z, g.path, D)) + self.assertLess(gap, swkern.s16_path_margin(g.s_f), "path change outside an S16 near-tie") + + def test_q90_lne_bits_and_decisions_within_s16(self): + f1 = _f1() + centres, lam, lnK = _evalue_table() + codes = np.concatenate([p["codes"] for p in f1]) + offsets = np.concatenate([[0], np.cumsum([len(p["codes"]) for p in f1])]).astype(np.int64) + out = {} + for mode in (True, False): + q90, lne, bits = [], [], [] + for a, p in enumerate(f1): + res = swkern.score_fwd_rev(swkern.query_prep(p["q16"]), codes, offsets, D, reference_sim=mode) + S = res.s_fwd.astype(np.float64) - res.s_rev.astype(np.float64) + q = float(np.quantile(S, 0.9)) + e, b = ref.reference_evalue(S[a] - q, len(p["q16"]), 1e9, centres, lam, lnK) + q90.append(q) + lne.append(float(e)) + bits.append(float(b)) + out[mode] = (np.array(q90), np.array(lne), np.array(bits)) + (gq, ge, gb), (bq, be, bb) = out[True], out[False] + self.assertLessEqual(float(np.max(np.abs(bq - gq))), 1e-4) + self.assertLessEqual(float(np.max(np.abs(be - ge))), 1e-4) + for t in (1e-3, 1.0, 10.0): + same = (ge <= np.log(t)) == (be <= np.log(t)) + boundary = np.abs(ge - np.log(t)) < 1e-4 + self.assertTrue(np.all(same | boundary), t) + rb_same = np.floor(gb + 0.5) == np.floor(bb + 0.5) + near = np.abs((gb + 0.5) - np.round(gb + 0.5)) < 2e-4 + self.assertTrue(np.all(rb_same | near)) + + def test_gemm_operand_and_output_layout_is_enforced(self): + p = _f1()[0] + qp = swkern.query_prep(p["q16"]) + n, m = len(qp), len(p["codes"]) + out = swkern.aligned_empty(n * m).reshape(n, m) + np.testing.assert_array_equal(swkern.similarity(qp, p["codes"], out), qp @ swkern.decode_codes(p["codes"]).T) + self.assertEqual(out.ctypes.data % 64, 0) + with self.assertRaises(TypeError): + swkern.similarity(qp.astype(np.float64), p["codes"], out) + with self.assertRaises(ValueError): + swkern.similarity(np.asfortranarray(qp), p["codes"], out) + raw = swkern.aligned_empty(n * m + 1) + with self.assertRaises(ValueError): + swkern.similarity(qp, p["codes"], raw[1:].reshape(n, m)) # 4-byte misaligned + with self.assertRaises(ValueError): + swkern.similarity(qp, p["codes"], np.empty((m, n), np.float32).T) # not C-contiguous + with self.assertRaises(TypeError): + swkern.similarity(qp, p["codes"].astype(np.int16), out) + # the kernel itself refuses a pair matrix off the 64-byte grid + k = swkern.load() + sim = swkern.aligned_empty(n * m + 16) + sim[:] = 0.1 + ws = np.empty(k.ws_floats(n, m), np.float32) + sf, sr = np.empty(1, np.float32), np.empty(1, np.float32) + mm = np.array([m], np.int32) + for off, want in ((0, 0), (16, 0), (1, 2), (4, 2)): + po = np.array([off], np.int64) + st = k.lib.fmsw_score_fwd_rev(sim.ctypes.data, n, po.ctypes.data, mm.ctypes.data, 1, 1.0, 5.0, 0.5, + sf.ctypes.data, sr.ctypes.data, ws.ctypes.data, ws.size) + self.assertEqual(st, want, off) + + def test_block_plan_keeps_every_pair_on_the_64_byte_grid(self): + n = 37 + lens = [0, 1, 5, 300, 7, 4000, 2, 0, 33] + for bf in (1 << 12, 1 << 17): + blocks = swkern.plan_blocks(n, lens, [False] * len(lens), bf) + self.assertEqual([k for b in blocks for k in range(*b)], list(range(len(lens)))) + for k0, k1 in blocks: + sizes = [-(-n * m // 16) * 16 for m in lens[k0:k1]] + self.assertTrue(k1 - k0 == 1 or sum(sizes) <= bf) + self.assertEqual(swkern.plan_blocks(n, [5, 5, 5], [False, True, False], 1 << 20), [(0, 1), (2, 3)]) + ws = swkern.Workspace() + for size in (1, 17, 1000, 123457): + self.assertEqual(ws.sim(size).ctypes.data % 64, 0) + + def test_workers_blas_threads_one(self): + try: + import threadpoolctl + except ImportError: + self.skipTest("threadpoolctl not installed") + if not [d for d in threadpoolctl.threadpool_info() if d.get("user_api") == "blas"]: + self.skipTest("threadpoolctl cannot introspect this BLAS (e.g. Accelerate)") + env = _subprocess_env() + for v in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS", "MKL_NUM_THREADS", "VECLIB_MAXIMUM_THREADS"): + env[v] = "1" + code = ("import json; from foldmatch.search import swkern; " + "print(json.dumps(swkern.blas_info()['threadpools']))") + out = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, check=True) + pools = json.loads(out.stdout.strip().splitlines()[-1]) + self.assertTrue(pools) + self.assertTrue(all(p["num_threads"] == 1 for p in pools), pools) + + def test_no_torch_or_biotite_in_the_kernel_path(self): + p = _f1()[3] + with tempfile.TemporaryDirectory() as d: + np.savez(Path(d) / "pair.npz", q16=p["q16"], codes=p["codes"]) + code = ( + "import sys; sys.modules['torch'] = None; sys.modules['biotite'] = None\n" + "import numpy as np\n" + "from foldmatch.search import swkern\n" + f"z = np.load({str(Path(d) / 'pair.npz')!r})\n" + "qp = swkern.query_prep(z['q16'])\n" + "off = np.array([0, len(z['codes'])], np.int64)\n" + "r = swkern.score_fwd_rev(qp, z['codes'], off, swkern.SETTING_D)\n" + "a = swkern.align(qp, z['codes'], off, swkern.SETTING_D)\n" + "m = swkern.path_metrics(a.path_of(0), 'A' * len(qp), 'A' * len(z['codes']), strings=True)\n" + "print(float(r.s_fwd[0]), float(a.scores[0]), m.cigar[:3])\n") + out = subprocess.run([sys.executable, "-c", code], env=_subprocess_env(), capture_output=True, text=True) + self.assertEqual(out.returncode, 0, out.stderr) + sf, sa, _ = out.stdout.split() + self.assertEqual(float(sf), float(sa)) + self.assertEqual(np.float32(float(sf)), np.float32(swkern.run_pair(p["q16"], p["codes"], + reference_sim=False).s_f)) + + +# ===================================================================================================================== +# 5. degenerate inputs (F-deg) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class DegenerateTest(unittest.TestCase): + + def test_synthetic_cases_match_the_s7_reference(self): + z = np.load(_RES / "degenerate_v1.npz") + qo, to = z["syn_q_offsets"], z["syn_t_offsets"] + for i, name in enumerate(z["syn_ids"].astype(str)): + q16, codes = z["syn_q"][qo[i]:qo[i + 1]], z["syn_codes"][to[i]:to[i + 1]] + if z["syn_expect_error"][i]: + with self.assertRaises(ValueError, msg=name): + swkern.query_prep(q16) + continue + for mode in (False, True): + r = swkern.run_pair(q16, codes, reference_sim=mode) + self.assertTrue(np.isfinite(r.s_f) and np.isfinite(r.s_r), name) + # abs <= 1e-5 on the degenerate scores (all <= 2.1); the full-size controls zero_row_q and + # subnormal_row_q (S_f ~ 322, float32 ulp 3e-5) get the same bound relative: 1e-5 max(|ref|, 1) + self.assertLessEqual(abs(r.s_f - z["syn_S_f"][i]), 1e-5 * max(abs(z["syn_S_f"][i]), 1.0), (name, mode)) + self.assertLessEqual(abs(r.s_r - z["syn_S_r"][i]), 1e-5 * max(abs(z["syn_S_r"][i]), 1.0), (name, mode)) + self.assertEqual(r.s_tb, r.s_f) + if name in ("identical_codes", "identical_q_rows", "identical_both", "n1"): + self.assertLessEqual(abs(r.s_f - r.s_r), 1e-5, name) + if name in ("identical_both", "all_zero_q"): + self.assertEqual((r.s_f, len(r.path)), (0.0, 0), name) + + def test_real_esm3_tiny_and_low_complexity_pairs(self): + z = np.load(_RES / "degenerate_v1.npz") + ro = z["real_offsets"] + items = [(z["real_q"][ro[i]:ro[i + 1]], z["real_codes"][ro[i]:ro[i + 1]]) for i in range(len(ro) - 1)] + items.append((z["base_q"], z["base_codes"])) + got_f, got_r = [], [] + for a, b in zip(z["pair_q"], z["pair_t"]): + r = swkern.run_pair(items[a][0], items[b][1], reference_sim=False) + self.assertTrue(np.isfinite(r.s_f) and np.isfinite(r.s_r)) + got_f.append(r.s_f) + got_r.append(r.s_r) + _assert_s16_score(self, got_f, z["pair_S_f"], "F-deg real S_f") + _assert_s16_score(self, got_r, z["pair_S_r"], "F-deg real S_r") + + def test_s7_threshold_on_hand_built_similarities(self): + """S7 pins the cutoff at population std < 1e-6, on both sides: a row (column) of std 5e-7 contributes exactly + 0 and one of std 2e-6 does not. Neither the golden pairs nor F1 / F-deg have a row or column with std in + [1e-7, 1e-5), so the kernel is fed hand-built similarities directly; the float64 oracle with the cutoff at + 1e-7 and at 1e-5 gives clearly different scores, so a moved cutoff cannot pass.""" + for kind in ("rows", "cols"): + sim = _s7_similarity(kind) + want = {eps: _s7_oracle(sim, eps) for eps in (1e-7, 1e-6, 1e-5)} + n_deg = want[1e-6]["ndeg"] + self.assertEqual(n_deg, (1, 0) if kind == "rows" else (0, 1), kind) + self.assertEqual(want[1e-5]["ndeg"], (2, 0) if kind == "rows" else (0, 2), kind) + for eps in (1e-7, 1e-5): + self.assertGreater(abs(want[eps]["S_f"] - want[1e-6]["S_f"]), 0.5, (kind, eps)) + for k in _kernels(): + s_f, s_r = _score_sim(k, sim) + _assert_s16_score(self, [s_f, s_r], [want[1e-6]["S_f"], want[1e-6]["S_r"]], f"S7 {kind} {k.isa}") + + def test_nan_and_inf_query_raise(self): + q = _f1()[0]["q16"].copy() + for bad in (np.nan, np.inf, -np.inf): + q2 = q.copy() + q2[3, 7] = bad + with self.assertRaises(ValueError): + swkern.query_prep(q2) + with self.assertRaises(TypeError): + swkern.query_prep(q.astype(np.float32)) + + +def _s7_similarity(kind, n=20, seed=20261014): + """[n, n] fp32 similarity with a strong diagonal (the forward alignment runs through every row and column) and + random values elsewhere, except two noise-only rows (``kind='rows'``) or columns (``'cols'``) of mean 0 and + population std 5e-7 (degenerate under S7) and 2e-6 (not degenerate), largest on their diagonal cell.""" + if kind == "cols": + return np.ascontiguousarray(_s7_similarity("rows", n, seed + 1).T) + rng = np.random.default_rng(seed) + s = 0.1 * rng.standard_normal((n, n)) + s[np.arange(n), np.arange(n)] += 1.0 + for r, sd in ((n // 3, 5e-7), (2 * n // 3, 2e-6)): + v = rng.standard_normal(n) + v[r] = np.abs(v).max() + 1.0 + s[r] = sd * (v - v.mean()) / v.std() + return s.astype(np.float32) + + +def _s7_oracle(sim, eps): + """float64 oracle S_f / S_r (setting D) of a given similarity, with the S7 cutoff at ``eps``.""" + out = {} + with mock.patch.object(ref, "S7_EPS", eps): + for d, x in (("f", sim), ("r", sim[::-1])): + z, nr, nc = ref.reference_rowcol_s7(np.asarray(x, np.float64)) + out[f"S_{d}"] = float(ref.smith_waterman(z - D.offset, D.gap_open, D.gap_extend).score) + out["ndeg"] = (nr, nc) + return out + + +def _score_sim(kernel, sim): + """fmsw_score_fwd_rev on one given [n, m] similarity (64-byte aligned copy): (S_f, S_r).""" + n, m = sim.shape + buf = swkern.aligned_empty(n * m) + buf[:] = np.asarray(sim, np.float32).ravel() + ws = np.empty(kernel.ws_floats(n, m), np.float32) + sf, sr = np.empty(1, np.float32), np.empty(1, np.float32) + po, mm = np.zeros(1, np.int64), np.array([m], np.int32) + st = kernel.lib.fmsw_score_fwd_rev(buf.ctypes.data, n, po.ctypes.data, mm.ctypes.data, 1, D.offset, D.gap_open, + D.gap_extend, sf.ctypes.data, sr.ctypes.data, ws.ctypes.data, ws.size) + assert st == 0, st + return float(sf[0]), float(sr[0]) + + +# ===================================================================================================================== +# 6. long pair and max_pair_cells +# ===================================================================================================================== +def _rss_bytes(): + r = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return r if sys.platform == "darwin" else r * 1024 + + +@pytest.mark.usefixtures("fmsw_libs") +class LongPairTest(unittest.TestCase): + + def test_oversize_pair_is_skipped_with_one_warning(self): + p = _f1()[0] + qp = swkern.query_prep(p["q16"]) + codes = np.concatenate([p["codes"], _f1()[1]["codes"], p["codes"]]) + m0, m1 = len(p["codes"]), len(_f1()[1]["codes"]) + offsets = np.array([0, m0, m0 + m1, 2 * m0 + m1], np.int64) + limit = len(qp) * m0 - 1 # pairs 0 and 2 are too large, pair 1 fits + self.assertLess(len(qp) * m1, limit) + swkern.reset_skipped_pairs() + with self.assertLogs("foldmatch.search.swkern", level="WARNING") as logs: + r = swkern.score_fwd_rev(qp, codes, offsets, D, max_pair_cells=limit) + a = swkern.align(qp, codes, offsets, D, max_pair_cells=limit) + self.assertEqual(len(logs.records), 1) + self.assertEqual(swkern.skipped_pair_count(), 4) + self.assertEqual(r.skipped.tolist(), [True, False, True]) + self.assertTrue(np.isnan(r.s_fwd[0]) and np.isnan(r.s_rev[2]) and np.isnan(a.scores[0])) + self.assertEqual(len(a.path_of(0)), 0) + single = swkern.run_pair(p["q16"], _f1()[1]["codes"], reference_sim=False) + self.assertEqual(r.s_fwd[1], np.float32(single.s_f)) + self.assertTrue(np.array_equal(a.path_of(1), single.path)) + swkern.reset_skipped_pairs() + + @unittest.skipUnless(os.environ.get("FOLDMATCH_SLOW_TESTS") == "1", + "slow: 5,000 x 5,000 pair; set FOLDMATCH_SLOW_TESTS=1 to run") + def test_5000_by_5000(self): + rng = np.random.default_rng(5000) + q16 = rng.standard_normal((5000, 64)).astype(np.float16) + t = q16.astype(np.float32)[200:4900] + 0.8 * rng.standard_normal((4700, 64)).astype(np.float32) + t = np.concatenate([rng.standard_normal((150, 64)).astype(np.float32), t, + rng.standard_normal((150, 64)).astype(np.float32)]) + codes = np.packbits(t >= 0, axis=1) + self.assertEqual(codes.shape, (5000, 8)) + off = np.array([0, 5000], np.int64) + ws = swkern.Workspace() + rss0 = _rss_bytes() + qp = swkern.query_prep(q16) + r = swkern.score_fwd_rev(qp, codes, off, D, workspace=ws) + a = swkern.align(qp, codes, off, D, workspace=ws) + grown = _rss_bytes() - rss0 + self.assertEqual(a.scores[0], r.s_fwd[0]) + self.assertGreater(float(r.s_fwd[0]), 1000.0) + path = a.path_of(0) + _assert_path_invariants(self, path, a.ends[0], float(a.scores[0]), 5000, 5000) + # 25 M cells: similarity 100 MB + flags 25 MB + workspace and path; allow 2x headroom + self.assertLess(ws.nbytes(), 160 * 2 ** 20) + self.assertLess(grown, 400 * 2 ** 20) + z = swkern.z64_pair(q16, codes) + self.assertLessEqual(abs(swkern.path_score(z, path, D) - float(a.scores[0])), + 5e-5 * float(a.scores[0])) + with self.assertLogs("foldmatch.search.swkern", level="WARNING"): + swkern.reset_skipped_pairs() + s = swkern.score_fwd_rev(qp, codes, off, D, max_pair_cells=5000 * 5000 - 1, workspace=ws) + self.assertTrue(s.skipped[0] and np.isnan(s.s_fwd[0])) + swkern.reset_skipped_pairs() + + +# ===================================================================================================================== +# 7. ABI, dispatch, missing and stale libraries +# ===================================================================================================================== +class _FakeKernel: + def __init__(self, isa, avx2): + self.isa, self._avx2 = isa, avx2 + + def cpu_has_avx2(self): + return self._avx2 + + +@pytest.mark.usefixtures("fmsw_libs") +class AbiDispatchTest(unittest.TestCase): + + def setUp(self): + self.tmp = Path(tempfile.mkdtemp(prefix="fmsw_test_")) + self.isa = _native.machine_isas()[0] + + def tearDown(self): + shutil.rmtree(self.tmp, ignore_errors=True) + + def _build_variant(self, edit): + src = self.tmp / "src" + src.mkdir() + for name in _native.SOURCE_FILES: + shutil.copy(_native.SOURCE_DIR / name, src / name) + edit(src) + out = self.tmp / "lib" + native_build.compile_library(self.isa, out, source_dir=src, quiet=True) + return out + + def test_default_dispatch_and_info(self): + k = swkern.load() + info = k.info() + self.assertEqual(set(info) >= {"isa", "abi", "source_sha256", "path"}, True) + self.assertEqual(info["abi"], swkern.ABI_VERSION) + self.assertEqual(info["source_sha256"], _native.source_sha256()) + self.assertEqual(info["dp_id"], "fmsw/1") + machine = _native.normalized_machine() + if machine == "arm64": + self.assertEqual(k.isa, "neon") + elif machine == "x86_64": + self.assertEqual(k.isa, "avx2" if swkern.load("sse2").cpu_has_avx2() else "sse2") + else: + self.assertEqual(k.isa, "generic") + + def test_isa_override(self): + with mock.patch.dict(os.environ, {swkern.ISA_ENV: "generic"}): + self.assertEqual(swkern.load().isa, "generic") + with mock.patch.dict(os.environ, {swkern.ISA_ENV: "GENERIC "}): + self.assertEqual(swkern.load().isa, "generic") + with mock.patch.dict(os.environ, {swkern.ISA_ENV: "fast"}): + with self.assertRaisesRegex(swkern.KernelError, "not a kernel ISA"): + swkern.load() + foreign = "sse2" if _native.normalized_machine() != "x86_64" else "neon" + with self.assertRaisesRegex(swkern.KernelError, "cannot run on this"): + swkern.load(foreign) + + def test_avx2_override_refused_without_avx2(self): + def fake_open(isa, dirs, check): + return _FakeKernel(isa, avx2=False) + with mock.patch.object(swkern, "_open", side_effect=fake_open): + with self.assertRaisesRegex(swkern.KernelError, "avx2 refused"): + swkern._select("avx2", "x86_64", None, True) + self.assertEqual(swkern._select(None, "x86_64", None, True).isa, "sse2") + with mock.patch.object(swkern, "_open", side_effect=lambda isa, d, c: _FakeKernel(isa, avx2=True)): + self.assertEqual(swkern._select("avx2", "x86_64", None, True).isa, "avx2") + self.assertEqual(swkern._select(None, "x86_64", None, True).isa, "avx2") + self.assertEqual(swkern._select(None, "aarch64", None, True).isa, "neon") + self.assertEqual(swkern._select(None, "riscv64", None, True).isa, "generic") + with self.assertRaisesRegex(swkern.KernelError, "cannot run on this"): + swkern._select("neon", "x86_64", None, True) + + def test_missing_library_names_the_rebuild_command(self): + with self.assertRaises(swkern.KernelError) as cm: + swkern.load(self.isa, search_dirs=[self.tmp]) + self.assertIn(_native.REBUILD_COMMAND, str(cm.exception)) + self.assertIn(_native.REINSTALL_COMMAND, str(cm.exception)) + + def test_abi_mismatch_refused(self): + def edit(src): + h = src / "fmsw.h" + h.write_text(h.read_text().replace("#define FMSW_ABI_VERSION 1", "#define FMSW_ABI_VERSION 2")) + lib = self._build_variant(edit) + with self.assertRaisesRegex(swkern.KernelError, "ABI 2"): + swkern.load(self.isa, search_dirs=[lib], check_source=False) + + def test_stale_library_refused(self): + lib = self._build_variant(lambda src: (src / "fmsw.c").write_text((src / "fmsw.c").read_text() + "\n/**/\n")) + self.assertFalse(native_build.library_is_current(lib / _native.library_filename(self.isa))) + self.assertTrue(native_build.library_is_current(_native.find_library(self.isa))) + with self.assertRaisesRegex(swkern.KernelError, "stale native kernel"): + swkern.load(self.isa, search_dirs=[lib]) + k = swkern.load(self.isa, search_dirs=[lib], check_source=False) + self.assertNotEqual(k.source_sha256, _native.source_sha256()) + + def test_not_a_kernel_library(self): + (self.tmp / _native.library_filename(self.isa)).write_bytes(b"not a library") + with self.assertRaises(swkern.KernelError): + swkern.load(self.isa, search_dirs=[self.tmp]) + + def test_x86_libraries_are_built_for_the_x86_64_baseline(self): + """Every x86_64 library gets -march=x86-64 -mtune=generic before its ISA flag, whatever the compiler's default + -march (a Cray wrapper's -march=znver3 made the sse2 library an AVX2 build); arm64 libraries get none.""" + def cmd(isa, machine=None): + return native_build.compile_command(isa, self.tmp / "lib.so", cc=sys.executable, machine=machine) + base = ["-march=x86-64", "-mtune=generic"] + for isa, machine in (("sse2", None), ("avx2", None), ("generic", "x86_64"), ("generic", "AMD64")): + c = cmd(isa, machine) + i = c.index("-march=x86-64") + self.assertEqual(c[i:i + 2], base, (isa, machine)) + if isa == "avx2": + self.assertGreater(c.index("-mavx2"), i + 1) + for isa, machine in (("neon", None), ("generic", "arm64"), ("generic", "aarch64"), ("generic", "riscv64")): + self.assertFalse([f for f in cmd(isa, machine) if f.startswith(("-march", "-mtune"))], (isa, machine)) + self.assertNotIn("-mfma", cmd("avx2")) + self.assertFalse([f for f in cmd("avx2") if f == "-march=native"]) + + def test_dp_params_validation(self): + with self.assertRaises(ValueError): + swkern.DPParams(0.1, 5.0, 0.5) # not exact in float32 + with self.assertRaises(ValueError): + swkern.DPParams(1.0, 0.5, 5.0) # gap_open < gap_extend + with self.assertRaises(ValueError): + swkern.DPParams(float("nan"), 5.0, 0.5) + self.assertEqual(swkern.SETTING_D, swkern.DPParams(1.0, 5.0, 0.5)) + + +# ===================================================================================================================== +# 8. workspace and status codes +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class WorkspaceStatusTest(unittest.TestCase): + + def setUp(self): + self.k = swkern.load() + rng = np.random.default_rng(3) + self.n, self.m = 37, 53 + self.sim = swkern.aligned_empty(self.n * self.m) + self.sim[:] = rng.standard_normal(self.n * self.m).astype(np.float32) * 0.1 + self.po = np.zeros(1, np.int64) + self.mm = np.array([self.m], np.int32) + + def _score(self, ws_len, n=None, po=None, mm=None, go=5.0, ge=0.5, offset=1.0, K=1): + n = self.n if n is None else n + ws = np.empty(max(ws_len, 1), np.float32) + sf, sr = np.empty(1, np.float32), np.empty(1, np.float32) + po = self.po if po is None else po + mm = self.mm if mm is None else mm + return self.k.lib.fmsw_score_fwd_rev(self.sim.ctypes.data, n, po.ctypes.data, mm.ctypes.data, K, offset, go, + ge, sf.ctypes.data, sr.ctypes.data, ws.ctypes.data, ws_len) + + def _align(self, ws_len, fl_len, cap, tb=1): + ws = np.empty(max(ws_len, 1), np.float32) + fl = np.empty(max(fl_len, 1), np.uint8) + pb = np.empty(2 * max(cap, 1), np.int32) + poff = np.empty(2, np.int64) + sc, ends = np.empty(1, np.float32), np.empty(2, np.int32) + return self.k.lib.fmsw_align(self.sim.ctypes.data, self.n, self.po.ctypes.data, self.mm.ctypes.data, 1, 1.0, + 5.0, 0.5, tb, sc.ctypes.data, ends.ctypes.data, pb.ctypes.data, cap, + poff.ctypes.data, ws.ctypes.data, ws_len, fl.ctypes.data, fl_len) + + def test_undersized_buffers_return_status_1(self): + need_ws, need_fl = self.k.ws_floats(self.n, self.m), self.k.flag_bytes(self.n, self.m) + cap = self.n + self.m + self.assertEqual(self._score(need_ws), 0) + self.assertEqual(self._score(need_ws - 1), 1) + self.assertEqual(self._align(need_ws, need_fl, cap), 0) + self.assertEqual(self._align(need_ws - 1, need_fl, cap), 1) + self.assertEqual(self._align(need_ws, need_fl - 1, cap), 1) + self.assertEqual(self._align(need_ws, need_fl, cap - 1), 1) + self.assertEqual(self._align(need_ws - 1, 0, 0, tb=0), 1) + self.assertEqual(self._align(need_ws, 0, 0, tb=0), 0) + + def test_bad_arguments_return_status_2(self): + need = self.k.ws_floats(self.n, self.m) + self.assertEqual(self._score(need, n=0), 2) + self.assertEqual(self._score(need, mm=np.array([-1], np.int32)), 2) + self.assertEqual(self._score(need, po=np.array([-16], np.int64)), 2) + self.assertEqual(self._score(need, go=0.4, ge=0.5), 2) + self.assertEqual(self._score(need, ge=-0.5, go=-0.5), 2) + self.assertEqual(self._score(need, offset=float("nan")), 2) + self.assertEqual(self._score(need, K=-1), 2) + self.assertEqual(self._score(need, K=0), 0) + q = np.ones((2, 64), np.float32) + self.assertEqual(self.k.lib.fmsw_query_prep(q.ctypes.data, -1, q.ctypes.data), 2) + + def test_empty_target_scores_zero(self): + self.assertEqual(self._score(self.k.ws_floats(self.n, 0), mm=np.array([0], np.int32)), 0) + qp = swkern.query_prep(_f1()[0]["q16"]) + codes = _f1()[0]["codes"] + res = swkern.align(qp, codes, np.array([0, 0, len(codes), len(codes)], np.int64), D) + self.assertEqual(res.scores[0], 0.0) + self.assertEqual(res.scores[2], 0.0) + self.assertEqual(res.path_off.tolist()[:2], [0, 0]) + self.assertGreater(res.scores[1], 0.0) + + def test_only_empty_targets_in_a_fresh_workspace(self): + """A call (or a first block) whose targets are all empty, in fresh workspaces: scores 0, no paths, never the + 64-byte guard. An empty numpy slice keeps its base's address, so aligned_empty(0) took the alignment of a + fresh allocation (off in about 3 of 4); the allocations here are made to start 4 bytes past a 64-byte + boundary, so the check does not depend on the allocator.""" + real_empty = np.empty + + def shifted(shape, dtype=float, *args, **kwargs): + """np.empty whose data starts 4 bytes past a 64-byte boundary.""" + dtype = np.dtype(dtype) + count = int(np.prod(shape)) + raw = real_empty(count * dtype.itemsize + 2 * swkern.ALIGN_BYTES, np.uint8) + start = -raw.ctypes.data % swkern.ALIGN_BYTES + 4 + return raw[start:start + count * dtype.itemsize].view(dtype).reshape(shape) + + with mock.patch.object(np, "empty", shifted): + self.assertEqual(np.empty(3, np.float32).ctypes.data % swkern.ALIGN_BYTES, 4) + bufs = {n: swkern.aligned_empty(n) for n in (0, 1, 15, 16, 17)} + fresh = [swkern.Workspace() for _ in range(8)] + for n, buf in bufs.items(): + self.assertEqual((buf.size, buf.ctypes.data % swkern.ALIGN_BYTES), (n, 0)) + qp = swkern.query_prep(_f1()[0]["q16"]) + codes = _f1()[0]["codes"] + m = len(codes) + for lens in ([0], [0, 0, 0], [0, 0, m], [0, m, 0]): + offsets = np.concatenate([[0], np.cumsum(lens)]).astype(np.int64) + with self.subTest(lens=lens): + with mock.patch.object(np, "empty", shifted): + ws, ws2 = swkern.Workspace(), swkern.Workspace() # their first buffers: shifted allocations + r = swkern.score_fwd_rev(qp, codes, offsets, D, workspace=ws, block_floats=1) + a = swkern.align(qp, codes, offsets, D, workspace=ws2, block_floats=1) + empty = np.asarray(lens) == 0 + self.assertTrue(np.all(r.s_fwd[empty] == 0) and np.all(r.s_rev[empty] == 0)) + self.assertTrue(np.all(a.scores[empty] == 0)) + self.assertTrue(np.all(np.diff(a.path_off)[empty] == 0)) + self.assertTrue(np.array_equal(a.scores.view(np.uint32), r.s_fwd.view(np.uint32))) + for ws in fresh: + self.assertEqual(ws.sim(0).ctypes.data % swkern.ALIGN_BYTES, 0) + + def test_release_drops_the_buffers_and_keeps_the_results(self): + qp = swkern.query_prep(_f1()[0]["q16"]) + codes = _f1()[0]["codes"] + offsets = np.array([0, len(codes)], np.int64) + ws = swkern.Workspace() + a = swkern.align(qp, codes, offsets, D, workspace=ws) + size = ws.nbytes() + self.assertGreater(size, 0) + self.assertFalse(ws.release(size)) + self.assertEqual(ws.nbytes(), size) + self.assertTrue(ws.release(size - 1)) + self.assertEqual(ws.nbytes(), 0) + b = swkern.align(qp, codes, offsets, D, workspace=ws) + for x, y in ((a.scores, b.scores), (a.ends, b.ends), (a.path, b.path), (a.path_off, b.path_off)): + self.assertTrue(np.array_equal(x, y)) + self.assertEqual(a.scores.view(np.uint32).tolist(), b.scores.view(np.uint32).tolist()) + + def test_workspace_grows_and_is_per_thread(self): + ws = swkern.Workspace() + a = ws.kernel_floats(10) + self.assertGreaterEqual(a.size, 10) + self.assertIs(ws.kernel_floats(5), a) + self.assertGreaterEqual(ws.kernel_floats(100).size, 100) + seen = [] + t = threading.Thread(target=lambda: seen.append(swkern.default_workspace())) + t.start() + t.join() + self.assertIsNot(seen[0], swkern.default_workspace()) + self.assertIs(swkern.default_workspace(), swkern.default_workspace()) + + +# ===================================================================================================================== +# 9. regression: align() output is in forward order (the research codepath.py:261 double reversal is not ported) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class ForwardPathRegressionTest(unittest.TestCase): + + def test_align_path_is_forward_and_not_reversed(self): + p = max(_f1(), key=lambda x: len(x["path"])) + r = swkern.run_pair(p["q16"], p["codes"], reference_sim=False) + path = r.path + self.assertTrue(np.array_equal(path, p["path"])) + self.assertLess(path[0, 0], path[-1, 0]) + self.assertLess(path[0, 1], path[-1, 1]) + self.assertEqual((path[-1, 0] + 1, path[-1, 1] + 1), r.end) + self.assertFalse(np.array_equal(path, path[::-1])) + qp = swkern.query_prep(p["q16"]) + a = swkern.align(qp, p["codes"], np.array([0, len(p["codes"])], np.int64), D) + self.assertTrue(np.array_equal(a.path, path)) + + +# ===================================================================================================================== +# CIGAR round trip on every F1 alignment plus the 1,000 golden alignments (P3.5 acceptance d) +# ===================================================================================================================== +@pytest.mark.usefixtures("fmsw_libs") +class CigarRoundTripTest(unittest.TestCase): + + def test_round_trip_on_1050_alignments(self): + rng = np.random.default_rng(1050) + done = 0 + for p in _f1(): + r = swkern.run_pair(p["q16"], p["codes"], reference_sim=False) + _assert_cigar_round_trip(self, r.path, p["qseq"], p["tseq"]) + done += 1 + for _kind, q16, codes in _golden_pairs(): + r = swkern.run_pair(q16, codes, reference_sim=False) + qs, ts = _random_seq(rng, len(q16)), _random_seq(rng, len(codes)) + if len(r.path) and rng.random() < 0.5: # force identities along half of the paths + ts = list(ts) + for i, j in r.path.tolist(): + if i >= 0 and j >= 0 and rng.random() < 0.5: + ts[j] = qs[i] + ts = "".join(ts) + _assert_cigar_round_trip(self, r.path, qs, ts) + done += 1 + self.assertEqual(done, 1050) + + def test_cigar_helpers(self): + path = np.array([[0, 0], [1, -1], [2, -1], [-1, 1], [3, 2], [4, 3]], np.int32) + self.assertEqual(swkern.cigar_from_path(path), "1M2I1D2M") + self.assertEqual(swkern.expand_cigar("1M2I1D2M"), "MIIDMM") + self.assertEqual(swkern.cigar_from_path(np.zeros((0, 2), np.int32)), "") + with self.assertRaises(ValueError): + swkern.expand_cigar("3M2X") + pm = swkern.path_metrics(path, "ACDEF", "AQEF", strings=True) + self.assertEqual((pm.q_aln, pm.t_aln), ("ACD-EF", "A--QEF")) + self.assertEqual((pm.n_ident, pm.mismatch, pm.gap_open, pm.q_start, pm.q_end, pm.t_start, pm.t_end), + (3, 0, 2, 1, 5, 1, 4)) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 359518a..f682436 100644 --- a/uv.lock +++ b/uv.lock @@ -607,25 +607,25 @@ wheels = [ [[package]] name = "faiss-cpu" -version = "1.13.2" +version = "1.15.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "packaging" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c9/671f66f6b31ec48e5825d36435f0cb91189fa8bb6b50724029dbff4ca83c/faiss_cpu-1.13.2-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9064eb34f8f64438dd5b95c8f03a780b1a3f0b99c46eeacb1f0b5d15fc02dc1", size = 3452776 }, - { url = "https://files.pythonhosted.org/packages/5a/4a/97150aa1582fb9c2bca95bd8fc37f27d3b470acec6f0a6833844b21e4b40/faiss_cpu-1.13.2-cp310-abi3-macosx_14_0_x86_64.whl", hash = "sha256:c8d097884521e1ecaea6467aeebbf1aa56ee4a36350b48b2ca6b39366565c317", size = 7896434 }, - { url = "https://files.pythonhosted.org/packages/0b/d0/0940575f059591ca31b63a881058adb16a387020af1709dcb7669460115c/faiss_cpu-1.13.2-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ee330a284042c2480f2e90450a10378fd95655d62220159b1408f59ee83ebf1", size = 11485825 }, - { url = "https://files.pythonhosted.org/packages/e7/e1/a5acac02aa593809f0123539afe7b4aff61d1db149e7093239888c9053e1/faiss_cpu-1.13.2-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab88ee287c25a119213153d033f7dd64c3ccec466ace267395872f554b648cd7", size = 23845772 }, - { url = "https://files.pythonhosted.org/packages/9c/7b/49dcaf354834ec457e85ca769d50bc9b5f3003fab7c94a9dcf08cf742793/faiss_cpu-1.13.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:85511129b34f890d19c98b82a0cd5ffb27d89d1cec2ee41d2621ee9f9ef8cf3f", size = 13477567 }, - { url = "https://files.pythonhosted.org/packages/f7/6b/12bb4037921c38bb2c0b4cfc213ca7e04bbbebbfea89b0b5746248ce446e/faiss_cpu-1.13.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8b32eb4065bac352b52a9f5ae07223567fab0a976c7d05017c01c45a1c24264f", size = 25102239 }, - { url = "https://files.pythonhosted.org/packages/87/ff/35ed875423200c17bdd594ce921abfc1812ddd21e09355290b9a94e170ab/faiss_cpu-1.13.2-cp312-cp312-win_amd64.whl", hash = "sha256:b82c01d30430dd7b1fa442001b9099735d1a82f6bb72033acdc9206d5ac66a64", size = 18890300 }, - { url = "https://files.pythonhosted.org/packages/c5/3a/bbdf5deaf6feb34b46b469c0a0acd40216c3d3c6ecf5aeb71d56b8a650e3/faiss_cpu-1.13.2-cp312-cp312-win_arm64.whl", hash = "sha256:2c4f696ae76e7c97cbc12311db83aaf1e7f4f7be06a3ffea7e5b0e8ec1fd805b", size = 8553157 }, - { url = "https://files.pythonhosted.org/packages/60/4b/903d85bf3a8264d49964ec799e45c7ffc91098606b8bc9ef2c904c1a56cb/faiss_cpu-1.13.2-cp313-cp313-win_amd64.whl", hash = "sha256:cb4b5ee184816a4b099162ac93c0d7f0033d81a88e7c1291d0a9cc41ec348984", size = 18891330 }, - { url = "https://files.pythonhosted.org/packages/b2/52/5d10642da628f63544aab27e48416be4a7ea25c6b81d8bd65016d8538b00/faiss_cpu-1.13.2-cp313-cp313-win_arm64.whl", hash = "sha256:1243967eeb2298791ff7f3683a4abd2100d7e6ec7542ca05c3b75d47a7f621e5", size = 8553088 }, - { url = "https://files.pythonhosted.org/packages/b0/b1/daaab8046f56c60079648bd83774f61b283b59a9930a2f60790ee4cdedfe/faiss_cpu-1.13.2-cp314-cp314-win_amd64.whl", hash = "sha256:c8b645e7d56591aa35dc75415bb53a62e4a494dba010e16f4b67daeffd830bd7", size = 18892621 }, - { url = "https://files.pythonhosted.org/packages/06/6f/5eaf3e249c636e616ebb52e369a4a2f1d32b1caf9a611b4f917b3dd21423/faiss_cpu-1.13.2-cp314-cp314-win_arm64.whl", hash = "sha256:8113a2a80b59fe5653cf66f5c0f18be0a691825601a52a614c30beb1fca9bc7c", size = 8556374 }, + { url = "https://files.pythonhosted.org/packages/9b/ed/d1b8e6720e9947469cab45dbfbf1b82e1d5acf9fe063dc97a6e82db83094/faiss_cpu-1.15.1-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:ea9e12d540ca8ac0347b831d034c0f6d7ff5eed20523a247db44b3543ad2aad4", size = 4987669 }, + { url = "https://files.pythonhosted.org/packages/ef/75/eb2f36334a58b343a87a2c1feaa747655fde7efdaad9c5d9eb367da89f15/faiss_cpu-1.15.1-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f52e727992ce86a783f61657f0c4f3498a235883083b982ba1be49d05f924450", size = 7237206 }, + { url = "https://files.pythonhosted.org/packages/a3/90/695eeab44921bb475611fc71ec0a74af82080f496cb7586c6490e4f322d2/faiss_cpu-1.15.1-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffa71b14b3090bc076f8b026554178868fdbfe2f26fe644da629405836369039", size = 9890446 }, + { url = "https://files.pythonhosted.org/packages/6c/f4/098bd9d178ae36fa078c66068d3264e27fff4308d5131655e5e743153d4c/faiss_cpu-1.15.1-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2c31b7f2f6647eb76829a5cfe3c398fb9346df9f26b1d4db35269c91eb58c33", size = 18834180 }, + { url = "https://files.pythonhosted.org/packages/3c/a7/d9e88b337f9636e0e80b651bfd27dbff533820d26c250bb60d2122de18a9/faiss_cpu-1.15.1-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0a59d8ee9ffcac34608f591d16b617d9056e12a26a8b8cf0015b6b334e33e1", size = 11447194 }, + { url = "https://files.pythonhosted.org/packages/01/28/0855b161a081556a1df0ff14d5e7e73db23bd24ed85505009387fb61762e/faiss_cpu-1.15.1-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:d4a250000112ac26ae79530e67a18fa986c8b7b0329154aefeb7692b270ed366", size = 19574480 }, + { url = "https://files.pythonhosted.org/packages/69/19/a4bd07c73f17556eff1599e27918b8a97eaab468aea7b143bd49ca0535eb/faiss_cpu-1.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:38d192695210a51ff72449d8802ff62601568fcfc6372222a64a069da0ecdb10", size = 16293368 }, + { url = "https://files.pythonhosted.org/packages/56/35/c79cd7321c6d8af277691e7a7ca1dd362e0fff24a9697aa944781cdb8c75/faiss_cpu-1.15.1-cp312-cp312-win_arm64.whl", hash = "sha256:4fd6623ed931d16256b268ac2984f672cdf1929702e24b3e741798d0bb08804f", size = 9039754 }, + { url = "https://files.pythonhosted.org/packages/98/ae/e31e9c30f686681b78bd089edbefd3675602132612ce5dd187275be8b773/faiss_cpu-1.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:8a577dd6d52f685326570105c3d18feb3776799d080534e329a191740d6362b6", size = 16292975 }, + { url = "https://files.pythonhosted.org/packages/dc/49/96bfac5586cc84bad3dae85dd29595512883327789573e6e81541646b5ef/faiss_cpu-1.15.1-cp313-cp313-win_arm64.whl", hash = "sha256:a26acb421037b030c1e9eea342adff5a0e1b6faab9e626be64b5f598241e5592", size = 9038412 }, + { url = "https://files.pythonhosted.org/packages/98/82/4b1866e93b85247774dbd67afc95fbe5d02097ee125cf4ed11c90515717b/faiss_cpu-1.15.1-cp314-cp314-win_amd64.whl", hash = "sha256:c18b569ec5d5e79f2156f0059fdb3ea79976f365d79291252ab6b45d40523c2c", size = 16574394 }, + { url = "https://files.pythonhosted.org/packages/61/23/8da811ff180c8f4f96f23bed84a1a235fad371f6b21ae5395d3e42d4ca95/faiss_cpu-1.15.1-cp314-cp314-win_arm64.whl", hash = "sha256:dc1cd974cd5477ca5d01d9f9ecba6a7fc555b6ef2eda7b16c97e20903431dc6b", size = 9340275 }, ] [[package]] @@ -639,10 +639,11 @@ wheels = [ [[package]] name = "foldmatch" -version = "0.8.0" +version = "0.10.0" source = { editable = "." } dependencies = [ { name = "biotite" }, + { name = "click" }, { name = "esm" }, { name = "faiss-cpu" }, { name = "hf-xet" }, @@ -650,9 +651,9 @@ dependencies = [ { name = "huggingface-hub" }, { name = "igraph" }, { name = "importlib-metadata" }, - { name = "leidenalg" }, { name = "lightning" }, { name = "pyarrow" }, + { name = "threadpoolctl" }, { name = "typer" }, ] @@ -664,16 +665,17 @@ dev = [ [package.metadata] requires-dist = [ { name = "biotite", specifier = "==1.7.1" }, + { name = "click", specifier = "==8.3.1" }, { name = "esm", specifier = "==3.2.3" }, - { name = "faiss-cpu", specifier = "==1.13.2" }, + { name = "faiss-cpu", specifier = "==1.15.1" }, { name = "hf-xet", specifier = "==1.4.2" }, { name = "httpx", specifier = "==0.28.1" }, { name = "huggingface-hub", specifier = "==0.36.2" }, { name = "igraph", specifier = "==1.0.0" }, { name = "importlib-metadata", specifier = "==8.7.1" }, - { name = "leidenalg", specifier = "==0.11.0" }, { name = "lightning", specifier = "==2.6.1" }, { name = "pyarrow", specifier = "==24.0.0" }, + { name = "threadpoolctl", specifier = "==3.6.0" }, { name = "typer", specifier = "==0.24.1" }, ] @@ -1148,26 +1150,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/91/53255615acd2a1eaca307ede3c90eb550bae9c94581f8c00081b6b1c8f44/kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57", size = 75987 }, ] -[[package]] -name = "leidenalg" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "igraph" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/a5/853e93441aed7f82b0389f86f37e19413e817ba0c54cc790895935256968/leidenalg-0.11.0.tar.gz", hash = "sha256:f454be96bbc8089ea2a90ca853d8d389ab646de964a03bd58417f8b29ff8ef5d", size = 452850 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/28/a9/4ab4e244215db0c8b626e4bed0d3e0fbd191c52d2d5f5cb9d160139ecc7e/leidenalg-0.11.0-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:5607589050bfc1926e657b4d8a3b5341fe1eb81018c22cf4a3d3a39e368d1fcb", size = 2256514 }, - { url = "https://files.pythonhosted.org/packages/98/f4/98db342d603671ae0a233f0a624939a47161044a2716cbd62a50440a1132/leidenalg-0.11.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:9b5781876b1f1faed72a4f9926ff52de286843556b9d6791fe25a2acb33b7a5c", size = 1926003 }, - { url = "https://files.pythonhosted.org/packages/9b/38/fd6ac21af10b12828b472eada4fce0edf2a212581238ad0c8d1afebc6f98/leidenalg-0.11.0-cp38-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a80f49477f8e793f27d8e08949177f19e3834cd878af50a662b4f87335d06549", size = 2545535 }, - { url = "https://files.pythonhosted.org/packages/77/87/b087584750a788535b4a8d56ddeb82a175d32b472aa5338a4e2cc593a42c/leidenalg-0.11.0-cp38-abi3-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:2143be3e80485584ccbdf927323fce65345da17facd0f8b438f11015f5dc6c27", size = 2845029 }, - { url = "https://files.pythonhosted.org/packages/b0/a4/a89e2ce16a580f7bea066ed49364f0b3e04a6412f0c3692975bee8515141/leidenalg-0.11.0-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:571a0934f831a69442d82889d319bdba93de924bd9e09b720cd8cbe6fdc08c17", size = 2738084 }, - { url = "https://files.pythonhosted.org/packages/e8/fe/8923cac6cd7c9e0ac5f38aaa69a4744c93d025575763d05f7a3baae8020d/leidenalg-0.11.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:aec03e7178b19102dd271b453a39b9865cf283b4113151ba60514e5681046294", size = 4070307 }, - { url = "https://files.pythonhosted.org/packages/fe/94/beaab5ee9968f9389f705532c31ffb868bad8a5ce68fb699ddde5ddc5409/leidenalg-0.11.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:310b9269a11fd1e960590c1a2b6ff685a2cc42aa3234ce67bc2a623ab61f26a9", size = 3797863 }, - { url = "https://files.pythonhosted.org/packages/6e/8e/8caf4ba38fd7d8e6197348b348a4ab666b1b3117225ea2f0934a98a93176/leidenalg-0.11.0-cp38-abi3-win32.whl", hash = "sha256:5ea4cd7ee054540112b28f7e2d64658dcccd59f61a5d6a08a41df808645f96e9", size = 1643351 }, - { url = "https://files.pythonhosted.org/packages/47/15/7d459a8e2a43f17c1db129b997b7bb7aa7f000a0967bab87c28b8c5cf448/leidenalg-0.11.0-cp38-abi3-win_amd64.whl", hash = "sha256:5e789c0960008d185413344a402d0587580c441644d4d20bf57c96f25d4d1710", size = 1990321 }, -] - [[package]] name = "lightning" version = "2.6.1"