diff --git a/docs/source/cluster_runs.rst b/docs/source/cluster_runs.rst new file mode 100644 index 0000000..c9dd4cc --- /dev/null +++ b/docs/source/cluster_runs.rst @@ -0,0 +1,178 @@ +Large runs and clusters +======================= + +``mofstructure_database``, ``mofstructure_topology`` and +``mofstructure_porosity`` are built to run over tens of thousands of +structures without stopping. This page describes how they do that, how to +split a run over a cluster array and what the output looks like. + +What protects a long run +------------------------ + +* **One worker process per structure.** Each structure is analysed in a + worker process. If compiled code crashes (a segmentation fault or abort in + Open Babel, zeo++ or numpy) only the worker dies. The structure is recorded + as ``crashed`` and a new worker carries on with the next structure. +* **A hard time limit.** ``--max-time`` is a wall-clock limit for the whole + analysis of one structure, including reading, guest removal, + deconstruction, the canonical form and the embedding. A structure that runs + past it is killed and recorded as ``timeout``. +* **An optional memory limit.** ``--memory-limit`` caps the memory of each + worker in GB (Linux). A structure that needs more fails on its own instead + of the scheduler killing the whole job. +* **Results are saved as they finish.** Each result is appended as one line + to a checkpoint file and flushed to disk. A job killed by the scheduler + loses at most the structures in progress. +* **Runs resume.** Repeating the same command skips every structure already + recorded. ``--retry-failed`` runs the failures again. +* **Database files are never left half written.** They are written to a + temporary file and renamed into place. + +Options shared by the three commands +------------------------------------ + +======================== ==================================================== +``-j, --workers N`` worker processes (default 1) +``--max-time S`` seconds per structure before it is killed + (database 7200, topology 1800, porosity 2400; + 0 for no limit) +``--memory-limit GB`` memory per worker, Linux only +``--shard i/N`` process part *i* of *N* (0-based); ``slurm/N`` + reads ``SLURM_ARRAY_TASK_ID`` +``--retry-failed`` run again structures recorded as error, timeout + or crashed +``--recycle N`` structures per worker before it is replaced + (default 50), which releases memory +``--merge-every MIN`` rebuild the json and csv files during the run + every MIN minutes (default 60, 0 only at the end; + not used with ``--shard``) +======================== ==================================================== + +On one node +----------- + +.. code-block:: bash + + mofstructure_database cif_folder -s MOFdb -j 32 --oms --memory-limit 8 + +If the job is killed, submit the same command again. + +On a SLURM array +---------------- + +Each array task processes one shard and writes its own checkpoint part. +Join the parts once all tasks have finished. + +.. code-block:: bash + + #!/bin/bash + #SBATCH --job-name=mofstructure + #SBATCH --array=0-31 + #SBATCH --cpus-per-task=16 + #SBATCH --mem=64G + #SBATCH --time=24:00:00 + #SBATCH --output=logs/mofstructure_%A_%a.log + + mofstructure_database cif_folder -s MOFdb \ + -j ${SLURM_CPUS_PER_TASK} --memory-limit 3.5 \ + --shard slurm/32 --oms + +.. code-block:: bash + + # after the array has finished, or at any time to inspect progress + mofstructure_merge MOFdb + +A task that hit its time limit is resubmitted with the same array index and +continues from its checkpoint. All shards must use the same ``N``. + +Topology, porosity and open metal sites as separate jobs +-------------------------------------------------------- + +The three analyses can be submitted as independent jobs on the same folder +and the same save directory. Each writes its own checkpoints, so they do not +interfere, and each rebuilds only the files it produces. + +.. code-block:: bash + + # topology.sh + #SBATCH --cpus-per-task=32 --mem=128G --time=48:00:00 + mofstructure_topology cif_folder -s MOFdb --method all_node \ + -j ${SLURM_CPUS_PER_TASK} --timeout 600 --max-time 1800 \ + --memory-limit 3.5 --quiet + + # porosity.sh + #SBATCH --cpus-per-task=32 --mem=128G --time=48:00:00 + mofstructure_porosity cif_folder -s MOFdb \ + -j ${SLURM_CPUS_PER_TASK} --timeout 1800 --memory-limit 3.5 + + # oms.sh + #SBATCH --cpus-per-task=32 --mem=128G --time=24:00:00 + mofstructure_oms cif_folder -s MOFdb \ + -j ${SLURM_CPUS_PER_TASK} --memory-limit 3.5 + +When a job reaches its time limit, submit the same script again and it +continues. When all three have finished, ``mofstructure_merge MOFdb`` writes +the final files and a ``run_status.csv`` covering all three. + +``mofstructure_topology`` names a structure by its file name without the +last suffix (``ABAVIJ.MOF_subset`` for ``ABAVIJ.MOF_subset.cif``), while the +porosity, OMS and database commands keep the text before the first dot +(``ABAVIJ``). Normalise the names before joining the tables. + +Output +------ + +.. code-block:: text + + MOFdb/ + Structure_Data/ + _progress/ checkpoints, one line per structure + database[.part-iiii-of-nnnn].jsonl + topology.[.part-...].jsonl + porosity[.part-...].jsonl + sbus_and_linkers.json/.csv + ligands_data.json/.csv + porosity_data.json/.csv + structure_oms_and_general_info.json/.csv + topology_data.json/.csv + fingerprint_data.json/.csv + run_status.csv + XYZ_DB/ building units as xyz files + +The checkpoints grow by one line per structure throughout the run. The json +and csv files have the same names and layout as in earlier versions; a json +file is one object that has to be rewritten whole, so they are rebuilt from +the checkpoints every ``--merge-every`` minutes, at the end of a run that is +not sharded, and by ``mofstructure_merge``, which can be run at any time, +also while a run is in progress. Records already in the json files +from an earlier version are kept. + +``run_status.csv`` has one row per structure and command: + +============ ============================================================= +``status`` ``ok``; ``error`` (the analysis raised, see ``detail``); + ``timeout`` (killed after ``--max-time``); ``crashed`` + (the worker died, ``detail`` names the signal) +``elapsed_s`` wall-clock seconds for the structure +``analysis_errors`` for ``mofstructure_database``, the analyses that + failed within a structure that otherwise completed, for + example ``{"topology": "..."}`` +============ ============================================================= + +A structure that timed out or crashed has a row in ``porosity_data.json`` +with ``porosity_status`` set to ``timeout`` or ``crashed`` and missing values +elsewhere, so the tables keep one row per input structure. + +Each checkpoint line is a json object: + +.. code-block:: json + + {"name": "ABAVIJ", "file": "cif_folder/ABAVIJ.cif", "status": "ok", + "elapsed_s": 1.83, "data": {"topology": {...}, "fingerprint": {...}, + "sbu": {...}, "ligand": {...}, "porosity": {...}}} + +Debugging one structure +----------------------- + +``-j 0`` runs everything in the calling process without isolation or time +limit, so a debugger or print statements work as usual. diff --git a/docs/source/index.rst b/docs/source/index.rst index 79e4863..df9a18e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -60,6 +60,7 @@ The `mofstructure` module includes a variety of features that simplify common op installation usage + cluster_runs examples api_reference updates diff --git a/docs/source/updates.rst b/docs/source/updates.rst index 2df8f38..cec16b4 100644 --- a/docs/source/updates.rst +++ b/docs/source/updates.rst @@ -1,3 +1,44 @@ +Unreleased +========== + +Topology keys no longer depend on how a crystal is written +------------------------------------------------------------ + +The canonical key is meant to identify a net whatever the cell, origin or +atom order of the input. Testing every reduction on CSD MOFs, CoRE COFs and +IZA zeolites under origin shifts, permutations, supercells and a change of +cell setting found three errors, now fixed: + +* ``all_node``: edges built inside a rod or layered metal SBU took their + lattice translation with the wrong sign. The error vanished when no bond + crossed the cell boundary, so it appeared only after an origin shift or in + a different cell. Before the fix 90 % of transformed MOFs kept their key; + after it all of them do. +* ``cof``: a linkage joining a building unit to its own neighbouring image + was discarded, and splicing two-connected units into edges used the wrong + sign. Small-cell COFs could reduce to a one-periodic net. +* Nets whose barycentric placement has collisions between vertices that are + not neighbours received a key that changed with the cell. They are now + reported as ``unidentified`` with the reason, as Systre does and as the + method states. A timeout is now reported as ``timeout`` rather than + ``unidentified``. + +Keys computed with 0.1.9.1 for MOFs with rod or layered SBUs and for small +cell COFs can differ from the corrected keys. The ``sbus`` and +``ligand_cluster`` reductions collapse a rod SBU into one vertex, which has +no cell-independent periodic graph; use ``all_node`` or ``single_node`` for +rod MOFs. + +Batch commands no longer stop +----------------------------- + +``mofstructure_database``, ``mofstructure_topology`` and +``mofstructure_porosity`` now run every structure in a worker process with a +hard time limit, record crashes and timeouts instead of stopping, save each +result as it finishes, resume when repeated and can be split over a cluster +array. ``mofstructure_merge`` joins the parts. Database json files are +written atomically. See :doc:`cluster_runs`. + Updates Version 0.1.9.1 ======================= diff --git a/mofstructure/batch.py b/mofstructure/batch.py new file mode 100644 index 0000000..3575976 --- /dev/null +++ b/mofstructure/batch.py @@ -0,0 +1,516 @@ +#!/usr/bin/env python3 +''' +Run an analysis over many structures so that one structure cannot stop it. + +A long batch over a structure database fails in ways that have nothing to do +with the structure being analysed at the time it fails: + + - compiled code (Open Babel, zeo++, numpy) can crash with a segmentation + fault or abort, which ends the Python process and everything in it; + - a structure can run for hours, for example a very large cell in the + deconstruction, the canonical form or the embedding, and a signal based + timeout cannot interrupt compiled code; + - memory can grow until the scheduler kills the whole job; + - results kept in memory and written at the end, or rewritten in full after + every structure, are lost or corrupted when the job is killed. + +`run_batch` therefore runs every structure in a worker process that the parent +can kill. Each worker is started with the spawn method, which is safe on macOS +and on Linux, handles structures one at a time and is replaced after a fixed +number of structures to release memory. The parent enforces a wall-clock +limit per structure, records a crash or a timeout as the result for that +structure and starts a new worker. Every result is appended as one line to a +json-lines checkpoint file and flushed to disk at once, so a job that is +killed loses at most the structures in flight, and repeating the same command +continues where it stopped. + +A run can also be split across the tasks of a cluster array job with +`shard`, each task writing its own checkpoint file, and the parts are joined +by `consolidate`. + +**checkpoint record:** + One json object per line:: + + {"name": "ABAVIJ", "file": ".../ABAVIJ.cif", "status": "ok", + "elapsed_s": 1.83, "data": {...}} + + `status` is "ok", "error" (the analysis raised; `detail` holds the + exception), "timeout" (killed after the wall-clock limit) or "crashed" + (the worker died; `detail` names the signal or exit code). +''' +from __future__ import annotations + +import importlib +import json +import logging +import multiprocessing as mp +import os +import signal +import sys +import time +import traceback +from collections.abc import Callable, Iterable, Sequence +from multiprocessing.connection import wait +from pathlib import Path + +logger = logging.getLogger(__name__) + +CHECKPOINT_DIR = "_progress" + + +# --------------------------------------------------------------------------- +# Worker side +# --------------------------------------------------------------------------- + +def _limit_memory(memory_gb: float | None) -> None: + ''' + Cap the address space of the worker so that a runaway structure fails in + its own process instead of drawing the scheduler's out-of-memory killer + onto the whole job. Linux enforces the limit; macOS does not, and the call + is skipped there. + ''' + if not memory_gb: + return + try: + import resource + + limit = int(memory_gb * 1024 ** 3) + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + except (ImportError, ValueError, OSError): + logger.debug("memory limit not supported on this platform") + + +def _resolve(task: str | Callable) -> Callable: + '''Import a task given as "module:function", or return a callable.''' + if callable(task): + return task + module, _, name = task.partition(":") + return getattr(importlib.import_module(module), name) + + +def _worker(conn, task, kwargs, memory_gb, quiet) -> None: + ''' + Worker loop: receive (name, path), run the task, send the result back. + + A None message ends the worker. Anything the task raises is returned as + an error record, so only a crash in compiled code can end the process + early, and that is detected by the parent. + ''' + # The parent decides when to stop; a Ctrl-C reaches every process in the + # group, so the worker leaves the interrupt to the parent. + signal.signal(signal.SIGINT, signal.SIG_IGN) + _limit_memory(memory_gb) + if quiet: + import warnings + + # zeo++, voro++ and Open Babel print from compiled code, which no + # Python setting can silence, and over a large folder that output + # buries the progress lines of a cluster log. Pointing the worker's + # file descriptors at /dev/null silences them and the child + # interpreters they start; errors still reach the checkpoint record. + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, 1) + os.dup2(devnull, 2) + warnings.filterwarnings("ignore") + logging.disable(logging.WARNING) + try: + from rdkit import RDLogger + + RDLogger.DisableLog("rdApp.*") + except ImportError: + pass + function = _resolve(task) + while True: + try: + message = conn.recv() + except EOFError: + return + if message is None: + return + name, path = message + start = time.perf_counter() + try: + data = function(path, **kwargs) + reply = {"status": "ok", "data": data} + except MemoryError: + reply = {"status": "error", "detail": "MemoryError"} + except Exception as exc: # noqa: BLE001 - reported, never swallowed + reply = { + "status": "error", + "detail": f"{type(exc).__name__}: {exc}"[:500], + "traceback": traceback.format_exc(limit=8)[-2000:], + } + reply["elapsed_s"] = round(time.perf_counter() - start, 3) + try: + conn.send((name, reply)) + except (TypeError, ValueError, AttributeError) as exc: + # The task returned something that cannot be pickled. + conn.send((name, {"status": "error", + "detail": f"unpicklable result: {exc}", + "elapsed_s": reply["elapsed_s"]})) + + +# --------------------------------------------------------------------------- +# Checkpoint files +# --------------------------------------------------------------------------- + +def shard_suffix(shard: tuple[int, int] | None) -> str: + '''File suffix for one shard of a split run, empty for a whole run.''' + if not shard: + return "" + index, count = shard + return f".part-{index:04d}-of-{count:04d}" + + +def parse_shard(text: str | None) -> tuple[int, int] | None: + ''' + Parse "i/N" into (i, N) with 0 <= i < N. Also accepts the value of + SLURM_ARRAY_TASK_ID when given as "slurm/N". + ''' + if not text: + return None + index, _, count = text.partition("/") + if index.lower() in ("slurm", "auto"): + index = os.environ.get("SLURM_ARRAY_TASK_ID", + os.environ.get("PBS_ARRAY_INDEX", "")) + index, count = int(index), int(count) + if not 0 <= index < count: + raise ValueError(f"shard index {index} is outside 0..{count - 1}") + return index, count + + +def select_shard(items: Sequence[tuple[str, str]], + shard: tuple[int, int] | None) -> list[tuple[str, str]]: + ''' + The items one shard is responsible for. Items are sorted by name and dealt + out in turn, so the split does not depend on the order the files were + listed in and every shard receives a similar mix of sizes. + ''' + ordered = sorted(items) + if not shard: + return ordered + index, count = shard + return ordered[index::count] + + +def read_checkpoints(paths: Iterable[Path]) -> dict[str, dict]: + ''' + Read checkpoint files into name -> record, later lines winning. A final + line cut short by a killed job is ignored. + ''' + records: dict[str, dict] = {} + for path in paths: + if not path.exists(): + continue + with open(path, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + records[record["name"]] = record + return records + + +class Checkpoint: + '''Append-only json-lines writer that flushes every record to disk.''' + + def __init__(self, path: Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self.handle = open(self.path, "a", encoding="utf-8") + + def write(self, record: dict) -> None: + self.handle.write(json.dumps(record, default=str) + "\n") + self.handle.flush() + os.fsync(self.handle.fileno()) + + def close(self) -> None: + self.handle.close() + + +# --------------------------------------------------------------------------- +# Parent side +# --------------------------------------------------------------------------- + +def _describe_exit(code: int | None) -> str: + if code is None: + return "worker vanished" + if code < 0: + try: + return f"killed by {signal.Signals(-code).name}" + except ValueError: + return f"killed by signal {-code}" + return f"exit code {code}" + + +class _Slot: + '''One worker process and the structure it is working on.''' + + def __init__(self, context, task, kwargs, memory_gb, quiet): + self.args = (task, kwargs, memory_gb, quiet) + self.context = context + self.start_worker() + + def start_worker(self): + parent, child = self.context.Pipe() + self.process = self.context.Process( + target=_worker, args=(child, *self.args), daemon=True) + self.process.start() + child.close() + self.conn = parent + self.current = None + self.started = 0.0 + self.done = 0 + + def assign(self, item): + self.current = item + self.started = time.monotonic() + self.conn.send(item) + + def stop(self, force=False): + try: + if force: + self.process.kill() + else: + self.conn.send(None) + except (BrokenPipeError, OSError): + pass + self.process.join(timeout=10) + if self.process.is_alive(): + self.process.kill() + self.process.join(timeout=10) + self.conn.close() + + +def run_batch( + items: Sequence[tuple[str, str]], + task: str | Callable, + checkpoint: Path, + *, + kwargs: dict | None = None, + workers: int = 1, + timeout: float | None = 3600, + memory_gb: float | None = None, + recycle: int = 50, + skip: Iterable[str] = (), + quiet: bool = True, + on_record: Callable[[dict, int, int], None] | None = None, +) -> dict[str, int]: + ''' + Run `task(path, **kwargs)` on every (name, path) item in isolated workers. + + **parameters:** + - items: sequence of (name, path) + + - task: "module:function" or a picklable top-level function + Called as task(path, **kwargs); must return json-serialisable + data. + + - checkpoint: pathlib.Path + json-lines file results are appended to. + + - kwargs: python dictionary + Keyword arguments for the task. + + - workers: int + Number of worker processes. 0 runs everything in the calling + process without isolation or time limit, for debugging. + + - timeout: float or None + Wall-clock seconds per structure before its worker is killed. + + - memory_gb: float or None + Address-space limit per worker (Linux). + + - recycle: int + Structures a worker handles before it is replaced. + + - skip: iterable of str + Names already done, for resuming. + + - quiet: bool + Silence warnings and library logging in the workers. + + - on_record: callable(record, done, total), optional + Called after every structure, for progress output. + + **returns:** + python dictionary + Count of records by status. + ''' + kwargs = kwargs or {} + skip = set(skip) + queue = [item for item in items if item[0] not in skip] + total = len(queue) + tally: dict[str, int] = {} + if not queue: + return tally + + if workers == 0: + return _run_in_process(queue, task, kwargs, checkpoint, tally, + on_record) + + context = mp.get_context("spawn") + writer = Checkpoint(checkpoint) + slots = [_Slot(context, task, kwargs, memory_gb, quiet) + for _ in range(max(1, min(workers, total)))] + files = dict(queue) + done = 0 + + def finish(slot, reply): + nonlocal done + name = slot.current[0] + record = {"name": name, "file": files[name], **reply} + writer.write(record) + tally[record["status"]] = tally.get(record["status"], 0) + 1 + done += 1 + slot.current = None + slot.done += 1 + if on_record: + on_record(record, done, total) + + try: + pending = list(reversed(queue)) + while pending or any(slot.current for slot in slots): + for slot in slots: + if slot.current is None and pending: + if slot.done >= recycle or not slot.process.is_alive(): + slot.stop() + slot.start_worker() + item = pending.pop() + try: + slot.assign(item) + except (BrokenPipeError, OSError): + slot.stop(force=True) + slot.start_worker() + slot.assign(item) + + busy = [slot for slot in slots if slot.current] + ready = wait([slot.conn for slot in busy] + + [slot.process.sentinel for slot in busy], timeout=1.0) + now = time.monotonic() + for slot in busy: + if slot.conn in ready or slot.conn.poll(): + try: + _name, reply = slot.conn.recv() + except (EOFError, OSError): + slot.process.join(timeout=5) + reply = {"status": "crashed", + "detail": _describe_exit( + slot.process.exitcode), + "elapsed_s": round(now - slot.started, 3)} + finish(slot, reply) + slot.stop(force=True) + slot.start_worker() + continue + finish(slot, reply) + elif not slot.process.is_alive(): + reply = {"status": "crashed", + "detail": _describe_exit(slot.process.exitcode), + "elapsed_s": round(now - slot.started, 3)} + finish(slot, reply) + slot.stop(force=True) + slot.start_worker() + elif timeout and now - slot.started > timeout: + reply = {"status": "timeout", + "detail": f"exceeded {timeout:g} s", + "elapsed_s": round(now - slot.started, 3)} + finish(slot, reply) + slot.stop(force=True) + slot.start_worker() + finally: + for slot in slots: + slot.stop(force=bool(slot.current)) + writer.close() + return tally + + +def _run_in_process(queue, task, kwargs, checkpoint, tally, on_record): + ''' + Run the items in the calling process, for debugging and tests. Exceptions + are recorded as for a worker, but a crash in compiled code ends the run + and no time limit is applied. + ''' + function = _resolve(task) + writer = Checkpoint(checkpoint) + try: + for done, (name, path) in enumerate(queue, start=1): + start = time.perf_counter() + try: + reply = {"status": "ok", "data": function(path, **kwargs)} + except Exception as exc: # noqa: BLE001 - recorded + reply = {"status": "error", + "detail": f"{type(exc).__name__}: {exc}"[:500]} + reply["elapsed_s"] = round(time.perf_counter() - start, 3) + record = {"name": name, "file": path, **reply} + writer.write(record) + tally[record["status"]] = tally.get(record["status"], 0) + 1 + if on_record: + on_record(record, done, len(queue)) + finally: + writer.close() + return tally + + +def progress_printer(label: str = "") -> Callable[[dict, int, int], None]: + ''' + Progress callback printing one flushed line per structure with an + estimate of the time left. Output is flushed so that a log file on a + cluster shows progress as it happens rather than in buffered blocks. + ''' + start = time.monotonic() + + def show(record: dict, done: int, total: int) -> None: + elapsed = time.monotonic() - start + left = elapsed / done * (total - done) if done else 0 + detail = record.get("detail", "") + print(f"{label}{done}/{total} {record['name']:<30} " + f"{record['status']:<8} {record.get('elapsed_s', 0):8.1f} s " + f"eta {left / 3600:6.2f} h {detail[:60]}", + flush=True) + + return show + + +def structure_items(folder_or_files: Sequence[str], + suffixes: Sequence[str] = (".cif",), + naming: str = "stem") -> list[tuple[str, str]]: + ''' + List (name, path) for the structures in folders or file lists. + + Hidden files, which on macOS include the "._" companions an archive + leaves behind, are skipped. + + **parameters:** + - folder_or_files: paths to folders or files + + - suffixes: file suffixes read from a folder + + - naming: "stem" uses the file name without its last suffix; + "first_dot" keeps the text before the first dot, the convention + the database commands have always used. + ''' + paths: list[Path] = [] + for entry in folder_or_files: + entry = Path(entry) + if entry.is_dir(): + paths.extend( + p for p in entry.iterdir() + if p.is_file() and not p.name.startswith(".") + and p.suffix.lower() in suffixes + ) + elif entry.exists(): + paths.append(entry) + items = [] + for path in paths: + name = path.stem if naming == "stem" else path.name.split(".")[0] + items.append((name, str(path))) + return items + + +if __name__ == "__main__": # pragma: no cover + sys.exit("mofstructure.batch is a library; use the mofstructure commands") diff --git a/mofstructure/batch_tasks.py b/mofstructure/batch_tasks.py new file mode 100644 index 0000000..efff6f6 --- /dev/null +++ b/mofstructure/batch_tasks.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +''' +Per-structure tasks run by the batch commands, and the step that joins their +checkpoint files into the usual structure database. + +Each task takes the path of one structure and returns json-serialisable data. +They are run by `mofstructure.batch.run_batch` in worker processes, so an +exception is recorded for that structure and a crash or a stall costs only +that structure. + +**database layout:** + The commands write json-lines checkpoints to + + /Structure_Data/_progress/[.part-iiii-of-nnnn].jsonl + + with one of "database", "topology", "porosity" and "oms", and + `consolidate` turns them into the files the commands have always written: + + Structure_Data/sbus_and_linkers.json database + Structure_Data/ligands_data.json database + Structure_Data/porosity_data.json database, porosity + Structure_Data/structure_oms_and_general_info.json database --oms, oms + Structure_Data/topology_data.json database, topology + Structure_Data/fingerprint_data.json database + Structure_Data/run_status.csv every command + + each json with a csv summary beside it. run_status.csv has one row per + structure and command with the status (ok, error, timeout, crashed), the + time taken and the reason for a failure, so a structure that did not + finish is visible instead of missing. +''' +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pandas as pd + +import mofstructure.filetyper as read_write +from mofstructure.batch import CHECKPOINT_DIR, read_checkpoints + +DATABASE_FILES = { + "sbu": "sbus_and_linkers.json", + "ligand": "ligands_data.json", + "porosity": "porosity_data.json", + "oms": "structure_oms_and_general_info.json", + "topology": "topology_data.json", + "fingerprint": "fingerprint_data.json", +} + +# Fields of a topology record that describe the run rather than the net. +TOPOLOGY_UNSAVED = ("source", "status") + + +def _plain(data): + '''Round-trip through json so numpy scalars and similar become plain.''' + return json.loads(json.dumps(read_write.convert_numpy_types(data), + default=str)) + + +def database_name(path: str) -> str: + '''Structure name used by the database commands: text before first dot.''' + return os.path.basename(path).split(".")[0] + + +# --------------------------------------------------------------------------- +# Tasks +# --------------------------------------------------------------------------- + +def topology_task(path, method="auto", all_methods=False, timeout=300, + descriptors=False, symmetry=False): + '''Topology record of one structure, as `mofstructure_topology` reports.''' + from mofstructure.topology import analyse, analyse_methods + + options = {"timeout": timeout or None, "descriptors": descriptors, + "symmetry": symmetry} + if all_methods: + return _plain(analyse_methods(str(path), **options)) + return _plain(analyse(str(path), method=method, **options)) + + +def porosity_task(path, probe_radius=1.86, number_of_steps=10000, + rad_file=None, high_accuracy=True, timeout=1800): + '''Pore geometry of one guest-free structure.''' + from mofstructure import structure + + mof = structure.MOFstructure(filename=str(path)) + return _plain(mof.get_porosity( + probe_radius=probe_radius, number_of_steps=number_of_steps, + rad_file=rad_file, high_accuracy=high_accuracy, timeout=timeout)) + + +def oms_task(path, max_atoms=5000): + '''Open metal sites and general information of one structure.''' + from mofstructure import structure + + mof = structure.MOFstructure(filename=str(path)) + if max_atoms and len(mof.ase_atoms) > max_atoms: + raise ValueError(f"skipped: {len(mof.ase_atoms)} atoms is more than " + f"--max-atoms {max_atoms}") + return _plain(mof.get_oms()) + + +def database_task(path, xyz_path, oms=False, topology=True, + topology_method="auto", topology_timeout=300, + porosity_timeout=1800, probe_radius=1.86, + high_accuracy=True, oms_max_atoms=5000, only=None): + ''' + Every analysis `mofstructure_database` records for one structure. + + Each analysis is attempted on its own, so a failure in one leaves the + others in place; the failures are listed under "errors". `only` limits + the run to the named analyses, which is how a database written by an + earlier version is filled in without repeating what it already holds. + ''' + from mofstructure.scripts import collect_data + + name = database_name(path) + mof = collect_data.structure.MOFstructure(filename=str(path)) + out, errors = {}, {} + wanted = set(only) if only else None + + def attempt(key, function): + if wanted is not None and key not in wanted: + return + try: + out[key] = _plain(function()) + except Exception as exc: # noqa: BLE001 - recorded per analysis + out[key] = None + errors[key] = f"{type(exc).__name__}: {exc}"[:300] + + if topology: + def net(): + record = mof.get_topology(method=topology_method, + timeout=topology_timeout) + return {k: v for k, v in record.items() if k != "status"} + attempt("topology", net) + + attempt("fingerprint", mof.get_ligand_cluster_fingerprint) + + def sbu(): + found = mof.get_sbu() + return None if found is None else collect_data.collect_sbus( + *found, name, xyz_path) + attempt("sbu", sbu) + + def ligand(): + found = mof.get_ligands() + return None if found is None else collect_data.collect_ligand( + found[1], name, xyz_path) + attempt("ligand", ligand) + + attempt("porosity", lambda: mof.get_porosity( + probe_radius=probe_radius, timeout=porosity_timeout, + high_accuracy=high_accuracy)) + + if oms and (wanted is None or "oms" in wanted): + if len(mof.ase_atoms) > oms_max_atoms: + out["oms"] = None + errors["oms"] = f"skipped: more than {oms_max_atoms} atoms" + else: + attempt("oms", mof.get_oms) + + if errors: + out["errors"] = errors + return out + + +# --------------------------------------------------------------------------- +# Joining checkpoints into the database +# --------------------------------------------------------------------------- + +def checkpoint_files(structure_db: Path, command: str) -> list[Path]: + ''' + Every checkpoint part a command has written, whole run or shards. + + A command name ending in ".*" matches every variant, which is how the + topology checkpoints, written one file per deconstruction method, are + collected together. + ''' + folder = Path(structure_db) / CHECKPOINT_DIR + if command.endswith(".*"): + stem = command[:-2] + return sorted(folder.glob(f"{stem}.jsonl")) + \ + sorted(folder.glob(f"{stem}.*.jsonl")) + return sorted(folder.glob(f"{command}.jsonl")) + \ + sorted(folder.glob(f"{command}.part-*.jsonl")) + + +def done_names(structure_db: Path, command: str, + retry_failed: bool = False) -> set[str]: + '''Names a command has already recorded, for resuming a run.''' + records = read_checkpoints(checkpoint_files(structure_db, command)) + if retry_failed: + return {n for n, r in records.items() if r.get("status") == "ok"} + return set(records) + + +def _load(path: Path) -> dict: + if path.exists() and path.stat().st_size: + return read_write.load_data(str(path)) + return {} + + +def _write(records: dict, path: Path, drop=()) -> None: + read_write.write_json_atomic(records, str(path)) + frame = read_write.summary_frame(records, drop=drop) + if not frame.empty: + frame.to_csv(path.with_suffix(".csv")) + + +def consolidate(save_dir: str) -> dict[str, int]: + ''' + Join every checkpoint under `save_dir` into the structure database files. + + Records already in the json files, for example from a run made with an + older version, are kept, and a checkpoint record replaces the stored one + for the same structure. Safe to run at any time, including while a run is + in progress: it only reads the checkpoints. + + **returns:** + python dictionary + Number of structures written per database file. + ''' + from mofstructure.porosity import empty_porosity_record + + structure_db = Path(save_dir) / read_write.STRUCTURE_DATA + structure_db.mkdir(parents=True, exist_ok=True) + status_rows = [] + tables: dict[str, dict] = {} + + def table(key): + if key not in tables: + tables[key] = _load(structure_db / DATABASE_FILES[key]) + return tables[key] + + def note(command, record): + status_rows.append({ + "name": record["name"], "command": command, + "status": record.get("status"), + "elapsed_s": record.get("elapsed_s"), + "detail": record.get("detail"), + "analysis_errors": json.dumps( + (record.get("data") or {}).get("errors")) + if command == "database" and isinstance(record.get("data"), dict) + else None, + "file": record.get("file"), + }) + + for record in read_checkpoints(checkpoint_files(structure_db, + "database")).values(): + note("database", record) + name = record["name"] + if record.get("status") == "ok": + for key, value in record["data"].items(): + if key in DATABASE_FILES: + table(key)[name] = value + else: + table("porosity")[name] = empty_porosity_record(record["status"]) + + topology_parts = checkpoint_files(structure_db, "topology.*") + for record in read_checkpoints(topology_parts).values(): + note("topology", record) + if record.get("status") != "ok": + continue + data = record["data"] + if data and all(isinstance(v, dict) and "method" in v + for v in data.values()): + for method, sub in data.items(): + table("topology")[f"{record['name']}:{method}"] = { + k: v for k, v in sub.items() if k not in TOPOLOGY_UNSAVED} + else: + table("topology")[record["name"]] = { + k: v for k, v in data.items() if k not in TOPOLOGY_UNSAVED} + + for record in read_checkpoints(checkpoint_files(structure_db, + "porosity")).values(): + note("porosity", record) + if record.get("status") == "ok": + table("porosity")[record["name"]] = record["data"] + else: + table("porosity")[record["name"]] = empty_porosity_record( + record["status"]) + + for record in read_checkpoints(checkpoint_files(structure_db, + "oms")).values(): + note("oms", record) + if record.get("status") == "ok": + table("oms")[record["name"]] = record["data"] + + written = {} + for key, records in tables.items(): + path = structure_db / DATABASE_FILES[key] + drop = ["cgd"] if key == "topology" else () + if key == "fingerprint": + read_write.write_json_atomic(records, str(path)) + frame = read_write.summary_frame(records) + if not frame.empty: + columns = [c for c in ("fingerprint_hash", "cluster_units") + if c in frame] + frame[columns].to_csv(path.with_suffix(".csv")) + else: + _write(records, path, drop=drop) + written[DATABASE_FILES[key]] = len(records) + + if status_rows: + pd.DataFrame(status_rows).to_csv(structure_db / "run_status.csv", + index=False) + written["run_status.csv"] = len(status_rows) + return written + + +# --------------------------------------------------------------------------- +# Command-line helpers shared by the batch commands +# --------------------------------------------------------------------------- + +BATCH_EPILOG = """\ +long runs and clusters: + Every structure runs in a separate worker process with a hard time limit, + so a crash or a stall costs only that structure. Results are appended to + /Structure_Data/_progress/*.jsonl as each structure finishes, and + repeating the same command continues where a killed job stopped. The + database files and run_status.csv are rebuilt at the end of every run and + can be rebuilt at any time with mofstructure_merge . + + Split a folder across a cluster array with --shard slurm/N (or i/N); every + task writes its own part, and mofstructure_merge joins them. +""" + + +def add_batch_arguments(parser, max_time=7200): + '''Add the options every batch command shares.''' + group = parser.add_argument_group("batch execution") + group.add_argument("-j", "--workers", type=int, default=1, + help="worker processes (default 1)") + group.add_argument("--max-time", type=float, default=max_time, + help="hard wall-clock limit per structure in seconds; " + "the worker is killed and the structure recorded " + f"as a timeout (default {max_time:g}, 0 for none)") + group.add_argument("--memory-limit", type=float, default=None, + help="memory limit per worker in GB (Linux)") + group.add_argument("--shard", type=str, default=None, + help="process one part of the folder, given as i/N " + "(0-based) or slurm/N to read " + "SLURM_ARRAY_TASK_ID") + group.add_argument("--retry-failed", action="store_true", + help="run again structures recorded as error, " + "timeout or crashed") + group.add_argument("--recycle", type=int, default=50, + help="structures per worker before it is replaced " + "(default 50)") + group.add_argument("--merge-every", type=float, default=60, + help="rebuild the json and csv files from the " + "checkpoints every this many minutes during the " + "run (default 60, 0 only at the end); ignored " + "with --shard") + return parser + + +def periodic_merge(save_dir, minutes, on_record=None): + ''' + Wrap a progress callback so that the database files are rebuilt from the + checkpoints at most every `minutes` minutes while a run is going. + + The checkpoints grow by one line per structure throughout the run; the + json files are single objects that must be rewritten whole, so they are + refreshed on a timer rather than after every structure. + ''' + import time + + last = [time.monotonic()] + + def callback(record, done, total): + if on_record: + on_record(record, done, total) + if minutes and done < total and \ + time.monotonic() - last[0] >= minutes * 60: + written = consolidate(save_dir) + last[0] = time.monotonic() + print(f"database files refreshed after {done}/{total} " + f"structures ({sum(written.values())} records)", flush=True) + + return callback + + +def run_command(command, items, task, kwargs, save_dir, args, + on_record=None, extra_skip=()): + ''' + Run a batch command with the shared options and rebuild the database. + + **returns:** + python dictionary + Count of records by status for this run. + ''' + from mofstructure.batch import (parse_shard, progress_printer, run_batch, + select_shard, shard_suffix) + + structure_db = Path(save_dir) / read_write.STRUCTURE_DATA + shard = parse_shard(args.shard) + selected = select_shard(items, shard) + skip = done_names(structure_db, command, retry_failed=args.retry_failed) + skip |= set(extra_skip) + checkpoint = structure_db / CHECKPOINT_DIR / \ + f"{command}{shard_suffix(shard)}.jsonl" + todo = sum(1 for name, _ in selected if name not in skip) + print(f"{command}: {len(selected)} structures" + f"{' in this shard' if shard else ''}, " + f"{len(selected) - todo} already recorded, {todo} to run, " + f"{args.workers} worker(s)", flush=True) + callback = on_record or progress_printer() + minutes = getattr(args, "merge_every", 0) + if minutes and not shard: + callback = periodic_merge(save_dir, minutes, callback) + tally = run_batch( + selected, task, checkpoint, kwargs=kwargs, workers=args.workers, + timeout=args.max_time or None, memory_gb=args.memory_limit, + recycle=args.recycle, skip=skip, on_record=callback, + ) + print(f"{command}: this run {tally or 'nothing to do'}", flush=True) + if shard: + # Array tasks finish at nearly the same time, and each rebuilding + # the same large files would only repeat the work, so the parts are + # joined once, after the last task, by mofstructure_merge. + print(f"shard written to {checkpoint}; join all shards with " + f"mofstructure_merge {save_dir}", flush=True) + return tally + written = consolidate(save_dir) + for name, count in written.items(): + print(f" {structure_db / name}: {count} records", flush=True) + return tally diff --git a/mofstructure/filetyper.py b/mofstructure/filetyper.py index 9a01e1d..fbb879b 100644 --- a/mofstructure/filetyper.py +++ b/mofstructure/filetyper.py @@ -10,6 +10,7 @@ __author__ = "Dr. Dinga Wonanke" __status__ = "production" import os +import tempfile import pickle import csv import json @@ -96,26 +97,81 @@ def json_to_ase_atom(data, encoder, filename): return -def append_json_atom(data, encoder, filename): +def write_json_atomic(data, filename, encoder=None, indent=4, sort_keys=True): ''' - append a data containing an ase atom object + Write a json file so that it is never left half written. + + The data go to a temporary file in the same directory, which is flushed + to disk and then renamed over the target. A rename within one file system + is atomic, so a job killed at any moment leaves either the old file or the + new one, never a truncated file that cannot be loaded. + + **parameters:** + - data: json-serialisable object + + - filename: str + Destination. + + - encoder: json.JSONEncoder subclass, optional + + - indent: int or None + + - sort_keys: bool ''' - if not os.path.exists(filename): - with open(filename, 'w', encoding='utf-8') as f_obj: - f_obj.write('{}') - elif os.path.getsize(filename) == 0: - with open(filename, 'w', encoding='utf-8') as f_obj: - f_obj.write('{}') - with open(filename, 'r+', encoding='utf-8') as f_obj: - # First we load existing data into a dict. - file_data = json.load(f_obj) - # Join new_data with file_data inside emp_details - file_data.update(data) - # Sets file's current position at offset. - f_obj.seek(0) - # convert back to json. + directory = os.path.dirname(os.path.abspath(filename)) or "." + os.makedirs(directory, exist_ok=True) + handle, temporary = tempfile.mkstemp( + prefix=".tmp_", suffix=".json", dir=directory) + try: + with os.fdopen(handle, "w", encoding="utf-8") as file: + json.dump(data, file, indent=indent, sort_keys=sort_keys, + cls=encoder) + file.flush() + os.fsync(file.fileno()) + os.replace(temporary, filename) + except BaseException: + if os.path.exists(temporary): + os.remove(temporary) + raise - json.dump(data, f_obj, indent=4, sort_keys=False, cls=encoder) + +def _load_json_or_empty(filename): + ''' + Load a json mapping for updating, treating a missing or empty file as {}. + + A file that exists but cannot be parsed is not silently replaced, since + that would discard every record it holds. The error names the file so it + can be repaired or moved aside. + ''' + if not os.path.exists(filename) or os.path.getsize(filename) == 0: + return {} + try: + with open(filename, encoding="utf-8") as file: + return json.load(file) + except json.JSONDecodeError as error: + raise ValueError( + f"{filename} is not valid json ({error}). It was probably " + "truncated by an interrupted run with an older version of " + "mofstructure. Move it aside and rebuild it with " + "mofstructure_merge, or repair it by hand." + ) from error + + +def append_json_atom(data, encoder, filename): + ''' + Add records containing ASE atoms objects to a json file. + + **parameters:** + - data: python dictionary + New records; existing keys are overwritten. + + - encoder: json.JSONEncoder subclass able to serialise the atoms. + + - filename: str + ''' + file_data = _load_json_or_empty(filename) + file_data.update(data) + write_json_atomic(file_data, filename, encoder=encoder, sort_keys=False) def summary_frame(records, drop=()): @@ -147,25 +203,21 @@ def summary_frame(records, drop=()): def append_json(new_data, filename): ''' - append a new data in an existing json file - ''' - if not os.path.exists(filename): - with open(filename, 'w', encoding='utf-8') as file: - file.write('{}') - elif os.path.getsize(filename) == 0: - with open(filename, 'w', encoding='utf-8') as file: - file.write('{}') - with open(filename, 'r+', encoding='utf-8') as file: - # First we load existing data into a dict. - file_data = json.load(file) - # Overwrite existing keys with new_data - file_data.update(new_data) - # Sets file's current position at offset. - file.seek(0) - # convert back to json. - json.dump(file_data, file, indent=4, sort_keys=True) - # drop any tail left by a longer previous version of the file. - file.truncate() + Add records to a json file, overwriting existing keys. + + The file is rewritten atomically, so an interrupted job cannot leave it + truncated. Rewriting a large file after every structure is slow, which is + why the batch commands record progress in json-lines files and only call + this when they consolidate. + + **parameters:** + - new_data: python dictionary + + - filename: str + ''' + file_data = _load_json_or_empty(filename) + file_data.update(new_data) + write_json_atomic(file_data, filename) def read_json(file_name): diff --git a/mofstructure/generate_cgd.py b/mofstructure/generate_cgd.py index 8b888e5..2c5d604 100644 --- a/mofstructure/generate_cgd.py +++ b/mofstructure/generate_cgd.py @@ -451,9 +451,6 @@ def base_edges_with_shifts( u = atom_to_comp[i] v = atom_to_comp[j] - if u == v: - continue - sij = np.array([ int(pair[2]) if len(pair) > 2 else 0, int(pair[3]) if len(pair) > 3 else 0, @@ -466,6 +463,14 @@ def base_edges_with_shifts( tuv = iu - iv + sij sx, sy, sz = int(tuv[0]), int(tuv[1]), int(tuv[2]) + # A cut between two atoms of one component joins that unit to one of + # its own periodic images. In a small cell this is how a building unit + # meets its neighbour, so it is an edge of the net and must be kept; + # in the supercell the same bond joins two different components. Only + # a cut that returns to the same image carries no connectivity. + if u == v and not (sx or sy or sz): + continue + edges_raw.append((u, v, sx, sy, sz)) seen = set() @@ -1127,8 +1132,10 @@ def splice_two_connected_nodes(edges, candidates): if u not in removed and v not in removed ] for node in removed: + # incident[node] holds each neighbour with its translation relative + # to the spliced vertex, so b sits at sb - sa as seen from a (a, sa), (b, sb) = incident[node] - kept.append((a, b, sa[0] - sb[0], sa[1] - sb[1], sa[2] - sb[2])) + kept.append((a, b, sb[0] - sa[0], sb[1] - sa[1], sb[2] - sa[2])) return dedup_periodic_edges(kept), removed @@ -1859,10 +1866,12 @@ def node_id(key, is_organic): incidences.append((neigh, tuple(int(x) for x in shift))) for a in range(len(incidences)): for b in range(a + 1, len(incidences)): + # kept_offsets[(o, x)] is the image of x relative to + # the contracted atom, so v sits at sv - su from u (u_atom, su), (v_atom, sv) = incidences[a], incidences[b] edges.append(( atom_node[u_atom], atom_node[v_atom], - su[0] - sv[0], su[1] - sv[1], su[2] - sv[2], + sv[0] - su[0], sv[1] - su[1], sv[2] - su[2], )) # linkers contract to an edge (ditopic) or a node (polytopic) between the diff --git a/mofstructure/graph_net/canonical.py b/mofstructure/graph_net/canonical.py index 318536e..d29db91 100644 --- a/mofstructure/graph_net/canonical.py +++ b/mofstructure/graph_net/canonical.py @@ -83,6 +83,7 @@ UnstableNetError, barycentric_placement, edge_vectors, + find_collisions, find_local_collisions, ) from mofstructure.graph_net.lattice import ( @@ -634,8 +635,10 @@ def canonical_form( `PeriodicGraph.components()` first if it may be disconnected. - check_stability: bool - When True, refuse to canonicalise a net that is not locally - stable rather than returning a key that depends on tie-breaking. + When True, refuse to canonicalise a net that is not stable, that + is one in which two vertices share a position in the barycentric + placement, rather than returning a key that depends on the cell + or on tie-breaking. - return_winners: bool When True, also return the reduced graph and every traversal that @@ -655,15 +658,35 @@ def canonical_form( **raises:** UnstableNetError - If the net is not locally stable and `check_stability` is set. + If the net is not stable and `check_stability` is set. ValueError If the quotient graph is disconnected or has no valid basis. ''' - reduced = primitive(graph.reduced()).reduced() - dim = reduced.dim - if dim == 0: + base = graph.reduced() + if base.dim == 0: raise ValueError("the graph has no periodicity and is not a net") + # The primitive reduction finds extra translations from the barycentric + # placement, which identifies a translation uniquely only when no two + # vertices share a position. With a collision, a translation composed with + # a swap of the colliding vertices has the same displacement, so neither + # the primitive cell nor the key is well defined and the result would + # depend on the cell the net was written in. Such a net is refused, as + # Systre refuses it. Instability is a property of the net, so the test + # gives the same answer in every cell. + base_placement = barycentric_placement(base) + if check_stability: + clashes = find_collisions(base, base_placement) + if clashes: + raise UnstableNetError( + "net is not stable: vertices collide in the barycentric " + "placement, so no canonical key exists", + collisions=clashes, + ) + + reduced = primitive(base, base_placement).reduced() + dim = reduced.dim + # The placement is a function of the reduced graph alone, and both the # stability check and the edge vectors need it, so it is solved once and # handed on. It is the most expensive rational arithmetic in the routine. diff --git a/mofstructure/graph_net/primitive.py b/mofstructure/graph_net/primitive.py index d3283f7..8fc6e1d 100644 --- a/mofstructure/graph_net/primitive.py +++ b/mofstructure/graph_net/primitive.py @@ -146,7 +146,10 @@ def additional_translations( return found -def primitive(graph: PeriodicGraph) -> PeriodicGraph: +def primitive( + graph: PeriodicGraph, + placement: Sequence[RatVec] | None = None, +) -> PeriodicGraph: ''' Rewrite a net on its primitive cell. @@ -159,6 +162,11 @@ def primitive(graph: PeriodicGraph) -> PeriodicGraph: - graph: PeriodicGraph Net whose quotient graph is connected. + - placement: sequence or None + Barycentric placement of `graph`; computed when omitted. Passing + it avoids solving the placement twice when the caller has + already needed it. + **returns:** PeriodicGraph Equivalent net with the smallest number of vertex orbits and the @@ -168,7 +176,7 @@ def primitive(graph: PeriodicGraph) -> PeriodicGraph: if dim == 0 or graph.n_vertices == 1: return graph - placement = barycentric_placement(graph) + placement = placement if placement is not None else barycentric_placement(graph) extra = additional_translations(graph, placement) if not extra: return graph diff --git a/mofstructure/scripts/collect_data.py b/mofstructure/scripts/collect_data.py index 3d19852..7e49494 100644 --- a/mofstructure/scripts/collect_data.py +++ b/mofstructure/scripts/collect_data.py @@ -141,29 +141,46 @@ def collect_ligand(organic_ligands, base_name, xyz_path): return read_write.convert_numpy_types(data_to_json) +def _legacy_state(structure_db, topology): + ''' + What a database written before the batch runner already holds. + + **returns:** + (complete, backfill) + Names with every requested analysis, which are skipped, and + names with building units but no fingerprint or net, for which + only the missing analyses are run, as the command has always done. + ''' + def keys(name): + path = os.path.join(structure_db, name) + if not os.path.exists(path) or not os.path.getsize(path): + return set() + return {k for k, v in read_write.load_data(path).items() if v} + + sbu = keys('sbus_and_linkers.json') + complete = sbu & keys('fingerprint_data.json') + if topology: + complete &= keys('topology_data.json') + return complete, sbu - complete + + def compile_data(cif_files, result_folder, verbose=False, oms=False, topology=True, topology_method='auto', porosity_timeout=default_porosity_timeout, - probe_radius=1.86, high_accuracy=True): + probe_radius=1.86, high_accuracy=True, workers=1, + max_time=7200, memory_limit=None, shard=None, + retry_failed=False, recycle=50, topology_timeout=300, + merge_every=60): ''' - A workflow to remove guest, compute porosity and deconstructure - mofs and creates a MOF database. The function starts with checking and - removing any unbound guest molecule present in the MOF. After that it - computed the porosity of all the MOFs and load them in a single csv. - Finally the deconstructs the MOFs into the various building units and - creates a three json files - - 1. ase_atoms_building_units.json - Json file containing ase atom object of all the building uints and for - each ase atom object there are additional information in the info[] key. + Build or extend a structure database from a list of structure files. - 2. sbus_and_linkers.json - A json file containing all the information about the linkers and metal - sbu. - - 3. cluster_and_ligands.json - A json file containing all the information about the ligands and metal - cluster. + For each structure the guests are removed and the building units, ligand + cluster fingerprint, porosity, optionally the open metal sites and the + underlying net are recorded. Every structure runs in a separate worker + process with a hard time limit (see `mofstructure.batch`), results are + checkpointed as they finish, and the json and csv files under + `Structure_Data` are rebuilt at the end. Repeating the call continues an + interrupted run. **parameters:** - cif_files: list of str @@ -179,180 +196,106 @@ def compile_data(cif_files, result_folder, verbose=False, oms=False, Record open metal sites. - topology: bool - Compute the underlying net. Identification runs in process and - costs about a sixth of the run, so it is on by default. + Compute the underlying net. - topology_method: str - Node definition passed to `MOFstructure.get_topology`, or "auto" - to choose it from the structure. + Node definition passed to `MOFstructure.get_topology`, or "auto". - porosity_timeout: float - Seconds allowed per structure before zeo++ is killed. + Seconds allowed for zeo++ per structure. - probe_radius: float Probe radius used for the porosity. - high_accuracy: bool Run zeo++ at high accuracy. - ''' - if not os.path.exists(result_folder): - os.makedirs(result_folder) - structure_db = os.path.join(result_folder, read_write.STRUCTURE_DATA) - if not os.path.exists(structure_db): - os.makedirs(structure_db) - xyz_path = os.path.join(result_folder, read_write.XYZ_DB) - if not os.path.exists(xyz_path): - os.makedirs(xyz_path) - - path2sbu = os.path.join(structure_db, 'sbus_and_linkers.json') - path2ligand = os.path.join(structure_db, 'ligands_data.json') - porosity_path = os.path.join(structure_db, 'porosity_data.json') - oms_path = os.path.join(structure_db, 'structure_oms_and_general_info.json') - topology_path = os.path.join(structure_db, 'topology_data.json') - fingerprint_path = os.path.join(structure_db, 'fingerprint_data.json') - - if os.path.exists(path2sbu): - all_sbu_data = read_write.load_data(path2sbu) - else: - all_sbu_data = {} - if os.path.exists(path2ligand): - all_ligand_data = read_write.load_data(path2ligand) - else: - all_ligand_data = {} + - workers: int + Worker processes. - if os.path.exists(porosity_path): - all_porosity_data = read_write.load_data(porosity_path) - else: - all_porosity_data = {} + - max_time: float + Hard wall-clock limit per structure in seconds. - if os.path.exists(oms_path): - all_oms_data = read_write.load_data(oms_path) - else: - all_oms_data = {} + - memory_limit: float or None + Memory limit per worker in GB (Linux). - if os.path.exists(topology_path): - all_topology_data = read_write.load_data(topology_path) - else: - all_topology_data = {} + - shard: str or None + "i/N" to process one part of the list. - if os.path.exists(fingerprint_path): - all_fingerprint_data = read_write.load_data(fingerprint_path) - else: - all_fingerprint_data = {} - - seen = list(all_sbu_data.keys()) - - for cif_file in cif_files: - try: - mof_object = structure.MOFstructure(filename=cif_file) - - base_name = os.path.basename(cif_file).split('.')[0] - # Computed outside the "already done" branch below so that a - # database built before this ran, or built without the flag, is - # filled in rather than skipped. - if topology and not all_topology_data.get(base_name): - try: - record = mof_object.get_topology(method=topology_method) - # `status` describes the run rather than the net, and - # mofstructure_topology already leaves it out of this - # file. Dropping it here too keeps one shape whichever - # command wrote the row. `get_topology` still returns it. - all_topology_data[base_name] = { - field: value for field, value in record.items() - if field != "status" - } - except Exception: - # A failed net must not discard the rest of the record. - all_topology_data[base_name] = None - read_write.append_json(all_topology_data, topology_path) - - if not all_fingerprint_data.get(base_name): - try: - all_fingerprint_data[base_name] = ( - mof_object.get_ligand_cluster_fingerprint()) - except Exception: - # A fingerprint failure must not discard porosity, SBU, - # ligand or OMS results for an otherwise readable MOF. - all_fingerprint_data[base_name] = None - read_write.append_json(all_fingerprint_data, fingerprint_path) - - if base_name not in seen: - print("======================================\n") - print(f' processing : {base_name} \n') - print("======================================") - sbu_data = mof_object.get_sbu() - ligand_data = mof_object.get_ligands() - - if sbu_data is not None: - metal_sbus, organic_sbus = sbu_data - data_to_json = collect_sbus(metal_sbus, - organic_sbus, - base_name, - xyz_path) - all_sbu_data[base_name] = data_to_json - read_write.append_json(all_sbu_data, path2sbu) - - if ligand_data is not None: - metal_clusters, organic_ligands = ligand_data - - data_to_json = collect_ligand(organic_ligands, base_name, xyz_path) - all_ligand_data[base_name] = data_to_json - read_write.append_json(all_ligand_data, path2ligand) - porosity = mof_object.get_porosity( - probe_radius=probe_radius, - timeout=porosity_timeout, - high_accuracy=high_accuracy) - # The record has the same keys whether or not zeo++ managed - # the structure, so a failure is a row of missing values - # carrying its reason rather than a gap in the table. - all_porosity_data[base_name] = porosity - read_write.append_json(all_porosity_data, porosity_path) - - if len(mof_object.ase_atoms) > 5000: - print('system size too large, will run out of application memory') - print('so will skip') - all_oms_data[base_name] = None - continue - all_oms_data[base_name] = mof_object.get_oms() - read_write.append_json(all_oms_data, oms_path) - else: - print("======================================\n") - print(f" !!! {base_name} is already done !!! \n") - print("======================================") - except Exception: - pass - - read_write.summary_frame(all_porosity_data).to_csv(structure_db+'/porosity_data.csv') - - if all_topology_data: - # the cgd net is a multi line block, so keep it out of the summary - read_write.summary_frame(all_topology_data, drop=['cgd']).to_csv( - structure_db+'/topology_data.csv') - - if all_fingerprint_data: - # The full nested chemical/connectivity description remains in JSON; - # the compact columns used for database indexing go into the CSV. - fingerprint_summary = read_write.summary_frame(all_fingerprint_data) - if not fingerprint_summary.empty: - fingerprint_summary[ - ['fingerprint_hash', 'cluster_units'] - ].to_csv(structure_db+'/fingerprint_data.csv') + - retry_failed: bool + Run again structures recorded as failed. + - recycle: int + Structures per worker before it is replaced. + + - topology_timeout: int + Seconds allowed for net identification. + + - merge_every: float + Minutes between refreshes of the json and csv files during the + run; 0 refreshes only at the end. + ''' + from types import SimpleNamespace + from mofstructure.batch import progress_printer + from mofstructure.batch_tasks import database_name, run_command + + structure_db = os.path.join(result_folder, read_write.STRUCTURE_DATA) + xyz_path = os.path.join(result_folder, read_write.XYZ_DB) + os.makedirs(structure_db, exist_ok=True) + os.makedirs(xyz_path, exist_ok=True) + + items = [(database_name(f), f) for f in cif_files] + options = SimpleNamespace(workers=workers, max_time=max_time, + memory_limit=memory_limit, shard=shard, + retry_failed=retry_failed, recycle=recycle, + merge_every=merge_every) + kwargs = dict(xyz_path=xyz_path, oms=oms, topology=topology, + topology_method=topology_method, + topology_timeout=topology_timeout, + porosity_timeout=porosity_timeout, + probe_radius=probe_radius, high_accuracy=high_accuracy) + + printer = progress_printer() + + def quiet(record, done, total): + # Failures always, otherwise every hundredth structure, so a long + # run leaves a readable log. + if record['status'] != 'ok' or done == total or done % 100 == 0: + printer(record, done, total) + + complete, backfill = _legacy_state(structure_db, topology) + if backfill: + # Building units recorded by an earlier version: add only the + # fingerprint and the net, keeping the stored porosity and units. + only = ['fingerprint'] + (['topology'] if topology else []) + run_command('database', [i for i in items if i[0] in backfill], + 'mofstructure.batch_tasks:database_task', + {**kwargs, 'only': only}, result_folder, options, + on_record=None if verbose else quiet, + extra_skip=complete) + run_command('database', [i for i in items if i[0] not in backfill], + 'mofstructure.batch_tasks:database_task', + kwargs, result_folder, options, + on_record=None if verbose else quiet, + extra_skip=complete) if verbose: print(f"Saved results to {result_folder}") - return def main(): ''' Command line interface to deconstruct MOFs to building units, - compute porosity and open metal sites + compute porosity, topology, fingerprints and open metal sites. ''' + from mofstructure.batch_tasks import BATCH_EPILOG, add_batch_arguments + parser = argparse.ArgumentParser( - description='Run work_flow function with optional verbose output') + description='Build a structure database from a folder of ' + 'structures.', + epilog=BATCH_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('cif_folder', type=str, - help='list of cif files. like glob') + help='folder of cif files') parser.add_argument('--oms', action='store_true', help='run oms') @@ -371,6 +314,9 @@ def main(): 'chooses it from the structure, so a folder of ' 'MOFs, COFs and zeolites is handled in one go ' '(--topology_method is a deprecated alias)') + parser.add_argument('--topology_timeout', type=int, default=300, + help='seconds allowed for net identification ' + '(default: 300)') parser.add_argument('-pr', '--probe_radius', default=1.86, type=float, help='probe radius used for the porosity ' '(default: 1.86)') @@ -386,10 +332,20 @@ def main(): default=read_write.DEFAULT_SAVE_DIR, help='directory to save output files') parser.add_argument('-v', '--verbose', action='store_true', - help='print verbose output') + help='print a line for every structure') + add_batch_arguments(parser, max_time=7200) args = parser.parse_args() cif_files = [os.path.join(args.cif_folder, f) for f in os.listdir( - args.cif_folder) if f.endswith('.cif')] + args.cif_folder) if f.endswith('.cif') and not f.startswith('.')] compile_data(cif_files, args.save_dir, args.verbose, args.oms, args.topology, args.topology_method, args.porosity_timeout, - args.probe_radius, args.accuracy == 'high') + args.probe_radius, args.accuracy == 'high', + workers=args.workers, max_time=args.max_time, + memory_limit=args.memory_limit, shard=args.shard, + retry_failed=args.retry_failed, recycle=args.recycle, + merge_every=args.merge_every, + topology_timeout=args.topology_timeout) + + +if __name__ == '__main__': + main() diff --git a/mofstructure/scripts/collect_oms.py b/mofstructure/scripts/collect_oms.py index c83ca17..ce91371 100644 --- a/mofstructure/scripts/collect_oms.py +++ b/mofstructure/scripts/collect_oms.py @@ -3,100 +3,94 @@ Command line entry point for `mofstructure_oms`. Reports the open metal sites of every structure in a folder, along with the -general information about each framework, and collects them into one table. +general information about each framework, and collects them into +`/Structure_Data/structure_oms_and_general_info.json` and its csv, +the file `mofstructure_database --oms` fills. Every structure runs in a worker +process with a hard time limit, results are checkpointed as they finish and +repeating the command continues an interrupted run (see +`mofstructure.batch`). ''' from __future__ import print_function __author__ = "Dr. Dinga Wonanke" __status__ = "production" import os +import sys import argparse -import pandas as pd -from mofstructure import structure import mofstructure.filetyper as read_write -def compile_data(cif_files, result_folder, verbose=False): +def compile_data(cif_files, result_folder, verbose=False, max_atoms=5000, + workers=1, max_time=3600, memory_limit=None, shard=None, + retry_failed=False, recycle=50, merge_every=60): ''' - A workflow to remove guest and compute open metal sites from a folder containing - cif files and create a database of open metal sites. This code can work for all - periodic systems containing metals. + Remove guests and record the open metal sites of a list of structures. - The function starts with checking and removing any unbound - guest molecule present in the periodic system. + **parameters:** + - cif_files: list of str - 1. metal_info.json - A json file containing all information about the metals in the system. - This file can easily be converted to csv format. + - result_folder: str + Directory the database is written to. - :: - Parameters - ---------- - cif_file : a cif file or any ase readable file containing a MOF. - result_folder : path to output folder - ''' - metal_info = {} - seen = [] - if not os.path.exists(result_folder): - os.makedirs(result_folder) - else: - try: - metal_info = read_write.load_data(result_folder+'/structure_oms_and_general_info.json') - seen = list(metal_info.keys()) - except Exception: - pass + - verbose: bool - for cif_file in cif_files: - try: - base_name = os.path.basename(cif_file).split('.')[0] - print("======================================\n") - print(f' processing : {base_name} \n') - print("======================================") - if base_name not in seen: - print(f'processing: {base_name}') - mof_object = structure.MOFstructure(filename=cif_file) - if len(mof_object.ase_atoms) > 5000: - print('system size too large, will run out of application memory') - print('so will skip') - continue - oms_data = mof_object.get_oms() + - max_atoms: int + Structures with more atoms are skipped, since the analysis can + exhaust memory on very large cells; the skip is recorded. - metal_info[base_name] = oms_data - read_write.append_json(metal_info, result_folder+'/structure_oms_and_general_info.json') - else: - print("======================================\n") - print(f" Already Done {base_name}\n") - print("======================================") - except Exception: - print(f'failed processing {base_name}') - pass - - data_f = pd.DataFrame.from_dict(metal_info, orient='index') - data_f.to_csv(result_folder+'/structure_oms_and_general_info.csv') + - workers, max_time, memory_limit, shard, retry_failed, recycle, + merge_every: + batch settings, see `mofstructure.batch_tasks.add_batch_arguments`. + ''' + from types import SimpleNamespace + from mofstructure.batch_tasks import database_name, run_command + os.makedirs(os.path.join(result_folder, read_write.STRUCTURE_DATA), + exist_ok=True) + items = [(database_name(f), f) for f in cif_files] + options = SimpleNamespace(workers=workers, max_time=max_time, + memory_limit=memory_limit, shard=shard, + retry_failed=retry_failed, recycle=recycle, + merge_every=merge_every) + run_command('oms', items, 'mofstructure.batch_tasks:oms_task', + {'max_atoms': max_atoms}, result_folder, options) if verbose: print(f"Saved results to {result_folder}") - return def main(): ''' - mofstructure command line interface for computiong open metal sites + mofstructure command line interface for computing open metal sites ''' + from mofstructure.batch_tasks import BATCH_EPILOG, add_batch_arguments + parser = argparse.ArgumentParser( - description='Run work_flow function with optional verbose output') + description='Record the open metal sites of a folder of structures.', + epilog=BATCH_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('cif_folder', type=str, - help='list of cif files. like glob') - + help='folder of cif files, or one cif file') parser.add_argument('-s', '--save_dir', type=str, default=read_write.DEFAULT_SAVE_DIR, help='directory to save output files') + parser.add_argument('--max-atoms', type=int, default=5000, + help='skip structures with more atoms ' + '(default 5000, 0 for no limit)') parser.add_argument('-v', '--verbose', action='store_true', help='print verbose output') + add_batch_arguments(parser, max_time=3600) args = parser.parse_args() if os.path.isdir(args.cif_folder): cif_files = [os.path.join(args.cif_folder, f) for f in os.listdir(args.cif_folder) - if f.endswith('.cif')] + if f.endswith('.cif') and not f.startswith('.')] else: cif_files = [args.cif_folder] - compile_data(cif_files, args.save_dir, args.verbose) + compile_data(cif_files, args.save_dir, args.verbose, args.max_atoms, + workers=args.workers, max_time=args.max_time, + memory_limit=args.memory_limit, shard=args.shard, + retry_failed=args.retry_failed, recycle=args.recycle, + merge_every=args.merge_every) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/mofstructure/scripts/collect_porosity.py b/mofstructure/scripts/collect_porosity.py index 0fab58a..415344a 100644 --- a/mofstructure/scripts/collect_porosity.py +++ b/mofstructure/scripts/collect_porosity.py @@ -12,10 +12,8 @@ __status__ = "production" import os import argparse -from mofstructure import structure import mofstructure.filetyper as read_write from mofstructure.porosity import DEFAULT_TIMEOUT as porosity_timeout -from mofstructure.porosity import empty_porosity_record def compile_data(cif_files, @@ -25,104 +23,85 @@ def compile_data(cif_files, rad_file=None, verbose=False, timeout=porosity_timeout, - high_accuracy=True): + high_accuracy=True, + workers=1, + max_time=None, + memory_limit=None, + shard=None, + retry_failed=False, + recycle=50, merge_every=60): ''' - A workflow to remove guest and compute porosity from any porous periodic system. - The results is written in both a json format and csv file format. - The function starts with checking and removing any unbound - guest molecule present in the porous. After that it computed the porosity - of all the systems and load them in a single csv. The function always computes - the high accuracy calculation. - - 1. ase_atoms_building_units.json - Json file containing ase atom object of all the building uints and for - each ase atom object there are additional information in the info[] key. - - 2. sbus_and_linkers.json - A json file containing all the information about the linkers and metal - sbu. - - 3. cluster_and_ligands.json - A json file containing all the information about the ligands and metal - cluster. - :: - Parameters - ---------- - cif_file : a cif file or any ase readable file containing a MOF. - result_folder : path to output folder - ''' - if not os.path.exists(result_folder): - os.makedirs(result_folder) - structure_db = os.path.join(result_folder, read_write.STRUCTURE_DATA) - if not os.path.exists(structure_db): - os.makedirs(structure_db) - porosity_path = os.path.join(structure_db, 'porosity_data.json') - if os.path.exists(porosity_path): - all_porosity_data = read_write.load_data(porosity_path) - else: - all_porosity_data = {} - - seen = list(all_porosity_data.keys()) - - for cif_file in cif_files: - base_name = os.path.basename(cif_file).split('.')[0] - try: - mof_object = structure.MOFstructure(filename=cif_file) - if base_name not in seen: - print("======================================\n") - print(f' processing : {base_name} \n') - print("======================================") - - porosity = mof_object.get_porosity( - probe_radius=probe_radius, - number_of_steps=number_of_steps, - rad_file=rad_file, - timeout=timeout, - high_accuracy=high_accuracy) - # The record has the same keys whether or not zeo++ managed - # the structure, so a failure is a row of missing values - # carrying its reason rather than a gap in the table. - all_porosity_data[base_name] = porosity - read_write.append_json(all_porosity_data, porosity_path) - else: - print("======================================\n") - print(f" !!! {base_name} is already done !!! \n") - print("======================================") - except Exception as error: - # A structure that cannot even be read still belongs in the - # table, with the reason attached. Skipping it silently leaves - # the caller unable to tell a missing row from a failed one. - print(f'!!! {base_name} could not be analysed: ' - f'{type(error).__name__}: {error} !!!') - all_porosity_data[base_name] = empty_porosity_record( - f'failed:{type(error).__name__}') - - read_write.summary_frame(all_porosity_data).to_csv( - structure_db + '/porosity_data.csv') + Remove guests and compute the pore geometry of every structure in a list. + + Every structure runs in a separate worker process (see + `mofstructure.batch`), so a crash or a stall in zeo++ or in the guest + removal costs only that structure. Results are checkpointed as they + finish and `Structure_Data/porosity_data.json` and its csv are rebuilt at + the end; a structure that failed has a row of missing values and its + reason in `porosity_status`. Repeating the call continues an interrupted + run. + + **parameters:** + - cif_files: list of str + + - result_folder: str + Directory the database is written to. + + - probe_radius, number_of_steps, rad_file, high_accuracy: + zeo++ settings, see `MOFstructure.get_porosity`. + - timeout: float + Seconds allowed for zeo++ per structure. + + - workers, max_time, memory_limit, shard, retry_failed, recycle: + batch settings, see `mofstructure.batch_tasks.add_batch_arguments`. + max_time defaults to the zeo++ timeout plus ten minutes for the + guest removal. + ''' + from types import SimpleNamespace + from mofstructure.batch_tasks import database_name, run_command + + os.makedirs(os.path.join(result_folder, read_write.STRUCTURE_DATA), + exist_ok=True) + items = [(database_name(f), f) for f in cif_files] + options = SimpleNamespace( + workers=workers, + max_time=max_time if max_time is not None else timeout + 600, + memory_limit=memory_limit, shard=shard, + retry_failed=retry_failed, recycle=recycle, + merge_every=merge_every) + kwargs = dict(probe_radius=probe_radius, number_of_steps=number_of_steps, + rad_file=rad_file, high_accuracy=high_accuracy, + timeout=timeout) + run_command('porosity', items, 'mofstructure.batch_tasks:porosity_task', + kwargs, result_folder, options) if verbose: print(f"Saved results to {result_folder}") - return def main(): ''' - mofstructure command line interface to compute the porosity of any periodic - system. + mofstructure command line interface to compute the porosity of any + periodic system. ''' + from mofstructure.batch_tasks import BATCH_EPILOG, add_batch_arguments + parser = argparse.ArgumentParser( - description='Run work_flow function with optional verbose output') + description='Compute the pore geometry of a folder of structures.', + epilog=BATCH_EPILOG, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('cif_folder', type=str, - help='list of cif files. like glob') + help='folder of cif files') parser.add_argument('-pr', '--probe_radius', default=1.86, type=float, help='probe radius (default: 1.86)') parser.add_argument('-ns', '--number_of_steps', default=10000, type=int, - help='Number of GCMC simulation cycles (default: 10000)') + help='Number of Monte Carlo samples (default: 10000)') parser.add_argument('-rf', '--rad_file', default=None, type=str, - help='path to radii file (default: None). rad file must have .rad file extension') + help='path to radii file (default: None). rad file ' + 'must have .rad file extension') parser.add_argument('-a', '--accuracy', default='high', choices=['high', 'low'], help='zeo++ accuracy (default: high). low uses the ' @@ -138,9 +117,18 @@ def main(): help='directory to save output files') parser.add_argument('-v', '--verbose', action='store_true', help='print verbose output') + add_batch_arguments(parser, max_time=porosity_timeout + 600) args = parser.parse_args() cif_files = [os.path.join(args.cif_folder, f) for f in os.listdir( - args.cif_folder) if f.endswith('.cif')] + args.cif_folder) if f.endswith('.cif') and not f.startswith('.')] compile_data(cif_files, args.save_dir, args.probe_radius, args.number_of_steps, args.rad_file, args.verbose, - args.timeout, args.accuracy == 'high') + args.timeout, args.accuracy == 'high', + workers=args.workers, max_time=args.max_time, + memory_limit=args.memory_limit, shard=args.shard, + retry_failed=args.retry_failed, recycle=args.recycle, + merge_every=args.merge_every) + + +if __name__ == '__main__': + main() diff --git a/mofstructure/scripts/merge.py b/mofstructure/scripts/merge.py new file mode 100644 index 0000000..25284ed --- /dev/null +++ b/mofstructure/scripts/merge.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +''' +Command-line entry point `mofstructure_merge`. + +Joins the checkpoint files written by `mofstructure_database`, +`mofstructure_topology` and `mofstructure_porosity`, including the parts of a +run split across a cluster array with --shard, into the json and csv files +under /Structure_Data, and writes run_status.csv with one row per +structure and command. + +**usage:** + mofstructure_merge MOFstructureDB +''' +import argparse +import sys + +from mofstructure.batch_tasks import consolidate + + +def main(argv=None): + '''Rebuild the database files from the checkpoints in a save directory.''' + parser = argparse.ArgumentParser( + description="Join batch checkpoints, including the shards of an " + "array job, into the structure database files.") + parser.add_argument("save_dir", help="database directory to rebuild") + args = parser.parse_args(argv) + written = consolidate(args.save_dir) + if not written: + print(f"no checkpoints found under {args.save_dir}") + return 1 + for name, count in written.items(): + print(f"{name:<45} {count} records") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/mofstructure/scripts/topology.py b/mofstructure/scripts/topology.py index f7cf8a8..8c55a2e 100644 --- a/mofstructure/scripts/topology.py +++ b/mofstructure/scripts/topology.py @@ -71,6 +71,14 @@ exit status: 0 every structure was identified 1 at least one was not; the reason is in the status column + +long runs and clusters: + Each structure runs in a worker process with a hard limit (--max-time), so + a crash or a stall costs only that structure; it is reported as crashed or + timeout. Results are checkpointed under + /Structure_Data/_progress/ as they finish, and repeating the + command continues a killed run. Use -j for several workers and + --shard slurm/N on a cluster array, then mofstructure_merge . """ @@ -321,6 +329,9 @@ def main(argv: list[str] = None) -> int: action="store_true", help="print the closing summary only, without a line per structure", ) + from mofstructure.batch_tasks import add_batch_arguments + + add_batch_arguments(parser, max_time=1800) args = parser.parse_args(argv) logging.basicConfig( @@ -356,33 +367,78 @@ def main(argv: list[str] = None) -> int: print(f"{str(path):<28} missing") failures += 1 - total = len(targets) + # Every structure runs in a worker process with a hard time limit, so a + # crash or a stall in one structure cannot end the run, and results are + # checkpointed as they finish so a killed job resumes where it stopped. + import tempfile + from types import SimpleNamespace + + from mofstructure.batch import (parse_shard, run_batch, select_shard, + shard_suffix) + from mofstructure.batch_tasks import consolidate, done_names + + label = "all-methods" if args.all_methods else args.method + command = f"topology.{label}" + if args.no_save or args.json: + scratch = tempfile.mkdtemp(prefix="mofstructure_topology_") + structure_db = Path(scratch) / read_write.STRUCTURE_DATA + else: + scratch = None + structure_db = _database_path(args.save_dir).parent + shard = parse_shard(args.shard) + items = select_shard([(path.stem, str(path)) for path in targets], shard) + skip = done_names(structure_db, command, retry_failed=args.retry_failed) + total = len(items) counter = len(str(total)) * 2 + 2 if total > 1 else 0 - if targets and not args.quiet: + if skip and not args.quiet: + print(f"{len(skip & {n for n, _ in items})} structures already " + f"recorded by an earlier run are skipped") + if items and not args.quiet: print( f"{'':<{counter}}{'structure':<28} {'status':<22} " - f"{'kind':<8} {'method':<14} net" + f"{'kind':<8} {'method':<14} net", flush=True ) - for number, path in enumerate(targets, start=1): - # A long run is mostly waiting, so each finished line says how far - # along it is. The width is fixed for the run, so it stays a column. - stamp = f"{number:>{len(str(total))}}/{total} " if counter else "" + def unpack(batch_record): + '''The analyse record(s) a checkpoint line stands for.''' + name = batch_record["name"] + if batch_record["status"] != "ok": + failed = {"status": batch_record["status"], + "detail": batch_record.get("detail"), + "method": label, "components": []} + return [(name, failed)] + data = batch_record["data"] if args.all_methods: - results = analyse_methods(str(path), **options) - for method, record in results.items(): - records.append(record) - collected[f"{path.stem}:{method}"] = record - if not args.quiet: - print(stamp + _row(record, f"{path.name}:{method}")) - failures += record["status"] != "ok" - else: - record = analyse(str(path), method=args.method, **options) + return [(f"{name}:{method}", sub) for method, sub in data.items()] + return [(name, data)] + + def show(batch_record, done, _total): + nonlocal failures + stamp = f"{done:>{len(str(total))}}/{total} " if counter else "" + for key, record in unpack(batch_record): records.append(record) - collected[path.stem] = record - if not args.quiet: - print(stamp + _row(record, path.name)) + collected[key] = record failures += record["status"] != "ok" + if not args.quiet: + shown = f"{Path(batch_record['file']).name}" + ( + f":{key.split(':', 1)[1]}" if ":" in key else "") + print(stamp + _row(record, shown), flush=True) + + run_batch( + items, + "mofstructure.batch_tasks:topology_task", + structure_db / "_progress" / f"{command}{shard_suffix(shard)}.jsonl", + kwargs={"method": args.method, "all_methods": args.all_methods, + "timeout": options["timeout"], + "descriptors": options["descriptors"], + "symmetry": options["symmetry"]}, + workers=args.workers, + timeout=args.max_time or None, + memory_gb=args.memory_limit, + recycle=args.recycle, + skip=skip, + on_record=show, + ) if not args.no_save and records: if args.json: @@ -402,25 +458,26 @@ def main(argv: list[str] = None) -> int: }, destination, ) + elif shard: + destination = structure_db / "_progress" + table = None + print(f"\nshard written to {destination}; join all shards with " + f"mofstructure_merge {args.save_dir}") else: + consolidate(args.save_dir) destination = _database_path(args.save_dir) - read_write.append_json( - json.loads(json.dumps( - { - name: _for_disk(record) - for name, record in collected.items() - }, - default=str, - )), - str(destination), - ) # Built from the merged file rather than from this run, so the # table covers the database and not just the last command. table = _write_csv( read_write.load_data(str(destination)), destination ) print(f"\nwrote {len(records)} records to {destination}") - print(f"summary table {table}") + if table: + print(f"summary table {table}") + if scratch: + import shutil + + shutil.rmtree(scratch, ignore_errors=True) if records: tally = Counter(record["status"] for record in records) diff --git a/mofstructure/topology.py b/mofstructure/topology.py index 41fe85f..734eb3c 100644 --- a/mofstructure/topology.py +++ b/mofstructure/topology.py @@ -115,8 +115,14 @@ ) -class _Timeout(Exception): - '''Raised when identification exceeds the budget it was given.''' +class _Timeout(BaseException): + ''' + Raised when identification exceeds the budget it was given. + + Derived from BaseException so that the per-component error handling in + `graph_net.identify`, which catches Exception, does not record a timeout + as an ordinary failure and report it as "unidentified". + ''' def _alarm(signum, frame): # noqa: ARG001 diff --git a/pyproject.toml b/pyproject.toml index c1e2557..6d69c0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ mofstructure_curate = "mofstructure.scripts.curate_structures:main" mofstructure_generate_cgd="mofstructure.generate_cgd:main" mofstructure_topology = "mofstructure.scripts.topology:main" mofstructure_fingerprint = "mofstructure.scripts.fingerprint:main" +mofstructure_merge = "mofstructure.scripts.merge:main" cof_stacking = "mofstructure.cof_stacking:Main" [tool.poetry.group.dev.dependencies] diff --git a/tests/_batch_helpers.py b/tests/_batch_helpers.py new file mode 100644 index 0000000..16b27f5 --- /dev/null +++ b/tests/_batch_helpers.py @@ -0,0 +1,19 @@ +"""Tasks that misbehave on purpose, for tests/test_batch.py.""" +import ctypes +import os +import time + + +def behave(path, **kwargs): + mode = os.path.basename(path) + if mode.startswith("ok"): + return {"mode": mode, **kwargs} + if mode.startswith("raise"): + raise ValueError("deliberate failure") + if mode.startswith("abort"): + os.abort() + if mode.startswith("segfault"): + ctypes.string_at(0) + if mode.startswith("hang"): + time.sleep(600) + return None diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 0000000..53da732 --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,99 @@ +""" +The batch runner must finish a run whatever individual structures do. + +Each misbehaving task stands for a failure seen in long database runs: an +exception, an abort or a segmentation fault in compiled code, and a structure +that never finishes. None of them may stop the run, every structure must be +recorded with what happened to it, and a repeated run must continue rather +than start again. +""" +import json +import os + +import pytest + +from mofstructure import filetyper +from mofstructure.batch import (parse_shard, read_checkpoints, run_batch, + select_shard) + +TASK = "tests._batch_helpers:behave" + + +def _items(*modes): + return [(mode, f"/nowhere/{mode}") for mode in modes] + + +def test_failures_are_recorded_and_the_run_continues(tmp_path): + checkpoint = tmp_path / "run.jsonl" + items = _items("ok-1", "raise-1", "abort-1", "segfault-1", "hang-1", + "ok-2", "ok-3") + tally = run_batch(items, TASK, checkpoint, workers=2, timeout=5, + kwargs={"flag": 1}) + records = read_checkpoints([checkpoint]) + assert set(records) == {name for name, _ in items} + assert records["ok-1"]["status"] == "ok" + assert records["ok-1"]["data"] == {"mode": "ok-1", "flag": 1} + assert records["ok-3"]["status"] == "ok" + assert records["raise-1"]["status"] == "error" + assert "deliberate failure" in records["raise-1"]["detail"] + assert records["abort-1"]["status"] == "crashed" + assert "SIGABRT" in records["abort-1"]["detail"] + assert records["segfault-1"]["status"] == "crashed" + assert records["hang-1"]["status"] == "timeout" + assert sum(tally.values()) == len(items) + + +def test_a_repeated_run_continues_instead_of_starting_again(tmp_path): + checkpoint = tmp_path / "run.jsonl" + run_batch(_items("ok-1", "raise-1"), TASK, checkpoint, timeout=5) + done = set(read_checkpoints([checkpoint])) + tally = run_batch(_items("ok-1", "raise-1", "ok-2"), TASK, checkpoint, + timeout=5, skip=done) + assert tally == {"ok": 1} + assert len(checkpoint.read_text().splitlines()) == 3 + + +def test_a_line_cut_short_by_a_killed_job_is_ignored(tmp_path): + checkpoint = tmp_path / "run.jsonl" + run_batch(_items("ok-1"), TASK, checkpoint, timeout=5) + with open(checkpoint, "a", encoding="utf-8") as handle: + handle.write('{"name": "ok-2", "status": "o') + assert set(read_checkpoints([checkpoint])) == {"ok-1"} + + +def test_shards_cover_every_structure_once(): + items = _items(*(f"ok-{i}" for i in range(23))) + parts = [select_shard(items, (i, 4)) for i in range(4)] + names = [name for part in parts for name, _ in part] + assert sorted(names) == sorted(name for name, _ in items) + assert len(names) == len(set(names)) + + +def test_shard_index_can_come_from_the_scheduler(monkeypatch): + monkeypatch.setenv("SLURM_ARRAY_TASK_ID", "3") + assert parse_shard("slurm/8") == (3, 8) + with pytest.raises(ValueError): + parse_shard("8/8") + + +def test_an_interrupted_json_write_leaves_the_old_file(tmp_path): + target = tmp_path / "data.json" + filetyper.append_json({"a": 1}, str(target)) + + class Broken(json.JSONEncoder): + def default(self, o): + raise RuntimeError("killed mid write") + + with pytest.raises(RuntimeError): + filetyper.write_json_atomic({"a": 2, "b": object()}, str(target), + encoder=Broken) + assert json.loads(target.read_text()) == {"a": 1} + assert not [p for p in os.listdir(tmp_path) if p.startswith(".tmp_")] + + +def test_a_corrupt_database_file_is_reported_not_overwritten(tmp_path): + target = tmp_path / "data.json" + target.write_text('{"a": 1, "b":') + with pytest.raises(ValueError, match="not valid json"): + filetyper.append_json({"c": 3}, str(target)) + assert target.read_text() == '{"a": 1, "b":' diff --git a/tests/test_data/COF_586.cif b/tests/test_data/COF_586.cif new file mode 100644 index 0000000..4752b26 --- /dev/null +++ b/tests/test_data/COF_586.cif @@ -0,0 +1,92 @@ +data_586 +_audit_creation_date 2023-09-08 +_audit_creation_method 'Materials Studio' +_symmetry_space_group_name_H-M 'P1' +_symmetry_Int_Tables_number 1 +_symmetry_cell_setting triclinic +loop_ +_symmetry_equiv_pos_as_xyz + x,y,z +_cell_length_a 10.5000 +_cell_length_b 10.5000 +_cell_length_c 3.3900 +_cell_angle_alpha 90.0000 +_cell_angle_beta 90.0000 +_cell_angle_gamma 120.0000 +loop_ +_atom_site_label +_atom_site_type_symbol +_atom_site_fract_x +_atom_site_fract_y +_atom_site_fract_z +_atom_site_U_iso_or_equiv +_atom_site_adp_type +_atom_site_occupancy +O1 O 0.14092 0.35102 0.49079 0.00000 Uiso 1.00 +C2 C 0.24547 0.48389 0.49217 0.00000 Uiso 1.00 +C3 C 0.40084 0.52140 0.48745 0.00000 Uiso 1.00 +O4 O 0.65557 0.70406 0.48484 0.00000 Uiso 1.00 +C5 C 0.52270 0.67573 0.48903 0.00000 Uiso 1.00 +C6 C 0.48521 0.79359 0.49553 0.00000 Uiso 1.00 +C7 C 0.59493 0.94019 0.49729 0.00000 Uiso 1.00 +C8 C 0.33089 0.76111 0.50043 0.00000 Uiso 1.00 +O9 O 0.30256 0.86566 0.50599 0.00000 Uiso 1.00 +C10 C 0.21303 0.60576 0.49863 0.00000 Uiso 1.00 +C11 C 0.06642 0.56886 0.50303 0.00000 Uiso 1.00 +N12 N 0.55053 0.42683 0.47231 0.00000 Uiso 1.00 +N13 N 0.70279 0.19302 0.49132 0.00000 Uiso 1.00 +N14 N 0.80683 0.44663 0.50334 0.00000 Uiso 1.00 +C15 C 0.43031 0.40763 0.48308 0.00000 Uiso 1.00 +N16 N 0.57519 0.05784 0.49964 0.00000 Uiso 1.00 +N17 N 0.55725 0.29868 0.46341 0.00000 Uiso 1.00 +C18 C 0.68964 0.31141 0.47525 0.00000 Uiso 1.00 +N19 N 0.94585 0.46604 0.49385 0.00000 Uiso 1.00 +H20 H 0.03611 0.65889 0.52119 0.00000 Uiso 1.00 +H21 H 0.81360 0.20414 0.49769 0.00000 Uiso 1.00 +H22 H 0.45444 0.19053 0.44697 0.00000 Uiso 1.00 +H23 H 0.96150 0.36925 0.47274 0.00000 Uiso 1.00 +H24 H 0.64600 0.54044 0.47007 0.00000 Uiso 1.00 +H25 H 0.33608 0.28894 0.49001 0.00000 Uiso 1.00 +H26 H 0.71544 0.97051 0.49680 0.00000 Uiso 1.00 +H27 H 0.46661 1.05179 0.50756 0.00000 Uiso 1.00 +Br28 Br 0.88757 0.73556 1.28308 0.00000 Uiso 1.00 +H29 H 0.79146 0.54333 0.53441 0.00000 Uiso 1.00 +loop_ +_geom_bond_atom_site_label_1 +_geom_bond_atom_site_label_2 +_geom_bond_distance +_geom_bond_site_symmetry_2 +_ccdc_geom_bond_type +O1 C2 1.273 . D +C2 C3 1.475 . S +C2 C10 1.480 . S +C3 C15 1.376 . D +C3 C5 1.480 . S +O4 C5 1.273 . D +C5 C6 1.474 . S +C6 C8 1.480 . S +C6 C7 1.387 . D +C7 H26 1.140 . S +C7 N16 1.351 1_565 S +C8 C10 1.474 . S +C8 O9 1.273 . D +C10 C11 1.387 . D +C11 H20 1.140 . S +C11 N19 1.184 1_455 S +N12 N17 1.383 . S +N12 H24 1.110 . S +N12 C15 1.175 . S +N13 N16 1.382 . S +N13 H21 1.110 . S +N13 C18 1.319 . D +N14 C18 1.339 . S +N14 N19 1.370 . S +N14 H29 1.110 . S +C15 H25 1.140 . S +N16 C7 1.351 1_545 S +N16 H27 1.110 1_545 S +N17 C18 1.329 . S +N17 H22 1.110 . S +N19 H23 1.110 . S +N19 C11 1.184 1_655 S +H27 N16 1.110 1_565 S diff --git a/tests/test_data/EMIYUV.cif b/tests/test_data/EMIYUV.cif new file mode 100755 index 0000000..c606e6a --- /dev/null +++ b/tests/test_data/EMIYUV.cif @@ -0,0 +1,50 @@ +data_image0 +_cell_length_a 6.9989 +_cell_length_b 10.3114 +_cell_length_c 12.9438 +_cell_angle_alpha 90 +_cell_angle_beta 118.441 +_cell_angle_gamma 90 + +_symmetry_space_group_name_H-M "P 1" +_symmetry_int_tables_number 1 + +loop_ + _symmetry_equiv_pos_as_xyz + 'x, y, z' + +loop_ + _atom_site_label + _atom_site_occupancy + _atom_site_fract_x + _atom_site_fract_y + _atom_site_fract_z + _atom_site_thermal_displace_type + _atom_site_B_iso_or_equiv + _atom_site_type_symbol + Cd1 1.0000 0.26534 0.95065 0.62317 Biso 1.000 Cd + S1 1.0000 0.93830 0.83105 0.44826 Biso 1.000 S + C1 1.0000 0.77197 0.80024 0.50836 Biso 1.000 C + N1 1.0000 0.59834 0.86825 0.48659 Biso 1.000 N + Cd2 1.0000 0.73466 0.04935 0.37683 Biso 1.000 Cd + I1 1.0000 0.57798 0.16582 0.16871 Biso 1.000 I + S2 1.0000 0.06170 0.16895 0.55174 Biso 1.000 S + C2 1.0000 0.22803 0.19976 0.49164 Biso 1.000 C + N2 1.0000 0.40166 0.13175 0.51341 Biso 1.000 N + N3 1.0000 0.49508 0.18641 0.45245 Biso 1.000 N + C3 1.0000 0.37789 0.28627 0.39572 Biso 1.000 C + H1 1.0000 0.40905 0.35010 0.33956 Biso 1.000 H + N4 1.0000 0.20838 0.29985 0.41835 Biso 1.000 N + C4 1.0000 0.03375 0.39382 0.36955 Biso 1.000 C + H2 1.0000 0.08931 0.48106 0.34439 Biso 1.000 H + H3 1.0000 0.89379 0.35173 0.29264 Biso 1.000 H + H4 1.0000 0.98536 0.41689 0.43612 Biso 1.000 H + N5 1.0000 0.50492 0.81359 0.54755 Biso 1.000 N + C5 1.0000 0.62211 0.71373 0.60428 Biso 1.000 C + H5 1.0000 0.59095 0.64990 0.66044 Biso 1.000 H + N6 1.0000 0.79162 0.70015 0.58165 Biso 1.000 N + C6 1.0000 0.96625 0.60618 0.63045 Biso 1.000 C + H6 1.0000 0.01464 0.58311 0.56388 Biso 1.000 H + H7 1.0000 0.91069 0.51894 0.65561 Biso 1.000 H + H8 1.0000 0.10621 0.64827 0.70736 Biso 1.000 H + I2 1.0000 0.42202 0.83418 0.83129 Biso 1.000 I diff --git a/tests/test_data/EVAKAQ.cif b/tests/test_data/EVAKAQ.cif new file mode 100755 index 0000000..f767d89 --- /dev/null +++ b/tests/test_data/EVAKAQ.cif @@ -0,0 +1,113 @@ +data_image0 +_cell_length_a 8.5714 +_cell_length_b 10.486 +_cell_length_c 11.086 +_cell_angle_alpha 117.21 +_cell_angle_beta 93.14 +_cell_angle_gamma 102.22 + +_symmetry_space_group_name_H-M "P 1" +_symmetry_int_tables_number 1 + +loop_ + _symmetry_equiv_pos_as_xyz + 'x, y, z' + +loop_ + _atom_site_label + _atom_site_occupancy + _atom_site_fract_x + _atom_site_fract_y + _atom_site_fract_z + _atom_site_thermal_displace_type + _atom_site_B_iso_or_equiv + _atom_site_type_symbol + Cd1 1.0000 0.00000 0.50000 0.50000 Biso 1.000 Cd + Cd2 1.0000 0.07414 0.18977 0.16487 Biso 1.000 Cd + Cd3 1.0000 0.92586 0.81023 0.83513 Biso 1.000 Cd + O1 1.0000 0.11492 0.51423 0.69462 Biso 1.000 O + O2 1.0000 0.88508 0.48577 0.30538 Biso 1.000 O + O3 1.0000 0.05937 0.68830 0.88725 Biso 1.000 O + O4 1.0000 0.94063 0.31170 0.11275 Biso 1.000 O + O5 1.0000 0.34192 0.97317 0.83524 Biso 1.000 O + O6 1.0000 0.65808 0.02683 0.16476 Biso 1.000 O + O7 1.0000 0.15739 0.03087 0.97019 Biso 1.000 O + O8 1.0000 0.84261 0.96913 0.02981 Biso 1.000 O + O9 1.0000 0.40736 0.62605 0.46855 Biso 1.000 O + O10 1.0000 0.59264 0.37395 0.53145 Biso 1.000 O + O11 1.0000 0.20451 0.41904 0.36914 Biso 1.000 O + O12 1.0000 0.79549 0.58096 0.63086 Biso 1.000 O + O13 1.0000 0.84267 0.26842 0.42305 Biso 1.000 O + O14 1.0000 0.15733 0.73158 0.57695 Biso 1.000 O + H1 1.0000 0.74942 0.30507 0.45808 Biso 1.000 H + H2 1.0000 0.25058 0.69493 0.54192 Biso 1.000 H + H3 1.0000 0.80440 0.20813 0.32437 Biso 1.000 H + H4 1.0000 0.19560 0.79187 0.67563 Biso 1.000 H + N1 1.0000 0.35153 0.31998 0.12912 Biso 1.000 N + N2 1.0000 0.64847 0.68002 0.87088 Biso 1.000 N + N3 1.0000 0.13437 0.08290 0.28130 Biso 1.000 N + N4 1.0000 0.86563 0.91710 0.71870 Biso 1.000 N + N5 1.0000 0.21755 0.03640 0.44262 Biso 1.000 N + N6 1.0000 0.78245 0.96360 0.55738 Biso 1.000 N + C1 1.0000 0.11537 0.57954 0.82110 Biso 1.000 C + C2 1.0000 0.88463 0.42046 0.17890 Biso 1.000 C + C3 1.0000 0.19024 0.51968 0.90514 Biso 1.000 C + C4 1.0000 0.80976 0.48032 0.09486 Biso 1.000 C + C5 1.0000 0.21640 0.59235 0.04892 Biso 1.000 C + C6 1.0000 0.78360 0.40765 0.95108 Biso 1.000 C + H5 1.0000 0.18922 0.69885 0.10379 Biso 1.000 H + H6 1.0000 0.81078 0.30115 0.89621 Biso 1.000 H + C7 1.0000 0.27447 0.53029 0.12272 Biso 1.000 C + C8 1.0000 0.72553 0.46971 0.87728 Biso 1.000 C + H7 1.0000 0.28974 0.59010 0.23390 Biso 1.000 H + H8 1.0000 0.71026 0.40990 0.76610 Biso 1.000 H + C9 1.0000 0.30803 0.39126 0.05542 Biso 1.000 C + C10 1.0000 0.69197 0.60874 0.94458 Biso 1.000 C + C11 1.0000 0.28830 0.32189 0.91136 Biso 1.000 C + C12 1.0000 0.71170 0.67811 0.08864 Biso 1.000 C + H9 1.0000 0.31554 0.21630 0.85414 Biso 1.000 H + H10 1.0000 0.68446 0.78370 0.14586 Biso 1.000 H + C13 1.0000 0.23006 0.38474 0.83848 Biso 1.000 C + C14 1.0000 0.76994 0.61526 0.16152 Biso 1.000 C + H11 1.0000 0.21173 0.32678 0.72686 Biso 1.000 H + H12 1.0000 0.78827 0.67322 0.27314 Biso 1.000 H + C15 1.0000 0.44589 0.42254 0.26894 Biso 1.000 C + C16 1.0000 0.55411 0.57746 0.73106 Biso 1.000 C + H13 1.0000 0.54176 0.50956 0.26810 Biso 1.000 H + H14 1.0000 0.45824 0.49044 0.73190 Biso 1.000 H + H15 1.0000 0.50236 0.35906 0.30504 Biso 1.000 H + H16 1.0000 0.49764 0.64094 0.69496 Biso 1.000 H + C17 1.0000 0.34243 0.49617 0.37751 Biso 1.000 C + C18 1.0000 0.65757 0.50383 0.62249 Biso 1.000 C + C19 1.0000 0.29680 0.05411 0.94133 Biso 1.000 C + C20 1.0000 0.70320 0.94589 0.05867 Biso 1.000 C + C21 1.0000 0.42423 0.19626 0.05158 Biso 1.000 C + C22 1.0000 0.57577 0.80374 0.94842 Biso 1.000 C + H17 1.0000 0.47497 0.16500 0.12368 Biso 1.000 H + H18 1.0000 0.52503 0.83500 0.87632 Biso 1.000 H + H19 1.0000 0.52212 0.22952 0.00379 Biso 1.000 H + H20 1.0000 0.47788 0.77048 0.99621 Biso 1.000 H + C23 1.0000 0.22563 0.14155 0.40362 Biso 1.000 C + C24 1.0000 0.77437 0.85845 0.59638 Biso 1.000 C + H21 1.0000 0.29779 0.25504 0.46407 Biso 1.000 H + H22 1.0000 0.70221 0.74496 0.53593 Biso 1.000 H + C25 1.0000 0.06719 0.93582 0.24023 Biso 1.000 C + C26 1.0000 0.93281 0.06418 0.75977 Biso 1.000 C + H23 1.0000 0.98906 0.86270 0.14229 Biso 1.000 H + H24 1.0000 0.01094 0.13730 0.85771 Biso 1.000 H + C27 1.0000 0.11790 0.90510 0.34018 Biso 1.000 C + C28 1.0000 0.88210 0.09490 0.65982 Biso 1.000 C + H25 1.0000 0.09212 0.80505 0.34918 Biso 1.000 H + H26 1.0000 0.90788 0.19495 0.65082 Biso 1.000 H + C29 1.0000 0.30596 0.05234 0.56664 Biso 1.000 C + C30 1.0000 0.69404 0.94766 0.43336 Biso 1.000 C + H27 1.0000 0.22018 0.00797 0.61656 Biso 1.000 H + H28 1.0000 0.77982 0.99203 0.38344 Biso 1.000 H + H29 1.0000 0.36139 0.17086 0.63482 Biso 1.000 H + H30 1.0000 0.63861 0.82914 0.36518 Biso 1.000 H + C31 1.0000 0.43666 0.96845 0.53213 Biso 1.000 C + C32 1.0000 0.56334 0.03155 0.46787 Biso 1.000 C + H31 1.0000 0.49338 0.97821 0.62791 Biso 1.000 H + H32 1.0000 0.50662 0.02179 0.37209 Biso 1.000 H + H33 1.0000 0.38061 0.85044 0.46173 Biso 1.000 H + H34 1.0000 0.61939 0.14956 0.53827 Biso 1.000 H diff --git a/tests/test_data/LORLEJ.cif b/tests/test_data/LORLEJ.cif new file mode 100755 index 0000000..f58cc0b --- /dev/null +++ b/tests/test_data/LORLEJ.cif @@ -0,0 +1,65 @@ +data_image0 +_cell_length_a 7.95 +_cell_length_b 7.95 +_cell_length_c 27.946 +_cell_angle_alpha 90 +_cell_angle_beta 90 +_cell_angle_gamma 120 + +_symmetry_space_group_name_H-M "P 1" +_symmetry_int_tables_number 1 + +loop_ + _symmetry_equiv_pos_as_xyz + 'x, y, z' + +loop_ + _atom_site_label + _atom_site_occupancy + _atom_site_fract_x + _atom_site_fract_y + _atom_site_fract_z + _atom_site_thermal_displace_type + _atom_site_B_iso_or_equiv + _atom_site_type_symbol + Pb1 1.0000 0.96249 0.96410 0.15383 Biso 1.000 Pb + O1 1.0000 0.75471 0.69301 0.20936 Biso 1.000 O + Cs1 1.0000 0.33307 0.66713 0.16665 Biso 1.000 Cs + O2 1.0000 0.91193 0.64093 0.12372 Biso 1.000 O + C1 1.0000 0.98186 0.58949 0.08986 Biso 1.000 C + C2 1.0000 0.84458 0.38126 0.06907 Biso 1.000 C + H1 1.0000 0.90977 0.29183 0.07748 Biso 1.000 H + H2 1.0000 0.84209 0.39659 0.03017 Biso 1.000 H + N1 1.0000 0.64744 0.28811 0.08882 Biso 1.000 N + C3 1.0000 0.53372 0.37810 0.07387 Biso 1.000 C + H3 1.0000 0.62421 0.53581 0.07643 Biso 1.000 H + H4 1.0000 0.47932 0.33422 0.03739 Biso 1.000 H + C4 1.0000 0.35958 0.30767 0.10925 Biso 1.000 C + O3 1.0000 0.40231 0.35763 0.15174 Biso 1.000 O + Pb2 1.0000 0.70408 0.36972 0.17930 Biso 1.000 Pb + O4 1.0000 0.66324 0.02685 0.15586 Biso 1.000 O + C5 1.0000 0.64031 0.98736 0.11231 Biso 1.000 C + C6 1.0000 0.54009 0.07713 0.08261 Biso 1.000 C + H5 1.0000 0.39208 0.01774 0.09632 Biso 1.000 H + H6 1.0000 0.53706 0.03738 0.04501 Biso 1.000 H + O5 1.0000 0.69330 0.88549 0.09082 Biso 1.000 O + O6 1.0000 0.19540 0.20430 0.09201 Biso 1.000 O + O7 1.0000 0.14203 0.68121 0.07156 Biso 1.000 O + O8 1.0000 0.00398 0.30701 0.17762 Biso 1.000 O + C7 1.0000 0.02603 0.34585 0.22120 Biso 1.000 C + C8 1.0000 0.12629 0.25602 0.25089 Biso 1.000 C + H7 1.0000 0.27438 0.31572 0.23722 Biso 1.000 H + H8 1.0000 0.12900 0.29547 0.28851 Biso 1.000 H + N2 1.0000 0.01919 0.04514 0.24457 Biso 1.000 N + C9 1.0000 0.82214 0.95164 0.26434 Biso 1.000 C + H9 1.0000 0.82470 0.93596 0.30323 Biso 1.000 H + H10 1.0000 0.75680 0.04104 0.25604 Biso 1.000 H + C10 1.0000 0.68505 0.74367 0.24345 Biso 1.000 C + O9 1.0000 0.52508 0.65139 0.26182 Biso 1.000 O + C11 1.0000 0.13312 0.95514 0.25930 Biso 1.000 C + H11 1.0000 0.04285 0.79745 0.25656 Biso 1.000 H + H12 1.0000 0.18746 0.99836 0.29580 Biso 1.000 H + C12 1.0000 0.30737 0.02653 0.22392 Biso 1.000 C + O10 1.0000 0.26488 0.97659 0.18143 Biso 1.000 O + O11 1.0000 0.47140 0.13019 0.24121 Biso 1.000 O + O12 1.0000 0.97235 0.44708 0.24272 Biso 1.000 O diff --git a/tests/test_data/MOZ.cif b/tests/test_data/MOZ.cif new file mode 100644 index 0000000..c92e56d --- /dev/null +++ b/tests/test_data/MOZ.cif @@ -0,0 +1,79 @@ +data_MOZ + +#************************************************************************** +# +# CIF taken from the IZA-SC Database of Zeolite Structures +# Ch. Baerlocher and L.B. McCusker +# Database of Zeolite Structures: http://www.iza-structure.org/databases/ +# +# The atom coordinates and the cell parameters were optimized with DLS76 +# assuming a pure SiO2 composition. +# +#************************************************************************** + +_cell_length_a 31.2020(0) +_cell_length_b 31.2020(0) +_cell_length_c 7.5510(0) +_cell_angle_alpha 90.0000(0) +_cell_angle_beta 90.0000(0) +_cell_angle_gamma 120.0000(0) + +_symmetry_space_group_name_H-M 'P 6/m m m' +_symmetry_Int_Tables_number 191 +_symmetry_cell_setting hexagonal + +loop_ +_symmetry_equiv_pos_as_xyz +'+x,+y,+z' +'-y,+x-y,+z' +'-x+y,-x,+z' +'-x,-y,+z' +'+y,-x+y,+z' +'+x-y,+x,+z' +'-y,-x,+z' +'-x+y,+y,+z' +'+x,+x-y,+z' +'+y,+x,+z' +'+x-y,-y,+z' +'-x,-x+y,+z' +'-x,-y,-z' +'+y,-x+y,-z' +'+x-y,+x,-z' +'+x,+y,-z' +'-y,+x-y,-z' +'-x+y,-x,-z' +'+y,+x,-z' +'+x-y,-y,-z' +'-x,-x+y,-z' +'-y,-x,-z' +'-x+y,+y,-z' +'+x,+x-y,-z' + +loop_ +_atom_site_label +_atom_site_type_symbol +_atom_site_fract_x +_atom_site_fract_y +_atom_site_fract_z + O7 O 0.1631 0.1631 0.5000 + O8 O 0.1896 0.0948 0.5000 + O9 O 0.2421 0.1783 0.3259 + O10 O 0.3075 0.1538 0.2241 + O11 O 0.2761 0.1965 0.0000 + O12 O 0.3341 0.2476 0.2639 + O13 O 0.3854 0.3055 0.0000 + O14 O 0.4037 0.3395 0.3259 + O15 O 0.4278 0.2737 0.2244 + O16 O 0.4183 0.4183 0.5000 + O17 O 0.4871 0.3926 0.5000 + O18 O 0.5816 0.4184 0.5000 + O19 O 0.5176 0.3396 0.3260 + O20 O 0.4954 0.2477 0.2627 + O21 O 0.5019 0.3059 0.0000 + T1 Si 0.2900 0.1940 0.2037 + T2 Si 0.3877 0.2916 0.2038 + T3 Si 0.4857 0.2917 0.2037 + T4 Si 0.2092 0.1533 0.5000 + T5 Si 0.4285 0.3725 0.5000 + T6 Si 0.5258 0.3727 0.5000 + diff --git a/tests/test_fingerprint_cli.py b/tests/test_fingerprint_cli.py index 8c333a8..15e47ca 100644 --- a/tests/test_fingerprint_cli.py +++ b/tests/test_fingerprint_cli.py @@ -55,7 +55,10 @@ def get_ligand_cluster_fingerprint(self): return SAMPLE_FINGERPRINT monkeypatch.setattr(collect_data.structure, "MOFstructure", FakeMOF) - collect_data.compile_data([str(cif)], str(tmp_path / "database")) + # workers=0 runs in this process, where the substitute class is visible; + # a worker process would import the real one. + collect_data.compile_data([str(cif)], str(tmp_path / "database"), + workers=0) records = json.loads( (structure_data / "fingerprint_data.json").read_text(encoding="utf-8") diff --git a/tests/test_topology_invariance.py b/tests/test_topology_invariance.py new file mode 100644 index 0000000..68209f4 --- /dev/null +++ b/tests/test_topology_invariance.py @@ -0,0 +1,137 @@ +""" +The canonical key must not depend on how a crystal is written down. + +Each test changes only the representation of one structure: the origin of the +cell, the cell itself or the order of the atoms. A key that moves under any of +these is not an identifier, and two files of one material would be counted as +two materials. + +LORLEJ and EVAKAQ are a layered and a rod MOF whose all-node nets contain +edges built inside the metal SBU. An origin shift moves some of those bonds +across the cell boundary, which exposed a sign error in the translation given +to those edges. EMIYUV deconstructs to a one-periodic net whose barycentric +placement has collisions, for which neither the primitive cell nor the key is +well defined. COF_586 (CoRE COF 586) is a single hydrazone layer in a cell +small enough that one linkage joins a building unit to its own neighbouring +image, and its two-connected units are spliced into edges; it exposed a cut +that was discarded because both ends lay in one component and a sign error +in the splice. + +The sbus and ligand_cluster reductions are not tested here. They collapse a +rod SBU to a single vertex, which is not a well-defined periodic graph, so +for rod MOFs their nets depend on the cell by construction. +""" + +from __future__ import annotations + +import warnings +from pathlib import Path + +import numpy as np +import pytest +from ase.build import make_supercell +from ase.io import read + +from mofstructure.graph_net.canonical import UnstableNetError, canonical_key +from mofstructure.graph_net.periodic_graph import PeriodicGraph +from mofstructure.structure import MOFstructure +from mofstructure.topology import _identify_into, analyse + +DATA = Path(__file__).resolve().parent / "test_data" + + +def _read(name): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return read(DATA / name) + + +def _key(atoms, method="all_node"): + record = analyse(MOFstructure(ase_atoms=atoms).remove_guest(), + method=method, timeout=300) + return record["status"], record.get("key") + + +def _variants(atoms, seed=0): + rng = np.random.default_rng(seed) + for k in range(3): + moved = atoms.copy() + moved.translate(rng.random(3) @ atoms.cell) + moved.wrap() + yield f"origin shift {k}", moved + yield "permutation", atoms[rng.permutation(len(atoms))] + yield "2x1x1 supercell", atoms.repeat((2, 1, 1)) + yield "cell setting", make_supercell(atoms, [[1, 1, 0], [0, 1, 0], [0, 0, 1]], + wrap=True) + + +@pytest.mark.parametrize("filename", ["LORLEJ.cif", "EVAKAQ.cif", "Cr.cif"]) +def test_all_node_key_is_independent_of_the_representation(filename): + atoms = _read(filename) + status, reference = _key(atoms) + assert status == "ok" and reference + for label, variant in _variants(atoms): + assert _key(variant) == (status, reference), f"key moved under {label}" + + +def test_cof_key_is_independent_of_the_representation(): + atoms = _read("COF_586.cif") + status, reference = _key(atoms, method="cof") + assert status == "ok" and reference + for label, variant in _variants(atoms, seed=3): + assert _key(variant, method="cof") == (status, reference), \ + f"key moved under {label}" + + +def test_an_unstable_net_is_refused_in_every_cell(): + atoms = _read("EMIYUV.cif") + statuses = {_key(variant)[0] for _, variant in _variants(atoms)} + statuses.add(_key(atoms)[0]) + assert statuses == {"unidentified"} + + +def _graph_supercell(graph, axis, n=2): + edges = [] + for copy in range(n): + for (u, v, shift) in graph.edges: + shift = list(shift) + total = copy + shift[axis] + shift[axis] = total // n + edges.append((u + graph.n_vertices * copy, + v + graph.n_vertices * (total % n), tuple(shift))) + return PeriodicGraph.build(dim=graph.dim, n_vertices=graph.n_vertices * n, + edges=edges) + + +def test_a_net_with_collisions_has_no_key(): + # Quotient graph of the EMIYUV chain. Vertices 0 and 3, and 1 and 2, share + # a position in the barycentric placement although no vertex has two + # neighbours at the same point, so the net is locally stable but not + # stable. Before the global test it received a key that changed when the + # same chain was written in a doubled cell. + chain = PeriodicGraph.build(dim=3, n_vertices=4, edges=[ + (0, 2, (-1, 0, 0)), (0, 2, (0, 0, 0)), (0, 3, (0, 1, 0)), + (1, 2, (0, -1, 0)), (1, 3, (0, 0, 0)), (1, 3, (1, 0, 0)), + ]) + for graph in (chain, _graph_supercell(chain, 0), _graph_supercell(chain, 0, 3)): + with pytest.raises(UnstableNetError): + canonical_key(graph) + + +def test_a_stable_net_has_the_same_key_in_a_supercell(): + # pcu written in its primitive cell and as 2x1x1, 1x2x1 and 1x1x3 cells. + pcu = PeriodicGraph.build(dim=3, n_vertices=1, edges=[ + (0, 0, (1, 0, 0)), (0, 0, (0, 1, 0)), (0, 0, (0, 0, 1)), + ]) + reference = canonical_key(pcu) + for axis, n in ((0, 2), (1, 2), (2, 3)): + assert canonical_key(_graph_supercell(pcu, axis, n)) == reference + + +def test_a_timeout_is_reported_as_a_timeout(): + # MOZ takes several seconds to key, so a one-second budget expires inside + # the canonical form. The timeout used to be caught by the per-component + # error handling and reported as "unidentified", which reads as a property + # of the net rather than of the budget. + record = analyse(str(DATA / "MOZ.cif"), method="zeol", timeout=1) + assert record["status"] == "timeout"