diff --git a/CHANGELOG.md b/CHANGELOG.md index 39e9bab..6808064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.7.0 + +SerialEM `.mdoc` decoding. + +- New decoder for `.mdoc`, the acquisition record SerialEM and Thermo Fisher + Tomography write beside a tilt series, montage or saved movie, dispatched by + extension with a 1 MiB leading read. Also importable as `decode_mdoc`. +- Returns `voltageKv`, `pixelSizeA`, `doseEA2`, `fractions`, `magnification` and + `exposureS` under the names the EPU decoder uses, so an mdoc goes straight into + `compare_acquisition`; plus the software and version, tilt axis, tilt range, + total dose, acquisition times, the first image's stage position, defocus and + frame path, and montage settings. +- All four section kinds: `[ZValue]` (stack images), `[FrameSet]` (the file + beside a single saved movie, including its unbracketed title line), + `[MontSection]` (montage settings, kept out of the per-image values) and IMOD's + `[Image = name]` series. NaN and unparseable values decode as absent; CRLF, a + UTF-8 byte order mark and Latin-1 text are accepted. +- Checked on 116 public mdoc files from GitHub: every value compared agrees with + mdocfile on the 111 it can read, and the other five (deliberately malformed test + files) decode with the bad values left out. 46,400 mutations of those files + raised nothing. + ## 0.6.0 Light microscopy and light-sheet formats, and the reader model they needed. diff --git a/README.md b/README.md index 4ba928a..2f20401 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,13 @@ Decoders today: nibabel. Big- and little-endian. - **CryoSPARC `.cs`**: a structured-array dataset. Reports record count and the field schema. +- **SerialEM `.mdoc`**: the acquisition record SerialEM (and Thermo Fisher + Tomography) writes beside a tilt series, montage or saved movie. Voltage, pixel + size, dose, frame count, exposure, magnification and tilt axis, the tilt range + and total dose over every image, the first image's stage position and defocus, + and the montage settings. Handles tilt series (`[ZValue]`), single-movie files + (`[FrameSet]`), montages (`[MontSection]`) and IMOD image series (`[Image]`). + Cross-checked against mdocfile on 111 public mdoc files. - **Parquet**: reads the footer, not a leading header, and returns the column names, physical types, and row count. Cross-checked against pyarrow. - **FCS**: flow cytometry (Flow Cytometry Standard). Returns parameter and event @@ -217,12 +224,14 @@ or exported by different people. A wrong one does not fail; it quietly caps the resolution. `compare_acquisition` lines the sources up and reports disagreements: ```python -from scigantic_headers import compare_acquisition, decode_epu_xml, read_star_optics +from scigantic_headers import compare_acquisition, decode_epu_xml, decode_file, read_star_optics epu = decode_epu_xml(open("FoilHole_..._Data_....xml", "rb").read()).fields +mdoc = decode_file("TS_01.mrc.mdoc").fields # SerialEM, if the session used it for f in compare_acquisition({ "sheet": {"voltageKv": 300, "pixelSizeA": 0.84, "doseEA2": 42.4, "fractions": 40}, "epu": epu, + "serialem": mdoc, "relion": read_star_optics("Import/movies.star"), }, target_resolution_a=3.3): print(f.severity, f.field, f.message) @@ -232,6 +241,8 @@ It does not decide which source is right. EPU records the pixel size of the binn readout grid, so `epu_pixel_size_candidates(epu)` lists the likely values for the saved movie; compare with the movie's own dimensions. `decode_epu_xml` is not auto-dispatched (`.xml` is too generic), so call it on a file you know is EPU's. +An mdoc's `doseEA2` is the dose of one image (one tilt, or one movie), the value to +compare with a per-movie dose; `totalDoseEA2` is the sum over the series. A raw MRC movie header has no pixel size: CELLA is 0, so `decode_*` returns `pixelSizeA` as `None`. The value is in the RELION STAR or CryoSPARC `.cs` file @@ -279,13 +290,14 @@ detector-pixel-size / magnification pair are all handled. ics.py Image Cytometry Standard header zarr.py OME-Zarr / Zarr v2+v3 / N5 sidecars (by file name) xml_sidecar.py .xml dispatcher: BDV SpimData, OME-XML, EPU + mdoc.py SerialEM / Tomography .mdoc acquisition record sources.py read bytes from file or URL (leading, footer, or seeking); bounded parallel batch star.py RELION STAR optics reader cryosparc.py CryoSPARC .cs optics reader optics.py read_session_optics: find a data file's optics file cli.py scigantic-headers benchmark.py scigantic-headers-bench, measures the speed levers - tests/ pytest (440 tests, incl. fuzz, golden and sparse real-file fixtures) + tests/ pytest (454 tests, incl. fuzz, golden and sparse real-file fixtures) ## Robustness diff --git a/pyproject.toml b/pyproject.toml index 7eb1ca3..2cb4f87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "scigantic-headers" -version = "0.6.0" +version = "0.7.0" description = "Decode scientific file headers (cryo-EM, light microscopy and light-sheet imaging incl. TIFF/OME-TIFF, CZI, HDF5/Imaris/BigDataViewer, LIF, ND2, Zarr/N5; tabular, flow cytometry, mass spec, sequencing, structural biology) into typed fields. Pure, zero-dependency, with bounded readers that follow internal pointers without downloading the file." readme = "README.md" requires-python = ">=3.9" diff --git a/src/scigantic_headers/__init__.py b/src/scigantic_headers/__init__.py index d32cc96..4f1417a 100644 --- a/src/scigantic_headers/__init__.py +++ b/src/scigantic_headers/__init__.py @@ -18,6 +18,7 @@ from .ics import decode_ics from .klb import decode_klb from .lif import decode_lif +from .mdoc import decode_mdoc from .nd2 import decode_nd2 from .obf import decode_obf from .ome_xml import parse_ome_xml @@ -78,7 +79,7 @@ except PackageNotFoundError: # a source checkout with no install metadata __version__ = "0.0.0+unknown" __all__ = [ - "Finding", "compare_acquisition", "decode_epu_xml", "epu_pixel_size_candidates", + "Finding", "compare_acquisition", "decode_epu_xml", "epu_pixel_size_candidates", "decode_mdoc", "HEADER_BYTES", "SEEK_MAX_READS", "DecodedHeader", "Read", "Source", "BytesSource", "FileSource", "UrlSource", "decode_bytes", "decode_source", "decode_mrc_header", "decode_nifti_header", "decode_npy_header", "decode_cryosparc_header", diff --git a/src/scigantic_headers/mdoc.py b/src/scigantic_headers/mdoc.py new file mode 100644 index 0000000..d4f810f --- /dev/null +++ b/src/scigantic_headers/mdoc.py @@ -0,0 +1,332 @@ +"""Decode a SerialEM .mdoc, the acquisition record written next to an image stack or movie. + +SerialEM (and Thermo Fisher Tomography, which writes the same format) saves a +plain-text `.mdoc` beside each stack, tilt series, montage or saved movie. It holds +the settings the data was recorded with, image by image: tilt angle, dose, exposure +time, pixel spacing, defocus, stage position, magnification, number of frames and +the time of each exposure. The MRC or TIFF it describes usually carries none of +this, so the mdoc is the only place to read it from. + +Layout: + + PixelSpacing = 1.35 global lines describe the whole stack + Voltage = 300 + ImageFile = TS_01.mrc + [T = SerialEM: Titan Krios ... 05-Jun-22 16:13:00] free-text titles + [T = Tilt axis angle = 85.3, binning = 1 spot = 9 camera = 0] + [ZValue = 0] one section per image in the stack + TiltAngle = -0.01 + ExposureDose = 3.1 + ... + +Three section kinds exist. `[ZValue = n]` is one image of a stack (one tilt, or one +piece of a montage). `[FrameSet = n]` is the whole of a single saved movie; the +small mdoc SerialEM writes beside each movie has one and no ZValue at all. +`[MontSection = n]` holds the settings of a montage as a whole; it is not an image, +so it is reported separately and left out of the per-image values. IMOD also reads +an image-series form (`ImageSeries = 1`, then one `[Image = name.tif]` section per +separate file); those sections count as images, and `imageFiles` lists the names. + +Units are SerialEM's own and are kept: pixel spacing in angstrom, voltage in kV, +dose in electrons per square angstrom, defocus in micrometers (negative is +underfocus), exposure in seconds, stage position in micrometers. The fields that +`compare_acquisition` reads (`voltageKv`, `pixelSizeA`, `doseEA2`, `fractions`, +`magnification`) use the same names as the EPU decoder, so an mdoc can be checked +against a collection sheet, a RELION optics table or an EPU file directly. + +`pixelSizeA` is the pixel spacing SerialEM recorded for the saved image. Movies saved +in super-resolution or with different binning from the image can sit on another +grid; check the movie's own dimensions before relying on it for the frames. +""" + +from __future__ import annotations + +import math +import re +from datetime import datetime +from typing import Dict, List, Optional, Tuple + +from .decoders import DecodedHeader, Read, register_decoder + +# A 60-tilt series is about 60 KiB; a large montage runs to a few hundred. Past this +# the per-image values are reported for the sections read, and `truncated` says so. +MDOC_READ_BYTES = 1024 * 1024 + +_SECTION_RE = re.compile(r"^\[\s*(ZValue|FrameSet|MontSection)\s*=\s*(-?\d+)\s*\]$") +# IMOD's image-series form: one section per separate image file, named in the header. +_IMAGE_SECTION_RE = re.compile(r"^\[\s*Image\s*=\s*(.+?)\s*\]$") +_TITLE_RE = re.compile(r"^\[\s*T\s*=(.*)\]$") +# SerialEM's keys are identifiers; a few camera-specific ones add '-', '(' and ')'. +_KV_RE = re.compile(r"^([A-Za-z][A-Za-z0-9_\-()]*)\s*=\s?(.*)$") +# Two title forms carry the tilt axis: "Tilt axis angle = 85.3, binning = 1 ..." +# (SerialEM 3 and later) and "TiltAxisAngle = 85.3 Binning = 1 ..." (older builds). +_TILT_AXIS_RE = re.compile(r"(?:Tilt axis angle|TiltAxisAngle)\s*=\s*(-?\d+(?:\.\d+)?)", re.I) +_TITLE_DATE_RE = re.compile(r"(\d{1,2}-[A-Za-z]{3}-\d{2,4}\s+\d{1,2}:\d{2}:\d{2})") +_VERSION_RE = re.compile(r"SerialEM Version\s+([0-9][0-9A-Za-z.\-]*)") + +_MAX_TITLES = 10 + + +def _num(v: Optional[str]) -> Optional[float]: + """First number of a value, or None. SerialEM writes NaN for a reading it does + not have (CameraLength in TEM mode); that is absent, not a number.""" + if v is None: + return None + parts = v.split() + if not parts: + return None + try: + x = float(parts[0]) + except ValueError: + return None + return x if math.isfinite(x) else None + + +def _nums(v: Optional[str]) -> Optional[List[float]]: + """Every number of a value (StagePosition is two, MinMaxMean three). Any token + that is not a finite number makes the whole value absent, not a partial list.""" + if v is None: + return None + parts = v.split() + if not parts: + return None + out = [] + for p in parts: + try: + x = float(p) + except ValueError: + return None + if not math.isfinite(x): + return None + out.append(x) + return out + + +def _int(v: Optional[str]) -> Optional[int]: + x = _num(v) + return int(x) if x is not None and x == int(x) and abs(x) < 2**53 else None + + +def _ints(v: Optional[str]) -> Optional[List[int]]: + xs = _nums(v) + if xs is None or any(x != int(x) or abs(x) >= 2**53 for x in xs): + return None + return [int(x) for x in xs] + + +def _text(v: Optional[str]) -> Optional[str]: + v = (v or "").strip() + return v or None + + +def _positive(x: Optional[float]) -> Optional[float]: + return x if x is not None and x > 0 else None + + +def _iso(stamp: Optional[str]) -> Optional[str]: + """'05-Jun-22 16:13:00' -> '2022-06-05T16:13:00'. SerialEM writes local time + with no zone, so none is added. None if the stamp does not parse.""" + if not stamp: + return None + s = " ".join(stamp.split()) + for fmt in ("%d-%b-%y %H:%M:%S", "%d-%b-%Y %H:%M:%S"): + try: + return datetime.strptime(s, fmt).isoformat() + except ValueError: + continue + return None + + +def _parse(text: str) -> Tuple[Dict[str, str], List[str], List[Tuple[str, int, Dict[str, str]]]]: + glob: Dict[str, str] = {} + titles: List[str] = [] + sections: List[Tuple[str, int, Dict[str, str]]] = [] + cur: Optional[Dict[str, str]] = None + for raw in text.splitlines(): + line = raw.strip() + if not line: + continue + m = _SECTION_RE.match(line) + if m: + cur = {} + sections.append((m.group(1), int(m.group(2)), cur)) + continue + m = _IMAGE_SECTION_RE.match(line) + if m and not line.startswith("[T"): + cur = {"__file": m.group(1)} + sections.append(("Image", len(sections), cur)) + continue + m = _TITLE_RE.match(line) + if m: + titles.append(m.group(1).strip()) + continue + m = _KV_RE.match(line) + if not m: + continue # a line cut off by the read, or one SerialEM's own readers would skip + key, value = m.group(1), m.group(2).strip() + # The mdoc SerialEM writes beside a saved movie has its title without + # brackets ("T = SerialEM: ..."); it is still a title, not a setting. + if key == "T" and cur is None: + titles.append(value) + continue + (cur if cur is not None else glob)[key] = value + return glob, titles, sections + + +def decode_mdoc(data: bytes) -> Optional[DecodedHeader]: + if not data: + return None + if data.startswith(b"\xef\xbb\xbf"): + data = data[3:] + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + # Windows builds can write a title or note in the system code page. + text = data.decode("latin-1") + if "\x00" in text: + return None + + glob, titles, sections = _parse(text) + + images = [s for s in sections if s[0] in ("ZValue", "FrameSet", "Image")] + montage_sections = [s for s in sections if s[0] == "MontSection"] + # A real file has a section, or (written before the first image lands) a global + # PixelSpacing or ImageFile line. Anything else is not an mdoc. + if not sections and "PixelSpacing" not in glob and "ImageFile" not in glob: + return None + + first = images[0][2] if images else {} + + def pick(key: str) -> Optional[str]: + """Global value if present, else the first image's.""" + return glob[key] if key in glob else first.get(key) + + tilt_axis = None + for t in titles: + m = _TILT_AXIS_RE.search(t) + if m: + tilt_axis = float(m.group(1)) + break + + software = None + m = _VERSION_RE.search(glob.get("Version", "")) + if m: + software = "SerialEM " + m.group(1) + else: + for t in titles: + head = t.split(":", 1)[0].strip() + if ":" in t and head in ("SerialEM", "Tomography"): + software = head + break + + title_date = None + for t in titles: + m = _TITLE_DATE_RE.search(t) + if m: + title_date = m.group(1) + break + + tilts = [x for x in (_num(s[2].get("TiltAngle")) for s in images) if x is not None] + doses = [x for x in (_num(s[2].get("ExposureDose")) for s in images) if x is not None] + stamps = [_text(s[2].get("DateTime")) for s in images] + stamps = [s for s in stamps if s] + pieces = sum(1 for s in images if "PieceCoordinates" in s[2]) + + total_dose = round(sum(doses), 4) if doses and any(d > 0 for d in doses) else None + first_dose = _positive(_num(first.get("ExposureDose"))) + + fields: Dict[str, object] = { + # The values compare_acquisition reads, in the names the EPU decoder uses. + "voltageKv": _positive(_num(pick("Voltage"))), + "pixelSizeA": _positive(_num(pick("PixelSpacing"))), + "magnification": _positive(_int(first.get("Magnification"))), + "doseEA2": first_dose, # one image (one tilt, or one movie); see totalDoseEA2 + "fractions": _positive(_int(first.get("NumSubFrames"))), + "exposureS": _positive(_num(first.get("ExposureTime"))), + # The stack as a whole. + "software": software, + "imageFile": _text(glob.get("ImageFile")), + "imageSize": _ints(glob.get("ImageSize")), + "dataMode": _int(glob.get("DataMode")), + "montage": (_int(glob.get("Montage")) == 1) if "Montage" in glob else bool(montage_sections) or None, + "tiltAxisAngle": tilt_axis, + "titles": titles[:_MAX_TITLES], + "sectionKind": images[0][0] if images else None, + "imageCount": len(images), + "montageSectionCount": len(montage_sections), + "pieceCount": pieces or None, + "imageFiles": [s[2]["__file"] for s in images if s[0] == "Image"][:_MAX_TITLES] or None, + "truncated": len(data) >= MDOC_READ_BYTES, + # Over every image read. + "tiltAngleMin": min(tilts) if tilts else None, + "tiltAngleMax": max(tilts) if tilts else None, + "totalDoseEA2": total_dose, + "acquiredAt": _iso(stamps[0] if stamps else title_date), + "lastAcquiredAt": _iso(stamps[-1]) if len(stamps) > 1 else None, + # The first image, as SerialEM wrote it (the settings are the same for every + # tilt of a series; this is also the first file to land in a live session). + "firstImage": { + "tiltAngle": _num(first.get("TiltAngle")), + "stagePosition": _nums(first.get("StagePosition")), + "stageZ": _num(first.get("StageZ")), + "defocusUm": _num(first.get("Defocus")), + "targetDefocusUm": _num(first.get("TargetDefocus")), + "imageShift": _nums(first.get("ImageShift")), + "binning": _num(first.get("Binning")), + "spotSize": _int(first.get("SpotSize")), + "magIndex": _int(first.get("MagIndex")), + "intensity": _num(first.get("Intensity")), + "doseRate": _num(first.get("DoseRate")), + "countsPerElectron": _num(first.get("CountsPerElectron")), + "cameraIndex": _int(first.get("CameraIndex")), + "rotationAngle": _num(first.get("RotationAngle")), + "filterSlitAndLoss": _nums(first.get("FilterSlitAndLoss")), + "subFramePath": _text(first.get("SubFramePath")), + "frameDosesAndNumber": _nums(first.get("FrameDosesAndNumber")), + "pieceCoordinates": _ints(first.get("PieceCoordinates")), + "navigatorLabel": _text(first.get("NavigatorLabel")), + "dateTime": _text(first.get("DateTime")), + } if images else None, + } + if montage_sections: + ms = montage_sections[0][2] + fields["montageSection"] = { + "fullMontageSize": _ints(ms.get("FullMontSize")), + "pixelSizeA": _positive(_num(ms.get("PixelSpacing"))), + "magnification": _positive(_int(ms.get("Magnification"))), + "binning": _num(ms.get("Binning")), + } + + summary = _summary(fields) + return DecodedHeader(format="serialem-mdoc", summary=summary, fields=fields) + + +def _summary(f: Dict[str, object]) -> str: + kind, n = f["sectionKind"], f["imageCount"] + if kind == "FrameSet": + head = "mdoc for one movie" if n == 1 else "mdoc, %d frame sets" % n + elif f["montage"]: + head = "montage mdoc, %d pieces" % (f["pieceCount"] or n) + elif n: + head = "mdoc, %d image%s" % (n, "" if n == 1 else "s") + else: + head = "mdoc, no images yet" + parts = [head] + if f["tiltAngleMin"] is not None and n > 1 and f["tiltAngleMax"] != f["tiltAngleMin"]: + parts.append("tilt %g to %g deg" % (f["tiltAngleMin"], f["tiltAngleMax"])) + if f["pixelSizeA"]: + parts.append("%.4g A/px" % f["pixelSizeA"]) + if f["voltageKv"]: + parts.append("%g kV" % f["voltageKv"]) + if f["totalDoseEA2"] and n > 1: + parts.append("%.4g e/A^2 total" % f["totalDoseEA2"]) + elif f["doseEA2"]: + parts.append("%.4g e/A^2" % f["doseEA2"]) + if f["fractions"]: + parts.append("%d frame%s" % (f["fractions"], "" if f["fractions"] == 1 else "s")) + if f["truncated"]: + parts.append("first %d KiB read" % (MDOC_READ_BYTES // 1024)) + return ", ".join(parts) + + +register_decoder("mdoc", decode_mdoc, read=Read(leading=MDOC_READ_BYTES)) diff --git a/tests/test_mdoc.py b/tests/test_mdoc.py new file mode 100644 index 0000000..d4c5a9a --- /dev/null +++ b/tests/test_mdoc.py @@ -0,0 +1,221 @@ +"""SerialEM .mdoc decoding. + +The mdoc documents below are synthetic: they follow the layout and key names of +real SerialEM and Thermo Fisher Tomography output, but every value is made up, so +no collection data lives in this public repo. During development the decoder was +also run over 116 public mdoc files from GitHub and agreed with mdocfile +(github.com/teamtomo/mdocfile) on every value compared, for all 111 that mdocfile +can read. +""" + +import json + +from scigantic_headers import compare_acquisition, decode_bytes, decode_mdoc, has_decoder_for + +TILT_SERIES = b"""PixelSpacing = 1.35 +Voltage = 300 +ImageFile = TS_01.mrc +ImageSize = 4096 4096 +DataMode = 1 +Version = SerialEM Version 4.1.10 64-bit, built Jan 1 2024 10:00:00 + +[T = SerialEM: Example Krios 01-Mar-24 09:15:00 ] + +[T = Tilt axis angle = 85.3, binning = 1 spot = 9 camera = 0] + +[ZValue = 0] +TiltAngle = 0.01 +StagePosition = 10.5 -20.25 +StageZ = 3.5 +Magnification = 81000 +Intensity = 0.12 +ExposureDose = 3.1 +PixelSpacing = 1.35 +SpotSize = 9 +Defocus = -3.02 +ImageShift = 0.1 -0.2 +RotationAngle = 175.3 +ExposureTime = 0.8 +Binning = 1 +MagIndex = 31 +CountsPerElectron = 32 +MinMaxMean = -12 400 30.5 +TargetDefocus = -3 +SubFramePath = X:\\frames\\TS_01_000_0.0.tif +NumSubFrames = 8 +FrameDosesAndNumber = 0.3875 8 +DateTime = 01-Mar-24 09:16:02 +FilterSlitAndLoss = 20 0 +CameraLength = NaN + +[ZValue = 1] +TiltAngle = 3.0 +ExposureDose = 3.1 +ExposureTime = 0.8 +NumSubFrames = 8 +DateTime = 01-Mar-24 09:16:40 + +[ZValue = 2] +TiltAngle = -3.0 +ExposureDose = 3.2 +ExposureTime = 0.8 +NumSubFrames = 8 +DateTime = 01-Mar-24 09:17:15 +""" + +# The file SerialEM writes beside each saved movie: title without brackets, one FrameSet. +FRAME_MDOC = b"""T = SerialEM: Example Krios 02-Mar-24 11:00:00 + +Voltage = 300 +[FrameSet = 0] +TiltAngle = 12.0 +StagePosition = 1 2 +Magnification = 105000 +ExposureDose = 2.5 +PixelSpacing = 0.83 +ExposureTime = 2 +Binning = 1 +NumSubFrames = 40 +DateTime = 02-Mar-24 11:00:05 +""" + +# A montage: the [MontSection] describes the montage as a whole and is not an image. +MONTAGE = b"""PixelSpacing = 120.5 +ImageFile = grid.mrc +ImageSize = 2880 2046 +Montage = 1 +DataMode = 1 + +[T = SerialEM: Example Krios 03-Mar-24 08:00:00 ] + +[ZValue = 0] +PieceCoordinates = 0 0 0 +TiltAngle = 0.0 +Magnification = 135 +ExposureDose = 0 +PixelSpacing = 120.5 +DateTime = 03-Mar-24 08:01:00 + +[ZValue = 1] +PieceCoordinates = 2600 0 0 +TiltAngle = 0.0 +Magnification = 135 +ExposureDose = 0 +DateTime = 03-Mar-24 08:01:30 + +[MontSection = 0] +TiltAngle = 0.0 +Magnification = 135 +PixelSpacing = 120.5 +Binning = 2 +FullMontSize = 5480 2046 +""" + + +def test_tilt_series_fields(): + h = decode_mdoc(TILT_SERIES) + f = h.fields + assert h.format == "serialem-mdoc" + assert f["voltageKv"] == 300 and f["pixelSizeA"] == 1.35 + assert f["magnification"] == 81000 and f["doseEA2"] == 3.1 + assert f["fractions"] == 8 and f["exposureS"] == 0.8 + assert f["software"] == "SerialEM 4.1.10" + assert f["imageFile"] == "TS_01.mrc" and f["imageSize"] == [4096, 4096] and f["dataMode"] == 1 + assert f["tiltAxisAngle"] == 85.3 + assert f["sectionKind"] == "ZValue" and f["imageCount"] == 3 and f["montageSectionCount"] == 0 + assert (f["tiltAngleMin"], f["tiltAngleMax"]) == (-3.0, 3.0) + assert f["totalDoseEA2"] == 9.4 + assert f["acquiredAt"] == "2024-03-01T09:16:02" and f["lastAcquiredAt"] == "2024-03-01T09:17:15" + first = f["firstImage"] + assert first["stagePosition"] == [10.5, -20.25] and first["imageShift"] == [0.1, -0.2] + assert first["defocusUm"] == -3.02 and first["targetDefocusUm"] == -3 + assert first["subFramePath"] == "X:\\frames\\TS_01_000_0.0.tif" + assert first["frameDosesAndNumber"] == [0.3875, 8] + assert h.summary == "mdoc, 3 images, tilt -3 to 3 deg, 1.35 A/px, 300 kV, 9.4 e/A^2 total, 8 frames" + + +def test_frame_mdoc_with_unbracketed_title(): + f = decode_mdoc(FRAME_MDOC).fields + assert f["sectionKind"] == "FrameSet" and f["imageCount"] == 1 + assert f["software"] == "SerialEM" + assert f["titles"][0].startswith("SerialEM: Example Krios") + # global PixelSpacing is absent here, so the frame set's own is used + assert f["pixelSizeA"] == 0.83 and f["voltageKv"] == 300 + assert f["fractions"] == 40 and f["doseEA2"] == 2.5 + assert decode_mdoc(FRAME_MDOC).summary.startswith("mdoc for one movie") + + +def test_montage_section_is_not_an_image(): + f = decode_mdoc(MONTAGE).fields + assert f["montage"] is True + assert f["imageCount"] == 2 and f["pieceCount"] == 2 and f["montageSectionCount"] == 1 + assert f["firstImage"]["pieceCoordinates"] == [0, 0, 0] + assert f["montageSection"] == {"fullMontageSize": [5480, 2046], "pixelSizeA": 120.5, + "magnification": 135, "binning": 2.0} + # a zero dose is "not calibrated", not a measurement + assert f["doseEA2"] is None and f["totalDoseEA2"] is None + assert decode_mdoc(MONTAGE).summary.startswith("montage mdoc, 2 pieces") + + +def test_tomography_title_and_older_tilt_axis_form(): + data = (b"PixelSpacing = 2.0\nImageFile = p.mrc\n" + b"[T = Tomography: EXAMPLE 04-Mar-24 10:00:00]\n" + b"[T = TiltAxisAngle = -174.2 Binning = 1 SpotSize = 7]\n" + b"[ZValue = 0]\nTiltAngle = 1\n") + f = decode_mdoc(data).fields + assert f["software"] == "Tomography" and f["tiltAxisAngle"] == -174.2 + assert f["acquiredAt"] == "2024-03-04T10:00:00" + + +def test_imod_image_series(): + data = b"ImageSeries = 1\n\n[Image = a.tif]\nTiltAngle = 3\n\n[Image = b.tif]\nTiltAngle = 4\n" + f = decode_mdoc(data).fields + assert f["imageCount"] == 2 and f["imageFiles"] == ["a.tif", "b.tif"] + assert (f["tiltAngleMin"], f["tiltAngleMax"]) == (3, 4) + + +def test_global_only_file_written_before_the_first_image(): + f = decode_mdoc(b"PixelSpacing = 1.1\nImageFile = new.mrc\nImageSize = 10 10\n").fields + assert f["imageCount"] == 0 and f["firstImage"] is None and f["pixelSizeA"] == 1.1 + + +def test_crlf_bom_and_latin1(): + crlf = b"\xef\xbb\xbf" + TILT_SERIES.replace(b"\n", b"\r\n") + assert decode_mdoc(crlf).fields == decode_mdoc(TILT_SERIES).fields + latin = TILT_SERIES.replace(b"Example Krios", b"Example Krios \xb5") + assert decode_mdoc(latin).fields["titles"][0].startswith("SerialEM: Example Krios \u00b5") + + +def test_bad_values_are_absent_not_errors(): + data = (b"PixelSpacing = abc\nVoltage = NaN\n[ZValue = 0]\nTiltAngle = inf\n" + b"StagePosition = 1 x\nMagnification = 1.5\nExposureDose = -2\n") + f = decode_mdoc(data).fields + assert f["pixelSizeA"] is None and f["voltageKv"] is None and f["tiltAngleMin"] is None + assert f["firstImage"]["stagePosition"] is None and f["magnification"] is None and f["doseEA2"] is None + json.dumps(decode_mdoc(data).to_dict(), allow_nan=False) + + +def test_truncated_read_is_reported(): + from scigantic_headers.mdoc import MDOC_READ_BYTES + big = TILT_SERIES + b"".join(b"[ZValue = %d]\nTiltAngle = 1\nExposureDose = 1\n" % i + for i in range(3, 40000)) + h = decode_mdoc(big[:MDOC_READ_BYTES]) + assert h.fields["truncated"] is True and "first 1024 KiB read" in h.summary + assert decode_mdoc(TILT_SERIES).fields["truncated"] is False + + +def test_rejects_what_is_not_an_mdoc(): + for data in (b"", b"\x00\x01\x02" * 50, b"hello world\n", b"# a comment\nkey: value\n", + b"", b"[section]\nname = x\n"): + assert decode_mdoc(data) is None + + +def test_dispatched_by_extension(): + assert has_decoder_for("TS_01.mrc.mdoc") and has_decoder_for("frames/x.tif.MDOC") + assert decode_bytes("TS_01.mrc.mdoc", TILT_SERIES).format == "serialem-mdoc" + + +def test_feeds_compare_acquisition(): + mdoc = decode_mdoc(TILT_SERIES).fields + out = compare_acquisition({"mdoc": mdoc, "sheet": {"voltageKv": 200, "pixelSizeA": 1.35, "fractions": 8}}) + assert [(x.field, x.severity) for x in out] == [("voltageKv", "error")]