diff --git a/.gitignore b/.gitignore index d45ba09f5..980d33e02 100644 --- a/.gitignore +++ b/.gitignore @@ -219,3 +219,6 @@ HANDOFF*.md # Curation figures and notes of the mito / nucleus EM generalists (kept locally). mito_nuc_figures/ + +# Generated paper figures and results tables (kept locally). +scripts/plotting/v2/for_paper/figures/ diff --git a/finetuning/v2/evaluation/common.py b/finetuning/v2/evaluation/common.py index 232e72c74..4c338cfec 100644 --- a/finetuning/v2/evaluation/common.py +++ b/finetuning/v2/evaluation/common.py @@ -25,7 +25,6 @@ from micro_sam.v1.evaluation.livecell import _get_livecell_paths from micro_sam.v2.normalization import normalize_raw -from micro_sam.v2.models.util import joint_unetr_state from micro_sam.v2.datasets.generalist_loader import ( ASTIH_SUBSETS, AXONEM_TEST_VOLUMES, AXONEM_VAL_VOLUMES, FAFB_TEST_BOXES, FAFB_VAL_BOXES, FIB25_TEST_SAMPLE, LICONN_ROI, MALECNS_TEST_BOXES, MALECNS_VAL_BOXES, TUMOR_SPHEROID_TEST_SLICES, TUMOR_SPHEROID_VAL_SLICES, @@ -184,6 +183,36 @@ } +def use_tmpdir_for_embeddings(): + """Write the temporary embeddings of micro-sam to the job's node-local TMPDIR, not to the micro-sam cache. + + In the cache of the home directory, the leftovers of jobs killed at their time limit fill the quota. + """ + if not os.environ.get("TMPDIR"): + return + import uuid + import atexit + import shutil + import micro_sam.util + import micro_sam.v2.util + import micro_sam.v2.instance_segmentation + import micro_sam.v2.automatic_prompt_generation + + def make_temp_embedding_path(): + parent = Path(os.environ["TMPDIR"], "micro_sam_tmp_embeddings") + parent.mkdir(parents=True, exist_ok=True) + path = str(parent / f"{uuid.uuid4().hex}.zarr") + atexit.register(shutil.rmtree, path, ignore_errors=True) + return path + + modules = ( + micro_sam.util, micro_sam.v2.util, micro_sam.v2.instance_segmentation, + micro_sam.v2.automatic_prompt_generation, + ) + for module in modules: + module.make_temp_embedding_path = make_temp_embedding_path + + def drop_excluded_test_images(dataset_name, raw_paths, label_paths) -> Tuple[List[str], List[str]]: """Remove the test images of a dataset that have a copy in the training data, see EXCLUDED_TEST_IMAGES.""" excluded = EXCLUDED_TEST_IMAGES.get(dataset_name) @@ -243,41 +272,43 @@ def _joint_export_root() -> str: # The four E. coli pathways on disk; the remaining four are 6 GB archives each and are not fetched. ECOLI_GENES = ("cib", "crosstalk", "recA", "rpsM") -# Light microscopy, 2d. The in-domain (ID) panels hold the blind test splits of the v5 training datasets that the -# main comparison scores: the sources shared with the baselines. The supplementary panels hold the blind splits of -# the remaining training datasets ("extra training"), resolvable and scorable on request but outside the main -# panel. Out-of-domain (OOD): datasets kept out of training. Datasets with a cell and a nucleus target are listed -# per target, omnipose per subset. +# Light microscopy, 2d. The in-domain (ID) and out-of-domain (OOD) lists hold exactly the datasets of the Figure 2 +# panels, six each per modality (see PANEL_DATASETS in 'plots_for_figure2.py'). ID: the blind test splits of v5 +# training datasets. OOD: datasets kept out of training. The supplementary lists hold the blind splits of the +# remaining training datasets, resolvable and scorable on request but outside the main panels. Datasets with a cell +# and a nucleus target are listed per target, omnipose per subset. DATASETS_2D_LM_CELL_ID = [ - "tissuenet", "omnipose_bact_fluor", "cvz_fluo_cell", "flywing", "enseg", "neurips_cellseg_fluorescence", -] -DATASETS_2D_LM_CELL_SUPPLEMENTARY = ["dememseg", "pan_multiplex", "xenium_cells", "deepbacs_fluorescence"] -DATASETS_2D_LM_CELL_OOD = ["covid_if_cells", "medussa", "hpa"] -DATASETS_2D_LM_NUCLEUS_ID = [ - "dsb", "cvz_fluo_dapi", "dynamicnuclearnet", "bitdepth_nucseg", "bmgd", "cellbindb", "u20s", "ifnuclei", - "tsakiroglou", + "tissuenet", "omnipose_bact_fluor", "cvz_fluo_cell", "flywing", "neurips_cellseg_fluorescence", + "deepbacs_fluorescence", ] +# xenium_cells and xenium_nuclei are not evaluated: their labels are the automatic segmentation of the Xenium +# instrument, not annotations. Their data loading below is kept. +DATASETS_2D_LM_CELL_SUPPLEMENTARY = ["dememseg", "pan_multiplex", "enseg"] +DATASETS_2D_LM_CELL_OOD = ["covid_if_cells", "medussa", "hpa", "mif_tonsil_cells", "fusionx", "wing_disc_timelapse"] +DATASETS_2D_LM_NUCLEUS_ID = ["dsb", "dynamicnuclearnet", "bmgd", "cellbindb", "u20s", "ifnuclei"] # mnDINO images are BBBC022 U2OS fields: 30 of its 64 test images are byte-identical to u20s training images and # 3 more to DSB training images, so it is not out of domain. -DATASETS_2D_LM_NUCLEUS_SUPPLEMENTARY = ["xenium_nuclei", "mndino"] -DATASETS_2D_LM_NUCLEUS_OOD = ["cardioblast_nuclei", "hela_cytonuc", "covid_if_nuclei", "arvidsson", "micro_bench"] +DATASETS_2D_LM_NUCLEUS_SUPPLEMENTARY = ["mndino", "cvz_fluo_dapi", "bitdepth_nucseg", "tsakiroglou"] +DATASETS_2D_LM_NUCLEUS_OOD = [ + "cardioblast_nuclei", "hela_cytonuc", "covid_if_nuclei", "arvidsson", "micro_bench", "mif_tonsil", +] # sPATCH DAPI shares tissue sections with the sPATCH H&E training data, so it is a held-out platform evaluation # rather than an independent OOD collection until the patient overlap is resolved. DATASETS_2D_LM_NUCLEUS_HELD_OUT_PLATFORM = ["spatch_dapi"] DATASETS_2D_LM_LABEL_FREE_ID = [ - "livecell", "deepbacs_label_free", "omnipose_bact_phase", "neurips_cellseg_label_free", "cell_acdc", "cellular", - "vicar", "microbeseg", + "livecell", "omnipose_bact_phase", "neurips_cellseg_label_free", "cell_acdc", "vicar", "orgline", ] # yeaz is split per image, so every test image is a time point of a field of view that also trains. DATASETS_2D_LM_LABEL_FREE_SUPPLEMENTARY = [ - "orgasegment", "organoidnet", "omnipose_worm", "omnipose_worm_high_res", "bccd", "cisd", "orgline", "organoid", - "mcellseg", "toiam", "bbbc030", "yeaz", + "orgasegment", "organoidnet", "omnipose_worm", "omnipose_worm_high_res", "bccd", "cisd", "organoid", + "mcellseg", "toiam", "bbbc030", "yeaz", "deepbacs_label_free", "cellular", "microbeseg", ] DATASETS_2D_LM_LABEL_FREE_OOD = [ - "cellapp", "deepseas", "dic_hepg2", "yeastsam", "bac_mother", "ecoli_microcolony_lineage", "bbbc010", + "cellapp", "deepseas", "yeastsam", "bac_mother", "ecoli_microcolony_lineage", "bbbc010", ] -# Reserve: kept resolvable, not part of any panel until its labels are checked. -DATASETS_2D_LM_LABEL_FREE_RESERVE = ["yeastcellseg"] +# Reserve: kept out of training and resolvable, but not part of any panel. yeastcellseg: labels not checked yet. +# dic_hepg2: all methods score close to 0 on it, which suggests a problem with its labels. +DATASETS_2D_LM_LABEL_FREE_RESERVE = ["yeastcellseg", "dic_hepg2"] DATASETS_2D_LM = list(dict.fromkeys( DATASETS_2D_LM_CELL_ID + DATASETS_2D_LM_CELL_SUPPLEMENTARY + DATASETS_2D_LM_CELL_OOD + DATASETS_2D_LM_NUCLEUS_ID + DATASETS_2D_LM_NUCLEUS_SUPPLEMENTARY + DATASETS_2D_LM_NUCLEUS_OOD @@ -288,14 +319,19 @@ def _joint_export_root() -> str: # Histopathology nuclei. The main panel holds the official test splits of the v5 training datasets shared with the # baselines, which the generalist loader never touches. Supplementary: training datasets with a blind split outside # the main panel; lizard (overlap with CoNSeP and PanNuke), lynsec and tnbc_celltype (baseline exposure) stay there -# until those are resolved. Out-of-domain: datasets kept out of training. lynsec is split by stain so H&E and IHC -# are reported apart. +# until those are resolved. Out-of-domain: datasets kept out of training, with dense manual or fully proofread +# nucleus annotations. lynsec is split by stain so H&E and IHC are reported apart. DATASETS_2D_HP_ID = ["cpm17", "glysac", "histo_miner", "monuseg", "pannuke", "puma"] DATASETS_2D_HP_SUPPLEMENTARY = [ "lizard", "lizard_mitosis", "lynsec_he", "lynsec_ihc", "srsanet", "tnbc_celltype", "cellbindb_he", ] -DATASETS_2D_HP_OOD = ["cytodark0", "deepliif", "khoshdeli", "panoptils", "pcns"] -DATASETS_HP = DATASETS_2D_HP_ID + DATASETS_2D_HP_SUPPLEMENTARY + DATASETS_2D_HP_OOD +DATASETS_2D_HP_OOD = ["amonuseg", "beacon", "cd47_ihcnusc", "cytodark0", "khoshdeli", "pcns"] +# Reserve: kept out of training and resolvable, but not part of any panel, since their annotations are not dense +# or not manual. deepliif: masks drawn on the mpIF DAPI channel, slightly off the IHC image. panoptils: +# bootstrapped, semi-manual labels. nusec: many nuclei, most of all stromal ones, are not annotated. bc_multiset: +# some nuclei are not annotated, and no paper describes the protocol yet. +DATASETS_2D_HP_RESERVE = ["deepliif", "panoptils", "nusec", "bc_multiset"] +DATASETS_HP = DATASETS_2D_HP_ID + DATASETS_2D_HP_SUPPLEMENTARY + DATASETS_2D_HP_OOD + DATASETS_2D_HP_RESERVE # 2D EM: ASTIH (neurites), the tumor spheroid slices (cells) and the mitochondria and nucleus panels. DATASETS_2D_EM_MITO = [ @@ -345,15 +381,22 @@ def _joint_export_root() -> str: # Light microscopy, 3d, grouped as the 2d datasets. nis3d is the Drosophila pair the loader trains on. Of MorphoNet # only the C. elegans nuclei (CTC Fluo-N3DH-CE) are scored: its Arabidopsis volumes are the PNAS plant1 time series -# and its Phallusia volumes come from the PhMamm source, both of which train. +# and its Phallusia volumes come from the PhMamm source, both of which train. mouse_embryo_membrane is the light-sheet +# membrane stain of the mouse embryo data, mucic_mda231 the CTC Fluo-C3DL-MDA231 movies with the ISBI 2025 full +# annotations; no model trains on either. morphonet_patiria is the MorphoNet starfish embryo with its curated labels. +# nus_organoids is the nucleus benchmark of the Digitalized Organoid paper (three of its four manually annotated +# volumes). DATASETS_3D_LM_CELL_ID = ["plantseg_root", "pnas_arabidopsis", "cartocell", "phmamm", "wing_disc", "embedseg_organoid"] -DATASETS_3D_LM_CELL_OOD = ["plantseg_ovules", "cshaper", "vibrio_cholerae"] +DATASETS_3D_LM_CELL_OOD = [ + "plantseg_ovules", "cshaper", "vibrio_cholerae", "mouse_embryo_membrane", "mucic_mda231", "morphonet_patiria", +] DATASETS_3D_LM_NUCLEUS_ID = [ "embedseg_mouse_skull", "embedseg_platy_nuclei", "nis3d", "celegans_atlas", "gonuclear", "nucverse3d", ] DATASETS_3D_LM_NUCLEUS_SUPPLEMENTARY = ["embedseg_platy_ish"] DATASETS_3D_LM_NUCLEUS_OOD = [ "parhyale_regen", "mouse_embryo", "blastospim", "morphonet_celegans", "bbbc032", "bbbc033", "bbbc050", + "nus_organoids", ] DATASETS_3D_LM = ( DATASETS_3D_LM_CELL_ID + DATASETS_3D_LM_CELL_OOD + DATASETS_3D_LM_NUCLEUS_ID + DATASETS_3D_LM_NUCLEUS_SUPPLEMENTARY @@ -361,15 +404,20 @@ def _joint_export_root() -> str: ) # Neurite segmentation: the blind regions of the v5 training sets (see EM_ROIS and the path resolver). The main -# panel holds the sources shared with the baselines, the supplementary panel the remaining training sets. nisb is +# panel holds the sources shared with the baselines, the supplementary panel the remaining training sets. snemi +# (mouse ATUM-SEM) is in the main panel, as the third EM technique next to cremi and zebrafinch_j0126. nisb is # synthetic and kept out of training, so it is a synthetic OOD test. synapseweb is scored inside its annotated cores # only. humanneurons (the cached H01 crop) is no OOD set: H01 trains through EMNeuron, so it is supplementary at most. -DATASETS_3D_EM_NEURITE_ID = ["cremi", "fafb", "hemibrain", "zebrafinch_j0126", "liconn", "xpress"] +# microns_basil and ionsem are the out-of-distribution validation volumes of EMNeuron, which neither micro-sam2 nor +# SegNeuron trains on (MICrONS basil: mouse visual cortex, ssTEM). cortex_human_stg is a central crop of the human +# superior temporal gyrus volume of CortexConnectomics (Loomba et al. 2022, SBEM), whose proofread labels keep some +# block-stitching seams. +DATASETS_3D_EM_NEURITE_ID = ["cremi", "fafb", "hemibrain", "zebrafinch_j0126", "snemi", "liconn", "xpress"] DATASETS_3D_EM_NEURITE_SUPPLEMENTARY = [ - "snemi", "axonem", "fib25", "manc", "malecns", "wafer4", "minnie65", "zebrafinch_j0251", "wildenberg", + "axonem", "fib25", "manc", "malecns", "wafer4", "minnie65", "zebrafinch_j0251", "wildenberg", "humanneurons", ] -DATASETS_3D_EM_NEURITE_OOD = ["isbi2012", "synapseweb", "nisb"] +DATASETS_3D_EM_NEURITE_OOD = ["isbi2012", "synapseweb", "nisb", "microns_basil", "ionsem", "cortex_human_stg"] # Cell segmentation: Platynereis volume 9 and the DenseCell val volume are blind; the tumor spheroid slices are 2d. DATASETS_3D_EM_CELL_ID = ["platynereis_cells", "densecell", "synapsenet_compartments"] @@ -446,7 +494,23 @@ def _joint_export_root() -> str: DATASETS_3D_EM_NUCLEUS_OOD = [ "human_liver_em_nucleus", "etchacell_nucleus", "nucmm_zebrafish", "nucmm_mouse", "aive_nucleus", ] -DATASETS_3D_EM_NUCLEUS = DATASETS_3D_EM_NUCLEUS_ID + DATASETS_3D_EM_NUCLEUS_OOD +# In-domain test sets of the EM nucleus panel, written by 'preprocess_nucleus_dataset.py': the OpenOrganelle +# dauer larva volume, which the nucleus model never saw, and the validation part of the human organoid volumes. +NUCLEUS_TEST_SET_ROOTS = { + "dauer_larva_nucleus": "dauer_larva_nucleus", "human_organoids_nucleus": "human_organoids_nucleus_val", +} +# The human placenta array tomography volumes (S-BIAD3408, torch-em 'placenta_at'): manual nuclei of one CTB cell and +# an arbitrary STB region per volume, so they are scored with LABELLED_ONLY_DATASETS (user, 2026-11-05). +PLACENTA_NUCLEUS_ROOT = "placenta_at" +# Automatic nucleus segmentations of connectome volumes (torch-em loaders of the same names, 6 crops of +# 64 x 1024 x 1024 each): the first crop tunes, the others are scored (user, 2026-11-05). +CONNECTOME_NUCLEUS_TUNING = { + "malecns_nuclei": "brain_0.h5", "zheng_ca3_nuclei": "crop_0.h5", "acson_nuclei": "crop_0.h5", +} +DATASETS_3D_EM_NUCLEUS_CANDIDATES = ( + list(NUCLEUS_TEST_SET_ROOTS) + ["placenta_nucleus"] + list(CONNECTOME_NUCLEUS_TUNING) +) +DATASETS_3D_EM_NUCLEUS = DATASETS_3D_EM_NUCLEUS_ID + DATASETS_3D_EM_NUCLEUS_OOD + DATASETS_3D_EM_NUCLEUS_CANDIDATES DATASETS_EM_NUCLEUS = DATASETS_3D_EM_NUCLEUS + DATASETS_2D_EM_NUCLEUS DATASETS_3D_EM = ( @@ -469,7 +533,8 @@ def _joint_export_root() -> str: DATASETS_SUPPLEMENTARY = ( DATASETS_2D_LM_CELL_SUPPLEMENTARY + DATASETS_2D_LM_NUCLEUS_SUPPLEMENTARY + DATASETS_2D_LM_NUCLEUS_HELD_OUT_PLATFORM + DATASETS_2D_LM_LABEL_FREE_SUPPLEMENTARY + DATASETS_2D_LM_LABEL_FREE_RESERVE + DATASETS_3D_LM_NUCLEUS_SUPPLEMENTARY - + DATASETS_3D_EM_NEURITE_SUPPLEMENTARY + DATASETS_2D_HP_SUPPLEMENTARY + DATASETS_3D_EM_MITO_SUPPLEMENTARY + + DATASETS_3D_EM_NEURITE_SUPPLEMENTARY + DATASETS_2D_HP_SUPPLEMENTARY + DATASETS_2D_HP_RESERVE + + DATASETS_3D_EM_MITO_SUPPLEMENTARY ) # The split to tune on, or None where the loader has no splits and VAL_Z_RANGE holds out a z-slab. @@ -490,9 +555,10 @@ def _joint_export_root() -> str: "covid_if_cells": "val", "covid_if_nuclei": "val", "medussa": "train", "cardioblast_nuclei": "train", "hela_cytonuc": "val", "arvidsson": "val", "mndino": "val", "cellapp": "train", "deepseas": "train", "dic_hepg2": "val", "bac_mother": "val", "plantseg_ovules": "val", "cshaper": "train", "mouse_embryo": "train", - "blastospim": "val", "bbbc010": "val", "bbbc050": "train", + "mouse_embryo_membrane": "train", "mucic_mda231": "val", "blastospim": "val", "bbbc010": "val", "bbbc050": "train", "wing_disc": None, "embedseg_mouse_skull": None, "embedseg_platy_ish": None, "nis3d": None, - "humanneurons": None, "bbbc032": None, "bbbc033": None, + "humanneurons": None, "bbbc032": None, "bbbc033": None, "morphonet_patiria": None, "nus_organoids": None, + "microns_basil": None, "ionsem": None, "cortex_human_stg": None, }) # Volumes whose tuning data is a z-slab of the test volume, as (file name, z-slab), following the loader; the @@ -531,7 +597,7 @@ def _joint_export_root() -> str: "cpm17": "train", "glysac": "train", "histo_miner": "val", "lizard": "val", "lizard_mitosis": "val", "lynsec_he": "val", "lynsec_ihc": "val", "monuseg": "train", "pannuke": "val", "puma": "val", "srsanet": "val", "tnbc_celltype": "val", "cytodark0": "val", "deepliif": "val", "pcns": "train", - "cellbindb_he": "val", + "cellbindb_he": "val", "cd47_ihcnusc": "train", "nusec": "train", "bc_multiset": "train", }) # The tuning slab for volumes with no splits, disjoint from the slab the evaluation scores. Indices @@ -706,6 +772,18 @@ def _sorted_pairs(raw_paths, label_paths) -> Tuple[List[str], List[str]]: return [pair[0] for pair in pairs], [pair[1] for pair in pairs] +def pad_to_min_shape(image, min_shape): + """Pad a 2d image at both sides with zeros to at least 'min_shape', and return it with the crop that undoes it. + + The padding follows the PathoSAM evaluation, which pads the 256 x 256 PanNuke tiles to its 512 x 512 training + patch shape. + """ + import numpy as np + pad = [(max(t - s, 0) // 2, max(t - s, 0) - max(t - s, 0) // 2) for s, t in zip(image.shape[:2], min_shape)] + crop = tuple(slice(before, before + s) for (before, _), s in zip(pad, image.shape[:2])) + return np.pad(image, pad + [(0, 0)] * (image.ndim - 2), mode="constant"), crop + + def _tiles_from_stack(stack_path: str, raw_key: str, label_key: str, out_dir: str) -> List[str]: """Write every tile of a stacked h5 (raw (C, N, H, W), labels (N, H, W)) into its own h5 file once. @@ -757,6 +835,83 @@ def _frames_from_movie(raw_path: str, label_path: str, out_dir: str, stride: int return paths +# The tile shape that the large images of a dataset are cut into, instead of the center crop. A center crop of +# MEDUSSA misses most of the sparse bacteria; the best tile of every image is scored instead. +GRID_TILE_SHAPE_2D = {"medussa": (512, 512)} + +# The size that the HPA images are resized to fit, then zero-padded to. A center crop or a tile of the native +# images cuts the large cells, and the cells of the native images are too large for micro-sam2. +HPA_RESIZE_2D = 512 + + +def _grid_tiles(raw_paths, label_paths, raw_key, label_key, out_dir, tile_shape, best_per_image=False): + """Cut every image into the tiles of a grid that covers it, and keep the tiles with labelled objects, once. + + The last row and column of tiles are shifted to end at the image border, so the tiles cover the whole image. + Every tile is written to its own h5 file, with the raw data under 'raw' (one key per channel for a tuple of + raw keys) and the labels under 'labels'. With 'best_per_image', only the tile with the most objects of an + image is kept. + """ + import h5py + from glob import glob + + marker = os.path.join(out_dir, ".complete") + if not os.path.exists(marker): + os.makedirs(out_dir, exist_ok=True) + keys = raw_key if isinstance(raw_key, tuple) else (raw_key,) + for raw_path, label_path in zip(raw_paths, label_paths): + labels = read_2d(label_path, label_key) + raws = [read_2d(raw_path, key) for key in keys] + height, width = labels.shape[:2] + th, tw = min(tile_shape[0], height), min(tile_shape[1], width) + ys = sorted(set(list(range(0, height - th + 1, th)) + [height - th])) + xs = sorted(set(list(range(0, width - tw + 1, tw)) + [width - tw])) + stem = os.path.splitext(os.path.basename(raw_path))[0] + windows = [(y, x) for y in ys for x in xs] + if best_per_image: + counts = [len(np.unique(labels[y:y + th, x:x + tw])) for y, x in windows] + windows = [windows[int(np.argmax(counts))]] + for y, x in windows: + window = np.s_[y:y + th, x:x + tw] + if labels[window].max() == 0: + continue + with h5py.File(os.path.join(out_dir, f"{stem}_y{y:05d}_x{x:05d}.h5"), "w") as f: + for key, raw in zip(keys, raws): + f.create_dataset(key, data=raw[window], compression="gzip") + f.create_dataset("labels", data=labels[window], compression="gzip") + open(marker, "w").close() + return sorted(glob(os.path.join(out_dir, "*.h5"))) + + +def _resized_images(raw_paths, label_paths, raw_key, label_key, out_dir, size): + """Resize every image so that its longest side is 'size', zero-pad it to (size, size) and write it once. + + The raw channels are resized with linear interpolation, the labels with nearest neighbours. + """ + import h5py + from skimage.transform import resize as sk_resize + + keys = raw_key if isinstance(raw_key, tuple) else (raw_key,) + os.makedirs(out_dir, exist_ok=True) + out_paths = [] + for raw_path, label_path in zip(raw_paths, label_paths): + path = os.path.join(out_dir, os.path.basename(raw_path).rsplit(".", 1)[0] + ".h5") + out_paths.append(path) + if os.path.exists(path): + continue + labels = read_2d(label_path, label_key) + scale = size / max(labels.shape[:2]) + shape = tuple(int(round(s * scale)) for s in labels.shape[:2]) + pad = [(0, size - s) for s in shape] + with h5py.File(path, "w") as f: + for key in keys: + raw = sk_resize(read_2d(raw_path, key).astype("float32"), shape, order=1, anti_aliasing=True) + f.create_dataset(key, data=np.pad(raw, pad), compression="gzip") + lab = sk_resize(labels, shape, order=0, anti_aliasing=False, preserve_range=True).astype(labels.dtype) + f.create_dataset("labels", data=np.pad(lab, pad), compression="gzip") + return out_paths + + def _tiles_from_slide( slide_path: str, keys: Tuple[str, ...], count_key: str, out_dir: str, n_tiles: int, tile_shape: Tuple[int, int] = (1024, 1024), min_instances: int = 20, seed: int = 42, @@ -909,7 +1064,9 @@ def _cmito_dauer_manual_patches(data_root: str, download: bool) -> List[str]: # the TissueNet layout pan_multiplex trains with. LM_IMAGE_CHANNELS = { "cvz_fluo_cell": (0, 2, 1), "cvz_fluo_dapi": (0,), "enseg": (1,), "xenium_cells": (1, 2, 3), - "hela_cytonuc": (2,), "micro_bench": (2,), "spatch_dapi": (0,), "bac_mother": (0,), + "hela_cytonuc": (2,), "micro_bench": (2,), "bac_mother": (0,), + # The sPATCH DAPI tiles store the stain in the blue channel; the red and green channels are empty. + "spatch_dapi": (2,), # The Lewy Mito crops are gray values stored as RGB. "lewy_mito": (0,), } @@ -1033,6 +1190,36 @@ def _get_hp_data_paths( paths = [path for path in paths if int(os.path.splitext(os.path.basename(path))[0]) not in PCNS_MONUSEG_PATCHES] return sorted(paths), sorted(paths), "raw", "labels/instances" + if dataset_name == "amonuseg": + # No split and no held-out data: all 48 images are scored, against the annotations of the first expert. + paths = datasets.amonuseg.get_amonuseg_paths(path=os.path.join(p, "amonuseg"), download=download) + return sorted(paths), sorted(paths), "raw", "labels/instances/exp1" + + if dataset_name == "beacon": + # No official split and no held-out data: all 67 tiles of both cohorts are scored. + paths = datasets.beacon.get_beacon_paths(path=os.path.join(p, "beacon"), download=download) + return sorted(paths), sorted(paths), "raw", "labels/instances" + + if dataset_name == "cd47_ihcnusc": + # The authors' validation / test split is scored, their training split tunes. + paths = datasets.cd47_ihcnusc.get_cd47_ihcnusc_paths( + path=os.path.join(p, "cd47_ihcnusc"), split="val" if split == "test" else "train", download=download, + ) + return sorted(paths), sorted(paths), "raw", "labels/instances" + + if dataset_name == "nusec": + paths = datasets.nusec.get_nusec_paths( + path=os.path.join(p, "nusec"), split="test" if split == "test" else "train", download=download, + ) + return sorted(paths), sorted(paths), "raw", "labels/instances" + + if dataset_name == "bc_multiset": + # Fold 4 is scored and fold 1 tunes. The patches of a fold overlap, see the loader. + paths = datasets.bc_multiset.get_bc_multiset_paths( + path=os.path.join(p, "bc_multiset"), folds=[4] if split == "test" else [1], download=download, + ) + return sorted(paths), sorted(paths), "raw", "labels/instances" + raise ValueError(f"Unknown histopathology dataset: {dataset_name!r}") @@ -1517,12 +1704,75 @@ def _get_2d_lm_data_paths( return sorted(paths), sorted(paths), "raw/nuclei/s0", "labels/nuclei/s0" if dataset_name == "medussa": - paths = lm.medussa.get_medussa_paths(path=os.path.join(p, "medussa"), split=split, download=download) - return sorted(paths), sorted(paths), "raw", "labels" + paths = sorted(lm.medussa.get_medussa_paths(path=os.path.join(p, "medussa"), split=split, download=download)) + # Only the tile with the most bacteria of every image is scored, since most tiles hold few or no bacteria. + out_dir = os.path.join(p, "eval_tiles", f"medussa_{split}_best_roi") + tiles = _grid_tiles(paths, paths, "raw", "labels", out_dir, GRID_TILE_SHAPE_2D["medussa"], best_per_image=True) + return tiles, tiles, "raw", "labels" if dataset_name == "hpa": - paths = lm.hpa.get_hpa_segmentation_paths(path=os.path.join(p, "hpa"), split="val", download=download) - return sorted(paths), sorted(paths), HPA_CHANNELS, "labels" + paths = sorted(lm.hpa.get_hpa_segmentation_paths(path=os.path.join(p, "hpa"), split="val", download=download)) + # EVAL_HPA_CHANNEL_ORDER: experiment that stacks the three HPA channels in another order, e.g. '1,0,2'. + order = os.environ.get("EVAL_HPA_CHANNEL_ORDER") + channels = HPA_CHANNELS if order is None else tuple(HPA_CHANNELS[int(i)] for i in order.split(",")) + # Every image is resized to fit (HPA_RESIZE_2D, HPA_RESIZE_2D) and zero-padded, so no cell is cut. + out_dir = os.path.join(p, "eval_tiles", f"hpa_val_resize{HPA_RESIZE_2D}") + images = _resized_images(paths, paths, HPA_CHANNELS, "labels", out_dir, HPA_RESIZE_2D) + return images, images, channels, "labels" + + if dataset_name == "mif_tonsil_cells": + # The six marker channels form the membrane, DAPI (the last channel) the nuclei, written once per region. + # None of the regions trains, so every split returns all of them. + import h5py + img, gt = lm.mif_tonsil.get_mif_tonsil_paths(path=os.path.join(p, "mif_tonsil"), label_type="cells") + out_dir = os.path.join(p, "mif_tonsil", "cells_h5") + os.makedirs(out_dir, exist_ok=True) + paths = [] + for raw_path, label_path in zip(img, gt): + path = os.path.join(out_dir, os.path.basename(raw_path).replace(".tif", ".h5")) + if not os.path.exists(path): + raw = imageio.imread(raw_path) + with h5py.File(path, "w") as f: + f.create_dataset("raw/membrane", data=raw[:-1].max(axis=0)) + f.create_dataset("raw/nuclei", data=raw[-1]) + f.create_dataset("labels", data=imageio.imread(label_path)) + paths.append(path) + return paths, paths, ("raw/membrane", "raw/nuclei"), "labels" + + if dataset_name == "fusionx": + # Membrane stain stored as magenta RGB; it is scored as one gray channel, written once per image. + img, gt = lm.fusionx.get_fusionx_paths(path=os.path.join(p, "fusionx"), split="test", download=download) + out_dir = os.path.join(p, "fusionx", "gray") + os.makedirs(out_dir, exist_ok=True) + gray = [] + for path in img: + gray_path = os.path.join(out_dir, os.path.basename(path).replace(".png", ".tif")) + if not os.path.exists(gray_path): + imageio.imwrite(gray_path, imageio.imread(path)[..., :3].max(axis=-1)) + gray.append(gray_path) + return (*_sorted_pairs(gray, gt), None, None) + + if dataset_name == "wing_disc_timelapse": + # None of the five movies trains. Adjacent frames are highly correlated, so every 16th frame of a movie is + # scored, which gives 10 to 12 frames per movie. The tissue around the annotated pouch carries a few huge + # labels that are not cells; labels larger than 25 times the median cell are set to background, once. + img, gt = [], [] + out_dir = os.path.join(p, "eval_tiles", "wing_disc_timelapse_labels") + os.makedirs(out_dir, exist_ok=True) + for movie in lm.wing_disc_timelapse.MOVIES: + i, g = lm.wing_disc_timelapse.get_wing_disc_timelapse_paths( + path=os.path.join(p, "wing_disc_timelapse"), movies=[movie], download=download, + ) + for label_path in g[::16]: + out_path = os.path.join(out_dir, f"{movie}_{os.path.basename(label_path)}") + if not os.path.exists(out_path): + labels = imageio.imread(label_path) + ids, sizes = np.unique(labels[labels > 0], return_counts=True) + labels[np.isin(labels, ids[sizes > 25 * np.median(sizes)])] = 0 + imageio.imwrite(out_path, labels) + gt.append(out_path) + img += i[::16] + return img, gt, None, None if dataset_name == "cardioblast_nuclei": # Time-lapse projections of 350 frames; every 35th frame is scored. @@ -1547,6 +1797,20 @@ def _get_2d_lm_data_paths( img, gt = lm.arvidsson.get_arvidsson_paths(path=os.path.join(p, "arvidsson"), split=split, download=download) return (*_sorted_pairs(img, gt), None, None) + if dataset_name == "mif_tonsil": + # The ten regions hold seven channels, the last one DAPI. The nuclei are scored on DAPI alone, which is + # written once per region. None of the regions trains, so every split returns all of them. + img, gt = lm.mif_tonsil.get_mif_tonsil_paths(path=os.path.join(p, "mif_tonsil"), download=download) + out_dir = os.path.join(p, "mif_tonsil", "dapi") + os.makedirs(out_dir, exist_ok=True) + dapi_paths = [] + for path in img: + dapi_path = os.path.join(out_dir, os.path.basename(path)) + if not os.path.exists(dapi_path): + imageio.imwrite(dapi_path, imageio.imread(path)[-1]) + dapi_paths.append(dapi_path) + return (*_sorted_pairs(dapi_paths, gt), None, None) + if dataset_name == "mndino": # The nuclei are scored; the disjoint micronuclei labels are left out. paths = lm.mndino.get_mndino_paths(path=os.path.join(p, "mndino"), split=split, download=download) @@ -1734,6 +1998,23 @@ def _get_3d_lm_data_paths( paths = sorted(paths)[::10] return paths, paths, "raw", "labels" + if dataset_name == "morphonet_patiria": + # Out of domain, a single volume. The labels are the curated segmentation of MorphoNet, which resolves all + # cells; the published segmentation of the loader leaves many cells out. + root = os.path.join(p, "morphonet", "Patiria miniata") + raw = glob(os.path.join(root, "published", "RAW", "*.tif")) + labels = [os.path.join(root, "curated", "SEG", "SeaStar.tiff")] + return raw, labels, None, None + + if dataset_name == "nus_organoids": + # Out of domain, three volumes (cell monolayer, colon organoid, breast cancer spheroid). TM00099 is left + # out: its nuclei are dim and sparse, and all methods fail on it. The raw data of the original download for + # HCT116 is a copy of its labels, the raw data here is the correct one. + root = os.path.join(p, "beghin_data", "nus_organoids") + labels = [path for path in glob(os.path.join(root, "*_groundtruth.tif")) if "TM00099" not in path] + raw = [path.replace("_groundtruth.tif", ".tif") for path in labels] + return (*_sorted_pairs(raw, labels), None, None) + if dataset_name == "parhyale_regen": paths = lm.parhyale_regen.get_parhyale_regen_paths(path=os.path.join(p, "parhyale_regen"), download=download) return sorted(paths), sorted(paths), "raw", "labels" @@ -1742,6 +2023,12 @@ def _get_3d_lm_data_paths( img, gt = lm.vibrio_cholerae.get_vibrio_cholerae_paths( path=os.path.join(p, "vibrio_cholerae"), download=download, ) + if split == "test": + # The test data is the fully manual annotation, which a human drew from scratch on a crop of the first + # biofilm. The semi-manual labels of the full volumes leave many cells unlabeled. + folder = os.path.join(os.path.dirname(os.path.dirname(img[0])), "fully-manual-annotation") + img = [os.path.join(folder, "biofilm_1_cropped_raw.tif")] + gt = [os.path.join(folder, "biofilm_1_cropped_labels.tif")] return (*_sorted_pairs(img, gt), None, None) if dataset_name == "mouse_embryo": @@ -1752,6 +2039,29 @@ def _get_3d_lm_data_paths( ) return sorted(paths), sorted(paths), "raw", "label" + if dataset_name == "mouse_embryo_membrane": + # Out of domain: the official val split (one light-sheet volume) is scored, the train split is the tuning + # data. The labels are GASP segmentations, corrected by hand. The train file of timepoint 73 is a crop of + # the scored volume (z 30-160, y 280-704), so it is left out of the tuning data. + paths = lm.mouse_embryo.get_mouse_embryo_paths( + path=os.path.join(p, "mouse_embryo"), name="membrane", split="val" if split == "test" else "train", + download=download, + ) + if split != "test": + paths = [path for path in paths if not os.path.basename(path).startswith("fused_paral_tp00073")] + return sorted(paths), sorted(paths), "raw", "label" + + if dataset_name == "mucic_mda231": + # Out of domain, without an official split. The two CTC movies are split, so that no frame of a scored movie + # tunes: the 12 frames of movie 02 (44-54 cells each) are scored, the 12 frames of movie 01 (29-31 cells + # each) are the tuning data. + paths = lm.mucic.get_mucic_paths( + path=os.path.join(p, "mucic"), cell_line="mda231", variant="default", download=download, + ) + movie = "02" if split == "test" else "01" + paths = [path for path in paths if os.path.basename(path).startswith(f"mda231_{movie}_")] + return sorted(paths), sorted(paths), "raw", "labels/instances" + if dataset_name == "blastospim": paths = lm.blastospim.get_blastospim_paths(path=os.path.join(p, "blastospim"), download=download) volumes = BLASTOSPIM_VAL_VOLUMES if split == "val" else BLASTOSPIM_TEST_VOLUMES @@ -2065,6 +2375,25 @@ def _get_3d_em_data_paths( label_paths = [os.path.join(root, "Label", "train", name) for name in names] return raw_paths, label_paths, "main", "main" + if dataset_name in CONNECTOME_NUCLEUS_TUNING: + # One file per crop; the tuning crop is held out from the scored ones. + paths = [ + path for path in sorted(glob(os.path.join(p, dataset_name, "*.h5"))) + if (os.path.basename(path) == CONNECTOME_NUCLEUS_TUNING[dataset_name]) == is_val + ] + return paths, paths, "raw", "labels" + + if dataset_name == "placenta_nucleus": + # Both volumes are scored and also tune: the dataset has no other annotated volume. + paths = sorted(glob(os.path.join(p, PLACENTA_NUCLEUS_ROOT, "placenta*.h5"))) + return paths, paths, "raw", "labels/nuclei" + + if dataset_name in NUCLEUS_TEST_SET_ROOTS: + # The scored boxes or slabs are in 'test', the tuning ones in 'val'. Instance labels. + folder = os.path.join(p, NUCLEUS_TEST_SET_ROOTS[dataset_name], "val" if is_val else "test") + paths = sorted(glob(os.path.join(folder, "*.h5"))) + return paths, paths, "raw", "labels" + if dataset_name == "aive_nucleus": # Scored and tuned in the regions of EM_ROIS. One nucleus per volume. paths = [os.path.join(p, AIVE_NUCLEUS_ROOT, name) for name in AIVE_NUCLEUS_EVAL_VOLUMES] @@ -2096,6 +2425,16 @@ def _get_3d_em_data_paths( path = em.isbi2012.get_isbi_paths(path=os.path.join(p, "isbi2012"), download=download) return [path], [path], "raw", "labels/gt_segmentation" + if dataset_name == "cortex_human_stg": + # The 64 x 1024 x 1024 crop at the volume center, streamed with the torch-em loader. + paths = sorted(glob(os.path.join(p, "cortex_connectomics", "human_stg_*.zarr"))) + return paths, paths, "raw", "labels" + + if dataset_name in ("microns_basil", "ionsem"): + folder = os.path.join(p, "emneuron", "valid", "OutofDistribution", + "Microns[B]" if dataset_name == "microns_basil" else "Ionsem") + return [os.path.join(folder, "raw.tif")], [os.path.join(folder, "label.tif")], None, None + if dataset_name == "humanneurons": # The cached H01 crop. Resolved directly: it predates the torch-em loader for this dataset. paths = sorted(glob(os.path.join(p, "humanneurons", "*.h5"))) @@ -2187,8 +2526,8 @@ def load_volume( """Load a 3D volume, apply dataset-specific preprocessing, and center-crop. valid_roi is a boolean mask that is True where the data is annotated. It is None except for the datasets - that are annotated only in part (platynereis_cells, synapseweb, plantseg_root, plantseg_ovules, kasthuri) and - for donglai_wei_mito, whose blank raw strips are excluded. + that are annotated only in part (platynereis_cells, synapseweb, plantseg_root, plantseg_ovules, kasthuri, + mouse_embryo_membrane) and for donglai_wei_mito, whose blank raw strips are excluded. 'split' selects the test or the tuning region of the EM volumes, see EM_ROIS. 'z_range' restricts the volume to a z-slab before the center crop, which is how a dataset without splits holds tuning data out @@ -2233,7 +2572,13 @@ def load_volume( labels = labels.astype("int64") valid_roi = labels != -1 labels[labels == -1] = 0 - elif dataset_name == "pnas_arabidopsis": + elif dataset_name == "mouse_embryo_membrane": + # Id 65535 marks tissue that is not resolved into cells (only in the tuning volumes); it is excluded from + # scoring. + labels = labels.astype("int64") + valid_roi = labels != 65535 + labels[~valid_roi] = 0 + elif dataset_name in ("pnas_arabidopsis", "morphonet_patiria"): # The background carries id 1. labels[labels == 1] = 0 elif dataset_name == "hemibrain_mito": @@ -2363,6 +2708,7 @@ def export_joint_checkpoint( Returns: The paths to the interactive (SAM2) and the automatic (UniSAM2 decoder) weight files. """ + from micro_sam.v2.models.util import joint_unetr_state export_root = export_root or _joint_export_root() checkpoint_path = get_joint_checkpoint(model_type, checkpoint) source_checksum = source_checksum or checkpoint_checksum(checkpoint_path) @@ -2388,13 +2734,9 @@ def export_joint_checkpoint( VOLUME_SPEED_OPTIONS = {"offload_to_cpu": False} +# The z/xy voxel ratios of the EM volumes, from their acquisition parameters. The light microscopy volumes are not +# here: all methods get them as acquired, without a correction for their anisotropy. DATASET_SPACING: dict = { - # z/xy voxel ratios from published acquisition parameters - "embedseg_mouse_skull": (4, 1, 1), # z=1µm, xy=0.25µm - "embedseg_organoid": (6, 1, 1), # z=1µm, xy=0.1733µm - "embedseg_platy_nuclei": (5, 1, 1), # z=2.031µm, xy=0.406µm - "blastospim": (10, 1, 1), # SPIM: z≈2µm, xy≈0.208µm - "mouse_embryo": (4, 1, 1), # confocal: z≈1µm, xy≈0.22µm "densecell": (5, 1, 1), # SBF-SEM: 50 nm sections, 10 nm pixels "densecell_mito": (5, 1, 1), "cmito_l1": (50 / 5.12, 1, 1), # ssTEM: 50 nm sections, 5.12 nm pixels @@ -2416,6 +2758,12 @@ def export_joint_checkpoint( "nucmm_zebrafish": (0.48 / 0.51, 1, 1), # ssEM: about 0.5 um voxels "nucmm_mouse": (1, 1, 1), # micro-CT, 0.72 um isotropic "aive_nucleus": (30 / 26, 1, 1), # FIB-SEM, downscaled to 30 x 26 x 26 nm + "dauer_larva_nucleus": (70 / 40, 1, 1), # ssTEM: 70 nm sections, labels at 40 nm pixels + "human_organoids_nucleus": (1, 1, 1), # FIB-SEM, binned 2x, isotropic + "placenta_nucleus": (4, 1, 1), # array tomography: 80 nm sections, 16 or 24 nm pixels + "malecns_nuclei": (1, 1, 1), # FIB-SEM, 16 nm isotropic + "zheng_ca3_nuclei": (45 / 72, 1, 1), # ssTEM: 45 nm sections, 72 nm pixels + "acson_nuclei": (1, 1, 1), # SBEM, downsampled to 50 nm isotropic "mitoem_human": (30 / 8, 1, 1), # ATUM-SEM: 30 nm sections, 8 nm pixels "mitoem_rat": (30 / 8, 1, 1), # ATUM-SEM: 30 nm sections, 8 nm pixels **{name: (1, 1, 1) for name in ("mitoemv2_beta", "mitoemv2_jurkat", "mitoemv2_macro", "mitoemv2_podo")}, @@ -2430,8 +2778,6 @@ def export_joint_checkpoint( "donglai_wei_mito": (40 / 8, 1, 1), # ssEM, 8 x 8 x 40 nm "synapsenet_compartments": (1, 1, 1), # electron tomography, isotropic "nisb": (2.2, 1, 1), # synthetic: 20 nm sections, 9 nm pixels - "bbbc032": (5, 1, 1), # spinning disk confocal: z=0.5µm, xy=0.101µm - "bbbc050": (2.5, 1, 1), # CV1000 (test split): z=2.0µm, xy=0.8µm } @@ -2692,6 +3038,7 @@ def build_ais_model_from_checkpoint(joint_checkpoint_path, model_type="hvit_t", Returns: The UniSAM2 decoder in eval mode, loaded from exactly this checkpoint. """ + from micro_sam.v2.models.util import joint_unetr_state if cache_dir is None: cache_dir = "/tmp/micro_sam2_checkpoint_cache" os.makedirs(cache_dir, exist_ok=True) @@ -2725,6 +3072,7 @@ def build_apg_model_from_checkpoint(joint_checkpoint_path, model_type="hvit_t", The automatic prompt generator, built through the same library factory `build_apg_segmenter` uses. """ from micro_sam.v2.util import get_sam2_model + from micro_sam.v2.models.util import joint_unetr_state from micro_sam.v2.instance_segmentation import get_instance_segmentation_generator if cache_dir is None: @@ -2875,6 +3223,18 @@ def sample_row_path(save_path: str, sample_index: int) -> str: return os.path.join(f"{save_path[:-4]}_samples", f"sample_{sample_index:04d}.csv") +# The datasets that label only some of their nuclei (e.g. the central cell of a ROI). Their predicted objects that do +# not touch a labelled object are removed before scoring, so that unlabelled nuclei do not count as false positives. +# Splits and merges of the labelled objects still count (user, 2026-11-05). +LABELLED_ONLY_DATASETS = {"etchacell_nucleus", "placenta_nucleus"} + + +def keep_labelled_objects(seg, labels): + """Return the segmentation with only the objects that overlap the labelled foreground.""" + keep = np.setdiff1d(np.unique(seg[labels > 0]), [0]) + return np.where(np.isin(seg, keep), seg, 0).astype(seg.dtype) + + def evaluate_samples( segment_fn, dataset_name, data_root, save_path, desc, limit=None, crop_shape=None, sample_index=None, extra_columns=None, normalize=True, shard=None, @@ -2951,6 +3311,8 @@ def evaluate_samples( seg = segment_fn(raw) if valid_roi is not None: seg[~valid_roi] = 0 + if dataset_name in LABELLED_ONLY_DATASETS: + seg = keep_labelled_objects(seg, labels) if ndim == 2: # The ground truth has no severed objects either, so predicting one is not a false positive. seg = drop_severed_objects(seg, GT_MIN_SIZE_2D.get(dataset_name, 0)) @@ -3137,9 +3499,26 @@ def check_data_download(dataset_name: str, data_root: str, download: bool = True print(f"Data check passed for '{dataset_name}': {len(raw_paths)} sample(s).") -CROP_SHAPE_2D = (512, 512) +# The center crop that 2d samples are scored on. EVAL_CROP_SHAPE_2D overrides it for a crop-size experiment, as one +# side length or 'full' for the whole image; such runs belong in their own experiment folder. +_crop_override = os.environ.get("EVAL_CROP_SHAPE_2D") +CROP_SHAPE_2D = ( + (512, 512) if _crop_override is None else + (10 ** 9, 10 ** 9) if _crop_override == "full" else (int(_crop_override), int(_crop_override)) +) CROP_SHAPE_3D = (8, 512, 512) +# The integer factor that the cropped 2d samples of a dataset are upscaled by, the image linearly and the labels by +# repeating pixels, for every method. The wing disc timelapse cells are about 11 pixels wide, too small for the +# image encoder at the native size. EVAL_UPSCALE_2D overrides it for all datasets in an upscaling experiment; such +# runs belong in their own experiment folder. +UPSCALE_2D = {"wing_disc_timelapse": 2} +_upscale_override = os.environ.get("EVAL_UPSCALE_2D") + +# The 2d datasets scored on the whole stored image instead of the center crop, e.g. the TEM benchmark images (most of +# them 1024 x 1024, whose center crop drops half of the mitochondria). +FULL_IMAGE_2D = frozenset(DATASETS_2D_EM_MITO + DATASETS_2D_EM_NUCLEUS + list(GRID_TILE_SHAPE_2D) + ["hpa"]) + # Ground-truth size floor of 3d datasets whose instances come from semantic labels: the connected components of # a semantic mask include specks of a few voxels that are no objects. GT_MIN_SIZE_3D = { @@ -3150,16 +3529,28 @@ def check_data_download(dataset_name: str, data_root: str, download: bool = True **{name: 100 for name in DATASETS_3D_EM_NUCLEUS}, # The NucMM-Z nuclei are only about 50 voxels large. "nucmm_zebrafish": 10, + # The curated labels hold specks of a few voxels next to the cells, which have at least 5000 voxels. + "morphonet_patiria": 500, + # The nuclei have at least 500 voxels; some labels are specks of a few voxels. + "nus_organoids": 100, } -# The fixed crops per dataset: the crop shape and the crop starts per sample, in stored array coordinates. +# The fixed crops per dataset: the crop shape of the dataset and the (crop start, crop shape) pairs per sample, in +# stored array coordinates. A dataset with 'crop_shapes' gives every crop its own shape. def _read_crops(file_name): with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), file_name)) as f: - return { - name: (tuple(entry["crop_shape"]), {sample: tuple(map(tuple, v)) for sample, v in entry["starts"].items()}) - for name, entry in json.load(f)["datasets"].items() - } + crops = {} + for name, entry in json.load(f)["datasets"].items(): + shape, shapes = tuple(entry["crop_shape"]), entry.get("crop_shapes", {}) + crops[name] = (shape, { + sample: tuple( + (tuple(start), tuple(shapes[sample][i]) if sample in shapes else shape) + for i, start in enumerate(starts) + ) + for sample, starts in entry["starts"].items() + }) + return crops # The test crops replace the center crop, and also hold the rule that placed them and a note per dataset. @@ -3169,12 +3560,14 @@ def _read_crops(file_name): def crops_2d(dataset_name, raw_path, split): - """Return the (crop start, crop shape) pairs for one image: its fixed crops or (None, None) for the center crop.""" + """Return the (crop start, crop shape) pairs for one image: its fixed crops, or (None, None) for no fixed crop. + + An image without fixed crops is scored on the whole image (FULL_IMAGE_2D) or on the center crop. + """ crops = {"test": EVAL_CROPS, "val": TUNING_CROPS}.get(split, {}) if dataset_name not in crops: return [(None, None)] - shape, starts = crops[dataset_name] - return [(start, shape) for start in starts[os.path.basename(raw_path)]] + return list(crops[dataset_name][1][os.path.basename(raw_path)]) def eval_crops_3d(dataset_name, raw_path, split, crop_shape): @@ -3184,14 +3577,12 @@ def eval_crops_3d(dataset_name, raw_path, split, crop_shape): 'crop_shape', which the parameter search passes for every volume. """ if split == "val" and dataset_name in TUNING_CROPS: - shape, starts = TUNING_CROPS[dataset_name] - return [(start, shape) for start in starts[os.path.basename(raw_path)]] + return list(TUNING_CROPS[dataset_name][1][os.path.basename(raw_path)]) if split != "test" or dataset_name not in EVAL_CROPS: return [(None, crop_shape or CROP_SHAPE_3D)] if crop_shape is not None: raise ValueError(f"'{dataset_name}' has fixed crops in EVAL_CROPS. Do not pass a crop shape.") - shape, starts = EVAL_CROPS[dataset_name] - return [(start, shape) for start in starts[os.path.basename(raw_path)]] + return list(EVAL_CROPS[dataset_name][1][os.path.basename(raw_path)]) def ensure_8bit_range(raw): @@ -3255,8 +3646,8 @@ def interactive_run_tag(ndim=2, use_masks=True, mask_threshold=0.0, min_size=0, tag += f"_t{mask_threshold:g}" if min_size: tag += f"_min{min_size}" - # Only the 2d path chooses the click protocol, see 'CLICK_PROTOCOLS'. - if ndim == 2 and click_protocol != "random": + # The click protocol, see 'CLICK_PROTOCOLS'. + if click_protocol != "random": tag += f"_{click_protocol}" return tag @@ -3402,6 +3793,8 @@ def load_evaluation_sample_2d( image = ensure_8bit_range(image) if normalize else np.clip(image, 0, 255).astype("uint8") if crop_start is not None: roi = tuple(slice(start, start + size) for start, size in zip(crop_start, crop_shape)) + elif dataset_name in FULL_IMAGE_2D: + roi = tuple(slice(None) for _ in image.shape[:2]) else: roi = _center_crop_roi(image.shape[:2], CROP_SHAPE_2D) labels = read_2d(label_path, label_key)[roi] @@ -3411,7 +3804,14 @@ def load_evaluation_sample_2d( elif dataset_name == "lewy_mito": labels = _lewy_mito_instances(labels) gt = connected_components(labels).astype("uint32") - return image[roi], drop_severed_objects(gt, GT_MIN_SIZE_2D.get(dataset_name, 0)) + gt = drop_severed_objects(gt, GT_MIN_SIZE_2D.get(dataset_name, 0)) + image = image[roi] + factor = UPSCALE_2D.get(dataset_name, 1) if _upscale_override is None else int(_upscale_override) + if factor > 1: + from skimage.transform import rescale + image = rescale(image, factor, order=1, preserve_range=True, channel_axis=-1 if image.ndim == 3 else None) + gt = np.repeat(np.repeat(gt, factor, axis=0), factor, axis=1) + return image, gt def load_evaluation_sample_3d( @@ -3491,6 +3891,34 @@ def n_samples(dataset_name, data_root, split="test"): return len(raw_paths) +# The interactive evaluation scores at most this many crops of a 3d dataset. Every object of a crop is prompted +# one by one over all iterations, so the datasets with many crops would take most of the compute. The automatic +# evaluation scores all crops. +INTERACTIVE_MAX_CROPS_3D = 8 + + +def interactive_sample_ids_3d(dataset_name, data_root, limit=None): + """Return the ids of the 3d test samples that the interactive evaluation scores, in increasing order. + + A dataset with more crops than INTERACTIVE_MAX_CROPS_3D keeps that many, spread evenly over its crops. + + Args: + dataset_name: The dataset. + data_root: The root the data lives in. + limit: Only use the first samples, for a check. + + Returns: + The sample ids. + """ + total = n_samples(dataset_name, data_root) + if limit is not None: + total = min(total, limit) + if total <= INTERACTIVE_MAX_CROPS_3D: + return list(range(total)) + step = (total - 1) / (INTERACTIVE_MAX_CROPS_3D - 1) + return [round(i * step) for i in range(INTERACTIVE_MAX_CROPS_3D)] + + def has_val_split(dataset_name: str) -> bool: """Whether a dataset holds data out from the samples the evaluation scores. diff --git a/finetuning/v2/evaluation/eval_crops.json b/finetuning/v2/evaluation/eval_crops.json index f477a84ff..10dccc24d 100644 --- a/finetuning/v2/evaluation/eval_crops.json +++ b/finetuning/v2/evaluation/eval_crops.json @@ -1,5 +1,5 @@ { - "rule": "The crops are 32x512x512, capped by the test region. They tile the bounding box of the annotation without overlap and stay clear of tuning data. Crops with fewer than 20% of the median object count of the dataset are left out. 2d sets without an entry are scored on the 512x512 center crop. The note of a dataset gives its exceptions.", + "rule": "The crops are 32x512x512, capped by the test region. They tile the bounding box of the annotation without overlap and stay clear of tuning data. Crops with fewer than 20% of the median object count of the dataset are left out. 2d sets without an entry are scored on the whole image (FULL_IMAGE_2D in common.py) or on the 512x512 center crop. The note of a dataset gives its exceptions.", "datasets": { "gonuclear": { "crop_shape": [32, 512, 512], @@ -950,17 +950,12 @@ } }, "densecell_mito": { - "crop_shape": [24, 256, 256], - "note": "24 x 256 x 256 tiles of the test volume (24 x 800 x 800), for more crops than 512 tiles give. 7 crops; tiles whose labels span fewer than 8 slices are left out.", + "crop_shape": [24, 512, 512], + "note": "Mitochondria panel: the crops tile the annotation inside the test region (the tuning slab at the start of the stack is left out). Tiles thinner than 16 slices or 256 px in plane are dropped, and of the tiles that pass the object-count rule at most 16 per volume are kept, the ones with the most objects. The crops span all 24 slices of the blind val volume; the tuning data is z >= 35 of the train volume. 2 crops hold 2-13 mitochondria.", "starts": { "densecell_val.h5": [ - [0, 0, 0], - [0, 0, 256], - [0, 0, 512], - [0, 256, 0], - [0, 256, 256], - [0, 512, 256], - [0, 512, 512] + [0, 8, 105], + [0, 520, 105] ] } }, @@ -1653,18 +1648,14 @@ } }, "snemi_mito": { - "crop_shape": [15, 512, 512], - "note": "15 x 512 x 512 tiles of the test split (30 x 1024 x 1024): its z is split in two for more crops. 8 crops, holding 11-16 objects.", + "crop_shape": [32, 512, 512], + "note": "Mitochondria panel (in-domain): the test split of the SNEMI Mito stack (z 70-99); the train and valid splits are training data, the valid split tunes. Manual semantic labels, made instances by connected components. 4 of 4 tiles are kept (at most 16 per volume, the ones with the most objects), holding 16-19 mitochondria.", "starts": { "snemi_mito.h5": [ [70, 0, 0], [70, 0, 512], [70, 512, 0], - [70, 512, 512], - [85, 0, 0], - [85, 0, 512], - [85, 512, 0], - [85, 512, 512] + [70, 512, 512] ] } }, @@ -1912,33 +1903,4508 @@ ] } }, - "elmi_mito": { - "crop_shape": [512, 512], - "note": "512 x 512 tiles of the test images that hold a labeled mitochondrion; only 3-5 are labeled per image. 16 crops, holding 1-2 objects.", + "pnas_arabidopsis": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", "starts": { - "ELMI_01.tif": [ - [0, 0], - [512, 0], - [512, 512], - [512, 1024], - [1024, 512] + "0hrs_plant18_trim-acylYFP.h5": [ + [0, 52, 74], + [31, 52, 74], + [62, 52, 74], + [92, 52, 74] + ], + "12hrs_plant18_trim-acylYFP.h5": [ + [0, 36, 47], + [27, 36, 47], + [54, 36, 47], + [80, 36, 47], + [107, 36, 47] + ], + "16hrs_plant18_trim-acylYFP.h5": [ + [0, 31, 68], + [31, 31, 68], + [62, 31, 68], + [94, 31, 68] + ], + "20hrs_plant18_trim-acylYFP.h5": [ + [0, 37, 30], + [28, 37, 30], + [56, 37, 30], + [84, 37, 30], + [113, 37, 30], + [141, 37, 30] + ], + "24hrs_plant18_trim-acylYFP.h5": [ + [0, 11, 38], + [28, 11, 38], + [56, 11, 38], + [84, 11, 38], + [112, 11, 38], + [140, 11, 38] + ], + "28hrs_plant18_trim-acylYFP.h5": [ + [0, 10, 21], + [27, 10, 21], + [54, 10, 21], + [80, 10, 21], + [107, 10, 21], + [134, 10, 21] + ], + "32hrs_plant18_trim-acylYFP.h5": [ + [0, 8, 40], + [30, 8, 40], + [59, 8, 40], + [88, 8, 40], + [118, 8, 40] + ], + "36hrs_plant18_trim-acylYFP.h5": [ + [0, 18, 48], + [28, 18, 48], + [56, 18, 48], + [84, 18, 48], + [111, 18, 48] + ], + "44hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 28], + [29, 0, 28], + [59, 0, 28], + [88, 0, 28], + [117, 0, 28], + [146, 0, 28] + ], + "48hrs_plant18_trim-acylYFP.h5": [ + [0, 10, 39], + [31, 10, 39], + [62, 10, 39], + [93, 10, 39], + [124, 10, 39] + ], + "4hrs_plant18_trim-acylYFP.h5": [ + [0, 64, 78], + [27, 64, 78], + [54, 64, 78], + [81, 64, 78], + [108, 64, 78] + ], + "52hrs_plant18_trim-acylYFP.h5": [ + [0, 6, 51], + [30, 6, 51], + [61, 6, 51], + [92, 6, 51], + [122, 6, 51] + ], + "56hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 12], + [29, 0, 12], + [58, 0, 12], + [87, 0, 12], + [116, 0, 12], + [144, 0, 12], + [173, 0, 12] + ], + "60hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 43], + [32, 0, 43], + [63, 0, 43], + [94, 0, 43], + [126, 0, 43] + ], + "64hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 19], + [31, 0, 19], + [61, 0, 19], + [92, 0, 19], + [122, 0, 19], + [153, 0, 19] + ], + "68hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 12], + [31, 0, 12], + [63, 0, 12], + [94, 0, 12], + [126, 0, 12], + [157, 0, 12], + [188, 0, 12] + ], + "72hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 18], + [32, 0, 18], + [63, 0, 18], + [95, 0, 18], + [127, 0, 18], + [158, 0, 18] + ], + "76hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 8], + [31, 0, 8], + [62, 0, 8], + [93, 0, 8], + [124, 0, 8], + [154, 0, 8], + [185, 0, 8] + ], + "80hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 0], + [28, 0, 0], + [55, 0, 0], + [83, 0, 0], + [111, 0, 0] ], - "ELMI_02.tif": [ - [0, 0], - [0, 512], - [512, 1024] + "84hrs_plant18_trim-acylYFP.h5": [ + [0, 0, 0], + [28, 0, 0], + [57, 0, 0], + [85, 0, 0], + [114, 0, 0], + [142, 0, 0], + [170, 0, 0] + ], + "8hrs_plant18_trim-acylYFP.h5": [ + [0, 50, 23], + [27, 50, 23], + [55, 50, 23], + [82, 50, 23], + [109, 50, 23], + [137, 50, 23] + ] + }, + "crop_shapes": { + "0hrs_plant18_trim-acylYFP.h5": [ + [31, 356, 348], + [31, 356, 348], + [30, 356, 348], + [31, 356, 348] + ], + "12hrs_plant18_trim-acylYFP.h5": [ + [27, 417, 401], + [27, 417, 401], + [26, 417, 401], + [27, 417, 401], + [27, 417, 401] + ], + "16hrs_plant18_trim-acylYFP.h5": [ + [31, 425, 393], + [31, 425, 393], + [32, 425, 393], + [31, 425, 393] + ], + "20hrs_plant18_trim-acylYFP.h5": [ + [28, 443, 464], + [28, 443, 464], + [28, 443, 464], + [29, 443, 464], + [28, 443, 464], + [28, 443, 464] + ], + "24hrs_plant18_trim-acylYFP.h5": [ + [28, 462, 451], + [28, 462, 451], + [28, 462, 451], + [28, 462, 451], + [28, 462, 451], + [28, 462, 451] + ], + "28hrs_plant18_trim-acylYFP.h5": [ + [27, 476, 443], + [27, 476, 443], + [26, 476, 443], + [27, 476, 443], + [27, 476, 443], + [27, 476, 443] + ], + "32hrs_plant18_trim-acylYFP.h5": [ + [30, 492, 433], + [29, 492, 433], + [29, 492, 433], + [30, 492, 433], + [30, 492, 433] + ], + "36hrs_plant18_trim-acylYFP.h5": [ + [28, 478, 427], + [28, 478, 427], + [28, 478, 427], + [27, 478, 427], + [28, 478, 427] + ], + "44hrs_plant18_trim-acylYFP.h5": [ + [29, 511, 465], + [30, 511, 465], + [29, 511, 465], + [29, 511, 465], + [29, 511, 465], + [30, 511, 465] + ], + "48hrs_plant18_trim-acylYFP.h5": [ + [31, 499, 432], + [31, 499, 432], + [31, 499, 432], + [31, 499, 432], + [31, 499, 432] + ], + "4hrs_plant18_trim-acylYFP.h5": [ + [27, 401, 395], + [27, 401, 395], + [27, 401, 395], + [27, 401, 395], + [27, 401, 395] + ], + "52hrs_plant18_trim-acylYFP.h5": [ + [30, 497, 418], + [31, 497, 418], + [31, 497, 418], + [30, 497, 418], + [30, 497, 418] + ], + "56hrs_plant18_trim-acylYFP.h5": [ + [29, 482, 463], + [29, 482, 463], + [29, 482, 463], + [29, 482, 463], + [28, 482, 463], + [29, 482, 463], + [29, 482, 463] + ], + "60hrs_plant18_trim-acylYFP.h5": [ + [32, 492, 442], + [31, 492, 442], + [31, 492, 442], + [32, 492, 442], + [32, 492, 442] + ], + "64hrs_plant18_trim-acylYFP.h5": [ + [31, 486, 474], + [30, 486, 474], + [31, 486, 474], + [30, 486, 474], + [31, 486, 474], + [30, 486, 474] + ], + "68hrs_plant18_trim-acylYFP.h5": [ + [31, 495, 472], + [32, 495, 472], + [31, 495, 472], + [32, 495, 472], + [31, 495, 472], + [31, 495, 472], + [32, 495, 472] + ], + "72hrs_plant18_trim-acylYFP.h5": [ + [32, 502, 467], + [31, 502, 467], + [32, 502, 467], + [32, 502, 467], + [31, 502, 467], + [32, 502, 467] + ], + "76hrs_plant18_trim-acylYFP.h5": [ + [31, 499, 485], + [31, 499, 485], + [31, 499, 485], + [31, 499, 485], + [30, 499, 485], + [31, 499, 485], + [31, 499, 485] + ], + "80hrs_plant18_trim-acylYFP.h5": [ + [28, 498, 486], + [27, 498, 486], + [28, 498, 486], + [28, 498, 486], + [27, 498, 486] + ], + "84hrs_plant18_trim-acylYFP.h5": [ + [28, 477, 475], + [29, 477, 475], + [28, 477, 475], + [29, 477, 475], + [28, 477, 475], + [28, 477, 475], + [29, 477, 475] + ], + "8hrs_plant18_trim-acylYFP.h5": [ + [27, 415, 442], + [28, 415, 442], + [27, 415, 442], + [27, 415, 442], + [28, 415, 442], + [27, 415, 442] + ] + } + }, + "plantseg_ovules": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "N_294_final_crop_ds2.h5": [ + [2, 0, 14], + [2, 480, 14], + [34, 0, 14], + [34, 480, 14], + [66, 0, 14], + [66, 0, 490], + [66, 480, 14], + [97, 0, 14], + [97, 0, 490], + [97, 480, 14], + [97, 480, 490], + [129, 0, 14], + [129, 0, 490], + [129, 480, 14], + [129, 480, 490], + [161, 0, 14], + [161, 0, 490], + [161, 480, 14], + [161, 480, 490], + [193, 0, 14], + [193, 0, 490], + [193, 480, 14], + [193, 480, 490], + [225, 0, 14], + [225, 0, 490], + [225, 480, 14], + [225, 480, 490], + [256, 0, 14], + [256, 0, 490], + [256, 480, 14], + [256, 480, 490], + [288, 0, 14], + [288, 0, 490], + [288, 480, 14], + [288, 480, 490] + ], + "N_435_final_crop_ds2.h5": [ + [0, 721, 22], + [30, 360, 22], + [30, 721, 22], + [59, 360, 22], + [59, 721, 22], + [89, 0, 502], + [89, 360, 22], + [89, 721, 22], + [119, 0, 502], + [119, 360, 22], + [119, 360, 502], + [119, 721, 22], + [148, 0, 22], + [148, 0, 502], + [148, 360, 22], + [148, 360, 502], + [148, 721, 22], + [178, 0, 22], + [178, 0, 502], + [178, 360, 22], + [178, 360, 502], + [178, 721, 22], + [208, 0, 22], + [208, 0, 502], + [208, 360, 22], + [208, 360, 502], + [208, 721, 22], + [238, 0, 22], + [238, 0, 502], + [238, 360, 22], + [238, 360, 502], + [238, 721, 22], + [267, 0, 502], + [267, 360, 22], + [267, 721, 22], + [297, 0, 502], + [297, 360, 22], + [297, 721, 22], + [327, 360, 22], + [327, 721, 22], + [356, 360, 22], + [356, 721, 22] + ], + "N_441_final_crop_ds2.h5": [ + [2, 362, 426], + [2, 723, 426], + [34, 362, 0], + [34, 362, 426], + [34, 723, 426], + [65, 362, 0], + [65, 362, 426], + [65, 723, 426], + [97, 362, 0], + [97, 362, 426], + [97, 723, 426], + [128, 362, 0], + [128, 362, 426], + [128, 723, 426], + [160, 362, 0], + [160, 362, 426], + [160, 723, 426], + [191, 0, 426], + [191, 362, 0], + [191, 362, 426], + [191, 723, 426], + [223, 0, 0], + [223, 0, 426], + [223, 362, 0], + [223, 362, 426], + [223, 723, 426], + [255, 0, 0], + [255, 0, 426], + [255, 362, 0], + [255, 362, 426], + [255, 723, 426], + [286, 0, 0], + [286, 0, 426], + [286, 362, 0], + [286, 362, 426], + [286, 723, 426], + [318, 0, 0], + [318, 0, 426], + [318, 362, 0], + [318, 362, 426], + [349, 0, 0], + [349, 362, 0], + [381, 0, 0], + [412, 0, 0] + ], + "N_511_final_crop_ds2.h5": [ + [6, 0, 21], + [6, 0, 355], + [37, 0, 21], + [37, 0, 355], + [37, 396, 21], + [37, 396, 355], + [68, 0, 21], + [68, 0, 355], + [68, 396, 21], + [68, 396, 355], + [99, 0, 21], + [99, 0, 355], + [99, 396, 21], + [99, 396, 355], + [130, 0, 21], + [130, 0, 355], + [130, 396, 21], + [130, 396, 355], + [161, 0, 21], + [161, 0, 355], + [161, 396, 21], + [161, 396, 355], + [192, 0, 355], + [192, 396, 21], + [192, 396, 355], + [223, 0, 355], + [223, 396, 355] + ], + "N_522_final_crop_ds2.h5": [ + [11, 21, 13], + [11, 415, 13], + [43, 21, 13], + [43, 415, 13], + [74, 21, 13], + [74, 415, 13], + [74, 415, 474], + [106, 21, 13], + [106, 415, 13], + [106, 415, 474], + [138, 21, 13], + [138, 415, 13], + [138, 415, 474], + [170, 21, 13], + [170, 415, 13], + [170, 415, 474], + [201, 21, 13], + [201, 21, 474], + [201, 415, 13], + [201, 415, 474], + [233, 21, 13], + [233, 21, 474], + [233, 415, 13], + [233, 415, 474], + [265, 21, 13], + [265, 21, 474], + [265, 415, 13], + [265, 415, 474], + [297, 21, 474], + [297, 415, 13], + [297, 415, 474], + [328, 415, 474] + ], + "N_590_final_crop_ds2.h5": [ + [2, 17, 0], + [2, 17, 385], + [2, 286, 0], + [2, 286, 385], + [30, 17, 0], + [30, 17, 385], + [30, 286, 0], + [30, 286, 385], + [58, 17, 0], + [58, 17, 385], + [58, 286, 0], + [58, 286, 385], + [86, 17, 0], + [86, 17, 385], + [86, 286, 0], + [86, 286, 385], + [114, 17, 0], + [114, 17, 385], + [114, 286, 0], + [114, 286, 385], + [142, 17, 385], + [142, 286, 0], + [142, 286, 385] + ], + "N_593_final_crop_ds2.h5": [ + [0, 0, 0], + [0, 0, 401], + [0, 0, 802], + [28, 0, 0], + [28, 0, 401], + [28, 0, 802], + [56, 0, 0], + [56, 0, 401], + [56, 0, 802], + [84, 0, 0], + [84, 0, 401], + [84, 0, 802], + [112, 0, 0], + [112, 0, 401], + [112, 0, 802] + ] + }, + "crop_shapes": { + "N_294_final_crop_ds2.h5": [ + [32, 480, 476], + [32, 480, 476], + [32, 480, 476], + [32, 480, 476], + [31, 480, 476], + [31, 480, 477], + [31, 480, 476], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [31, 480, 476], + [31, 480, 477], + [31, 480, 476], + [31, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477], + [32, 480, 476], + [32, 480, 477] + ], + "N_435_final_crop_ds2.h5": [ + [30, 360, 480], + [29, 361, 480], + [29, 360, 480], + [30, 361, 480], + [30, 360, 480], + [30, 360, 480], + [30, 361, 480], + [30, 360, 480], + [29, 360, 480], + [29, 361, 480], + [29, 361, 480], + [29, 360, 480], + [30, 360, 480], + [30, 360, 480], + [30, 361, 480], + [30, 361, 480], + [30, 360, 480], + [30, 360, 480], + [30, 360, 480], + [30, 361, 480], + [30, 361, 480], + [30, 360, 480], + [30, 360, 480], + [30, 360, 480], + [30, 361, 480], + [30, 361, 480], + [30, 360, 480], + [29, 360, 480], + [29, 360, 480], + [29, 361, 480], + [29, 361, 480], + [29, 360, 480], + [30, 360, 480], + [30, 361, 480], + [30, 360, 480], + [30, 360, 480], + [30, 361, 480], + [30, 360, 480], + [29, 361, 480], + [29, 360, 480], + [30, 361, 480], + [30, 360, 480] + ], + "N_441_final_crop_ds2.h5": [ + [32, 361, 427], + [32, 362, 427], + [31, 361, 426], + [31, 361, 427], + [31, 362, 427], + [32, 361, 426], + [32, 361, 427], + [32, 362, 427], + [31, 361, 426], + [31, 361, 427], + [31, 362, 427], + [32, 361, 426], + [32, 361, 427], + [32, 362, 427], + [31, 361, 426], + [31, 361, 427], + [31, 362, 427], + [32, 362, 427], + [32, 361, 426], + [32, 361, 427], + [32, 362, 427], + [32, 362, 426], + [32, 362, 427], + [32, 361, 426], + [32, 361, 427], + [32, 362, 427], + [31, 362, 426], + [31, 362, 427], + [31, 361, 426], + [31, 361, 427], + [31, 362, 427], + [32, 362, 426], + [32, 362, 427], + [32, 361, 426], + [32, 361, 427], + [32, 362, 427], + [31, 362, 426], + [31, 362, 427], + [31, 361, 426], + [31, 361, 427], + [32, 362, 426], + [32, 361, 426], + [31, 362, 426], + [32, 362, 426] + ], + "N_511_final_crop_ds2.h5": [ + [31, 396, 334], + [31, 396, 333], + [31, 396, 334], + [31, 396, 333], + [31, 395, 334], + [31, 395, 333], + [31, 396, 334], + [31, 396, 333], + [31, 395, 334], + [31, 395, 333], + [31, 396, 334], + [31, 396, 333], + [31, 395, 334], + [31, 395, 333], + [31, 396, 334], + [31, 396, 333], + [31, 395, 334], + [31, 395, 333], + [31, 396, 334], + [31, 396, 333], + [31, 395, 334], + [31, 395, 333], + [31, 396, 333], + [31, 395, 334], + [31, 395, 333], + [31, 396, 333], + [31, 395, 333] + ], + "N_522_final_crop_ds2.h5": [ + [32, 394, 461], + [32, 395, 461], + [31, 394, 461], + [31, 395, 461], + [32, 394, 461], + [32, 395, 461], + [32, 395, 461], + [32, 394, 461], + [32, 395, 461], + [32, 395, 461], + [32, 394, 461], + [32, 395, 461], + [32, 395, 461], + [31, 394, 461], + [31, 395, 461], + [31, 395, 461], + [32, 394, 461], + [32, 394, 461], + [32, 395, 461], + [32, 395, 461], + [32, 394, 461], + [32, 394, 461], + [32, 395, 461], + [32, 395, 461], + [32, 394, 461], + [32, 394, 461], + [32, 395, 461], + [32, 395, 461], + [31, 394, 461], + [31, 395, 461], + [31, 395, 461], + [32, 395, 461] + ], + "N_590_final_crop_ds2.h5": [ + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385], + [28, 269, 385] + ], + "N_593_final_crop_ds2.h5": [ + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401], + [28, 480, 401] + ] + } + }, + "cshaper": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "Sample02_024.h5": [ + [29, 43, 10], + [60, 43, 10], + [91, 43, 10], + [122, 43, 10], + [153, 43, 10] + ], + "Sample02_034.h5": [ + [28, 48, 12], + [60, 48, 12], + [91, 48, 12], + [123, 48, 12], + [154, 48, 12] + ], + "Sample02_044.h5": [ + [28, 42, 11], + [60, 42, 11], + [91, 42, 11], + [123, 42, 11], + [154, 42, 11] + ], + "Sample02_054.h5": [ + [27, 41, 14], + [59, 41, 14], + [91, 41, 14], + [123, 41, 14], + [155, 41, 14] + ], + "Sample02_064.h5": [ + [27, 45, 12], + [54, 45, 12], + [81, 45, 12], + [107, 45, 12], + [134, 45, 12], + [161, 45, 12] + ], + "Sample02_074.h5": [ + [27, 41, 12], + [59, 41, 12], + [91, 41, 12], + [123, 41, 12], + [155, 41, 12] + ], + "Sample02_084.h5": [ + [27, 42, 13], + [59, 42, 13], + [91, 42, 13], + [123, 42, 13], + [155, 42, 13] + ], + "Sample03_024.h5": [ + [67, 26, 11], + [96, 26, 11], + [125, 26, 11], + [154, 26, 11] + ], + "Sample03_034.h5": [ + [39, 26, 13], + [68, 26, 13], + [97, 26, 13], + [125, 26, 13], + [154, 26, 13] + ], + "Sample03_044.h5": [ + [39, 24, 11], + [68, 24, 11], + [97, 24, 11], + [126, 24, 11], + [155, 24, 11] + ], + "Sample03_054.h5": [ + [40, 24, 16], + [69, 24, 16], + [98, 24, 16], + [127, 24, 16], + [156, 24, 16] + ], + "Sample03_064.h5": [ + [39, 25, 14], + [68, 25, 14], + [97, 25, 14], + [127, 25, 14], + [156, 25, 14] + ], + "Sample03_074.h5": [ + [39, 26, 19], + [68, 26, 19], + [97, 26, 19], + [127, 26, 19], + [156, 26, 19] + ], + "Sample03_084.h5": [ + [39, 27, 17], + [68, 27, 17], + [98, 27, 17], + [127, 27, 17], + [157, 27, 17] + ], + "Sample04_024.h5": [ + [20, 26, 11], + [52, 26, 11], + [83, 26, 11], + [115, 26, 11], + [146, 26, 11] + ], + "Sample04_034.h5": [ + [21, 25, 16], + [52, 25, 16], + [83, 25, 16], + [114, 25, 16], + [145, 25, 16] + ], + "Sample04_044.h5": [ + [22, 25, 18], + [54, 25, 18], + [85, 25, 18], + [117, 25, 18], + [148, 25, 18] + ], + "Sample04_054.h5": [ + [23, 24, 18], + [55, 24, 18], + [86, 24, 18], + [118, 24, 18], + [149, 24, 18] + ], + "Sample04_064.h5": [ + [25, 25, 15], + [56, 25, 15], + [87, 25, 15], + [119, 25, 15], + [150, 25, 15] + ], + "Sample04_074.h5": [ + [24, 25, 14], + [56, 25, 14], + [87, 25, 14], + [119, 25, 14], + [150, 25, 14] + ], + "Sample04_084.h5": [ + [26, 23, 10], + [57, 23, 10], + [88, 23, 10], + [120, 23, 10], + [151, 23, 10] + ] + }, + "crop_shapes": { + "Sample02_024.h5": [ + [31, 218, 106], + [31, 218, 106], + [31, 218, 106], + [31, 218, 106], + [31, 218, 106] + ], + "Sample02_034.h5": [ + [32, 214, 104], + [31, 214, 104], + [32, 214, 104], + [31, 214, 104], + [32, 214, 104] + ], + "Sample02_044.h5": [ + [32, 217, 105], + [31, 217, 105], + [32, 217, 105], + [31, 217, 105], + [32, 217, 105] + ], + "Sample02_054.h5": [ + [32, 210, 105], + [32, 210, 105], + [32, 210, 105], + [32, 210, 105], + [32, 210, 105] + ], + "Sample02_064.h5": [ + [27, 208, 106], + [27, 208, 106], + [26, 208, 106], + [27, 208, 106], + [27, 208, 106], + [27, 208, 106] + ], + "Sample02_074.h5": [ + [32, 216, 106], + [32, 216, 106], + [32, 216, 106], + [32, 216, 106], + [32, 216, 106] + ], + "Sample02_084.h5": [ + [32, 216, 103], + [32, 216, 103], + [32, 216, 103], + [32, 216, 103], + [32, 216, 103] + ], + "Sample03_024.h5": [ + [29, 229, 95], + [29, 229, 95], + [29, 229, 95], + [29, 229, 95] + ], + "Sample03_034.h5": [ + [29, 231, 94], + [29, 231, 94], + [28, 231, 94], + [29, 231, 94], + [29, 231, 94] + ], + "Sample03_044.h5": [ + [29, 232, 96], + [29, 232, 96], + [29, 232, 96], + [29, 232, 96], + [29, 232, 96] + ], + "Sample03_054.h5": [ + [29, 232, 94], + [29, 232, 94], + [29, 232, 94], + [29, 232, 94], + [29, 232, 94] + ], + "Sample03_064.h5": [ + [29, 231, 95], + [29, 231, 95], + [30, 231, 95], + [29, 231, 95], + [29, 231, 95] + ], + "Sample03_074.h5": [ + [29, 232, 92], + [29, 232, 92], + [30, 232, 92], + [29, 232, 92], + [29, 232, 92] + ], + "Sample03_084.h5": [ + [29, 233, 91], + [30, 233, 91], + [29, 233, 91], + [30, 233, 91], + [29, 233, 91] + ], + "Sample04_024.h5": [ + [32, 232, 87], + [31, 232, 87], + [32, 232, 87], + [31, 232, 87], + [32, 232, 87] + ], + "Sample04_034.h5": [ + [31, 232, 86], + [31, 232, 86], + [31, 232, 86], + [31, 232, 86], + [31, 232, 86] + ], + "Sample04_044.h5": [ + [32, 231, 86], + [31, 231, 86], + [32, 231, 86], + [31, 231, 86], + [32, 231, 86] + ], + "Sample04_054.h5": [ + [32, 229, 87], + [31, 229, 87], + [32, 229, 87], + [31, 229, 87], + [32, 229, 87] + ], + "Sample04_064.h5": [ + [31, 231, 86], + [31, 231, 86], + [32, 231, 86], + [31, 231, 86], + [31, 231, 86] + ], + "Sample04_074.h5": [ + [32, 231, 84], + [31, 231, 84], + [32, 231, 84], + [31, 231, 84], + [32, 231, 84] + ], + "Sample04_084.h5": [ + [31, 233, 87], + [31, 233, 87], + [32, 233, 87], + [31, 233, 87], + [31, 233, 87] + ] + } + }, + "vibrio_cholerae": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel. Scored on the fully manual annotation (biofilm_1_cropped, 129 x 257 x 257, drawn from scratch; the dataset's own evaluation set) since 2026-10-29, in place of the semi-manual labels of the five full volumes, which leave many cells unlabeled. The annotation is split along z into the fewest equal tiles of at most 32 slices; all five crops hold at least 20% of the median object count.", + "starts": { + "biofilm_1_cropped_raw.tif": [ + [0, 0, 0], + [26, 0, 0], + [52, 0, 0], + [78, 0, 0], + [104, 0, 0] + ] + }, + "crop_shapes": { + "biofilm_1_cropped_raw.tif": [ + [26, 257, 257], + [26, 257, 257], + [26, 257, 257], + [26, 257, 257], + [25, 257, 257] + ] + } + }, + "parhyale_regen": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "Parhyale_H2B-EGFP_1.h5": [ + [3, 8, 46] ], - "ELMI_03.tif": [ - [0, 1024], - [512, 1024], - [1024, 1024], - [1536, 1024] + "Parhyale_H2B-EGFP_11.h5": [ + [1, 1, 37] ], - "ELMI_04.tif": [ - [512, 512], - [512, 1024], - [1024, 0], - [1024, 1024] + "Parhyale_H2B-EGFP_21.h5": [ + [1, 17, 40] + ], + "Parhyale_H2B-EGFP_31.h5": [ + [0, 35, 36] + ], + "Parhyale_H2B-EGFP_41.h5": [ + [0, 30, 40] + ], + "Parhyale_H2B-EGFP_50.h5": [ + [0, 4, 31] + ] + }, + "crop_shapes": { + "Parhyale_H2B-EGFP_1.h5": [ + [22, 485, 466] + ], + "Parhyale_H2B-EGFP_11.h5": [ + [28, 476, 475] + ], + "Parhyale_H2B-EGFP_21.h5": [ + [28, 492, 472] + ], + "Parhyale_H2B-EGFP_31.h5": [ + [29, 467, 476] + ], + "Parhyale_H2B-EGFP_41.h5": [ + [28, 456, 472] + ], + "Parhyale_H2B-EGFP_50.h5": [ + [29, 482, 481] + ] + } + }, + "mouse_embryo": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "E15_Ab1.h5": [ + [14, 12, 34], + [14, 347, 34], + [44, 12, 34], + [44, 347, 34], + [74, 12, 34], + [74, 347, 34] + ], + "E15_Ab2.h5": [ + [17, 0, 9], + [17, 328, 9], + [40, 0, 9], + [40, 328, 9], + [64, 0, 9], + [64, 328, 9] + ], + "E16_Ab2.h5": [ + [24, 476, 195], + [24, 802, 195], + [50, 476, 195], + [50, 802, 195], + [77, 476, 195], + [77, 802, 195] + ], + "E17_Ab1.h5": [ + [0, 0, 40], + [0, 407, 40], + [27, 0, 40], + [27, 407, 40], + [53, 0, 40], + [53, 407, 40] + ], + "E18_Ab1.h5": [ + [19, 73, 55], + [19, 395, 55], + [45, 73, 55], + [45, 395, 55], + [72, 73, 55], + [72, 395, 55] + ], + "E18_Ab2.h5": [ + [30, 172, 67], + [30, 487, 67], + [62, 172, 67], + [62, 487, 67] + ], + "E19_Ab2.h5": [ + [21, 403, 61], + [51, 403, 61] + ], + "E20_Ab1.h5": [ + [18, 367, 17], + [47, 17, 17], + [47, 367, 17], + [75, 17, 17], + [75, 367, 17] + ], + "E20_Ab2.h5": [ + [16, 8, 15], + [16, 303, 15], + [40, 8, 15], + [40, 303, 15], + [63, 8, 15], + [63, 303, 15] + ], + "E21_Ab1.h5": [ + [8, 350, 37], + [8, 350, 307], + [8, 700, 37], + [8, 700, 307], + [35, 0, 37], + [35, 0, 307], + [35, 350, 37], + [35, 350, 307], + [35, 700, 37], + [35, 700, 307], + [62, 0, 37], + [62, 0, 307], + [62, 350, 37], + [62, 350, 307], + [62, 700, 37], + [62, 700, 307], + [88, 0, 37], + [88, 0, 307], + [88, 350, 37], + [88, 350, 307], + [88, 700, 37], + [88, 700, 307] + ], + "E21_Ab2.h5": [ + [24, 524, 126], + [49, 524, 126], + [75, 524, 126] + ], + "E7_Ab1.h5": [ + [0, 13, 5], + [0, 370, 5], + [28, 13, 5], + [28, 370, 5], + [57, 13, 5], + [57, 370, 5] + ], + "E9_Ab3.h5": [ + [36, 125, 61], + [64, 125, 61], + [91, 125, 61] + ] + }, + "crop_shapes": { + "E15_Ab1.h5": [ + [30, 335, 444], + [30, 335, 444], + [30, 335, 444], + [30, 335, 444], + [30, 335, 444], + [30, 335, 444] + ], + "E15_Ab2.h5": [ + [23, 328, 459], + [23, 329, 459], + [24, 328, 459], + [24, 329, 459], + [23, 328, 459], + [23, 329, 459] + ], + "E16_Ab2.h5": [ + [26, 326, 449], + [26, 325, 449], + [27, 326, 449], + [27, 325, 449], + [26, 326, 449], + [26, 325, 449] + ], + "E17_Ab1.h5": [ + [27, 407, 332], + [27, 407, 332], + [26, 407, 332], + [26, 407, 332], + [27, 407, 332], + [27, 407, 332] + ], + "E18_Ab1.h5": [ + [26, 322, 325], + [26, 323, 325], + [27, 322, 325], + [27, 323, 325], + [26, 322, 325], + [26, 323, 325] + ], + "E18_Ab2.h5": [ + [32, 315, 265], + [32, 315, 265], + [31, 315, 265], + [31, 315, 265] + ], + "E19_Ab2.h5": [ + [30, 352, 277], + [31, 352, 277] + ], + "E20_Ab1.h5": [ + [29, 350, 400], + [28, 350, 400], + [28, 350, 400], + [29, 350, 400], + [29, 350, 400] + ], + "E20_Ab2.h5": [ + [24, 295, 333], + [24, 295, 333], + [23, 295, 333], + [23, 295, 333], + [24, 295, 333], + [24, 295, 333] + ], + "E21_Ab1.h5": [ + [27, 350, 270], + [27, 350, 271], + [27, 350, 270], + [27, 350, 271], + [27, 350, 270], + [27, 350, 271], + [27, 350, 270], + [27, 350, 271], + [27, 350, 270], + [27, 350, 271], + [26, 350, 270], + [26, 350, 271], + [26, 350, 270], + [26, 350, 271], + [26, 350, 270], + [26, 350, 271], + [27, 350, 270], + [27, 350, 271], + [27, 350, 270], + [27, 350, 271], + [27, 350, 270], + [27, 350, 271] + ], + "E21_Ab2.h5": [ + [25, 435, 369], + [26, 435, 369], + [25, 435, 369] + ], + "E7_Ab1.h5": [ + [28, 357, 354], + [28, 357, 354], + [29, 357, 354], + [29, 357, 354], + [28, 357, 354], + [28, 357, 354] + ], + "E9_Ab3.h5": [ + [28, 326, 406], + [27, 326, 406], + [28, 326, 406] + ] + } + }, + "blastospim": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "Blast_074_image_0001.h5": [ + [12, 268, 854], + [12, 268, 1208], + [12, 612, 854], + [12, 612, 1208], + [35, 268, 854], + [35, 268, 1208], + [35, 612, 854], + [35, 612, 1208], + [57, 268, 854], + [57, 268, 1208], + [57, 612, 854], + [57, 612, 1208] + ], + "Blast_075_image_0001.h5": [ + [20, 340, 888], + [20, 340, 1182], + [20, 648, 888], + [20, 648, 1182], + [50, 340, 888], + [50, 340, 1182], + [50, 648, 888], + [50, 648, 1182] + ], + "F11_070_image_0001.h5": [ + [16, 394, 235] + ], + "F24_002_image_0001.h5": [ + [11, 343, 191] + ], + "F24_010_image_0001.h5": [ + [10, 337, 181], + [28, 337, 181] + ], + "F25_008_image_0001.h5": [ + [13, 395, 275] + ], + "F27_009_image_0001.h5": [ + [11, 303, 173] + ], + "F27_010_image_0001.h5": [ + [11, 311, 168] + ], + "F29_003_image_0001.h5": [ + [14, 636, 392] + ], + "F29_004_image_0001.h5": [ + [14, 637, 391] + ], + "F2_012_image_0001.h5": [ + [5, 340, 170], + [5, 596, 170], + [35, 340, 170], + [35, 596, 170] + ], + "F30_004_image_0001.h5": [ + [11, 368, 337] + ], + "F30_008_image_0001.h5": [ + [12, 385, 330] + ], + "F30_009_image_0001.h5": [ + [13, 381, 329] + ], + "F33_067_image_0001.h5": [ + [11, 438, 397], + [27, 438, 397] + ], + "F34_073_image_0001.h5": [ + [15, 473, 395], + [31, 473, 395] + ], + "F39_117_image_0001.h5": [ + [11, 392, 357], + [31, 392, 357] + ], + "F40_136_image_0001.h5": [ + [11, 396, 327], + [32, 396, 327] + ], + "F44_087_image_0001.h5": [ + [9, 403, 336], + [25, 403, 336] + ], + "F49_148_image_0001.h5": [ + [5, 385, 256], + [29, 385, 256] + ], + "F9_071_image_0001.h5": [ + [6, 344, 185], + [24, 344, 185] + ], + "H1_006_image_0001.h5": [ + [6, 459, 152] + ], + "H2_016_image_0001.h5": [ + [4, 451, 181] + ], + "H3_002_image_0001.h5": [ + [3, 417, 139] + ], + "H4_012_image_0001.h5": [ + [5, 400, 126] + ], + "H5_007_image_0001.h5": [ + [3, 480, 158], + [21, 480, 158] + ], + "H7_004_image_0001.h5": [ + [20, 252, 168] + ], + "H7_008_image_0001.h5": [ + [20, 255, 171] + ], + "H8_016_image_0001.h5": [ + [20, 293, 194] + ], + "H8_021_image_0001.h5": [ + [18, 202, 220] + ], + "H9_008_image_0001.h5": [ + [14, 317, 290] + ], + "M10_015_image_0001.h5": [ + [0, 494, 236] + ], + "M14_020_image_0001.h5": [ + [6, 234, 346], + [22, 234, 346] + ], + "M3_008_image_0001.h5": [ + [14, 295, 132] + ], + "M4_012_image_0001.h5": [ + [14, 302, 134] + ], + "M6_021_image_0001.h5": [ + [13, 541, 177] + ], + "M7_000_image_0001.h5": [ + [2, 468, 132], + [18, 468, 132] + ], + "M7_007_image_0001.h5": [ + [1, 522, 137] + ], + "M8_015_image_0001.h5": [ + [1, 536, 215] + ], + "M8_016_image_0001.h5": [ + [0, 538, 243] + ] + }, + "crop_shapes": { + "Blast_074_image_0001.h5": [ + [23, 344, 354], + [23, 344, 354], + [23, 344, 354], + [23, 344, 354], + [22, 344, 354], + [22, 344, 354], + [22, 344, 354], + [22, 344, 354], + [23, 344, 354], + [23, 344, 354], + [23, 344, 354], + [23, 344, 354] + ], + "Blast_075_image_0001.h5": [ + [30, 308, 294], + [30, 308, 295], + [30, 307, 294], + [30, 307, 295], + [29, 308, 294], + [29, 308, 295], + [29, 307, 294], + [29, 307, 295] + ], + "F11_070_image_0001.h5": [ + [11, 319, 314] + ], + "F24_002_image_0001.h5": [ + [30, 388, 384] + ], + "F24_010_image_0001.h5": [ + [18, 389, 392], + [18, 389, 392] + ], + "F25_008_image_0001.h5": [ + [26, 239, 273] + ], + "F27_009_image_0001.h5": [ + [27, 279, 321] + ], + "F27_010_image_0001.h5": [ + [27, 267, 323] + ], + "F29_003_image_0001.h5": [ + [30, 316, 318] + ], + "F29_004_image_0001.h5": [ + [29, 321, 317] + ], + "F2_012_image_0001.h5": [ + [30, 256, 461], + [30, 257, 461], + [29, 256, 461], + [29, 257, 461] + ], + "F30_004_image_0001.h5": [ + [28, 247, 302] + ], + "F30_008_image_0001.h5": [ + [26, 248, 302] + ], + "F30_009_image_0001.h5": [ + [25, 246, 298] + ], + "F33_067_image_0001.h5": [ + [16, 334, 339], + [17, 334, 339] + ], + "F34_073_image_0001.h5": [ + [16, 323, 382], + [17, 323, 382] + ], + "F39_117_image_0001.h5": [ + [20, 425, 414], + [20, 425, 414] + ], + "F40_136_image_0001.h5": [ + [21, 433, 440], + [21, 433, 440] + ], + "F44_087_image_0001.h5": [ + [16, 359, 306], + [17, 359, 306] + ], + "F49_148_image_0001.h5": [ + [24, 496, 478], + [25, 496, 478] + ], + "F9_071_image_0001.h5": [ + [18, 314, 399], + [18, 314, 399] + ], + "H1_006_image_0001.h5": [ + [26, 314, 323] + ], + "H2_016_image_0001.h5": [ + [29, 299, 349] + ], + "H3_002_image_0001.h5": [ + [32, 310, 348] + ], + "H4_012_image_0001.h5": [ + [30, 355, 336] + ], + "H5_007_image_0001.h5": [ + [18, 356, 323], + [19, 356, 323] + ], + "H7_004_image_0001.h5": [ + [25, 329, 370] + ], + "H7_008_image_0001.h5": [ + [25, 321, 369] + ], + "H8_016_image_0001.h5": [ + [27, 262, 334] + ], + "H8_021_image_0001.h5": [ + [28, 390, 346] + ], + "H9_008_image_0001.h5": [ + [27, 354, 348] + ], + "M10_015_image_0001.h5": [ + [31, 337, 327] + ], + "M14_020_image_0001.h5": [ + [16, 366, 294], + [17, 366, 294] + ], + "M3_008_image_0001.h5": [ + [26, 244, 302] + ], + "M4_012_image_0001.h5": [ + [27, 242, 295] + ], + "M6_021_image_0001.h5": [ + [29, 205, 326] + ], + "M7_000_image_0001.h5": [ + [16, 263, 344], + [17, 263, 344] + ], + "M7_007_image_0001.h5": [ + [31, 306, 366] + ], + "M8_015_image_0001.h5": [ + [29, 365, 342] + ], + "M8_016_image_0001.h5": [ + [31, 344, 271] + ] + } + }, + "morphonet_celegans": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files. The volumes t0000.h5, t0010.h5, t0020.h5, t0030.h5 hold too few objects and have no crop.", + "starts": { + "t0000.h5": [], + "t0010.h5": [], + "t0020.h5": [], + "t0030.h5": [], + "t0040.h5": [ + [5, 83, 96] + ], + "t0050.h5": [ + [4, 78, 97] + ], + "t0060.h5": [ + [4, 78, 91] + ], + "t0070.h5": [ + [4, 83, 96] + ], + "t0080.h5": [ + [4, 73, 89], + [4, 73, 355] + ], + "t0090.h5": [ + [4, 84, 94], + [4, 84, 359] + ], + "t0100.h5": [ + [4, 77, 102] + ], + "t0110.h5": [ + [4, 72, 97], + [4, 72, 366] + ], + "t0120.h5": [ + [4, 80, 93], + [4, 80, 361] + ], + "t0130.h5": [ + [4, 81, 93], + [4, 81, 361] + ], + "t0140.h5": [ + [4, 77, 94], + [4, 77, 362] + ], + "t0150.h5": [ + [4, 79, 90], + [4, 79, 355] + ], + "t0160.h5": [ + [4, 73, 92], + [4, 73, 359] + ], + "t0170.h5": [ + [4, 75, 89], + [4, 75, 357] + ], + "t0180.h5": [ + [4, 75, 95], + [4, 75, 361] + ] + }, + "crop_shapes": { + "t0000.h5": [], + "t0010.h5": [], + "t0020.h5": [], + "t0030.h5": [], + "t0040.h5": [ + [25, 339, 506] + ], + "t0050.h5": [ + [26, 363, 510] + ], + "t0060.h5": [ + [26, 369, 511] + ], + "t0070.h5": [ + [27, 356, 509] + ], + "t0080.h5": [ + [27, 376, 266], + [27, 376, 267] + ], + "t0090.h5": [ + [27, 368, 265], + [27, 368, 265] + ], + "t0100.h5": [ + [27, 371, 508] + ], + "t0110.h5": [ + [27, 378, 269], + [27, 378, 269] + ], + "t0120.h5": [ + [27, 369, 268], + [27, 369, 268] + ], + "t0130.h5": [ + [27, 367, 268], + [27, 367, 267] + ], + "t0140.h5": [ + [28, 376, 268], + [28, 376, 268] + ], + "t0150.h5": [ + [28, 374, 265], + [28, 374, 265] + ], + "t0160.h5": [ + [28, 380, 267], + [28, 380, 267] + ], + "t0170.h5": [ + [27, 378, 268], + [27, 378, 269] + ], + "t0180.h5": [ + [27, 377, 266], + [27, 377, 265] + ] + } + }, + "bbbc032": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files. The tuning slab z < 20 of LM_VAL_Z_SLABS is left out. The crops starting at [50, 696, 150] and [142, 696, 519] are left out by hand (2026-11-01), since nuclei in them are not labelled.", + "starts": { + "BMP4blastocystC3.tif": [ + [20, 316, 150], + [20, 316, 519], + [20, 696, 150], + [20, 696, 519], + [50, 316, 150], + [50, 316, 519], + [50, 696, 519], + [81, 316, 150], + [81, 316, 519], + [81, 696, 150], + [81, 696, 519], + [111, 316, 150], + [111, 316, 519], + [111, 696, 150], + [111, 696, 519], + [142, 316, 150], + [142, 316, 519], + [142, 696, 150] + ] + }, + "crop_shapes": { + "BMP4blastocystC3.tif": [ + [30, 380, 369], + [30, 380, 369], + [30, 380, 369], + [30, 380, 369], + [31, 380, 369], + [31, 380, 369], + [31, 380, 369], + [30, 380, 369], + [30, 380, 369], + [30, 380, 369], + [30, 380, 369], + [31, 380, 369], + [31, 380, 369], + [31, 380, 369], + [31, 380, 369], + [30, 380, 369], + [30, 380, 369], + [30, 380, 369] + ] + } + }, + "bbbc033": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "BBBC033.h5": [ + [11, 228, 143], + [11, 228, 445], + [11, 577, 143], + [11, 577, 445] + ] + }, + "crop_shapes": { + "BBBC033.h5": [ + [21, 349, 302], + [21, 349, 301], + [21, 349, 302], + [21, 349, 301] + ] + } + }, + "bbbc050": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-27 in place of the 8-slice center crop: each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out. The tuning data are separate files.", + "starts": { + "Emb1_t001.tif": [ + [8, 38, 43] + ], + "Emb1_t051.tif": [ + [7, 44, 27], + [24, 44, 27] + ], + "Emb1_t101.tif": [ + [9, 39, 30] + ], + "Emb1_t151.tif": [ + [10, 28, 31] + ], + "Emb1_t201.tif": [ + [5, 27, 26], + [25, 27, 26] + ], + "Emb1_t251.tif": [ + [6, 30, 27], + [26, 30, 27] + ], + "Emb1_t301.tif": [ + [0, 14, 25], + [24, 14, 25] + ], + "Emb1_t351.tif": [ + [9, 25, 11], + [27, 25, 11] + ], + "Emb1_t401.tif": [ + [3, 15, 19], + [25, 15, 19] + ], + "Emb1_t451.tif": [ + [3, 14, 13], + [24, 14, 13] + ], + "Emb1_t501.tif": [ + [4, 18, 15], + [25, 18, 15] + ], + "Emb2_t001.tif": [ + [11, 31, 45] + ], + "Emb2_t051.tif": [ + [10, 30, 47] + ], + "Emb2_t101.tif": [ + [13, 25, 41] + ], + "Emb2_t151.tif": [ + [12, 22, 41] + ], + "Emb2_t201.tif": [ + [4, 35, 29], + [25, 35, 29] + ], + "Emb2_t251.tif": [ + [5, 19, 30], + [26, 19, 30] + ], + "Emb2_t301.tif": [ + [8, 18, 32], + [25, 18, 32] + ], + "Emb2_t351.tif": [ + [4, 24, 18], + [24, 24, 18] + ], + "Emb2_t401.tif": [ + [4, 23, 19], + [25, 23, 19] + ], + "Emb2_t451.tif": [ + [5, 10, 13], + [25, 10, 13] + ], + "Emb2_t501.tif": [ + [5, 10, 13], + [24, 10, 13] + ], + "Emb3_t001.tif": [ + [9, 52, 44] + ], + "Emb3_t051.tif": [ + [11, 22, 34] + ], + "Emb3_t101.tif": [ + [8, 42, 33], + [25, 42, 33] + ], + "Emb3_t151.tif": [ + [8, 35, 31] + ], + "Emb3_t201.tif": [ + [7, 31, 30] + ], + "Emb3_t251.tif": [ + [8, 23, 36], + [27, 23, 36] + ], + "Emb3_t301.tif": [ + [8, 27, 35], + [26, 27, 35] + ], + "Emb3_t351.tif": [ + [7, 27, 22], + [25, 27, 22] + ], + "Emb3_t401.tif": [ + [11, 13, 19], + [27, 13, 19] + ], + "Emb3_t451.tif": [ + [8, 16, 15], + [28, 16, 15] + ], + "Emb3_t501.tif": [ + [8, 18, 15], + [27, 18, 15] + ], + "Emb4_t001.tif": [ + [9, 42, 50] + ], + "Emb4_t051.tif": [ + [6, 42, 50], + [22, 42, 50] + ], + "Emb4_t101.tif": [ + [8, 34, 45], + [27, 34, 45] + ], + "Emb4_t151.tif": [ + [6, 35, 43], + [26, 35, 43] + ], + "Emb4_t201.tif": [ + [6, 31, 47], + [26, 31, 47] + ], + "Emb4_t251.tif": [ + [5, 30, 19], + [26, 30, 19] + ], + "Emb4_t301.tif": [ + [5, 23, 23], + [25, 23, 23] + ], + "Emb4_t351.tif": [ + [0, 26, 27], + [22, 26, 27] + ], + "Emb4_t401.tif": [ + [5, 23, 14], + [27, 23, 14] + ], + "Emb4_t451.tif": [ + [4, 17, 17], + [26, 17, 17] + ], + "Emb4_t501.tif": [ + [6, 21, 14], + [27, 21, 14] + ] + }, + "crop_shapes": { + "Emb1_t001.tif": [ + [22, 29, 41] + ], + "Emb1_t051.tif": [ + [17, 29, 61], + [17, 29, 61] + ], + "Emb1_t101.tif": [ + [31, 43, 53] + ], + "Emb1_t151.tif": [ + [28, 63, 47] + ], + "Emb1_t201.tif": [ + [20, 65, 61], + [20, 65, 61] + ], + "Emb1_t251.tif": [ + [20, 62, 56], + [21, 62, 56] + ], + "Emb1_t301.tif": [ + [24, 78, 62], + [24, 78, 62] + ], + "Emb1_t351.tif": [ + [18, 73, 84], + [19, 73, 84] + ], + "Emb1_t401.tif": [ + [22, 79, 75], + [22, 79, 75] + ], + "Emb1_t451.tif": [ + [21, 87, 86], + [21, 87, 86] + ], + "Emb1_t501.tif": [ + [21, 87, 81], + [21, 87, 81] + ], + "Emb2_t001.tif": [ + [28, 45, 31] + ], + "Emb2_t051.tif": [ + [31, 60, 27] + ], + "Emb2_t101.tif": [ + [28, 67, 33] + ], + "Emb2_t151.tif": [ + [29, 74, 33] + ], + "Emb2_t201.tif": [ + [21, 65, 64], + [21, 65, 64] + ], + "Emb2_t251.tif": [ + [21, 77, 61], + [21, 77, 61] + ], + "Emb2_t301.tif": [ + [17, 82, 58], + [17, 82, 58] + ], + "Emb2_t351.tif": [ + [20, 63, 77], + [21, 63, 77] + ], + "Emb2_t401.tif": [ + [21, 82, 83], + [21, 82, 83] + ], + "Emb2_t451.tif": [ + [20, 93, 90], + [21, 93, 90] + ], + "Emb2_t501.tif": [ + [19, 92, 87], + [19, 92, 87] + ], + "Emb3_t001.tif": [ + [26, 34, 46] + ], + "Emb3_t051.tif": [ + [29, 73, 66] + ], + "Emb3_t101.tif": [ + [17, 44, 53], + [17, 44, 53] + ], + "Emb3_t151.tif": [ + [31, 50, 54] + ], + "Emb3_t201.tif": [ + [32, 57, 58] + ], + "Emb3_t251.tif": [ + [19, 62, 64], + [19, 62, 64] + ], + "Emb3_t301.tif": [ + [18, 67, 59], + [17, 67, 59] + ], + "Emb3_t351.tif": [ + [18, 81, 86], + [19, 81, 86] + ], + "Emb3_t401.tif": [ + [16, 83, 74], + [17, 83, 74] + ], + "Emb3_t451.tif": [ + [20, 76, 82], + [19, 76, 82] + ], + "Emb3_t501.tif": [ + [19, 78, 87], + [19, 78, 87] + ], + "Emb4_t001.tif": [ + [29, 31, 28] + ], + "Emb4_t051.tif": [ + [16, 27, 26], + [17, 27, 26] + ], + "Emb4_t101.tif": [ + [19, 49, 32], + [19, 49, 32] + ], + "Emb4_t151.tif": [ + [20, 49, 27], + [20, 49, 27] + ], + "Emb4_t201.tif": [ + [20, 54, 26], + [20, 54, 26] + ], + "Emb4_t251.tif": [ + [21, 61, 64], + [21, 61, 64] + ], + "Emb4_t301.tif": [ + [20, 71, 66], + [20, 71, 66] + ], + "Emb4_t351.tif": [ + [22, 62, 73], + [22, 62, 73] + ], + "Emb4_t401.tif": [ + [22, 71, 90], + [22, 71, 90] + ], + "Emb4_t451.tif": [ + [22, 83, 88], + [21, 83, 88] + ], + "Emb4_t501.tif": [ + [21, 83, 87], + [21, 83, 87] + ] + } + }, + "mouse_embryo_membrane": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-29: the official val volume (light-sheet, 200 x 1024 x 512, hand-corrected GASP labels). Each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out (1 of 8; the kept crops hold 59-176 cells). The tuning data are separate files; the train file of timepoint 73 is a crop of the scored volume and is left out of the tuning data.", + "starts": { + "fused_paral_tp00073_raw_nuclei_membrane_crop_label_corrected_postprocessed.h5": [ + [32, 97, 0], + [63, 97, 0], + [63, 504, 0], + [94, 97, 0], + [94, 504, 0], + [125, 97, 0], + [125, 504, 0] + ] + }, + "crop_shapes": { + "fused_paral_tp00073_raw_nuclei_membrane_crop_label_corrected_postprocessed.h5": [ + [31, 407, 431], + [31, 407, 431], + [31, 407, 431], + [31, 407, 431], + [31, 407, 431], + [31, 407, 431], + [31, 407, 431] + ] + } + }, + "mucic_mda231": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-10-29: the 12 frames of CTC Fluo-C3DL-MDA231 movie 02 with the ISBI 2025 full annotations (movie 01 tunes). Each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every frame gives one crop of its own shape (crop_shapes). All 12 crops hold 44-54 cells.", + "starts": { + "mda231_02_000.h5": [ + [4, 9, 14] + ], + "mda231_02_001.h5": [ + [5, 10, 18] + ], + "mda231_02_002.h5": [ + [5, 11, 12] + ], + "mda231_02_003.h5": [ + [4, 0, 17] + ], + "mda231_02_004.h5": [ + [5, 6, 13] + ], + "mda231_02_005.h5": [ + [4, 14, 19] + ], + "mda231_02_006.h5": [ + [4, 26, 16] + ], + "mda231_02_007.h5": [ + [5, 13, 10] + ], + "mda231_02_008.h5": [ + [4, 0, 23] + ], + "mda231_02_009.h5": [ + [5, 1, 15] + ], + "mda231_02_010.h5": [ + [5, 0, 13] + ], + "mda231_02_011.h5": [ + [5, 0, 14] + ] + }, + "crop_shapes": { + "mda231_02_000.h5": [ + [22, 503, 495] + ], + "mda231_02_001.h5": [ + [21, 502, 494] + ], + "mda231_02_002.h5": [ + [21, 501, 499] + ], + "mda231_02_003.h5": [ + [23, 512, 495] + ], + "mda231_02_004.h5": [ + [22, 489, 499] + ], + "mda231_02_005.h5": [ + [23, 498, 493] + ], + "mda231_02_006.h5": [ + [24, 486, 496] + ], + "mda231_02_007.h5": [ + [22, 484, 502] + ], + "mda231_02_008.h5": [ + [23, 496, 480] + ], + "mda231_02_009.h5": [ + [22, 492, 496] + ], + "mda231_02_010.h5": [ + [21, 496, 497] + ], + "mda231_02_011.h5": [ + [22, 494, 492] + ] + } + }, + "elmi_mito": { + "crop_shape": [512, 512], + "note": "512 x 512 tiles of the test images that hold a labeled mitochondrion; only 3-5 are labeled per image. 16 crops, holding 1-2 objects.", + "starts": { + "ELMI_01.tif": [ + [0, 0], + [512, 0], + [512, 512], + [512, 1024], + [1024, 512] + ], + "ELMI_02.tif": [ + [0, 0], + [0, 512], + [512, 1024] + ], + "ELMI_03.tif": [ + [0, 1024], + [512, 1024], + [1024, 1024], + [1536, 1024] + ], + "ELMI_04.tif": [ + [512, 512], + [512, 1024], + [1024, 0], + [1024, 1024] + ] + } + }, + "cortex_human_stg": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-02: the 64 x 1024 x 1024 crop at the center of the CortexConnectomics human STG volume (SBEM, 11.24 x 11.24 x 30 nm), dense proofread neuron labels with some block-stitching seams. The crops tile the annotation; tiles with a blank section or fewer than 20% of the median object count are dropped. 8 of 8 tiles are kept, holding 911-1295 objects.", + "starts": { + "human_stg_08f31f40fc22.zarr": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ] + } + }, + "morphonet_patiria": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-11-02: the single MorphoNet Patiria miniata volume (confocal, 174 x 375 x 375) with the curated labels; objects below 500 voxels are specks and left out. Each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out (0 of 4; the kept crops hold 73-213 cells).", + "starts": { + "embryo1_20200812_pos6_stk010.tif": [ + [23, 33, 41], + [49, 33, 41], + [75, 33, 41], + [102, 33, 41] + ] + }, + "crop_shapes": { + "embryo1_20200812_pos6_stk010.tif": [ + [26, 309, 307], + [26, 309, 307], + [27, 309, 307], + [26, 309, 307] + ] + } + }, + "ionsem": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-02: the EMNeuron out-of-distribution validation volume (Ionsem, 30 x 1000 x 1000), dense neurite labels. The crops tile the annotation; tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 4 of 4 tiles are kept, holding 160-181 objects.", + "starts": { + "raw.tif": [ + [0, 0, 0], + [0, 0, 500], + [0, 500, 0], + [0, 500, 500] + ] + }, + "crop_shapes": { + "raw.tif": [ + [30, 500, 500], + [30, 500, 500], + [30, 500, 500], + [30, 500, 500] + ] + } + }, + "microns_basil": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-02: the EMNeuron out-of-distribution validation volume (MICrONS basil, ssTEM, 72 x 512 x 512), dense neurite labels. The crops tile the annotation; tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 1 of 3 tiles are kept, holding 225-225 objects.", + "starts": { + "raw.tif": [ + [0, 0, 0] + ] + }, + "crop_shapes": { + "raw.tif": [ + [24, 512, 512] + ] + } + }, + "isbi2012": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 1 of 1 tiles are kept, holding 465-465 objects.", + "starts": { + "isbi.h5": [ + [0, 0, 0] + ] + }, + "crop_shapes": { + "isbi.h5": [ + [30, 512, 512] + ] + } + }, + "nisb": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 25 of 1025 tiles are kept, holding 128-153 objects.", + "starts": { + "data.zarr": [ + [276, 1760, 1248], + [308, 1760, 1248], + [372, 1760, 1760], + [404, 1760, 736], + [404, 1760, 1760], + [404, 2272, 736], + [404, 2272, 1760], + [436, 1248, 736], + [468, 1248, 736], + [564, 736, 224], + [564, 736, 1760], + [564, 1760, 736], + [596, 736, 1760], + [628, 1760, 2272], + [660, 1248, 1248], + [692, 1248, 736], + [692, 1248, 1248], + [692, 1248, 1760], + [788, 736, 736], + [820, 736, 736], + [852, 1760, 2272], + [884, 736, 224], + [916, 736, 224], + [1044, 224, 1248], + [1300, 736, 1760] + ] + } + }, + "synapseweb": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 25 of 179 tiles are kept, holding 94-1034 objects.", + "starts": { + "synapseweb_hippocampus_apical.h5": [ + [10, 192, 896], + [10, 1728, 384], + [10, 1728, 2432], + [10, 2240, 2944], + [10, 2752, 896], + [42, 1728, 384], + [42, 2752, 896], + [42, 2752, 1408], + [42, 3264, 896], + [74, 704, 1920], + [74, 704, 3456], + [74, 1216, 384], + [74, 1216, 896], + [74, 1216, 1408], + [74, 1728, 384], + [74, 1728, 896], + [74, 1728, 1408], + [74, 2240, 1408], + [74, 2752, 3456], + [74, 3264, 2432] + ], + "synapseweb_hippocampus_oblique.h5": [ + [8, 2496, 1568], + [8, 2496, 2592], + [40, 1472, 2592], + [40, 3008, 2592] + ], + "synapseweb_hippocampus_spine.h5": [ + [5, 1376, 1248] + ] + } + }, + "densecell": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 1 of 1 tiles are kept, holding 13-13 objects.", + "starts": { + "densecell_val.h5": [ + [0, 144, 144] + ] + }, + "crop_shapes": { + "densecell_val.h5": [ + [24, 512, 512] + ] + } + }, + "fafb": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 44 of 48 tiles are kept, holding 568-1849 objects.", + "starts": { + "ea798aee7bab.zarr": [ + [13, 0, 0], + [13, 0, 512], + [13, 512, 0], + [13, 512, 512], + [77, 0, 0], + [77, 0, 512], + [77, 512, 0], + [77, 512, 512], + [109, 0, 0], + [109, 0, 512], + [109, 512, 0], + [109, 512, 512], + [141, 0, 0], + [141, 0, 512], + [141, 512, 0], + [141, 512, 512], + [173, 0, 0], + [173, 0, 512], + [173, 512, 0], + [173, 512, 512], + [205, 0, 0], + [205, 0, 512], + [205, 512, 0], + [205, 512, 512], + [237, 0, 0], + [237, 0, 512], + [237, 512, 0], + [237, 512, 512], + [269, 0, 0], + [269, 0, 512], + [269, 512, 0], + [269, 512, 512], + [301, 0, 0], + [301, 0, 512], + [301, 512, 0], + [301, 512, 512], + [333, 0, 0], + [333, 0, 512], + [333, 512, 0], + [333, 512, 512], + [365, 0, 0], + [365, 0, 512], + [365, 512, 0], + [365, 512, 512] + ] + } + }, + "hemibrain": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 24 of 24 tiles are kept, holding 160-306 objects.", + "starts": { + "neurons_e6712bb65fa6.h5": [ + [826, 0, 0], + [826, 0, 512], + [826, 512, 0], + [826, 512, 512], + [858, 0, 0], + [858, 0, 512], + [858, 512, 0], + [858, 512, 512], + [890, 0, 0], + [890, 0, 512], + [890, 512, 0], + [890, 512, 512], + [922, 0, 0], + [922, 0, 512], + [922, 512, 0], + [922, 512, 512], + [954, 0, 0], + [954, 0, 512], + [954, 512, 0], + [954, 512, 512], + [986, 0, 0], + [986, 0, 512], + [986, 512, 0], + [986, 512, 512] + ] + } + }, + "xpress": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 1 of 1 tiles are kept, holding 23-23 objects.", + "starts": { + "xpress-training.h5": [ + [308, 128, 128] + ] + }, + "crop_shapes": { + "xpress-training.h5": [ + [20, 200, 200] + ] + } + }, + "liconn": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 25 of 48 tiles are kept, holding 630-745 objects.", + "starts": { + "liconn.zarr": [ + [586, 320, 384], + [586, 320, 896], + [586, 320, 1408], + [586, 320, 1920], + [586, 320, 2432], + [586, 320, 2944], + [586, 832, 384], + [586, 832, 896], + [586, 832, 1408], + [586, 832, 2944], + [586, 1344, 384], + [586, 1344, 896], + [586, 1344, 1408], + [586, 1344, 1920], + [586, 1344, 2944], + [586, 1856, 384], + [586, 1856, 1920], + [586, 1856, 2432], + [586, 1856, 2944], + [586, 2368, 896], + [586, 2368, 2944], + [586, 2880, 2432], + [586, 3392, 384], + [586, 3392, 2944], + [586, 3904, 384] + ] + } + }, + "zebrafinch_j0126": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 25 of 324 tiles are kept, holding 409-827 objects.", + "starts": { + "j0126_mip0_5a3523fabf23.zarr": [ + [512, 272, 288], + [512, 272, 1312], + [512, 1296, 288], + [512, 2832, 800], + [544, 272, 288], + [544, 272, 1312], + [544, 272, 1824], + [544, 1296, 288], + [544, 4368, 2848], + [576, 272, 288], + [576, 272, 1312], + [576, 784, 288], + [576, 1296, 2336], + [576, 1808, 3360], + [576, 2320, 3360], + [576, 4368, 2848], + [608, 272, 288], + [608, 272, 1312], + [608, 272, 1824], + [608, 784, 288], + [608, 784, 800], + [608, 1808, 3360], + [608, 2320, 3360], + [608, 2832, 800], + [608, 4368, 2848] + ] + } + }, + "synapsenet_compartments": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count. 8 of 8 tiles are kept, holding 9-30 objects.", + "starts": { + "36859_H2_SP_11_rec_2Kb1dawbp_crop.h5": [ + [3, 0, 0], + [35, 0, 0] + ], + "36859_J1_66K_TS_CA3_MF_19_rec_2Kb1dawbp_crop.h5": [ + [11, 0, 0], + [43, 0, 0], + [75, 0, 0] + ], + "36859_J2_66K_TS_R04_PS11_rec_2Kb1dawbp_crop.h5": [ + [12, 0, 0], + [44, 0, 0], + [76, 0, 0] + ] + }, + "crop_shapes": { + "36859_H2_SP_11_rec_2Kb1dawbp_crop.h5": [ + [32, 478, 479], + [32, 478, 479] + ], + "36859_J1_66K_TS_CA3_MF_19_rec_2Kb1dawbp_crop.h5": [ + [32, 440, 480], + [32, 440, 480], + [32, 440, 480] + ], + "36859_J2_66K_TS_R04_PS11_rec_2Kb1dawbp_crop.h5": [ + [32, 480, 480], + [32, 480, 480], + [32, 480, 480] + ] + } + }, + "axonem": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 4 tiles are kept, holding 142-200 objects. The centred tile of im_700-3584-0_pad.h5 has blank sections (z 21, 25, 34), so its crop starts at z 35 instead.", + "starts": { + "im_700-0-3584_pad.h5": [ + [28, 512, 512] + ], + "im_700-3584-0_pad.h5": [ + [35, 512, 512] + ], + "im_950-0-3584_pad.h5": [ + [29, 512, 512] + ], + "im_950-3584-0_pad.h5": [ + [29, 512, 512] + ] + } + }, + "fib25": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 16 of 16 tiles are kept, holding 584-1286 objects.", + "starts": { + "raw.h5": [ + [4, 4, 4], + [36, 4, 4], + [68, 4, 4], + [100, 4, 4], + [132, 4, 4], + [164, 4, 4], + [196, 4, 4], + [228, 4, 4], + [260, 4, 4], + [292, 4, 4], + [324, 4, 4], + [356, 4, 4], + [388, 4, 4], + [420, 4, 4], + [452, 4, 4], + [484, 4, 4] + ] + } + }, + "manc": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 24 of 24 tiles are kept, holding 190-462 objects.", + "starts": { + "d44819d9e109.h5": [ + [826, 0, 0], + [826, 0, 512], + [826, 512, 0], + [826, 512, 512], + [858, 0, 0], + [858, 0, 512], + [858, 512, 0], + [858, 512, 512], + [890, 0, 0], + [890, 0, 512], + [890, 512, 0], + [890, 512, 512], + [922, 0, 0], + [922, 0, 512], + [922, 512, 0], + [922, 512, 512], + [954, 0, 0], + [954, 0, 512], + [954, 512, 0], + [954, 512, 512], + [986, 0, 0], + [986, 0, 512], + [986, 512, 0], + [986, 512, 512] + ] + } + }, + "malecns": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 25 of 128 tiles are kept, holding 301-540 objects.", + "starts": { + "4c17ab1c9772.h5": [ + [160, 0, 512], + [288, 0, 512], + [288, 512, 0], + [288, 512, 512], + [384, 0, 512], + [416, 0, 512], + [416, 512, 0], + [448, 512, 0], + [480, 0, 512], + [480, 512, 0], + [512, 0, 512], + [544, 0, 512], + [608, 0, 512], + [640, 0, 512], + [672, 0, 512], + [704, 0, 0], + [704, 0, 512], + [704, 512, 0], + [704, 512, 512], + [736, 0, 512], + [768, 0, 512], + [832, 512, 0], + [896, 512, 0], + [928, 512, 0], + [960, 512, 0] + ] + } + }, + "wafer4": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 4 of 4 tiles are kept, holding 85-137 objects.", + "starts": { + "wafer4_inputs.h5": [ + [100, 113, 113], + [100, 113, 625], + [100, 625, 113], + [100, 625, 625] + ] + }, + "crop_shapes": { + "wafer4_inputs.h5": [ + [25, 512, 512], + [25, 512, 512], + [25, 512, 512], + [25, 512, 512] + ] + } + }, + "minnie65": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 24 of 4096 tiles are kept, holding 401-554 objects.", + "starts": { + "1573f88c7587.zarr": [ + [0, 0, 0], + [0, 0, 1024], + [0, 1536, 512], + [0, 2560, 1536], + [192, 1024, 512], + [416, 3072, 512] + ], + "49705991c41e.zarr": [ + [96, 0, 3584], + [96, 3072, 3584], + [96, 3584, 3584], + [128, 0, 3584], + [160, 0, 3584], + [192, 0, 3072] + ], + "5c85ec2c1e55.zarr": [ + [32, 0, 3072], + [32, 0, 3584], + [32, 512, 512], + [32, 1024, 0], + [32, 3072, 1024], + [32, 3584, 2048] + ], + "99ee6e54c159.zarr": [ + [0, 0, 512], + [0, 0, 2048], + [0, 1536, 3584], + [0, 3072, 3072], + [32, 1536, 3584], + [96, 3072, 0] + ] + } + }, + "zebrafinch_j0251": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 25 of 128 tiles are kept, holding 313-385 objects.", + "starts": { + "j0251_mip0_4f24046ab754.zarr": [ + [0, 0, 0], + [0, 0, 1536], + [0, 512, 0], + [0, 1024, 1536], + [32, 0, 0], + [32, 0, 512], + [32, 0, 1536], + [32, 512, 0], + [32, 1024, 1536], + [64, 0, 512], + [64, 512, 0], + [64, 1024, 1536], + [64, 1536, 1536], + [96, 0, 512], + [96, 512, 0], + [96, 1024, 1536], + [128, 512, 0], + [160, 0, 512], + [160, 512, 0], + [160, 1024, 0], + [160, 1024, 1536], + [192, 512, 0], + [192, 1024, 1536], + [224, 512, 0], + [224, 1024, 1536] + ] + } + }, + "wildenberg": { + "crop_shape": [32, 512, 512], + "note": "EM panel, added 2026-11-01 in place of the 8-slice center crop: the crops tile the annotation inside the test region. Tiles thinner than 16 slices or 256 px in plane and tiles with a blank section are dropped, and so are tiles with fewer than 20% of the median object count, at most 25 kept, shared over the volumes by their number of tiles (at least one each), the ones with the most objects of a volume. 8 of 9 tiles are kept, holding 3648-7752 objects.", + "starts": { + "p105_saturated_4eac7f2899e7.zarr": [ + [120, 181, 181], + [120, 181, 693], + [120, 181, 1205], + [120, 693, 181], + [120, 693, 1205], + [120, 1205, 181], + [120, 1205, 693], + [120, 1205, 1205] + ] + }, + "crop_shapes": { + "p105_saturated_4eac7f2899e7.zarr": [ + [30, 512, 512], + [30, 512, 512], + [30, 512, 512], + [30, 512, 512], + [30, 512, 512], + [30, 512, 512], + [30, 512, 512], + [30, 512, 512] + ] + } + }, + "nus_organoids": { + "crop_shape": [32, 512, 512], + "note": "Light microscopy panel, added 2026-11-02: three of the four volumes of the nucleus benchmark of the Digitalized Organoid paper with their manual labels (TM00099 is left out: its nuclei are dim and sparse, and all methods fail on it); objects below 100 voxels are specks and left out. Each axis of the annotation is split into the fewest equal tiles of at most 32 x 512 x 512, so every crop has its own shape (crop_shapes). Crops with fewer than 20% of the median object count are left out (0 of 4; the kept crops hold 38-73 nuclei).", + "starts": { + "HCT116_cells_monolayer.tif": [ + [0, 0, 55] + ], + "Human_colon_organoid.tif": [ + [0, 83, 155] + ], + "ZeroG_breast_cancer_spheroid.tif": [ + [13, 147, 150], + [44, 147, 150] + ] + }, + "crop_shapes": { + "HCT116_cells_monolayer.tif": [ + [15, 415, 457] + ], + "Human_colon_organoid.tif": [ + [16, 313, 208] + ], + "ZeroG_breast_cancer_spheroid.tif": [ + [31, 224, 184], + [32, 224, 184] + ] + } + }, + "mitolab_tem": { + "crop_shape": [512, 512], + "note": "512 x 512 tiles that cover each test image, spread evenly with a small overlap; an image axis below 512 px is one tile. 328 of 371 tiles are kept (at most 16 per image, the ones with the most objects), holding 2-86 mitochondria.", + "starts": { + "DccMhwElHJmbuS2J8lrU.tiff": [ + [0, 512] + ], + "KDdqT7pjt4RB6DP9YhBv.tiff": [ + [0, 0], + [0, 512] + ], + "gxdh9YGZMJUB58YvyzgM.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "MWyu8XxfJUz0tPpjMG68.tiff": [ + [0, 0], + [0, 459], + [340, 0], + [340, 459] + ], + "sN90grldkJUpRIAJ64Bz.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "1vNHcbV4bWuC1zywuBdP.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "iidFXTWegya1bm1hl5to.tiff": [ + [0, 0], + [0, 384], + [0, 768], + [284, 0], + [284, 384], + [284, 768], + [568, 0], + [568, 384], + [568, 768] + ], + "j1k88xV7tmKNMLM4bj1h.tiff": [ + [0, 0], + [0, 512], + [512, 0] + ], + "9SNS8ZHu7wx8HstQg2Vo.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "KYJvyQylQ6GoiyoABzCi.tiff": [ + [0, 0], + [0, 342], + [0, 684], + [376, 0], + [376, 342], + [376, 684] + ], + "VFu9kl8plvKxrBEwRfhL.tiff": [ + [0, 0], + [424, 0], + [424, 512] + ], + "qqRmOpdujNXIqEuWrFLk.tiff": [ + [0, 0], + [0, 386], + [378, 0], + [378, 386], + [756, 0], + [756, 386] + ], + "HIv1HOOLTWhvBGs2GLAW.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "QWlbQZ1v00zxmk2Fp3Le.tiff": [ + [0, 0], + [0, 238], + [145, 0], + [145, 238] + ], + "dYaOkBZPfpE3oHiplIst.tiff": [ + [0, 512], + [512, 0] + ], + "ALknkSgr7aoWr14RfrSV.tiff": [ + [0, 0], + [0, 494], + [334, 0], + [334, 494], + [334, 988], + [669, 0], + [669, 494], + [669, 988] + ], + "QLwSiZlPfFaKn9UkVw5A.tiff": [ + [0, 0], + [512, 0], + [512, 512] + ], + "xHSMEPuKIY4T3X4UURT5.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "6plriVrEepvhHyOBmtUc.tiff": [ + [0, 0], + [0, 512], + [512, 512] + ], + "ghAKYp44DzBUhlyn7Kec.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "7Z7jSCJZ23zSW2ZztTPJ.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "SXnwBMNzR18D8sKSjh8g.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "pLeNHL0RTZwYh5EwN7up.tiff": [ + [0, 0] + ], + "IsdoTn0CZN3NNBHw9eHs.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "uBBsOocNXDq5poteflbh.tiff": [ + [0, 0], + [128, 0] + ], + "2Tlqk9n8mmGv6G0EGxwl.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "cwpw4Zu4rqVzbTMPhql2.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "ch1HZ72XmfK9rD5K40QW.tiff": [ + [0, 0], + [0, 258], + [0, 516], + [324, 0], + [324, 258], + [324, 516] + ], + "NFlQQXCuLN49baKGNHc8.tiff": [ + [0, 0], + [0, 512], + [512, 512] + ], + "AOVAPBvPacOXcUoVM3uZ.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "TbVlEbkOaXvgqXuOC2y5.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "N9aIKs7lUVsYd6bT3LxC.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "3pbuH37yaTFrTt52QdNO.tiff": [ + [0, 0], + [0, 406], + [390, 0], + [390, 406], + [780, 0], + [780, 406] + ], + "vziaz0e1YZ9BJHIY7tJV.tiff": [ + [0, 0], + [512, 0] + ], + "HWJC0I0n5pPosm91fPN9.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "DprwEW2YlVIuqe8iiBUM.tiff": [ + [0, 0], + [512, 0], + [512, 512] + ], + "kF3EaTAoYeTRf12uLsIX.tiff": [ + [0, 0], + [0, 324], + [0, 648], + [494, 0], + [494, 324], + [494, 648], + [988, 0], + [988, 324], + [988, 648] + ], + "Gp5MEeb8iAuYfrAcOb3T.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "YnjZLcFb5mRSIQR7oTJ1.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "9jHLkwpVONtdgyvLyhds.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "GC18X3wh91fultTzXk0n.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "1unGsOC0dfeB02LCtQwi.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "AvUtncbaV60ITOmLlAkQ.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "P4f6REHlY0j5oPr8hyFr.tiff": [ + [0, 0] + ], + "5BzcbKwHruGS296dUugB.tiff": [ + [0, 0] + ], + "k0Ij8iyjBDimmOPwtFHL.tiff": [ + [0, 0], + [0, 114], + [238, 0], + [238, 114] + ], + "3ib2iOCHNQIgrUaxnLwm.tiff": [ + [0, 0] + ], + "bUsTjDOJHiHMPCNIWkuT.tiff": [ + [0, 0], + [512, 0] + ], + "NWNws2Vo4AQhIw579f4E.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "qs9OOX5WsQhRE3Kg2rpU.tiff": [ + [512, 512] + ], + "VcJ1fV2RlrJrmLHIzdIv.tiff": [ + [0, 512], + [512, 0], + [512, 512] + ], + "zRf1ZWOS08TxjOvROo86.tiff": [ + [0, 0], + [512, 0] + ], + "V52bj9CYW97AUqevB2Rv.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "QWqaLBDT0uuxxNX25kP9.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "UH2Xa2sqyug07qibeLI2.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "d2Cvt7UfKPIAnaJGlKpw.tiff": [ + [0, 0], + [0, 258], + [0, 516], + [310, 0], + [310, 258], + [310, 516] + ], + "RoECvy0RbILoKiv8jYab.tiff": [ + [0, 0] + ], + "d7McLI44TDhNmnd1QSpo.tiff": [ + [0, 0], + [0, 512], + [512, 0] + ], + "Y5Uy1ACZO4MkqTvSzJcS.tiff": [ + [0, 0], + [0, 238], + [142, 0], + [142, 238] + ], + "yayA9ov0gTMPe5oUDmCY.tiff": [ + [0, 0], + [0, 157], + [174, 0], + [174, 157] + ], + "k4Uw1csHJKTrviecgXyz.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "QQZlSgIkiL7OdZ2a8oIm.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "X1DAYM5MVaMXPhpeeTBT.tiff": [ + [0, 0], + [512, 0], + [512, 512] + ], + "TCzgvIZVHqNFtdJEPaXf.tiff": [ + [0, 0], + [0, 371], + [404, 0], + [404, 371] + ], + "yn4W2WociugOlXWN83Ps.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "WCqmS6R1MFjZfQXMEbov.tiff": [ + [0, 0], + [0, 48], + [411, 0], + [411, 48] + ], + "iDXfFDvQLRHsB4XZHb3A.tiff": [ + [0, 512], + [512, 0] + ], + "MTeOXdixYD6o3ysTjtlY.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "v64IdYxtp9vzYzojtxir.tiff": [ + [0, 512], + [512, 0], + [512, 512] + ], + "XsisEi7EmDiq7esbhl3k.tiff": [ + [0, 0], + [0, 512], + [512, 512] + ], + "aLriAAXSpMzI1tSLzl05.tiff": [ + [0, 512], + [512, 0], + [512, 512] + ], + "ngygM944aKxBq6rf72Zl.tiff": [ + [0, 512], + [512, 0] + ], + "NpIsioJcraoQ7UM5w5KA.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "JbZdraW8jUWOkWYenUQW.tiff": [ + [0, 0], + [124, 0] + ], + "17pOyGcUusCVRfzUZxTs.tiff": [ + [0, 0], + [0, 73], + [157, 0], + [157, 73] + ], + "Ub2PGMk15slpiDOMy6gd.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "FJkyZfNswUHYI8UXjInE.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "1U0jYrdPwdlRDsQzkUNc.tiff": [ + [0, 512], + [512, 0], + [512, 512] + ], + "FcQtYwd3CZhEoqWrZQ6a.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "qOA9ml35XYXQjtWmFvvg.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "sXnNKReR54pzKRZ5rpfP.tiff": [ + [0, 0], + [512, 0], + [512, 512] + ], + "hR5EYnvM2Ysjfddi9ZTt.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "pu4kmugMyXvbFtAgLogm.tiff": [ + [512, 0], + [512, 512] + ], + "lCH8UM6Mv2AG7zDPrqHA.tiff": [ + [0, 0], + [0, 268], + [0, 536], + [307, 0], + [307, 268], + [307, 536] + ], + "E3DjzU1Hv0lKtRerU0dr.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "XzLxoE9qQ3O9Gq28iveF.tiff": [ + [0, 512], + [512, 0], + [512, 512] + ], + "eL79mWeRWlqFAqOO6w8o.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "AH9rwncuiyjonmnTpzc6.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "snDIBSvwQMaqGKBbGF6V.tiff": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "UTbbCmNhBbRwzOznV6eM.tiff": [ + [0, 0], + [0, 512] + ] + }, + "crop_shapes": { + "pLeNHL0RTZwYh5EwN7up.tiff": [ + [315, 224] + ], + "uBBsOocNXDq5poteflbh.tiff": [ + [512, 451], + [512, 451] + ], + "P4f6REHlY0j5oPr8hyFr.tiff": [ + [359, 456] + ], + "5BzcbKwHruGS296dUugB.tiff": [ + [324, 460] + ], + "3ib2iOCHNQIgrUaxnLwm.tiff": [ + [470, 501] + ], + "JbZdraW8jUWOkWYenUQW.tiff": [ + [512, 449], + [512, 449] + ] + } + }, + "probtem": { + "crop_shape": [512, 512], + "note": "512 x 512 tiles that cover each test image, spread evenly with a small overlap; an image axis below 512 px is one tile. 96 of 150 tiles are kept (at most 16 per image, the ones with the most objects), holding 1-7 mitochondria.", + "starts": { + "x_0.h5": [ + [0, 512], + [0, 1024], + [512, 0], + [512, 512], + [512, 1024], + [512, 1536], + [512, 2048], + [1024, 0], + [1024, 512], + [1024, 1024], + [1024, 1536], + [1024, 2048], + [1536, 1536], + [1536, 2048], + [2048, 1536], + [2048, 2048] + ], + "x_1.h5": [ + [0, 0], + [0, 512], + [0, 1024], + [0, 1536], + [0, 2048], + [512, 0], + [512, 1024], + [512, 1536], + [512, 2048], + [1024, 0], + [1024, 512], + [1536, 512], + [1536, 2048], + [2048, 512], + [2048, 1536], + [2048, 2048] + ], + "x_2.h5": [ + [0, 0], + [0, 512], + [0, 1024], + [512, 0], + [512, 1024], + [1024, 0], + [1024, 512], + [1024, 1024], + [1024, 1536], + [1536, 0], + [1536, 512], + [1536, 1024], + [1536, 1536], + [2048, 512], + [2048, 1536], + [2048, 2048] + ], + "x_3.h5": [ + [0, 0], + [0, 512], + [0, 1024], + [0, 1536], + [0, 2048], + [512, 0], + [512, 512], + [512, 1024], + [512, 1536], + [512, 2048], + [1024, 0], + [1024, 1024], + [1024, 1536], + [2048, 1024], + [2048, 1536], + [2048, 2048] + ], + "x_4.h5": [ + [0, 0], + [0, 512], + [0, 1024], + [0, 1536], + [0, 2048], + [512, 0], + [512, 512], + [512, 1024], + [512, 1536], + [1024, 0], + [1024, 512], + [1024, 1024], + [1536, 1024], + [2048, 0], + [2048, 1024], + [2048, 2048] + ], + "x_5.h5": [ + [0, 512], + [0, 1024], + [0, 1536], + [0, 2048], + [512, 0], + [512, 512], + [512, 1024], + [512, 1536], + [512, 2048], + [1024, 0], + [1024, 512], + [1024, 1536], + [2048, 0], + [2048, 512], + [2048, 1536], + [2048, 2048] + ] + } + }, + "tnbc_mito": { + "crop_shape": [512, 512], + "note": "512 x 512 tiles that cover each test image, spread evenly with a small overlap; an image axis below 512 px is one tile. 136 of 552 tiles are kept (at most 16 per image, the ones with the most objects), holding 1-27 mitochondria.", + "starts": { + "wt_3_HM_2.tif": [ + [0, 1536], + [512, 0], + [512, 512], + [512, 1024], + [1024, 0], + [1024, 512], + [1024, 1024], + [1024, 1536], + [1536, 1536] + ], + "wt_7Kx_2.tif": [ + [0, 512], + [0, 1024], + [0, 1536], + [1024, 512], + [1024, 1024], + [1536, 512], + [1536, 1024] + ], + "wt_9_HM_2.tif": [ + [0, 1024], + [512, 0], + [512, 1536], + [1024, 0], + [1024, 512], + [1024, 1024], + [1024, 1536], + [1536, 512] + ], + "R20_72_G01_2913_11.tif": [ + [0, 512], + [0, 2048], + [0, 2560], + [464, 2048], + [464, 2560], + [928, 2048], + [1392, 1024], + [1392, 1536], + [1856, 1024], + [1856, 1536], + [1856, 2560], + [2320, 1536], + [2784, 1536], + [2784, 3072], + [3248, 3072], + [3248, 3584] + ], + "R20_72_G01_2913_18.tif": [ + [928, 2560], + [928, 3072], + [1392, 0], + [1392, 512], + [1392, 1024], + [1392, 1536], + [1392, 2048], + [1392, 2560], + [1392, 3072], + [1392, 3584], + [1856, 0], + [1856, 512], + [1856, 1536], + [1856, 2048], + [1856, 2560], + [1856, 3072] + ], + "R20_120C_55A_01.tif": [ + [0, 2560], + [0, 3072], + [464, 2560], + [464, 3072], + [928, 2560], + [928, 3072], + [1392, 2048], + [1392, 2560], + [1392, 3072], + [2320, 0], + [2320, 1536], + [2784, 1536], + [2784, 2048], + [2784, 2560], + [3248, 1536], + [3248, 3584] + ], + "R20_120E_56A_25.tif": [ + [464, 1536], + [928, 1024], + [928, 1536], + [928, 2048], + [1392, 0], + [1392, 1024], + [1856, 512], + [1856, 1536], + [1856, 2048], + [2320, 512], + [2320, 1536], + [2320, 2048], + [2784, 0], + [2784, 512], + [2784, 3584], + [3248, 2560] + ], + "R20_120E_56A_39.tif": [ + [0, 0], + [464, 0], + [464, 2560], + [928, 0], + [928, 2048], + [1392, 1536], + [1392, 2560], + [1392, 3072], + [1392, 3584], + [1856, 2560], + [1856, 3584], + [2320, 3072], + [2784, 2048], + [3248, 1536], + [3248, 2048], + [3712, 3584] + ], + "R20_120F_56B_32.tif": [ + [0, 3072], + [464, 1024], + [464, 3072], + [928, 2560], + [928, 3072], + [1392, 2048], + [1392, 2560], + [1392, 3072], + [1392, 3584], + [1856, 3584], + [2320, 2560], + [2784, 1024], + [2784, 2048], + [2784, 2560], + [2784, 3072], + [2784, 3584] + ], + "R20_120F_56B_37.tif": [ + [0, 2048], + [464, 2560], + [1392, 512], + [1392, 1024], + [1392, 2048], + [1856, 512], + [1856, 1024], + [1856, 2048], + [1856, 3584], + [2320, 2560], + [2784, 512], + [2784, 2560], + [2784, 3072], + [3248, 512], + [3712, 512], + [3712, 1536] + ] + } + }, + "perez_mito": { + "crop_shape": [512, 512], + "note": "512 x 512 tiles that cover each test image, spread evenly with a small overlap; an image axis below 512 px is one tile. 151 of 160 tiles are kept (at most 16 per image, the ones with the most objects), holding 1-9 mitochondria.", + "starts": { + "000.png": [ + [0, 488], + [488, 0], + [488, 488] + ], + "001.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "002.png": [ + [0, 488], + [488, 0], + [488, 488] + ], + "003.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "004.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "005.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "006.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "007.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "008.png": [ + [0, 488], + [488, 0], + [488, 488] + ], + "009.png": [ + [0, 0], + [0, 488], + [488, 0] + ], + "010.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "011.png": [ + [0, 0], + [0, 488], + [488, 488] + ], + "012.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "013.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "014.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "015.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "016.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "017.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "018.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "019.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "020.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "021.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "022.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "023.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "024.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "025.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "026.png": [ + [0, 0], + [0, 488] + ], + "027.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "028.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "029.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "030.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "031.png": [ + [0, 0], + [0, 488], + [488, 488] + ], + "032.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "033.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "034.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "035.png": [ + [0, 488], + [488, 0], + [488, 488] + ], + "036.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "037.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "038.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ], + "039.png": [ + [0, 0], + [0, 488], + [488, 0], + [488, 488] + ] + } + }, + "cardiomyocyte_tem_mito": { + "crop_shape": [512, 512], + "note": "512 x 512 tiles that cover each test image, spread evenly with a small overlap; an image axis below 512 px is one tile. 113 of 124 tiles are kept (at most 16 per image, the ones with the most objects), holding 6-88 mitochondria.", + "starts": { + "914.11_grid1_nucA_m2_M3500x_tile_y0_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.11_grid1_nucA_m2_M3500x_tile_y1024_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.11_grid1_nucA_m2_M3500x_tile_y1024_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.11_grid1_nucA_m2_M3500x_tile_y2048_x2048.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.11_grid4_nucA_m2_M3500x_tile_y0_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.11_grid4_nucA_m2_M3500x_tile_y1024_x0.h5": [ + [0, 0], + [0, 512], + [512, 512] + ], + "914.11_grid4_nucA_m2_M3500x_tile_y1024_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.11_grid4_nucA_m2_M3500x_tile_y2048_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.16_18.12.2023_nucD_m2_3500x_tile_y0_x0.h5": [ + [512, 0], + [512, 512] + ], + "914.16_18.12.2023_nucD_m2_3500x_tile_y0_x1024.h5": [ + [0, 512], + [512, 0], + [512, 512] + ], + "914.16_18.12.2023_nucD_m2_3500x_tile_y0_x2048.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.16_18.12.2023_nucD_m2_3500x_tile_y0_x3072.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.1_nucA_m3_3500x_tile_y0_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.1_nucA_m3_3500x_tile_y0_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.1_nucA_m3_3500x_tile_y1024_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.1_nucA_m3_3500x_tile_y2048_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.1_nucC_m3_3500x_tile_y0_x0.h5": [ + [0, 512], + [512, 512] + ], + "914.1_nucC_m3_3500x_tile_y1024_x0.h5": [ + [0, 512], + [512, 0], + [512, 512] + ], + "914.1_nucC_m3_3500x_tile_y2048_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucC_m3_3500x_tile_y0_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucC_m3_3500x_tile_y1024_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucC_m3_3500x_tile_y2048_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucC_m3_3500x_tile_y2048_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucE_m2_3500x_tile_y0_x0.h5": [ + [512, 512] + ], + "914.22_nucE_m2_3500x_tile_y0_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucE_m2_3500x_tile_y1024_x0.h5": [ + [0, 512], + [512, 0], + [512, 512] + ], + "914.22_nucE_m2_3500x_tile_y1024_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.9_grid1_nucA_m3_M3500x_tile_y3072_x3072.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.9_grid1_nucA_m3_M3500x_tile_y4096_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.9_grid1_nucA_m3_M3500x_tile_y5120_x0.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ], + "914.9_grid1_nucA_m3_M3500x_tile_y5120_x1024.h5": [ + [0, 0], + [0, 512], + [512, 0], + [512, 512] + ] + } + }, + "dauer_larva_nucleus": { + "crop_shape": [32, 512, 512], + "note": "Nucleus panel (in-domain): boxes of the OpenOrganelle jrc_dauer-larva volume (ssTEM, 70 x 40 x 40 nm), which the nucleus model never saw; automatic COSEM nucleus instances. 14 of 24 tiles are kept (at most 16 per volume, the ones with the most objects), holding 6-38 nuclei.", + "starts": { + "box_192_1029_11266.h5": [ + [0, 512, 0], + [0, 512, 512], + [32, 512, 0], + [32, 512, 512] + ], + "box_192_1029_12290.h5": [ + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "box_256_1029_11778.h5": [ + [0, 512, 0], + [0, 512, 512], + [32, 512, 0], + [32, 512, 512] + ] + } + }, + "human_organoids_nucleus": { + "crop_shape": [32, 512, 512], + "note": "Nucleus panel (in-domain): the validation part of the four human organoid volumes (the last 10% of the slices, manual instances), which selected the training checkpoint. 64 of 201 tiles are kept (at most 16 per volume, the ones with the most objects), holding 1-6 nuclei.", + "starts": { + "F059.h5": [ + [0, 0, 1024], + [0, 512, 512], + [0, 512, 1024], + [0, 512, 1536], + [0, 1024, 1024], + [0, 1024, 1536], + [32, 0, 1024], + [32, 512, 512], + [32, 512, 1024], + [32, 512, 1536], + [32, 1024, 1024], + [32, 1024, 1536], + [64, 0, 1024], + [64, 512, 1024], + [64, 512, 1536], + [64, 1024, 1024] + ], + "F107_A1.h5": [ + [0, 0, 1024], + [0, 512, 0], + [0, 512, 512], + [0, 512, 1024], + [0, 1024, 512], + [0, 1024, 1024], + [32, 0, 1024], + [32, 512, 0], + [32, 512, 512], + [32, 512, 1024], + [32, 1024, 512], + [32, 1024, 1024], + [64, 0, 1024], + [64, 512, 0], + [64, 512, 512], + [96, 0, 1024] + ], + "F107_A2.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 512, 512], + [32, 1024, 512], + [64, 512, 0], + [64, 512, 512], + [64, 1024, 512], + [96, 0, 0], + [96, 0, 1024], + [96, 512, 512], + [96, 1024, 512], + [128, 0, 0], + [128, 512, 0] + ], + "F107_A3.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [0, 1024, 0], + [0, 1024, 512], + [32, 512, 0], + [32, 512, 512], + [32, 1024, 512], + [64, 512, 0], + [64, 512, 512], + [64, 512, 1024], + [64, 1024, 0], + [64, 1024, 512], + [96, 512, 0], + [96, 512, 512] + ] + } + }, + "placenta_nucleus": { + "crop_shape": [32, 512, 512], + "note": "Nucleus panel (out-of-domain): the two human placenta array tomography volumes (S-BIAD3408, 80 nm sections); manual nuclei of one CTB cell and an STB region, scored only where labelled. 19 of 56 tiles are kept (at most 16 per volume, the ones with the most objects), holding 1-7 nuclei.", + "starts": { + "placenta1.h5": [ + [0, 0, 1024], + [0, 0, 1536], + [0, 512, 1024], + [0, 512, 1536], + [0, 1024, 1024], + [0, 1024, 1536], + [32, 0, 512], + [32, 0, 1024], + [32, 0, 1536], + [32, 512, 512], + [32, 512, 1024], + [32, 512, 1536], + [32, 1024, 1024], + [32, 1024, 1536] + ], + "placenta2.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 512], + [128, 512, 0], + [160, 512, 0] + ] + } + }, + "malecns_nuclei": { + "crop_shape": [32, 512, 512], + "note": "Nucleus panel (out-of-domain): Drosophila male CNS FIB-SEM crops at 16 nm; automatic nucleus segmentation, not filtered for glia or noise. 40 of 40 tiles are kept (at most 16 per volume, the ones with the most objects), holding 5-23 nuclei.", + "starts": { + "brain_1.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "brain_2.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "brain_3.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "brain_4.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "vnc_0.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ] + } + }, + "zheng_ca3_nuclei": { + "crop_shape": [32, 512, 512], + "note": "Nucleus panel (out-of-domain): mouse CA3 ssTEM crops at 72 x 72 x 45 nm; automatic nucleus segmentation. 27 of 40 tiles are kept (at most 16 per volume, the ones with the most objects), holding 3-14 nuclei.", + "starts": { + "crop_1.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 512] + ], + "crop_2.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512] + ], + "crop_3.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512] + ], + "crop_4.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "crop_5.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ] + } + }, + "acson_nuclei": { + "crop_shape": [32, 512, 512], + "note": "Nucleus panel candidate (out-of-domain): rat white matter SBEM crops at 50 nm (DeepACSON); automatic nucleus segmentation. 37 of 40 tiles are kept (at most 16 per volume, the ones with the most objects), holding 1-6 nuclei.", + "starts": { + "crop_1.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 512] + ], + "crop_2.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "crop_3.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "crop_4.h5": [ + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] + ], + "crop_5.h5": [ + [0, 0, 0], + [0, 0, 512], + [0, 512, 0], + [0, 512, 512], + [32, 0, 0], + [32, 0, 512], + [32, 512, 0], + [32, 512, 512] ] } } diff --git a/finetuning/v2/evaluation/evaluate_automatic_baselines.py b/finetuning/v2/evaluation/evaluate_automatic_baselines.py index ef29febaa..8cf337195 100644 --- a/finetuning/v2/evaluation/evaluate_automatic_baselines.py +++ b/finetuning/v2/evaluation/evaluate_automatic_baselines.py @@ -4,9 +4,15 @@ cellpose: CellPose generalist models (cpsam, cpsam_v2, cpdino, cpdino-vitb; cyto3 and nuclei from CellPose 3) stardist: StarDist pretrained (2D_versatile_fluo / 3D_demo) cellsam: CellSAM pipeline (2d only) - cellvit: CellViT++ nucleus segmentation (2d histopathology only) + cellvit: CellViT from the original repository, in the 'cellvit' environment (2d histopathology only) + cellvitpp: CellViT from the CellViT++ code, in this environment (2d histopathology only) microsam_ais: micro-sam v1 automatic instance segmentation (vit_*_em_organelles for the EM organelle sets) microsam_apg: micro-sam v1 automatic prompt generation (vit_*_em_organelles for the EM organelle sets) + (on histopathology, the PathoSAM models pad images smaller than 512 x 512 to that shape, as PathoSAM does) + sam_amg: SAM automatic mask generation with vit_h (2d only) + sam2_amg: SAM2 automatic mask generation with the original SAM2 weights (2d only) + sam3: SAM3.1 with the text prompt 'cell', for nuclei too (2d only) + instanseg: InstanSeg pretrained (2d fluorescence and histopathology only) segneuron: SegNeuron (3d EM only) focus3d: FOCUS-3D (3d LM only) mitonet: MitoNet from empanada (EM mitochondria, 2d and 3d) @@ -20,6 +26,10 @@ python evaluate_automatic_baselines.py -d livecell -e --method cellsam python evaluate_automatic_baselines.py -d pannuke -e --method cellvit -m sam_h_x40 python evaluate_automatic_baselines.py -d embedseg -e --method microsam_ais -m vit_b + python evaluate_automatic_baselines.py -d livecell -e --method sam_amg -m vit_b + python evaluate_automatic_baselines.py -d livecell -e --method sam2_amg + python evaluate_automatic_baselines.py -d livecell -e --method sam3 + python evaluate_automatic_baselines.py -d dsb -e --method instanseg python evaluate_automatic_baselines.py -d cremi -e --method segneuron python evaluate_automatic_baselines.py -d gonuclear -e --method focus3d -m nuclei python evaluate_automatic_baselines.py -d lucchi -e --method mitonet @@ -27,31 +37,134 @@ """ import os +import sys import json import warnings import argparse import numpy as np +from skimage.segmentation import relabel_sequential import torch -from elf.segmentation import features, multicut, watershed +# The original CellViT runs in the 'cellvit' environment, which has none of the packages imported below. In it, this +# script runs as the CellViT worker ('--cellvit_worker', see 'CellViTProcess'), with numpy, torch and the CellViT +# repository only. +CELLVIT_ORIGINAL_ROOT = os.path.expanduser("~/CellViT") -from common import ( - CROP_SHAPE_3D, DATA_ROOT, DATASETS_2D, DATASETS_3D, DATASETS_3D_LM, DATASETS_EM, DATASETS_EM_MITO, - DATASETS_EM_NUCLEUS, DATASETS_HP, EVAL_CROPS, GT_MIN_SIZE_2D, check_data_download, drop_severed_objects, - ensure_8bit_range, evaluate_samples, load_data, n_samples, + +def _load_cellvit_original(checkpoint, device): + """Load the CellViT model of the checkpoint, as the inference scripts of the repository do.""" + sys.path.insert(0, CELLVIT_ORIGINAL_ROOT) + from utils.tools import unflatten_dict + from models.segmentation.cell_segmentation.cellvit import CellViT, CellViT256, CellViTSAM + + state = torch.load(checkpoint, map_location="cpu") + run_conf = unflatten_dict(state["config"], ".") + data_conf, model_conf = run_conf["data"], run_conf["model"] + classes = { + "num_nuclei_classes": data_conf["num_nuclei_classes"], "num_tissue_classes": data_conf["num_tissue_classes"], + } + regression_loss = model_conf.get("regression_loss", False) + if state["arch"] == "CellViTSAM": + model = CellViTSAM( + model_path=None, vit_structure=model_conf["backbone"], regression_loss=regression_loss, **classes, + ) + elif state["arch"] == "CellViT256": + model = CellViT256(model256_path=None, regression_loss=regression_loss, **classes) + elif state["arch"] == "CellViT": + model = CellViT( + embed_dim=model_conf["embed_dim"], input_channels=model_conf.get("input_channels", 3), + depth=model_conf["depth"], num_heads=model_conf["num_heads"], extract_layers=model_conf["extract_layers"], + regression_loss=regression_loss, **classes, + ) + else: + raise ValueError(f"Unsupported CellViT architecture '{state['arch']}' in '{checkpoint}'.") + model.load_state_dict(state["model_state_dict"]) + return model.eval().to(device), run_conf + + +def _segment_cellvit_original(image, model, run_conf, magnification, device): + """Segment one RGB patch of 8-bit values with the original CellViT. + + The encoder adds a square slice of its positional embedding to the tokens, so the patch is padded up to a square + with a side that is a multiple of the token size. + """ + if image.ndim == 2: + image = np.stack([image] * 3, axis=-1) + image = image[..., :3] + extent = max(image.shape[:2]) + extent += -extent % model.patch_size + padding = [(0, extent - size) for size in image.shape[:2]] + padded = np.pad(image, padding + [(0, 0)], mode="reflect") + + normalization = run_conf.get("transformations", {}).get("normalize", {}) + mean = torch.tensor(normalization.get("mean", (0.5, 0.5, 0.5))).view(3, 1, 1) + std = torch.tensor(normalization.get("std", (0.5, 0.5, 0.5))).view(3, 1, 1) + inputs = torch.from_numpy(np.ascontiguousarray(padded)).float().permute(2, 0, 1) / 255.0 + inputs = ((inputs - mean) / std)[None].to(device) + + with torch.no_grad(): + predictions = model(inputs) + predictions["nuclei_binary_map"] = torch.softmax(predictions["nuclei_binary_map"], dim=1) + predictions["nuclei_type_map"] = torch.softmax(predictions["nuclei_type_map"], dim=1) + instances, _ = model.calculate_instance_map(predictions, magnification=magnification) + return instances[0].numpy()[:image.shape[0], :image.shape[1]].astype("uint32") + + +def _cellvit_worker(argv): + """Segment the images that 'CellViTProcess' sends, one path per line on stdin. + + For each image the worker writes the instance segmentation next to it and answers with the path of that file. + """ + parser = argparse.ArgumentParser(prog="evaluate_automatic_baselines.py --cellvit_worker") + parser.add_argument("--checkpoint", required=True, help="The CellViT checkpoint.") + parser.add_argument("--magnification", type=int, default=40, choices=(20, 40), help="The magnification.") + args = parser.parse_args(argv) + + device = "cuda" if torch.cuda.is_available() else "cpu" + model, run_conf = _load_cellvit_original(args.checkpoint, device) + print("ready", flush=True) + for line in sys.stdin: + input_path = line.strip() + output_path = f"{input_path[:-4]}_seg.npy" + seg = _segment_cellvit_original(np.load(input_path), model, run_conf, args.magnification, device) + np.save(output_path, seg) + print(output_path, flush=True) + + +if __name__ == "__main__" and sys.argv[1:2] == ["--cellvit_worker"]: + _cellvit_worker(sys.argv[2:]) + sys.exit(0) + +from elf.segmentation import features, multicut, watershed # noqa: E402 + +from common import ( # noqa: E402 + pad_to_min_shape, + CROP_SHAPE_3D, DATA_ROOT, DATASETS_2D, DATASETS_3D, DATASETS_3D_LM, DATASETS_EM, DATASETS_HP, EVAL_CROPS, + DATASETS_2D_LM_CELL_ID, DATASETS_2D_LM_CELL_SUPPLEMENTARY, DATASETS_2D_LM_CELL_OOD, + DATASETS_2D_LM_NUCLEUS_ID, DATASETS_2D_LM_NUCLEUS_SUPPLEMENTARY, DATASETS_2D_LM_NUCLEUS_OOD, + DATASETS_2D_LM_NUCLEUS_HELD_OUT_PLATFORM, DATASETS_EM_MITO, DATASETS_EM_NUCLEUS, GT_MIN_SIZE_2D, + check_data_download, drop_severed_objects, ensure_8bit_range, evaluate_samples, load_data, n_samples, ) LM_DATASETS = set(DATASETS_2D + DATASETS_3D_LM) EM_DATASETS = set(DATASETS_EM) HP_DATASETS = set(DATASETS_HP) -EM_ORGANELLE_DATASETS = set(DATASETS_EM_MITO + DATASETS_EM_NUCLEUS) METHODS = [ - "cellpose", "stardist", "cellsam", "cellvit", "microsam_ais", "microsam_apg", "segneuron", "focus3d", "mitonet", - "nucleonet", + "cellpose", "stardist", "cellsam", "cellvit", "cellvitpp", "microsam_ais", "microsam_apg", "sam_amg", "sam2_amg", + "sam3", "instanseg", "segneuron", "focus3d", "mitonet", "nucleonet", ] +# The 2d datasets whose objects are nuclei. Histopathology has nuclei only. +NUCLEUS_DATASETS = set( + DATASETS_2D_LM_NUCLEUS_ID + DATASETS_2D_LM_NUCLEUS_SUPPLEMENTARY + DATASETS_2D_LM_NUCLEUS_OOD + + DATASETS_2D_LM_NUCLEUS_HELD_OUT_PLATFORM + DATASETS_HP +) +FLUORESCENCE_CELL_DATASETS = set(DATASETS_2D_LM_CELL_ID + DATASETS_2D_LM_CELL_SUPPLEMENTARY + DATASETS_2D_LM_CELL_OOD) + +EM_ORGANELLE_DATASETS = set(DATASETS_EM_MITO + DATASETS_EM_NUCLEUS) + SEGNEURON_ROOT = "/mnt/vast-nhr/home/archit/u12090/SegNeuron" SEGNEURON_CHECKPOINT = "/mnt/vast-nhr/projects/cidas/cca/models/segneuron/SegNeuronModel.ckpt" @@ -71,12 +184,18 @@ # The IoU that links the masks of adjacent slices of a volume. It is the default of `cellpose.utils.stitch3D`. # The default of `eval` is 0, which leaves the slices unlinked. CELLPOSE_STITCH_THRESHOLD = 0.25 +# How CellPose segments a volume: 'stitch' segments every z-slice in 2d and links the masks of adjacent slices +# (CELLPOSE_STITCH_THRESHOLD); 'do3d' predicts the flows in the xy, xz and yz views and combines them into 3d flows. +# Both get the volume as acquired, without a correction for its anisotropy. The 'do3d' results get the suffix +# '_do3d'. +CELLPOSE_3D_MODES = ("stitch", "do3d") # The channels of the whole-cell and the nuclear marker in the loaded channel-last image, for the baselines that # name their input channels. The other datasets are passed as loaded. MARKER_CHANNELS = { "tissuenet": {"cell": 0, "nucleus": 1}, "cvz_fluo_cell": {"cell": 0, "nucleus": 1}, "covid_if_cells": {"cell": 0, "nucleus": 1}, "pan_multiplex": {"cell": 0, "nucleus": 1}, + "mif_tonsil_cells": {"cell": 0, "nucleus": 1}, } # The marker in a grayscale image, for CellSAM. CellSAM returns a nuclear segmentation if only its nuclear slot is @@ -99,20 +218,45 @@ # Histopathology uses the model trained on H&E, which reads the RGB image directly. STARDIST_2D_HP_MODEL = "2D_versatile_he" STARDIST_3D_MODEL = "3D_demo" +# How StarDist segments a volume: '3d' runs its only pretrained 3d model (STARDIST_3D_MODEL, the model of its 3d +# example, trained on one demo volume); 'stitch' runs STARDIST_2D_MODEL on every z-slice and links the masks of +# adjacent slices, as the CellPose 'stitch' mode (CELLPOSE_STITCH_THRESHOLD). The 'stitch' results are named after +# the 2d model, e.g. 'stardist_2D_versatile_fluo_stitch'. +STARDIST_3D_MODES = ("3d", "stitch") # micro-sam v1 model types SAM_V1_MODEL_TYPE = "vit_b_lm" +SAM_V1_HP_MODEL_TYPE = "vit_b_histopathology" + +# The original SAM and SAM2 weights for automatic mask generation. +SAM_AMG_MODEL_TYPE = "vit_h" +# The smallest input of PathoSAM, the micro-sam histopathology models: their training patch shape. +PATHOSAM_MIN_SHAPE = (512, 512) +SAM2_AMG_MODEL_TYPE = "hvit_b" + +SAM3_VERSION = "sam3.1" +SAM3_CONFIDENCE_THRESHOLD = 0.5 + +# The InstanSeg model per modality. InstanSeg has no model for label-free images. +INSTANSEG_HP_MODEL = "brightfield_nuclei" +INSTANSEG_FLUORESCENCE_MODEL = "fluorescence_nuclei_and_cells" SAM_V1_EM_ORGANELLE_MODEL_TYPE = "vit_b_em_organelles" # CellViT++ ships a whole-slide pipeline that needs pathopatch, ray and openslide. The patch-level model and its # post-processor are imported from the checkout instead, which only adds einops to the environment. CELLVIT_ROOT = os.path.expanduser("~/CellViT-plus-plus") CELLVIT_CHECKPOINT_ROOT = "/mnt/vast-nhr/projects/cidas/cca/models/cellvit" +# The original CellViT runs in its own environment, with this script as its worker (see '_cellvit_worker'). +CELLVIT_PYTHON = "/mnt/lustre-grete/usr/u12090/micromamba/envs/cellvit/bin/python" CELLVIT_CHECKPOINTS = { "sam_h_x40": "CellViT-SAM-H-x40.pth", "sam_h_x20": "CellViT-SAM-H-x20.pth", "vit256_x40": "CellViT-256-x40.pth", "vit256_x20": "CellViT-256-x20.pth", + # The CellViT++ segmentation models: CellViT retrained on PanNuke with mixed precision, and a Virchow backbone. + "sam_h_x40_amp": "cellvit_plus_plus/CellViT-SAM-H-x40-AMP.pth", + "vit256_x40_amp": "cellvit_plus_plus/CellViT-256-x40-AMP.pth", + "virchow_x40_amp": "cellvit_plus_plus/CellViT-Virchow-x40-AMP.pth", } # MitoNet and NucleoNet run from the empanada-napari checkout, which contains the empanada package. @@ -135,17 +279,6 @@ # A downsampling replaces the default of 1 only if it scores this much higher, since a few crops are a noisy choice. EMPANADA_MIN_GAIN = 0.02 -# The z/xy anisotropy of each dataset (z_voxel / xy_voxel). FOCUS-3D takes it as its z_ratio. -DATASET_ANISOTROPY = { - "embedseg_mouse_skull": 4.0, - "embedseg_organoid": 6.0, - "embedseg_platy_nuclei": 5.0, - "blastospim": 10.0, - "mouse_embryo": 4.0, - "cremi": 10.0, # z=40nm, xy=4nm - "snemi": 5.0, # z=30nm, xy=6nm -} - def _load_cellpose(model_type, device): """Load a CellPose model, supporting both the CellPose 3 and the CellPose 4 APIs. @@ -170,15 +303,18 @@ def _load_cellpose(model_type, device): return models.CellposeModel(gpu=use_gpu, model_type=model_type) -def _load_stardist(ndim, dataset_name): +def _load_stardist(ndim, dataset_name, mode_3d="3d"): + # TensorFlow takes all GPU memory by default, which starves the other workers on the same GPU. + os.environ.setdefault("TF_FORCE_GPU_ALLOW_GROWTH", "true") from stardist.models import StarDist2D, StarDist3D + if ndim == 3 and mode_3d == "stitch": + return StarDist2D.from_pretrained(STARDIST_2D_MODEL) if ndim == 3: return StarDist3D.from_pretrained(STARDIST_3D_MODEL) return StarDist2D.from_pretrained(STARDIST_2D_HP_MODEL if dataset_name in DATASETS_HP else STARDIST_2D_MODEL) def _load_segneuron(checkpoint_path, device): - import sys sys.path.insert(0, os.path.join(SEGNEURON_ROOT, "Train_and_Inference")) from collections import OrderedDict import torch @@ -206,7 +342,6 @@ def _stub_cupy(): Only the GPU variants of its helpers use cupy, and the post-processing this runs calls the numpy ones. """ - import sys import types try: import cupy # noqa @@ -224,7 +359,6 @@ def _stub_cupy(): def _load_cellvit(model_type, checkpoint, magnification, device): """Load a CellViT model and the post-processor that turns its branches into instances.""" - import sys if CELLVIT_ROOT not in sys.path: sys.path.insert(0, CELLVIT_ROOT) _stub_cupy() @@ -232,6 +366,7 @@ def _load_cellvit(model_type, checkpoint, magnification, device): from cellvit.models.cell_segmentation.cellvit import CellViT from cellvit.models.cell_segmentation.cellvit_256 import CellViT256 from cellvit.models.cell_segmentation.cellvit_sam import CellViTSAM + from cellvit.models.cell_segmentation.cellvit_virchow import CellViTVirchow from cellvit.models.cell_segmentation.postprocessing import DetectionCellPostProcessor from cellvit.utils.tools import unflatten_dict @@ -260,6 +395,12 @@ def _load_cellvit(model_type, checkpoint, magnification, device): num_tissue_classes=run_conf["data"]["num_tissue_classes"], regression_loss=model_conf.get("regression_loss", False), ) + elif architecture == "CellViTVirchow": + model = CellViTVirchow( + model_virchow_path=None, + num_nuclei_classes=run_conf["data"]["num_nuclei_classes"], + num_tissue_classes=run_conf["data"]["num_tissue_classes"], + ) elif architecture == "CellViT": model = CellViT( num_nuclei_classes=run_conf["data"]["num_nuclei_classes"], @@ -331,13 +472,33 @@ def _load_empanada(method, ndim, device, inference_scale): def _load_microsam_v1(method, model_type, checkpoint, device): from micro_sam.v1.automatic_segmentation import get_predictor_and_segmenter - mode = {"microsam_ais": "ais", "microsam_apg": "apg"}[method] + mode = {"microsam_ais": "ais", "microsam_apg": "apg", "sam_amg": "amg"}[method] return get_predictor_and_segmenter( model_type=model_type, checkpoint=checkpoint, device=device, segmentation_mode=mode, ) -def _segment_cellpose(image_or_volume, model, ndim, dataset_name): +def _load_sam2_amg(model_type, checkpoint, device): + from micro_sam.v2.automatic_segmentation import get_predictor_and_segmenter + return get_predictor_and_segmenter( + model_type=model_type, checkpoint=checkpoint, device=device, segmentation_mode="amg", + ) + + +def _load_sam3(checkpoint, device): + from micro_sam3.util import get_sam3_model + return get_sam3_model( + checkpoint_path=checkpoint, device=device, input_type="image", version=SAM3_VERSION, + confidence_threshold=SAM3_CONFIDENCE_THRESHOLD, + ) + + +def _load_instanseg(model_type): + from instanseg import InstanSeg + return InstanSeg(model_type, verbosity=0) + + +def _segment_cellpose(image_or_volume, model, ndim, dataset_name, mode_3d="stitch"): markers = MARKER_CHANNELS.get(dataset_name) if type(model).__name__ == "Cellpose": # CellPose 3 names the channel to segment and the nuclear one, 1-based with 0 for grayscale. @@ -350,19 +511,65 @@ def _segment_cellpose(image_or_volume, model, ndim, dataset_name): if ndim == 3: # CellPose 3 cannot estimate the diameter of a volume, so pass the mean diameter, its own fallback. diameter = model.diam_mean if type(model).__name__ == "Cellpose" else None - masks = model.eval( - image_or_volume, diameter=diameter, stitch_threshold=CELLPOSE_STITCH_THRESHOLD, z_axis=0, **channel_kwargs, - )[0] + if mode_3d == "do3d": + masks = model.eval( + image_or_volume, diameter=diameter, do_3D=True, z_axis=0, **channel_kwargs, + )[0] + else: + masks = model.eval( + image_or_volume, diameter=diameter, stitch_threshold=CELLPOSE_STITCH_THRESHOLD, z_axis=0, + **channel_kwargs, + )[0] else: masks = model.eval(image_or_volume, diameter=None, **channel_kwargs)[0] return masks.astype("uint32") -def _segment_stardist(image_or_volume, model, ndim): +def _stitch_slices(masks, threshold=CELLPOSE_STITCH_THRESHOLD): + """Link the 2d masks of adjacent z-slices into 3d objects, as `cellpose.utils.stitch3D`. + + A mask takes the label of the mask in the slice above with which it has the highest IoU, if that IoU is at least + 'threshold' and the mask above has no better match in this slice. All other masks get new labels. + """ + stitched = np.zeros(masks.shape, dtype="uint32") + next_label = 1 + for z, current in enumerate(masks): + ids = np.unique(current[current > 0]) + mapping = {} + if z > 0 and len(ids): + above = stitched[z - 1] + above_ids = np.unique(above[above > 0]) + if len(above_ids): + both = (current > 0) & (above > 0) + pairs, overlap = np.unique(np.stack([current[both], above[both]]), axis=1, return_counts=True) + rows, cols = np.searchsorted(ids, pairs[0]), np.searchsorted(above_ids, pairs[1]) + areas = np.bincount(current.ravel())[ids], np.bincount(above.ravel())[above_ids] + iou = np.zeros((len(ids), len(above_ids))) + iou[rows, cols] = overlap / (areas[0][rows] + areas[1][cols] - overlap) + iou[iou < threshold] = 0 + iou[iou < iou.max(axis=0)] = 0 + for row in np.nonzero(iou.max(axis=1) > 0)[0]: + mapping[ids[row]] = above_ids[iou[row].argmax()] + for i in ids: + if i not in mapping: + mapping[i] = next_label + next_label += 1 + lookup = np.zeros(int(current.max()) + 1, dtype="uint32") + lookup[list(mapping)] = list(mapping.values()) + stitched[z] = lookup[current] + next_label = max(next_label, int(stitched[z].max()) + 1) + return stitched + + +def _segment_stardist(image_or_volume, model, ndim, mode_3d="3d"): from csbdeep.utils import normalize as csbdeep_normalize if ndim == 2 and image_or_volume.ndim == 3 and model.config.n_channel_in == 1: image_or_volume = image_or_volume.mean(axis=-1) inp = csbdeep_normalize(image_or_volume, 1.0, 99.8) + if ndim == 3 and mode_3d == "stitch": + # The volume is normalized as a whole, so that all slices share the intensity scale. + masks = np.stack([model.predict_instances(plane, scale=1)[0] for plane in inp]) + return _stitch_slices(masks) seg, _ = model.predict_instances(inp) if ndim == 3 else model.predict_instances(inp, scale=1) return seg.astype("uint32") @@ -406,6 +613,13 @@ def _segment_cellvit(image, model, processor, run_conf, device): # patch is padded up to a square, and that square to a multiple of the token size. extent = max(image.shape[:2]) extent += -extent % model.patch_size + # The Virchow backbone only reads the input sizes it rescales to its token grid, so pad up to the next one. + allowed = sorted(getattr(model, "input_rescale_dict", {})) + if allowed: + larger = [size for size in allowed if size >= max(image.shape[:2])] + if not larger: + raise ValueError(f"CellViT-Virchow reads at most {allowed[-1]} pixels per side, got {image.shape[:2]}.") + extent = larger[0] padding = [(0, extent - size) for size in image.shape[:2]] padded = np.pad(image, padding + [(0, 0)], mode="reflect") @@ -504,13 +718,24 @@ def _segment_focus3d(volume, segmenter, dataset_name, cell_radius, z_ratio): """Segment one volume with FOCUS-3D, which rescales it to its own scale first. `z_ratio` is the z-to-xy spacing ratio. `cell_radius` is the xy radius that the rescaling starts from. - The two are coupled: at a z_ratio of 1, FOCUS-3D rescales the z axis together with xy. + The two are coupled: at a z_ratio of 1, FOCUS-3D rescales the z axis together with xy. Without a z_ratio, + the volume is taken as acquired, with a z_ratio of 1. """ if z_ratio is None: - z_ratio = DATASET_ANISOTROPY.get(dataset_name, 1.0) + z_ratio = 1.0 return segmenter(volume, z_ratio=z_ratio, cell_radius=cell_radius) +def _segment_microsam_v1_padded(image, predictor, segmenter): + """Segment a 2d image smaller than PATHOSAM_MIN_SHAPE padded to that shape, and crop the segmentation back. + + PathoSAM trains on 512 x 512 patches, and its own evaluation pads the 256 x 256 PanNuke tiles to that shape. + """ + padded, crop = pad_to_min_shape(image, PATHOSAM_MIN_SHAPE) + seg = _segment_microsam_v1(padded, predictor, segmenter, 2)[crop] + return relabel_sequential(seg)[0].astype("uint32") + + def _segment_empanada(image_or_volume, inference, engine, config, ndim): """Segment one image or volume with MitoNet or NucleoNet.""" from skimage.segmentation import relabel_sequential @@ -540,9 +765,88 @@ def _segment_microsam_v1(image_or_volume, predictor, segmenter, ndim): return seg.astype("uint32") if seg is not None else np.zeros(image_or_volume.shape, dtype="uint32") +def _segment_sam2_amg(image, predictor, segmenter): + from micro_sam.v2.automatic_segmentation import automatic_instance_segmentation + seg = automatic_instance_segmentation( + predictor=predictor, segmenter=segmenter, input_path=image, ndim=2, verbose=False, + ) + return seg.astype("uint32") if seg is not None else np.zeros(image.shape[:2], dtype="uint32") + + +def _segment_sam3(image, processor, prompt): + """Segment all objects of the text prompt, and write the masks from low to high score. + + SAM3 returns overlapping masks, so a mask with a higher score wins the overlap. + """ + from PIL import Image + from micro_sam.util import _to_image + + state = processor.set_image(Image.fromarray(_to_image(image))) + processor.reset_all_prompts(state) + output = processor.set_text_prompt(state=state, prompt=prompt) + + seg = np.zeros(image.shape[:2], dtype="uint32") + if len(output["masks"]) == 0: + return seg + masks = output["masks"].squeeze(1).cpu().numpy() + for label, index in enumerate(torch.argsort(output["scores"]).cpu().numpy(), start=1): + seg[masks[index]] = label + return seg + + +def _segment_instanseg(image, model, target): + # InstanSeg does not read one-channel images, so a grayscale image is passed as three identical channels. + if image.ndim == 2: + image = np.stack([image] * 3, axis=-1) + labels, _ = model.eval_small_image(image=image, target=target) + return labels.squeeze().cpu().numpy().astype("uint32") + + +class CellViTProcess: + """The original CellViT in the 'cellvit' environment, which segments the images that this process sends it.""" + + def __init__(self, checkpoint, magnification): + import tempfile + import subprocess + self.folder = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + # The PYTHONPATH of this environment would put its packages before the ones of the older environment. + env = {key: value for key, value in os.environ.items() if key != "PYTHONPATH"} + self.process = subprocess.Popen( + [CELLVIT_PYTHON, os.path.abspath(__file__), "--cellvit_worker", "--checkpoint", checkpoint, + "--magnification", str(magnification)], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, env=env, + ) + self._read_until(lambda line: line == "ready") + self.count = 0 + + def _read_until(self, condition): + for line in self.process.stdout: + if condition(line.strip()): + return line.strip() + raise RuntimeError(f"The CellViT worker stopped with the exit code {self.process.wait()}.") + + def __call__(self, image): + input_path = os.path.join(self.folder, f"{self.count}.npy") + self.count += 1 + np.save(input_path, image) + self.process.stdin.write(input_path + "\n") + self.process.stdin.flush() + output_path = self._read_until(lambda line: line.endswith("_seg.npy")) + seg = np.load(output_path) + os.remove(input_path) + os.remove(output_path) + return seg.astype("uint32") + + def close(self): + import shutil + self.process.stdin.close() + self.process.wait() + shutil.rmtree(self.folder, ignore_errors=True) + + def _run_evaluation( segment_fn, dataset_name, data_root, save_path, desc, limit, sample_index, crop_shape=None, normalize=True, - extra_columns=None, + shard=None, extra_columns=None, ): if limit is not None: # Name the file after the truncation, so it cannot pass for the full evaluation. save_path = f"{save_path[:-4]}_n{limit}.csv" @@ -551,38 +855,38 @@ def _run_evaluation( return evaluate_samples( segment_fn, dataset_name, data_root, save_path, desc, limit=limit, crop_shape=crop_shape, - sample_index=sample_index, normalize=normalize, extra_columns=extra_columns, + sample_index=sample_index, normalize=normalize, shard=shard, extra_columns=extra_columns, ) -def run_cellpose_evaluation(dataset_name, data_root, experiment_folder, model_type, device, limit, sample_index): - if model_type in CELLPOSE3_MODELS and dataset_name in DATASETS_HP: - warnings.warn( - f"{model_type} is not a histopathology model and does not run on '{dataset_name}'. Skipping.", - UserWarning, stacklevel=2, - ) - return - +def run_cellpose_evaluation( + dataset_name, data_root, experiment_folder, model_type, device, limit, sample_index, shard=None, mode_3d="stitch", +): ndim = 3 if dataset_name in DATASETS_3D else 2 model = _load_cellpose(model_type, device) - save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_cellpose_{model_type}.csv") + suffix = "_do3d" if ndim == 3 and mode_3d == "do3d" else "" + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_cellpose_{model_type}{suffix}.csv") _run_evaluation( - lambda x: _segment_cellpose(x, model, ndim, dataset_name), + lambda x: _segment_cellpose(x, model, ndim, dataset_name, mode_3d), dataset_name, data_root, save_path, desc=f"cellpose-{model_type}", limit=limit, sample_index=sample_index, + shard=shard, ) -def run_stardist_evaluation(dataset_name, data_root, experiment_folder, limit, sample_index): +def run_stardist_evaluation( + dataset_name, data_root, experiment_folder, limit, sample_index, shard=None, mode_3d="3d", +): ndim = 3 if dataset_name in DATASETS_3D else 2 - model = _load_stardist(ndim, dataset_name) - save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_stardist.csv") + model = _load_stardist(ndim, dataset_name, mode_3d) + name = f"stardist_{STARDIST_2D_MODEL}_stitch" if ndim == 3 and mode_3d == "stitch" else "stardist" + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_{name}.csv") _run_evaluation( - lambda x: _segment_stardist(x, model, ndim), - dataset_name, data_root, save_path, desc="stardist", limit=limit, sample_index=sample_index, + lambda x: _segment_stardist(x, model, ndim, mode_3d), + dataset_name, data_root, save_path, desc="stardist", limit=limit, sample_index=sample_index, shard=shard, ) -def run_cellsam_evaluation(dataset_name, data_root, experiment_folder, limit, sample_index): +def run_cellsam_evaluation(dataset_name, data_root, experiment_folder, limit, sample_index, shard=None): if dataset_name in DATASETS_3D: warnings.warn( f"CellSAM is 2D-only and does not support 3D dataset '{dataset_name}'. Skipping.", @@ -593,12 +897,41 @@ def run_cellsam_evaluation(dataset_name, data_root, experiment_folder, limit, sa save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_cellsam.csv") _run_evaluation( lambda x: _segment_cellsam(x, dataset_name), - dataset_name, data_root, save_path, desc="cellsam", limit=limit, sample_index=sample_index, + dataset_name, data_root, save_path, desc="cellsam", limit=limit, sample_index=sample_index, shard=shard, ) def run_cellvit_evaluation( - dataset_name, data_root, experiment_folder, model_type, checkpoint, device, limit, sample_index, + dataset_name, data_root, experiment_folder, model_type, checkpoint, limit, sample_index, shard=None, +): + if dataset_name not in HP_DATASETS: + warnings.warn( + f"CellViT is a histopathology nucleus model and does not support dataset '{dataset_name}'. Skipping.", + UserWarning, stacklevel=2, + ) + return + + if checkpoint is None: + checkpoint = os.path.join(CELLVIT_CHECKPOINT_ROOT, CELLVIT_CHECKPOINTS[model_type]) + magnification = 20 if model_type.endswith("_x20") else 40 + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_cellvit_{model_type}.csv") + if os.path.exists(save_path): + print(f"Results already stored at '{save_path}'.") + return + + # CellViT reads the stored 8-bit RGB, as in `run_cellvitpp_evaluation`. + process = CellViTProcess(checkpoint, magnification) + try: + _run_evaluation( + process, dataset_name, data_root, save_path, desc=f"cellvit-{model_type}", limit=limit, + sample_index=sample_index, normalize=False, shard=shard, + ) + finally: + process.close() + + +def run_cellvitpp_evaluation( + dataset_name, data_root, experiment_folder, model_type, checkpoint, device, limit, sample_index, shard=None, ): if dataset_name not in HP_DATASETS: warnings.warn( @@ -611,16 +944,16 @@ def run_cellvit_evaluation( # channel by channel, so this is the one baseline scored on the stored 8-bit RGB. magnification = 20 if model_type.endswith("_x20") else 40 model, processor, run_conf = _load_cellvit(model_type, checkpoint, magnification, device) - save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_cellvit_{model_type}.csv") + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_cellvitpp_{model_type}.csv") _run_evaluation( lambda x: _segment_cellvit(x, model, processor, run_conf, device), - dataset_name, data_root, save_path, desc=f"cellvit-{model_type}", limit=limit, - sample_index=sample_index, normalize=False, + dataset_name, data_root, save_path, desc=f"cellvitpp-{model_type}", limit=limit, + sample_index=sample_index, normalize=False, shard=shard, ) def run_segneuron_evaluation( - dataset_name, data_root, experiment_folder, device, limit, sample_index, checkpoint_path=None, + dataset_name, data_root, experiment_folder, device, limit, sample_index, checkpoint_path=None, shard=None, ): if dataset_name not in EM_DATASETS: warnings.warn( @@ -636,13 +969,13 @@ def run_segneuron_evaluation( save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_segneuron.csv") _run_evaluation( lambda x: _segment_segneuron(x, model, device), - dataset_name, data_root, save_path, desc="segneuron", limit=limit, sample_index=sample_index, + dataset_name, data_root, save_path, desc="segneuron", limit=limit, sample_index=sample_index, shard=shard, ) def run_focus3d_evaluation( dataset_name, data_root, experiment_folder, model_type, checkpoint, device, cell_radius, limit, sample_index, - z_ratio=None, crop_shape=None, batch_size=FOCUS3D_BATCH_SIZE, num_workers=FOCUS3D_NUM_WORKERS, + z_ratio=None, crop_shape=None, batch_size=FOCUS3D_BATCH_SIZE, num_workers=FOCUS3D_NUM_WORKERS, shard=None, ): if dataset_name not in DATASETS_3D_LM: warnings.warn( @@ -652,10 +985,11 @@ def run_focus3d_evaluation( return segmenter = _load_focus3d(model_type, checkpoint, device, batch_size, num_workers) - if crop_shape is None and dataset_name not in EVAL_CROPS: - # The harness default of 8 slices would leave most of the model's 32-deep window as padding. - crop_shape = (segmenter.patch_size[0],) + tuple(CROP_SHAPE_3D[1:]) - depth = crop_shape[0] if crop_shape else EVAL_CROPS[dataset_name][0][0] + # FOCUS-3D is scored on the crops of all other methods, also where they are shallower than its window. + if crop_shape is not None: + depth = crop_shape[0] + else: + depth = EVAL_CROPS[dataset_name][0][0] if dataset_name in EVAL_CROPS else CROP_SHAPE_3D[0] print(f"FOCUS-3D has the patch size {segmenter.patch_size} and runs on crops of depth {depth}.") # The crop depth is in the name: a baseline scored on another crop is not comparable. @@ -663,7 +997,7 @@ def run_focus3d_evaluation( _run_evaluation( lambda x: _segment_focus3d(x, segmenter, dataset_name, cell_radius, z_ratio), dataset_name, data_root, save_path, desc=f"focus3d-{model_type}", limit=limit, sample_index=sample_index, - crop_shape=crop_shape, + crop_shape=crop_shape, shard=shard, ) @@ -729,20 +1063,89 @@ def run_empanada_evaluation(dataset_name, data_root, experiment_folder, method, def run_microsam_v1_evaluation( - dataset_name, data_root, experiment_folder, method, model_type, checkpoint, device, limit, sample_index, + dataset_name, data_root, experiment_folder, method, model_type, checkpoint, device, limit, sample_index, shard=None, ): - # Of the EM data, the v1 models only cover mitochondria and nuclei, with the organelle models. - if dataset_name in EM_DATASETS and not (dataset_name in EM_ORGANELLE_DATASETS and "em_organelles" in model_type): + # Of the EM data, micro-sam v1 AIS and APG only cover mitochondria and nuclei, with the organelle models. + em_organelle_model = dataset_name in EM_ORGANELLE_DATASETS and "em_organelles" in (model_type or "") + if method != "sam_amg" and dataset_name in EM_DATASETS and not em_organelle_model: raise ValueError( - f"micro-sam v1 runs on EM only with the 'vit_*_em_organelles' models, and only on the mitochondria and " - f"nucleus datasets. Got '{model_type}' on '{dataset_name}'." + f"micro-sam v1 AIS and APG run on EM only with the 'vit_*_em_organelles' models, and only on the " + f"mitochondria and nucleus datasets. Got '{model_type}' on '{dataset_name}'." ) ndim = 3 if dataset_name in DATASETS_3D else 2 + if method == "sam_amg" and ndim == 3: + warnings.warn(f"SAM AMG is 2d only and does not run on '{dataset_name}'. Skipping.", UserWarning, stacklevel=2) + return + if model_type is None: + if method == "sam_amg": + model_type = SAM_AMG_MODEL_TYPE + else: + model_type = SAM_V1_HP_MODEL_TYPE if dataset_name in HP_DATASETS else SAM_V1_MODEL_TYPE + predictor, segmenter = _load_microsam_v1(method, model_type, checkpoint, device) save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_{method}_{model_type}.csv") _run_evaluation( - lambda x: _segment_microsam_v1(x, predictor, segmenter, ndim), - dataset_name, data_root, save_path, desc=method, limit=limit, sample_index=sample_index, + # PathoSAM only: SAM AMG runs through this function too and keeps the image as it is. + (lambda x: _segment_microsam_v1_padded(x, predictor, segmenter)) + if dataset_name in HP_DATASETS and method != "sam_amg" else + (lambda x: _segment_microsam_v1(x, predictor, segmenter, ndim)), + dataset_name, data_root, save_path, desc=method, limit=limit, sample_index=sample_index, shard=shard, + ) + + +def run_sam2_amg_evaluation( + dataset_name, data_root, experiment_folder, model_type, checkpoint, device, limit, sample_index, shard=None, +): + if dataset_name in DATASETS_3D: + warnings.warn(f"SAM2 AMG is 2d only and does not run on '{dataset_name}'. Skipping.", UserWarning, stacklevel=2) + return + + predictor, segmenter = _load_sam2_amg(model_type, checkpoint, device) + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_sam2_amg_{model_type}.csv") + _run_evaluation( + lambda x: _segment_sam2_amg(x, predictor, segmenter), + dataset_name, data_root, save_path, desc="sam2_amg", limit=limit, sample_index=sample_index, shard=shard, + ) + + +def run_sam3_evaluation( + dataset_name, data_root, experiment_folder, checkpoint, device, limit, sample_index, shard=None, +): + if dataset_name in DATASETS_3D: + warnings.warn(f"SAM3 runs here on 2d only, not on '{dataset_name}'. Skipping.", UserWarning, stacklevel=2) + return + + # SAM3 finds almost no nuclei with 'nucleus' or 'nuclei', in fluorescence and in H&E: the best object of an + # image scores below the confidence threshold. So every dataset uses 'cell', as in the APG paper. + prompt = "cell" + processor = _load_sam3(checkpoint, device) + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_sam3_{SAM3_VERSION}.csv") + _run_evaluation( + lambda x: _segment_sam3(x, processor, prompt), + dataset_name, data_root, save_path, desc=f"sam3-{prompt}", limit=limit, sample_index=sample_index, shard=shard, + ) + + +def run_instanseg_evaluation(dataset_name, data_root, experiment_folder, limit, sample_index, shard=None): + if dataset_name in HP_DATASETS: + model_type, target = INSTANSEG_HP_MODEL, "nuclei" + elif dataset_name in NUCLEUS_DATASETS: + model_type, target = INSTANSEG_FLUORESCENCE_MODEL, "nuclei" + elif dataset_name in FLUORESCENCE_CELL_DATASETS: + model_type, target = INSTANSEG_FLUORESCENCE_MODEL, "cells" + else: + warnings.warn( + f"InstanSeg has no model for '{dataset_name}', which is not 2d fluorescence or histopathology. Skipping.", + UserWarning, stacklevel=2, + ) + return + + model = _load_instanseg(model_type) + save_path = os.path.join(experiment_folder, "results", f"{dataset_name}_instanseg_{model_type}.csv") + _run_evaluation( + lambda x: _segment_instanseg(x, model, target), + dataset_name, data_root, save_path, desc=f"instanseg-{target}", limit=limit, sample_index=sample_index, + shard=shard, ) @@ -754,67 +1157,106 @@ def main(): parser.add_argument("--method", type=str, required=True, choices=METHODS) parser.add_argument( "-m", "--model_type", type=str, default=None, - help="Model type override, e.g. cyto3 for cellpose, sam_h_x40 for cellvit or vit_b_em_organelles for " - "micro-sam v1." + help="Model type override, e.g. cyto3 for cellpose, sam_h_x40 for cellvit, vit_b for micro-sam v1 or SAM " + "AMG, or vit_b_em_organelles for micro-sam v1 on EM organelles." ) parser.add_argument( "-c", "--checkpoint", type=str, default=None, - help="Checkpoint path for the cellvit, micro-sam v1, segneuron and focus3d methods." + help="Checkpoint path for the cellvit, micro-sam v1, SAM, SAM2, SAM3, segneuron and focus3d methods." ) parser.add_argument( "--crop_3d", type=int, nargs=3, default=None, help="Override the 3d center crop. FOCUS-3D defaults to its own window depth." ) parser.add_argument("--cell_radius", type=float, default=FOCUS3D_CELL_RADIUS, help="The xy radius to rescale from.") + parser.add_argument( + "--cellpose_3d_mode", type=str, default="stitch", choices=CELLPOSE_3D_MODES, + help="How CellPose segments a volume, see CELLPOSE_3D_MODES.", + ) + parser.add_argument( + "--stardist_3d_mode", type=str, default="3d", choices=STARDIST_3D_MODES, + help="How StarDist segments a volume, see STARDIST_3D_MODES.", + ) parser.add_argument( "--z_ratio", type=float, default=None, - help="The z-to-xy spacing ratio. Defaults to the anisotropy." + help="The z-to-xy spacing ratio. Defaults to 1, the volume as acquired." ) parser.add_argument("--batch_size", type=int, default=FOCUS3D_BATCH_SIZE, help="Patches per forward pass.") parser.add_argument("--num_workers", type=int, default=FOCUS3D_NUM_WORKERS, help="Data loader workers.") parser.add_argument("--n_samples", type=int, default=None, help="Score only the first N samples, for a check.") parser.add_argument("--sample_index", type=int, default=None, help="Score only this sample, as one array task.") + parser.add_argument("--n_shards", type=int, default=None, help="Split the samples over this many tasks.") + parser.add_argument("--shard_index", type=int, default=None, help="The shard this task scores.") args = parser.parse_args() check_data_download(args.dataset_name, args.input_path) print("Device:", torch.cuda.get_device_name() if torch.cuda.is_available() else "CPU") device = "cuda" if torch.cuda.is_available() else "cpu" + shard = None if args.n_shards is None else (args.shard_index, args.n_shards) if args.method == "cellpose": for model_type in ((args.model_type,) if args.model_type else CELLPOSE_MODELS): run_cellpose_evaluation( args.dataset_name, args.input_path, args.experiment_folder, model_type=model_type, device=device, - limit=args.n_samples, sample_index=args.sample_index, + limit=args.n_samples, sample_index=args.sample_index, shard=shard, mode_3d=args.cellpose_3d_mode, ) elif args.method == "stardist": run_stardist_evaluation( args.dataset_name, args.input_path, args.experiment_folder, limit=args.n_samples, - sample_index=args.sample_index, + sample_index=args.sample_index, shard=shard, mode_3d=args.stardist_3d_mode, ) elif args.method == "cellsam": run_cellsam_evaluation( args.dataset_name, args.input_path, args.experiment_folder, limit=args.n_samples, - sample_index=args.sample_index, + sample_index=args.sample_index, shard=shard, ) elif args.method == "cellvit": for model_type in ((args.model_type,) if args.model_type else tuple(CELLVIT_CHECKPOINTS)): run_cellvit_evaluation( + args.dataset_name, args.input_path, args.experiment_folder, model_type=model_type, + checkpoint=args.checkpoint, limit=args.n_samples, sample_index=args.sample_index, shard=shard, + ) + + elif args.method == "cellvitpp": + for model_type in ((args.model_type,) if args.model_type else tuple(CELLVIT_CHECKPOINTS)): + run_cellvitpp_evaluation( args.dataset_name, args.input_path, args.experiment_folder, model_type=model_type, checkpoint=args.checkpoint, device=device, limit=args.n_samples, sample_index=args.sample_index, + shard=shard, ) - elif args.method in ("microsam_ais", "microsam_apg"): + elif args.method in ("microsam_ais", "microsam_apg", "sam_amg"): run_microsam_v1_evaluation( args.dataset_name, args.input_path, args.experiment_folder, method=args.method, model_type=args.model_type or ( - SAM_V1_EM_ORGANELLE_MODEL_TYPE if args.dataset_name in EM_ORGANELLE_DATASETS else SAM_V1_MODEL_TYPE + SAM_V1_EM_ORGANELLE_MODEL_TYPE + if args.dataset_name in EM_ORGANELLE_DATASETS and args.method != "sam_amg" else None ), checkpoint=args.checkpoint, device=device, - limit=args.n_samples, sample_index=args.sample_index, + limit=args.n_samples, sample_index=args.sample_index, shard=shard, + ) + + elif args.method == "sam2_amg": + run_sam2_amg_evaluation( + args.dataset_name, args.input_path, args.experiment_folder, + model_type=args.model_type or SAM2_AMG_MODEL_TYPE, checkpoint=args.checkpoint, device=device, + limit=args.n_samples, sample_index=args.sample_index, shard=shard, + ) + + elif args.method == "sam3": + run_sam3_evaluation( + args.dataset_name, args.input_path, args.experiment_folder, checkpoint=args.checkpoint, device=device, + limit=args.n_samples, sample_index=args.sample_index, shard=shard, + ) + + elif args.method == "instanseg": + run_instanseg_evaluation( + args.dataset_name, args.input_path, args.experiment_folder, limit=args.n_samples, + sample_index=args.sample_index, shard=shard, ) elif args.method == "focus3d": @@ -824,13 +1266,14 @@ def main(): cell_radius=args.cell_radius, z_ratio=args.z_ratio, crop_shape=tuple(args.crop_3d) if args.crop_3d else None, batch_size=args.batch_size, num_workers=args.num_workers, limit=args.n_samples, - sample_index=args.sample_index, + sample_index=args.sample_index, shard=shard, ) elif args.method == "segneuron": run_segneuron_evaluation( args.dataset_name, args.input_path, args.experiment_folder, device=device, checkpoint_path=args.checkpoint, limit=args.n_samples, sample_index=args.sample_index, + shard=shard, ) elif args.method in EMPANADA_MODELS: diff --git a/finetuning/v2/evaluation/evaluate_automatic_segmentation.py b/finetuning/v2/evaluation/evaluate_automatic_segmentation.py index 6f7baf74b..a6458ba84 100644 --- a/finetuning/v2/evaluation/evaluate_automatic_segmentation.py +++ b/finetuning/v2/evaluation/evaluate_automatic_segmentation.py @@ -21,6 +21,7 @@ import argparse import warnings +import numpy as np import pandas as pd import torch @@ -30,37 +31,52 @@ from common import ( DATA_ROOT, DATASETS_2D, DATASETS_3D, DATASET_SPACING, MODEL_TYPES, MODES, VOLUME_SPEED_OPTIONS, build_model, check_data_download, evaluate_samples, has_val_split, load_apg_overrides, postprocess_unisam2, predict_unisam2, - read_tuned_params, resolve_checkpoint_identity, + pad_to_min_shape, read_tuned_params, resolve_checkpoint_identity, use_tmpdir_for_embeddings, ) +# The 2d datasets whose test images are smaller than their training images at the same pixel size. They are +# padded with zeros to the training shape, so that micro-sam2 sees the objects at the scale it was trained on: +# the TissueNet test images are 256 x 256 and its training images 512 x 512. +TRAINING_SCALE_MIN_SHAPE_2D = {"tissuenet": (512, 512)} + +# EVAL_NORM_PERCENTILES ('lower,upper', e.g. '1,99.8'): experiment that replaces the 2nd / 98th percentile bounds +# of the micro-sam2 input normalization; such runs belong in their own experiment folder. +if os.environ.get("EVAL_NORM_PERCENTILES"): + import micro_sam.v2.normalization as _normalization + _lower, _upper = (float(value) for value in os.environ["EVAL_NORM_PERCENTILES"].split(",")) + for _function in (_normalization.compute_percentile_bounds, _normalization.normalize_raw): + _names = _function.__code__.co_varnames[:_function.__code__.co_argcount] + _defaults = list(_function.__defaults__) + _first = len(_names) - len(_defaults) + _defaults[_names.index("lower_percentile") - _first] = _lower + _defaults[_names.index("upper_percentile") - _first] = _upper + _function.__defaults__ = tuple(_defaults) + REGISTRY_MODELS = [model for model in FINETUNED_MODELS if has_registered_decoder(model)] -# The temporary embeddings of APG go to the job's node-local TMPDIR, as in `parameter_search.py`. -# In the micro-sam cache of the home directory, the leftovers of killed jobs fill the quota. -if os.environ.get("TMPDIR"): - import uuid - import atexit - import shutil - from pathlib import Path - import micro_sam.util - import micro_sam.v2.util - import micro_sam.v2.instance_segmentation - import micro_sam.v2.automatic_prompt_generation - - def _make_temp_embedding_path(): - parent = Path(os.environ["TMPDIR"], "micro_sam_tmp_embeddings") - parent.mkdir(parents=True, exist_ok=True) - path = str(parent / f"{uuid.uuid4().hex}.zarr") - atexit.register(shutil.rmtree, path, ignore_errors=True) - return path - - for _module in (micro_sam.util, micro_sam.v2.util, micro_sam.v2.instance_segmentation, - micro_sam.v2.automatic_prompt_generation): - _module.make_temp_embedding_path = _make_temp_embedding_path - - -def segment(model, mode, raw, ndim, dataset_name, model_type, params, device, spacing=None, devices=None): - """Segment one sample with the tuned parameters of a mode.""" +use_tmpdir_for_embeddings() + + +def segment( + model, mode, raw, ndim, dataset_name, model_type, params, device, spacing=None, devices=None, min_shape=None, +): + """Segment one sample with the tuned parameters of a mode. + + With 'min_shape' (y, x), a 2d image smaller than it is padded with zeros to that shape, as PathoSAM does with the + 256 x 256 PanNuke tiles, and the segmentation is cropped back. A volume is padded in y and x, slice by slice. + """ + if min_shape is not None and ndim == 2 and min(raw.shape[:2]) < min(min_shape): + from skimage.segmentation import relabel_sequential + padded, crop = pad_to_min_shape(raw, min_shape) + seg = segment(model, mode, padded, ndim, dataset_name, model_type, params, device, spacing, devices) + return relabel_sequential(seg[crop])[0].astype("uint32") + if min_shape is not None and ndim == 3 and min(raw.shape[1:3]) < min(min_shape): + from skimage.segmentation import relabel_sequential + # The z axis is moved last, so that pad_to_min_shape pads the y and x axes of every slice. + padded, crop = pad_to_min_shape(np.moveaxis(raw, 0, -1), min_shape) + seg = segment(model, mode, np.moveaxis(padded, -1, 0), ndim, dataset_name, model_type, params, device, spacing, + devices) + return relabel_sequential(seg[(slice(None),) + crop])[0].astype("uint32") if mode == "apg": model.clear_state() model.initialize(raw, ndim=ndim, **(VOLUME_SPEED_OPTIONS if ndim == 3 else {})) @@ -74,7 +90,7 @@ def segment(model, mode, raw, ndim, dataset_name, model_type, params, device, sp def run_evaluation( model, mode, dataset_name, data_root, experiment_folder, model_type, params, device, limit, crop_shape=None, checkpoint_id=None, devices=None, tuned=None, result_tag=None, config_name=None, sample_index=None, - shard=None, + shard=None, min_shape=None, ): """Score the test split with the given parameters and write the result CSV. @@ -135,6 +151,7 @@ def run_evaluation( return evaluate_samples( lambda raw: segment( model, mode, raw, ndim, dataset_name, model_type, params or {}, device, spacing=spacing, devices=devices, + min_shape=min_shape, ), dataset_name, data_root, save_path, desc=f"{mode}-{model_type}", limit=limit, crop_shape=crop_shape, sample_index=sample_index, extra_columns=extra_columns, shard=shard, @@ -179,6 +196,11 @@ def main(): "--result_tag", type=str, default=None, help="Tag appended to the result file name. Defaults to the --apg_params configuration name.", ) + parser.add_argument( + "--pad_to_min_shape", type=int, nargs=2, default=None, + help="Pad images (or the slices of a volume) smaller than this (Y X) with zeros and crop the segmentation " + "back. Defaults to TRAINING_SCALE_MIN_SHAPE_2D of the dataset.", + ) args = parser.parse_args() check_data_download(args.dataset_name, args.input_path) @@ -235,6 +257,10 @@ def main(): devices=args.devices or None, tuned=tuned, result_tag=result_tag, config_name=config_name, limit=args.n_samples, sample_index=args.sample_index, shard=None if args.n_shards is None else (args.shard_index, args.n_shards), + min_shape=( + TRAINING_SCALE_MIN_SHAPE_2D.get(args.dataset_name) if args.pad_to_min_shape is None + else tuple(args.pad_to_min_shape) + ), ) diff --git a/finetuning/v2/evaluation/evaluate_automatic_tracking.py b/finetuning/v2/evaluation/evaluate_automatic_tracking.py new file mode 100644 index 000000000..69928681b --- /dev/null +++ b/finetuning/v2/evaluation/evaluate_automatic_tracking.py @@ -0,0 +1,357 @@ +"""Automatic tracking with micro-sam2 and Trackastra on the Cell Tracking Challenge (CTC) data, 2d+t and 3d+t. + +micro-sam2 segments each frame on its own (AIS or APG, the library defaults of the model), then Trackastra links the +objects over time. The result is written in CTC format and scored against the CTC ground truth: the tracking metrics +of traccuracy (TRA, DET, LNK, divisions) on the TRA markers, and the CTC SEG score on the annotated frames. + +Besides the CTC sets it runs on the BacMother test videos and the TOIAM test movie, which are in the CTC format too. +'--method' replaces the micro-sam2 segmentation by a baseline, segmented as in `evaluate_automatic_baselines.py`: +'cpsam' (Cellpose-SAM) or 'microsam_v1' (micro-sam v1 AIS with 'vit_l_lm'). Trackastra links all of them the same way. +The result table marks the datasets that micro-sam2 trained on ('in_training'). + +A large 3d+t sequence can be split: '--frames START END' segments only these frames and stores them, one file per +frame, as short GPU jobs; '--link_only' then links all stored frames with Trackastra and scores the result, e.g. on a +CPU node with enough memory. The result is the same as of a run in one piece. + +'--prepare_data' converts the movies of other sources (E. coli, ALFI, Cardioblast, wing disc, YeaZ, Cell-ACDC and +DynamicNuclearNet, see 'tracking_util.PREPARED_DATASETS') to the CTC format once. '--collect' joins the result tables +of all runs and, with '--figure_folder', writes the central table 'cells_automatic_tracking_results.csv'. + +Usage examples: + python evaluate_automatic_tracking.py -d Fluo-N2DH-GOWT1 -s 01 --mode ais + python evaluate_automatic_tracking.py -d Fluo-C3DH-A549 -s 02 --mode apg + python evaluate_automatic_tracking.py -d Fluo-N3DL-DRO -s 01 --mode apg --frames 0 5 + python evaluate_automatic_tracking.py -d Fluo-N3DL-DRO -s 01 --mode apg --link_only + python evaluate_automatic_tracking.py --list + python evaluate_automatic_tracking.py --prepare_data ALFI + python evaluate_automatic_tracking.py --collect --figure_folder ../../../scripts/plotting/v2/for_paper/figures +""" + +import os +import time +import shutil +import argparse +import tempfile +import warnings + +import tifffile +import numpy as np +import pandas as pd +from tqdm import trange + +import torch + +from micro_sam.v1.multi_dimensional_segmentation import track_across_frames +from micro_sam.v2.automatic_segmentation import automatic_instance_segmentation, get_predictor_and_segmenter + +from common import use_tmpdir_for_embeddings +from tracking_util import ( + MICROSAM_TRAINING, PREPARED_DATASETS, TRACKING_DATA, modality, prepare_tracking_data, scores_of, write_tables, + EXPERIMENT_ROOT, SPARSE_GT_DATASETS, LazyFrames, automatic_datasets, ctc_seg_score, gt_folder_of, in_training, + is_3d, load_frames, load_sequence, restrict_to_annotated, sequences_of, set_norm_percentiles, tracking_metrics, +) + + +METHODS = ("microsam2", "cpsam", "microsam_v1") +MICROSAM_V1_MODEL = "vit_l_lm" + + +def method_model(method, model_type): + """Get the model that a method segments with.""" + return {"microsam2": model_type, "cpsam": "cpsam", "microsam_v1": MICROSAM_V1_MODEL}[method] + + +def build_frame_segmenter(method, mode, model_type, dataset_name, ndim, device): + """Get a function that segments one frame with a method.""" + if method == "cpsam": + from evaluate_automatic_baselines import _load_cellpose, _segment_cellpose + model = _load_cellpose("cpsam", device) + return lambda frame: _segment_cellpose(frame, model, ndim, dataset_name) + if method == "microsam_v1": + from evaluate_automatic_baselines import _load_microsam_v1, _segment_microsam_v1 + predictor, segmenter = _load_microsam_v1("microsam_ais", MICROSAM_V1_MODEL, None, device) + return lambda frame: _segment_microsam_v1(frame, predictor, segmenter, ndim) + predictor, segmenter = get_predictor_and_segmenter( + model_type=model_type, segmentation_mode=mode, ndim=ndim, device=device, + ) + return lambda frame: automatic_instance_segmentation( + predictor=predictor, segmenter=segmenter, input_path=frame, ndim=ndim, device=device, verbose=False, + ) + + +def segment_frames(timeseries, segment): + """Segment every frame on its own and give the objects ids that are unique over all frames.""" + segmentation = np.zeros(timeseries.shape, dtype="uint32") + offset = 0 + for t in trange(timeseries.shape[0], desc="Segment frames"): + frame_seg = np.asarray(segment(timeseries[t])).astype("uint32") + max_id = int(frame_seg.max()) + frame_seg[frame_seg != 0] += offset + offset += max_id + segmentation[t] = frame_seg + return segmentation + + +def segment_frame_range(dataset_name, sequence, method, model_type, mode, start, end, frames_folder): + """Segment the frames start to end (exclusive) and store each one, with its time, in 'frames_folder'. + + A frame that is already stored is skipped, so a task that was stopped resumes. + """ + os.makedirs(frames_folder, exist_ok=True) + todo = [t for t in range(start, end) if not os.path.exists(os.path.join(frames_folder, f"seg_t{t:04d}.tif"))] + if not todo: + print(f"Frames {start} to {end} are already stored in '{frames_folder}'.") + return + ndim = 3 if is_3d(dataset_name) else 2 + device = "cuda" if torch.cuda.is_available() else "cpu" + segment = build_frame_segmenter(method, mode, model_type, dataset_name, ndim, device) + for t in todo: + # Another task may have stored this frame meanwhile, e.g. a smaller shard of the same frames. + if os.path.exists(os.path.join(frames_folder, f"seg_t{t:04d}.tif")): + continue + frame = load_frames(dataset_name, sequence, t, t + 1)[0] + begin = time.time() + seg = np.asarray(segment(frame)).astype("uint32") + seconds = time.time() - begin + # The time goes first, so a stored frame always has its time. + with open(os.path.join(frames_folder, "times.txt"), "a") as f: + f.write(f"{t} {seconds:.1f}\n") + partial = os.path.join(frames_folder, f"seg_t{t:04d}.partial.tif") + tifffile.imwrite(partial, seg, compression="zlib") + os.replace(partial, os.path.join(frames_folder, f"seg_t{t:04d}.tif")) + print(f"frame {t}: {int(seg.max())} objects in {seconds:.0f} s", flush=True) + + +def load_stored_frames(frames_folder, n_frames): + """Load the stored frame segmentations, with ids that are unique over all frames, and their summed time.""" + paths = [os.path.join(frames_folder, f"seg_t{t:04d}.tif") for t in range(n_frames)] + missing = [t for t, path in enumerate(paths) if not os.path.exists(path)] + if missing: + raise RuntimeError(f"{len(missing)} frames are not segmented yet in '{frames_folder}', e.g. {missing[:5]}.") + first = tifffile.imread(paths[0]) + segmentation = np.zeros((n_frames,) + first.shape, dtype="uint32") + offset = 0 + for t, path in enumerate(paths): + frame_seg = first if t == 0 else tifffile.imread(path) + max_id = int(frame_seg.max()) + frame_seg = frame_seg.astype("uint32") + frame_seg[frame_seg != 0] += offset + offset += max_id + segmentation[t] = frame_seg + times = {} + with open(os.path.join(frames_folder, "times.txt")) as f: + for line in f: + # Shards append to the same file, so their writes can mix and leave empty or broken lines. + parts = line.split() + if len(parts) != 2: + continue + times[int(parts[0])] = float(parts[1]) + return segmentation, sum(times.values()) + + +def score_tracking(dataset_name, sequence, pred_folder, tracked): + """Score a CTC prediction; 'tracked' gives its frames for the SEG score. + + For a dataset in `SPARSE_GT_DATASETS` the tracking metrics ignore the predictions of unannotated cells. + """ + gt_folder = gt_folder_of(dataset_name, sequence) + if not os.path.exists(os.path.join(pred_folder, "man_track.txt")): + # Trackastra writes nothing for an empty segmentation, which detects and links nothing. + warnings.warn(f"The tracking result in {pred_folder} is empty. It scores 0.") + scores = {"TRA": 0.0, "DET": 0.0, "LNK": 0.0} + else: + eval_folder = pred_folder + if dataset_name in SPARSE_GT_DATASETS: + eval_folder = restrict_to_annotated(pred_folder, os.path.join(gt_folder, "TRA"), f"{pred_folder}_annotated") + scores = tracking_metrics(os.path.join(gt_folder, "TRA"), eval_folder) + scores["SEG"] = ctc_seg_score(gt_folder, tracked) + scores["annotated_only"] = dataset_name in SPARSE_GT_DATASETS + return scores + + +def rescore_tracking(dataset_name, sequence, pred_folder, result_path): + """Score the stored prediction of a run again and update its result table.""" + old = pd.read_csv(result_path) + scores = score_tracking(dataset_name, sequence, pred_folder, LazyFrames(pred_folder)) + columns = ( + "dataset", "sequence", "ndim", "in_training", "model", "mode", "tracking_model", "linking_mode", "n_frames", + "n_segmented_objects", "segmentation_time", "tracking_time", + ) + keep = [c for c in columns if c in old.columns] + result = pd.DataFrame([{**old.iloc[0][keep].to_dict(), **scores}]) + result.to_csv(result_path, index=False) + print(result.T.to_string(header=False)) + return result + + +def run_tracking( + dataset_name, sequence, model_type, mode, output_root, linking_mode, tracking_model, score_only=False, + norm_percentiles=None, frames=None, link_only=False, method="microsam2", +): + """Segment and track one CTC sequence, write the CTC result and return the row of scores. + + With 'score_only', the stored prediction of an earlier run is scored again instead. With 'frames' (start, end), + only these frames are segmented and stored; with 'link_only', the stored frames are linked and scored. + """ + volumetric = is_3d(dataset_name) + if method == "microsam2": + tag = f"{dataset_name}_{sequence}_{model_type}_{mode}_{tracking_model}_{linking_mode}" + else: # A baseline has no AIS / APG mode. + tag = f"{dataset_name}_{sequence}_{method}_{method_model(method, model_type)}_{tracking_model}_{linking_mode}" + if norm_percentiles is not None: + tag += f"_norm{norm_percentiles[0]}-{norm_percentiles[1]}" + pred_folder = os.path.join(output_root, "predictions", tag) + result_path = os.path.join(output_root, "results", f"{tag}.csv") + frames_folder = os.path.join(output_root, "frames", tag) + if score_only: + return rescore_tracking(dataset_name, sequence, pred_folder, result_path) + if os.path.exists(result_path): + print(f"The result is already stored at '{result_path}'.") + return pd.read_csv(result_path) + if frames is not None: + return segment_frame_range(dataset_name, sequence, method, model_type, mode, *frames, frames_folder) + + timeseries = load_sequence(dataset_name, sequence) + print(f"{dataset_name} {sequence}: {timeseries.shape} {timeseries.dtype}") + if link_only: + segmentation, segmentation_time = load_stored_frames(frames_folder, timeseries.shape[0]) + else: + device = "cuda" if torch.cuda.is_available() else "cpu" + ndim = 3 if volumetric else 2 + segment = build_frame_segmenter(method, mode, model_type, dataset_name, ndim, device) + start = time.time() + segmentation = segment_frames(timeseries, segment) + segmentation_time = time.time() - start + + # Trackastra with pretrained features writes them to './embeddings', so each run links in a folder of its own. + work_dir, cwd = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")), os.getcwd() + os.chdir(work_dir) + try: + start = time.time() + tracked, _ = track_across_frames( + timeseries=timeseries, segmentation=segmentation, mode=linking_mode, tracking_model=tracking_model, + output_folder=pred_folder, + ) + tracking_time = time.time() - start + finally: + os.chdir(cwd) + shutil.rmtree(work_dir, ignore_errors=True) + + scores = score_tracking(dataset_name, sequence, pred_folder, tracked) + row = { + "dataset": dataset_name, "sequence": sequence, "ndim": "3d+t" if volumetric else "2d+t", + "in_training": in_training(dataset_name), "method": method, "model": method_model(method, model_type), + "mode": mode if method == "microsam2" else "-", + "tracking_model": tracking_model, "linking_mode": linking_mode, "n_frames": timeseries.shape[0], + "norm_percentiles": "2,98" if norm_percentiles is None else ",".join(map(str, norm_percentiles)), + "n_segmented_objects": int(len(np.unique(segmentation)) - 1), + "segmentation_time": segmentation_time, "tracking_time": tracking_time, **scores, + } + os.makedirs(os.path.dirname(result_path), exist_ok=True) + result = pd.DataFrame([row]) + result.to_csv(result_path, index=False) + print(result.T.to_string(header=False)) + return result + + +# The (method, model) names of the central table, as in the segmentation tables. +TABLE_METHODS = { + "microsam2": ("micro-sam2", "hvit_l"), "microsam_v1": ("micro-sam", "vit_l_lm"), "cpsam": ("cellpose", "cpsam") +} + + +def central_automatic(table): + """The automatic table of the paper: one row per dataset, method, mode and Trackastra model.""" + # Only the default 2/98 percentile normalization; the other normalizations were tests. + table = table[table["norm_percentiles"].isna() | (table["norm_percentiles"] == "2,98")].copy() + # The first micro-sam2 runs have no method column. + table["method"] = table["method"].fillna("microsam2") + table = table.drop_duplicates(["dataset", "sequence", "method", "mode", "tracking_model"], keep="last") + table["model"] = [TABLE_METHODS[method][1] for method in table["method"]] + table["method"] = [TABLE_METHODS[method][0] for method in table["method"]] + # micro-sam runs AIS; the baselines have no mode. + table.loc[table["method"] == "micro-sam", "mode"] = "ais" + table.loc[table["method"] == "cellpose", "mode"] = None + table["ndim"] = table["ndim"].str[0].astype(int) + keys = ["dataset", "ndim", "in_training", "method", "model", "mode", "tracking_model", "linking_mode"] + result = scores_of(table, keys) + result.insert(1, "category", [modality(d)[0] for d in result["dataset"]]) + result.insert(2, "structure", [modality(d)[1] for d in result["dataset"]]) + # The domain is that of micro-sam2: in domain if micro-sam2 trained on the dataset. + result.insert(3, "domain", ["ID" if seen else "OOD" for seen in result.pop("in_training")]) + result.insert(5, "in_microsam_training", result["dataset"].isin(MICROSAM_TRAINING)) + result["primary_metric"], result["lower_is_better"] = "tra", False + return result.sort_values(["ndim", "dataset", "method", "mode", "tracking_model"], na_position="first") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "-d", "--dataset_name", type=str, default=None, help="A CTC set like 'Fluo-N2DH-GOWT1', 'BacMother' or 'TOIAM'." + ) + parser.add_argument("-s", "--sequence", type=str, default="01", help="The sequence, see --list.") + parser.add_argument("-m", "--model_type", type=str, default="hvit_l_cells") + parser.add_argument("--mode", type=str, default="ais", choices=("ais", "apg")) + parser.add_argument( + "--method", type=str, default="microsam2", choices=METHODS, + help="The segmentation: micro-sam2, or a baseline ('cpsam', 'microsam_v1').", + ) + parser.add_argument("-o", "--output_root", type=str, default=os.path.join(EXPERIMENT_ROOT, "automatic")) + parser.add_argument("--linking_mode", type=str, default="greedy", choices=("greedy", "greedy_nodiv", "ilp")) + parser.add_argument( + "--tracking_model", type=str, default=None, + help="The Trackastra model. By default 'general_2d' for 2d+t and 'ctc' for 3d+t (the only 3d model).", + ) + parser.add_argument("--list", action="store_true", help="Print the datasets and their sequences and exit.") + parser.add_argument("--score_only", action="store_true", help="Score the stored prediction of a run again.") + parser.add_argument( + "--frames", type=int, nargs=2, default=None, help="Only segment and store the frames START to END (exclusive).", + ) + parser.add_argument("--link_only", action="store_true", help="Link and score the stored frames of '--frames'.") + parser.add_argument( + "--norm_percentiles", type=float, nargs=2, default=None, + help="Lower and upper percentile of the input normalization instead of 2 and 98, e.g. 1 99.8.", + ) + parser.add_argument( + "--prepare_data", nargs="*", default=None, choices=list(PREPARED_DATASETS), + help="Convert these datasets (all if none is given) to the CTC format of the evaluation and exit.", + ) + parser.add_argument("--data_root", default=TRACKING_DATA, help="Where --prepare_data writes the movies.") + parser.add_argument("--collect", action="store_true", help="Collect the result tables of all runs.") + parser.add_argument("--figure_folder", default=None, help="Where --collect writes the central table.") + args = parser.parse_args() + if args.prepare_data is not None: + prepare_tracking_data(args.prepare_data, args.data_root) + return + if args.collect: + keys = ["dataset", "sequence", "ndim", "in_training", "mode"] + root = os.path.dirname(args.output_root.rstrip("/")) + write_tables("automatic", keys, central_automatic, root, args.figure_folder) + return + + if args.list: + for name in automatic_datasets(): + print( + name, "3d+t" if is_3d(name) else "2d+t", "trained" if in_training(name) else "held-out", + " ".join(sequences_of(name)), + ) + return + if args.dataset_name not in automatic_datasets(): + parser.error(f"Choose one of the datasets: {automatic_datasets()}") + if args.sequence not in sequences_of(args.dataset_name): + parser.error(f"Choose one of the sequences of {args.dataset_name}: {sequences_of(args.dataset_name)}") + + use_tmpdir_for_embeddings() + if args.norm_percentiles is not None: + set_norm_percentiles(*args.norm_percentiles) + tracking_model = args.tracking_model or ("ctc" if is_3d(args.dataset_name) else "general_2d") + run_tracking( + args.dataset_name, args.sequence, args.model_type, args.mode, args.output_root, args.linking_mode, + tracking_model, score_only=args.score_only, + norm_percentiles=None if args.norm_percentiles is None else tuple(args.norm_percentiles), + frames=None if args.frames is None else tuple(args.frames), link_only=args.link_only, method=args.method, + ) + + +if __name__ == "__main__": + main() diff --git a/finetuning/v2/evaluation/evaluate_interactive_baselines.py b/finetuning/v2/evaluation/evaluate_interactive_baselines.py index ac5a5c504..995c65a8b 100644 --- a/finetuning/v2/evaluation/evaluate_interactive_baselines.py +++ b/finetuning/v2/evaluation/evaluate_interactive_baselines.py @@ -21,6 +21,7 @@ import os import sys +import time import uuid import shutil import argparse @@ -32,14 +33,16 @@ import numpy as np import imageio.v3 as imageio from tqdm import tqdm -from scipy.ndimage import distance_transform_edt, find_objects -from skimage.measure import label as connected_components import torch +from bioimage_cpp.segmentation import label as relabel_components + +from micro_sam.prompt_generators import get_error_centers +from micro_sam.v1.evaluation.inference import _get_batched_prompts from common import ( CLICK_PROTOCOLS, DATA_ROOT, DATASETS_2D, DATASETS_3D, DATASETS_EM, - check_data_download, interactive_result_name, interactive_run_tag, load_data, n_samples, + check_data_download, interactive_result_name, interactive_run_tag, interactive_sample_ids_3d, load_data, n_samples, run_dataset_evaluation, ) @@ -48,6 +51,10 @@ NNINTERACTIVE_CHECKPOINT = "/mnt/vast-nhr/home/archit/u12090/nnInteractive/pretrained_weights/nnInteractive_v1.0" SAM3_ROOT = "/mnt/vast-nhr/home/archit/u12090/SAM3_Experiments" +# The SAM3 3d evaluation saves its state after an object at most this often, so that an interrupted volume goes on +# from there: one 32-slice crop can take up to a day (2026-11-05). +CHECKPOINT_SECONDS = 600 + SAM_V1_MODEL_TYPE = "vit_b" MICROSAM_V1_LM_MODEL = "vit_b_lm" MICROSAM_V1_EM_MODEL = "vit_b_em_organelles" @@ -63,38 +70,59 @@ def _load_samples(dataset_name, data_root, ndim, min_size=0, limit=None): return samples, total -def _get_largest_region_center(mask): - """Return the most interior point of the largest connected region of a mask, or None if it is empty. +# The baselines without a micro-sam pipeline (nnInteractive, SAM3) are prompted with the protocol of the micro-sam2 +# evaluation (see 'micro_sam.v2.evaluation.inference'), so that every method gets the same prompts: the connected +# components of the labels are prompted with the same first box or point, the 'center' correction clicks are placed +# on the worst slice in 3d, and the smaller object is kept where two predicted objects overlap. +PROMPT_DILATION = 5 +PROMPT_SEED = 0 + + +def _prompting_labels(labels): + """Return the objects that get prompted: the connected components of the labels.""" + return relabel_components(labels).astype("uint32") + + +def _first_prompt_2d(mask, start_with_box): + """Return the first prompt of one object in 2d: its box as (x0, y0, x1, y1), or a point as (x, y).""" + points, _, boxes = _get_batched_prompts( + gt=mask.astype("uint32"), gt_ids=[1], use_points=not start_with_box, use_boxes=start_with_box, + n_positives=0 if start_with_box else 1, n_negatives=0, dilation=PROMPT_DILATION, + ) + return [float(v) for v in (boxes[0] if start_with_box else points[0][0])] + + +def _center_slice(mask): + """Return the slice of the first prompt of one object in 3d: the middle of its z-extent.""" + z = np.where(mask.any(axis=(1, 2)))[0] + return int(np.ceil(np.mean([z.min(), z.max()]))) + + +def _worst_slice_clicks(gt_mask, pred_mask): + """Return the correction clicks of one object in 3d: the 'center' clicks on its slice with the most errors. - The centroid is not used, since it can lie outside the region. False foreground often forms a - shell around the object, and the centroid of such a shell is a point on the true object. + Returns: + The slice, the positive click as (x, y) or None, and the negative click as (x, y) or None. """ - labeled = connected_components(mask) - if labeled.max() == 0: - return None - region_id = np.bincount(labeled.ravel())[1:].argmax() + 1 - bbox = find_objects(labeled)[region_id - 1] - region = np.pad(labeled[bbox] == region_id, 1) - distances = distance_transform_edt(region) - point = np.unravel_index(distances.argmax(), distances.shape) - return [int(c) - 1 + sl.start for c, sl in zip(point, bbox)] + z_slices = np.where(gt_mask.any(axis=(1, 2)))[0] + z = int(z_slices[np.argmax([np.sum(gt_mask[z] != pred_mask[z]) for z in z_slices])]) + pos, neg = get_error_centers(gt_mask[z], pred_mask[z]) + return z, pos, neg -def _get_correction_points(gt_mask, pred_mask): - """Return one positive FN point and one negative FP point if available.""" - positive = _get_largest_region_center(gt_mask & ~pred_mask) - negative = _get_largest_region_center(~gt_mask & pred_mask) - return positive, negative +def _add_object(segmentation, areas, mask, gt_id): + """Add one predicted object to a segmentation. Where objects overlap, the smaller one is kept.""" + area = mask.sum() + existing = segmentation[mask] + segmentation[mask] = np.where(areas[existing] > area, gt_id, existing) + areas[gt_id] = area -def _get_middle_slice_prompt(gt_mask): - """Return a representative z slice and the 2D object mask on this slice.""" - z_indices = np.where(gt_mask)[0] - z_mid = int(np.round((int(z_indices.min()) + int(z_indices.max())) / 2.0)) - z_values = np.unique(z_indices) - z = min(z_values, key=lambda zz: abs(int(zz) - z_mid)) - mask_2d = gt_mask[z] - return int(z), mask_2d +def _empty_predictions(labels, n_iterations): + """Return the segmentations and object areas of all iterations, before any object is added.""" + segmentations = [np.zeros(labels.shape, dtype="uint32") for _ in range(n_iterations)] + areas = [np.full(int(labels.max()) + 1, np.inf) for _ in range(n_iterations)] + return segmentations, areas def _load_nninteractive(checkpoint_path, device): @@ -110,49 +138,40 @@ def _segment_nninteractive_iterative(volume, labels, session, start_with_box, n_ buffer = np.zeros(volume.shape, dtype="float32") session.set_target_buffer(buffer) - gt_ids = np.unique(labels)[1:] - seg_per_iter = [np.zeros(volume.shape, dtype="uint32") for _ in range(n_iterations)] - - for gt_id in gt_ids: + labels = _prompting_labels(labels) + np.random.seed(PROMPT_SEED) + seg_per_iter, areas = _empty_predictions(labels, n_iterations) + for gt_id in np.setdiff1d(np.unique(labels), [0]): gt_mask = labels == gt_id session.reset_interactions() - z, gt_mask_2d = _get_middle_slice_prompt(gt_mask) - yx_coords = np.where(gt_mask_2d) - + z = _center_slice(gt_mask) + prompt = [int(round(v)) for v in _first_prompt_2d(gt_mask[z], start_with_box)] if start_with_box: - bbox = [ - [z, z + 1], - [int(yx_coords[0].min()), int(yx_coords[0].max()) + 1], - [int(yx_coords[1].min()), int(yx_coords[1].max()) + 1], - ] - session.add_bbox_interaction(bbox, include_interaction=True) + x0, y0, x1, y1 = prompt + session.add_bbox_interaction([[z, z + 1], [y0, y1], [x0, x1]], include_interaction=True) else: - center = (z, *_get_largest_region_center(gt_mask_2d)) - session.add_point_interaction(center, include_interaction=True) - + session.add_point_interaction((z, prompt[1], prompt[0]), include_interaction=True) pred_mask = buffer > 0.5 - seg_per_iter[0][pred_mask] = gt_id + _add_object(seg_per_iter[0], areas[0], pred_mask, gt_id) for it in range(1, n_iterations): - positive_point, negative_point = _get_correction_points(gt_mask, pred_mask) - if positive_point is not None: - session.add_point_interaction(tuple(positive_point), include_interaction=True) - pred_mask = buffer > 0.5 - if negative_point is not None: - session.add_point_interaction(tuple(negative_point), include_interaction=False) - pred_mask = buffer > 0.5 - seg_per_iter[it][pred_mask] = gt_id + z, pos, neg = _worst_slice_clicks(gt_mask, pred_mask) + for click, include in ((pos, True), (neg, False)): + if click is not None: + session.add_point_interaction((z, int(click[1]), int(click[0])), include_interaction=include) + pred_mask = buffer > 0.5 + _add_object(seg_per_iter[it], areas[it], pred_mask, gt_id) return seg_per_iter -def _nninteractive_prediction_paths(prediction_root, sample_id, n_iterations): +def _cached_prediction_paths(prediction_root, sample_id, n_iterations): """The per-iteration prediction files of one sample.""" sample_dir = os.path.join(prediction_root, f"sample_{sample_id:05d}") return [os.path.join(sample_dir, f"iteration{it:02d}.tif") for it in range(n_iterations)] -def _write_nninteractive_predictions(segs, paths): +def _write_cached_predictions(segs, paths): """Write the predictions of one sample. Each file is renamed into place, so no partial file is left.""" os.makedirs(os.path.dirname(paths[0]), exist_ok=True) for seg, path in zip(segs, paths): @@ -188,10 +207,8 @@ def run_nninteractive_evaluation( return prediction_root = os.path.join(experiment_folder, "predictions", "nninteractive", dataset_name, prompt_str) - total = n_samples(dataset_name, data_root) - if limit is not None: - total = min(total, limit) - sample_ids = range(total) if shard is None else range(shard[0], total, shard[1]) + selected = interactive_sample_ids_3d(dataset_name, data_root, limit) + sample_ids = selected if shard is None else selected[shard[0]::shard[1]] samples = load_data(dataset_name, data_root, 3, sample_ids=set(sample_ids)) # The model is loaded on the first sample that needs it, so that scoring from the cache loads no model. @@ -200,7 +217,9 @@ def run_nninteractive_evaluation( pred_paths_per_iter = [[] for _ in range(n_iterations)] samples = tqdm(samples, total=len(sample_ids), desc="nninteractive") for sample_id, (raw, labels, valid_roi) in zip(sample_ids, samples): - paths = _nninteractive_prediction_paths(prediction_root, sample_id, n_iterations) + if labels.max() == 0: # Nothing to prompt, and nothing to score, as in the micro-sam2 evaluation. + continue + paths = _cached_prediction_paths(prediction_root, sample_id, n_iterations) if not all(os.path.exists(path) for path in paths): if score_only: missing.append(sample_id) @@ -208,7 +227,7 @@ def run_nninteractive_evaluation( if session is None: session = _load_nninteractive(checkpoint_path, device) segs = _segment_nninteractive_iterative(raw, labels, session, start_with_box, n_iterations) - _write_nninteractive_predictions(segs, paths) + _write_cached_predictions(segs, paths) all_gt.append(labels) all_valid_rois.append(valid_roi) for it, path in enumerate(paths): @@ -291,8 +310,12 @@ def run_sam_v1_evaluation( "The slice-wise micro-sam v1 evaluation is 2d only. Use '--method microsam_vol' for a volume." ) - if name_tag == "micro-sam" and dataset_name in EM_DATASETS: - raise ValueError(f"micro-sam interactive does not support EM datasets (LM model only); got '{dataset_name}'.") + # Of the EM data, micro-sam v1 covers only the mitochondria and nuclei, with the organelle models. + if name_tag == "micro-sam" and dataset_name in EM_DATASETS and "em_organelles" not in model_type: + raise ValueError( + f"micro-sam interactive runs on EM only with the 'vit_*_em_organelles' models; got '{model_type}' on " + f"'{dataset_name}'." + ) prompt_str = "box" if start_with_box else "point" run_tag = interactive_run_tag(ndim=2, use_masks=use_masks, min_size=min_size, click_protocol=click_protocol) @@ -367,18 +390,174 @@ def run_sam_v1_evaluation( shutil.rmtree(work_dir, ignore_errors=True) +def _segment_sam3_2d(image, labels, model, processor, start_with_box, n_iterations): + """Prompt SAM3 for every object of an image, with the protocol of the micro-sam2 evaluation. + + The box stays in all iterations, the correction clicks add up, and the previous mask logits are fed back, + as for micro-sam2 and SAM2. + """ + from PIL import Image + + # SAM3 reads the size of an array as channels-first, so the image is passed as an RGB PIL image, as in its + # examples. Grayscale images are repeated into the three channels. + rgb = image if image.ndim == 3 else np.stack([image] * 3, axis=-1) + state = processor.set_image(Image.fromarray(np.clip(np.round(rgb), 0, 255).astype("uint8"))) + + labels = _prompting_labels(labels) + np.random.seed(PROMPT_SEED) + seg_per_iter, areas = _empty_predictions(labels, n_iterations) + for gt_id in np.setdiff1d(np.unique(labels), [0]): + gt_mask = labels == gt_id + prompt = _first_prompt_2d(gt_mask, start_with_box) + box = np.array([prompt], dtype="float32") if start_with_box else None + points = None if start_with_box else np.array([prompt], dtype="float32") + point_labels = None if start_with_box else np.array([1]) + mask_input, pred_mask = None, None + for it in range(n_iterations): + if it > 0: + pos, neg = get_error_centers(gt_mask, pred_mask) + clicks = [(click, label) for click, label in ((pos, 1), (neg, 0)) if click is not None] + if clicks: + new_points = np.array([click for click, _ in clicks], dtype="float32") + new_labels = np.array([label for _, label in clicks]) + points = new_points if points is None else np.concatenate([points, new_points]) + point_labels = new_labels if point_labels is None else np.concatenate([point_labels, new_labels]) + # A single mask for the first point, as in the 'center' protocol of micro-sam2. + masks, _, low_res_masks = model.predict_inst( + state, point_coords=points, point_labels=point_labels, box=box, mask_input=mask_input, + multimask_output=False, + ) + pred_mask = masks[0] > 0 + mask_input = low_res_masks[:1] + _add_object(seg_per_iter[it], areas[it], pred_mask, gt_id) + return seg_per_iter + + +def _save_object_checkpoint(path, segmentations, areas, last_id): + """Save the state of a volume after its object 'last_id': the segmentations and object areas of all iterations + and the random state. The file is renamed into place, so no partial file is left. + """ + _, keys, pos, has_gauss, gauss = np.random.get_state() + os.makedirs(os.path.dirname(path), exist_ok=True) + partial_path = f"{path[:-4]}.{uuid.uuid4().hex}.partial.npz" + np.savez_compressed( + partial_path, segmentations=np.stack(segmentations), areas=np.stack(areas), last_id=last_id, + rng_keys=keys, rng_pos=pos, rng_has_gauss=has_gauss, rng_gauss=gauss, + ) + os.replace(partial_path, path) + + +def _load_object_checkpoint(path, labels, n_iterations): + """Return the segmentations, object areas and last finished object of an interrupted volume, or None. + + The random state is restored too, so the run goes on as if it had not been interrupted. A checkpoint of other + labels or another number of iterations is not used. + """ + if path is None or not os.path.exists(path): + return None + try: + with np.load(path) as f: + segmentations, areas, last_id = list(f["segmentations"]), list(f["areas"]), int(f["last_id"]) + state = ("MT19937", f["rng_keys"], int(f["rng_pos"]), int(f["rng_has_gauss"]), float(f["rng_gauss"])) + except (OSError, KeyError, ValueError) as e: + print(f"The checkpoint '{path}' cannot be read and is not used: {e}") + return None + if len(segmentations) != n_iterations or segmentations[0].shape != labels.shape or \ + len(areas[0]) != int(labels.max()) + 1: + print(f"The checkpoint '{path}' belongs to another run and is not used.") + return None + np.random.set_state(state) + return segmentations, areas, last_id + + +def _segment_sam3_3d(volume, labels, predictor, start_with_box, n_iterations, checkpoint_path=None): + """Prompt SAM3 for every object of a volume, with the protocol of the micro-sam2 evaluation. + + The first prompt is on the middle slice of the object, the 'center' correction clicks are on its worst slice, + and the masks are propagated through the volume after every prompt, as for micro-sam2 and SAM2. + With 'checkpoint_path', the state is saved every CHECKPOINT_SECONDS after an object, and an interrupted volume + goes on after its last saved object: the objects are prompted one by one, each from cleared points, so the + result is the same as without the interruption. + """ + import tempfile + from micro_sam3.evaluation.inference import _propagate_sam3, _volume_to_frames + + n_frames, height, width = volume.shape + labels = _prompting_labels(labels) + np.random.seed(PROMPT_SEED) + seg_per_iter, areas = _empty_predictions(labels, n_iterations) + last_id = 0 + checkpoint = _load_object_checkpoint(checkpoint_path, labels, n_iterations) + if checkpoint is not None: + seg_per_iter, areas, last_id = checkpoint + print(f"Going on after object {last_id} of {int(labels.max())} from '{checkpoint_path}'.") + saved = time.time() + + def to_points(clicks): + coords = torch.tensor([[x / width, y / height] for (x, y), _ in clicks], dtype=torch.float32) + return coords, torch.tensor([label for _, label in clicks], dtype=torch.int32) + + def propagate(frame): + pred_mask = np.zeros(labels.shape, dtype=bool) + for z, mask in _propagate_sam3(predictor, state, frame, n_frames).items(): + pred_mask[z] = mask + return pred_mask + + tmp_dir = tempfile.mkdtemp() + try: + state = predictor.init_state(video_path=_volume_to_frames(volume, tmp_dir)) + for gt_id in np.setdiff1d(np.unique(labels), [0]): + if gt_id <= last_id: + continue + gt_mask = labels == gt_id + predictor.clear_all_points_in_video(state) + z = _center_slice(gt_mask) + prompt = _first_prompt_2d(gt_mask[z], start_with_box) + if start_with_box: + x0, y0, x1, y1 = prompt + box = np.clip(np.array([[x0 / width, y0 / height, x1 / width, y1 / height]], dtype="float32"), 0, 1) + predictor.add_new_points_or_box(inference_state=state, frame_idx=z, obj_id=int(gt_id), box=box) + else: + coords, point_labels = to_points([(prompt, 1)]) + predictor.add_new_points_or_box( + inference_state=state, frame_idx=z, obj_id=int(gt_id), points=coords, labels=point_labels, + clear_old_points=False, + ) + pred_mask = propagate(z) + _add_object(seg_per_iter[0], areas[0], pred_mask, gt_id) + + for it in range(1, n_iterations): + z, pos, neg = _worst_slice_clicks(gt_mask, pred_mask) + clicks = [(click, label) for click, label in ((pos, 1), (neg, 0)) if click is not None] + if clicks: + coords, point_labels = to_points(clicks) + predictor.add_new_points_or_box( + inference_state=state, frame_idx=z, obj_id=int(gt_id), points=coords, labels=point_labels, + clear_old_points=False, + ) + pred_mask = propagate(z) + _add_object(seg_per_iter[it], areas[it], pred_mask, gt_id) + if checkpoint_path is not None and time.time() - saved > CHECKPOINT_SECONDS: + _save_object_checkpoint(checkpoint_path, seg_per_iter, areas, int(gt_id)) + saved = time.time() + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + return seg_per_iter + + def run_sam3_evaluation( dataset_name, data_root, experiment_folder, - start_with_box=True, n_iterations=8, ndim=None, limit=None, + start_with_box=True, n_iterations=8, ndim=None, shard=None, score_only=False, limit=None, ): + """Run SAM3 with iterative prompting on the test split and write one result CSV per iteration. + + The predictions are cached per sample, with the same sharding and scoring as 'run_nninteractive_evaluation'. + """ if ndim is None: ndim = 3 if dataset_name in DATASETS_3D else 2 sys.path.insert(0, SAM3_ROOT) - from micro_sam3.evaluation.inference import ( - build_sam3_image_predictor, build_sam3_video_predictor, - run_interactive_segmentation_2d_sam3, run_interactive_segmentation_3d_sam3, - ) + from micro_sam3.evaluation.inference import build_sam3_image_predictor, build_sam3_video_predictor prompt_str = "box" if start_with_box else "point" dim_suffix = "" if ndim == 2 else "_3d" @@ -391,39 +570,64 @@ def run_sam3_evaluation( print(f"Results already stored at '{results_dir}'.") return - if ndim == 2: - model, processor = build_sam3_image_predictor() - predictor = None + prediction_root = os.path.join(experiment_folder, "predictions", f"sam3{dim_suffix}", dataset_name, prompt_str) + if ndim == 3: + selected = interactive_sample_ids_3d(dataset_name, data_root, limit) else: - predictor = build_sam3_video_predictor() - model, processor = None, None - - samples, n = _load_samples(dataset_name, data_root, ndim, limit=limit) - all_gt = [] - all_seg_per_iter = [[] for _ in range(n_iterations)] - - for raw, labels, valid_roi in tqdm(samples, total=n, desc=f"sam3-{ndim}d"): - if ndim == 2: - segs = run_interactive_segmentation_2d_sam3( - image=raw, gt=labels, model=model, processor=processor, - start_with_box_prompt=start_with_box, n_iterations=n_iterations, - ) - else: - segs = run_interactive_segmentation_3d_sam3( - raw=raw, gt=labels, predictor=predictor, - start_with_box_prompt=start_with_box, n_iterations=n_iterations, - ) + total = n_samples(dataset_name, data_root) + selected = list(range(total if limit is None else min(total, limit))) + sample_ids = selected if shard is None else selected[shard[0]::shard[1]] + samples = load_data(dataset_name, data_root, ndim, sample_ids=set(sample_ids)) + + # The model is loaded on the first sample that needs it, so that scoring from the cache loads no model. + model, processor, predictor = None, None, None + all_gt, all_valid_rois, missing = [], [], [] + pred_paths_per_iter = [[] for _ in range(n_iterations)] + samples = tqdm(samples, total=len(sample_ids), desc=f"sam3-{ndim}d") + for sample_id, (raw, labels, valid_roi) in zip(sample_ids, samples): + if labels.max() == 0: # Nothing to prompt, and nothing to score, as in the micro-sam2 evaluation. + continue + paths = _cached_prediction_paths(prediction_root, sample_id, n_iterations) + if not all(os.path.exists(path) for path in paths): + if score_only: + missing.append(sample_id) + continue + if ndim == 2: + if model is None: + model, processor = build_sam3_image_predictor() + segs = _segment_sam3_2d(raw, labels, model, processor, start_with_box, n_iterations) + else: + if predictor is None: + predictor = build_sam3_video_predictor() + checkpoint_path = os.path.join(os.path.dirname(paths[0]), "checkpoint.npz") + segs = _segment_sam3_3d(raw, labels, predictor, start_with_box, n_iterations, checkpoint_path) + _write_cached_predictions(segs, paths) + if ndim == 3 and os.path.exists(checkpoint_path): + os.remove(checkpoint_path) all_gt.append(labels) - for it, seg in enumerate(segs): - if valid_roi is not None: - seg[~valid_roi] = 0 - all_seg_per_iter[it].append(seg) + all_valid_rois.append(valid_roi) + for it, path in enumerate(paths): + pred_paths_per_iter[it].append(path) + + if shard is not None: + print(f"Shard {shard[0]}/{shard[1]} finished predicting {len(sample_ids)} sample(s).") + return + if missing: + raise RuntimeError( + f"{len(missing)} sample(s) of '{dataset_name}' have no predictions yet: {missing}. Run their shards first." + ) os.makedirs(results_dir, exist_ok=True) for it, save_path in enumerate(save_paths): if os.path.exists(save_path): continue - results = run_dataset_evaluation(all_gt, all_seg_per_iter[it], dataset_name, save_path) + predictions = [] + for path, valid_roi in zip(pred_paths_per_iter[it], all_valid_rois): + prediction = imageio.imread(path) + if valid_roi is not None: + prediction[~valid_roi] = 0 + predictions.append(prediction) + results = run_dataset_evaluation(all_gt, predictions, dataset_name, save_path) print(f"Iteration {it:02d}: {results}") @@ -477,10 +681,8 @@ def run_microsam_volumetric_evaluation( experiment_folder, "predictions", f"microsam_vol_{model_type}", dataset_name, f"{prompt}{interactive_run_tag(ndim=3, min_size=min_size)}", ) - total = n_samples(dataset_name, data_root) - if limit is not None: - total = min(total, limit) - sample_ids = range(total) if shard is None else range(shard[0], total, shard[1]) + selected = interactive_sample_ids_3d(dataset_name, data_root, limit) + sample_ids = selected if shard is None else selected[shard[0]::shard[1]] loaded = load_data(dataset_name, data_root, 3, min_size=min_size, sample_ids=set(sample_ids)) # Skip the volumes without objects: there is nothing to prompt, and nothing to score. @@ -488,7 +690,7 @@ def run_microsam_volumetric_evaluation( for sample_id, (raw, labels, valid_roi) in zip(sample_ids, loaded): if labels.max() == 0: continue - paths = _nninteractive_prediction_paths(prediction_root, sample_id, n_iterations) + paths = _cached_prediction_paths(prediction_root, sample_id, n_iterations) if not all(os.path.exists(path) for path in paths): if score_only: missing.append(sample_id) @@ -507,7 +709,7 @@ def run_microsam_volumetric_evaluation( with ProcessPoolExecutor(min(n_workers, len(todo)), mp_context=get_context("spawn")) as pool: segs_per_volume = pool.map(segment, *zip(*[(raw, labels) for raw, labels, _ in todo])) for segs, (_, _, paths) in tqdm(zip(segs_per_volume, todo), total=len(todo), desc="microsam_vol"): - _write_nninteractive_predictions(segs, paths) + _write_cached_predictions(segs, paths) if shard is not None: print(f"Shard {shard[0]}/{shard[1]} finished predicting {len(sample_ids)} sample(s).") @@ -590,8 +792,6 @@ def main(): raise ValueError("--shard_index and --n_shards must be given together.") if args.score_only and args.shard_index is not None: raise ValueError("--score_only scores the whole dataset, so it cannot be combined with a shard.") - if (args.shard_index is not None or args.score_only) and args.method == "sam3": - raise ValueError("--shard_index, --n_shards and --score_only are not supported for sam3.") if args.click_protocol != "random" and args.method not in ("sam", "micro-sam"): raise ValueError("--click_protocol is supported for sam and micro-sam only.") @@ -612,7 +812,8 @@ def main(): elif args.method == "sam3": run_sam3_evaluation( args.dataset_name, args.input_path, args.experiment_folder, - start_with_box=start_with_box, n_iterations=args.n_iterations, ndim=args.ndim, limit=args.n_samples, + start_with_box=start_with_box, n_iterations=args.n_iterations, ndim=args.ndim, shard=shard, + score_only=args.score_only, limit=args.n_samples, ) elif args.method == "sam": diff --git a/finetuning/v2/evaluation/evaluate_interactive_segmentation.py b/finetuning/v2/evaluation/evaluate_interactive_segmentation.py index b5718c360..1a332fe6e 100644 --- a/finetuning/v2/evaluation/evaluate_interactive_segmentation.py +++ b/finetuning/v2/evaluation/evaluate_interactive_segmentation.py @@ -41,7 +41,8 @@ from common import ( CLICK_PROTOCOLS, DATA_ROOT, DATASETS_2D, DATASETS_3D, MODEL_TYPES, CHECKPOINT_PATHS, check_data_download, checkpoint_checksum, export_joint_checkpoint, get_joint_checkpoint, - interactive_result_name, interactive_run_tag, load_data, n_samples, run_dataset_evaluation, + interactive_result_name, interactive_run_tag, interactive_sample_ids_3d, load_data, n_samples, + run_dataset_evaluation, ) @@ -232,7 +233,7 @@ def run_interactive_evaluation_2d( def run_interactive_evaluation_3d( dataset_name, data_root, experiment_folder, device, model_type, checkpoint_path, tag, legacy_tag, start_with_box=True, n_iterations=8, min_size=0, crop_shape=None, shard=None, score_only=False, limit=None, - correction_margin=0.0, + correction_margin=0.0, click_protocol="random", ): """Run iterative prompting on the 3d test split and write one result CSV per iteration. @@ -249,12 +250,14 @@ def run_interactive_evaluation_3d( save_paths = [ os.path.join(results_dir, interactive_result_name( dataset_name, tag, model_type, prompt, iteration, ndim=3, min_size=min_size, + click_protocol=click_protocol, )) for iteration in range(n_iterations) ] legacy_paths = [ os.path.join(results_dir, interactive_result_name( dataset_name, legacy_tag, model_type, prompt, iteration, ndim=3, min_size=min_size, + click_protocol=click_protocol, )) for iteration in range(n_iterations) ] @@ -264,15 +267,13 @@ def run_interactive_evaluation_3d( # Keyed by model type and dataset, since the cached predictions are reused across runs and the # per-sample names are otherwise identical for every dataset. 'min_size' changes the ground truth - # and therefore the prompts, so it has to key the cache too. + # and therefore the prompts, and the click protocol changes the prompts, so both key the cache too. prediction_root = os.path.join( experiment_folder, "predictions", f"{tag}_{model_type}", - dataset_name if not min_size else f"{dataset_name}_min{min_size}", + f"{dataset_name}{interactive_run_tag(3, min_size=min_size, click_protocol=click_protocol)}", ) - total = n_samples(dataset_name, data_root) - if limit is not None: - total = min(total, limit) - sample_ids = range(total) if shard is None else range(shard[0], total, shard[1]) + selected = interactive_sample_ids_3d(dataset_name, data_root, limit) + sample_ids = selected if shard is None else selected[shard[0]::shard[1]] samples = load_data( dataset_name, data_root, 3, min_size=min_size, crop_shape=crop_shape, sample_ids=set(sample_ids) ) @@ -305,6 +306,7 @@ def run_interactive_evaluation_3d( device=device, n_iterations=n_iterations, correction_margin=correction_margin, + click_protocol=click_protocol, ) all_gt.append(labels) all_valid_rois.append(valid_roi) @@ -361,8 +363,9 @@ def main(): ) parser.add_argument( "--click_protocol", type=str, default="random", choices=CLICK_PROTOCOLS, - help="2d only. The correction clicks: 'random' positions in the error regions, or the 'center' of the " - "largest error regions with a single mask for the first point, see 'CLICK_PROTOCOLS'.", + help="The correction clicks: 'random' positions in the error regions, or the 'center' of the largest " + "error regions with a single mask for the first point in 2d, see 'CLICK_PROTOCOLS'. In 3d the clicks " + "go on the worst slice of an object.", ) parser.add_argument("--crop_3d", type=int, nargs=3, default=None, help="Override the 3d center crop (Z Y X).") parser.add_argument( @@ -388,8 +391,6 @@ def main(): raise ValueError("--score_only scores the whole dataset, so it cannot be combined with a shard.") ndim = args.ndim or (3 if args.dataset_name in DATASETS_3D else 2) - if ndim == 3 and args.click_protocol != "random": - raise ValueError("--click_protocol applies to 2d only.") check_data_download(args.dataset_name, args.input_path) @@ -416,6 +417,7 @@ def main(): start_with_box=(args.prompt_choice == "box"), n_iterations=args.n_iterations, min_size=args.min_size, crop_shape=tuple(args.crop_3d) if args.crop_3d else None, shard=shard, score_only=args.score_only, limit=args.n_samples, correction_margin=args.correction_margin, + click_protocol=args.click_protocol, ) diff --git a/finetuning/v2/evaluation/evaluate_interactive_tracking.py b/finetuning/v2/evaluation/evaluate_interactive_tracking.py new file mode 100644 index 000000000..778d505bb --- /dev/null +++ b/finetuning/v2/evaluation/evaluate_interactive_tracking.py @@ -0,0 +1,291 @@ +"""Interactive tracking with micro-sam2: a simulated user drives a headless copy of the tracking annotator. + +The user annotates every ground-truth lineage of a movie as described in `interactive_tracking.SimulatedUser`: a box +on the first frame of each track, a stop where it ends, a division where it divides, and corrections on the worst +frame. The committed result for each correction budget is written in CTC format and scored with traccuracy (TRA, DET, +LNK, divisions with an IoU matching) and the SEG score over all frames. + +The datasets are in `tracking_util.INTERACTIVE_DATASETS`. '--collect' joins the result tables of all runs and, with +'--figure_folder', writes the central table 'cells_interactive_tracking_results.csv'. + +Usage examples: + python evaluate_interactive_tracking.py -d dynamicnuclearnet -s HeLa-S3_nuc_pos4_q1_5c + python evaluate_interactive_tracking.py -d tracking_fm_real -s microb_s3z1 --n_corrections 2 + python evaluate_interactive_tracking.py -d dynamicnuclearnet --list + python evaluate_interactive_tracking.py --collect --figure_folder ../../../scripts/plotting/v2/for_paper/figures +""" + +import os +import glob +import json +import time +import pickle +import shutil +import argparse +import tempfile +import warnings + +import tifffile +import numpy as np +import pandas as pd +from tqdm import tqdm + +import torch + +from micro_sam.v1.multi_dimensional_segmentation import export_tracking_result_to_ctc + +from common import use_tmpdir_for_embeddings +from interactive_tracking import SimulatedUser, TrackingAnnotator, V1TrackingAnnotator +from tracking_util import ( + MICROSAM_TRAINING, modality, scores_of, write_tables, + EXPERIMENT_ROOT, INTERACTIVE_DATASETS, dense_seg_score, interactive_samples, load_interactive_sample, + set_norm_percentiles, split_track_gaps, tracking_metrics, write_ctc_tracks, +) + + +def export_budgets(user, gt_masks, gt_tracks, budgets, pred_root, info): + """Export the ground-truth and the result of each correction budget in CTC format. + + The prompt counts and timings go to 'info.json', and 'complete' marks that all budgets are written, so a later + run can score again without annotating again. + """ + write_ctc_tracks(gt_masks, gt_tracks, os.path.join(pred_root, "gt", "TRA")) + info = dict(info, n_prompts={}) + for budget, committed, lineages, n_prompts in user.results_per_budget(budgets): + tracks = committed.copy() + pred_folder = os.path.join(pred_root, f"corrections_{budget}") + export_tracking_result_to_ctc(tracks, split_track_gaps(tracks, lineages), pred_folder) + info["n_prompts"][str(budget)] = n_prompts + with open(os.path.join(pred_root, "info.json"), "w") as f: + json.dump(info, f) + open(os.path.join(pred_root, "complete"), "w").close() + + +def score_budgets(gt_masks, budgets, pred_root): + """Score the exported result of each correction budget. A budget whose scoring fails gets NaN, with a warning.""" + with open(os.path.join(pred_root, "info.json")) as f: + info = json.load(f) + gt_folder = os.path.join(pred_root, "gt", "TRA") + rows = [] + for budget in budgets: + pred_folder = os.path.join(pred_root, f"corrections_{budget}", "TRA") + try: + scores = tracking_metrics(gt_folder, pred_folder, dense_gt=True) + except Exception as e: + warnings.warn(f"The tracking metrics of budget {budget} in {pred_root} failed: {e}") + scores = {"TRA": np.nan, "DET": np.nan, "LNK": np.nan} + paths = sorted(glob.glob(os.path.join(pred_folder, "man_track*.tif"))) + pred = np.stack([tifffile.imread(path) for path in paths]) + scores["SEG"] = dense_seg_score(gt_masks, pred) + rows.append({"n_corrections": budget, "n_prompts": info["n_prompts"][str(budget)], **scores}) + return info, rows + + +# The interactive tools: the micro-sam2 annotator with the micro-sam2 model or stock SAM2, and the micro-sam v1 +# annotator with its light microscopy model. Each has its default model. +METHODS = {"microsam2": "hvit_l_cells", "sam2": "hvit_l", "microsam_v1": "vit_l_lm"} + + +def merge_parts(user, parts_folder): + """Put the lineage records of all annotated ranges into 'user', in the order of its roots. + + Returns: + The summed embedding and annotation times, or None if a range is missing. + """ + n_roots = len(user.roots()) + parts = sorted(glob.glob(os.path.join(parts_folder, "part_*.pkl"))) + covered, records, times = 0, [], {"embedding_time": 0.0, "annotation_time": 0.0} + for path in parts: + with open(path, "rb") as f: + part = pickle.load(f) + if part["start"] != covered: + break + covered = part["end"] + records += part["records"] + for key in times: + times[key] += part[key] + if covered < n_roots: + print(f"The ranges in '{parts_folder}' cover {covered} of {n_roots} lineages.") + return None + user.lineage_records = records + return times + + +def run_sample( + dataset_name, sample, model_type, first_prompt, n_corrections, accept_iou, output_root, correction, + norm_percentiles=None, method="microsam2", lineages=None, merge=False, +): + """Annotate one movie with the simulated user and return its rows of scores. + + With 'lineages' (start, end), only that range of the lineages is annotated and its records are stored; 'merge' + then joins the records of all ranges, exports and scores them. The result equals that of a run in one piece. + """ + # The point corrections keep the plain name of the first runs. + correction_tag = "" if correction == "points" else f"_{correction}corr" + tag = f"{dataset_name}_{sample}_{model_type}_{first_prompt}{correction_tag}_c{n_corrections}_iou{accept_iou}" + if norm_percentiles is not None: + tag += f"_norm{norm_percentiles[0]}-{norm_percentiles[1]}" + result_path = os.path.join(output_root, "results", f"{tag}.csv") + if os.path.exists(result_path): + print(f"The result is already stored at '{result_path}'.") + return pd.read_csv(result_path) + + raw, gt_masks, gt_tracks = load_interactive_sample(dataset_name, sample) + print(f"{dataset_name} {sample}: {raw.shape} {raw.dtype}, {len(gt_tracks)} tracks") + budgets = range(n_corrections + 1) + pred_root = os.path.join(output_root, "predictions", tag) + parts_folder = os.path.join(pred_root, "lineage_parts") + if lineages is not None: + part_path = os.path.join(parts_folder, f"part_{lineages[0]:05d}-{lineages[1]:05d}.pkl") + if os.path.exists(part_path): + print(f"The lineages {lineages} are already stored at '{part_path}'.") + return None + if os.path.exists(os.path.join(pred_root, "complete")): + print(f"The annotation is already exported at '{pred_root}'. It is only scored.") + elif merge: + user = SimulatedUser( + None, gt_masks, gt_tracks, first_prompt=first_prompt, n_corrections=n_corrections, accept_iou=accept_iou, + correction=correction, + ) + info = merge_parts(user, parts_folder) + if info is None: + raise RuntimeError(f"Not all lineage ranges of {tag} are annotated yet.") + export_budgets(user, gt_masks, gt_tracks, budgets, pred_root, info) + else: + device = "cuda" if torch.cuda.is_available() else "cpu" + embedding_dir = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + try: + start = time.time() + annotator_class = V1TrackingAnnotator if method == "microsam_v1" else TrackingAnnotator + annotator = annotator_class(raw, model_type, os.path.join(embedding_dir, "embeddings.zarr"), device) + embedding_time = time.time() - start + + start = time.time() + user = SimulatedUser( + annotator, gt_masks, gt_tracks, first_prompt=first_prompt, n_corrections=n_corrections, + accept_iou=accept_iou, correction=correction, + ) + user.run(progress=lambda roots: tqdm(roots, desc="Annotate lineages"), lineages=lineages) + annotation_time = time.time() - start + finally: + shutil.rmtree(embedding_dir, ignore_errors=True) + info = {"embedding_time": embedding_time, "annotation_time": annotation_time} + if lineages is not None: + os.makedirs(parts_folder, exist_ok=True) + part = { + "start": lineages[0], "end": min(lineages[1], len(user.roots())), "records": user.lineage_records, + **info + } + with open(part_path + ".partial", "wb") as f: + pickle.dump(part, f) + os.replace(part_path + ".partial", part_path) + print(f"Stored the lineages {lineages} at '{part_path}'.") + return None + export_budgets(user, gt_masks, gt_tracks, budgets, pred_root, info) + + info, rows = score_budgets(gt_masks, budgets, pred_root) + result = pd.DataFrame([ + { + "dataset": dataset_name, "domain": INTERACTIVE_DATASETS[dataset_name], "sample": sample, + "method": method, "model": model_type, "first_prompt": first_prompt, "correction": correction, + "accept_iou": accept_iou, + "norm_percentiles": "2,98" if norm_percentiles is None else ",".join(map(str, norm_percentiles)), + "n_frames": raw.shape[0], "n_gt_tracks": len(gt_tracks), "embedding_time": info["embedding_time"], + "annotation_time": info["annotation_time"], **row, + } + for row in rows + ]) + os.makedirs(os.path.dirname(result_path), exist_ok=True) + result.to_csv(result_path, index=False) + print(result.T.to_string(header=False)) + return result + + +# The (method, model) names of the central table, as in the segmentation tables. +TABLE_METHODS = { + "hvit_l_cells": ("micro-sam2", "hvit_l"), "hvit_l": ("sam2", "hvit_l"), "vit_l_lm": ("micro-sam", "vit_l_lm") +} + + +def central_interactive(table): + """The interactive table of the paper: one row per dataset, method, first prompt, correction and iteration.""" + # Only the default 2/98 percentile normalization; the 1/99.8 runs were tests. + table = table[table["norm_percentiles"].isna() | (table["norm_percentiles"] == "2,98")].copy() + table["model"], table["method"] = ( + [TABLE_METHODS[model][1] for model in table["model"]], + [TABLE_METHODS[model][0] for model in table["model"]], + ) + # The first runs corrected with points and have no correction column. + table["correction"] = table["correction"].fillna("points") + table = table.rename(columns={"first_prompt": "prompt", "n_corrections": "iteration"}) + keys = ["dataset", "domain", "method", "model", "prompt", "correction", "accept_iou", "iteration"] + result = scores_of(table, keys) + result["n_prompts"] = table.groupby(keys, dropna=False)["n_prompts"].mean().values + result.insert(1, "category", [modality(d)[0] for d in result["dataset"]]) + result.insert(2, "structure", [modality(d)[1] for d in result["dataset"]]) + result["domain"] = result["domain"].map({"in-domain": "ID", "out-of-domain": "OOD"}) + result.insert(4, "in_microsam_training", result["dataset"].isin(MICROSAM_TRAINING)) + result["primary_metric"], result["lower_is_better"] = "tra", False + return result.sort_values(["dataset", "method", "correction", "iteration"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("-d", "--dataset_name", type=str, default=None, choices=sorted(INTERACTIVE_DATASETS)) + parser.add_argument("-s", "--sample", type=str, default=None, help="The movie. All movies by default.") + parser.add_argument( + "--method", type=str, default="microsam2", choices=sorted(METHODS), + help="The tool: the micro-sam2 annotator ('microsam2', 'sam2' with stock SAM2) or the micro-sam v1 annotator.", + ) + parser.add_argument("-m", "--model_type", type=str, default=None, help="By default the model of the method.") + parser.add_argument("--first_prompt", type=str, default="box", choices=("box", "point")) + parser.add_argument( + "--lineages", type=int, nargs=2, default=None, + help="Only annotate the lineages START to END (exclusive) and store them, to split a long movie.", + ) + parser.add_argument("--merge", action="store_true", help="Merge, export and score the stored lineage ranges.") + parser.add_argument("--n_corrections", type=int, default=4, help="The largest number of corrections per track.") + parser.add_argument( + "--correction", type=str, default="box", choices=("points", "box"), + help="How the user corrects the worst frame: a positive and a negative point, or a new box.", + ) + parser.add_argument( + "--norm_percentiles", type=float, nargs=2, default=None, + help="Lower and upper percentile of the input normalization instead of 2 and 98, e.g. 1 99.8.", + ) + parser.add_argument("--accept_iou", type=float, default=0.9, help="The user accepts a track above this IoU.") + parser.add_argument("-o", "--output_root", type=str, default=os.path.join(EXPERIMENT_ROOT, "interactive")) + parser.add_argument("--list", action="store_true", help="Print the movies of the dataset and exit.") + parser.add_argument("--collect", action="store_true", help="Collect the result tables of all runs.") + parser.add_argument("--figure_folder", default=None, help="Where --collect writes the central table.") + args = parser.parse_args() + if args.collect: + keys = ["dataset", "domain", "sample", "n_corrections", "n_prompts"] + root = os.path.dirname(args.output_root.rstrip("/")) + write_tables("interactive", keys, central_interactive, root, args.figure_folder) + return + + if args.dataset_name is None: + parser.error("Give a dataset with -d, or use --collect.") + samples = interactive_samples(args.dataset_name) + if args.list: + print("\n".join(samples)) + return + if args.sample is not None and args.sample not in samples: + parser.error(f"Choose one of the movies: {samples}") + + use_tmpdir_for_embeddings() + if args.norm_percentiles is not None: + set_norm_percentiles(*args.norm_percentiles) + model_type = args.model_type or METHODS[args.method] + for sample in samples if args.sample is None else [args.sample]: + run_sample( + args.dataset_name, sample, model_type, args.first_prompt, args.n_corrections, args.accept_iou, + args.output_root, args.correction, + norm_percentiles=None if args.norm_percentiles is None else tuple(args.norm_percentiles), + method=args.method, lineages=None if args.lineages is None else tuple(args.lineages), merge=args.merge, + ) + + +if __name__ == "__main__": + main() diff --git a/finetuning/v2/evaluation/interactive_tracking.py b/finetuning/v2/evaluation/interactive_tracking.py new file mode 100644 index 000000000..4c0c856c8 --- /dev/null +++ b/finetuning/v2/evaluation/interactive_tracking.py @@ -0,0 +1,590 @@ +"""A headless copy of the micro-sam2 tracking annotator and a simulated user that drives it. + +`TrackingAnnotator` keeps the state of the tracking annotator ('micro_sam.sam_annotator.annotator_tracking') without +napari: the point and box prompts with their track id and track state, the 'current_object' and 'committed_objects' +layers and the lineage. Its actions copy the annotator's: +- `segment` is 'Segment Object' with 'Apply to All Frames' ('UnifiedSegmentWidget._run_tracking'): propagate the + prompts of the current track forward in time, bounded by a stop annotation or a division. +- a 'division' point ends the mother track at its frame and seeds two daughter tracks ('_update_lineage'). +- `commit` is 'Commit' ('commit_track' / '_commit_impl'): move the lineage to the committed objects. +The predictor and the interactive segmenter come from 'AnnotatorState.initialize_predictor', as in the annotator. + +`SimulatedUser` annotates every lineage of the ground-truth like a human would: a box (or point) on the first frame +of a track, corrections on the worst frame, a stop where the track ends, and a division where it divides. +""" + +import hashlib +from dataclasses import dataclass + +import numpy as np +from scipy import ndimage + +from micro_sam.v2.util import resolve_default_tiling +from micro_sam.sam_annotator._state import AnnotatorState + + +@dataclass +class PointPrompt: + t: int + y: float + x: float + label: int # 1 positive, 0 negative. + state: str # 'track' or 'division'. + track_id: int + + +@dataclass +class BoxPrompt: + t: int + box: np.ndarray # (y0, x0, y1, x1). + track_id: int + + +def commit_objects(committed, committed_lineages, current_object, lineage): + """Commit a lineage like 'commit_track' with the default 'pixels' preserve mode. + + The track ids of the lineage are offset by the largest committed id, and committed pixels are not overwritten. + """ + id_offset = int(committed.max()) + mask = (current_object != 0) & (committed == 0) + committed[mask] = current_object[mask] + id_offset + committed_lineages.append( + {parent + id_offset: [child + id_offset for child in children] for parent, children in lineage.items()} + ) + + +class TrackingAnnotator: + """The state and the actions of the tracking annotator, without napari. + + Args: + image: The timeseries, (T, Y, X). + model_type: The SAM2 model, e.g. 'hvit_l_cells'. + embedding_path: Where the embeddings are cached. None caches them in a temporary folder. + device: The torch device. + """ + + def __init__(self, image, model_type, embedding_path=None, device=None): + state = AnnotatorState() + state.reset_state() + state.image_shape = image.shape + tile_shape, halo = resolve_default_tiling(image.shape, None, None, embedding_path) + state.initialize_predictor( + image, model_type=model_type, save_path=embedding_path, halo=halo, tile_shape=tile_shape, + prefer_decoder=True, ndim=3, device=device, use_cli=True, + ) + self.segmenter = state.interactive_segmenter + self.shape = image.shape + self.committed_objects = np.zeros(self.shape, dtype="uint32") + self.committed_lineages = [] + self.reset_track_state() + + def reset_track_state(self): + """Clear the prompts, the current object and the lineage ('_reset_tracking_state' and 'clear_annotations').""" + self.current_track_id = 1 + self.lineage = {1: []} + self.current_object = np.zeros(self.shape, dtype="uint32") + self.points, self.boxes = [], [] + # Every change of the current object, as (tag, kind, track id, flat indices or first frame). The caller sets + # 'log_tag', so that the changes can be replayed in part (see 'replay'). + self.log, self.log_tag = [], 0 + + def add_point(self, t, yx, positive=True, state="track", track_id=None): + """Place a point prompt for a track, by default the current one.""" + track_id = self.current_track_id if track_id is None else track_id + self.points.append(PointPrompt(int(t), float(yx[0]), float(yx[1]), int(positive), state, int(track_id))) + + def add_box(self, t, box, track_id=None): + """Place a box prompt, (y0, x0, y1, x1), for a track, by default the current one.""" + track_id = self.current_track_id if track_id is None else track_id + self.boxes.append(BoxPrompt(int(t), np.asarray(box, dtype="float64"), int(track_id))) + + def remove_prompts(self, t, track_id=None): + """Delete the point and box prompts of a track on a frame, as a user deletes them in the layers. + + The division markers stay. + """ + track_id = self.current_track_id if track_id is None else track_id + self.points = [p for p in self.points if not (p.t == t and p.track_id == track_id and p.state != "division")] + self.boxes = [b for b in self.boxes if not (b.t == t and b.track_id == track_id)] + + def add_division(self, t, yx, track_id=None): + """Mark the division of a track on its last frame, with a point in the 'division' state.""" + self.add_point(t, yx, positive=True, state="division", track_id=track_id) + + def _frame_points(self, t, track_id): + """Get the point prompts of a track on a frame without the division markers, and whether they are a stop.""" + points = [p for p in self.points if p.t == t and p.track_id == track_id and p.state != "division"] + has_box = any(b.t == t and b.track_id == track_id for b in self.boxes) + # A lone negative point beside no box is a stop annotation ('collect_frame_prompts'). + is_stop = len(points) == 1 and points[0].label == 0 and not has_box + return points, is_stop + + def _division_frame(self, track_id): + frames = [p.t for p in self.points if p.state == "division" and p.track_id == track_id] + return min(frames) if frames else None + + def _mother_division_frame(self, track_id): + for mother, daughters in self.lineage.items(): + if track_id in daughters: + return self._division_frame(mother) + return None + + def _update_lineage(self, mother): + """Seed two daughter tracks of a dividing track, once.""" + if mother not in self.lineage or len(self.lineage[mother]) > 0: + return + daughter1 = max(self.lineage.keys()) + 1 + daughter2 = daughter1 + 1 + self.lineage[mother] = [daughter1, daughter2] + self.lineage[daughter1] = [] + self.lineage[daughter2] = [] + + def _propagate_track(self, track_id, division_frame): + """Propagate the prompts of one track forward in time ('propagate_track').""" + self.segmenter.reset_predictor() + + prompted_frames, stop_frames = [], [] + have_positive_cue = False + for t in sorted({p.t for p in self.points}): + points, is_stop = self._frame_points(t, track_id) + if is_stop: + stop_frames.append(t) + continue + if not points: + continue + labels = np.array([p.label for p in points]) + have_positive_cue = have_positive_cue or bool(np.any(labels == 1)) + self.segmenter.add_point_prompts( + frame_ids=t, points=np.array([[p.y, p.x] for p in points]), point_labels=labels, + ) + prompted_frames.append(t) + + for t in sorted({b.t for b in self.boxes}): + boxes = [b.box for b in self.boxes if b.t == t and b.track_id == track_id] + for box in boxes: + self.segmenter.add_box_prompts(frame_ids=t, boxes=[box]) + if boxes: + have_positive_cue = True + prompted_frames.append(t) + + if not prompted_frames or not have_positive_cue: + return None + + annotated = sorted(set(prompted_frames) | set(stop_frames)) + stop_upper = annotated[-1] in stop_frames + z_lo = min(prompted_frames) + z_hi = max(prompted_frames) if stop_upper else self.shape[0] - 1 + if division_frame is not None: + z_hi = min(z_hi, division_frame) + if z_hi < z_lo: + return None + return self.segmenter.predict(early_stop_patience=None, z_range=(z_lo, z_hi)) + + def segment(self): + """Run 'Segment Object' for the current track over all frames ('_run_tracking').""" + track_id = self.current_track_id + division_frame = self._division_frame(track_id) + segmented = division_frame is not None and bool( + np.any(self.current_object[:division_frame + 1] == track_id) + ) + if segmented: + # A division of a segmented track only erases it after the division frame. + after = self.current_object[division_frame + 1:] + after[after == track_id] = 0 + self.log.append((self.log_tag, "truncate", track_id, division_frame + 1)) + self._update_lineage(track_id) + return + + seg = self._propagate_track(track_id, division_frame) + if seg is not None: + mother_division = self._mother_division_frame(track_id) + if mother_division is not None: + seg = seg.copy() + seg[:mother_division + 1] = 0 + self.current_object[self.current_object == track_id] = 0 + self.current_object[seg == 1] = track_id + self.log.append((self.log_tag, "write", track_id, np.flatnonzero(seg == 1))) + if division_frame is not None: + self._update_lineage(track_id) + + @staticmethod + def replay(log, shape, max_tag): + """Rebuild the current object from the logged changes whose tag is at most 'max_tag'.""" + current = np.zeros(shape, dtype="uint32") + for tag, kind, track_id, value in log: + if tag > max_tag: + continue + if kind == "write": + current[current == track_id] = 0 + current.flat[value] = track_id + else: + after = current[value:] + after[after == track_id] = 0 + return current + + def commit(self): + """Run 'Commit' for the current lineage and reset the tracking state.""" + commit_objects(self.committed_objects, self.committed_lineages, self.current_object, self.lineage) + self.segmenter.reset_predictor() + self.reset_track_state() + + +class V1TrackingAnnotator: + """The state and the actions of the micro-sam v1 tracking annotator, without napari. + + Copies the tracking of micro-sam v1 (before 'Refactor tracking annotator', #1251): the frames with prompts of the + current track are segmented from their prompts ('segment_slices_with_prompts'), then the track is propagated + frame by frame from the first of them with the mask of the previous frame and a motion model + ('track_from_prompts'), with the defaults of its widget. A frame with a 'division' point stops the track before + it, so a user marks a division on the first frame of the daughters. It has the interface of `TrackingAnnotator`. + + Args: + image: The timeseries, (T, Y, X). + model_type: The micro-sam v1 model, e.g. 'vit_l_lm'. + embedding_path: Where the embeddings are cached. + device: The torch device. + """ + + # The defaults of the v1 tracking widget. + projection = "single_point" + iou_threshold = 0.5 + box_extension = 0.05 + motion_smoothing = 0.5 + + def __init__(self, image, model_type="vit_l_lm", embedding_path=None, device=None): + from micro_sam.v1.util import get_sam_model, precompute_image_embeddings + self.predictor = get_sam_model(model_type=model_type, device=device) + self.image_embeddings = precompute_image_embeddings( + self.predictor, image, save_path=embedding_path, ndim=3, verbose=True, + ) + self.shape = image.shape + self.committed_objects = np.zeros(self.shape, dtype="uint32") + self.committed_lineages = [] + self.reset_track_state() + + reset_track_state = TrackingAnnotator.reset_track_state + add_point = TrackingAnnotator.add_point + add_box = TrackingAnnotator.add_box + remove_prompts = TrackingAnnotator.remove_prompts + _update_lineage = TrackingAnnotator._update_lineage + replay = TrackingAnnotator.replay + + def add_division(self, t, yx, track_id=None): + """Mark the division of a track whose last frame is t: v1 takes the marker on the next frame.""" + self.add_point(t + 1, yx, positive=True, state="division", track_id=track_id) + + def _frame_prompts(self, t, track_id): + """The point prompts (y, x), labels and boxes of a track on a frame; division points are prompts in v1.""" + points = [p for p in self.points if p.t == t and p.track_id == track_id] + boxes = [b.box for b in self.boxes if b.t == t and b.track_id == track_id] + return np.array([[p.y, p.x] for p in points]).reshape(-1, 2), np.array([p.label for p in points]), boxes + + def _segment_prompted_frames(self, track_id): + """Segment the frames with prompts of a track ('segment_slices_with_prompts').""" + from micro_sam.v1 import prompt_based_segmentation as pbs + seg = np.zeros(self.shape, dtype="uint32") + frames = sorted( + {p.t for p in self.points if p.track_id == track_id} | {b.t for b in self.boxes if b.track_id == track_id} + ) + stop_upper = False + for t in list(frames): + points, labels, boxes = self._frame_prompts(t, track_id) + if len(points) == 1 and labels[0] == 0 and not boxes: # A lone negative point is a stop annotation. + if t == frames[-1]: + stop_upper = True + elif t != frames[0]: + frames.remove(t) + continue + emb = dict(image_embeddings=self.image_embeddings, i=t) + if len(points) and boxes: + pred = pbs.segment_from_box_and_points(self.predictor, boxes[0], points, labels, **emb) + elif boxes: + pred = pbs.segment_from_box(self.predictor, boxes[0], **emb) + else: + pred = pbs.segment_from_points(self.predictor, points, labels, **emb) + seg[t] = np.asarray(pred).squeeze() > 0 + return seg, np.array(frames, dtype="int"), stop_upper + + def _is_division(self, t): + """'prompt_layer_to_state': a frame is in the division state if any point on it is.""" + return any(p.t == t and p.state == "division" for p in self.points) + + def _track(self, seg, frames, stop_upper): + """Propagate the track from its first prompted frame ('track_from_prompts').""" + from scipy.ndimage import shift + from micro_sam.util import compute_iou + from micro_sam.v1 import prompt_based_segmentation as pbs + from micro_sam.v1.multi_dimensional_segmentation import _validate_projection + use_box, use_mask, use_points, use_single_point = _validate_projection(self.projection) + + def center(t): + ys, xs = np.where(seg[t] == 1) + return np.array([ys.mean(), xs.mean()]) + + motion_model, has_division = None, False + t0 = int(frames.min()) + t = t0 + 1 + while t < seg.shape[0]: + if t == t0 + 2: + motion_model = center(t - 1) - center(t - 2) + elif t > t0 + 2: + motion_model = self.motion_smoothing * motion_model + (1 - self.motion_smoothing) * ( + center(t - 1) - center(t - 2) + ) + if t in frames: + seg_prev, seg_t = None, seg[t] + state = "division" if self._is_division(t) else "track" + else: + seg_prev = seg[t - 1] + if not seg_prev.any(): # The track is lost: SAM has no mask to continue from. + break + if motion_model is not None and not np.isnan(motion_model).any(): + shifted = np.zeros_like(seg_prev) + shift(seg_prev, motion_model, output=shifted, order=0, prefilter=False) + seg_prev = shifted + seg_t = pbs.segment_from_mask( + self.predictor, seg_prev, image_embeddings=self.image_embeddings, i=t, use_mask=use_mask, + use_box=use_box, use_points=use_points, box_extension=self.box_extension, + use_single_point=use_single_point, + ) + seg_t = (np.asarray(seg_t).squeeze() > 0).astype(seg.dtype) + state = "track" + if t < frames[-1]: + seg_prev = None + if seg_prev is not None and compute_iou(seg_prev, seg_t) < self.iou_threshold: + break + if state == "division": + has_division = True + break + seg[t] = seg_t + t += 1 + if t == frames[-1] and stop_upper: + break + return seg, has_division + + def segment(self): + """Run 'Track Object' of the v1 annotator for the current track.""" + track_id = self.current_track_id + seg, frames, stop_upper = self._segment_prompted_frames(track_id) + if len(frames) == 0: + return + seg, has_division = self._track(seg, frames, stop_upper) + if has_division and len(self.lineage.get(track_id, [])) == 0: + self._update_lineage(track_id) + self.current_object[self.current_object == track_id] = 0 + self.current_object[seg == 1] = track_id + self.log.append((self.log_tag, "write", track_id, np.flatnonzero(seg == 1))) + + def commit(self): + commit_objects(self.committed_objects, self.committed_lineages, self.current_object, self.lineage) + self.reset_track_state() + + +def center_of(mask): + """Get the point of a mask farthest from its boundary, in its largest component, as (y, x).""" + components, n = ndimage.label(mask) + if n > 1: + sizes = ndimage.sum_labels(mask, components, index=np.arange(1, n + 1)) + mask = components == (np.argmax(sizes) + 1) + distances = ndimage.distance_transform_edt(np.pad(mask, 1))[1:-1, 1:-1] + return np.unravel_index(np.argmax(distances), mask.shape) + + +def box_of(mask): + """Get the bounding box of a mask, (y0, x0, y1, x1).""" + ys, xs = np.where(mask) + return np.array([ys.min(), xs.min(), ys.max() + 1, xs.max() + 1]) + + +def digest(array): + """Get a hash of an array, to compare a large result without keeping a copy.""" + return hashlib.blake2b(np.ascontiguousarray(array).data).hexdigest() + + +def iou(a, b): + union = np.logical_or(a, b).sum() + return 1.0 if union == 0 else np.logical_and(a, b).sum() / union + + +class SimulatedUser: + """A simulated user who annotates every ground-truth lineage with the tracking annotator. + + For each track of a lineage, from the root down to the daughters: + 1. A box (or a point) on the first frame of the track, then 'Segment Object'. + 2. Where the track ends without dividing before the last frame, and the propagation goes on: a positive point on + its last frame and a stop (a lone negative point) on the next one. + 3. Up to 'n_corrections' corrections on the frame with the lowest IoU while it is below 'accept_iou'. With + 'points', a positive point in the largest missed region and a negative point in the largest false region, at + their centers. With 'box', the prompts of the track on that frame are deleted and its box is drawn again. + 4. Where the track divides: a division point on its last frame, then 'Segment Object' seeds the daughters. + Then 'Commit'. The corrections come before the division, because the annotator only truncates a segmented + mother once its division is marked and does not propagate it again. + + The changes of the annotator are tagged with the number of the correction of the track, so the result for a + smaller correction budget follows from the same run by a replay (see `results_per_budget`). + + Args: + annotator: The `TrackingAnnotator`. + gt_masks: The dense ground-truth track masks, (T, Y, X). + gt_tracks: The ground-truth lineage table, with the columns 'track', 'start', 'end', 'parent'. + first_prompt: 'box' or 'point'. + n_corrections: The largest number of corrections per track. + correction: 'points' or 'box', see above. + accept_iou: The IoU on every frame of a track above which the user accepts it. + """ + + def __init__( + self, annotator, gt_masks, gt_tracks, first_prompt="box", n_corrections=4, accept_iou=0.9, correction="box", + ): + if correction not in ("points", "box"): + raise ValueError(f"Invalid correction '{correction}', choose 'points' or 'box'.") + self.correction = correction + self.annotator = annotator + self.gt_masks = gt_masks + self.tracks = gt_tracks.set_index("track") + self.first_prompt = first_prompt + self.n_corrections = n_corrections + self.accept_iou = accept_iou + self.children = {track: [] for track in self.tracks.index} + for track, parent in self.tracks["parent"].items(): + if parent in self.children: + self.children[parent].append(track) + self.n_frames = gt_masks.shape[0] + self.lineage_records = [] + + def roots(self): + parents = self.tracks["parent"] + roots = [track for track in self.tracks.index if parents[track] not in self.children] + return sorted(roots, key=lambda track: (self.tracks.loc[track, "start"], track)) + + def _gt_frames(self, gt_track): + start, end = int(self.tracks.loc[gt_track, "start"]), int(self.tracks.loc[gt_track, "end"]) + return [t for t in range(start, end + 1) if np.any(self.gt_masks[t] == gt_track)] + + def _track_mask(self, track_id): + return self.annotator.current_object == track_id + + def _correct(self, track_id, gt_track, frames): + """Place one correction on the worst frame of a track; return the number of prompts, 0 if the user accepts.""" + pred = self._track_mask(track_id) + if self.correction == "box": + # A frame with the box of the track already has the box a redraw would give. + boxed = {b.t for b in self.annotator.boxes if b.track_id == track_id} + frames = [t for t in frames if t not in boxed] + if not frames: + return 0 + scores = [iou(pred[t], self.gt_masks[t] == gt_track) for t in frames] + if min(scores) >= self.accept_iou: + return 0 + t = frames[int(np.argmin(scores))] + gt, mask = self.gt_masks[t] == gt_track, pred[t] + if self.correction == "box": + self.annotator.remove_prompts(t) + self.annotator.add_box(t, box_of(gt)) + self.annotator.segment() + return 1 + missed, false = gt & ~mask, mask & ~gt + n_prompts = 0 + # A lone negative point would be a stop, so a positive point always comes with it. + positive = missed if missed.any() else gt + self.annotator.add_point(t, center_of(positive), positive=True) + n_prompts += 1 + if false.any(): + self.annotator.add_point(t, center_of(false), positive=False) + n_prompts += 1 + self.annotator.segment() + return n_prompts + + def annotate_track(self, track_id, gt_track): + """Annotate one track and return its record: the masks after each correction and the prompts placed.""" + annotator = self.annotator + annotator.current_track_id = track_id + annotator.log_tag = 0 + frames = self._gt_frames(gt_track) + start, end = frames[0], frames[-1] + first_mask = self.gt_masks[start] == gt_track + if self.first_prompt == "box": + annotator.add_box(start, box_of(first_mask)) + else: + annotator.add_point(start, center_of(first_mask), positive=True) + annotator.segment() + n_prompts = [1] + + children = self.children[gt_track] + if not children and end < self.n_frames - 1 and self._track_mask(track_id)[end + 1:].any(): + if end != start: + annotator.add_point(end, center_of(self.gt_masks[end] == gt_track), positive=True) + annotator.add_point(end + 1, center_of(self.gt_masks[end] == gt_track), positive=False) + annotator.segment() + n_prompts[0] += 1 if end == start else 2 + + for correction in range(1, self.n_corrections + 1): + annotator.log_tag = correction + placed = self._correct(track_id, gt_track, frames) + if placed == 0: + break + n_prompts.append(placed) + + division = None + annotator.log_tag = 0 + if children: + annotator.add_division(end, center_of(self.gt_masks[end] == gt_track)) + annotator.segment() + n_prompts[0] += 1 + division = end + return {"track_id": track_id, "gt_track": gt_track, "n_prompts": n_prompts, "division": division} + + def annotate_lineage(self, root): + """Annotate the lineage of a root track, commit it and keep the record of each track.""" + queue = [(root, 1)] + records = [] + while queue: + gt_track, track_id = queue.pop(0) + if not self._gt_frames(gt_track): + continue + record = self.annotate_track(track_id, gt_track) + records.append(record) + if record["division"] is not None: + daughters = self.annotator.lineage[track_id] + # A ground-truth track with more than two children keeps only the first two. + for gt_child, daughter in zip(sorted(self.children[gt_track]), daughters): + queue.append((gt_child, daughter)) + self.lineage_records.append({ + "lineage": {k: list(v) for k, v in self.annotator.lineage.items()}, "records": records, + "log": self.annotator.log, "final": digest(self.annotator.current_object), + }) + self.annotator.commit() + + def run(self, progress=None, lineages=None): + """Annotate all lineages, or only the roots lineages[0] to lineages[1] (exclusive) of `roots`. + + The lineages are independent until they are committed, so a range of them can be annotated on its own and the + records of all ranges merged in the order of `roots` (see `results_per_budget`). + """ + roots = self.roots() + if lineages is not None: + roots = roots[lineages[0]:lineages[1]] + for root in roots if progress is None else progress(roots): + self.annotate_lineage(root) + + def results_per_budget(self, budgets): + """Yield the committed result for each correction budget by a replay of the logged annotator changes. + + Each track is annotated to the end before the next one starts, so dropping the changes of the corrections + beyond a budget gives what a user with that budget would get. For the full budget this is the annotator's own + result, which is checked. The budgets are built one at a time to bound the memory. + + Yields: + The budget, the committed objects, the committed lineages and the number of prompts. + """ + shape = self.gt_masks.shape + for budget in budgets: + committed = np.zeros(shape, dtype="uint32") + committed_lineages, n_prompts = [], 0 + for entry in self.lineage_records: + n_prompts += sum(sum(record["n_prompts"][:budget + 1]) for record in entry["records"]) + current = TrackingAnnotator.replay(entry["log"], shape, budget) + if budget >= self.n_corrections and digest(current) != entry["final"]: + raise RuntimeError("The replayed lineage differs from the annotator's result.") + commit_objects(committed, committed_lineages, current, entry["lineage"]) + yield budget, committed, committed_lineages, n_prompts diff --git a/finetuning/v2/evaluation/parameter_search.py b/finetuning/v2/evaluation/parameter_search.py index 305fef8ec..e7b775d89 100644 --- a/finetuning/v2/evaluation/parameter_search.py +++ b/finetuning/v2/evaluation/parameter_search.py @@ -55,7 +55,6 @@ from bioimage_py.evaluation import symmetric_best_dice_score -import micro_sam.util from micro_sam.v2.postprocessing import watershed_heightmap, _compute_flow_density from micro_sam.v2.util import FINETUNED_MODELS, has_registered_decoder, models as model_registry @@ -64,29 +63,11 @@ EM_NUCLEUS_MOCK_2D_ROOT, GT_MIN_SIZE_2D, CROP_SHAPE_3D, DATA_ROOT, MODEL_TYPES, MODES, TUNING_CROPS, VOLUME_SPEED_OPTIONS, build_model, check_data_download, drop_severed_objects, em_roi, get_data_paths, has_val_split, load_data, load_evaluation_sample_2d, load_evaluation_sample_3d, n_samples, predict_unisam2, - read_tuned_params, resolve_checkpoint_identity, sorted_path_pairs, val_z_range, _read_crops, _read_window, + read_tuned_params, resolve_checkpoint_identity, sorted_path_pairs, use_tmpdir_for_embeddings, val_z_range, + _read_crops, _read_window, ) -# The temporary embeddings of 3d APG go to the job's node-local TMPDIR. In the micro-sam cache of the home -# directory, the leftovers of jobs killed at their time limit filled the quota (2026-10-22). -if os.environ.get("TMPDIR"): - import uuid - import atexit - import shutil - import micro_sam.v2.util - import micro_sam.v2.instance_segmentation - import micro_sam.v2.automatic_prompt_generation - - def _make_temp_embedding_path(): - parent = Path(os.environ["TMPDIR"], "micro_sam_tmp_embeddings") - parent.mkdir(parents=True, exist_ok=True) - path = str(parent / f"{uuid.uuid4().hex}.zarr") - atexit.register(shutil.rmtree, path, ignore_errors=True) - return path - - for _module in (micro_sam.util, micro_sam.v2.util, micro_sam.v2.instance_segmentation, - micro_sam.v2.automatic_prompt_generation): - _module.make_temp_embedding_path = _make_temp_embedding_path +use_tmpdir_for_embeddings() # Sparse (flow) grid for LM data. Keys map to flow_instance_segmentation arguments. LM_GRID = { diff --git a/finetuning/v2/evaluation/runtime_benchmarking_script.py b/finetuning/v2/evaluation/runtime_benchmarking_script.py new file mode 100644 index 000000000..0ae438fb9 --- /dev/null +++ b/finetuning/v2/evaluation/runtime_benchmarking_script.py @@ -0,0 +1,743 @@ +"""Runtime benchmark of micro-sam2 and its references on CPU, GPU and Apple silicon (Extended Data Figure 2). + +The benchmark times the evaluation pipelines of 'evaluate_automatic_segmentation.py' (AIS and APG with the library +defaults) and of 'evaluate_interactive_segmentation.py' (first prompt only) for the four '_cells' models of +micro-sam2, and the automatic references CellposeSAM (LIVECell, GoNuclear) and SegNeuron (CREMI). + +Each sample is timed once, after one untimed warm-up run per loaded model. The parts are: +- automatic: 'encoder' (image embeddings), 'decoder' (UniSAM2 decoder), 'postprocessing' (instances from the + decoder output; for APG also the prompting of SAM2 with the decoder seeds). The references have 'network' + instead of 'encoder' and 'decoder'. 'total' is the time of the whole call, so 'total' minus the parts is the + time spent outside of them. +- interactive: 'encoder' once per sample ('other' in 3d: the set-up of the video predictor state on the + embeddings), and the 'prompting' time of each object from its first prompt (a point or a box): one mask in 2d, + the propagation through all slices in 3d. The objects of a sample are capped at MAX_OBJECTS, + chosen at random with a fixed seed. + +Every sample is written to its own JSON file, with the device, the host and the per-object times, so that the +numbers can be regrouped later without a rerun ('collect' writes them into CSV tables). + +Usage: + + # Once, on the cluster: write the samples, so that every device times the same inputs. + python runtime_benchmarking_script.py export + # One group (dataset, method, model, prompt) on one device: + python runtime_benchmarking_script.py run --device_tag a100 -d livecell --method apg -m hvit_t + # All groups of a device in turn (e.g. on a Mac): + python runtime_benchmarking_script.py run --device_tag m2 --device mps --all + python runtime_benchmarking_script.py collect +""" + +import os +import sys +import json +import time +import glob +import socket +import argparse +import platform +import contextlib + +import h5py +import numpy as np +import torch + +ROOT = os.environ.get( + "RUNTIME_BENCHMARK_ROOT", "/mnt/vast-nhr/projects/cidas/cca/experiments/micro_sam2/experiments/runtime_benchmark" +) +MODEL_DIR = os.environ.get("MICRO_SAM2_MODEL_DIR", os.path.expanduser("~/.cache/micro_sam/models")) +DATASETS = {"livecell": 2, "gonuclear": 3, "cremi": 3} +MAX_SAMPLES = 50 +MAX_OBJECTS = 50 +# The number of slices of the 3d warm-up, per device type. The CPU needs few, so that a volume fits a 1 h job. +WARMUP_SLICES = {"cuda": 8, "cpu": 2} +SIZES = ["hvit_t", "hvit_s", "hvit_b", "hvit_l"] +REFERENCES = {"livecell": "cellposesam", "gonuclear": "cellposesam", "cremi": "segneuron"} +INTERACTIVE = ["interactive_point", "interactive_box"] + + +def groups(): + """All (dataset, method, model) groups of one device, the GPU-heavy ones first.""" + result = [] + for dataset in DATASETS: + for size in reversed(SIZES): + for method in ["apg", "ais"] + INTERACTIVE: + result.append((dataset, method, size)) + result.append((dataset, REFERENCES[dataset], "default")) + return result + + +# +# Timing +# + + +class Timer: + """Wall-clock timer that waits for the device before reading the clock.""" + + def __init__(self, device): + self.device = str(device) + + def sync(self): + if self.device.startswith("cuda"): + torch.cuda.synchronize() + elif self.device.startswith("mps"): + torch.mps.synchronize() + + @contextlib.contextmanager + def __call__(self, times, key): + self.sync() + start = time.perf_counter() + try: + yield + finally: + self.sync() + times[key] = times.get(key, 0.0) + time.perf_counter() - start + + +def timed_method(owner, name, timer, times, key): + """Replace 'owner.name' by a version that adds its run time to 'times[key]'. Returns the original.""" + original = getattr(owner, name) + + def wrapper(*args, **kwargs): + with timer(times, key): + return original(*args, **kwargs) + + setattr(owner, name, wrapper) + return original + + +def environment(device, threads): + """The device and the host a number was measured on.""" + info = { + "host": socket.gethostname(), "platform": platform.platform(), "processor": platform.processor(), + "torch": torch.__version__, "device": str(device), "threads": threads, + "slurm_job": os.environ.get("SLURM_JOB_ID"), "slurm_partition": os.environ.get("SLURM_JOB_PARTITION"), + } + try: + with open("/proc/cpuinfo") as f: + info["cpu_model"] = next(line.split(":", 1)[1].strip() for line in f if line.startswith("model name")) + except (OSError, StopIteration): + info["cpu_model"] = platform.processor() + if platform.system() == "Darwin": + import subprocess + info["cpu_model"] = subprocess.run( + ["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True + ).stdout.strip() + if str(device).startswith("cuda"): + info["gpu"] = torch.cuda.get_device_name() + elif str(device).startswith("mps"): + info["gpu"] = info["cpu_model"] + return info + + +# +# Data +# + + +def sample_paths(dataset): + return sorted(glob.glob(os.path.join(ROOT, "data", dataset, "sample_*.h5"))) + + +# In 3d, all methods are timed on the same three volumes on every device: APG takes up to 4 h per volume on the +# CPU. The volumes are those with the A100 APG time closest to the mean over all volumes (averaged over the sizes): +# their mean is within 13% of the mean over all volumes for every size. +VOLUMES_3D = { + "gonuclear": ["sample_0009.h5", "sample_0012.h5", "sample_0014.h5"], + "cremi": ["sample_0001.h5", "sample_0003.h5", "sample_0004.h5"], +} +# Groups that are timed on fewer volumes. CREMI APG L needs more than 12 h per volume on CPU, so only one volume. +VOLUMES_3D_GROUP = { + ("cpu", "cremi", "apg", "hvit_l"): ["sample_0001.h5"], + # CREMI is not in the figure; its CPU APG B runs are not completed. + ("cpu", "cremi", "apg", "hvit_b"): ["sample_0001.h5", "sample_0003.h5"], +} +# Devices that are timed on fewer volumes. The M2 takes about 1 h per volume for APG, so only the volume with the +# object count closest to the mean of the three. +VOLUMES_3D_DEVICE = { + ("m2", "gonuclear"): ["sample_0014.h5"], +} + + +def group_paths(device_tag, dataset, method, model=None): + """The samples that are timed for a group: all in 2d, the volumes of VOLUMES_3D in 3d. + + VOLUMES_3D_GROUP and then VOLUMES_3D_DEVICE replace the volumes of VOLUMES_3D for a group or for a device. + """ + paths = sample_paths(dataset) + volumes = VOLUMES_3D_GROUP.get( + (device_tag, dataset, method, model), VOLUMES_3D_DEVICE.get((device_tag, dataset), VOLUMES_3D.get(dataset)) + ) + if volumes is not None: + paths = [p for p in paths if os.path.basename(p) in volumes] + return paths + + +def read_sample(path): + with h5py.File(path, "r") as f: + return f["raw"][:], f["labels"][:] + + +def export(): + """Write the first MAX_SAMPLES test samples with labels of each dataset, as the evaluation loads them.""" + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from common import DATA_ROOT, load_data + for dataset, ndim in DATASETS.items(): + folder = os.path.join(ROOT, "data", dataset) + os.makedirs(folder, exist_ok=True) + n_written = len(sample_paths(dataset)) + if n_written >= MAX_SAMPLES: + continue + written = 0 + for sample_id, (raw, labels, valid_roi) in enumerate(load_data(dataset, DATA_ROOT, ndim)): + if labels.max() == 0: + continue + path = os.path.join(folder, f"sample_{sample_id:04d}.h5") + if not os.path.exists(path): + with h5py.File(path + ".tmp", "w") as f: + f.create_dataset("raw", data=raw, compression="gzip") + f.create_dataset("labels", data=labels.astype("uint32"), compression="gzip") + f.attrs["sample_id"] = sample_id + os.replace(path + ".tmp", path) + written += 1 + if written == MAX_SAMPLES: + break + print(f"{dataset}: {written} samples in '{folder}'.") + + +# +# Methods. Each runner is built once per group and returns the times of one sample. +# + + +def _common(): + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import common + common.use_tmpdir_for_embeddings() + return common + + +def automatic_runner(method, size, dataset, device, timer): + """AIS or APG of micro-sam2, in the pipeline of 'evaluate_automatic_segmentation.py' with the defaults.""" + common = _common() + ndim = DATASETS[dataset] + decoder = os.path.join(MODEL_DIR, f"{size}_cells_decoder") + import micro_sam.v2.automatic_prompt_generation as apg_module + from micro_sam.v2.instance_segmentation import UniSAM2InstanceSegmentation + if method == "apg": + model = common.build_model( + "apg", size, device, ndim, checkpoint_path=decoder, + interactive_checkpoint_path=os.path.join(MODEL_DIR, f"{size}_cells"), devices=[device], + ) + spacing = common.DATASET_SPACING.get(dataset) + + def run(raw): + times = {} + # The encoder is the embedding of the image or of the volume, the decoder the UniSAM2 decoder on it. + originals = [ + (model, "_encode", timed_method(model, "_encode", timer, times, "encoder")), + (apg_module, "precompute_image_embeddings", + timed_method(apg_module, "precompute_image_embeddings", timer, times, "encoder")), + (UniSAM2InstanceSegmentation, "initialize", + timed_method(UniSAM2InstanceSegmentation, "initialize", timer, times, "decoder")), + ] + try: + model.clear_state() + with timer(times, "total"), timer(times, "initialize"): + model.initialize(raw, ndim=ndim, **(common.VOLUME_SPEED_OPTIONS if ndim == 3 else {})) + with timer(times, "total"), timer(times, "generate"): + seg = model.generate(**({"spacing": spacing} if ndim == 3 else {})) + finally: + for owner, name, original in originals: + if owner is model: + del model._encode # The instance attribute hid the bound method. + else: + setattr(owner, name, original) + # The rest of 'initialize' builds the video predictor that 'generate' prompts, so it goes with it. + times["postprocessing"] = times["generate"] + times["initialize"] - times["encoder"] - times["decoder"] + model.clear_state() + return times, int(len(np.unique(seg)) - 1) + return run + + # AIS runs like the first stage of APG: the embeddings of the image or of the volume (once per slice, untiled), + # then the UniSAM2 decoder on them, then the AIS postprocessing. + from micro_sam.util import make_temp_embedding_path + model = common.build_model( + "apg", size, device, ndim, checkpoint_path=decoder, + interactive_checkpoint_path=os.path.join(MODEL_DIR, f"{size}_cells"), devices=[device], + ) + + def run(raw): + times = {} + path = make_temp_embedding_path() if ndim == 3 else None + try: + model.clear_state() + with timer(times, "total"): + with timer(times, "encoder"): + if ndim == 3: + # As in 'AutomaticPromptGenerator._initialize_volume'. + embeddings = apg_module.precompute_image_embeddings( + model._video_predictor, raw, save_path=path, ndim=3, verbose=False, + lazy_loading=True, devices=model._inference_devices(None), + ) + else: + embeddings = model._encode(raw) + with timer(times, "decoder"): + UniSAM2InstanceSegmentation.initialize(model, raw, ndim=ndim, image_embeddings=embeddings) + with timer(times, "postprocessing"): + seg = common.postprocess_unisam2(model.get_state()["prediction"], dataset, model_type=size) + finally: + model.clear_state() + if path is not None: + import shutil + shutil.rmtree(path, ignore_errors=True) + return times, int(len(np.unique(seg)) - 1) + return run + + +def cellposesam_runner(dataset, device, timer): + """CellposeSAM as in 'evaluate_automatic_baselines.py': 2d, or 'do_3D' for a volume.""" + from cellpose import models + gpu = not str(device).startswith("cpu") + model = models.CellposeModel(gpu=gpu, pretrained_model="cpsam", device=torch.device(device) if gpu else None) + ndim = DATASETS[dataset] + + def run(raw): + times = {} + originals = [ + ("_run_net", timed_method(model, "_run_net", timer, times, "network")), + ("_compute_masks", timed_method(model, "_compute_masks", timer, times, "postprocessing")), + ] + try: + with timer(times, "total"): + if ndim == 3: + masks = model.eval(raw, diameter=None, do_3D=True, z_axis=0)[0] + else: + masks = model.eval(raw, diameter=None)[0] + finally: + for name, _ in originals: + delattr(model, name) + return times, int(len(np.unique(masks)) - 1) + return run + + +def segneuron_runner(dataset, device, timer): + """SegNeuron as in 'evaluate_automatic_baselines.py': the network on blocks, then its multicut.""" + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import evaluate_automatic_baselines as baselines + checkpoint = os.environ.get("SEGNEURON_CHECKPOINT", baselines.SEGNEURON_CHECKPOINT) + if os.environ.get("SEGNEURON_ROOT"): + baselines.SEGNEURON_ROOT = os.environ["SEGNEURON_ROOT"] + model = baselines._load_segneuron(checkpoint, device) + + def run(raw): + times = {} + original = timed_method(baselines, "_segneuron_multicut", timer, times, "postprocessing") + try: + with timer(times, "total"): + seg = baselines._segment_segneuron(raw, model, device) + finally: + baselines._segneuron_multicut = original + times["network"] = times["total"] - times["postprocessing"] + return times, int(len(np.unique(seg)) - 1) + return run + + +def pick_objects(ids, seed=0): + """At most MAX_OBJECTS of the object ids, chosen at random with a fixed seed, in increasing order.""" + ids = np.asarray(ids) + if len(ids) <= MAX_OBJECTS: + return ids + return np.sort(np.random.default_rng(seed).choice(ids, MAX_OBJECTS, replace=False)) + + +def interactive_runner(method, size, dataset, device, timer): + """The first prompt of 'evaluate_interactive_segmentation.py', one object at a time.""" + from micro_sam.v1.evaluation.inference import _get_batched_prompts + from micro_sam.v2.normalization import normalize_raw + from micro_sam.v2.util import encode_image, get_sam2_image_predictor, get_sam2_model, precompute_image_embeddings + from micro_sam.v2.evaluation.inference import _embedding_tensors_to_numpy, _extract_prompts_per_object + from bioimage_cpp.segmentation import label as connected_components + + use_boxes = method == "interactive_box" + prompt_kwargs = dict(use_points=not use_boxes, use_boxes=use_boxes, n_positive=0 if use_boxes else 1) + checkpoint = os.path.join(MODEL_DIR, f"{size}_cells") + ndim = DATASETS[dataset] + + def to_uint8(raw): + return normalize_raw(raw, output_dtype="uint8") + + if ndim == 2: + predictor = get_sam2_image_predictor(get_sam2_model(model_type=size, device=device, checkpoint_path=checkpoint)) + + def run(raw, labels): + times = {} + # As 'run_interactive_segmentation_2d': a grayscale image is stacked to RGB, the labels are relabelled. + image = to_uint8(raw).astype("uint8") + if image.ndim == 2: + image = np.stack([image] * 3, axis=-1) + labels = connected_components(labels).astype("uint32") + np.random.seed(0) + gt_ids = pick_objects(np.setdiff1d(np.unique(labels), [0])) + points, point_labels, boxes = _get_batched_prompts( + gt=labels, gt_ids=gt_ids, use_points=not use_boxes, use_boxes=use_boxes, + n_positives=prompt_kwargs["n_positive"], n_negatives=0, dilation=5, + ) + with timer(times, "encoder"): + encode_image(predictor, image) + object_times = [] + for i in range(len(gt_ids)): + one = {} + with timer(one, "object"): + predictor.predict( + point_coords=None if use_boxes else points[i:i + 1], + point_labels=None if use_boxes else point_labels[i:i + 1], + box=boxes[i:i + 1] if use_boxes else None, multimask_output=False, return_logits=True, + ) + object_times.append(one["object"]) + return times, object_times, 1 + return run + + predictor = get_sam2_model(model_type=size, device=device, checkpoint_path=checkpoint, input_type="videos") + predictor.add_all_frames_to_correct_as_cond = True + + def run(raw, labels): + times = {} + volume = np.stack([to_uint8(frame) for frame in raw]) + labels = connected_components(labels).astype(labels.dtype) + gt_ids = [int(i) for i in pick_objects(np.setdiff1d(np.unique(labels), [0]))] + with timer(times, "encoder"): + embeddings = precompute_image_embeddings(predictor=predictor, input_=volume, ndim=3) + embeddings = _embedding_tensors_to_numpy(embeddings) + # The video predictor state holds the embeddings for the propagation: the 'other' part of the encoding. + with timer(times, "other"): + state = predictor.init_state(volume=volume, volume_embeddings=embeddings) + np.random.seed(0) + prompts = _extract_prompts_per_object( + labels=labels, gt_ids=gt_ids, first_prompt_at_center=True, dilation=5, **prompt_kwargs, + ) + object_times = [] + with torch.no_grad(): + for obj_id, (z, point, label, box) in prompts.items(): + predictor.reset_state(state) + one = {} + with timer(one, "object"): + predictor.add_new_points_or_box( + inference_state=state, frame_idx=z, obj_id=obj_id, points=point, labels=label, box=box, + ) + n_frames = 0 + for _, _, logits in predictor.propagate_in_video(state): + (logits > 0.0).cpu().numpy() + n_frames += 1 + if n_frames < volume.shape[0]: + for _, _, logits in predictor.propagate_in_video(state, reverse=True): + (logits > 0.0).cpu().numpy() + object_times.append(one["object"]) + return times, object_times, int(volume.shape[0]) + return run + + +# +# Running +# + + +def result_path(device_tag, dataset, method, model, sample_path): + name = os.path.basename(sample_path).replace(".h5", ".json") + return os.path.join(ROOT, "results", device_tag, dataset, f"{method}_{model}", name) + + +def release_memory(device): + """Empty the MPS cache between samples, untimed: without a watermark it grows into swap over the samples.""" + if str(device).startswith("mps"): + import gc + gc.collect() + torch.mps.empty_cache() + + +def run_group(device_tag, device, dataset, method, model, deadline, threads, max_samples=None): + """Time all samples of one group that have no result yet. Stops before 'deadline' (time.time()).""" + paths = group_paths(device_tag, dataset, method, model)[:max_samples] + paths = [p for p in paths if open_sample(result_path(device_tag, dataset, method, model, p))] + if not paths: + return True + # A job that has not enough time left for any sample of this group leaves it to a longer job. + shortest = min(expected_seconds(device_tag, dataset, method, model, p) for p in paths) + if time.time() + FIT_MARGIN * shortest > deadline: + print(f"Skipping the group: one sample takes about {shortest / 60:.0f} min.", flush=True) + return True + timer = Timer(device) + if method in ("ais", "apg"): + runner, interactive = automatic_runner(method, model, dataset, device, timer), False + elif method == "cellposesam": + runner, interactive = cellposesam_runner(dataset, device, timer), False + elif method == "segneuron": + runner, interactive = segneuron_runner(dataset, device, timer), False + else: + runner, interactive = interactive_runner(method, model, dataset, device, timer), True + info = environment(device, threads) + + def call(raw, labels): + return runner(raw, labels) if interactive else runner(raw) + + # The warm-up loads the weights to the device and compiles the kernels; its times are not kept. For 3d, it runs + # on the first slices of a volume only, which use the same kernels at a fraction of the time. + raw, labels = read_sample(paths[0]) + if raw.ndim == 3: + n_slices = WARMUP_SLICES.get(str(device), WARMUP_SLICES["cuda"]) + raw, labels = raw[:n_slices], labels[:n_slices] + call(raw, labels) + release_memory(device) + for path in paths: + # A sample that does not fit in the time left is left to a longer job; a shorter one may still fit. + if time.time() + FIT_MARGIN * expected_seconds(device_tag, dataset, method, model, path) > deadline: + print(f"Leaving {os.path.basename(path)} to a longer job.", flush=True) + continue + out = result_path(device_tag, dataset, method, model, path) + started = out + ".started" + # Several jobs can run the same group: each sample is claimed by one job. + if not claim_sample(out): + continue + start = time.time() + raw, labels = read_sample(path) + record = { + "wall_seconds": None, "dataset": dataset, "method": method, "model": model, "device_tag": device_tag, + "sample": os.path.basename(path), "shape": list(raw.shape), **info, + } + if interactive: + times, object_times, n_slices = call(raw, labels) + record.update({ + "times": times, "object_times": object_times, "n_slices": n_slices, + "n_objects_total": int(len(np.setdiff1d(np.unique(labels), [0]))), + }) + else: + times, n_objects = call(raw, labels) + record.update({"times": times, "n_objects": n_objects}) + if str(device).startswith("mps"): + record["mps_allocated_gb"] = torch.mps.current_allocated_memory() / 1e9 + record["mps_driver_gb"] = torch.mps.driver_allocated_memory() / 1e9 + release_memory(device) + record["wall_seconds"] = time.time() - start + with open(out + ".tmp", "w") as f: + json.dump(record, f, indent=1) + os.replace(out + ".tmp", out) + os.remove(started) + print(f"{dataset} {method} {model} {os.path.basename(path)}: {json.dumps(times)}", flush=True) + return True + + +def _owner(): + return os.environ.get("SLURM_JOB_ID", f"local{os.getpid()}") + + +def _started_owner(started): + try: + with open(started) as f: + return f.read().strip() + except OSError: + return None + + +def open_sample(out): + """Whether a sample has no result and no job works on it. + + A sample whose job ended without a result took longer than that job had left. Short jobs leave it to the long + jobs, so that it does not restart in every short job. + """ + if os.path.exists(out): + return False + started = out + ".started" + if not os.path.exists(started): + return True + if _job_alive(_started_owner(started)): + return False + return os.environ.get("RUNTIME_SHORT_JOB") != "1" + + +def claim_sample(out): + """Claim a sample for this job with an atomic create. A claim of a job that no longer runs is taken over.""" + started = out + ".started" + os.makedirs(os.path.dirname(out), exist_ok=True) + try: + fd = os.open(started, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError: + if not open_sample(out): + return False + # The claim of an ended job: take it over. + with open(started, "w") as f: + f.write(_owner()) + return True + with os.fdopen(fd, "w") as f: + f.write(_owner()) + return True + + +# For a group without results on the CPU yet, the time of a sample is estimated from the A100 times. APG measured a +# CPU / A100 ratio of 41 to 50 for T to B on GoNuclear, and the ratio grows with the model size. +CPU_SLOWDOWN = 70 +# A job starts a sample only if this multiple of its expected time is left. +FIT_MARGIN = 1.15 + + +def expected_seconds(device_tag, dataset, method, model, path): + """The expected wall time of one sample, to start it only in a job with enough time left. + + On the CPU, it is the A100 time of the sample times the largest CPU / A100 ratio of the samples of the group with + both times (CPU_SLOWDOWN if there are none). Else, it is the longest wall time of a sample of the group so far. + """ + times = _wall_seconds(device_tag, dataset, method, model) + if device_tag == "cpu": + gpu_times = _wall_seconds("a100", dataset, method, model) + sample = os.path.basename(path) + if sample in gpu_times: + ratios = [times[name] / gpu_times[name] for name in times if gpu_times.get(name)] + return max(ratios, default=CPU_SLOWDOWN) * gpu_times[sample] + if not times: + return CPU_SLOWDOWN * max(gpu_times.values(), default=0.0) + return max(times.values(), default=0.0) + + +def _wall_seconds(device_tag, dataset, method, model): + """The wall time of each finished sample of a group, by the file name of the sample.""" + times = {} + for path in glob.glob(os.path.join(ROOT, "results", device_tag, dataset, f"{method}_{model}", "*.json")): + try: + with open(path) as f: + record = json.load(f) + except (OSError, ValueError): + continue + seconds = record.get("wall_seconds") + if seconds is None: + parts = record["times"] + seconds = parts.get("total", sum(parts.values())) + sum(record.get("object_times", [])) + times[os.path.splitext(os.path.basename(path))[0] + ".h5"] = seconds + return times + + +def claim(device_tag, group): + """Claim a group for this job with an atomic mkdir. A lock of a job that no longer runs is taken over.""" + lock = os.path.join(ROOT, "locks", device_tag, "__".join(group)) + owner = os.environ.get("SLURM_JOB_ID", f"local{os.getpid()}") + os.makedirs(os.path.dirname(lock), exist_ok=True) + try: + os.mkdir(lock) + except FileExistsError: + try: + with open(os.path.join(lock, "owner")) as f: + previous = f.read().strip() + except OSError: + return None + if previous == owner or _job_alive(previous): + return None + with open(os.path.join(lock, "owner"), "w") as f: + f.write(owner) + return lock + + +def _job_alive(job): + if not job or job.startswith("local"): + return False + import subprocess + out = subprocess.run(["squeue", "-h", "-j", job, "-o", "%T"], capture_output=True, text=True).stdout + return bool(out.strip()) + + +def group_done(device_tag, group): + dataset, method, model = group + return all( + os.path.exists(result_path(device_tag, dataset, method, model, p)) + for p in group_paths(device_tag, dataset, method, model) + ) + + +def run(args): + threads = args.threads or int(os.environ.get("SLURM_CPUS_PER_TASK", os.cpu_count())) + torch.set_num_threads(threads) + device = args.device or ("cuda" if torch.cuda.is_available() else "cpu") + deadline = time.time() + args.seconds if args.seconds else float("inf") + if not args.all: + todo = [(args.dataset_name, args.method, args.model_type)] + else: + todo = [g for g in groups() if not args.only or g[1] in args.only or g[0] in args.only] + # First the groups that no job runs, then, with --all, the groups of other jobs that still have open samples: + # the jobs share the samples of a group. + passes = [(group, True) for group in todo] + ([(group, False) for group in todo] if args.all else []) + for group, own in passes: + if group_done(args.device_tag, group): + continue + lock = claim(args.device_tag, group) if args.all and own else None + if args.all and own and lock is None: + continue + print(f"Group {group} on {device} ({args.device_tag}).", flush=True) + try: + finished = run_group(args.device_tag, device, *group, deadline, threads, args.max_samples) + except Exception: + # One failing group must not end the job: the error is logged, and the next group runs. + import traceback + traceback.print_exc() + os.makedirs(os.path.join(ROOT, "failed"), exist_ok=True) + with open(os.path.join(ROOT, "failed", f"{args.device_tag}__{'__'.join(group)}.txt"), "a") as f: + f.write(traceback.format_exc()) + finished = True + finally: + if lock is not None: + import shutil + shutil.rmtree(lock, ignore_errors=True) + if not finished: + return + + +def collect(): + """Write the per-sample parts and the per-object times of all devices into two CSV tables.""" + import pandas as pd + samples, objects = [], [] + for path in sorted(glob.glob(os.path.join(ROOT, "results", "*", "*", "*", "*.json"))): + with open(path) as f: + record = json.load(f) + # All timed samples are kept; 'selected' marks the samples of the figure (see VOLUMES_3D). + selected = [os.path.basename(p) for p in group_paths( + record["device_tag"], record["dataset"], record["method"], record["model"])] + keys = ["device_tag", "dataset", "method", "model", "sample", "host", "cpu_model", "gpu", "threads"] + base = {key: record.get(key) for key in keys} + base["selected"] = record["sample"] in selected + for part, seconds in record["times"].items(): + samples.append({**base, "part": part, "seconds": seconds, + "n_objects": record.get("n_objects", len(record.get("object_times", [])))}) + for k, seconds in enumerate(record.get("object_times", [])): + objects.append({**base, "object": k, "seconds": seconds, "n_slices": record["n_slices"], + "seconds_per_slice": seconds / record["n_slices"]}) + pd.DataFrame(samples).to_csv(os.path.join(ROOT, "runtimes_per_sample.csv"), index=False) + pd.DataFrame(objects).to_csv(os.path.join(ROOT, "runtimes_per_object.csv"), index=False) + print(f"{len(samples)} part rows and {len(objects)} object rows in '{ROOT}'.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("command", choices=["export", "run", "collect", "status"]) + parser.add_argument("--device_tag", help="The name of the device in the results, e.g. 'cpu', 'a100', 'm2'.") + parser.add_argument("--device", default=None, help="The torch device. Default: cuda if available, else cpu.") + parser.add_argument("-d", "--dataset_name", choices=list(DATASETS)) + parser.add_argument("--method", choices=["ais", "apg", "cellposesam", "segneuron"] + INTERACTIVE) + parser.add_argument("-m", "--model_type", default="default") + parser.add_argument("--all", action="store_true", help="Run all open groups, claiming each with a lock.") + parser.add_argument("--only", nargs="*", help="With --all: only the groups of these methods or datasets.") + parser.add_argument("--threads", type=int, default=None, help="The torch threads. Default: the Slurm CPUs.") + parser.add_argument("--max_samples", type=int, default=None, help="Time only the first samples, for a check.") + parser.add_argument("--seconds", type=float, default=None, help="Stop starting new samples after this time.") + args = parser.parse_args() + if args.command == "export": + export() + elif args.command == "collect": + collect() + elif args.command == "status": + # The number of open groups of the device, for the job loops. + print(sum(not group_done(args.device_tag, group) for group in groups())) + else: + if args.device_tag is None: + parser.error("'run' needs --device_tag.") + run(args) + + +if __name__ == "__main__": + main() diff --git a/finetuning/v2/evaluation/tracking_util.py b/finetuning/v2/evaluation/tracking_util.py new file mode 100644 index 000000000..7cfc79850 --- /dev/null +++ b/finetuning/v2/evaluation/tracking_util.py @@ -0,0 +1,756 @@ +"""Shared helpers of the micro-sam2 tracking evaluation: the data, the CTC export and the metrics. + +The automatic tracking runs on data in the Cell Tracking Challenge (CTC) format: the CTC training sequences (2d+t and +3d+t), the BacMother test videos and the TOIAM test movie. The interactive tracking needs dense track masks in every +frame to place the prompts of the simulated user, so it runs on DynamicNuclearNet, TOIAM (in-domain), BacMother and the +real movies of 'tracking_fm_data' (out-of-domain). +""" + +import os +import re +import glob +import shutil +import sqlite3 +import zipfile +import tempfile +import warnings + +import zarr +import tifffile +import numpy as np +import pandas as pd +import scipy.io as sio +import imageio.v3 as imageio + +from common import DATA_ROOT + +CTC_ROOT = os.path.join(DATA_ROOT, "ctc", "train") +EXPERIMENT_ROOT = "/mnt/vast-nhr/projects/cidas/cca/experiments/micro_sam2/experiments/tracking" + +# The CTC sets of the micro-sam2 generalist training (see 'micro_sam.v2.datasets.generalist_loader.CTC_DATASETS'). +CTC_TRAINED = ( + "BF-C2DL-HSC", "BF-C2DL-MuSC", "DIC-C2DH-HeLa", "Fluo-C2DL-Huh7", "Fluo-C2DL-MSC", "Fluo-N2DH-SIM+", + "PhC-C2DH-U373", "PhC-C2DL-PSC", +) +CTC_SEQUENCES = ("01", "02") + +# The other 2D+t data in CTC format and whether micro-sam2 trained on it; TOIAM movie 04 is held out. +TRACKING_DATA = os.path.join(EXPERIMENT_ROOT, "data") +CONVERTED_DATASETS = ("EColi", "ALFI", "Cardioblast", "WingDisc", "YeaZ", "CellACDC", "DynamicNuclearNet") +# Held-out movies from the training sources: the YeaZ phase-contrast movies and the Cell-ACDC test movie. +CONVERTED_IN_DOMAIN = ("YeaZ", "CellACDC", "DynamicNuclearNet") +OTHER_CTC_DATASETS = { + "BacMother": (os.path.join(DATA_ROOT, "bac_mother", "CTC", "test"), False), + "TOIAM": (os.path.join(DATA_ROOT, "toiam", "data"), True), + **{name: (os.path.join(TRACKING_DATA, name), name in CONVERTED_IN_DOMAIN) for name in CONVERTED_DATASETS}, +} +TOIAM_TEST_MOVIES = ("04",) +# TOIAM 04 is too large for the simulated user: frames 300 to 399 only. +TOIAM_INTERACTIVE_FRAMES = (300, 100) + +DNN_ROOT = os.path.join(DATA_ROOT, "deepcell_tracking_data", "test") +FM_ROOT = os.path.join(DATA_ROOT, "tracking_fm_data", "new_data_131026", "real") + +# The interactive datasets and their domain: DynamicNuclearNet and TOIAM were part of the micro-sam2 training. +INTERACTIVE_DATASETS = { + "dynamicnuclearnet": "in-domain", "toiam": "in-domain", "bacmother": "out-of-domain", + "tracking_fm_real": "out-of-domain", "ecoli": "out-of-domain", "alfi": "out-of-domain", + "cardioblast": "out-of-domain", "wingdisc": "out-of-domain", "yeaz": "in-domain", "cellacdc": "in-domain", +} +# The interactive datasets that are read from the CTC folders of an automatic dataset. +INTERACTIVE_CTC = { + "toiam": "TOIAM", "bacmother": "BacMother", "ecoli": "EColi", "alfi": "ALFI", "cardioblast": "Cardioblast", + "wingdisc": "WingDisc", "yeaz": "YeaZ", "cellacdc": "CellACDC", +} + + +def ctc_datasets(): + """Get the names of all CTC sets on disk.""" + return sorted(name for name in os.listdir(CTC_ROOT) if os.path.isdir(os.path.join(CTC_ROOT, name, "01"))) + + +def automatic_datasets(): + """Get the names of all datasets for the automatic tracking: the CTC sets, BacMother and TOIAM.""" + return ctc_datasets() + sorted(OTHER_CTC_DATASETS) + + +def dataset_root(name): + return OTHER_CTC_DATASETS[name][0] if name in OTHER_CTC_DATASETS else os.path.join(CTC_ROOT, name) + + +def sequences_of(name): + """Get the sequences of a dataset: '01' and '02' for CTC, the test movie of TOIAM, the test videos of BacMother.""" + if name == "TOIAM": + return TOIAM_TEST_MOVIES + if name == "BacMother": + return sorted(folder for folder in os.listdir(dataset_root(name)) if folder.isdigit()) + if name in CONVERTED_DATASETS: + return sorted(folder for folder in os.listdir(dataset_root(name)) if not folder.endswith("_GT")) + return CTC_SEQUENCES + + +def is_3d(name): + """Check whether a dataset is 3d+t; a CTC set tells it by the second part of its name, e.g. 'C3DH'.""" + return name not in OTHER_CTC_DATASETS and name.split("-")[1][1] == "3" + + +def in_training(name): + return OTHER_CTC_DATASETS[name][1] if name in OTHER_CTC_DATASETS else name in CTC_TRAINED + + +def to_grayscale(frames): + """Convert RGB frames (T, Y, X, 3), as BacMother stores them, to grayscale.""" + if frames.ndim != 4 or frames.shape[-1] != 3: + return frames + if all(np.array_equal(frames[..., 0], frames[..., c]) for c in (1, 2)): + return frames[..., 0] + return frames.mean(axis=-1).astype(frames.dtype) + + +def load_sequence(name, sequence): + """Load the frames of one sequence as a (T, Y, X) or (T, Z, Y, X) array.""" + paths = sorted(glob.glob(os.path.join(dataset_root(name), sequence, "t*.tif"))) + return to_grayscale(np.stack([tifffile.imread(path) for path in paths])) + + +def gt_folder_of(name, sequence): + """Get the ground-truth folder of one sequence, with 'TRA' and 'SEG'.""" + return os.path.join(dataset_root(name), f"{sequence}_GT") + + +def load_ctc_tracks(folder): + """Load the dense track masks of a CTC folder and its lineage table, columns 'track', 'start', 'end', 'parent'.""" + paths = sorted(glob.glob(os.path.join(folder, "man_track*.tif"))) + masks = np.stack([tifffile.imread(path) for path in paths]) + tracks = pd.read_csv( + os.path.join(folder, "man_track.txt"), sep=" ", header=None, names=["track", "start", "end", "parent"] + ) + return masks, tracks + + +def frame_window(raw, masks, tracks, first, n_frames): + """Limit a movie and its tracks to a window of frames, numbered from 0. A track whose parent ends before the + window becomes a root. + """ + last = first + n_frames - 1 + tracks = tracks[(tracks["start"] <= last) & (tracks["end"] >= first)].copy() + tracks["parent"] = tracks["parent"].where(tracks["parent"].isin(tracks["track"]), 0) + tracks["start"] = tracks["start"].clip(lower=first) - first + tracks["end"] = tracks["end"].clip(upper=last) - first + return raw[first:last + 1], masks[first:last + 1], tracks + + +def interactive_samples(dataset_name): + """Get the names of the movies of an interactive dataset.""" + if dataset_name == "dynamicnuclearnet": + return sorted(os.listdir(DNN_ROOT)) + if dataset_name == "tracking_fm_real": + return sorted(os.path.basename(path)[:-len(".ome.zarr")] for path in glob.glob(f"{FM_ROOT}/*.ome.zarr")) + if dataset_name == "toiam": + return list(TOIAM_TEST_MOVIES) + if dataset_name in INTERACTIVE_CTC: + return list(sequences_of(INTERACTIVE_CTC[dataset_name])) + raise ValueError(f"Unknown interactive dataset '{dataset_name}'.") + + +def load_interactive_sample(dataset_name, sample): + """Load one movie with dense track masks: the raw frames, the track masks and the lineage table.""" + if dataset_name == "dynamicnuclearnet": + frames = sorted(glob.glob(os.path.join(DNN_ROOT, sample, "01", "*.tif"))) + raw = np.stack([tifffile.imread(path) for path in frames]) + masks, tracks = load_ctc_tracks(os.path.join(DNN_ROOT, sample, "01_GT", "TRA")) + return raw, masks, tracks + if dataset_name == "tracking_fm_real": + f = zarr.open(os.path.join(FM_ROOT, f"{sample}.ome.zarr"), mode="r") + raw, masks = f["s0"][:], f["labels/cells/s0"][:].astype("uint32") + return raw, masks, _geff_tracks(f) + if dataset_name in INTERACTIVE_CTC: + name = INTERACTIVE_CTC[dataset_name] + raw = load_sequence(name, sample) + masks, tracks = load_ctc_tracks(os.path.join(gt_folder_of(name, sample), "TRA")) + if dataset_name == "toiam": + raw, masks, tracks = frame_window(raw, masks, tracks, *TOIAM_INTERACTIVE_FRAMES) + return raw, masks, tracks + raise ValueError(f"Unknown interactive dataset '{dataset_name}'.") + + +def _geff_tracks(f): + """Read the CTC lineage table that the GEFF graph of an OME-Zarr stores; its labels are the CTC track ids.""" + lbep = f["tracks/cells.geff"].attrs["geff"]["extra"]["ctc"]["man_track_lbep"] + return pd.DataFrame(lbep, columns=["track", "start", "end", "parent"]) + + +def tracking_metrics(gt_folder, pred_folder, gt_track_file=None, dense_gt=False): + """Compute the tracking metrics of a prediction in CTC format with traccuracy. + + The CTC metrics (TRA, DET, LNK) use the CTC matching, which also works for the marker annotations of the real CTC + sets. The division metrics need a one-to-one matching: for markers, a linear assignment over any overlap; for + dense ground-truth, an IoU of at least 0.5. + """ + from traccuracy import run_metrics + from traccuracy.loaders import load_ctc_data + from traccuracy.matchers import CTCMatcher, IOUMatcher + from traccuracy.metrics import CTCMetrics, DivisionMetrics + + gt = load_ctc_data(gt_folder, track_path=gt_track_file or valid_track_file(gt_folder), name="gt") + pred_folder = trim_ctc_frames(pred_folder, len(glob.glob(os.path.join(gt_folder, "man_track*.tif")))) + try: + pred = load_ctc_data(pred_folder, name="pred") + except Exception as e: + # A predicted track can have a gap or an empty parent, which the CTC checks reject. + warnings.warn(f"The prediction fails the CTC checks ({e}). It is loaded without them.") + pred = load_ctc_data(pred_folder, name="pred", run_checks=False) + scores = {} + results, _ = run_metrics(gt, pred, matcher=CTCMatcher(), metrics=[CTCMetrics()]) + scores.update(results[0]["results"]) + division_matcher = IOUMatcher(iou_threshold=0.5 if dense_gt else 0.0, one_to_one=True) + try: + results, _ = run_metrics(gt, pred, matcher=division_matcher, metrics=[DivisionMetrics(max_frame_buffer=2)]) + for frame_buffer, values in results[0]["results"].items(): + for key, value in values.items(): + scores[f"{key} ({frame_buffer})"] = value + except Exception as e: + warnings.warn(f"The division metrics failed: {e}") + return {key: value for key, value in scores.items() if np.isscalar(value)} + + +def ctc_seg_score(gt_folder, segmentation): + """Compute the CTC SEG score: the mean IoU of each ground-truth object with the prediction that covers more than + half of it, 0 without one. The SEG annotations are a subset of the frames; for some 3d sets single slices. + """ + ious = [] + for path in sorted(glob.glob(os.path.join(gt_folder, "SEG", "man_seg*.tif"))): + numbers = [int(number) for number in re.findall(r"\d+", os.path.basename(path))] + gt = tifffile.imread(path) + pred = segmentation[numbers[0]] if len(numbers) == 1 else segmentation[numbers[0]][numbers[1]] + if gt.shape != pred.shape: + raise ValueError(f"The SEG annotation {path} has shape {gt.shape}, the prediction {pred.shape}.") + ious.extend(_seg_ious(gt, pred)) + return float(np.mean(ious)) if ious else np.nan + + +def _seg_ious(gt, pred): + """Get the SEG IoU of each ground-truth object of one frame.""" + gt_ids, gt_sizes = np.unique(gt[gt > 0], return_counts=True) + pairs, overlaps = np.unique(np.stack([gt[gt > 0], pred[gt > 0]]), axis=1, return_counts=True) + pred_ids, pred_sizes = np.unique(pred[pred > 0], return_counts=True) + pred_size = dict(zip(pred_ids, pred_sizes)) + ious = dict.fromkeys(gt_ids.tolist(), 0.0) + gt_size = dict(zip(gt_ids, gt_sizes)) + for (gt_id, pred_id), overlap in zip(pairs.T, overlaps): + if pred_id != 0 and overlap > 0.5 * gt_size[gt_id]: + ious[int(gt_id)] = overlap / (gt_size[gt_id] + pred_size[pred_id] - overlap) + return list(ious.values()) + + +def write_ctc_tracks(masks, tracks, folder): + """Write dense track masks and their lineage table in CTC format, for the scoring with traccuracy.""" + os.makedirs(folder, exist_ok=True) + present = set(np.unique(masks).tolist()) - {0} + tracks = tracks[tracks["track"].isin(present) & (tracks["start"] <= tracks["end"])] + tracks[["track", "start", "end", "parent"]].to_csv( + os.path.join(folder, "man_track.txt"), sep=" ", header=False, index=False + ) + for t in range(masks.shape[0]): + tifffile.imwrite(os.path.join(folder, f"man_track{t:04d}.tif"), masks[t].astype("uint32")) + + +def dense_seg_score(gt_masks, segmentation): + """Compute the CTC SEG score over all frames of dense ground-truth.""" + ious = [] + for gt, pred in zip(gt_masks, segmentation): + ious.extend(_seg_ious(gt, pred)) + return float(np.mean(ious)) if ious else np.nan + + +def split_track_gaps(segmentation, lineages): + """Split each track at the frames where it is missing, as the CTC format requires. + + Each later piece gets a new id whose parent is the piece before it, and the children of the track move to its last + piece. The links of a parent without pixels are dropped. The segmentation is changed in place. + + Returns: + The lineages with one more dict that holds the links of the new pieces. + """ + frames = {} + for t in range(segmentation.shape[0]): + for label_id in np.unique(segmentation[t]): + if label_id != 0: + frames.setdefault(int(label_id), []).append(t) + next_id = int(segmentation.max()) + 1 + last_piece, links = {}, {} + for label_id, present in frames.items(): + pieces = np.split(np.array(present), np.where(np.diff(present) > 1)[0] + 1) + previous = label_id + for piece in pieces[1:]: + for t in piece: + segmentation[t][segmentation[t] == label_id] = next_id + links[previous] = [next_id] + previous = next_id + next_id += 1 + last_piece[label_id] = previous + # A parent without pixels, e.g. a mother track that SAM2 lost, is not written; its children become roots. + updated = [ + {last_piece.get(parent, parent): children for parent, children in lineage.items() if parent in frames} + for lineage in lineages + ] + return updated + [links] + + +def valid_track_file(folder): + """Get the lineage file of a CTC folder, or a copy without the rows that end before they start. + + The official 'Fluo-C3DL-MDA231/02_GT/TRA/man_track.txt' has such a row ('26 4 2 0'), for a label that no mask holds. + """ + path = os.path.join(folder, "man_track.txt") + tracks = pd.read_csv(path, sep=" ", header=None, names=["track", "start", "end", "parent"]) + invalid = tracks["start"] > tracks["end"] + if not invalid.any(): + return path + warnings.warn(f"Dropping the rows of {path} that end before they start: {tracks[invalid].values.tolist()}") + fixed_path = os.path.join(EXPERIMENT_ROOT, "gt_fixed", path.replace(os.sep, "_").strip("_")) + os.makedirs(os.path.dirname(fixed_path), exist_ok=True) + tracks[~invalid].to_csv(fixed_path, sep=" ", header=False, index=False) + return fixed_path + + +def trim_ctc_frames(pred_folder, n_frames): + """Get a CTC prediction limited to its first 'n_frames', for ground-truth that annotates fewer frames. + + The ground-truth of 'Fluo-N3DH-CE' annotates the first 195 (01) and 190 (02) of 250 frames. + """ + paths = sorted(glob.glob(os.path.join(pred_folder, "man_track*.tif"))) + if len(paths) <= n_frames: + return pred_folder + trimmed = f"{pred_folder.rstrip(os.sep)}_first{n_frames}" + os.makedirs(trimmed, exist_ok=True) + for path in paths[:n_frames]: + link = os.path.join(trimmed, os.path.basename(path)) + if not os.path.exists(link): + os.symlink(path, link) + tracks = pd.read_csv( + os.path.join(pred_folder, "man_track.txt"), sep=" ", header=None, names=["track", "start", "end", "parent"] + ) + tracks = tracks[tracks["start"] < n_frames].copy() + tracks["end"] = tracks["end"].clip(upper=n_frames - 1) + tracks.to_csv(os.path.join(trimmed, "man_track.txt"), sep=" ", header=False, index=False) + return trimmed + + +# The CTC sets whose TRA annotation covers only some cells. A prediction of an unannotated cell is no error there. +SPARSE_GT_DATASETS = ("Fluo-N3DL-DRO", "Fluo-N3DL-TRIC") + + +def restrict_to_annotated(pred_folder, gt_folder, out_folder): + """Write a CTC prediction limited to the objects that overlap an annotated marker of their frame. + + For ground-truth that annotates only some cells (`SPARSE_GT_DATASETS`): the scoring then ignores the predictions of + unannotated cells, while missed annotated cells, wrong links, splits and merges still count. A track that loses + frames is split into linked pieces, as the CTC format requires. The frames are processed one at a time, since the + 3d+t sets do not fit in memory. + """ + pred_paths = sorted(glob.glob(os.path.join(pred_folder, "man_track*.tif"))) + gt_paths = sorted(glob.glob(os.path.join(gt_folder, "man_track*.tif")))[:len(pred_paths)] + os.makedirs(out_folder, exist_ok=True) + frames = {} + for t, (pred_path, gt_path) in enumerate(zip(pred_paths, gt_paths)): + pred, gt = tifffile.imread(pred_path), tifffile.imread(gt_path) + keep = np.setdiff1d(np.unique(pred[gt > 0]), [0]) + pred[~np.isin(pred, keep)] = 0 + tifffile.imwrite(os.path.join(out_folder, f"man_track{t:04d}.tif"), pred.astype("uint32")) + for label_id in keep: + frames.setdefault(int(label_id), []).append(t) + + tracks = pd.read_csv( + os.path.join(pred_folder, "man_track.txt"), sep=" ", header=None, names=["track", "start", "end", "parent"] + ) + parent_of = dict(zip(tracks["track"].astype(int), tracks["parent"].astype(int))) + next_id = max(max(frames, default=0), int(tracks["track"].max())) + 1 + relabel, rows, last_piece = {}, [], {} + for label_id in sorted(frames): + present = frames[label_id] + pieces = np.split(np.array(present), np.where(np.diff(present) > 1)[0] + 1) + previous = None + for i, piece in enumerate(pieces): + piece_id = label_id if i == 0 else next_id + if i > 0: + next_id += 1 + for t in piece: + relabel.setdefault(int(t), {})[label_id] = piece_id + rows.append([piece_id, int(piece[0]), int(piece[-1]), previous]) + previous = piece_id + last_piece[label_id] = previous + for row in rows: + if row[3] is None: # The first piece keeps the parent of the track, if that parent was kept. + parent = parent_of.get(row[0], 0) + row[3] = last_piece.get(parent, 0) + for t, mapping in relabel.items(): + path = os.path.join(out_folder, f"man_track{t:04d}.tif") + frame = tifffile.imread(path) + for old, new in mapping.items(): + frame[frame == old] = new + tifffile.imwrite(path, frame) + pd.DataFrame(rows, columns=["track", "start", "end", "parent"]).to_csv( + os.path.join(out_folder, "man_track.txt"), sep=" ", header=False, index=False + ) + return out_folder + + +class LazyFrames: + """The frames of a CTC folder, read one at a time, for the SEG score of a prediction that does not fit in memory.""" + + def __init__(self, folder): + self.paths = sorted(glob.glob(os.path.join(folder, "man_track*.tif"))) + + def __getitem__(self, t): + return tifffile.imread(self.paths[t]) + + +def set_norm_percentiles(lower, upper): + """Replace the 2nd and 98th percentile bounds of the micro-sam2 input normalization.""" + import micro_sam.v2.normalization as normalization + for function in (normalization.compute_percentile_bounds, normalization.normalize_raw): + names = function.__code__.co_varnames[:function.__code__.co_argcount] + defaults = list(function.__defaults__) + first = len(names) - len(defaults) + defaults[names.index("lower_percentile") - first] = lower + defaults[names.index("upper_percentile") - first] = upper + function.__defaults__ = tuple(defaults) + + +def frame_paths(name, sequence): + """Get the paths of the frames of one sequence.""" + return sorted(glob.glob(os.path.join(dataset_root(name), sequence, "t*.tif"))) + + +def load_frames(name, sequence, start, end): + """Load the frames start to end (exclusive) of one sequence.""" + return to_grayscale(np.stack([tifffile.imread(path) for path in frame_paths(name, sequence)[start:end]])) + + +# The conversion of the movies of other sources to the CTC format of the tracking evaluation. +WING_DISC_CROP = 160 + + +def split_gaps(masks, parents): + """Split the tracks with a gap in time; return the new masks and the lineage rows (id, start, end, parent).""" + masks = masks.copy() + frames = {} + for t in range(masks.shape[0]): + for track in np.unique(masks[t])[1:]: + frames.setdefault(int(track), []).append(t) + next_id = max(frames, default=0) + 1 + parts_of = {} + for track in sorted(frames): + ts = np.array(frames[track]) + parts_of[track] = [] + for k, part in enumerate(np.split(ts, np.flatnonzero(np.diff(ts) > 1) + 1)): + new_id = track + if k > 0: + new_id, next_id = next_id, next_id + 1 + for t in part: + masks[t][masks[t] == track] = new_id + parts_of[track].append((new_id, int(part[0]), int(part[-1]))) + rows = [] + for track, parts in parts_of.items(): + # A daughter hangs on the last part of its mother; a mother without pixels is dropped. + mother = parents.get(track, 0) + parent = parts_of[mother][-1][0] if mother in parts_of else 0 + for new_id, start, end in parts: + rows.append([new_id, start, end, parent]) + parent = new_id + return masks, rows + + +def write_sequence(root, sequence, raw, masks, parents): + """Write one movie and its dense track masks in the CTC format. + + Frames without any mask at the start or the end are left out: traccuracy shifts the frames of a prediction + against the ground truth when the first frame is empty, so that nothing matches (E. coli starts so). + """ + annotated = np.flatnonzero(masks.reshape(masks.shape[0], -1).max(axis=1) > 0) + raw, masks = raw[annotated[0]:annotated[-1] + 1], masks[annotated[0]:annotated[-1] + 1] + masks, rows = split_gaps(masks.astype("uint32"), parents) + digits = max(3, len(str(raw.shape[0] - 1))) + folders = {name: os.path.join(root, *parts) for name, parts in ( + ("raw", (sequence,)), ("tra", (f"{sequence}_GT", "TRA")), ("seg", (f"{sequence}_GT", "SEG")), + )} + for folder in folders.values(): + os.makedirs(folder, exist_ok=True) + for t in range(raw.shape[0]): + tifffile.imwrite(os.path.join(folders["raw"], f"t{t:0{digits}d}.tif"), raw[t]) + tifffile.imwrite(os.path.join(folders["tra"], f"man_track{t:0{digits}d}.tif"), masks[t].astype("uint16")) + tifffile.imwrite(os.path.join(folders["seg"], f"man_seg{t:0{digits}d}.tif"), masks[t].astype("uint16")) + np.savetxt(os.path.join(folders["tra"], "man_track.txt"), np.array(rows, dtype=int), fmt="%d") + n_divisions = len({row[3] for row in rows if row[3] > 0}) + print(f"{os.path.basename(root)} {sequence}: {raw.shape}, {len(rows)} tracks, {n_divisions} divisions") + + +def schnitz_parents(path): + """The parent position of each schnitz, from 1; MATLAB v7.3 files are HDF5 files and need h5py.""" + try: + schnitzes = sio.loadmat(path, simplify_cells=True)["schnitzcells"] + return {k: int(np.atleast_1d(s["P"])[0]) for k, s in enumerate(schnitzes, start=1)} + except NotImplementedError: + import h5py + with h5py.File(path, "r") as f: + refs = f["schnitzcells"]["P"] + return {k: int(np.array(f[refs[k - 1, 0]]).ravel()[0]) for k in range(1, refs.shape[0] + 1)} + + +def prepare_ecoli(output): + from torch_em.data.datasets.light_microscopy.ecoli_microcolony_lineage import _prepare_colony_labels + root = os.path.join(DATA_ROOT, "ecoli_microcolony_lineage") + cache = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + try: + for lineage_path in sorted(glob.glob(os.path.join(root, "*", "*", "data", "*_lin.mat"))): + if "__MACOSX" in lineage_path: + continue + colony_dir = os.path.dirname(os.path.dirname(lineage_path)) + if os.path.exists(os.path.join(output, f"{os.path.basename(colony_dir)}_GT", "TRA", "man_track.txt")): + continue + raw_paths, label_paths = _prepare_colony_labels(colony_dir, cache) + if not raw_paths: + continue + raw = np.stack([tifffile.imread(path) for path in raw_paths]) + masks = np.stack([tifffile.imread(path) for path in label_paths]) + # The track ids are the positions of the schnitzes in the file, from 1; 'P' is the parent position. + parents = {k: max(p, 0) for k, p in schnitz_parents(lineage_path).items()} + write_sequence(output, os.path.basename(colony_dir), raw, masks, parents) + finally: + shutil.rmtree(cache, ignore_errors=True) + + +def prepare_alfi(output): + import pandas as pd + root = os.path.join(DATA_ROOT, "alfi", "Data&Annotations") + for movie in sorted(os.listdir(root)): + table = pd.read_csv(os.path.join(root, movie, f"{movie}_DTLTruth.csv"), dtype={"ID": str}) + names = sorted(table["ID"].unique(), key=lambda name: [int(part) for part in name.split(".")]) + track_of = {name: k for k, name in enumerate(names, start=1)} + # '3.1' is a daughter of '3'. + parents = {track_of[name]: track_of.get(name.rsplit(".", 1)[0], 0) if "." in name else 0 for name in names} + frames = sorted(table["ImNo"].unique()) + raw, masks = [], [] + for number in frames: + image = imageio.imread(os.path.join(root, movie, "Images", f"I_{movie}_{number:04d}.png")) + labels = tifffile.imread(os.path.join(root, movie, "instances_labels", f"M_{movie}_{number:04d}.tif")) + boxes = table[table["ImNo"] == number] + mask = np.zeros(labels.shape, dtype="uint32") + for label in np.unique(labels)[1:]: + ys, xs = np.nonzero(labels == label) + inside = [ + ((ys >= b.ymin) & (ys < b.ymin + b.height) & (xs >= b.xmin) & (xs < b.xmin + b.width)).mean() + for b in boxes.itertuples() + ] + if max(inside) > 0.8: + mask[labels == label] = track_of[boxes["ID"].iloc[int(np.argmax(inside))]] + raw.append(image[..., 0] if image.ndim == 3 else image) + masks.append(mask) + write_sequence(output, movie, np.stack(raw), np.stack(masks), parents) + + +def prepare_cardioblast(output): + root = os.path.join(DATA_ROOT, "cardioblast_nuclei", "cardioblast_nuclei") + for folder in sorted(glob.glob(os.path.join(root, "*", "cardioblast_nuclei_*"))): + name = os.path.basename(folder) + raw = tifffile.imread(os.path.join(folder, f"{name}.tif")) + masks = tifffile.imread(os.path.join(folder, f"{name}_mask.tif")) + # One raw movie has unannotated trailing frames (see the torch_em loader). + annotated = np.flatnonzero(masks.reshape(masks.shape[0], -1).max(axis=1) > 0) + n_frames = annotated[-1] + 1 + write_sequence(output, name.replace("cardioblast_nuclei_", ""), raw[:n_frames], masks[:n_frames], {}) + + +def prepare_wing_disc(output): + root = os.path.join(DATA_ROOT, "wing_disc_timelapse") + for archive in sorted(glob.glob(os.path.join(root, "Ecd_*.zip"))): + movie = os.path.basename(archive)[:-len(".zip")] + if os.path.exists(os.path.join(output, f"{movie}_GT", "TRA", "man_track.txt")): + continue + work = tempfile.mkdtemp(dir=os.environ.get("TMPDIR")) + try: + with zipfile.ZipFile(archive) as f: + # Some movies store the colour images as tif instead of png. + members = [m for m in f.namelist() if m.endswith(".sqlite") or "tracked_cells_resized." in m] + f.extractall(work, members) + database = sqlite3.connect(glob.glob(os.path.join(work, movie, "*.sqlite"))[0]) + histories = database.execute( + "select cell_id, tissue_analyzer_group_id, first_occ, last_occ, left_daughter_cell_id, " + "right_daughter_cell_id from cell_histories" + ).fetchall() + parents = {} + for cell, _, _, _, left, right in histories: + for daughter in (left, right): + if daughter is not None: + parents[int(daughter)] = int(cell) + images = sorted(glob.glob(os.path.join(root, "converted", movie, "images", "*.tif"))) + frame_folders = { + int(re.search(r"_(\d+)$", os.path.dirname(path)).group(1)): path + for path in glob.glob(os.path.join(work, movie, "Segmentation", "*", "tracked_cells_resized.*")) + } + raw, masks = [], [] + for path in images: + t = int(os.path.basename(path)[:-len(".tif")]) + image = tifffile.imread(path) + path = frame_folders[t] + colours = tifffile.imread(path) if path.endswith(".tif") else imageio.imread(path) + colours = colours[..., :3].astype(np.int64) + group = colours[..., 0] * 65536 + colours[..., 1] * 256 + colours[..., 2] + assert group.shape == image.shape[:2], (group.shape, image.shape) + # The group id of a cell, with the frame, gives its cell id: one cell history between divisions. + cell_of = {g: c for c, g, first, last, _, _ in histories if first <= t <= last} + y0, x0 = (np.array(group.shape) - WING_DISC_CROP) // 2 + crop = (slice(y0, y0 + WING_DISC_CROP), slice(x0, x0 + WING_DISC_CROP)) + group = group[crop] + mask = np.zeros(group.shape, dtype="uint32") + for g in np.unique(group): + if g in cell_of: + mask[group == g] = cell_of[g] + raw.append(image[crop]) + masks.append(mask) + # The cell ids start at 10000; they fit uint16 masks after renumbering. + masks = np.stack(masks) + ids = np.unique(masks)[1:] + new = {int(old): k for k, old in enumerate(ids, start=1)} + lookup = np.zeros(int(masks.max()) + 1, dtype="uint32") + lookup[ids] = np.arange(1, len(ids) + 1) + parents = {new[d]: new.get(p, 0) for d, p in parents.items() if d in new} + write_sequence(output, movie, np.stack(raw), lookup[masks], parents) + finally: + shutil.rmtree(work, ignore_errors=True) + + +def prepare_yeaz(output): + """The YeaZ phase-contrast time-lapse movies of the 'val' and 'test' splits with at least 5 frames. + + micro-sam2 trained on YeaZ bright field only, and micro-sam on the 'train' split of the phase-contrast movies, so + neither model saw these movies. The masks keep the cell ids over time; buds have no parent link. + """ + import json + root = os.path.join(DATA_ROOT, "yeaz") + splits = json.load(open(os.path.join(root, "yeaz_phc_splits.json"))) + for name in splits["val"] + splits["test"]: + masks = tifffile.imread(os.path.join(root, "gold-standard-PhC-plus-2", name.replace("_im.tif", "_mask.tif"))) + if masks.ndim != 3 or masks.shape[0] < 5: + continue + raw = tifffile.imread(os.path.join(root, "gold-standard-PhC-plus-2", name)) + write_sequence(output, name[:-len("_im.tif")], raw, masks, {}) + + +def prepare_cell_acdc(output): + """The Cell-ACDC test movie of the micro-sam2 split ('CELL_ACDC_TEST_MOVIES'), with the phase-contrast frames. + + Only the frames listed in the 'acdc_output.csv' table are annotated. A bud keeps growing next to its mother, which + the CTC lineage cannot express (a mother must end where its daughters start), so the buds have no parent link. + """ + import pandas as pd + folder = os.path.join( + DATA_ROOT, "cell_acdc", "test_data_budding_yeast", "TimeLapse_2D", "MIA_KC_htb1_mCitrine_labeled", + "Position_8", "Images", + ) + prefix = os.path.join(folder, "19-03-2021_KCY050_SCGE_s08") + segmentation = np.load(f"{prefix}_segm.npz") + masks = segmentation[segmentation.files[0]] + raw = tifffile.imread(f"{prefix}_phase_contr.tif") + n_frames = int(pd.read_csv(f"{prefix}_acdc_output.csv")["frame_i"].max()) + 1 + write_sequence(output, "Position_8", raw[:n_frames], masks[:n_frames], {}) + + +def prepare_dynamicnuclearnet(output): + """The DynamicNuclearNet tracking test movies, the in-domain movies of the interactive tracking. + + micro-sam2 trained on the DynamicNuclearNet segmentation data, not on these movies. They are written again in the + layout of the automatic tracking, with their lineage table. + """ + from tracking_util import interactive_samples, load_interactive_sample + for sample in interactive_samples("dynamicnuclearnet"): + raw, masks, tracks = load_interactive_sample("dynamicnuclearnet", sample) + parents = dict(zip(tracks["track"].astype(int), tracks["parent"].astype(int))) + write_sequence(output, sample, raw, masks, parents) + + +PREPARED_DATASETS = { + "EColi": prepare_ecoli, "ALFI": prepare_alfi, "Cardioblast": prepare_cardioblast, "WingDisc": prepare_wing_disc, + "YeaZ": prepare_yeaz, "CellACDC": prepare_cell_acdc, "DynamicNuclearNet": prepare_dynamicnuclearnet +} + + +# The collection of the per-run result tables into one table per pipeline. +COLUMNS = ["TRA", "DET", "LNK", "SEG", "Division F1 (Frame Buffer 0)", "Division F1 (Frame Buffer 2)"] + +# The scores of the central tables, with their names there. +SCORES = { + "TRA": "tra", "DET": "det", "LNK": "lnk", "SEG": "seg", "AOGM": "aogm", + "Division F1 (Frame Buffer 0)": "division_f1", "Division F1 (Frame Buffer 2)": "division_f1_buffer2", + "Mitotic Branching Correctness (Frame Buffer 0)": "mbc", +} + +# The modality of each dataset that its CTC name does not give, as (category, structure). +MODALITIES = { + "BacMother": ("label-free", "cell"), "TOIAM": ("label-free", "cell"), + "dynamicnuclearnet": ("fluorescence", "nucleus"), "toiam": ("label-free", "cell"), + "bacmother": ("label-free", "cell"), "tracking_fm_real": ("mixed", "mixed"), + # ALFI labels interphase nuclei and whole mitotic cells; it counts as label-free cells. + "EColi": ("label-free", "cell"), "ecoli": ("label-free", "cell"), "ALFI": ("label-free", "cell"), + "alfi": ("label-free", "cell"), "Cardioblast": ("fluorescence", "nucleus"), + "cardioblast": ("fluorescence", "nucleus"), + "WingDisc": ("fluorescence", "cell"), "wingdisc": ("fluorescence", "cell"), + "YeaZ": ("label-free", "cell"), "yeaz": ("label-free", "cell"), "CellACDC": ("label-free", "cell"), + "cellacdc": ("label-free", "cell"), "DynamicNuclearNet": ("fluorescence", "nucleus"), +} + +# The datasets that micro-sam (vit_l_lm) trained on, from the v1 training script. +MICROSAM_TRAINING = { + "BF-C2DL-HSC", "BF-C2DL-MuSC", "DIC-C2DH-HeLa", "Fluo-C2DL-Huh7", "Fluo-C2DL-MSC", "Fluo-C3DH-A549", + "Fluo-C3DH-A549-SIM", "Fluo-C3DH-H157", "Fluo-C3DL-MDA231", "Fluo-N2DH-SIM+", "Fluo-N3DH-CE", "Fluo-N3DH-CHO", + "Fluo-N3DH-SIM+", "Fluo-N3DL-DRO", "Fluo-N3DL-TRIC", "PhC-C2DH-U373", "PhC-C2DL-PSC", "dynamicnuclearnet", +} + + +def collect(folder): + """Concatenate all result tables of one pipeline, or None without any.""" + paths = sorted(glob.glob(os.path.join(folder, "results", "*.csv"))) + if not paths: + return None + return pd.concat([pd.read_csv(path, dtype={"sample": str}) for path in paths], ignore_index=True) + + +def modality(dataset): + if dataset in MODALITIES: + return MODALITIES[dataset] + prefix, kind = dataset.split("-")[:2] + return ("fluorescence" if prefix == "Fluo" else "label-free", "nucleus" if kind.startswith("N") else "cell") + + +def scores_of(table, keys): + """Average the scores over the sequences or movies of each group, and name them as in the central tables.""" + scores = [column for column in SCORES if column in table.columns] + grouped = table.groupby(keys, dropna=False) + result = grouped[scores].mean().rename(columns=SCORES) + result.insert(0, "n_samples", grouped.size()) + return result.reset_index() + + +def write_tables(pipeline, keys, central, output_root, figure_folder=None): + """Join the per-run tables of one pipeline into '_tracking_results.csv' in the output root, print them, + and, with 'figure_folder', write the central table of the paper ('cells__tracking_results.csv').""" + table = collect(os.path.join(output_root, pipeline)) + if table is None: + print(f"No {pipeline} results yet.") + return + save_path = os.path.join(output_root, f"{pipeline}_tracking_results.csv") + table.to_csv(save_path, index=False) + columns = keys + [column for column in COLUMNS if column in table.columns] + print(f"{pipeline} ({save_path}):") + print(table[columns].round(3).to_string(index=False)) + if figure_folder is not None: + path = os.path.join(figure_folder, f"cells_{pipeline}_tracking_results.csv") + central(table).to_csv(path, index=False) + print(f"Wrote the central table '{path}'.") + + +def prepare_tracking_data(names=None, output_root=TRACKING_DATA): + """Convert the datasets of PREPARED_DATASETS (all by default) to the CTC format, see 'write_sequence'.""" + for name in names or list(PREPARED_DATASETS): + PREPARED_DATASETS[name](os.path.join(output_root, name)) diff --git a/finetuning/v2/generalist/create_data_overview.py b/finetuning/v2/generalist/create_data_overview.py new file mode 100644 index 000000000..6f73bb1e1 --- /dev/null +++ b/finetuning/v2/generalist/create_data_overview.py @@ -0,0 +1,680 @@ +"""Create the overview table of the training and evaluation data of the micro-sam2 generalist. + +The table has one row per dataset and these columns: + +- ``dataset``: The dataset name. Evaluated datasets use their name in ``finetuning/v2/evaluation/common.py``. +- ``modality``: fluorescence, label-free, EM or histopathology; XPRESS is X-ray. A dataset that mixes modalities + gets its majority modality, see the comments of cellpose and neurips_cellseg. +- ``structure``: The labeled structure, such as cell, nucleus or neurite. +- ``dimensionality``: 2d or 3d. +- ``role``: ``train`` (training data only), ``ID`` (training data with a scored test split) or ``OOD`` (scored + test split of a dataset that is meant to be kept out of training). ID and OOD follow the ``PANEL_DATASETS`` of + the Figure 2 (2d) and Figure 3 (3d) plotting scripts in ``scripts/plotting/v2/for_paper``. +- ``train_images_or_volumes``, ``train_objects``: The images or volumes and the labeled objects of the training + data (train + val split) of the 'all' generalist, i.e. the 'lm', 'em' and 'hp' domains of ``generalist_loader.py``. + They are counted with ``micro_sam.v2.datasets.data_counter``, so they follow the splits and the label + pre-processing of the loader. See its docstring for how stacks, regions of interest and multi-target images count. +- ``test_images_or_volumes``, ``test_objects``: The test samples that the evaluation scores and their objects, + loaded with ``common.load_data``. The evaluation scores a 512 x 512 center crop of most 2d images and fixed + crops of the 3d volumes (8 x 512 x 512 by default, see ``EVAL_CROPS_3D``), so for 3d data a test sample is one + crop and the objects are the ones inside the crops. Samples without labels are not scored and not counted. + The columns are blank for datasets without a scored test split. +- ``comment``: Notes, e.g. on the subsets, the annotation or the imaging technique of a dataset. + +The table (``data-overview/data_overview.csv`` by default) is also the cache: the counts are only computed for +rows whose count cells are empty, or for all requested rows with ``--force``. The table is written after every +finished dataset, so an interrupted run keeps its progress. Computing the training counts first builds the training +datasets of a domain like the loader does. This takes several minutes and up to 15 GB of memory per domain. +Counting all datasets needs a CPU job with about 64 GB of memory and 8 workers. + +Usage: + + python create_data_overview.py # fill in the missing counts + python create_data_overview.py --datasets livecell # only process this dataset + python create_data_overview.py --datasets livecell --force # recompute its counts +""" + +import os +import ast +import csv +import sys +import argparse +import warnings +from collections import Counter +from dataclasses import dataclass, replace +from concurrent.futures import FIRST_COMPLETED, ProcessPoolExecutor, wait +from typing import Callable, Dict, List, Optional, Tuple + +import numpy as np +from tqdm import tqdm + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.abspath(os.path.join(SCRIPT_DIR, *[os.pardir] * 3)) +EVALUATION_DIR = os.path.join(REPO_ROOT, "finetuning", "v2", "evaluation") +FIGURE_SCRIPTS = { + 2: os.path.join(REPO_ROOT, "scripts", "plotting", "v2", "for_paper", "plots_for_figure2.py"), + 3: os.path.join(REPO_ROOT, "scripts", "plotting", "v2", "for_paper", "plots_for_figure3.py"), +} +DEFAULT_DATA_ROOT = "/mnt/vast-nhr/projects/cidas/cca/data" +DEFAULT_OUTPUT = os.path.join(SCRIPT_DIR, "data-overview", "data_overview.csv") + +TRAIN_COLUMNS = ["train_images_or_volumes", "train_objects"] +TEST_COLUMNS = ["test_images_or_volumes", "test_objects"] +COLUMNS = ["dataset", "modality", "structure", "dimensionality", "role"] + TRAIN_COLUMNS + TEST_COLUMNS + ["comment"] +ROLE_ORDER = {"ID": 0, "OOD": 1, "train": 2} + +INTERNAL = "Internal data, not public." + + +def _common(): + """Import the evaluation module ``common.py``, which resolves and loads the test data.""" + if EVALUATION_DIR not in sys.path: + sys.path.insert(0, EVALUATION_DIR) + import common + return common + + +# +# The registry of the datasets. +# + +def _path_has(*parts: str) -> Callable: + """Select the label sources whose path contains one of the parts.""" + return lambda unit: any(part in unit.label_path for part in parts) + + +def _key_is(key: str) -> Callable: + """Select the label sources with this internal label key.""" + return lambda unit: unit.label_key == key + + +def _deepbacs_fluorescence(unit) -> bool: + return "/mixed/" in unit.label_path and _common().deepbacs_is_fluorescence(unit.label_path) + + +def _deepbacs_label_free(unit) -> bool: + return "/e_coli_stationary/" in unit.label_path or not _common().deepbacs_is_fluorescence(unit.label_path) + + +@dataclass(frozen=True) +class Train: + """The training label sources of a dataset: a domain of the loader, a top-level data folder and a filter.""" + domain: str + folder: str + select: Optional[Callable] = None + + +@dataclass(frozen=True) +class Entry: + """One row of the table. ``test`` is the dataset name of the evaluation, if its test split is scored.""" + name: str + modality: str + structure: str + ndim: int + train: Optional[Train] = None + test: Optional[str] = None + comment: str = "" + + +FL, LF, EM, HP = "fluorescence", "label-free", "EM", "histopathology" + + +def _lm(folder, select=None): + return Train("lm", folder, select) + + +def _em(folder, select=None): + return Train("em", folder, select) + + +def _hp(folder, select=None): + return Train("hp", folder, select) + + +CTC_LABEL_FREE = ("/BF-C2DL-HSC/", "/BF-C2DL-MuSC/", "/DIC-C2DH-HeLa/", "/PhC-C2DH-U373/", "/PhC-C2DL-PSC/") +NEURIPS_TRAIN = "Training data in row neurips_cellseg, whose images are not split by modality." + +ENTRIES = [ + # Light microscopy, 2d. + Entry("bbbc030", LF, "cell", 2, _lm("bbbc030")), + Entry("bccd", LF, "cell", 2, _lm("bccd"), comment="Blood cells in brightfield blood smears."), + Entry("bitdepth_nucseg", FL, "nucleus", 2, _lm("bitdepth_nucseg")), + Entry("bmgd", FL, "nucleus", 2, _lm("bmgd"), "bmgd"), + Entry("cell_acdc", LF, "cell", 2, _lm("cell_acdc"), "cell_acdc"), + Entry("cellbindb", FL, "nucleus", 2, _lm("cellbindb"), "cellbindb", "DAPI, ssDNA and mIF stains."), + Entry( + "cellpose", FL, "cell", 2, _lm("cellpose"), + comment="Mixed modalities (cyto and cyto2), counted as fluorescence, the majority: 45% of the images have a " + "dark fluorescence background, most others are dense fluorescence tissue or non-microscopy images, " + "few are brightfield.", + ), + Entry("cellular", LF, "cell", 2, _lm("cellular")), + Entry("cisd", LF, "cell", 2, _lm("cisd")), + Entry("ctc_label_free", LF, "cell", 2, _lm("ctc", _path_has(*CTC_LABEL_FREE)), + comment="Cell Tracking Challenge: BF-C2DL-HSC, BF-C2DL-MuSC, DIC-C2DH-HeLa, PhC-C2DH-U373, PhC-C2DL-PSC."), + Entry("ctc_fluorescence_cells", FL, "cell", 2, _lm("ctc", _path_has("/Fluo-C2DL-Huh7/", "/Fluo-C2DL-MSC/")), + comment="Cell Tracking Challenge: Fluo-C2DL-Huh7, Fluo-C2DL-MSC."), + Entry("ctc_fluorescence_nuclei", FL, "nucleus", 2, _lm("ctc", _path_has("/Fluo-N2DH-SIM+/")), + comment="Cell Tracking Challenge: Fluo-N2DH-SIM+, simulated nuclei."), + Entry("cvz_fluo_cell", FL, "cell", 2, _lm("cvz", _path_has("_Cell_")), "cvz_fluo_cell"), + Entry("cvz_fluo_dapi", FL, "nucleus", 2, _lm("cvz", _path_has("_Dapi_"))), + Entry("dcis_com_nuclei", FL, "nucleus", 2, _lm("dcis_com_nuclei")), + Entry("deepbacs_fluorescence", FL, "cell", 2, _lm("deepbacs", _deepbacs_fluorescence), "deepbacs_fluorescence"), + Entry("deepbacs_label_free", LF, "cell", 2, _lm("deepbacs", _deepbacs_label_free), + comment="Includes the E. coli stationary-phase images (train split only)."), + Entry("dememseg", FL, "prospore membrane", 2, _lm("dememseg")), + Entry("dsb", FL, "nucleus", 2, _lm("dsb"), "dsb", "The fluorescence images of DSB 2018."), + Entry("dynamicnuclearnet", FL, "nucleus", 2, _lm("dynamicnuclearnet"), "dynamicnuclearnet"), + Entry("enseg", FL, "cell", 2, _lm("enseg"), comment="Enteric neurons."), + Entry("epicure", FL, "cell", 2, _lm("epicure")), + Entry("flywing", FL, "cell", 2, _lm("flywing"), "flywing"), + Entry("ifnuclei", FL, "nucleus", 2, _lm("ifnuclei"), "ifnuclei"), + Entry("livecell", LF, "cell", 2, _lm("livecell"), "livecell"), + Entry("lpc_nucseg", FL, "nucleus", 2, _lm("lpc_nucseg")), + Entry("mcellseg", LF, "cell", 2, _lm("mcellseg")), + Entry("microbeseg", LF, "cell", 2, _lm("microbeseg")), + Entry( + "neurips_cellseg", LF, "cell", 2, _lm("neurips_cellseg"), + comment="Brightfield, phase contrast, DIC and fluorescence images, counted as label-free, the majority: " + "385 of the 1,101 images (35%) are fluorescence by a dark-background rule that classifies 98.7% of the " + "tuning and test images correctly. Its test split is scored as neurips_cellseg_fluorescence and " + "neurips_cellseg_label_free.", + ), + Entry("neurips_cellseg_fluorescence", FL, "cell", 2, test="neurips_cellseg_fluorescence", comment=NEURIPS_TRAIN), + Entry("neurips_cellseg_label_free", LF, "cell", 2, test="neurips_cellseg_label_free", comment=NEURIPS_TRAIN), + Entry("omnipose_bact_fluor", FL, "cell", 2, _lm("omnipose", _path_has("/bact_fluor/")), "omnipose_bact_fluor"), + Entry("omnipose_bact_phase", LF, "cell", 2, _lm("omnipose", _path_has("/bact_phase/")), "omnipose_bact_phase"), + Entry("omnipose_worm", LF, "worm", 2, _lm("omnipose", _path_has("/worm/"))), + Entry("omnipose_worm_high_res", LF, "worm", 2, _lm("omnipose", _path_has("/worm_high_res/"))), + Entry("organoid", LF, "organoid", 2, _lm("organoid")), + Entry("organoidnet", LF, "organoid", 2, _lm("organoidnet")), + Entry("orgasegment", LF, "organoid", 2, _lm("orgasegment")), + Entry("orgline", LF, "organoid", 2, _lm("orgline"), "orgline"), + Entry("pan_multiplex", FL, "cell", 2, _lm("pan_multiplex"), + comment="MIBI, CODEX and Vectra images. MIBI is mass spectrometry imaging, not fluorescence."), + Entry("tissuenet", FL, "cell", 2, _lm("tissuenet"), "tissuenet"), + Entry("toiam", LF, "cell", 2, _lm("toiam")), + Entry("tsakiroglou", FL, "nucleus", 2, _lm("tsakiroglou")), + Entry("u20s", FL, "nucleus", 2, _lm("u20s"), "u20s"), + Entry("usiigaci", LF, "cell", 2, _lm("usiigaci")), + Entry("vicar", LF, "cell", 2, _lm("vicar"), "vicar"), + Entry("xenium_cells", FL, "cell", 2, _lm("xenium", _key_is("labels/cells")), + comment="The labels are the automatic Xenium segmentation, not manual annotations."), + Entry("xenium_nuclei", FL, "nucleus", 2, _lm("xenium", _key_is("labels/nuclei")), + comment="The labels are the automatic Xenium segmentation, not manual annotations."), + Entry("yeaz", LF, "cell", 2, _lm("yeaz")), + # Light microscopy, 2d, OOD. + Entry("arvidsson", FL, "nucleus", 2, test="arvidsson"), + Entry("bac_mother", LF, "cell", 2, test="bac_mother", + comment="E. coli in a mother machine. Every 10th frame of the test movies is scored."), + Entry("bbbc010", LF, "worm", 2, test="bbbc010", comment="C. elegans."), + Entry("cardioblast_nuclei", FL, "nucleus", 2, test="cardioblast_nuclei"), + Entry("cellapp", LF, "cell", 2, test="cellapp", + comment="The HeLa, RPE1 and U2OS subsets. The labels are automatic, so not every cell is annotated."), + Entry("covid_if_cells", FL, "cell", 2, test="covid_if_cells"), + Entry("covid_if_nuclei", FL, "nucleus", 2, test="covid_if_nuclei"), + Entry("deepseas", LF, "cell", 2, test="deepseas"), + Entry("ecoli_microcolony_lineage", LF, "cell", 2, test="ecoli_microcolony_lineage", + comment="E. coli microcolonies (cib, crosstalk, recA, rpsM). Every 10th frame of the movies is scored."), + Entry("fusionx", FL, "cell", 2, test="fusionx"), + Entry("hela_cytonuc", FL, "nucleus", 2, test="hela_cytonuc"), + Entry("hpa", FL, "cell", 2, test="hpa"), + Entry("medussa", FL, "cell", 2, test="medussa"), + Entry("micro_bench", FL, "nucleus", 2, test="micro_bench", comment="The OpenCell nuclei of µBench."), + Entry("mif_tonsil", FL, "nucleus", 2, test="mif_tonsil"), + Entry("mif_tonsil_cells", FL, "cell", 2, test="mif_tonsil_cells"), + Entry("wing_disc_timelapse", FL, "cell", 2, test="wing_disc_timelapse"), + Entry("yeastsam", LF, "cell", 2, test="yeastsam"), + # Light microscopy, 3d. + Entry("cartocell", FL, "cell", 3, _lm("cartocell"), "cartocell"), + Entry("celegans_atlas", FL, "nucleus", 3, _lm("celegans_atlas"), "celegans_atlas"), + Entry("cochleanet_ihc", FL, "cell", 3, _lm("cochleanet_data", _path_has("/ihc/")), + comment=f"Inner hair cells. {INTERNAL}"), + Entry("cochleanet_sgn", FL, "cell", 3, _lm("cochleanet_data", _path_has("/sgn/")), + comment=f"Spiral ganglion neuron somata. {INTERNAL}"), + Entry("embedseg_mouse_skull", FL, "nucleus", 3, _lm("embedseg", _path_has("/Mouse-Skull-Nuclei-CBG/")), + "embedseg_mouse_skull"), + Entry("embedseg_organoid", FL, "cell", 3, _lm("embedseg", _path_has("/Mouse-Organoid-Cells-CBG/")), + "embedseg_organoid"), + Entry("embedseg_platy_ish", FL, "nucleus", 3, _lm("embedseg", _path_has("/Platynereis-ISH-Nuclei-CBG/"))), + Entry("embedseg_platy_nuclei", FL, "nucleus", 3, _lm("embedseg", _path_has("/Platynereis-Nuclei-CBG/")), + "embedseg_platy_nuclei"), + Entry("gonuclear", FL, "nucleus", 3, _lm("gonuclear"), "gonuclear"), + Entry("nis3d", FL, "nucleus", 3, _lm("nis3d"), "nis3d"), + Entry("nucverse3d", FL, "nucleus", 3, _lm("nucverse3d"), "nucverse3d"), + Entry("phmamm", FL, "cell", 3, _lm("phmamm"), "phmamm"), + Entry("plantseg_root", FL, "cell", 3, _lm("plantseg"), "plantseg_root"), + Entry("pnas_arabidopsis", FL, "cell", 3, _lm("pnas_arabidopsis"), "pnas_arabidopsis"), + Entry("wing_disc", FL, "cell", 3, _lm("wing_disc"), "wing_disc"), + # Light microscopy, 3d, OOD. + Entry("bbbc032", FL, "nucleus", 3, test="bbbc032"), + Entry("bbbc033", FL, "nucleus", 3, test="bbbc033"), + Entry("bbbc050", FL, "nucleus", 3, test="bbbc050"), + Entry("blastospim", FL, "nucleus", 3, test="blastospim"), + Entry("cshaper", FL, "cell", 3, test="cshaper"), + Entry("morphonet_celegans", FL, "nucleus", 3, test="morphonet_celegans"), + Entry("morphonet_patiria", FL, "cell", 3, test="morphonet_patiria", + comment="The MorphoNet starfish (Patiria miniata) embryo with its curated labels."), + Entry("nus_organoids", FL, "nucleus", 3, test="nus_organoids", + comment="Three of the four manually annotated volumes of the nucleus benchmark of the Digitalized " + "Organoid paper: cell monolayer, colon organoid and breast cancer spheroid."), + Entry("plantseg_ovules", FL, "cell", 3, test="plantseg_ovules"), + # Electron microscopy (and EM-like), 2d. + Entry("astih_bf", LF, "axon", 2, _em("astih", _path_has("/BF1/", "/BF2/")), + comment="ASTIH BF1 and BF2: brightfield light microscopy of nerve sections, myelinated axons."), + Entry("astih_sem", EM, "axon", 2, _em("astih", _path_has("/SEM1/")), comment="ASTIH SEM1: myelinated axons."), + Entry("tumor_spheroid", EM, "cell", 2, _em("tumor_spheroid_em")), + # Electron microscopy (and EM-like), 3d. + Entry("axonem", EM, "neurite", 3, _em("axonem")), + Entry("cremi", EM, "neurite", 3, _em("cremi"), "cremi"), + Entry("densecell", EM, "cell", 3, _em("densecell")), + Entry( + "emneuron", EM, "neurite", 3, _em("emneuron"), + comment="A compilation of vEM volumes (without its AxonEM, CREMI and FIB-25 folders). It also holds " + "Kasthuri (ATUM) and J0126 volumes, the sources of the snemi and zebrafinch_j0126 test data. Its " + "out-of-distribution validation volumes, the sources of ionsem, microns_basil and synapseweb, do not train.", + ), + Entry("fafb", EM, "neurite", 3, _em("fafb"), "fafb"), + Entry("fib25", EM, "neurite", 3, _em("fib25")), + Entry("hemibrain", EM, "neurite", 3, _em("hemibrain"), "hemibrain"), + Entry("igor_cells", EM, "cell", 3, _em("igor_cells"), comment=f"Internal annotations. {INTERNAL}"), + Entry("liconn", FL, "neurite", 3, _em("liconn"), "liconn", comment="Expansion light microscopy (EM-like), not EM."), + Entry("malecns", EM, "neurite", 3, _em("malecns")), + Entry("manc", EM, "neurite", 3, _em("manc")), + Entry("microns_minnie65", EM, "neurite", 3, _em("microns-minnie65")), + Entry("microns_pinky", EM, "neurite", 3, _em("microns")), + Entry("platynereis_cells", EM, "cell", 3, _em("platynereis")), + Entry("snemi", EM, "neurite", 3, _em("snemi"), "snemi"), + Entry("synapsenet_compartments", EM, "synaptic compartment", 3, _em("synapsenet_compartments_data"), + comment=INTERNAL), + Entry("wafer4", EM, "neurite", 3, _em("wafer4")), + Entry("wildenberg", EM, "neurite", 3, _em("wildenberg2023"), comment="The labels are an automatic segmentation."), + Entry("xpress", "X-ray", "axon", 3, _em("xpress"), + comment="X-ray holographic nano-tomography (EM-like), not EM. Myelinated axons."), + Entry("zebrafinch_j0126", EM, "neurite", 3, _em("zebrafinch", _path_has("/j0126_")), "zebrafinch_j0126"), + Entry("zebrafinch_j0251", EM, "neurite", 3, _em("zebrafinch", _path_has("/j0251_"))), + # Electron microscopy, 3d, OOD. + Entry("humanneurons", EM, "neurite", 3, test="humanneurons", + comment="A 64 x 2048 x 2048 crop of the H01 human cortex volume."), + Entry("ionsem", EM, "neurite", 3, test="ionsem", + comment="Ionsem, an out-of-distribution validation volume of EMNeuron."), + Entry("isbi2012", EM, "neurite", 3, test="isbi2012"), + Entry("microns_basil", EM, "neurite", 3, test="microns_basil", + comment="MICrONS basil (mouse visual cortex, ssTEM), an out-of-distribution validation volume of " + "EMNeuron."), + Entry("nisb", EM, "neurite", 3, test="nisb", comment="Synthetic data."), + Entry("synapseweb", EM, "neurite", 3, test="synapseweb"), + # Histopathology, 2d. + Entry("cellbindb_he", HP, "nucleus", 2, _hp("cellbindb")), + Entry("consep", HP, "nucleus", 2, _hp("consep")), + Entry("cpm15", HP, "nucleus", 2, _hp("cpm15")), + Entry("cpm17", HP, "nucleus", 2, _hp("cpm17"), "cpm17"), + Entry("cryonuseg", HP, "nucleus", 2, _hp("cryonuseg")), + Entry("dsb_he", HP, "nucleus", 2, _hp("dsb"), comment="The H&E images of DSB 2018."), + Entry("glysac", HP, "nucleus", 2, _hp("glysac"), "glysac"), + Entry("histo_miner", HP, "nucleus", 2, _hp("histo_miner"), "histo_miner"), + Entry("lizard", HP, "nucleus", 2, _hp("lizard")), + Entry("lizard_mitosis", HP, "nucleus", 2, _hp("lizard_mitosis")), + Entry("lynsec_he", HP, "nucleus", 2, _hp("lynsec", _path_has("/h&e/"))), + Entry("lynsec_ihc", HP, "nucleus", 2, _hp("lynsec", _path_has("/ihc/")), comment="IHC."), + Entry("monuseg", HP, "nucleus", 2, _hp("monuseg"), "monuseg"), + Entry("nuinsseg", HP, "nucleus", 2, _hp("nuinsseg")), + Entry("pannuke", HP, "nucleus", 2, _hp("pannuke"), "pannuke"), + Entry("puma", HP, "nucleus", 2, _hp("puma"), "puma"), + Entry("spatch_he", HP, "nucleus", 2, _hp("spatch"), comment="The H&E subsets of sPATCH."), + Entry("srsanet", HP, "nucleus", 2, _hp("srsanet"), comment="IHC."), + Entry("tnbc_celltype", HP, "nucleus", 2, _hp("tnbc_celltype")), + # Histopathology, 2d, OOD. + Entry("amonuseg", HP, "nucleus", 2, test="amonuseg", + comment="H&E. Semantic masks: the instances are connected components, so touching nuclei merge."), + Entry("beacon", HP, "nucleus", 2, test="beacon", comment="H&E biopsies of Barrett's esophagus."), + Entry("cd47_ihcnusc", HP, "nucleus", 2, test="cd47_ihcnusc", comment="CD47 IHC of esophageal cancer."), + Entry("cytodark0", HP, "nucleus", 2, test="cytodark0"), + Entry("khoshdeli", HP, "nucleus", 2, test="khoshdeli"), + Entry("pcns", HP, "nucleus", 2, test="pcns"), +] + + +def read_figure_literal(path: str, name: str): + """Read the literal value of a module-level assignment from a plotting script, without importing it.""" + with open(path) as f: + tree = ast.parse(f.read()) + nodes = [ + node.value for node in tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == name for target in node.targets) + ] + if len(nodes) != 1: + raise RuntimeError(f"Expected one {name} assignment in {path}, found {len(nodes)}.") + return ast.literal_eval(nodes[0]) + + +def read_panels(figure_scripts: Optional[Dict[int, str]] = None) -> Dict[str, Tuple[int, str]]: + """Read the ID and OOD datasets of the figure panels from the plotting scripts. + + Args: + figure_scripts: The plotting script of each dimensionality. ``FIGURE_SCRIPTS`` (Figures 2 and 3) by default. + + Returns: + The dimensionality and the role ('ID' or 'OOD') of every evaluated dataset. + """ + panels = {} + for ndim, path in (FIGURE_SCRIPTS if figure_scripts is None else figure_scripts).items(): + for panel in read_figure_literal(path, "PANEL_DATASETS").values(): + for role, names in panel.items(): + for name in names: + if name in panels and panels[name] != (ndim, role): + raise RuntimeError(f"{name} has two roles in the figure panels.") + panels[name] = (ndim, role) + return panels + + +def check_registry(entries: List[Entry], panels: Dict[str, Tuple[int, str]]) -> Dict[str, str]: + """Check the registry against the figure panels and return the role of each entry.""" + names = [entry.name for entry in entries] + duplicates = [name for name, n in Counter(names).items() if n > 1] + if duplicates: + raise RuntimeError(f"Duplicate dataset names in the registry: {duplicates}") + + roles = {} + for entry in entries: + if entry.test is None: + if entry.train is None: + raise RuntimeError(f"{entry.name} has neither training nor test data.") + roles[entry.name] = "train" + continue + if entry.test not in panels: + raise RuntimeError(f"The test data {entry.test} of {entry.name} is not in the figure panels.") + ndim, role = panels[entry.test] + if ndim != entry.ndim: + raise RuntimeError(f"{entry.name} is {entry.ndim}d in the registry, but in a {ndim}d figure.") + roles[entry.name] = role + + missing = sorted(set(panels) - {entry.test for entry in entries}) + if missing: + raise RuntimeError(f"The registry has no row for these evaluated datasets: {missing}") + return roles + + +def assign_units(units, entries: List[Entry]) -> Dict[str, list]: + """Assign every training label source of the built domains to exactly one registry entry. + + Raises: + RuntimeError: If a label source matches no entry or several entries. + """ + domains = {unit.domain for unit in units} + candidates = [entry for entry in entries if entry.train is not None and entry.train.domain in domains] + assigned = {entry.name: [] for entry in candidates} + unassigned, ambiguous = [], [] + for unit in units: + matches = [ + entry for entry in candidates if entry.train.domain == unit.domain and entry.train.folder == unit.dataset + and (entry.train.select is None or entry.train.select(unit)) + ] + if not matches: + unassigned.append(f"{unit.domain}: {unit.label_path} ({unit.label_key})") + elif len(matches) > 1: + ambiguous.append(f"{unit.label_path}: {[entry.name for entry in matches]}") + else: + assigned[matches[0].name].append(unit) + if unassigned or ambiguous: + raise RuntimeError( + "The registry does not partition the training data. Update its entries.\n" + f"Unassigned label sources ({len(unassigned)}): {unassigned[:10]}\n" + f"Label sources with several entries ({len(ambiguous)}): {ambiguous[:10]}" + ) + empty = [name for name, entry_units in assigned.items() if not entry_units] + if empty: + raise RuntimeError(f"These entries select no training data: {empty}") + return assigned + + +# +# Counting. +# + +def count_test_shard(dataset_name: str, ndim: int, data_root: str, shard: int, n_shards: int) -> Tuple[int, int]: + """Count the scored test samples and their objects for every n_shards-th sample of a dataset. + + Args: + dataset_name: The dataset name of the evaluation. + ndim: The dimensionality of the data. + data_root: The root folder of the data. + shard: The index of the first sample of this shard. + n_shards: The number of shards. + + Returns: + The number of samples with labels and the number of their objects. + """ + common = _common() + total = common.n_samples(dataset_name, data_root) + sample_ids = set(range(shard, total, n_shards)) + n_samples, n_objects = 0, 0 + if not sample_ids: + return n_samples, n_objects + for _, labels, _ in common.load_data(dataset_name, data_root, ndim, normalize=False, sample_ids=sample_ids): + if labels.max() == 0: # The evaluation does not score samples without labels. + continue + n_samples += 1 + n_objects += int(np.count_nonzero(np.unique(labels))) + return n_samples, n_objects + + +def build_training_units(domains, data_root): + """Build the training datasets of the domains like the loader and collect their label sources.""" + from micro_sam.v2.datasets.data_counter import build_units + + units, transforms = [], {} + for domain in domains: + print(f"Build the training datasets of the domain '{domain}'. This takes several minutes.", flush=True) + domain_units, domain_transforms = build_units(domain, data_root) + units.extend(domain_units) + transforms.update(domain_transforms) + return units, transforms + + +def submit_train(pool, units, transforms): + """Submit the counting of the training label sources of one dataset. + + Returns: + The counting jobs and their futures. A file split into several regions of interest is one volume, see + ``micro_sam.v2.datasets.data_counter.counting_jobs``. + """ + from micro_sam.v2.datasets.data_counter import count_job, counting_jobs + + jobs = counting_jobs(units, transforms) + return [(job, pool.submit(count_job, job)) for job in jobs] + + +def finish_train(name, units, futures): + """Sum the counts of the training label sources of one dataset.""" + from micro_sam.v2.datasets.data_counter import summarize + + counts = { + unit.key: count for job, future in futures for unit, count in zip(job.units, future.result()) + } + _, rows = summarize([replace(unit, dataset=name) for unit in units], counts) + return sum(row["n_images"] for row in rows), sum(row["n_objects"] for row in rows) + + +def submit_test(pool, dataset_names, ndim, data_root, n_workers): + """Submit the counting of the scored test samples of one or several pooled datasets, split into shards.""" + futures = [] + for dataset_name in dataset_names: + n_samples = _common().n_samples(dataset_name, data_root) + n_shards = max(1, min(n_workers, n_samples)) + futures.extend( + pool.submit(count_test_shard, dataset_name, ndim, data_root, i, n_shards) for i in range(n_shards) + ) + return futures + + +def finish_test(futures): + """Sum the counts of the test shards of one dataset.""" + results = [future.result() for future in futures] + return sum(r[0] for r in results), sum(r[1] for r in results) + + +# +# The table. +# + +def read_table(path: str) -> Dict[str, dict]: + """Read the existing table. A missing table is empty.""" + if not os.path.exists(path): + return {} + with open(path, newline="") as f: + return {row["dataset"]: row for row in csv.DictReader(f)} + + +def write_table(path: str, rows: Dict[str, dict]) -> None: + """Write the table sorted by dimensionality, modality, structure and role, via a temporary file.""" + ordered = sorted( + rows.values(), + key=lambda row: (row["dimensionality"], row["modality"], row["structure"], ROLE_ORDER[row["role"]], + row["dataset"]), + ) + tmp_path = f"{path}.tmp{os.getpid()}" + with open(tmp_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=COLUMNS) + writer.writeheader() + writer.writerows(ordered) + os.replace(tmp_path, path) + + +def _futures(submitted) -> list: + """The futures of the submitted counts: the train counts are (job, future) pairs, the test counts futures.""" + return [item[1] if isinstance(item, tuple) else item for item in submitted] + + +def _is_missing(row: dict, columns: List[str]) -> bool: + return any(row.get(column, "") in ("", None) for column in columns) + + +def run( + output: str, data_root: str, datasets: Optional[List[str]], force: bool, n_workers: int, + registry: Optional[List[Entry]] = None, roles: Optional[Dict[str, str]] = None, + pooled: Optional[Dict[str, List[str]]] = None, +) -> None: + """Fill in the missing counts of the table, or recompute them with 'force'. + + Args: + output: The path of the CSV table, which is read first and also serves as the cache. + data_root: The root folder of the data. + datasets: Only process these datasets. All by default. + force: Recompute the counts of the processed datasets even if the table has them. + n_workers: The number of processes that count in parallel. + registry: The rows of the table. ``ENTRIES`` by default. + roles: The role of each entry. By default derived from the Figure 2 and 3 panels with ``check_registry``. + pooled: The evaluation datasets that a figure pools into one test dataset, by the name of the pooled + dataset. The test counts of a pooled dataset are the sums over its parts. + """ + registry = ENTRIES if registry is None else registry + if roles is None: + roles = check_registry(registry, read_panels()) + entries = {entry.name: entry for entry in registry} + if datasets is not None: + unknown = sorted(set(datasets) - set(entries)) + if unknown: + raise ValueError(f"Unknown datasets: {unknown}. Choose from {sorted(entries)}.") + + # The static columns always come from the registry, the counts from the existing table. + old_rows = read_table(output) + dropped = sorted(set(old_rows) - set(entries)) + if dropped: + warnings.warn(f"Dropping the rows that are no longer in the registry: {dropped}") + rows = {} + for name, entry in entries.items(): + old = old_rows.get(name, {}) + # Counts of a split the entry no longer has, e.g. after a dataset left the evaluation, are cleared. + kept = (TRAIN_COLUMNS if entry.train is not None else []) + (TEST_COLUMNS if entry.test is not None else []) + rows[name] = { + "dataset": name, "modality": entry.modality, "structure": entry.structure, + "dimensionality": f"{entry.ndim}d", "role": roles[name], "comment": entry.comment, + **{column: old.get(column, "") if column in kept else "" for column in TRAIN_COLUMNS + TEST_COLUMNS}, + } + + selected = list(entries) if datasets is None else datasets + need_train = [ + name for name in selected if entries[name].train is not None + and (force or _is_missing(rows[name], TRAIN_COLUMNS)) + ] + need_test = [ + name for name in selected if entries[name].test is not None and (force or _is_missing(rows[name], TEST_COLUMNS)) + ] + todo = [name for name in selected if name in need_train or name in need_test] + if not todo: + write_table(output, rows) + print(f"All counts are in {output}. Pass --force to recompute them.") + return + + units_per_entry, transforms = {}, {} + if need_train: + domains = sorted({entries[name].train.domain for name in need_train}) + units, transforms = build_training_units(domains, data_root) + units_per_entry = assign_units(units, registry) + + failed = [] + with ProcessPoolExecutor(max_workers=n_workers) as pool: + pending = {} + for name in todo: + jobs = {} + try: + if name in need_train: + jobs["train"] = submit_train(pool, units_per_entry[name], transforms) + if name in need_test: + test = entries[name].test + parts = (pooled or {}).get(test, [test]) + jobs["test"] = submit_test(pool, parts, entries[name].ndim, data_root, n_workers) + except Exception as e: + warnings.warn(f"Could not start counting {name}: {e!r}") + failed.append(name) + if jobs: + pending[name] = jobs + + with tqdm(total=len(todo), desc="Count the datasets") as pbar: + pbar.update(len(todo) - len(pending)) + while pending: + futures = [f for jobs in pending.values() for fs in jobs.values() for f in _futures(fs)] + wait(futures, return_when=FIRST_COMPLETED) + for name in [ + n for n, jobs in pending.items() if all(f.done() for fs in jobs.values() for f in _futures(fs)) + ]: + jobs = pending.pop(name) + try: + if "train" in jobs: + values = finish_train(name, units_per_entry[name], jobs["train"]) + rows[name].update(zip(TRAIN_COLUMNS, values)) + if "test" in jobs: + rows[name].update(zip(TEST_COLUMNS, finish_test(jobs["test"]))) + except Exception as e: + tqdm.write(f"Counting {name} failed: {e!r}") + failed.append(name) + write_table(output, rows) + pbar.set_postfix_str(name) + pbar.update(1) + + write_table(output, rows) + print(f"Wrote {output}.") + if failed: + print(f"The counts of these datasets failed and stay empty: {sorted(set(failed))}") + + +def main(): + parser = argparse.ArgumentParser( + description="Create the overview table of the micro-sam2 generalist training and evaluation data.", + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, + ) + parser.add_argument("-o", "--output", default=DEFAULT_OUTPUT, help="The CSV table, also used as the cache.") + parser.add_argument("--data_root", default=DEFAULT_DATA_ROOT, help="The root folder of the data.") + parser.add_argument("-d", "--datasets", nargs="+", help="Only process these datasets. All by default.") + parser.add_argument("-f", "--force", action="store_true", help="Recompute the counts that the table has.") + parser.add_argument( + "-n", "--n_workers", type=int, default=min(8, len(os.sched_getaffinity(0))), + help="The number of processes that count in parallel. Use at most 8 on a 64 GB allocation.", + ) + args = parser.parse_args() + run(args.output, args.data_root, args.datasets, args.force, args.n_workers) + + +if __name__ == "__main__": + main() diff --git a/finetuning/v2/generalist/create_data_overview_em_organelles.py b/finetuning/v2/generalist/create_data_overview_em_organelles.py new file mode 100644 index 000000000..76fd847cf --- /dev/null +++ b/finetuning/v2/generalist/create_data_overview_em_organelles.py @@ -0,0 +1,192 @@ +"""Create the overview table of the training data of the micro-sam2 EM organelle models (mitochondria and nuclei). + +The table combines the two models: the mitochondria model ('em_mito', see ``EM_MITO_DATASETS`` in +``micro_sam/v2/datasets/generalist_loader.py`` and ``em_organelles/train_em_mito.py``) and the nucleus model +('em_nucleus', see ``EM_NUCLEUS_DATASETS`` and ``em_organelles/train_em_nucleus.py``). It has the columns of the +table of ``create_data_overview.py``: + +- ``dataset``: The dataset name. It follows the dataset names of the loader, except for vEM nuclei, which has a 2d + and a 3d row. Rows with test data only use the dataset name of Figure 6, except for 'mitolab_3d'. +- ``modality``: EM; the soft X-ray tomography of SXT INS-1E is X-ray. +- ``structure``: The labeled structure, mitochondrion or nucleus. It also tells which model the dataset trains. +- ``dimensionality``: 2d or 3d. +- ``role``: ``train`` (training data only), ``ID`` or ``OOD`` (scored test data), following the ``PANEL_DATASETS`` + of the Figure 6 plotting script in ``scripts/plotting/v2/for_paper``. An ID row has training data if the test data + comes from the same source as a training dataset. The ID rows without training data (parlakgul_liver_eval, + dauer_larva_nucleus) are other samples of a dataset whose other samples train the model. +- ``train_images_or_volumes``, ``train_objects``: The images or volumes and the labeled objects of the training + data (train + val split), counted with ``micro_sam.v2.datasets.data_counter``. So they follow the splits and the + label pre-processing of the loader. See its docstring for how stacks and regions of interest count. +- ``test_images_or_volumes``, ``test_objects``: The scored test crops and their objects, counted as for + ``create_data_overview.py``. A dataset that Figure 6 pools from several evaluation datasets (``POOLED``, e.g. the + MitoEM 2.0 volumes) sums the counts of its parts. +- ``comment``: Notes, e.g. on the subsets or the annotation of a dataset. + +The table (``data-overview/data_overview_em_organelles.csv`` by default) is also the cache, as for +``create_data_overview.py``: the counts are only computed for rows whose count cells are empty, or for all requested +rows with ``--force``. Building the training datasets of a domain takes several minutes. + +Usage: + + python create_data_overview_em_organelles.py # fill in the missing counts + python create_data_overview_em_organelles.py --datasets lucchi # only process this dataset + python create_data_overview_em_organelles.py --datasets lucchi --force # recompute its counts +""" + +import os +import argparse +from typing import Callable + +from create_data_overview import ( + DEFAULT_DATA_ROOT, REPO_ROOT, SCRIPT_DIR, Entry, Train, _path_has, check_registry, read_figure_literal, + read_panels, run, +) + +DEFAULT_OUTPUT = os.path.join(SCRIPT_DIR, "data-overview", "data_overview_em_organelles.csv") +FIGURE_SCRIPTS = {3: os.path.join(REPO_ROOT, "scripts", "plotting", "v2", "for_paper", "plots_for_figure6.py")} + +EM, XRAY = "EM", "X-ray" +MITO, NUCLEUS = "mitochondrion", "nucleus" + + +def _ndim_is(ndim: int) -> Callable: + """Select the label sources of this dimensionality.""" + return lambda unit: unit.ndim == ndim + + +def _mito(folder, select=None): + return Train("em_mito", folder, select) + + +def _nucleus(folder, select=None): + return Train("em_nucleus", folder, select) + + +ENTRIES = [ + # Mitochondria, 2d. + Entry("mitolab", EM, MITO, 2, _mito("mitolab"), + comment="CEM-MitoLab: tiles from about 600 source volumes and images. The groups whose names match a " + "mitochondria evaluation dataset are left out."), + Entry("probtem", EM, MITO, 2, _mito("probtem_full")), + Entry("u2os_fibsem_mito", EM, MITO, 2, _mito("u2os_fibsem"), comment="Single FIB-SEM slices of a U2-OS cell."), + Entry("vnc", EM, MITO, 2, _mito("vnc"), comment="Single slices of the 20-slice ssTEM train stack."), + # Mitochondria, 3d. + Entry("aive_mito", EM, MITO, 3, _mito("aive")), + Entry("asem_mito", EM, MITO, 3, _mito("asem"), test="asem_mito", comment="Boxes of the volumes 46, 58 and 2E."), + Entry("brain_pfibsem_mito", EM, MITO, 3, _mito("brain_pfibsem"), + comment="The noisy stack is registered to the reference stack and shares its labels."), + Entry("cellmap_mito", EM, MITO, 3, _mito("cellmap_mito_crops"), + comment="Crops of the CellMap Segmentation Challenge."), + Entry("densecell_mito", EM, MITO, 3, _mito("densecell"), comment="The mitochondria class of the semantic labels."), + Entry("hela_stard3_mito", EM, MITO, 3, _mito("hela_stard3")), + Entry("human_organoids_mito", EM, MITO, 3, _mito("human_organoids_mito"), + comment="The instances are precomputed with a neck watershed (em_organelles/preprocess_mito_dataset.py)."), + Entry("lucchi", EM, MITO, 3, _mito("lucchi"), test="lucchi", + comment="Lucchi++, the re-annotated Lucchi train volume."), + Entry("microns_pinky_mito", EM, MITO, 3, _mito("microns"), + comment="The densely annotated stitched volume vol19-vol34 of MICrONS pinky."), + Entry("mito_segem", EM, MITO, 3, _mito("mito_segem"), + comment="The testis and brainstem boxes. The intestine box is evaluation data."), + Entry("mitoem", EM, MITO, 3, _mito("mitoem"), test="mitoem", comment="The train volumes of MitoEM-H and MitoEM-R."), + Entry("mitoemv2", EM, MITO, 3, _mito("mitoemv2"), test="mitoemv2", + comment="MitoEM 2.0: beta, jurkat, macro, podo, sperm and stem. pyra repeats MitoEM-H and is left out. " + "The test data also has the mossy fiber boutons (mitoemv2_mossy), which Figure 6 pools into MitoEM 2.0."), + Entry("mitoemv2_mossy", EM, MITO, 3, _mito("mitoverse_derived", _path_has("/han24_")), + comment="The mossy fiber boutons of MitoEM 2.0 (MitoVerse han24). Only the boutons are annotated."), + Entry("mitoverse_haberl18", EM, MITO, 3, _mito("mitoverse", _path_has("/haberl18/"))), + Entry("mitoverse_microns1", EM, MITO, 3, _mito("mitoverse_derived", _path_has("/turner22_")), + comment="MICrONS phase 1."), + Entry("mitoverse_xiao18", EM, MITO, 3, _mito("mitoverse", _path_has("/xiao18/"))), + Entry("parlakgul_fasting_liver_mito", EM, MITO, 3, _mito("parlakgul_fasting_liver"), + comment="Other samples than the Parlakgul liver samples of the evaluation."), + Entry("snemi_mito", EM, MITO, 3, _mito("snemi_mito"), test="snemi_mito"), + Entry("uro_cell", EM, MITO, 3, _mito("uro_cell"), comment="The 5 of 8 UroCell volumes with mitochondria labels."), + # Nuclei, 2d. + Entry("cardiomyocyte_tem_nucleus", EM, NUCLEUS, 2, _nucleus("cardiomyocyte_tem_nucleus")), + Entry("perez_nucleus", EM, NUCLEUS, 2, _nucleus("perez_nucleus"), + comment="The test tiles of the dataset are the validation split of the training."), + Entry("tumor_spheroid_nuclei", EM, NUCLEUS, 2, _nucleus("tumor_spheroid_nucleus"), + comment="Single SBF-SEM slices."), + Entry("vem_nuclei_2d", EM, NUCLEUS, 2, _nucleus("vem_nuclei_nucleus", _ndim_is(2)), + comment="vEM nuclei, array tomography SEM. Proofread NucleoNet predictions."), + Entry("waenet_sstem_nucleus", EM, NUCLEUS, 2, _nucleus("waenet_nucleus", _path_has("/dataset_3_")), + comment="WAE-Net dataset 3: serial ssTEM sections of one cell."), + Entry("waenet_tem_nucleus", EM, NUCLEUS, 2, _nucleus("waenet_nucleus", _path_has("/tem_")), + comment="WAE-Net datasets 4-7: TEM images of lymphocytes and B cells. Only the central cell is labeled."), + # Nuclei, 3d. + Entry("betaseg_nucleus", EM, NUCLEUS, 3, _nucleus("betaseg_nucleus_boxes"), + comment="Only the nucleus of one target cell per volume is labeled."), + Entry("cellmap_nucleus", EM, NUCLEUS, 3, _nucleus("cellmap_nucleus_crops"), test="cellmap_nucleus", + comment="Crops of the CellMap Segmentation Challenge. The test data is the held-out crop357."), + Entry("cerebellum_nucleus", EM, NUCLEUS, 3, _nucleus("cerebellum_fibsem_nucleus")), + Entry("fafb_nuclei", EM, NUCLEUS, 3, _nucleus("fafb_nuclei"), test="fafb_nuclei", + comment="The labels are the automatic FlyWire segmentation."), + Entry("human_organoids_nucleus", EM, NUCLEUS, 3, _nucleus("human_organoids_nucleus"), + test="human_organoids_nucleus", + comment="The test data is the validation part of the volumes (the last 10% of the slices), which is also in " + "the train + val counts."), + Entry("janelia_nucleus", EM, NUCLEUS, 3, _nucleus("janelia_nucleus"), + comment="Cellpose pseudo-labels, except for the 3 manually labeled jrc_mus-nacc volumes."), + Entry("macrophage_a431_nucleus", EM, NUCLEUS, 3, _nucleus("macrophage_a431_nucleus"), + comment="Quick semantic annotations."), + Entry("microns_nuclei", EM, NUCLEUS, 3, _nucleus("microns_nuclei"), test="microns_nuclei", + comment="The labels are the automatic MICrONS minnie65 segmentation."), + Entry("microsam_em_nuclei", EM, NUCLEUS, 3, _nucleus("microsam_em_nuclei"), + comment="The train crop of the microSAM user study."), + Entry("openorganelle_nucleus", EM, NUCLEUS, 3, _nucleus("openorganelle_nucleus"), comment="Automatic labels."), + Entry("platynereis_nuclei", EM, NUCLEUS, 3, _nucleus("platynereis"), test="platynereis_nuclei"), + Entry("sxt_nucleus", XRAY, NUCLEUS, 3, _nucleus("sxt_ins1e_mito"), + comment="SXT INS-1E: soft X-ray tomography (EM-like), not EM. Semi-automatic labels."), + Entry("vem_nuclei_3d", EM, NUCLEUS, 3, _nucleus("vem_nuclei_nucleus", _ndim_is(3)), + comment="vEM nuclei, FIB-SEM. Proofread NucleoNet predictions."), + Entry("waenet_fibsem_nucleus", EM, NUCLEUS, 3, _nucleus("waenet_nucleus", _path_has("/dataset_1.h5")), + comment="WAE-Net dataset 1: FIB-SEM of pancreatic carcinoid cells."), + # Mitochondria, test data only. + Entry("parlakgul_liver_eval", EM, MITO, 3, test="parlakgul_liver_eval", + comment="The manual Parlakgul liver evaluation crops. Other Parlakgul liver samples train the model."), + Entry("cerebellum_fibsem_mito", EM, MITO, 3, test="cerebellum_fibsem_mito"), + Entry("cmito", EM, MITO, 3, test="cmito", comment="The CMito volumes L1, L2, L3, adult and dauer, pooled."), + Entry("human_liver_em", EM, MITO, 3, test="human_liver_em"), + Entry("kasthuri", EM, MITO, 3, test="kasthuri"), + Entry("macrophage_a431_mito", EM, MITO, 3, test="macrophage_a431_mito"), + Entry("mitolab_3d", EM, MITO, 3, test="mitolab", + comment="The CEM-MitoLab volumes c_elegans, fly_brain, glycolytic_muscle, hela_cell and salivary_gland, " + "pooled. They are left out of the training data (see mitolab)."), + # Nuclei, test data only. + Entry("dauer_larva_nucleus", EM, NUCLEUS, 3, test="dauer_larva_nucleus", + comment="Boxes of the OpenOrganelle jrc_dauer-larva volume, with automatic COSEM nucleus instances. Other " + "OpenOrganelle volumes train the model."), + Entry("aive_nucleus", EM, NUCLEUS, 3, test="aive_nucleus"), + Entry("etchacell_nucleus", EM, NUCLEUS, 3, test="etchacell_nucleus"), + Entry("human_liver_em_nucleus", EM, NUCLEUS, 3, test="human_liver_em_nucleus"), + Entry("malecns_nuclei", EM, NUCLEUS, 3, test="malecns_nuclei"), + Entry("placenta_nucleus", EM, NUCLEUS, 3, test="placenta_nucleus"), + Entry("zheng_ca3_nuclei", EM, NUCLEUS, 3, test="zheng_ca3_nuclei"), +] + + +def main(): + parser = argparse.ArgumentParser( + description="Create the overview table of the micro-sam2 EM organelle training and evaluation data.", + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, + ) + parser.add_argument("-o", "--output", default=DEFAULT_OUTPUT, help="The CSV table, also used as the cache.") + parser.add_argument("--data_root", default=DEFAULT_DATA_ROOT, help="The root folder of the data.") + parser.add_argument("-d", "--datasets", nargs="+", help="Only process these datasets. All by default.") + parser.add_argument("-f", "--force", action="store_true", help="Recompute the counts that the table has.") + parser.add_argument( + "-n", "--n_workers", type=int, default=min(8, len(os.sched_getaffinity(0))), + help="The number of processes that count in parallel. Use at most 8 on a 64 GB allocation.", + ) + args = parser.parse_args() + + roles = check_registry(ENTRIES, read_panels(FIGURE_SCRIPTS)) + pooled = read_figure_literal(FIGURE_SCRIPTS[3], "POOLED") + run( + args.output, args.data_root, args.datasets, args.force, args.n_workers, registry=ENTRIES, roles=roles, + pooled=pooled, + ) + + +if __name__ == "__main__": + main() diff --git a/finetuning/v2/generalist/em_organelles/preprocess_mito_dataset.py b/finetuning/v2/generalist/em_organelles/preprocess_mito_dataset.py index 019eda2c7..c87acd7cc 100644 --- a/finetuning/v2/generalist/em_organelles/preprocess_mito_dataset.py +++ b/finetuning/v2/generalist/em_organelles/preprocess_mito_dataset.py @@ -8,8 +8,10 @@ -> /macrophage_a431/fib_control_z-.h5. - cellmap: the CellMap Segmentation Challenge crops with a mito label array -> /cellmap_mito_crops/_crop.zarr and index.json. +- eval_crops_2d: the 512 x 512 test crops of the 2d EM mitochondria sets -> finetuning/v2/evaluation/eval_crops.json. """ import os +import sys import json import time import shutil @@ -35,6 +37,13 @@ DATA_ROOT = "/mnt/vast-nhr/projects/cidas/cca/data" +EVALUATION_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "evaluation")) +EVAL_CROPS_PATH = os.path.join(EVALUATION_ROOT, "eval_crops.json") +# The 2d EM mitochondria test sets that get 512 x 512 evaluation crops, and the rules of their placement. +EVAL_DATASETS_2D = ["mitolab_tem", "probtem", "tnbc_mito", "perez_mito", "cardiomyocyte_tem_mito"] +EVAL_TILE_2D = 512 +EVAL_MIN_RELATIVE_COUNT_2D = 0.2 +EVAL_MAX_TILES_PER_IMAGE_2D = 16 NECK_RATIO = 0.5 MIN_PIECE = 5000 SEED_HEIGHT = 1.0 @@ -457,8 +466,111 @@ def preprocess_cellmap(input_root, output_root): json.dump(index, f, indent=1) +def import_evaluation_common(): + """Import common.py of the evaluation folder. Only the evaluation crops need it, and it loads torch.""" + sys.path.insert(0, EVALUATION_ROOT) + import common + return common + + +def dump_crops(obj, level=0): + """JSON in the layout of eval_crops.json: one space per level, number lists on one line.""" + pad, inner = " " * level, " " * (level + 1) + if isinstance(obj, dict): + items = [f"{inner}{json.dumps(k)}: {dump_crops(v, level + 1)}" for k, v in obj.items()] + return "{\n" + ",\n".join(items) + "\n" + pad + "}" + if isinstance(obj, list): + if all(not isinstance(x, (list, dict)) for x in obj): + return json.dumps(obj) + return "[\n" + ",\n".join(inner + dump_crops(x, level + 1) for x in obj) + "\n" + pad + "]" + return json.dumps(obj) + + +def update_eval_crops(placed, crops_path, dry): + """Add the placed crops to eval_crops.json; it is read after the placement, so that edits meanwhile are kept.""" + if dry: + return + with open(crops_path) as f: + crops = json.load(f) + crops["datasets"].update(placed) + tmp_path = f"{crops_path}.tmp{os.getpid()}" + with open(tmp_path, "w") as f: + f.write(dump_crops(crops) + "\n") + os.replace(tmp_path, crops_path) + print(f"Saved the crops of {len(placed)} datasets to {crops_path}.") + + +def cover_starts(n): + """The tile starts that cover an axis of length n with the fewest tiles, spread evenly, and the tile size.""" + if n <= EVAL_TILE_2D: + return [0], n + k = int(np.ceil(n / EVAL_TILE_2D)) + return [int(round(i * (n - EVAL_TILE_2D) / (k - 1))) for i in range(k)], EVAL_TILE_2D + + +def count_objects_2d(common, raw, labels, start, shape, min_size): + """The number of objects in a tile, or None for a tile with a constant raw image.""" + window = tuple(slice(s, s + t) for s, t in zip(start, shape)) + patch = raw[window] + if patch.min() == patch.max(): + return None + gt = common.drop_severed_objects(label(labels[window]).astype("uint32"), min_size) + return int(len(np.unique(gt)) - 1) + + +def place_crops_2d(common, dataset_name): + """The eval_crops.json entry of a 2d dataset, with the crop shape of each image in 'crop_shapes'.""" + raw_paths, label_paths, raw_key, label_key = common.get_data_paths(dataset_name, common.DATA_ROOT, split="test")[:4] + min_size = common.GT_MIN_SIZE_2D.get(dataset_name, 0) + counts, shapes = {}, {} + for raw_path, label_path in zip(raw_paths, label_paths): + raw, labels = common.read_2d(raw_path, raw_key), common.read_2d(label_path, label_key) + if raw.ndim == 3: + raw = raw.mean(axis=-1) + (ys, height), (xs, width) = cover_starts(raw.shape[0]), cover_starts(raw.shape[1]) + name = os.path.basename(raw_path) + shapes[name] = [height, width] + counts[name] = { + (y, x): count_objects_2d(common, raw, labels, (y, x), (height, width), min_size) for y in ys for x in xs + } + valid = [n for tiles in counts.values() for n in tiles.values() if n] + threshold = EVAL_MIN_RELATIVE_COUNT_2D * float(np.median(valid)) + starts, crop_shapes, kept, n_tiles = {}, {}, [], 0 + for name, tiles in counts.items(): + n_tiles += len(tiles) + passing = sorted(((n, s) for s, n in tiles.items() if n and n >= threshold), key=lambda t: -t[0]) + passing = passing[:EVAL_MAX_TILES_PER_IMAGE_2D] + if not passing: + continue + starts[name] = [list(start) for _, start in sorted(passing, key=lambda t: t[1])] + if shapes[name] != [EVAL_TILE_2D, EVAL_TILE_2D]: + crop_shapes[name] = [shapes[name]] * len(passing) + kept.extend(n for n, _ in passing) + note = ( + "512 x 512 tiles that cover each test image, spread evenly with a small overlap; an image axis below 512 px " + f"is one tile. {len(kept)} of {n_tiles} tiles are kept (at most {EVAL_MAX_TILES_PER_IMAGE_2D} per image, the " + f"ones with the most objects), holding {min(kept)}-{max(kept)} mitochondria." + ) + print(f"{dataset_name}: {note}") + entry = {"crop_shape": [EVAL_TILE_2D, EVAL_TILE_2D], "note": note, "starts": starts} + if crop_shapes: + entry["crop_shapes"] = crop_shapes + return entry + + +def preprocess_eval_crops_2d(datasets, crops_path, dry): + """Place the 512 x 512 test crops of the 2d EM mitochondria datasets and add them to eval_crops.json. + + The crops cover each test image with the fewest tiles, spread evenly, so neighbors overlap by a few pixels. Tiles + with a constant raw image or with fewer than 20% of the median object count of the dataset are dropped; of the + rest, at most 16 per image stay, the ones with the most objects. + """ + common = import_evaluation_common() + update_eval_crops({name: place_crops_2d(common, name) for name in datasets}, crops_path, dry) + + def main(): - choices = ["human_organoids", "parlakgul_fasting", "macrophage_fib_control", "cellmap"] + choices = ["human_organoids", "parlakgul_fasting", "macrophage_fib_control", "cellmap", "eval_crops_2d"] parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--dataset", required=True, choices=choices, help="The dataset to preprocess.") parser.add_argument("--input_root", default=DATA_ROOT, help="The data root to read the source data from.") @@ -470,6 +582,9 @@ def main(): parser.add_argument("--start", type=int, default=848, help="The first macrophage fib_control slice (zero-based).") parser.add_argument("--stop", type=int, default=976, help="The macrophage fib_control slice after the last one.") parser.add_argument("--n_threads", type=int, default=8, help="Parallel macrophage fib_control downloads.") + parser.add_argument("--eval_datasets", nargs="+", default=EVAL_DATASETS_2D, help="The 2d sets for eval_crops_2d.") + parser.add_argument("--crops_path", default=EVAL_CROPS_PATH, help="The eval_crops.json file to update.") + parser.add_argument("--dry", action="store_true", help="Print the evaluation crops without writing them.") args = parser.parse_args() if args.dataset == "human_organoids": volumes = args.volumes or [name.replace("_bin2.h5", "") for name in HUMAN_ORGANOIDS_VOLUMES] @@ -478,8 +593,10 @@ def main(): download_parlakgul_fasting(args.output_root, args.index) elif args.dataset == "macrophage_fib_control": download_macrophage_fib_control(args.output_root, args.start, args.stop, args.n_threads) - else: + elif args.dataset == "cellmap": preprocess_cellmap(args.input_root, args.output_root) + else: + preprocess_eval_crops_2d(args.eval_datasets, args.crops_path, args.dry) if __name__ == "__main__": diff --git a/finetuning/v2/generalist/em_organelles/preprocess_nucleus_dataset.py b/finetuning/v2/generalist/em_organelles/preprocess_nucleus_dataset.py index 8316becac..ff583fe02 100644 --- a/finetuning/v2/generalist/em_organelles/preprocess_nucleus_dataset.py +++ b/finetuning/v2/generalist/em_organelles/preprocess_nucleus_dataset.py @@ -8,12 +8,13 @@ - cardiomyocyte, perez, vem, u2os, etchacell, aive, cerebellum, macrophage: the nucleus data of these torch-em datasets -> // (see BUILDERS) with build_info.json. - human_liver_eval: the human liver EM nucleus evaluation boxes -> /human_liver_em_nucleus_boxes/. +- dauer_larva_test, human_organoids_test: the in-domain test sets of the EM nucleus panel -> + /dauer_larva_nucleus/ and /human_organoids_nucleus_val/, with test/ and val/ (then run eval_crops). - eval_crops: the fixed evaluation crops of the EM nucleus panel -> finetuning/v2/evaluation/eval_crops.json. """ import io import os import re -import sys import glob import json import time @@ -42,7 +43,10 @@ from torch_em.data.datasets.electron_microscopy.human_liver_em import _HttpFile -from preprocess_mito_dataset import CELLMAP_ZIP, DATA_ROOT, get_cellmap_em_levels, load_cellmap_crop +from preprocess_mito_dataset import ( + CELLMAP_ZIP, DATA_ROOT, EVAL_CROPS_PATH, get_cellmap_em_levels, import_evaluation_common, + load_cellmap_crop, update_eval_crops, +) IGNORE_LABEL = np.iinfo("uint32").max @@ -120,8 +124,6 @@ "tune": {"bbox_xxyyzz": [7168, 11264, 1024, 5120, 160, 176], "role": "tuning"}, } -EVALUATION_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "evaluation")) -EVAL_CROPS_PATH = os.path.join(EVALUATION_ROOT, "eval_crops.json") EVAL_OBJECT_NAMES = { "parlakgul_liver": "mitochondria", "cerebellum_fibsem_mito": "mitochondria", "macrophage_a431_mito": "mitochondria", "mito_anf": "mitochondria", "snemi_mito": "mitochondria", @@ -138,6 +140,30 @@ EVAL_MAX_TILES_PER_VOLUME = 16 EVAL_MIN_RELATIVE_COUNT = 0.2 EVAL_NOTES = { + "malecns_nuclei": ( + "Nucleus panel (out-of-domain): Drosophila male CNS FIB-SEM crops at 16 nm; automatic nucleus " + "segmentation, not filtered for glia or noise." + ), + "zheng_ca3_nuclei": ( + "Nucleus panel (out-of-domain): mouse CA3 ssTEM crops at 72 x 72 x 45 nm; automatic nucleus " + "segmentation." + ), + "acson_nuclei": ( + "Nucleus panel candidate (out-of-domain): rat white matter SBEM crops at 50 nm (DeepACSON); automatic nucleus " + "segmentation." + ), + "placenta_nucleus": ( + "Nucleus panel (out-of-domain): the two human placenta array tomography volumes (S-BIAD3408, 80 nm " + "sections); manual nuclei of one CTB cell and an STB region, scored only where labelled." + ), + "dauer_larva_nucleus": ( + "Nucleus panel (in-domain): boxes of the OpenOrganelle jrc_dauer-larva volume (ssTEM, 70 x 40 x 40 " + "nm), which the nucleus model never saw; automatic COSEM nucleus instances." + ), + "human_organoids_nucleus": ( + "Nucleus panel (in-domain): the validation part of the four human organoid volumes (the last 10% " + "of the slices, manual instances), which selected the training checkpoint." + ), "cellmap_nucleus": "Nucleus panel: the held-out jrc_mus-liver-zon-2 crop357 (32 nm, manual instances).", "fafb_nuclei": "Nucleus panel: the four official FAFB test blocks (automatic FlyWire nucleus segmentation).", "microns_nuclei": "Nucleus panel: the four official MICrONS test blocks (automatic minnie65 nucleus segmentation).", @@ -1234,24 +1260,91 @@ def preprocess_human_liver(input_root, output_root, steps, names, map_step, dept json.dump(infos, f, indent=2) -def import_evaluation_common(): - """Import common.py of the evaluation folder. Only the eval_crops choice needs it, and it loads torch.""" - sys.path.insert(0, EVALUATION_ROOT) - import common - return common - - -def dump_crops(obj, level=0): - """JSON in the layout of eval_crops.json: one space per level, number lists on one line.""" - pad, inner = " " * level, " " * (level + 1) - if isinstance(obj, dict): - items = [f"{inner}{json.dumps(k)}: {dump_crops(v, level + 1)}" for k, v in obj.items()] - return "{\n" + ",\n".join(items) + "\n" + pad + "}" - if isinstance(obj, list): - if all(not isinstance(x, (list, dict)) for x in obj): - return json.dumps(obj) - return "[\n" + ",\n".join(inner + dump_crops(x, level + 1) for x in obj) + "\n" + pad + "]" - return json.dumps(obj) +# The dauer larva boxes on the label grid (z, y, x) and the number of scored boxes; one more box tunes. +DAUER_BOX_SHAPE = (64, 1024, 1024) +DAUER_TEST_BOXES = 3 +DAUER_NAME = "jrc_dauer-larva" +DAUER_LABELS = "labels/inference/nucleus_seg" +DAUER_RAW = "em/tem-uint8" +# The raw level closest to the 40 nm label grid, and the coarse label level that ranks the boxes. +DAUER_RAW_LEVEL = "s3" +DAUER_COARSE_LEVEL = "s4" +# The slices of each organoid volume before its validation part that tune. +ORGANOID_TUNING_SLICES = 32 + + +def write_test_h5(path, raw, labels): + os.makedirs(os.path.dirname(path), exist_ok=True) + write_h5(path, raw, labels) + print(f"Wrote {path}: {raw.shape}, {len(np.unique(labels)) - 1} objects.") + + +def rank_dauer_boxes(): + """Return the starts of the non-overlapping boxes on the label grid, the ones with the most nuclei first.""" + from torch_em.data.datasets.electron_microscopy import openorganelle_nucleus as oo + url = f"s3://janelia-cosem-datasets/{DAUER_NAME}/{DAUER_NAME}.zarr/recon-1/{DAUER_LABELS}" + coarse = np.asarray(oo._open_remote_zarr(f"{url}/{DAUER_COARSE_LEVEL}")[:]) + full_shape = oo._open_remote_zarr(f"{url}/s0").shape + factor = np.array(full_shape) / np.array(coarse.shape) + window = [max(1, int(round(s / f))) for s, f in zip(DAUER_BOX_SHAPE, factor)] + candidates = [] + for z in range(0, coarse.shape[0] - window[0] + 1, window[0]): + for y in range(0, coarse.shape[1] - window[1] + 1, window[1] // 2): + for x in range(0, coarse.shape[2] - window[2] + 1, window[2] // 2): + ids = np.unique(coarse[z:z + window[0], y:y + window[1], x:x + window[2]]) + candidates.append((len(ids) - (ids[0] == 0), (z, y, x))) + chosen = [] + for count, start in sorted(candidates, reverse=True): + box = [(s, s + w) for s, w in zip(start, window)] + if all(any(b[1] <= c[0] or c[1] <= b[0] for b, c in zip(box, other)) for other in chosen): + chosen.append(box) + print(f"Box at {start} (coarse grid) with {count} nuclei.") + if len(chosen) == DAUER_TEST_BOXES + 1: + break + return [tuple(int(round(lo * f)) for (lo, _), f in zip(box, factor)) for box in chosen] + + +def preprocess_dauer_larva_test(output_root): + """The dauer larva boxes -> /dauer_larva_nucleus/{test,val}/, the raw resampled to the label grid.""" + from torch_em.data.datasets.electron_microscopy import openorganelle_nucleus as oo + base = f"s3://janelia-cosem-datasets/{DAUER_NAME}/{DAUER_NAME}.zarr/recon-1" + labels = oo._open_remote_zarr(f"{base}/{DAUER_LABELS}/s0") + raw = oo._open_remote_zarr(f"{base}/{DAUER_RAW}/{DAUER_RAW_LEVEL}") + label_scale = oo._multiscale_levels(f"{oo.BUCKET_URL}{DAUER_NAME}/{DAUER_NAME}.zarr/recon-1/{DAUER_LABELS}")["s0"] + raw_levels = oo._multiscale_levels(f"{oo.BUCKET_URL}{DAUER_NAME}/{DAUER_NAME}.zarr/recon-1/{DAUER_RAW}") + ratio = np.array(label_scale) / np.array(raw_levels[DAUER_RAW_LEVEL]) + for k, start in enumerate(rank_dauer_boxes()): + box = tuple(slice(s, min(s + w, n)) for s, w, n in zip(start, DAUER_BOX_SHAPE, labels.shape)) + label_data = np.asarray(labels[box]).astype("uint32") + raw_box = tuple(slice(int(sl.start * r), int(np.ceil(sl.stop * r))) for sl, r in zip(box, ratio)) + raw_native = np.asarray(raw[raw_box]) + raw_data = ndimage.zoom(raw_native, [t / s for t, s in zip(label_data.shape, raw_native.shape)], order=1) + split = "test" if k < DAUER_TEST_BOXES else "val" + name = "_".join(str(sl.start) for sl in box) + path = os.path.join(output_root, "dauer_larva_nucleus", split, f"box_{name}.h5") + write_test_h5(path, raw_data.astype("uint8"), label_data) + + +def preprocess_human_organoids_test(input_root, output_root): + """The validation part of the human organoid volumes -> /human_organoids_nucleus_val/{test,val}/.""" + from micro_sam.v2.datasets.generalist_loader import ( + HUMAN_ORGANOIDS_VOLUMES, HUMAN_ORGANOIDS_VAL_FRACTION, HUMAN_ORGANOIDS_NUCLEUS_IGNORE_LABEL, + ) + for volume in HUMAN_ORGANOIDS_VOLUMES: + name = volume.replace("_bin2.h5", "") + with h5py.File(os.path.join(input_root, "human_organoids", "11380", "data", volume), "r") as f: + raw = f["raw"] + n_train = int(raw.shape[0] * (1 - HUMAN_ORGANOIDS_VAL_FRACTION)) + parts = {"test": np.s_[n_train:], "val": np.s_[n_train - ORGANOID_TUNING_SLICES:n_train]} + label_path = os.path.join(input_root, "human_organoids_nucleus", f"{name}_nucleus_labels.zarr") + labels = zarr.open(label_path, mode="r") + labels = labels["labels"] if not hasattr(labels, "shape") else labels + for split, part in parts.items(): + label_data = np.asarray(labels[part]).astype("uint32") + # the ignored fragments and the entotic cell are not scored + label_data[label_data == HUMAN_ORGANOIDS_NUCLEUS_IGNORE_LABEL] = 0 + path = os.path.join(output_root, "human_organoids_nucleus_val", split, f"{name}.h5") + write_test_h5(path, raw[part], label_data) def tile_starts(region, crop_shape, min_tile_shape): @@ -1325,21 +1418,13 @@ def preprocess_eval_crops(input_root, crops_path, datasets, dry): """ common = import_evaluation_common() datasets = datasets or common.DATASETS_3D_EM_NUCLEUS - placed = {name: place_crops(common, input_root, name) for name in datasets} - # The file is read after the crops are placed, so edits made in the meantime are kept. - with open(crops_path) as f: - crops = json.load(f) - crops["datasets"].update(placed) - if not dry: - tmp_path = f"{crops_path}.tmp{os.getpid()}" - with open(tmp_path, "w") as f: - f.write(dump_crops(crops) + "\n") - os.replace(tmp_path, crops_path) + update_eval_crops({name: place_crops(common, input_root, name) for name in datasets}, crops_path, dry) def main(): choices = [ - "betaseg", "human_organoids", "tumor_spheroid", "waenet", "cellmap", *BUILDERS, "human_liver_eval", "eval_crops" + "betaseg", "human_organoids", "tumor_spheroid", "waenet", "cellmap", *BUILDERS, "human_liver_eval", + "dauer_larva_test", "human_organoids_test", "eval_crops", ] parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--dataset", required=True, choices=choices, help="The dataset to preprocess.") @@ -1373,6 +1458,10 @@ def main(): preprocess_human_liver( args.input_root, args.output_root, args.steps, args.names, args.map_step, args.depth, args.n_best ) + elif args.dataset == "dauer_larva_test": + preprocess_dauer_larva_test(args.output_root) + elif args.dataset == "human_organoids_test": + preprocess_human_organoids_test(args.input_root, args.output_root) else: preprocess_eval_crops(args.input_root, args.crops_path, args.eval_datasets, args.dry) diff --git a/finetuning/v2/generalist/summarize_data_overview.py b/finetuning/v2/generalist/summarize_data_overview.py new file mode 100644 index 000000000..66c8ef88e --- /dev/null +++ b/finetuning/v2/generalist/summarize_data_overview.py @@ -0,0 +1,265 @@ +"""Summarize the data overview table that ``create_data_overview.py`` writes. + +The readout is chosen with the positional argument: + +- ``train`` (default): The number of annotated objects in the training data (train + val split) in five groups: + cells and nuclei in 2d and 3d light microscopy (LM), and all structures in electron microscopy (EM). + EM covers the EM datasets and the EM-like LICONN (expansion microscopy) and XPRESS (X-ray) datasets. Everything + else is LM, including histopathology and label-free data. LM rows labeled 'nucleus' count as nuclei, all other + LM structures (cells, organoids, worms, prospore membranes, axons) count as cells. +- ``test-2d``: The number of annotated objects in the scored 2d test data (see ``create_data_overview.py``) for + the four panels of Figure 2: label-free cells, fluorescence cells, fluorescence nuclei and histopathology nuclei. + For each the ID, the OOD and the summed (ID + OOD) objects are printed; the summed objects of all groups are all + annotations of the scored 2d test splits. All label-free datasets count as cells, including the organoids of + OrgLine and the C. elegans of BBBC010, which are in the label-free cells panel. +- ``test-3d``: The number of annotated objects in the scored 3d test data (test crops, see + ``create_data_overview.py``) for the three panels of Figure 3: LM cells, LM nuclei and EM neurites. For each the + ID, the OOD and the summed (ID + OOD) objects are printed. LICONN (expansion microscopy) counts as EM, as in the + EM neurites panel of Figure 3. +- ``detail``: A Markdown table with the structures and the images or volumes per modality, structure and + dimensionality, for the training data and the scored 2d and 3d test data (ID and OOD). The training rows use the + groups of ``train``, split by modality: label-free and histopathology datasets count as cells and nuclei, EM and + the EM-like LICONN and XPRESS count as 'Cell / Neurite'. Cellpose (fluorescence) and NeurIPS CellSeg + (label-free) mix modalities and count under their majority modality. The test rows are the groups of + ``test-2d`` and ``test-3d``; for 3d the images / volumes are the scored test crops. + +Usage: + + python summarize_data_overview.py # same as 'train' + python summarize_data_overview.py train + python summarize_data_overview.py test-2d + python summarize_data_overview.py test-3d + python summarize_data_overview.py detail +""" + +import csv +import argparse +from collections import defaultdict + +from create_data_overview import DEFAULT_OUTPUT + +# The datasets that count as EM although their modality is not EM. +EM_LIKE = {"liconn", "xpress"} +GROUPS = ["2d LM cells", "2d LM nuclei", "3d LM cells", "3d LM nuclei", "EM structures"] +# The groups of the 2d test data, the panels of Figure 2. +TEST_2D_GROUPS = ["label-free cells", "fluorescence cells", "fluorescence nuclei", "histopathology nuclei"] +# The groups of the 3d test data, the panels of Figure 3. +TEST_3D_GROUPS = ["LM cells", "LM nuclei", "EM neurites"] + + +def read_table(path): + """Read the rows of the data overview table.""" + with open(path, newline="") as f: + return list(csv.DictReader(f)) + + +def select_train_rows(rows): + """Return the rows with training counts. Raise an error if a dataset with training data has none.""" + missing = [ + row["dataset"] for row in rows if row["role"] in ("train", "ID") and row["train_objects"] == "" + and not row["dataset"].startswith("neurips_cellseg_") # their training data is the 'neurips_cellseg' row + ] + if missing: + raise RuntimeError( + f"The table has no training counts for {missing}. Run create_data_overview.py to fill them in." + ) + return [row for row in rows if row["role"] in ("train", "ID") and row["train_objects"] != ""] + + +def select_test_rows(rows, ndim): + """Return the rows of the given dimensionality ('2d' or '3d') with a scored test split. + + Raise an error if one has no test counts. + """ + test_rows = [row for row in rows if row["dimensionality"] == ndim and row["role"] in ("ID", "OOD")] + missing = [row["dataset"] for row in test_rows if row["test_objects"] == ""] + if missing: + raise RuntimeError(f"The table has no test counts for {missing}. Run create_data_overview.py to fill them in.") + return test_rows + + +def train_group(row): + """Return the group of a dataset with training data.""" + if row["modality"] == "EM" or row["dataset"] in EM_LIKE: + return "EM structures" + kind = "nuclei" if row["structure"] == "nucleus" else "cells" + return f"{row['dimensionality']} LM {kind}" + + +def print_train(rows): + """Print the annotated objects of the training data per group.""" + train_rows = select_train_rows(rows) + totals = defaultdict(lambda: {"objects": 0, "images": 0, "datasets": 0, "structures": set()}) + for row in train_rows: + total = totals[train_group(row)] + total["objects"] += int(row["train_objects"]) + total["images"] += int(row["train_images_or_volumes"]) + total["datasets"] += 1 + total["structures"].add(row["structure"]) + + print("Annotated objects in the training data (train + val split):") + print(f"{'group':<15} {'objects':>12} {'images / volumes':>17} {'datasets':>9} structures") + for group in GROUPS: + total = totals[group] + print( + f"{group:<15} {total['objects']:>12,} {total['images']:>17,} {total['datasets']:>9} " + f"{', '.join(sorted(total['structures']))}" + ) + print( + f"{'total':<15} {sum(t['objects'] for t in totals.values()):>12,} " + f"{sum(t['images'] for t in totals.values()):>17,} {sum(t['datasets'] for t in totals.values()):>9}" + ) + + +def test_2d_group(row): + """Return the group (the Figure 2 panel) of a 2d dataset with a scored test split.""" + if row["modality"] == "label-free": + return "label-free cells" + if row["modality"] == "histopathology": + return "histopathology nuclei" + if row["modality"] == "fluorescence": + return "fluorescence nuclei" if row["structure"] == "nucleus" else "fluorescence cells" + raise ValueError(f"{row['dataset']} has the modality {row['modality']}, which is in no 2d test group.") + + +def test_3d_group(row): + """Return the group (the Figure 3 panel) of a 3d dataset with a scored test split.""" + if row["modality"] == "EM" or row["dataset"] in EM_LIKE: + return "EM neurites" + if row["modality"] == "fluorescence": + return "LM nuclei" if row["structure"] == "nucleus" else "LM cells" + raise ValueError(f"{row['dataset']} has the modality {row['modality']}, which is in no 3d test group.") + + +def print_test(rows, ndim, group_fn, groups): + """Print the annotated objects of the scored test splits per group, for ID, OOD and both.""" + test_rows = select_test_rows(rows, ndim) + totals = defaultdict(lambda: {"ID": 0, "OOD": 0, "images": 0, "datasets": 0}) + for row in test_rows: + total = totals[group_fn(row)] + total[row["role"]] += int(row["test_objects"]) + total["images"] += int(row["test_images_or_volumes"]) + total["datasets"] += 1 + + # The groups must cover every scored object. + expected = sum(int(row["test_objects"]) for row in test_rows) + grand_total = sum(t["ID"] + t["OOD"] for t in totals.values()) + assert grand_total == expected, (grand_total, expected) + + images = "images" if ndim == "2d" else "crops" + print(f"Annotated objects in the scored {ndim} test data:") + print(f"{'group':<22} {'ID':>10} {'OOD':>10} {'ID + OOD':>10} {images:>8} {'datasets':>9}") + for group in groups: + total = totals[group] + print( + f"{group:<22} {total['ID']:>10,} {total['OOD']:>10,} {total['ID'] + total['OOD']:>10,} " + f"{total['images']:>8,} {total['datasets']:>9}" + ) + print( + f"{'total':<22} {sum(t['ID'] for t in totals.values()):>10,} {sum(t['OOD'] for t in totals.values()):>10,} " + f"{grand_total:>10,} {sum(t['images'] for t in totals.values()):>8,} " + f"{sum(t['datasets'] for t in totals.values()):>9}" + ) + + +# The rows of the detail table: (modality, structure, dimensionality). +DETAIL_TRAIN_ROWS = [ + ("Label-free", "Cell", "2"), + ("Fluorescence", "Cell", "2"), + ("Fluorescence", "Nucleus", "2"), + ("Histopathology", "Nucleus", "2"), + ("Electron Microscopy", "Cell / Neurite", "2"), + ("Fluorescence", "Cell", "3"), + ("Fluorescence", "Nucleus", "3"), + ("Electron Microscopy", "Cell / Neurite", "3"), +] +DETAIL_TEST_2D_ROWS = { + "label-free cells": ("Label-free", "Cell", "2"), + "fluorescence cells": ("Fluorescence", "Cell", "2"), + "fluorescence nuclei": ("Fluorescence", "Nucleus", "2"), + "histopathology nuclei": ("Histopathology", "Nucleus", "2"), +} +DETAIL_TEST_3D_ROWS = { + "LM cells": ("Fluorescence", "Cell", "3"), + "LM nuclei": ("Fluorescence", "Nucleus", "3"), + "EM neurites": ("Electron Microscopy", "Cell / Neurite", "3"), +} + + +def detail_train_row(row): + """Return the detail table row (modality, structure, dimensionality) of a dataset with training data.""" + ndim = row["dimensionality"].rstrip("d") + if row["modality"] == "EM" or row["dataset"] in EM_LIKE: + return ("Electron Microscopy", "Cell / Neurite", ndim) + if row["modality"] == "label-free": + return ("Label-free", "Cell", ndim) + if row["modality"] == "histopathology": + return ("Histopathology", "Nucleus", ndim) + if row["modality"] == "fluorescence": + return ("Fluorescence", "Nucleus" if row["structure"] == "nucleus" else "Cell", ndim) + raise ValueError(f"{row['dataset']} has the modality {row['modality']}, which is in no detail row.") + + +def print_detail(rows): + """Print the detail table of the training data and the scored 2d and 3d test data as Markdown.""" + train_rows = select_train_rows(rows) + train = defaultdict(lambda: {"objects": 0, "images": 0}) + for row in train_rows: + key = detail_train_row(row) + if key not in DETAIL_TRAIN_ROWS: + raise ValueError(f"{row['dataset']} falls into {key}, which is not a row of the detail table.") + train[key]["objects"] += int(row["train_objects"]) + train[key]["images"] += int(row["train_images_or_volumes"]) + assert sum(t["objects"] for t in train.values()) == sum(int(row["train_objects"]) for row in train_rows) + + test = defaultdict(lambda: {"ID": 0, "OOD": 0, "ID images": 0, "OOD images": 0}) + for ndim, group_fn, detail_rows in [ + ("2d", test_2d_group, DETAIL_TEST_2D_ROWS), ("3d", test_3d_group, DETAIL_TEST_3D_ROWS) + ]: + test_rows = select_test_rows(rows, ndim) + for row in test_rows: + key = detail_rows[group_fn(row)] + test[key][row["role"]] += int(row["test_objects"]) + test[key][f"{row['role']} images"] += int(row["test_images_or_volumes"]) + assert sum(test[key]["ID"] + test[key]["OOD"] for key in detail_rows.values()) == sum( + int(row["test_objects"]) for row in test_rows + ) + + print("| Modality | Structure | Dimensionality | Split | # Structures | # Images / Volumes |") + print("|---|---|---|---|---|---|") + for key in DETAIL_TRAIN_ROWS: + print(f"| {' | '.join(key)} | Train | {train[key]['objects']:,} | {train[key]['images']:,} |") + for key in [*DETAIL_TEST_2D_ROWS.values(), *DETAIL_TEST_3D_ROWS.values()]: + counts = test[key] + print( + f"| {' | '.join(key)} | Test | {counts['ID']:,} (ID); {counts['OOD']:,} (OOD) | " + f"{counts['ID images']:,} (ID); {counts['OOD images']:,} (OOD) |" + ) + + +def print_test_2d(rows): + """Print the annotated objects of the scored 2d test splits per Figure 2 panel, for ID, OOD and both.""" + print_test(rows, "2d", test_2d_group, TEST_2D_GROUPS) + + +def print_test_3d(rows): + """Print the annotated objects of the scored 3d test crops per Figure 3 panel, for ID, OOD and both.""" + print_test(rows, "3d", test_3d_group, TEST_3D_GROUPS) + + +READOUTS = {"train": print_train, "test-2d": print_test_2d, "test-3d": print_test_3d, "detail": print_detail} + + +def main(): + parser = argparse.ArgumentParser( + description="Summarize the data overview table.", formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument("readout", nargs="?", default="train", choices=list(READOUTS), help="The readout to print.") + parser.add_argument("-i", "--input", default=DEFAULT_OUTPUT, help="The data overview table.") + args = parser.parse_args() + READOUTS[args.readout](read_table(args.input)) + + +if __name__ == "__main__": + main() diff --git a/finetuning/v2/generalist/summarize_data_overview_em_organelles.py b/finetuning/v2/generalist/summarize_data_overview_em_organelles.py new file mode 100644 index 000000000..c835bfda9 --- /dev/null +++ b/finetuning/v2/generalist/summarize_data_overview_em_organelles.py @@ -0,0 +1,178 @@ +"""Summarize the EM organelle data overview table that ``create_data_overview_em_organelles.py`` writes. + +The readout is chosen with the positional argument, as for ``summarize_data_overview.py``: + +- ``train`` (default): The number of annotated objects in the training data (train + val split) of the two EM + organelle models: mitochondria and nuclei. 2d and 3d data are not distinguished, so the images / volumes column + sums 2d images and 3d volumes. The structure column of the table decides the group, so SXT INS-1E (soft X-ray + tomography, EM-like) counts under nuclei. +- ``test``: The number of annotated objects in the scored test data (test crops, see + ``create_data_overview_em_organelles.py``) of the two panels of Figure 6: mitochondria and nuclei. For each the ID, + the OOD and the summed (ID + OOD) objects are printed. A dataset that Figure 6 pools from several volumes counts + as one dataset. +- ``detail``: A Markdown table with the structures and the images or volumes per structure and dimensionality, for + the training data (2d and 3d) and the scored test data (ID and OOD, all 3d), in the format of the ``detail`` + readout of ``summarize_data_overview.py``. SXT INS-1E counts as EM, as in ``train``. For the test data the + images / volumes are the scored test crops. + +Usage: + + python summarize_data_overview_em_organelles.py # same as 'train' + python summarize_data_overview_em_organelles.py train + python summarize_data_overview_em_organelles.py test + python summarize_data_overview_em_organelles.py detail +""" + +import argparse +from collections import defaultdict + +from create_data_overview_em_organelles import DEFAULT_OUTPUT, ENTRIES, MITO, NUCLEUS +from summarize_data_overview import read_table, select_test_rows + +# The groups of the training data: the structure of the table and the name it is printed with. +GROUPS = {MITO: "mitochondria", NUCLEUS: "nuclei"} + + +def select_train_rows(rows): + """Return the rows with training data. Raise an error if one has no training counts. + + Unlike in the generalist table, some ID rows have test data only, so the registry decides which rows have + training data. + """ + with_train = {entry.name for entry in ENTRIES if entry.train is not None} + train_rows = [row for row in rows if row["dataset"] in with_train] + missing = sorted(with_train - {row["dataset"] for row in train_rows if row["train_objects"] != ""}) + if missing: + raise RuntimeError( + f"The table has no training counts for {missing}. Run create_data_overview_em_organelles.py to fill " + "them in." + ) + return train_rows + + +def print_train(rows): + """Print the annotated objects of the training data per structure.""" + train_rows = select_train_rows(rows) + totals = defaultdict(lambda: {"objects": 0, "images": 0, "datasets": 0, "modalities": set()}) + for row in train_rows: + if row["structure"] not in GROUPS: + raise ValueError(f"{row['dataset']} has the structure {row['structure']}, which is in no group.") + total = totals[row["structure"]] + total["objects"] += int(row["train_objects"]) + total["images"] += int(row["train_images_or_volumes"]) + total["datasets"] += 1 + total["modalities"].add(row["modality"]) + + print("Annotated objects in the EM organelle training data (train + val split, 2d and 3d):") + print(f"{'group':<15} {'objects':>12} {'images / volumes':>17} {'datasets':>9} modalities") + for structure, group in GROUPS.items(): + total = totals[structure] + print( + f"{group:<15} {total['objects']:>12,} {total['images']:>17,} {total['datasets']:>9} " + f"{', '.join(sorted(total['modalities']))}" + ) + print( + f"{'total':<15} {sum(t['objects'] for t in totals.values()):>12,} " + f"{sum(t['images'] for t in totals.values()):>17,} {sum(t['datasets'] for t in totals.values()):>9}" + ) + + +def print_test(rows): + """Print the annotated objects of the scored test crops per structure, for ID, OOD and both.""" + test_rows = select_test_rows(rows, "3d") + totals = defaultdict(lambda: {"ID": 0, "OOD": 0, "crops": 0, "datasets": 0}) + for row in test_rows: + if row["structure"] not in GROUPS: + raise ValueError(f"{row['dataset']} has the structure {row['structure']}, which is in no group.") + total = totals[row["structure"]] + total[row["role"]] += int(row["test_objects"]) + total["crops"] += int(row["test_images_or_volumes"]) + total["datasets"] += 1 + + # The groups must cover every scored object; the test data of Figure 6 is all 3d. + expected = sum(int(row["test_objects"]) for row in rows if row["role"] in ("ID", "OOD")) + grand_total = sum(t["ID"] + t["OOD"] for t in totals.values()) + assert grand_total == expected, (grand_total, expected) + + print("Annotated objects in the scored EM organelle test data:") + print(f"{'group':<15} {'ID':>10} {'OOD':>10} {'ID + OOD':>10} {'crops':>8} {'datasets':>9}") + for structure, group in GROUPS.items(): + total = totals[structure] + print( + f"{group:<15} {total['ID']:>10,} {total['OOD']:>10,} {total['ID'] + total['OOD']:>10,} " + f"{total['crops']:>8,} {total['datasets']:>9}" + ) + print( + f"{'total':<15} {sum(t['ID'] for t in totals.values()):>10,} {sum(t['OOD'] for t in totals.values()):>10,} " + f"{grand_total:>10,} {sum(t['crops'] for t in totals.values()):>8,} " + f"{sum(t['datasets'] for t in totals.values()):>9}" + ) + + +# The rows of the detail table: (modality, structure, dimensionality) and the structure of the table. +DETAIL_STRUCTURES = {MITO: "Mitochondrion", NUCLEUS: "Nucleus"} +DETAIL_TRAIN_ROWS = [(MITO, "2"), (NUCLEUS, "2"), (MITO, "3"), (NUCLEUS, "3")] +DETAIL_TEST_ROWS = [(MITO, "3"), (NUCLEUS, "3")] +DETAIL_MODALITY = "Electron microscopy" + + +def _detail_key(row): + if row["structure"] not in GROUPS: + raise ValueError(f"{row['dataset']} has the structure {row['structure']}, which is in no group.") + return row["structure"], row["dimensionality"].rstrip("d") + + +def _detail_label(key): + structure, ndim = key + return f"{DETAIL_MODALITY} | {DETAIL_STRUCTURES[structure]} | {ndim}" + + +def print_detail(rows): + """Print the detail table of the training data and the scored test data as Markdown.""" + train_rows = select_train_rows(rows) + train = defaultdict(lambda: {"objects": 0, "images": 0}) + for row in train_rows: + key = _detail_key(row) + if key not in DETAIL_TRAIN_ROWS: + raise ValueError(f"{row['dataset']} falls into {key}, which is not a row of the detail table.") + train[key]["objects"] += int(row["train_objects"]) + train[key]["images"] += int(row["train_images_or_volumes"]) + assert sum(t["objects"] for t in train.values()) == sum(int(row["train_objects"]) for row in train_rows) + + test_rows = select_test_rows(rows, "3d") + test = defaultdict(lambda: {"ID": 0, "OOD": 0, "ID images": 0, "OOD images": 0}) + for row in test_rows: + key = _detail_key(row) + test[key][row["role"]] += int(row["test_objects"]) + test[key][f"{row['role']} images"] += int(row["test_images_or_volumes"]) + expected = sum(int(row["test_objects"]) for row in rows if row["role"] in ("ID", "OOD")) + assert sum(t["ID"] + t["OOD"] for t in test.values()) == expected + + print("| Modality | Structure | Dimensionality | Split | # Structures | # Images / Volumes |") + print("|---|---|---|---|---|---|") + for key in DETAIL_TRAIN_ROWS: + print(f"| {_detail_label(key)} | Train | {train[key]['objects']:,} | {train[key]['images']:,} |") + for key in DETAIL_TEST_ROWS: + counts = test[key] + print( + f"| {_detail_label(key)} | Test | {counts['ID']:,} (ID); {counts['OOD']:,} (OOD) | " + f"{counts['ID images']:,} (ID); {counts['OOD images']:,} (OOD) |" + ) + + +READOUTS = {"train": print_train, "test": print_test, "detail": print_detail} + + +def main(): + parser = argparse.ArgumentParser( + description="Summarize the EM organelle data overview table.", + formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, + ) + parser.add_argument("readout", nargs="?", default="train", choices=list(READOUTS), help="The readout to print.") + parser.add_argument("-i", "--input", default=DEFAULT_OUTPUT, help="The EM organelle data overview table.") + args = parser.parse_args() + READOUTS[args.readout](read_table(args.input)) + + +if __name__ == "__main__": + main() diff --git a/micro_sam/prompt_generators.py b/micro_sam/prompt_generators.py index 8746435a8..b32429135 100644 --- a/micro_sam/prompt_generators.py +++ b/micro_sam/prompt_generators.py @@ -378,22 +378,54 @@ def __call__( return net_coords, net_labels, None, None +def _error_center(error): + regions, n_regions = connected_components(error) + if n_regions == 0: + return None + largest = regions == np.bincount(regions.ravel())[1:].argmax() + 1 + distances = distance_transform_edt(np.pad(largest, 1))[1:-1, 1:-1] + y, x = np.unravel_index(distances.argmax(), distances.shape) + return [float(x), float(y)] + + +def get_error_centers( + true_mask: np.ndarray, predicted_mask: np.ndarray +) -> Tuple[Optional[List[float]], Optional[List[float]]]: + """Get the correction clicks of the 'center' protocol for one object in 2d. + + The positive click is the center of the largest missed region and the negative click the center of the + largest added region, like the 'center' click sampling of RITM and SAM2. The center is the point with the + largest distance to the region border. + + Args: + true_mask: The ground-truth mask of the object. + predicted_mask: The predicted mask of the object. + + Returns: + The positive click as (x, y), or None if nothing is missed. + The negative click as (x, y), or None if nothing is added. + """ + # Both error regions lie within the bounding box of the object and its prediction, so only this crop is + # searched. The crop keeps the raster order of the pixels, so the centers are the same. + union = true_mask | predicted_mask + if not union.any(): + return None, None + rows, cols = np.where(union.any(axis=1))[0], np.where(union.any(axis=0))[0] + crop = np.s_[rows[0]:rows[-1] + 1, cols[0]:cols[-1] + 1] + true_crop, predicted_crop = true_mask[crop], predicted_mask[crop] + pos = _error_center(true_crop & ~predicted_crop) + neg = _error_center(predicted_crop & ~true_crop) + pos = None if pos is None else [pos[0] + cols[0], pos[1] + rows[0]] + neg = None if neg is None else [neg[0] + cols[0], neg[1] + rows[0]] + return pos, neg + + class CenterIterativePromptGenerator(PromptGeneratorBase): """Generate point prompts at the centers of the prediction errors iteratively, for 2d evaluation. - Each object gets a positive click at the center of its largest missed region and a negative click at the - center of its largest added region, like the 'center' click sampling of RITM and SAM2. The center is the - point with the largest distance to the region border. A missing error type gets a padding point with the + Each object gets the clicks of `get_error_centers`. A missing error type gets a padding point with the label -1, which the model ignores. """ - def _error_center(self, error): - regions, n_regions = connected_components(error) - if n_regions == 0: - return None - largest = regions == np.bincount(regions.ravel())[1:].argmax() + 1 - distances = distance_transform_edt(np.pad(largest, 1))[1:-1, 1:-1] - y, x = np.unravel_index(distances.argmax(), distances.shape) - return [float(x), float(y)] def __call__( self, @@ -420,20 +452,7 @@ def __call__( predicted_objects = prediction.bool().cpu().numpy()[:, 0] coordinates, labels = [], [] for true_object, predicted_object in zip(true_objects, predicted_objects): - # Both error regions lie within the bounding box of the object and its prediction, so only this crop - # is searched. The crop keeps the raster order of the pixels, so the centers are the same. - union = true_object | predicted_object - if not union.any(): - coordinates.append([[0.0, 0.0], [0.0, 0.0]]) - labels.append([-1, -1]) - continue - rows, cols = np.where(union.any(axis=1))[0], np.where(union.any(axis=0))[0] - crop = np.s_[rows[0]:rows[-1] + 1, cols[0]:cols[-1] + 1] - true_crop, predicted_crop = true_object[crop], predicted_object[crop] - pos = self._error_center(true_crop & ~predicted_crop) - neg = self._error_center(predicted_crop & ~true_crop) - pos = None if pos is None else [pos[0] + cols[0], pos[1] + rows[0]] - neg = None if neg is None else [neg[0] + cols[0], neg[1] + rows[0]] + pos, neg = get_error_centers(true_object, predicted_object) coordinates.append([pos or [0.0, 0.0], neg or [0.0, 0.0]]) labels.append([1 if pos else -1, 0 if neg else -1]) diff --git a/micro_sam/v1/evaluation/inference.py b/micro_sam/v1/evaluation/inference.py index f785f2ed2..f107101ea 100644 --- a/micro_sam/v1/evaluation/inference.py +++ b/micro_sam/v1/evaluation/inference.py @@ -13,7 +13,7 @@ import torch -from bioimage_cpp.segmentation import relabel_sequential +from bioimage_cpp.segmentation import label as connected_components, relabel_sequential from ... import util as util from . import instance_segmentation @@ -393,7 +393,7 @@ def _run_inference_with_iterative_prompting_for_image( prompt_generator = CenterIterativePromptGenerator() if click_protocol == "center" else IterativePromptGenerator() - gt_ids = np.unique(gt)[1:] + gt_ids = np.setdiff1d(np.unique(gt), [0]) # Use multi-masking only if we have a single positive point without box if start_with_box_prompt: @@ -440,7 +440,9 @@ def _run_inference_with_iterative_prompting_for_image( # switching off multimasking after first iter, as next iters (with multiple prompts) don't expect multimasking multimasking = False - masks = torch.stack([m["segmentation"][None] for m in batched_outputs]).to(torch.float32) + # The masks are stacked on the CPU: the prompt generators move them there anyway, and many objects in a + # large image do not fit on the GPU. + masks = torch.stack([m["segmentation"][None].cpu() for m in batched_outputs]).to(torch.float32) next_coords, next_labels = _get_batched_iterative_prompts( sampled_binary_gt, masks, batch_size, prompt_generator @@ -521,7 +523,9 @@ def run_inference_with_iterative_prompting( image = imageio.imread(image_path) gt = imageio.imread(gt_path).astype("uint32") - gt = relabel_sequential(gt)[0] + # The objects that get prompted are the connected components of the labels, as in the micro-sam2 + # evaluation, so that both prompt the same objects. + gt = connected_components(gt).astype("uint32") if embedding_dir is None: embedding_path = None diff --git a/micro_sam/v1/evaluation/multi_dimensional_segmentation.py b/micro_sam/v1/evaluation/multi_dimensional_segmentation.py index 567c70411..2adf6242a 100644 --- a/micro_sam/v1/evaluation/multi_dimensional_segmentation.py +++ b/micro_sam/v1/evaluation/multi_dimensional_segmentation.py @@ -117,7 +117,7 @@ def segment_slices_from_ground_truth( ) # Compute instance ids (without the background) - label_ids = np.unique(ground_truth)[1:] + label_ids = np.setdiff1d(np.unique(ground_truth), [0]) assert len(label_ids) > 0, "There are no objects to perform volumetric segmentation." # Create an empty volume to store incoming segmentations @@ -231,7 +231,7 @@ def segment_slices_from_ground_truth( # Calculate dice per class. dice = [ dice_score(segmentation=(final_segmentation == i), groundtruth=(curr_gt == i)) - for i in np.unique(curr_gt)[1:] + for i in np.setdiff1d(np.unique(curr_gt), [0]) ] dice = np.mean(dice) results = {"Dice": dice} @@ -497,7 +497,7 @@ def evaluate_interactive_volume_segmentation( z_index = np.arange(volume.shape[0]) segmentations = [np.zeros(ground_truth.shape, dtype="uint32") for _ in range(n_iterations)] - for label_id in tqdm(np.unique(ground_truth)[1:], desc="Segment objects", disable=not verbose): + for label_id in tqdm(np.setdiff1d(np.unique(ground_truth), [0]), desc="Segment objects", disable=not verbose): gt = ground_truth == label_id z_values = np.where(gt.any(axis=(1, 2)))[0] if start_slice == "random": diff --git a/micro_sam/v2/datasets/generalist_loader.py b/micro_sam/v2/datasets/generalist_loader.py index b0629a0dc..7b71c9212 100644 --- a/micro_sam/v2/datasets/generalist_loader.py +++ b/micro_sam/v2/datasets/generalist_loader.py @@ -1972,7 +1972,7 @@ def _split(raw_paths, label_paths, small_keys): ) ) - # 16. Zebrafinch j0126 (neuron segmentation in FIB-SEM of zebra finch Area X, 10x10x20 nm, Kornfeld lab) + # 16. Zebrafinch j0126 (neuron segmentation in SBEM of zebra finch Area X, 10x10x20 nm, Kornfeld lab) # NOTE: Somata, vessels and missing tiles carry no id, so unlabelled blobs above 2 um^2 map to the ignore # label. The last fifth of z validates. j0251 (10x10x25 nm) uses boxes placed by a tissue scan, since # torch_em's cached boxes sit at the empty volume corner. diff --git a/micro_sam/v2/evaluation/inference.py b/micro_sam/v2/evaluation/inference.py index 711fa8b39..cac31be74 100644 --- a/micro_sam/v2/evaluation/inference.py +++ b/micro_sam/v2/evaluation/inference.py @@ -154,7 +154,7 @@ def _run_interactive_segmentation_2d_per_image( # Expects RGB-style images. encode_image(predictor, image.astype("uint8")) # uint8 is what SAM2 expects as input. - gt_ids = np.unique(gt)[1:] + gt_ids = np.setdiff1d(np.unique(gt), [0]) points, point_labels, boxes = _get_batched_prompts( gt=gt, @@ -329,8 +329,10 @@ def run_interactive_segmentation_2d( # Run connected components on labels gt = connected_components(gt).astype("uint32") - if len(np.unique(gt)) == 1: - continue # skipping the image as there are no labels. + # Skip the image if it has no labels. An image that one object covers has no background, but it + # still has a label to prompt. + if gt.max() == 0: + continue _run_interactive_segmentation_2d_per_image( image=image, @@ -399,6 +401,7 @@ def run_interactive_segmentation_3d( run_connected_components: bool = True, seed: int = 0, correction_margin: float = 0.0, + click_protocol: str = "random", ) -> str: """Run interactive segmentation on 3d inputs using Segment Anything 2. @@ -422,6 +425,8 @@ def run_interactive_segmentation_3d( seed: The seed of the prompt sampling of the volume. correction_margin: The fraction of each object's z-extent at either end that is excluded when the slice for the next correction click is chosen. 0 chooses among all slices of the object. + click_protocol: The correction clicks on the worst slice of an object: 'random' samples them at random + positions in the error regions, 'center' places them at the centers of the largest error regions. Returns: The folder where segmentations are stored. @@ -477,7 +482,7 @@ def run_interactive_segmentation_3d( # and with them the results, reproducible. np.random.seed(seed) - gt_ids = list(np.unique(labels))[1:] # Ignoring the background label + gt_ids = list(np.setdiff1d(np.unique(labels), [0])) # Ignoring the background label # An empty volume has nothing to prompt, so all iterations stay empty. segmentation = [np.zeros(labels.shape, dtype="uint64") for _ in range(n_iterations)] # The area of the object that holds each label, per iteration, to resolve overlaps. @@ -496,6 +501,7 @@ def run_interactive_segmentation_3d( batch_size=batch_size, n_iterations=n_iterations, correction_margin=correction_margin, + click_protocol=click_protocol, ) # Merge the incoming object. Where objects overlap, the smaller one is kept, as in the 2d @@ -533,6 +539,7 @@ def _run_interactive_segmentation_3d_per_object( batch_size: int = 32, n_iterations: int = 8, correction_margin: float = 0.0, + click_protocol: str = "random", ): """Functionality for interactive segmentation using iterative prompting. """ @@ -559,6 +566,7 @@ def _run_interactive_segmentation_3d_per_object( batch_size=batch_size, n_iterations=n_iterations, correction_margin=correction_margin, + click_protocol=click_protocol, ) assert len(gt_ids) == len(preds_per_object), "The number of label ids must match the number of objects segmented." @@ -616,11 +624,14 @@ def _extract_prompts_per_object( @torch.no_grad() def _get_iteratively_prompted_segmentation_per_image_dir( inference_state, labels, id_to_prompts, predictor, batch_size=32, n_iterations=8, correction_margin=0.0, + click_protocol="random", ): """Functionality for inference of 3d interactive segmentation. """ # Let's define the iterative prompt generator. - prompt_generator = IterativePromptGenerator() + if click_protocol not in ("random", "center"): + raise ValueError(f"Invalid click protocol '{click_protocol}', choose 'random' or 'center'.") + prompt_generator = CenterIterativePromptGenerator() if click_protocol == "center" else IterativePromptGenerator() pred_per_object = [] for _obj_id, _input_prompts in id_to_prompts.items(): @@ -645,6 +656,9 @@ def _get_iteratively_prompted_segmentation_per_image_dir( # previous propagation at that frame, matching the mask_inputs feedback used # during training (SAM2Train._iter_correct_pt_sampling, line 492). for per_point, per_label in zip(_corr_points, _corr_labels): + # The 'center' clicks pad a missing error type with the label -1. + if per_label < 0: + continue _, out_obj_ids, out_mask_logits = predictor.add_new_points_or_box( inference_state=inference_state, frame_idx=_corr_frame, obj_id=_obj_id, points=np.array([per_point]), labels=np.array([per_label]), diff --git a/micro_sam/v2/instance_segmentation.py b/micro_sam/v2/instance_segmentation.py index a0d9a846d..97f7ff80e 100644 --- a/micro_sam/v2/instance_segmentation.py +++ b/micro_sam/v2/instance_segmentation.py @@ -288,12 +288,15 @@ def generate( return self._masks if len(self._masks) == 0: # No masks were found. return np.zeros(self._original_size, dtype="uint32") + # Smaller masks paint over larger ones, as in micro-sam v1 AMG. With exclusive merging, the mask that + # covers the whole image, which AMG always predicts, claims every pixel before the objects in it. return mask_data_to_segmentation( masks=self._masks, shape=self._original_size, min_object_size=min_object_size, max_object_size=max_object_size, with_background=with_background, + merge_exclusively=False, ) def get_state(self) -> Dict[str, Any]: @@ -512,6 +515,7 @@ def generate( min_object_size=min_object_size, max_object_size=max_object_size, with_background=with_background, + merge_exclusively=False, ) # Offset the per-tile instance ids so that they stay unique across tiles. diff --git a/scripts/plotting/v2/README.md b/scripts/plotting/v2/README.md deleted file mode 100644 index 079eb7939..000000000 --- a/scripts/plotting/v2/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# plotting/v2 - -Scripts for postprocessing experiments and visualization of UniSAM2 predictions. - -## Scripts - -- **`run_predictions.py`** — Run UniSAM2 model inference on a dataset and cache raw/distances/labels - to H5 files for offline use (e.g. bulk re-runs without GPU). -- **`grid_search_postprocessing.py`** — Sweep postprocessing hyperparameters (flow thresholds, sigma, - etc.) using the cpp backend. Runs inference live; results saved to `OUTPUT_ROOT`. -- **`visualize.py`** — Load cached H5/TIF predictions and open them in napari for inspection. - -## Workflow - -```bash -# 1. Grid search (runs model live, sweeps postprocessing params, saves CSV) -python grid_search_postprocessing.py -d nis3d -m automatic -python grid_search_postprocessing.py -d plantseg_ovules -m automatic -python grid_search_postprocessing.py -d humanneurons -m automatic - -# 2. Full-volume prediction with best params (no crop, saves H5) -python grid_search_postprocessing.py -d nis3d -m automatic --full_volume -python grid_search_postprocessing.py -d plantseg_ovules -m automatic --full_volume -python grid_search_postprocessing.py -d humanneurons -m automatic --full_volume - -# 3. Visualize in napari -python visualize.py --dataset nis3d -``` - -## Output - -Grid search CSVs and full-volume H5s are written to: - - /mnt/vast-nhr/projects/cidas/cca/experiments/micro_sam2/experiments/grid-search-experiments/ - -## Results (unisam2-automatic, cpp backend) - -Full-volume evaluation on the first test sample per dataset (no crop). -mSA: higher is better. VI-split / VI-merge: lower is better (EM only). - -| dataset | shape (Z x Y x X) | mSA | SA50 | VI-split | VI-merge | CREMI | time (total) | -|---------|-------------------|-----|------|----------|----------|-------|--------------| -| nis3d | 198 x 978 x 987 | 0.4562 | 0.8069 | - | - | - | 5m48s | -| plantseg_ovules | 320 x 960 x 1000 | 0.3771 | 0.5941 | - | - | - | 12m33s | -| liconn (EM protocol) | 700 x 700 x 700 | 0.0004 | - | 8.2262 | 1.4307 | 3.1045 | 11m28s | -| mitoem-human (EM) | 100 x 4096 x 4096 | 0.0000 | - | 9.7326 | 0.2464 | 3.0998 | 46m14s | -| cremi-padded sampleB (EM) | 200 x 3072 x 3072 | 0.0180 | - | 2.0314 | 1.0036 | 0.9379 | 46m44s | -| microns minnie65 (EM) | 200 x 2048 x 2048 | 0.0015 | - | 3.4523 | 2.4439 | 1.6291 | 28m18s | diff --git a/scripts/plotting/v2/for_paper/README.md b/scripts/plotting/v2/for_paper/README.md new file mode 100644 index 000000000..e3d6d56dc --- /dev/null +++ b/scripts/plotting/v2/for_paper/README.md @@ -0,0 +1,13 @@ +# Plotting Scripts for the micro-sam2 Paper + +This directory has the python scripts that make the figures for the `micro-sam2` paper. + +## Scripts + +- `plot_style.py`: The shared plot style and the fixed color of each method. All figures import their colors from here. +- `plots_for_figure1.py`: Plots for Figure 1. +- `plots_for_figure2.py`: All plots for Figure 2, automatic and interactive, quantitative and qualitative. +- `plots_for_figure3.py`: Plots for Figure 3. +- `plots_for_figure4.py`: Plots for Figure 4, object and pixel classification with the features of each model over the number of labels, and the mean rank of the feature models. +- `plots_for_figure5.py`: Plots for Figure 5. +- `plots_for_figure6.py`: Plots for Figure 6. diff --git a/scripts/plotting/v2/for_paper/assets/micro_sam2_logo.png b/scripts/plotting/v2/for_paper/assets/micro_sam2_logo.png new file mode 100644 index 000000000..acbdc9ce3 Binary files /dev/null and b/scripts/plotting/v2/for_paper/assets/micro_sam2_logo.png differ diff --git a/scripts/plotting/v2/for_paper/plot_extended_data1.py b/scripts/plotting/v2/for_paper/plot_extended_data1.py new file mode 100644 index 000000000..a356ff983 --- /dev/null +++ b/scripts/plotting/v2/for_paper/plot_extended_data1.py @@ -0,0 +1,235 @@ +"""Plots for Extended Data Figure 1 of the micro-sam2 paper: the quantitative panels of Figures 2 and 3 for all model +sizes of micro-sam2. + +The figure repeats panels a and c of Figure 2 and panels a and c of Figure 3, on the same datasets, with the four +Hiera sizes of micro-sam2 (T, S, B, L) and of SAM2. The script writes these plots to +'figures/extendeddata1//quantitative': +- 'automatic_2d' (panel a): the mSA of CellposeSAM and of micro-sam2 (AIS and APG) per modality, for the in-domain and + the out-of-domain datasets. +- 'interactive_2d' (panel b): the mSA of SAM2 and micro-sam2 over the correction iterations, with a point or a box as + the first prompt. +- 'automatic_3d' (panel c): as panel a for 3d. The EM neurites compare SegNeuron instead of CellposeSAM. +- 'interactive_3d' (panel d): as panel b for 3d, with nnInteractive. +- 'extended_data_1' (in 'figures/final'): the four plots in one figure, as panels a to d. + +A model size is a tint of the color of its method: T is the lightest, L has the full color, as in Figures 2 and 3. +The automatic panels show micro-sam2 with one sub-bar per mode and size: AIS T to L, then APG T to L. + +Usage: + + python plot_extended_data1.py + python plot_extended_data1.py --only extended_data_1 +""" + +import os +import argparse + +import pandas as pd +import matplotlib.colors as colors +import matplotlib.pyplot as plt + +import plots_for_figure2 as fig2 +import plots_for_figure3 as fig3 +from plot_style import METHOD_COLORS, set_style + +ROOT = os.path.dirname(os.path.abspath(__file__)) +OUTPUT_DIR = os.path.join(ROOT, "figures", "extendeddata1") +COMBINED_NAME = "extended_data_1" + +# The Hiera sizes, from the smallest to the largest, with their short names and the share of the method color. +SIZES = ["hvit_t", "hvit_s", "hvit_b", "hvit_l"] +SIZE_NAMES = {"hvit_t": "T", "hvit_s": "S", "hvit_b": "B", "hvit_l": "L"} +SIZE_TINTS = {"hvit_t": 0.5, "hvit_s": 0.67, "hvit_b": 0.83, "hvit_l": 1.0} + +# The ('method', 'model') of the automatic baselines, per dimension. micro-sam2 runs AIS and APG. +AUTOMATIC_BASELINES_2D = {"CellposeSAM": ("cellpose", "cpsam")} +AUTOMATIC_BASELINES_3D = {"CellposeSAM": ("cellpose", "cpsam_do3d"), "SegNeuron": ("segneuron", "default")} +# The baseline of each 3d modality: SegNeuron for the EM neurites, where CellposeSAM has no results. +BASELINE_3D = {"fluorescence_cells": "CellposeSAM", "fluorescence_nuclei": "CellposeSAM", "em_neurites": "SegNeuron"} +MICROSAM2_MODES = {"ais": "AIS", "apg": "APG"} +# The sub-bars of micro-sam2 in the automatic plots, as '_', from the top to the bottom. +SIZE_MODES = [f"{mode}_{size}" for mode in MICROSAM2_MODES for size in SIZES] + +# The interactive methods with one size, as (method, model), and those with all sizes, as their method. +INTERACTIVE_SINGLE_3D = {"nnInteractive": ("nninteractive", "default")} +INTERACTIVE_SIZED = {"SAM2": "sam2", "micro-sam2": "micro-sam2"} + + +def tinted(color, tint): + """Return the color mixed with white, with the share 'tint' of the color.""" + return colors.to_hex([1 - tint * (1 - c) for c in colors.to_rgb(color)]) + + +def size_label(method, size): + """The name of a method with one model size in the plots, such as 'μSAM2 T'.""" + return f"{fig2.names_of([method], set())[0]} {SIZE_NAMES[size]}" + + +def interactive_methods(single): + """Return the interactive methods, as name -> ('method', models, mode), and their colors. + + Each size of a sized method is a method of its own, named ' '. + """ + methods, method_colors = {}, {} + for name, (method, model) in single.items(): + methods[name] = (method, (model,), "") + method_colors[name] = METHOD_COLORS[name] + for name, method in INTERACTIVE_SIZED.items(): + for size in SIZES: + key = f"{name} {size}" + methods[key] = (method, (size,), "") + method_colors[key] = tinted(METHOD_COLORS[name], SIZE_TINTS[size]) + return methods, method_colors + + +def load_automatic(path, category, structure, datasets, baselines, metric="msa"): + """Load the automatic scores of the baselines and of each size of micro-sam2 for one modality. + + The column 'mode' holds the mode and size of micro-sam2, such as 'ais_hvit_t', and an empty string for the + baselines. + """ + df = pd.read_csv(path) + df = df[(df["category"] == category) & (df["structure"] == structure)] + parts = [] + for name, (method, model) in baselines.items(): + rows = df[(df["method"] == method) & (df["model"] == model) & df["mode"].isna()] + parts.append(rows.assign(method=name, mode="")) + rows = df[(df["method"] == "micro-sam2") & df["model"].isin(SIZES) & df["mode"].isin(MICROSAM2_MODES)] + parts.append(rows.assign(mode=rows["mode"] + "_" + rows["model"])) + results = pd.concat(parts, ignore_index=True)[["dataset", "method", "mode", metric]] + return fig2.select_datasets(results.rename(columns={metric: "score"}), datasets) + + +def plot_automatic(dimension, region=None, compact=False): + """Plot the automatic results of one dimension, with micro-sam2 in all sizes, in the layout of Figure 2 or 3.""" + results, panels = {}, {} + if dimension == "2d": + for name, (category, structure, title) in fig2.PANELS.items(): + results[name] = load_automatic( + fig2.RESULTS_CSV, category, structure, fig2.PANEL_DATASETS[name], AUTOMATIC_BASELINES_2D, + ) + panels[name] = (title, fig2.SCORE_LABEL, ["CellposeSAM", "micro-sam2"]) + rows, grid, domain_offset = fig2.AUTOMATIC_ROWS, fig2.COMBINED_GRID, 0.1 + else: + for name, (category, structure, title, metric, score_label) in fig3.PANELS.items(): + baseline = BASELINE_3D[name] + results[name] = load_automatic( + fig3.RESULTS_CSV, category, structure, fig3.PANEL_DATASETS[name], + {baseline: AUTOMATIC_BASELINES_3D[baseline]}, metric, + ) + panels[name] = (title, score_label, [baseline, "micro-sam2"]) + rows, grid, domain_offset = fig3.AUTOMATIC_ROWS, fig3.COMBINED_GRID, 0.27 + return fig2.plot_automatic( + results, rows, panels, starred=set(), domain_offset=domain_offset, region=region, + row_height=fig2.FIGURE_ROW_HEIGHT if compact else fig2.BAR_ROW_HEIGHT, + grid=grid if compact else None, title_line=compact, ranks=True, + group_gap=fig2.COMBINED_GROUP_GAP if compact else fig2.GROUP_GAP, + row_gap=fig2.COMBINED_ROW_GAP if compact else None, + ) + + +def add_size_legend(fig, methods, method_colors): + """Add the legend of the interactive plot under its panels, in one row.""" + handles, labels = [], [] + for key in methods: + name, _, size = key.partition(" ") + handles.append(plt.Line2D( + [], [], color=method_colors[key], marker="o", markersize=2.2, linewidth=1.0, markeredgecolor="white", + markeredgewidth=0.4, + )) + labels.append(size_label(name, size) if size else fig2.names_of([name], set())[0]) + bottom = min(ax.get_position().y0 for ax in fig.axes) + offset = (fig2.X_LABEL_OFFSET + 0.14) / fig2.height_of(fig) + legend = fig.legend( + handles, labels, loc="upper center", bbox_to_anchor=(0.5, bottom - offset), ncol=len(labels), + fontsize=fig2.MAX_FONT_SIZE, handlelength=1.1, handletextpad=0.3, columnspacing=0.8, borderpad=0.25, + frameon=True, framealpha=1, facecolor="white", edgecolor=fig2.LEGEND_BORDER_COLOR, + ) + legend.get_frame().set_linewidth(0.6) + + +def plot_interactive(dimension, region=None, compact=False): + """Plot the interactive results of one dimension, with SAM2 and micro-sam2 in all sizes.""" + if dimension == "2d": + modalities = [name for row in fig2.AUTOMATIC_ROWS for name in row] + specs = {name: (category, structure, "msa", fig2.SCORE_LABEL) + for name, (category, structure, _) in fig2.PANELS.items()} + titles = {name: title for name, (_, _, title) in fig2.PANELS.items()} + datasets, path, grid, single = fig2.PANEL_DATASETS, fig2.INTERACTIVE_CSV, fig2.COMBINED_GRID, {} + else: + modalities = [name for row in fig3.AUTOMATIC_ROWS for name in row] + specs = {name: (category, structure, metric, score_label) + for name, (category, structure, _, metric, score_label) in fig3.PANELS.items()} + titles = {name: title for name, (_, _, title, _, _) in fig3.PANELS.items()} + datasets, path, grid = fig3.PANEL_DATASETS, fig3.INTERACTIVE_CSV, fig3.COMBINED_GRID + single = INTERACTIVE_SINGLE_3D + methods, method_colors = interactive_methods(single) + results = { + name: fig2.load_interactive( + category, structure, datasets[name], path=path, methods=methods, metric=metric, + ).dropna(subset=["score"]) + for name, (category, structure, metric, _) in specs.items() + } + score_labels = {name: score_label for name, (_, _, _, score_label) in specs.items()} + # The curves look up their colors in the color table of Figure 2. + fig2.METHOD_COLORS = {**METHOD_COLORS, **method_colors} + try: + fig = fig2.plot_interactive_by_domain( + results, modalities, titles, starred=set(), methods=methods, score_labels=score_labels, + region=region, panel_height=fig2.COMBINED_PANEL_HEIGHT if compact else fig2.PANEL_HEIGHT, + grid=grid if compact else None, title_line=compact, shared_rows=True, show_titles=not compact, + score_gap=0.32 if compact else None, row_gap=fig2.COMBINED_ROW_GAP if compact else None, + band_color=fig2.COMBINED_BAND_COLOR if compact else fig2.BOX_BAND_COLOR, prompts_on_top=True, + x_label=fig2.ITERATION_LABEL, + ) + finally: + fig2.METHOD_COLORS = METHOD_COLORS + add_size_legend(fig, methods, method_colors) + return fig + + +# The plots in their order in the combined figure, with their kind and style (see 'fig2.save_figure'). +PLOTS = { + "automatic_2d": ("automatic", "quantitative"), + "interactive_2d": ("interactive", "quantitative"), + "automatic_3d": ("automatic", "quantitative"), + "interactive_3d": ("interactive", "quantitative"), +} + + +def make_plot(name, region=None, compact=False): + """Make one plot, as its own figure or in a region of the combined figure (see 'fig2.plot_figure').""" + kind, dimension = name.split("_") + plot = plot_automatic if kind == "automatic" else plot_interactive + return plot(dimension, region=region, compact=compact) + + +def set_size_modes(): + """Let the bar plots of Figure 2 show each mode and size of micro-sam2 as a sub-bar, tinted by its size.""" + fig2.MODES = SIZE_MODES + fig2.MODE_NAMES = { + f"{mode}_{size}": f"{name} {SIZE_NAMES[size]}" for mode, name in MICROSAM2_MODES.items() for size in SIZES + } + fig2.MODE_TINTS = {f"{mode}_{size}": SIZE_TINTS[size] for mode in MICROSAM2_MODES for size in SIZES} + # The longer names, such as 'APG L', need more room on their sub-bars. + fig2.MODE_TAG_WIDTH = 20 + + +def main(): + names = list(PLOTS) + [COMBINED_NAME] + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--only", nargs="+", choices=names, default=names, help="Make only these plots.") + args = parser.parse_args() + set_style() + fig2.check_style() + set_size_modes() + fig2.OUTPUT_DIR = OUTPUT_DIR + for name, (kind, style) in PLOTS.items(): + if name in args.only: + fig2.save_figure(make_plot(name), name, kind, style) + if COMBINED_NAME in args.only: + fig2.save_combined(*fig2.plot_figure(PLOTS, make_plot), name=COMBINED_NAME) + + +if __name__ == "__main__": + main() diff --git a/scripts/plotting/v2/for_paper/plot_extended_data2.py b/scripts/plotting/v2/for_paper/plot_extended_data2.py new file mode 100644 index 000000000..ff9cfb1c2 --- /dev/null +++ b/scripts/plotting/v2/for_paper/plot_extended_data2.py @@ -0,0 +1,484 @@ +"""Plots for Extended Data Figure 2 of the micro-sam2 paper: the runtimes of micro-sam2 on CPU, GPU and Apple silicon. + +The numbers come from 'finetuning/v2/evaluation/runtime_benchmarking_script.py' (run 'collect' there first): one +timed run per sample of LIVECell (2d LM) and GoNuclear (3d LM), for the four sizes of micro-sam2 (T, S, B, L) and the +reference CellposeSAM. AIS and APG share their first stage: the embeddings of the image or of every slice (untiled), +then the decoder on them. The script writes these plots to +'figures/extendeddata2//quantitative': +- 'automatic_runtimes' (panel a): the mean time per image (2d) or per volume (3d) of AIS and APG, split into the + encoder, the decoder and the postprocessing (for APG also the prompting of SAM2 with the decoder seeds), and of the + references, split into their network and their postprocessing. The error bars are the standard deviation of the + total over the samples. +- 'interactive_runtimes' (panel b): the mean time of one object from its first prompt (a point or a box), per + object in 2d and per object and slice in 3d, with the standard deviation over the objects, and the time of the + encoder per image or volume (in 3d with the set-up of the video predictor), as two bars per row on a log axis. + The CPU row of panel a has a log axis too. +- 'extended_data_2' (in 'figures/final'): both plots in one figure. + +In 3d, all methods on all devices are timed on the same three volumes per dataset (see 'VOLUMES_3D' in +'runtime_benchmarking_script.py'). A device without results (e.g. the Mac before it is run) shows 'Results pending'. + +Usage: + + python plot_extended_data2.py +""" + +import glob +import os +import argparse + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.colors as colors +import matplotlib.ticker as ticker +import matplotlib.transforms as transforms +from matplotlib.text import Annotation + +import plots_for_figure2 as fig2 +from plot_style import METHOD_COLORS, TEXT_COLOR, set_style + +ROOT = os.path.dirname(os.path.abspath(__file__)) +OUTPUT_DIR = os.path.join(ROOT, "figures", "extendeddata2") +COMBINED_NAME = "extended_data_2" +BENCHMARK_ROOT = os.environ.get( + "RUNTIME_BENCHMARK_ROOT", "/mnt/vast-nhr/projects/cidas/cca/experiments/micro_sam2/experiments/runtime_benchmark" +) +SAMPLE_CSV = os.path.join(BENCHMARK_ROOT, "runtimes_per_sample.csv") +OBJECT_CSV = os.path.join(BENCHMARK_ROOT, "runtimes_per_object.csv") +# The tables of the devices timed off the cluster (the Mac) are in one folder per device tag, next to the main +# tables. They are separate, because 'runtime_benchmarking_script.py collect' rewrites the main tables. +DEVICE_CSV_ROOT = BENCHMARK_ROOT + +DEVICES = {"cpu": "CPU (16 cores)", "a100": "GPU (A100)", "m2": "Apple M2"} +# CREMI (3d EM, reference SegNeuron) is timed too, but not shown: APG L takes more than 12 h per volume on CPU. +DATASETS = {"livecell": "LIVECell (2d, LM)", "gonuclear": "GoNuclear (3d, LM)"} +REFERENCES = {"cellposesam": "CellposeSAM"} +# The text over a mode that was not run in a panel, because it takes too long there. +NOT_RUN_TEXT = "not run (hours per volume)" +# The short names of the references in the figure. +REFERENCE_NAMES = {"cellposesam": "CPSAM"} +SIZES = ["hvit_t", "hvit_s", "hvit_b", "hvit_l"] +SIZE_NAMES = {"hvit_t": "T", "hvit_s": "S", "hvit_b": "B", "hvit_l": "L"} +MODES = {"ais": "AIS", "apg": "APG"} +PROMPTS = {"interactive_point": "Point", "interactive_box": "Box"} +# The number of slices of the 3d samples (all evaluation crops are 32 x 512 x 512), for the time per slice. +N_SLICES = {"gonuclear": 32, "cremi": 32} +# The devices whose automatic runtimes span orders of magnitude get a log time axis. +LOG_DEVICES = {"cpu", "a100", "m2"} +# The gap between the stacked parts and the smallest width of a part, as fractions of the axis width. +PART_GAP = 0.006 +# The space between the end of a bar (or its error bar) and its number, in points. +LABEL_PAD = 2 +# The share of the axis width taken by the longest bar; the rest is for its number. +BAR_SHARE_2D, BAR_SHARE_3D = 0.8, 0.7 +# The largest share of a block that its number may fill. +TEXT_FILL = 0.8 +# The color of the time per slice next to the totals (contrast 5.3 on white). +SLICE_COLOR = "#6b6b6b" +# The panels (device, dataset) whose totals carry the unit. +SECONDS_PANELS = {(device, dataset) for device in ("cpu", "a100", "m2") + for dataset in ("livecell", "gonuclear", "cremi")} +# The panels (device, dataset) that write times over MINUTES_FROM seconds in minutes. +MINUTES_PANELS = {("cpu", "gonuclear"), ("cpu", "cremi"), ("m2", "gonuclear")} +MINUTES_FROM = 200 +MIN_PART_WIDTH = 0.012 +# The share of the micro-sam2 color for the interactive bars: a tint that no part of panel a uses. + +# The share of the method color per part: the encoder has the full color, the later parts are lighter. +PART_TINTS = {"encoder": 1.0, "network": 1.0, "decoder": 0.6, "postprocessing": 0.3} +PART_NAMES = {"encoder": "Encoder", "decoder": "Decoder", "network": "Network", "postprocessing": "Postprocessing"} + +BAR_HEIGHT = 0.72 +ROW_HEIGHT = 0.12 +PANEL_GRID = (0.75, 0.15, 0.25) + + +def tinted(color, tint): + return colors.to_hex([1 - tint * (1 - c) for c in colors.to_rgb(color)]) + + +def load(): + samples = pd.read_csv(SAMPLE_CSV) if os.path.exists(SAMPLE_CSV) else pd.DataFrame( + columns=["device_tag", "dataset", "method", "model", "sample", "part", "seconds"] + ) + objects = pd.read_csv(OBJECT_CSV) if os.path.exists(OBJECT_CSV) else pd.DataFrame( + columns=["device_tag", "dataset", "method", "model", "seconds_per_slice", "n_slices"] + ) + # The tables of a device folder replace the rows of this device in the main tables. + for folder in sorted(glob.glob(os.path.join(DEVICE_CSV_ROOT, "runtimes_*", ""))): + device_tag = os.path.basename(os.path.dirname(folder))[len("runtimes_"):] + for name, table in (("samples", "runtimes_per_sample.csv"), ("objects", "runtimes_per_object.csv")): + path = os.path.join(folder, table) + if not os.path.exists(path): + continue + current = samples if name == "samples" else objects + current = pd.concat([current[current["device_tag"] != device_tag], pd.read_csv(path)], ignore_index=True) + if name == "samples": + samples = current + else: + objects = current + # Only the selected samples: all images in 2d, the volumes of VOLUMES_3D per dataset in 3d (one on the M2). + if "selected" in samples: + samples = samples[samples["selected"].astype(bool)] + objects = objects[objects["selected"].astype(bool)] + return samples, objects + + +def automatic_rows(dataset): + """The bars of one automatic panel, top to bottom, as (label, method, model, method color name).""" + reference = "segneuron" if dataset == "cremi" else "cellposesam" + rows = [(REFERENCE_NAMES.get(reference, REFERENCES[reference]), reference, "default", REFERENCES[reference])] + for mode, mode_name in MODES.items(): + rows += [(f"μSAM2 {mode_name} {SIZE_NAMES[size]}", mode, size, "micro-sam2") for size in SIZES] + return rows + + +def draw_pending(ax): + ax.set_xticks([]) + ax.set_yticks([]) + ax.text(0.5, 0.5, "Results pending", ha="center", va="center", style="italic", color=fig2.GROUP_COLOR, + transform=ax.transAxes, fontsize=fig2.MAX_FONT_SIZE) + + +def style_axes(ax): + ax.grid(axis="x", color="#e6e5e0", linewidth=0.5, linestyle=(0, (3, 3)), zorder=0) + ax.set_axisbelow(True) + ax.tick_params(axis="y", length=0) + for side in ("top", "right"): + ax.spines[side].set_visible(False) + + +def grid_axes(fig, n_rows, n_columns, panel_height, top, row_gap=0.25): + """Axes in a grid of devices (rows) and datasets (columns), with sizes in inches from the top of the figure.""" + lefts, width = fig2.column_grid(PANEL_GRID, n_columns) + height = fig2.height_of(fig) + axes = np.empty((n_rows, n_columns), dtype=object) + for i in range(n_rows): + bottom = height - top - (i + 1) * panel_height - i * row_gap + for j, left in enumerate(lefts): + axes[i, j] = fig.add_axes([ + left / fig2.FIGURE_WIDTH, bottom / height, width / fig2.FIGURE_WIDTH, panel_height / height, + ]) + return axes + + +def label_grid(fig, axes): + for j, title in enumerate(DATASETS.values()): + axes[0, j].set_title(title, fontsize=fig2.MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR, pad=4) + for i, device in enumerate(DEVICES.values()): + box = axes[i, -1].get_position() + fig.text(box.x1 + 0.01, (box.y0 + box.y1) / 2, device, rotation=270, ha="left", va="center", + fontsize=fig2.MAX_FONT_SIZE, color=TEXT_COLOR, fontweight="bold") + + +def format_minutes(value): + """Minutes, or hours from 60 minutes on.""" + minutes = value / 60 + if minutes >= 60: + return f"{minutes / 60:.1f} h" + return f"{minutes:.1f} min" if minutes < 10 else f"{minutes:.0f} min" + + +def format_tick(value, _): + """Log axis ticks in seconds; from 1000 s on as powers of ten, which are shorter.""" + if value >= 1000: + # Unicode superscripts instead of mathtext, which would embed a second font. + return "10" + str(int(round(np.log10(value)))).translate(str.maketrans("0123456789", "⁰¹²³⁴⁵⁶⁷⁸⁹")) + " s" + return f"{value:g} s" + + +def format_seconds(value): + if value < 0.1: + return f"{value:.2g}" + return f"{value:.0f}" if value >= 10 else f"{value:.1f}" if value >= 1 else f"{value:.2f}" + + +def contrast(first, second): + """The WCAG contrast ratio of two colors.""" + def luminance(color): + rgb = [c / 12.92 if c <= 0.03928 else ((c + 0.055) / 1.055) ** 2.4 for c in colors.to_rgb(color)] + return 0.2126 * rgb[0] + 0.7152 * rgb[1] + 0.0722 * rgb[2] + high, low = sorted([luminance(first), luminance(second)], reverse=True) + return (high + 0.05) / (low + 0.05) + + +def text_color_on(color): + """White text on dark colors, dark text on light colors; black where neither reaches a contrast of 4.5.""" + red, green, blue = colors.to_rgb(color) + preferred = "white" if 0.2126 * red + 0.7152 * green + 0.0722 * blue < 0.5 else TEXT_COLOR + if contrast(preferred, color) >= 4.5: + return preferred + return max(["white", TEXT_COLOR, "black"], key=lambda text: contrast(text, color)) + + +def draw_segments(ax, segments, log, minutes=False, seconds=False): + """Draw the stacked parts with a small gap between them, so that each part shows as a block. + + The bar ends at the total. A part shorter than MIN_PART_WIDTH of the axis gets that width, taken from the + parts before it, so it is not to scale. + """ + x0, x1 = ax.get_xlim() + forward, back = (np.log10, lambda u: 10.0 ** u) if log else (lambda x: x, lambda u: u) + span = forward(x1) - forward(x0) + min_width, gap = MIN_PART_WIDTH * span, PART_GAP * span + for yi, start, values, parts, color_name, whisker in segments: + keep = [(part, value) for part, value in zip(parts, values) if value > 0] + if not keep: + continue + # The boundaries of the parts on the (log) axis, from the start to the total. + bounds = [forward(max(start, x0))] + total = 0.0 + for _, value in keep: + total += value + bounds.append(forward(max(total, start))) + # Going back from the end, every part gets at least the minimum width. + for k in range(len(keep) - 1, 0, -1): + bounds[k] = min(bounds[k], bounds[k + 1] - min_width) + for k, (part, _) in enumerate(keep): + low = bounds[k] + high = bounds[k + 1] - (gap if k < len(keep) - 1 else 0.0) + if high <= low: + continue + color = tinted(METHOD_COLORS[color_name], PART_TINTS[part]) + ax.barh(yi, back(high) - back(low), left=back(low), height=BAR_HEIGHT, linewidth=0, zorder=2, color=color) + # The time of the part is written in its block, if it fits; in the last block, left of the error bar. + right = min(high, forward(whisker)) if k == len(keep) - 1 and whisker > 0 else high + value = keep[k][1] + if minutes and value > MINUTES_FROM: + label = format_minutes(value) + else: + label = format_seconds(value) + (" s" if seconds else "") + text = ax.text(back((low + right) / 2), yi, label, ha="center", va="center", + fontsize=fig2.SMALL_FONT_SIZE - 1, color=text_color_on(color), zorder=4) + renderer = ax.figure.canvas.get_renderer() + block = ax.transData.transform([(back(low), yi), (back(right), yi)]) + if text.get_window_extent(renderer).width > TEXT_FILL * (block[1, 0] - block[0, 0]): + text.remove() + + +def write_total(ax, x, y, seconds, device, dataset, per_slice=None): + """Write a total right of 'x', followed by the time per slice in gray italics (3d).""" + if (device, dataset) in MINUTES_PANELS and seconds > MINUTES_FROM: + note = f" {format_minutes(seconds)}" + else: + note = " " + (f"{seconds:.2g}" if seconds < 0.1 else f"{seconds:.2f}" if seconds < 100 else f"{seconds:.0f}") + if (device, dataset) in SECONDS_PANELS: + note += " s" + text = ax.text(x, y, note, va="center", ha="left", fontsize=fig2.SMALL_FONT_SIZE, color=TEXT_COLOR, + transform=transforms.offset_copy(ax.transData, fig=ax.figure, x=LABEL_PAD, units="points")) + if per_slice is not None: + if per_slice > MINUTES_FROM: + per_slice = f"{per_slice / 60:.1f} min" + else: + per_slice = f"{per_slice:.2g} s" if per_slice < 100 else f"{per_slice:.0f} s" + ax.annotate(f" ({per_slice}/slice)", xy=(1, 0.5), xycoords=text, va="center", ha="left", + fontsize=fig2.SMALL_FONT_SIZE, color=SLICE_COLOR, fontstyle="italic") + + +def log_start(first): + """The power of ten under the shortest first part; one lower if that part would be a sliver.""" + start = 10.0 ** np.floor(np.log10(first.min())) + return start / 10 if first.min() < 2 * start else start + + +def set_log_axis(ax, start, end, share): + """A log time axis on which the longest bar takes 'share' of the width.""" + ax.set_xscale("log") + ax.set_xlim(start, start * (end / start) ** (1 / share)) + ax.xaxis.set_major_formatter(ticker.FuncFormatter(format_tick)) + ax.xaxis.set_minor_formatter(ticker.NullFormatter()) + + +def title_on_edge(fig, legend, title): + """Write the title of a legend card on the top left of its frame, over the frame line.""" + fig.add_artist(Annotation(title, xy=(0, 1), xycoords=legend, xytext=(6, 0), textcoords="offset points", + ha="left", va="center", fontsize=fig2.MAX_FONT_SIZE, color=TEXT_COLOR, + zorder=legend.get_zorder() + 1, bbox=dict(facecolor="white", edgecolor="none", pad=1.5))) + + +def plot_automatic(region=None, compact=False): + samples, _ = load() + n_rows = len(automatic_rows("livecell")) + panel_height = (n_rows + 0.6) * ROW_HEIGHT + legend_height = 0.3 + height = 0.3 + len(DEVICES) * (panel_height + 0.25) + 0.3 + legend_height + fig = fig2.new_figure(height, region) + axes = grid_axes(fig, len(DEVICES), len(DATASETS), panel_height, top=0.3) + for i, device in enumerate(DEVICES): + for j, dataset in enumerate(DATASETS): + ax = axes[i, j] + rows = automatic_rows(dataset) + data = samples[(samples["device_tag"] == device) & (samples["dataset"] == dataset)] + y = -np.arange(len(rows)) - np.array([0 if k == 0 else 0.4 if k <= 4 else 0.8 for k in range(len(rows))]) + if data.empty: + draw_pending(ax) + else: + means = data.groupby(["method", "model", "part"])["seconds"].mean() + totals = data[data["part"] == "total"].groupby(["method", "model"])["seconds"].agg(["mean", "std"]) + log = device in LOG_DEVICES + # On a log axis, the bars start at the power of ten under the fastest first part (encoder or + # network), so that every part stays visible. + first = means[means.index.get_level_values("part").isin(["encoder", "network"]) & (means > 0)] + start = 10.0 ** np.floor(np.log10(first.min())) if log else 0.0 + # A first part just above the power of ten would be a sliver, so start one power lower. + if log and first.min() < 2 * start: + start /= 10 + end, segments = 0.0, [] + for yi, (label, method, model, color_name) in zip(y, rows): + if (method, model) not in totals.index: + continue + parts = ["network", "postprocessing"] if method in REFERENCES else [ + "encoder", "decoder", "postprocessing"] + # The parts are stacked; on a log axis, the first part starts at 'start'. + values = [means.get((method, model, part), 0.0) for part in parts] + segments.append((yi, start, values, parts, color_name)) + mean, std = totals.loc[(method, model)] + std = 0.0 if np.isnan(std) else std + # On a log axis, the lower error bar stops at the start of the axis. + lower = min(std, mean - start * 1.01) if log else std + segments[-1] += (mean - lower,) + ax.errorbar(mean, yi, xerr=[[lower], [std]], color=fig2.GROUP_COLOR, linewidth=0.6, + capsize=1.2, zorder=3) + write_total(ax, mean + std, yi, mean, device, dataset, + mean / N_SLICES[dataset] if dataset in N_SLICES else None) + end = max(end, mean + std) + # The bars take the same share of the axis on a linear and on a log axis; the rest is for the notes. + share = BAR_SHARE_3D if dataset in N_SLICES else BAR_SHARE_2D + if log: + ax.set_xscale("log") + ax.set_xlim(start, start * (end / start) ** (1 / share)) + ax.xaxis.set_major_formatter(ticker.FuncFormatter(format_tick)) + ax.xaxis.set_minor_formatter(ticker.NullFormatter()) + else: + ax.set_xlim(0, end / share) + draw_segments(ax, segments, log, minutes=(device, dataset) in MINUTES_PANELS, + seconds=(device, dataset) in SECONDS_PANELS) + # A mode without any result in a panel (not run, too slow) gets one band over its rows, as the + # 'not applicable' bands of Figure 2. + for mode in MODES: + mode_y = [yi for yi, row in zip(y, rows) if row[1] == mode] + if any((mode, size) in totals.index for size in SIZES): + continue + ax.axhspan(min(mode_y) - BAR_HEIGHT / 2, max(mode_y) + BAR_HEIGHT / 2, + color=fig2.NOT_APPLICABLE_COLOR, linewidth=0, zorder=0) + ax.text(0.5, np.mean(mode_y), NOT_RUN_TEXT, transform=ax.get_yaxis_transform(), + ha="center", va="center", fontsize=fig2.MAX_FONT_SIZE, style="italic", + color=fig2.NOT_APPLICABLE_TEXT_COLOR) + style_axes(ax) + ax.set_ylim(y.min() - 0.6, y.max() + 0.6) + ax.set_yticks(y) + ax.set_yticklabels([row[0] for row in rows] if j == 0 else [], fontsize=fig2.SMALL_FONT_SIZE) + label_grid(fig, axes) + bottom_box = axes[-1, 0].get_position() + fig.text(0.5, bottom_box.y0 - 0.25 / height, "Time per image (2d) or volume (3d) [s, log scale] ↓", + ha="center", va="top", + fontsize=fig2.MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR) + # One legend card per method, side by side: micro-sam2 left of the center, the references right of it. + cards = [("μSAM2", "micro-sam2", ["encoder", "decoder", "postprocessing"])] + cards += [(REFERENCE_NAMES.get(method, name), name, ["network", "postprocessing"]) + for method, name in REFERENCES.items()] + for k, (title, color_name, parts) in enumerate(cards): + handles = [plt.Rectangle((0, 0), 1, 1, color=tinted(METHOD_COLORS[color_name], PART_TINTS[part])) + for part in parts] + legend = fig.legend(handles, [PART_NAMES[part] for part in parts], ncol=len(parts), + loc="lower right" if k == 0 else "lower left", + bbox_to_anchor=(0.49 if k == 0 else 0.51, 0.0), + fontsize=fig2.MAX_FONT_SIZE, frameon=True, + edgecolor=fig2.LEGEND_BORDER_COLOR, handlelength=1.1, columnspacing=0.8, borderpad=0.9) + legend.get_frame().set_linewidth(0.6) + title_on_edge(fig, legend, title) + return fig + + +def plot_interactive(region=None, compact=False): + samples, objects = load() + rows = [(f"{PROMPTS[method]} {SIZE_NAMES[size]}", method, size) for method in PROMPTS for size in SIZES] + panel_height = (len(rows) + 0.6) * ROW_HEIGHT + legend_height = 0.3 + height = 0.3 + len(DEVICES) * (panel_height + 0.25) + 0.3 + legend_height + fig = fig2.new_figure(height, region) + axes = grid_axes(fig, len(DEVICES), len(DATASETS), panel_height, top=0.3) + # Each row is one stacked bar on a log time axis, as in panel a: the image encoder, once per image (2d) or + # volume (3d), then the prompt encoder with the mask decoder for one object after its first prompt, in 3d through + # all slices. The gray italic number is the decoder time per slice. + y = -np.arange(len(rows)) - np.where(np.arange(len(rows)) >= len(SIZES), 0.4, 0) + for i, device in enumerate(DEVICES): + for j, dataset in enumerate(DATASETS): + ax = axes[i, j] + obj = objects[(objects["device_tag"] == device) & (objects["dataset"] == dataset)] + enc = samples[(samples["device_tag"] == device) & (samples["dataset"] == dataset) + & samples["part"].isin(["encoder", "other"])] + if obj.empty: + draw_pending(ax) + else: + encoder = enc.groupby(["method", "model", "sample"])["seconds"].sum() \ + .groupby(["method", "model"]).agg(["mean", "std"]) + decoder = obj.groupby(["method", "model"])["seconds"].agg(["mean", "std"]) + per_slice = obj.groupby(["method", "model"])["seconds_per_slice"].mean() + start = log_start(encoder["mean"][encoder["mean"] > 0]) + segments, end = [], 0.0 + for yi, (label, method, model) in zip(y, rows): + if (method, model) not in encoder.index or (method, model) not in decoder.index: + continue + enc_mean, enc_std = encoder.loc[(method, model)] + dec_mean, dec_std = decoder.loc[(method, model)] + mean = enc_mean + dec_mean + std = float(np.hypot(np.nan_to_num(enc_std), np.nan_to_num(dec_std))) + lower = min(std, mean - start * 1.01) + ax.errorbar(mean, yi, xerr=[[lower], [std]], color=fig2.GROUP_COLOR, linewidth=0.6, + capsize=1.2, zorder=3) + segments.append((yi, start, [enc_mean, dec_mean], ["encoder", "decoder"], "micro-sam2", + mean - lower)) + write_total(ax, mean + std, yi, mean, device, dataset, + per_slice.loc[(method, model)] if dataset in N_SLICES else None) + end = max(end, mean + std) + set_log_axis(ax, start, end, BAR_SHARE_3D if dataset in N_SLICES else BAR_SHARE_2D) + draw_segments(ax, segments, True, minutes=(device, dataset) in MINUTES_PANELS, + seconds=(device, dataset) in SECONDS_PANELS) + style_axes(ax) + ax.set_ylim(y.min() - 0.6, y.max() + 0.6) + ax.set_yticks(y) + ax.set_yticklabels([row[0] for row in rows] if j == 0 else [], fontsize=fig2.SMALL_FONT_SIZE) + label_grid(fig, axes) + bottom_box = axes[-1, 0].get_position() + fig.text(0.5, bottom_box.y0 - 0.25 / height, "Time per object [s, log scale] ↓", ha="center", va="top", + fontsize=fig2.MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR) + handles = [plt.Rectangle((0, 0), 1, 1, color=tinted(METHOD_COLORS["micro-sam2"], PART_TINTS[part])) + for part in ("encoder", "decoder")] + legend = fig.legend(handles, ["Encoder", "Prompt encoder & Mask decoder"], + loc="lower center", bbox_to_anchor=(0.5, 0.0), ncol=2, fontsize=fig2.MAX_FONT_SIZE, + frameon=True, edgecolor=fig2.LEGEND_BORDER_COLOR, handlelength=1.1, + columnspacing=0.8, borderpad=0.9) + legend.get_frame().set_linewidth(0.6) + title_on_edge(fig, legend, "μSAM2") + return fig + + +PLOTS = { + "automatic_runtimes": ("automatic", "quantitative"), + "interactive_runtimes": ("interactive", "quantitative"), +} + + +def make_plot(name, region=None, compact=False): + return (plot_automatic if name == "automatic_runtimes" else plot_interactive)(region=region, compact=compact) + + +def main(): + names = list(PLOTS) + [COMBINED_NAME] + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--only", nargs="+", choices=names, default=names, help="Make only these plots.") + args = parser.parse_args() + set_style() + fig2.check_style() + fig2.OUTPUT_DIR = OUTPUT_DIR + for name, (kind, style) in PLOTS.items(): + if name in args.only: + fig2.save_figure(make_plot(name), name, kind, style) + if COMBINED_NAME in args.only: + fig2.save_combined(*fig2.plot_figure(PLOTS, make_plot), name=COMBINED_NAME) + + +if __name__ == "__main__": + main() diff --git a/scripts/plotting/v2/for_paper/plot_style.py b/scripts/plotting/v2/for_paper/plot_style.py new file mode 100644 index 000000000..7d49e360d --- /dev/null +++ b/scripts/plotting/v2/for_paper/plot_style.py @@ -0,0 +1,94 @@ +"""The shared style of all micro-sam2 paper plots: the matplotlib settings and the fixed color of each method. + +Every figure imports its colors from here, so that a method has the same color in every plot of the paper. +""" + +import os +from glob import glob + +import matplotlib.pyplot as plt +from matplotlib import font_manager + + +# The colors of the methods, after the palette of the micro-sam (v1) paper plots. micro-sam2 gets the dark teal of +# the v1 method and the white hatch that marks it, micro-sam (and PathoSAM, its histopathology model) the lighter +# teal and CellposeSAM the pink of CellPose. The SAM family goes from sand over orange to brick red, the other cell +# models are lilac and plum. The histopathology and nucleus models are slate blue (StarDist), brown and dark brown +# (CellViT, CellViT++) and olive green (InstanSeg). Each color has its own hue name, so a caption can name it, and +# every pair is at least 12 OKLab units apart for normal vision. +METHOD_COLORS = { + "StarDist": "#5966A6", + "SAM": "#FCDE9C", + "CellViT": "#8F6E56", + "micro-sam": "#089099", + "CellSAM": "#C3A4DC", + "Cellpose3": "#6E2D7E", + "SAM2": "#F39B4F", + "InstanSeg": "#6A8A28", + "CellViT++": "#3C2920", + "CellposeSAM": "#E0479B", + "SAM3": "#A8321F", + "micro-sam2": "#045275", + # The 3d-only methods: FOCUS-3D mint, SegNeuron (EM) cornflower blue and nnInteractive light blue. SegNeuron is + # not grey, which marks the rows of methods that do not apply to a modality. + "FOCUS-3D": "#7CCBA2", + "SegNeuron": "#6495ED", + "nnInteractive": "#9ECAE1", + # The EM organelle models of empanada: MitoNet violet and NucleoNet green, the hues furthest from the others. + "MitoNet": "#9E39E6", + "NucleoNet": "#30BF30", + # The feature models of the classification plots: DINOv3 indigo and UNIv2 vermilion, at least 15 OKLab units + # from all colors above. The end-to-end baselines are dark grey with a dashed line. + "DINOv3": "#1E1E9E", + "UNIv2": "#FF4500", + "End-to-end": "#52514E", + # ilastik (classical features, classification plots only) royal blue: at least 18 OKLab units from every method + # of those plots, also for protan and deutan vision, and at least 12 from every color above. + "ilastik": "#1861F2", +} +TEXT_COLOR = "#212427" + + +# Nature Methods asks for Arial or Helvetica, and for all text between 5 and 7 pt. Liberation Sans has the letter +# widths of Arial and an open license; it is read from FONT_DIR when it is there, else DejaVu Sans is used. +FONT_DIR = os.path.expanduser("~/.local/share/fonts/liberation") +MAX_FONT_SIZE = 7 + + +def set_style(): + """Set the matplotlib style that all plots of the paper share.""" + for path in glob(os.path.join(FONT_DIR, "*.ttf")): + font_manager.fontManager.addfont(path) + available = {font.name for font in font_manager.fontManager.ttflist} + plt.rcParams.update({ + "font.family": "sans-serif", + "font.sans-serif": [name for name in ("Arial", "Helvetica", "Liberation Sans") if name in available] + [ + "DejaVu Sans" + ], + "font.size": MAX_FONT_SIZE, + "axes.titlesize": MAX_FONT_SIZE, + "axes.labelsize": MAX_FONT_SIZE, + "xtick.labelsize": 6, + "ytick.labelsize": 6, + "legend.fontsize": MAX_FONT_SIZE, + "axes.linewidth": 0.6, + "axes.edgecolor": "#52514e", + "axes.labelcolor": TEXT_COLOR, + "axes.titlecolor": TEXT_COLOR, + "axes.titleweight": "bold", + "axes.labelweight": "bold", + "xtick.color": "#52514e", + "ytick.color": "#52514e", + "xtick.labelcolor": TEXT_COLOR, + "ytick.labelcolor": TEXT_COLOR, + "hatch.linewidth": 0.8, + "xtick.major.width": 0.6, + "ytick.major.width": 0.6, + "xtick.major.size": 2.5, + "ytick.major.size": 2.5, + "axes.spines.top": False, + "axes.spines.right": False, + "savefig.dpi": 300, + "svg.fonttype": "none", + "pdf.fonttype": 42, + }) diff --git a/scripts/plotting/v2/for_paper/plots_for_figure1.py b/scripts/plotting/v2/for_paper/plots_for_figure1.py new file mode 100644 index 000000000..85b9005fa --- /dev/null +++ b/scripts/plotting/v2/for_paper/plots_for_figure1.py @@ -0,0 +1,4432 @@ +"""Plots for Figure 1 of the micro-sam2 paper. + +Panel a shows the architecture of micro-sam2: a shared image encoder, the prompt encoder, memory attention and mask +decoder for interactive segmentation, and the segmentation decoder for automatic segmentation. +Panel b shows the diversity of the training data: example crops of the datasets, grouped by domain, with their +annotations. It reads the crops from the training data. +Panel c shows the tasks that micro-sam2 supports around its hub: interactive analysis (segmentation in 2D and 3D, +tracking in 2D+T, pixel and object classification, with the prompts that the tools support) on the left, and +automatic analysis (segmentation in 2D and 3D, tracking in 2D+T and 3D+T, batch processing) on the right. +Panel d shows examples from the user studies as studio renders: the objects in 3D with distinct colours for close +objects, on a light grey floor with walls and shadows (the expanded Platynereis nuclei, the MCF7 spheroid nuclei and +the cells of a heart section). The renders run off screen with pyvista on a GPU node (--render) and are saved in +panel_d of the output folder; panel d puts the examples together from them, in rows. +The panels and the whole figure are saved as svg, pdf and png. +""" + +import io +import os +import re +import glob +import json +import time +import base64 +import argparse +import colorsys +import subprocess +import importlib.util +import urllib.request +from functools import lru_cache + +import vtk +import zarr +import tifffile +import numpy as np +import pyvista as pv +import imageio.v3 as imageio +from PIL import Image, ImageDraw, ImageFont + +import matplotlib +from matplotlib.path import Path +from matplotlib import font_manager +from matplotlib import pyplot as plt +from matplotlib.patches import ConnectionPatch, Polygon + +from scipy import ndimage +from scipy.spatial.distance import cdist +from scipy.interpolate import make_interp_spline + +from skimage.color import rgb2lab +from skimage.segmentation import find_boundaries + +from elf.io import open_file + +from micro_sam.v2.transforms.labels import DirectedPerObjectBoundaryDistanceTransform + + +PNG_DPI = 600 +# the fonts that render the text: Liberation Sans has the widths of Arial (see plot_style.py) +FONT_DIR = os.path.expanduser("~/.local/share/fonts/liberation") +# the smallest and the largest text in the figure, 5 pt and 7 pt in mm, as Nature Methods asks +MIN_TEXT, MAX_TEXT = 5 * 25.4 / 72, 7 * 25.4 / 72 +# the layout of the figure in mm +FIGURE_W, LETTER_W, ROW_GAP, PANEL_GAP, LETTER_SIZE = 183.0, 2.8, 3.5, 2.0, 8 * 25.4 / 72 +# the height of the figure: an A4 page with the same margins as the width leaves (13.5 mm) +FIGURE_H = 270.0 +SIZE = 8.5 +STACK_FRAME, STACK_OFFSET = 7.4, 1.4 +ITEM_GAP = 1.5 + +BLUE = "#3d4596" +PURPLE = "#8e4c9f" +RIM = "#b49ad6" +# the object segmented from the box prompt in the 2D output of the interactive band +BOX_OBJECT = "#e0a030" +TEAL = "#38b2c4" +TEAL_DARK = "#2a8f9c" +ORANGE = "#ef6f45" +TEXT = "#212427" +SUBTEXT = "#55505e" +INTERACTIVE_BACKGROUND = "#e7dcdd" +AUTOMATIC_BACKGROUND = "#dbe3e0" +FONT = "font-family:Arial,'Liberation Sans',sans-serif" + +# a cell body among the neurites of the EM block, and its nucleus, as spheres in the unit cube +SOMA = ((0.24, 0.6, 0.05), 0.3) +NUCLEUS = ((0.21, 0.62, 0.05), 0.17) +N_AXIAL, N_LATERAL = 34, 4 +SOMA_LABEL, FEATURED_LABEL = N_AXIAL + N_LATERAL, N_AXIAL + N_LATERAL + 1 + +# the logo of micro-sam, taken from the documentation of this repository +REPOSITORY = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), *[os.pardir] * 4)) +# the μSAM2 logo (the horizontal light version, 901 x 381 px) +LOGO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "micro_sam2_logo.png") +# the official Python logo, which is downloaded once into the assets folder next to this script +PYTHON_LOGO = os.path.join(os.path.dirname(os.path.abspath(__file__)), "assets", "python_logo.svg") +PYTHON_LOGO_URL = "https://upload.wikimedia.org/wikipedia/commons/c/c3/Python-logo-notext.svg" +NAPARI_LOGO = os.path.join( + importlib.util.find_spec("napari").submodule_search_locations[0], "resources", "logos", "gradient-plain-light.svg" +) + + +def fmt(value): + return f"{value:.3f}".rstrip("0").rstrip(".") + + +@lru_cache +def font(bold): + name = "LiberationSans-Bold.ttf" if bold else "LiberationSans-Regular.ttf" + return ImageFont.truetype(os.path.join(FONT_DIR, name), 1000) + + +def text_width(text, size, bold=False): + """Measure the width of a text in mm, for a font size in mm.""" + return font(bold).getlength(text) / 1000 * size + + +def blob_points(cx, cy, r, aspect, angle, seed, n=28): + """Get the points of a slightly irregular closed outline.""" + rng = np.random.default_rng(seed) + t = np.linspace(0, 2 * np.pi, n, endpoint=False) + noise = sum(rng.normal(0, 0.07 / k) * np.sin(k * t + rng.uniform(0, 2 * np.pi)) for k in (2, 3, 4)) + rad = r * (1 + noise) + px, py = rad * np.cos(t) * aspect, rad * np.sin(t) / aspect + ca, sa = np.cos(angle), np.sin(angle) + return np.stack([cx + ca * px - sa * py, cy + sa * px + ca * py], axis=1) + + +def blob_path(cx, cy, r, aspect, angle, seed, n=28): + """Get a smooth, slightly irregular closed outline as a cubic bezier path.""" + pts = blob_points(cx, cy, r, aspect, angle, seed, n) + d = f"M{fmt(pts[0, 0])},{fmt(pts[0, 1])}" + for i in range(n): + p0, p1, p2, p3 = pts[i - 1], pts[i], pts[(i + 1) % n], pts[(i + 2) % n] + c1, c2 = p1 + (p2 - p0) / 6, p2 - (p3 - p1) / 6 + d += f"C{fmt(c1[0])},{fmt(c1[1])} {fmt(c2[0])},{fmt(c2[1])} {fmt(p2[0])},{fmt(p2[1])}" + return d + "Z" + + +class Drawing: + """Collect the defs and the elements of an svg drawing.""" + + def __init__(self, text_scale=1.0, scale=1.0): + # the text scale keeps all printed text between MIN_TEXT and MAX_TEXT + self.defs, self.body, self.count, self.text_scale, self.scale = [], [], 0, text_scale, scale + + def uid(self, prefix): + self.count += 1 + return f"f1c-{prefix}-{self.count}" + + def add(self, element): + self.body.append(element) + + def text_size(self, size): + """Get the size that a text of this size gets: enlarged by the text scale, and between MIN_TEXT and MAX_TEXT in + print.""" + size = size * self.text_scale if size >= 1.5 else size + return min(max(size, MIN_TEXT / self.scale), MAX_TEXT / self.scale) + + def text(self, x, y, s, size, weight="normal", color=TEXT, anchor="middle", rotate=None): + size = self.text_size(size) + transform = f' transform="rotate({rotate},{fmt(x)},{fmt(y)})"' if rotate is not None else "" + self.add( + f'{s}' + ) + + def linear_gradient(self, x1, y1, x2, y2, start=BLUE, stop=PURPLE): + gid = self.uid("grad") + self.defs.append( + f'' + f'' + ) + return gid + + def clip_rect(self, x, y, w, h, rx): + cid = self.uid("clip") + self.defs.append( + f'' + ) + return cid + + +def shared_defs(): + return ( + '' + '' + '' + '' + f'' + f'' + f'' + '' + '' + '' + '' + '' + '' + '' + '' + '' + ) + + +def point_prompt(drawing, x, y, positive=True, r=0.75): + color = TEAL if positive else ORANGE + s = r * 0.5 + sign = f"M{fmt(x - s)},{fmt(y)}H{fmt(x + s)}" + if positive: + sign += f"M{fmt(x)},{fmt(y - s)}V{fmt(y + s)}" + drawing.add( + f'' + f'' + ) + + +def in_sphere(x, y, z, sphere): + (cx, cy, cz), r = sphere + return (x - cx) ** 2 + (y - cy) ** 2 + (z - cz) ** 2 < r ** 2 + + +def neurite_labels(x, y, z, seed, n_axial=N_AXIAL, n_lateral=N_LATERAL, soma=None, featured=False): + """Label points of the unit cube by their closest neurite; most neurites run along z, a few thin ones along x. + + A soma, given as a sphere, gets the label after the last neurite. The featured neurite starts near the center of + the front face and bends up to run just below the top face, so that it trails along the top face. + """ + rng = np.random.default_rng(seed) + # a smooth warp of space makes the membranes wavy + warped = [] + for coordinate in (x, y, z): + offset = 0.0 + for _ in range(3): + f, phase = rng.normal(0, 7, 3), rng.uniform(0, 2 * np.pi) + offset = offset + 0.018 * np.sin(f[0] * x + f[1] * y + f[2] * z + phase) + warped.append(coordinate + offset) + x, y, z = warped + distances = [] + for _ in range(n_axial): + c, v, w = rng.uniform(-0.1, 1.1, 2), rng.normal(0, 0.25, 2), rng.uniform(0, 0.05) + distances.append(np.hypot(x - c[0] - v[0] * (z - 0.5), y - c[1] - v[1] * (z - 0.5)) - w) + for _ in range(n_lateral): + c, v = rng.uniform(0.1, 0.9, 2), rng.normal(0, 0.15, 2) + distances.append(1.8 * np.hypot(y - c[0] - v[0] * (x - 0.5), z - c[1] - v[1] * (x - 0.5))) + if featured: + distances.append(np.full_like(x, np.inf)) + center_x, center_y = 0.62 + 0.1 * z, 0.0 + 0.48 * np.exp(-z / 0.16) + distances.append(np.hypot(x - center_x, y - center_y) - 0.12) + labels = np.argmin(distances, axis=0) + if soma is not None: + labels = np.where(in_sphere(x, y, z, soma), n_axial + n_lateral, labels) + return labels + + +def face_coordinates(face, n): + """Get the cube coordinates of the pixels of a visible face; u runs along the columns and v along the rows.""" + u, v = np.meshgrid(np.linspace(0, 1, n), np.linspace(0, 1, n)) + if face == "front": + return u, v, np.zeros_like(u) + if face == "top": + return u, np.zeros_like(u), 1 - v + return np.ones_like(u), v, u + + +def membranes(labels): + boundary = np.zeros(labels.shape, dtype=bool) + boundary[:, 1:] |= labels[:, 1:] != labels[:, :-1] + boundary[1:, :] |= labels[1:, :] != labels[:-1, :] + return ndimage.binary_dilation(boundary, iterations=1) + + +def stamp(canvas, points, radius): + """Mark the pixels around a dense list of (row, col) points.""" + n = canvas.shape[0] + for r, c in points: + r0, r1 = int(max(r - radius, 0)), int(min(r + radius + 1, n)) + c0, c1 = int(max(c - radius, 0)), int(min(c + radius + 1, n)) + canvas[r0:r1, c0:c1] = True + + +def arc_points(center, radius, start, span, rng, wobble=1.2, step=0.4): + angles = np.arange(start, start + span, step / radius) + rad = radius + wobble * np.sin(angles * rng.uniform(5, 9) + rng.uniform(0, 2 * np.pi)) + return np.stack([center[0] + rad * np.sin(angles), center[1] + rad * np.cos(angles)], axis=1) + + +def cell_body_organelles(image, rng, soma, nucleus): + """Draw the nucleus, rough ER, a Golgi stack and lysosomes of a cell body into an EM section.""" + n = image.shape[0] + rows, cols = np.mgrid[:n, :n] + if nucleus.any(): + depth = ndimage.distance_transform_edt(nucleus) + chromatin = ndimage.gaussian_filter(rng.normal(0, 1, image.shape), 2.0) + chromatin /= chromatin.std() + image[nucleus] = 0.56 + 0.05 * chromatin[nucleus] + # heterochromatin lines the envelope in patches and forms a few clumps inside + image[nucleus & (((depth < 5) & (chromatin > -0.3)) | (chromatin > 1.6))] = 0.34 + image[nucleus & (depth <= 1.4)] = 0.2 + r0, c0 = np.unravel_index(np.argmax(depth), depth.shape) + nucleolus = np.hypot(rows - r0 - 0.2 * depth.max(), cols - c0 + 0.25 * depth.max()) < 0.28 * depth.max() + image[nucleolus] = np.where(chromatin[nucleolus] > 0, 0.22, 0.3) + center, radius = (rows[nucleus].mean(), cols[nucleus].mean()), np.sqrt(nucleus.sum() / np.pi) + else: + center, radius = (rows[soma].mean(), cols[soma].mean()), 0.3 * np.sqrt(soma.sum() / np.pi) + cytoplasm = soma & ~ndimage.binary_dilation(nucleus, iterations=3) & ndimage.binary_erosion(soma, iterations=3) + image[cytoplasm] -= 0.05 + er, ribosomes = np.zeros(image.shape, dtype=bool), np.zeros(image.shape, dtype=bool) + for _ in range(9): + r, start, span = radius + rng.uniform(7, 30), rng.uniform(0, 2 * np.pi), rng.uniform(0.4, 0.9) + for offset in (0.0, 2.6): + points = arc_points(center, r + offset, start, span, rng) + stamp(er, points, 1) + stamp(ribosomes, points[rng.random(len(points)) < 0.15] + rng.normal(0, 1.8, 2), 1) + image[ribosomes & cytoplasm] = 0.3 + image[er & cytoplasm] = 0.24 + golgi, start = np.zeros(image.shape, dtype=bool), rng.uniform(0, 2 * np.pi) + for k in range(5): + stamp(golgi, arc_points(center, radius + 12 + 3.2 * k, start, 0.6 - 0.06 * k, rng, wobble=0.3), 1) + image[ndimage.binary_dilation(golgi) & cytoplasm] = 0.52 + image[golgi & cytoplasm] = 0.24 + for k in (-1, 1): + angle = start + (0.3 + 0.12 * k) + k * 0.35 + for d in range(3): + distance = radius + 14 + 3 * d + pr, pc = center[0] + distance * np.sin(angle), center[1] + distance * np.cos(angle) + image[(np.hypot(rows - pr, cols - pc) < 2.0) & cytoplasm] = 0.28 + candidates = np.argwhere(ndimage.binary_erosion(cytoplasm, iterations=5)) + for r, c in candidates[rng.integers(len(candidates), size=3)] if len(candidates) else []: + lysosome = np.hypot(rows - r, cols - c) < rng.uniform(3, 4.5) + image[lysosome] = 0.28 + image[lysosome & (rng.random(image.shape) < 0.15)] = 0.5 + return image + + +def em_texture(labels, boundary, seed, n_mitochondria=9, soma=None, nucleus=None, mitochondria=None, vesicles=None): + """Simulate an EM section: neurite profiles with dark membranes, a cell body, mitochondria, vesicles and grain. + + The mitochondria and the clusters of vesicles are placed at random, unless masks of their sections are given. + """ + rng = np.random.default_rng(seed) + n = labels.shape[0] + image = rng.uniform(0.6, 0.74, labels.max() + 1)[labels] + nucleus = np.zeros(labels.shape, dtype=bool) if nucleus is None else nucleus + if soma is not None and soma.any(): + image = cell_body_organelles(image, rng, soma, nucleus) + rows, cols = np.mgrid[:n, :n] + if mitochondria is not None: + mito = mitochondria & ~boundary + cristae = np.sin((0.8 * rows + 0.6 * cols) * 1.3) > 0.55 + image[mito] = np.where(cristae[mito], 0.3, 0.42) + image[mito & ~ndimage.binary_erosion(mito, iterations=1)] = 0.22 + n_mitochondria = 0 + for _ in range(n_mitochondria): + cy, cx = rng.integers(10, n - 10, 2) + a, b, angle = rng.uniform(7, 15), rng.uniform(3, 5.5), rng.uniform(0, np.pi) + du, dv = cols - cx, rows - cy + ru, rv = du * np.cos(angle) + dv * np.sin(angle), -du * np.sin(angle) + dv * np.cos(angle) + mito = ((ru / a) ** 2 + (rv / b) ** 2 < 1) & (labels == labels[cy, cx]) & ~boundary + mito &= ~ndimage.binary_dilation(nucleus, iterations=3) + cristae = np.sin(ru * 1.3) > 0.55 + image[mito] = np.where(cristae[mito], 0.3, 0.42) + image[mito & ~ndimage.binary_erosion(mito, iterations=1)] = 0.22 + if vesicles is not None: + image[vesicles & ~boundary] = 0.3 + for _ in range(0 if vesicles is not None else 7): + center = rng.integers(8, n - 8, 2) + for dy, dx in rng.normal(0, 3.0, (8, 2)): + image[np.hypot(rows - center[0] - dy, cols - center[1] - dx) < 1.3] = 0.34 + image[boundary] = 0.2 + image = ndimage.gaussian_filter(image, 0.9) + rng.normal(0, 0.045, image.shape) + image += 0.05 * ndimage.gaussian_filter(rng.normal(0, 1, image.shape), n / 8) * n / 8 + return np.clip(ndimage.gaussian_filter(image, 0.5), 0, 1) + + +def hex_to_rgb(color): + return np.array([int(color[i:i + 2], 16) for i in (1, 3, 5)]) / 255.0 + + +# the longer side the images are upsampled to, so that they do not look pixelated in print +IMAGE_SIZE = 360 +JPEG_QUALITY = 92 + + +def upscale(rgb, factor, step=0.25): + """Upsample an rgb image by an integer factor without pixel steps. + + Strong color steps, like the edges of masks and labels, are taken from a median filtered nearest neighbour upscale, + which rounds their staircase into smooth contours. Everything else, like the texture of the images, is taken from a + cubic upscale. Steps in any channel larger than 'step' count as strong. + """ + rgb = np.clip(np.asarray(rgb, dtype=float), 0, 1) + cubic = np.clip(ndimage.zoom(rgb, (factor, factor, 1), order=3), 0, 1) + smooth = np.repeat(np.repeat(rgb, factor, 0), factor, 1) + for _ in range(3): + smooth = ndimage.median_filter(smooth, size=(factor + 1, factor + 1, 1)) + contrast = np.max([ + ndimage.maximum_filter(rgb[..., c], 2) - ndimage.minimum_filter(rgb[..., c], 2) for c in range(3) + ], axis=0) + weight = ndimage.maximum_filter(np.clip((contrast - step) / step, 0, 1), 3) + weight = np.clip(ndimage.zoom(weight, factor, order=1), 0, 1)[..., None] + return weight * smooth + (1 - weight) * cubic + + +def array_uri(rgb): + buffer = io.BytesIO() + rgb = np.clip(rgb, 0, 1) + factor = int(np.ceil(IMAGE_SIZE / max(rgb.shape[:2]))) + # the median filter is slow at large factors: Lanczos does the rest + if factor > 1: + rgb = upscale(rgb, min(factor, 4)) + image = Image.fromarray((rgb * 255).astype("uint8")) + if factor > 4: + size = (rgb.shape[1] * factor // 4, rgb.shape[0] * factor // 4) + image = image.resize(size, Image.LANCZOS) + # the images have no transparency, so a high quality jpeg keeps the files small + image.save(buffer, format="JPEG", quality=JPEG_QUALITY, subsampling=0) + return "data:image/jpeg;base64," + base64.b64encode(buffer.getvalue()).decode() + + +def em_faces(seed, overlay, n=220): + """Render the three visible faces of a simulated EM block; overlay maps the labels and u, v to colors.""" + faces = {} + for k, (face, shade) in enumerate((("front", 1.0), ("top", 1.12), ("side", 0.82))): + x, y, z = face_coordinates(face, n) + labels = neurite_labels(x, y, z, seed, soma=SOMA, featured=True) + boundary = membranes(labels) + soma, nucleus = labels == SOMA_LABEL, in_sphere(x, y, z, NUCLEUS) + gray = em_texture(labels, boundary, seed + 17 * (k + 1), n_mitochondria=12, soma=soma, nucleus=nucleus) + rgb = np.repeat(gray[..., None], 3, axis=-1) + u, v = np.meshgrid(np.linspace(0, 1, n), np.linspace(0, 1, n)) + color, alpha = overlay(labels, u, v) + alpha = np.where(boundary, alpha * 0.3, alpha)[..., None] + rgb = (1 - alpha) * rgb + alpha * color * (0.55 + 0.6 * gray[..., None]) + faces[face] = (rgb * shade, labels) + return faces + + +def em_block(drawing, x, y, faces, front=8.5, depth=3.5): + """Draw an oblique EM block whose back top edge starts at (x + depth, y).""" + fx, fy, d = x, y + depth, depth + matrices = { + "front": (front, 0, 0, front, fx, fy), "top": (front, 0, -d, d, fx + d, fy - d), + "side": (d, -d, 0, front, fx + front, fy), + } + for face, matrix in matrices.items(): + drawing.add( + f'' + ) + outline = ( + f"M{fmt(fx)},{fmt(fy)}h{fmt(front)}v{fmt(front)}h{fmt(-front)}Z" + f"M{fmt(fx)},{fmt(fy)}l{fmt(d)},{fmt(-d)}h{fmt(front)}l{fmt(-d)},{fmt(d)}" + f"M{fmt(fx + front + d)},{fmt(fy - d)}v{fmt(front)}l{fmt(-d)},{fmt(d)}" + ) + drawing.add(f'') + return fx, fy + + +def neurite_colors(n_labels): + palette = [ + "#4f5bd5", "#9b59b6", "#38b2c4", "#e0679b", "#7fc97f", "#f0a35e", "#6c8ebf", "#c77dd6", "#2a9d8f", + "#d4a5e8", "#5fb3e8", "#e8c547", "#8e4c9f", "#57c49a", + ] + return np.array([hex_to_rgb(palette[k % len(palette)]) for k in range(n_labels)]) + + +EM_SEED = 9 +# the mother cell (index 3) is blue before the division, its daughters (indices 3 and 4) are pink and orange +TRACK_COLORS = {0: "f1c-mask", 1: "f1c-mask-teal", 2: "f1c-mask-gold"} +MOTHER_COLOR, DAUGHTER_COLORS = "f1c-mask-blue", ("f1c-mask-pink", "f1c-mask-orange") + + +def lineage_colors(t, tracked): + """Get the mask fill of each tracked cell in frame t; frame 0 is the last time point, after the division.""" + colors = {k: TRACK_COLORS[k] for k in tracked if k in TRACK_COLORS} + if t == 0: + colors.update({3: DAUGHTER_COLORS[0], 4: DAUGHTER_COLORS[1]}) + else: + colors[3] = MOTHER_COLOR + return colors + + +def frames_2dt(): + """Get the nuclei of the 2D+T example, front to back; the front frame is the last time point.""" + frames = [] + for t in range(3): + if t == 0: + cells = [ + (0.3, 0.3, 0.14, 1.1, 0.6), (0.78, 0.3, 0.13, 1.05, 2.0), (0.25, 0.8, 0.13, 1.1, 1.4), + (0.62, 0.68, 0.1, 1.1, 0.4), (0.84, 0.86, 0.1, 1.1, 0.9), + ] + else: + # the mother nucleus condenses and elongates before it divides + cells = [ + (0.3 - 0.05 * t, 0.3 + 0.04 * t, 0.14, 1.1, 0.6), (0.78, 0.3 - 0.05 * t, 0.13, 1.05, 2.0), + (0.25 + 0.04 * t, 0.8, 0.13, 1.1, 1.4), (0.72, 0.76, 0.13, 1.1 + 0.2 * (t == 1), 0.7), + ] + frames.append(cells) + return frames + + +# the label-free 2D example of panel c, as LIVECell: wild cells as (x, y, type) in unit coordinates +LIVECELL_CELLS = [ + (0.15, 0.17, "spread"), (0.46, 0.12, "spiky"), (0.8, 0.14, "neuron"), (0.12, 0.52, "spiky"), + (0.4, 0.44, "spread"), (0.7, 0.45, "spiky"), (0.93, 0.52, "spindle"), (0.22, 0.83, "neuron"), + (0.5, 0.76, "spiky"), (0.77, 0.8, "spread"), (0.96, 0.95, "spiky"), (0.55, 0.98, "spindle"), +] +# the cells of the box prompt, of the positive point and of the negative point +LIVECELL_BOX, LIVECELL_POINT, LIVECELL_NEGATIVE = 5, 7, 8 + + +def stamp_disks(labels, label, points, radii): + """Mark the pixels within the radii of points in unit coordinates, without overwriting other cells.""" + n = labels.shape[0] + for (px, py), r in zip(np.asarray(points) * n, np.asarray(radii) * n): + r = max(r, 0.7) + r0, r1 = int(max(py - r, 0)), int(min(py + r + 1, n)) + c0, c1 = int(max(px - r, 0)), int(min(px + r + 1, n)) + if r1 <= r0 or c1 <= c0: + continue + rows, cols = np.mgrid[r0:r1, c0:c1] + block = labels[r0:r1, c0:c1] + block[(np.hypot(cols - px, rows - py) <= r) & (block == 0)] = label + + +def branch(rng, start, heading, length, radius, depth, out): + """Grow a process with a wandering path and a tapering radius, which may fork into two thinner branches.""" + steps = int(length * 500) + 8 + turns = np.cumsum(rng.normal(0, 0.09, steps)) + angles = heading + turns + points = start + np.cumsum(np.stack([np.cos(angles), np.sin(angles)], axis=1) * length / steps, axis=0) + radii = np.linspace(radius, 0.4 * radius, steps) + out.append((points, radii)) + if depth > 0 and rng.uniform() < 0.85: + for side in (-1, 1): + spread = side * rng.uniform(0.35, 0.8) + branch(rng, points[-1], angles[-1] + spread, 0.6 * length, 0.55 * radius, depth - 1, out) + elif depth == 0 and rng.uniform() < 0.5: + # a ruffled lamellipodium at the tip + tip = points[-1] + out.append((tip + rng.normal(0, 0.008, (6, 2)), np.full(6, 2.2 * radius))) + + +def wild_cell(rng, cx, cy, kind): + """Describe one cell by the outline of its body, its nucleus and the strokes of its processes.""" + strokes = [] + if kind == "spindle": + angle = rng.uniform(0, np.pi) + t = np.linspace(-1, 1, 200)[:, None] + axis = np.array([np.cos(angle), np.sin(angle)]) + bend = np.array([-axis[1], axis[0]]) * 0.03 * np.sin(np.pi * t) + points = np.array([cx, cy]) + 0.16 * t * axis + bend + strokes.append((points, 0.04 * (1 - np.abs(t[:, 0])) ** 0.7 + 0.003)) + body = (cx, cy, 0.03, 1.6, angle) + tips = [points[0], points[-1]] + spikes = [(tip, angle + np.pi * (k == 0)) for k, tip in enumerate(tips)] + else: + size = {"spread": 0.075, "spiky": 0.05, "neuron": 0.04}[kind] + aspect = rng.uniform(1.0, 1.5) if kind != "spiky" else rng.uniform(1.0, 1.25) + body = (cx, cy, size, aspect, rng.uniform(0, np.pi)) + spikes = [] + if kind == "spread": + for heading in rng.uniform(0, 2 * np.pi, rng.integers(3, 6)): + start = np.array([cx, cy]) + 0.8 * size * np.array([np.cos(heading), np.sin(heading)]) + branch(rng, start, heading, rng.uniform(0.08, 0.16), 0.014, 2, strokes) + elif kind == "neuron": + for heading in rng.uniform(0, 2 * np.pi, rng.integers(2, 4)): + start = np.array([cx, cy]) + 0.8 * size * np.array([np.cos(heading), np.sin(heading)]) + branch(rng, start, heading, rng.uniform(0.18, 0.3), 0.007, 1, strokes) + else: + spikes = [(None, heading) for heading in rng.uniform(0, 2 * np.pi, rng.integers(14, 26))] + # thin straight filopodia, around the whole body or at the tips of a spindle + for tip, heading in spikes: + count = 1 if tip is None else 4 + for _ in range(count): + angle = heading + (0.0 if tip is None else rng.normal(0, 0.5)) + base = tip if tip is not None else np.array([cx, cy]) + 0.85 * body[2] * np.array( + [np.cos(angle), np.sin(angle)] + ) + length = rng.uniform(0.02, 0.065) + t = np.linspace(0, 1, 30)[:, None] + strokes.append((base + t * length * np.array([np.cos(angle), np.sin(angle)]), np.full(30, 0.0035))) + nucleus = (cx, cy, 0.5 * body[2], 1.25, body[4]) + return body, nucleus, strokes + + +@lru_cache +def livecell_scene(n=260): + """Label the cells of the label free 2D example and render them as a phase contrast image.""" + rng = np.random.default_rng(EM_SEED + 7) + labels = np.zeros((n, n), dtype="uint32") + rows, cols = np.mgrid[:n, :n] + grid = np.stack([cols.ravel() + 0.5, rows.ravel() + 0.5], axis=1) / n + cells = [wild_cell(rng, cx, cy, kind) for cx, cy, kind in LIVECELL_CELLS] + nuclei = np.zeros((n, n), dtype=bool) + # the bodies go first, so that processes grow around their neighbours + for label, (body, nucleus, _) in enumerate(cells, start=1): + outline = blob_points(*body, seed=label, n=40) + mask = Path(outline).contains_points(grid).reshape(n, n) + labels[mask & (labels == 0)] = label + nuclei |= Path(blob_points(*nucleus, seed=label + 50)).contains_points(grid).reshape(n, n) + for label, (_, _, strokes) in enumerate(cells, start=1): + for points, radii in strokes: + stamp_disks(labels, label, points, radii) + nuclei &= labels > 0 + # phase contrast: grainy cells with a dark rim and a bright halo + inside = labels > 0 + image = 0.58 + 0.03 * ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), 12) * 12 + grain = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), 1.2) + image = np.where(inside, 0.4 + 0.12 * grain, image) + image[nuclei] = 0.33 + 0.05 * grain[nuclei] + spots = ndimage.binary_erosion(nuclei, iterations=3) & (rng.uniform(size=(n, n)) < 0.012) + image[ndimage.binary_dilation(spots, iterations=1)] = 0.62 + halo = ndimage.binary_dilation(inside, iterations=4) & ~inside + image = image + 0.42 * ndimage.gaussian_filter(halo.astype(float), 1.6) + image[inside & ~ndimage.binary_erosion(inside, iterations=1)] = 0.2 + image = ndimage.gaussian_filter(image, 0.6) + rng.normal(0, 0.02, (n, n)) + gray = np.clip(image, 0, 1) + return labels, np.repeat(gray[..., None], 3, axis=-1) + + +def livecell_tile(drawing, x, y, rgb): + clip = drawing.clip_rect(x, y, SIZE, SIZE, 0.9) + drawing.add( + f'' + ) + + +def interactive_2d(drawing, x, cy): + """A spiky cell segmented from a box prompt, a neuron-like cell with its neurites from a positive point prompt, + and a negative point prompt on the neighbouring cell.""" + y = cy - SIZE / 2 + labels, image = livecell_scene() + n = labels.shape[0] + box, point = LIVECELL_BOX + 1, LIVECELL_POINT + 1 + livecell_tile(drawing, x, y, overlay(image, [labels == box, labels == point])) + rows, cols = np.nonzero(labels == box) + x0, y0 = x + cols.min() / n * SIZE - 0.3, y + rows.min() / n * SIZE - 0.3 + x1, y1 = x + cols.max() / n * SIZE + 0.3, y + rows.max() / n * SIZE + 0.3 + drawing.add( + f'' + ) + cx, cy_point, _ = LIVECELL_CELLS[LIVECELL_POINT] + point_prompt(drawing, x + cx * SIZE, y + cy_point * SIZE, r=0.5) + cx, cy_point, _ = LIVECELL_CELLS[LIVECELL_NEGATIVE] + point_prompt(drawing, x + cx * SIZE, y + cy_point * SIZE, positive=False, r=0.5) + + +def interactive_3d(drawing, x, cy, scribble=True, masked=True): + """A neuron segmented from a point prompt, and optionally the larger cell body segmented from a scribble.""" + blue, purple = hex_to_rgb(BLUE), hex_to_rgb(PURPLE) + teal_dark, teal_light = hex_to_rgb(TEAL_DARK), hex_to_rgb("#7fd0c4") + + def overlay(labels, u, v): + g = ((u + v) / 2)[..., None] + neuron, soma = (1 - g) * blue + g * purple, (1 - g) * teal_dark + g * teal_light + color = np.where((labels == SOMA_LABEL)[..., None], soma, neuron) + segmented = (labels == FEATURED_LABEL) | ((labels == SOMA_LABEL) & scribble) + return color, np.where(segmented & masked, 0.75, 0.0) + + faces = em_faces(seed=EM_SEED, overlay=overlay) + fx, fy = em_block(drawing, x, cy - (SIZE + 3.5) / 2, faces) + labels = faces["front"][1] + n = labels.shape[0] + rows, cols = np.nonzero(labels == FEATURED_LABEL) + point_prompt(drawing, fx + cols.mean() / n * SIZE, fy + rows.mean() / n * SIZE, r=0.62) + if not scribble: + return + region = ndimage.binary_erosion(labels == SOMA_LABEL, iterations=int(0.06 * n)) + points = freehand_stroke(region, np.random.default_rng(2), length=int(0.45 * n)) + if points is not None: + xs, ys = fx + points[:, 1] / n * SIZE, fy + points[:, 0] / n * SIZE + brush_stroke(drawing, xs, ys, "#ffffff", width=0.46) + brush_stroke(drawing, xs, ys, TEAL, width=0.3) + + +def frame_stack(drawing, x, cy, frames, seed): + """Draw frames as a stack; frames go from front to back and hold (cells, masked, mask fills). + + Returns the top left corner of each frame. + """ + size, offset = STACK_FRAME, STACK_OFFSET + top = cy - (size + 2 * offset) / 2 + origins = [(x + offset * t, top + offset * (len(frames) - 1 - t)) for t in range(len(frames))] + for t in reversed(range(len(frames))): + cells, masked, fills = frames[t] + nucleus_frame(drawing, *origins[t], size, cells, masked, fills, seed + t, mitotic=(t == 1,)) + return origins + + +def draw_tracks(drawing, tracks, split=None): + """Draw tracks through time on top of the frames; tracks are (points, color) and split marks a division.""" + parts = [] + for points, color in tracks: + path = "M" + "L".join(f"{fmt(px)},{fmt(py)}" for px, py in points) + parts.append( + f'' + f'' + ) + for points, color in tracks: + for px, py in points: + parts.append( + f'' + ) + if split is not None: + parts.append( + f'' + ) + drawing.add("".join(parts)) + + +def tracks_2dt(origins, tracked): + """Get the tracks of the 2D+T example through the frames, oldest first, and the point of the division.""" + frames, s = frames_2dt(), STACK_FRAME + + def position(t, k): + return origins[t][0] + frames[t][k][0] * s, origins[t][1] + frames[t][k][1] * s + + tracks = [([position(t, k) for t in (2, 1, 0)], MASK_EDGES[TRACK_COLORS[k]]) for k in tracked] + split = position(1, 3) + tracks.append(([position(2, 3), split], MASK_EDGES[MOTHER_COLOR])) + for k, color in zip((3, 4), DAUGHTER_COLORS): + tracks.append(([split, position(0, k)], MASK_EDGES[color])) + return tracks, split + + +def interactive_2dt(drawing, x, cy): + # a prompted nucleus is followed through time, and a prompted mother nucleus is followed through its division + frames = [] + for t, cells in enumerate(frames_2dt()): + colors = lineage_colors(t, tracked=(1,)) + frames.append((cells, list(colors), colors)) + origins = frame_stack(drawing, x, cy, frames, seed=81) + draw_tracks(drawing, *tracks_2dt(origins, tracked=(1,))) + fx, fy = origins[0] + point_prompt(drawing, fx + 0.78 * STACK_FRAME, fy + 0.3 * STACK_FRAME, r=0.55) + + +def split_line(drawing, x, y): + drawing.add( + f'' + ) + + +def freehand_stroke(region, rng, length, step=1.0, tries=300): + """Simulate a hand drawn stroke: the longest gently curved arc with a little jitter that stays inside a region.""" + candidates = np.argwhere(region) + if len(candidates) < 20: + return None + best = None + for _ in range(tries): + point = candidates[rng.integers(len(candidates))].astype(float) + heading, curvature = rng.uniform(0, 2 * np.pi), rng.normal(0, 0.012) + points = [point] + for _ in range(length): + heading += curvature + point = point + step * np.array([np.sin(heading), np.cos(heading)]) + r, c = int(round(point[0])), int(round(point[1])) + if not (0 <= r < region.shape[0] and 0 <= c < region.shape[1] and region[r, c]): + break + points.append(point) + if best is None or len(points) > len(best): + best = points + if len(best) < 10: + return None + jitter = ndimage.gaussian_filter1d(rng.normal(0, 1.2, (len(best), 2)), 4, axis=0) + return np.array(best) + jitter + + +def brush_stroke(drawing, xs, ys, color, width=0.24): + path = "M" + "L".join(f"{fmt(px)},{fmt(py)}" for px, py in zip(xs[::2], ys[::2])) + drawing.add( + f'' + ) + + +def stamp_nuclei(image, mask, rng, centers, size, elongation, angles=None): + """Stamp hematoxylin stained nuclei with some chromatin texture at the given centers inside a mask.""" + n = image.shape[0] + for k, (r, c) in enumerate(centers): + a = rng.uniform(*size) + angle = rng.uniform(0, np.pi) if angles is None else angles[k] + extent = int(np.ceil(a * elongation)) + 2 + r0, r1 = max(int(r) - extent, 0), min(int(r) + extent, n) + c0, c1 = max(int(c) - extent, 0), min(int(c) + extent, n) + rows, cols = np.mgrid[r0:r1, c0:c1] + du, dv = cols - c, rows - r + ru, rv = du * np.cos(angle) + dv * np.sin(angle), -du * np.sin(angle) + dv * np.cos(angle) + nucleus = ((ru / (a * elongation)) ** 2 + (rv / a) ** 2 < 1) & mask[r0:r1, c0:c1] + tone = rng.uniform(0.75, 1.15) * np.array([0.36, 0.25, 0.55]) + speckle = 1 + 0.25 * rng.normal(0, 1, nucleus.shape) + patch = image[r0:r1, c0:c1] + patch[nucleus] = np.clip(tone[None] * speckle[nucleus][:, None], 0, 1) + + +def he_tissue(seed, n=300): + """Simulate an H&E stained tissue patch; returns the image and the classes (0 stroma, 1 tumor, 2 fat).""" + rng = np.random.default_rng(seed) + rows, cols = np.mgrid[:n, :n] + field = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), n / 12) + tumor = field > 0.35 * field.std() + fat_field = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), n / 10) + fat = (fat_field > 0.5 * fat_field.std()) & ~ndimage.binary_dilation(tumor, iterations=8) + classes = np.where(tumor, 1, np.where(fat, 2, 0)) + + # eosin: stroma with wavy collagen fibres, tumor with paler, granular cytoplasm + fibres = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), (0.7, 5)) + fibres = ndimage.map_coordinates( + fibres, [rows + 6 * np.sin(cols / 23.0), cols + 4 * np.sin(rows / 31.0)], order=1, mode="reflect" + ) + fibres /= fibres.std() + stroma = np.array([0.94, 0.62, 0.76])[None, None] * (1 + 0.08 * fibres[..., None]) + granules = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), 1.2) + cytoplasm = np.array([0.86, 0.62, 0.8])[None, None] * (1 + 0.6 * granules[..., None]) + image = np.where(tumor[..., None], cytoplasm, stroma) + + # tumor glands: a lumen lined by a ring of crowded, polarized nuclei + lumens = np.zeros((n, n), dtype=bool) + candidates = np.argwhere(ndimage.binary_erosion(tumor, iterations=18)) + ring_centers, ring_angles = [], [] + for r, c in candidates[rng.integers(len(candidates), size=min(6, len(candidates)))] if len(candidates) else []: + a, b = rng.uniform(5, 10), rng.uniform(4, 7) + lumen = ((rows - r) / b) ** 2 + ((cols - c) / a) ** 2 < 1 + lumens |= lumen + for t in np.arange(0, 2 * np.pi, 0.5): + ring_centers.append((r + (b + 3.2) * np.sin(t), c + (a + 3.2) * np.cos(t))) + ring_angles.append(t) + image[lumens] = [0.97, 0.93, 0.96] + stamp_nuclei(image, tumor & ~lumens, rng, ring_centers, (1.8, 2.4), 1.5, ring_angles) + tumor_points = np.argwhere(tumor & ~ndimage.binary_dilation(lumens, iterations=6)) + stamp_nuclei(image, tumor, rng, tumor_points[rng.integers(len(tumor_points), size=330)], (1.8, 3.0), 1.2) + stroma_points = np.argwhere(classes == 0) + stamp_nuclei(image, classes == 0, rng, stroma_points[rng.integers(len(stroma_points), size=70)], (0.8, 1.2), 3.2) + + # adipocytes: large empty cells with thin pink walls + seeds = rng.uniform(0, n, (55, 2)) + nearest = np.argmin([np.hypot(rows - r, cols - c) for r, c in seeds], axis=0) + walls = membranes(nearest) + image[fat] = [0.98, 0.96, 0.98] + image[fat & walls] = [0.9, 0.6, 0.75] + + # slide appearance: slight blur, uneven staining and noise + stain = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), n / 6) + image = ndimage.gaussian_filter(image, (0.6, 0.6, 0)) * (1 + 0.6 * stain[..., None]) + image += rng.normal(0, 0.02, image.shape) + return np.clip(image, 0, 1), classes + + +def pixel_classification(drawing, x, cy, n=300): + """Tissue classification in a histopathology slide: scribbles on the left, the predicted tissue map on the right.""" + y = cy - SIZE / 2 + image, classes = he_tissue(seed=12, n=n) + colors = np.array([hex_to_rgb(TEAL), hex_to_rgb(ORANGE), hex_to_rgb("#e8c547")]) + # the prediction is a light overlay with outlined classes, so that both halves show the same tissue + outlines = ndimage.binary_dilation(membranes(classes), iterations=2)[..., None] + predicted = 0.65 * image + 0.35 * colors[classes] + predicted = np.where(outlines, colors[classes], predicted) + image[:, n // 2:] = predicted[:, n // 2:] + clip = drawing.clip_rect(x, y, SIZE, SIZE, 0.9) + drawing.add( + f'' + ) + split_line(drawing, x, y) + # freehand brush strokes for each tissue class, inside its region in the left half + rng = np.random.default_rng(4) + left = np.zeros(classes.shape, dtype=bool) + left[int(0.05 * n):int(0.95 * n), int(0.05 * n):int(0.45 * n)] = True + for k, color, n_strokes in ((0, TEAL, 1), (1, ORANGE, 1), (2, "#d9a520", 1)): + region = ndimage.binary_erosion(classes == k, iterations=6) & left + for _ in range(n_strokes): + points = freehand_stroke(region, rng, length=int(0.3 * n)) + if points is not None: + brush_stroke(drawing, x + points[:, 1] / n * SIZE, y + points[:, 0] / n * SIZE, color) + + +NUCLEUS_CLASSES = {"tumor": ORANGE, "lymphocyte": PURPLE, "fibroblast": TEAL} + + +def place_nuclei(rng, region, count, radius, aspect, nuclei, angle=None): + """Place non-overlapping nuclei, given as fractions of the patch, at random positions inside a region.""" + n = region.shape[0] + placed = 0 + for _ in range(3000): + if placed == count: + break + cy, cx = rng.uniform(0.04, 0.96, 2) + r = rng.uniform(*radius) + if not region[int(cy * n), int(cx * n)]: + continue + if any(np.hypot(cx - o[0], cy - o[1]) < (r * aspect + o[2] * o[3]) * 1.05 for o in nuclei): + continue + theta = rng.uniform(0, np.pi) if angle is None else angle + rng.normal(0, 0.2) + nuclei.append((cx, cy, r, aspect * rng.uniform(0.9, 1.15), theta)) + placed += 1 + + +def he_nuclei(seed, n=220): + """Simulate a zoomed H&E patch with tumor nuclei, lymphocytes and fibroblasts; returns image and nuclei.""" + rng = np.random.default_rng(seed) + rows, cols = np.mgrid[:n, :n] + # a tumor nest in the upper left, stroma with collagen fibres elsewhere + boundary = 0.55 + 0.08 * np.sin(cols / n * 7) + 0.05 * np.sin(rows / n * 11) + tumor = (rows / n + 0.8 * cols / n) < boundary * 1.25 + fibres = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), (0.8, 7)) + fibres = ndimage.rotate(fibres, -25, reshape=False, mode="reflect") + fibres /= fibres.std() + stroma = np.array([0.95, 0.66, 0.79])[None, None] * (1 + 0.07 * fibres[..., None]) + granules = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), 1.5) + cytoplasm = np.array([0.87, 0.66, 0.83])[None, None] * (1 + 0.5 * granules[..., None]) + image = np.where(tumor[..., None], cytoplasm, stroma) + + nuclei, classes = [], [] + for name, region, count, radius, aspect, angle in ( + ("tumor", ndimage.binary_erosion(tumor, iterations=8), 11, (0.05, 0.065), 1.2, None), + ("lymphocyte", ndimage.binary_erosion(~tumor, iterations=6), 9, (0.026, 0.03), 1.05, None), + ("fibroblast", ndimage.binary_erosion(~tumor, iterations=6), 6, (0.018, 0.022), 2.6, -25 / 180 * np.pi), + ): + before = len(nuclei) + place_nuclei(rng, region, count, radius, aspect, nuclei, angle) + classes += [name] * (len(nuclei) - before) + + grid = np.stack([cols.ravel() + 0.5, rows.ravel() + 0.5], axis=1) + hematoxylin = np.array([0.3, 0.2, 0.5]) + for k, ((cx, cy, r, aspect, angle), name) in enumerate(zip(nuclei, classes)): + mask = Path(blob_points(cx, cy, r, aspect, angle, seed + k) * n).contains_points(grid).reshape(n, n) + depth = ndimage.distance_transform_edt(mask) + grain = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), 0.8) + if name == "tumor": + # vesicular chromatin: a dark rim, a paler center and a prominent nucleolus + density = 0.55 + 0.35 * (depth < 2.5) + 0.25 * grain / grain.std() + r0, c0 = np.unravel_index(np.argmax(depth), depth.shape) + density += 0.6 * (np.hypot(rows - r0 - 1, cols - c0 + 1) < 2.2) + elif name == "lymphocyte": + density = 1.15 + 0.12 * grain / grain.std() + else: + density = 1.0 + 0.1 * grain / grain.std() + stain = 1 - np.clip(density, 0, 1.4)[..., None] * (1 - hematoxylin)[None, None] / 1.4 * 1.25 + image = np.where(mask[..., None], np.clip(stain, 0, 1), image) + stain = ndimage.gaussian_filter(rng.normal(0, 1, (n, n)), n / 5) + image = ndimage.gaussian_filter(image, (0.6, 0.6, 0)) * (1 + 0.5 * stain[..., None]) + image += rng.normal(0, 0.015, image.shape) + return np.clip(image, 0, 1), nuclei, classes + + +def object_classification(drawing, x, cy): + """Nucleus classification in a histopathology slide: tumor nuclei, lymphocytes and fibroblasts.""" + y = cy - SIZE / 2 + seed = 23 + image, nuclei, classes = he_nuclei(seed) + clip = drawing.clip_rect(x, y, SIZE, SIZE, 0.9) + parts = [ + f'', f'', + ] + for k, ((cx, cyy, r, aspect, angle), name) in enumerate(zip(nuclei, classes)): + path = blob_path(x + cx * SIZE, y + cyy * SIZE, r * SIZE, aspect, angle, seed + k) + color = NUCLEUS_CLASSES[name] + parts.append(f'') + parts.append("") + # one nucleus per class that the user labeled by hand + for name in NUCLEUS_CLASSES: + k = classes.index(name) + cx, cyy = nuclei[k][:2] + parts.append( + f'' + ) + drawing.add("".join(parts)) + + +def automatic_2d(drawing, x, cy): + """All cells of the label free 2D example, segmented automatically.""" + labels, image = livecell_scene() + livecell_tile(drawing, x, cy - SIZE / 2, overlay(image, [labels == k for k in range(1, labels.max() + 1)])) + + +def automatic_3d(drawing, x, cy): + colors = neurite_colors(64) + faces = em_faces(seed=EM_SEED, overlay=lambda labels, u, v: (colors[labels], np.full(labels.shape, 0.6))) + em_block(drawing, x, cy - (SIZE + 3.5) / 2, faces) + + +def automatic_2dt(drawing, x, cy): + frames = [] + for t, cells in enumerate(frames_2dt()): + colors = lineage_colors(t, tracked=(0, 1, 2)) + frames.append((cells, list(colors), colors)) + origins = frame_stack(drawing, x, cy, frames, seed=81) + draw_tracks(drawing, *tracks_2dt(origins, tracked=(0, 1, 2))) + + +# solid colors of the track masks, for their outlines +NUCLEUS_COLORS = { + "purple": "#7b68d6", "teal": "#34b4c0", "gold": "#e0b43c", "pink": "#e0679b", "blue": "#4f86d6", + "orange": "#ef7a45", +} +MASK_EDGES = { + "f1c-mask": "#9a86e0", "f1c-mask-teal": "#5fd0d8", "f1c-mask-gold": "#f3cf63", "f1c-mask-pink": "#f08dbd", + "f1c-mask-blue": "#7fb2f0", "f1c-mask-orange": "#f69a6c", +} + + +def nucleus_texture(mask, rng, brightness, condensed=False): + """Fill a nucleus mask with DNA stain: grainy chromatin, a few bright foci and a brighter rim.""" + grain = ndimage.gaussian_filter(rng.normal(0, 1, mask.shape), 0.9) + grain /= grain.std() + foci = np.zeros(mask.shape) + points = np.argwhere(ndimage.binary_erosion(mask, iterations=3)) + for r, c in points[rng.integers(len(points), size=min(5, len(points)))] if len(points) else []: + foci[r, c] = 1.0 + foci = ndimage.gaussian_filter(foci, 1.1) * 18 + rim = ndimage.binary_dilation(~mask, iterations=2) & mask + texture = brightness * (1 + (0.32 if condensed else 0.18) * grain) + 0.35 * foci + 0.08 * rim + if condensed: + texture *= 1.35 + return np.where(mask, texture, 0.0) + + +def stain_rgb(value): + """Map a DNA stain intensity to the blue-white lookup table of a Hoechst channel.""" + value = np.clip(value, 0, 1)[..., None] ** 0.85 + return np.clip(value * np.array([0.62, 0.74, 1.0]) + np.clip(value - 0.75, 0, 1) * 1.2, 0, 1) + + +def nucleus_frame(drawing, x, y, size, cells, masked, fills, seed, mitotic=(), n=170, rx=0.8): + """Draw a fluorescence frame of stained nuclei with outlined, lightly filled track masks.""" + rng = np.random.default_rng(seed) + image = np.zeros((n, n)) + rows, cols = np.mgrid[:n, :n] + grid = np.stack([cols.ravel() + 0.5, rows.ravel() + 0.5], axis=1) + paths = [] + for k, (cx, cy, r, aspect, angle) in enumerate(cells): + points = blob_points(cx, cy, r, aspect, angle, seed + k) + mask = Path(points * n).contains_points(grid).reshape(n, n) + condensed = k == 3 and bool(mitotic) and mitotic[0] + image += nucleus_texture(mask, rng, rng.uniform(0.42, 0.6), condensed) + paths.append(blob_path(x + cx * size, y + cy * size, r * size, aspect, angle, seed + k)) + image = ndimage.gaussian_filter(image, 0.9) + 0.035 + rng.normal(0, 0.018, image.shape) + clip = drawing.clip_rect(x, y, size, size, rx) + parts = [ + f'', + f'', + f'', + ] + for k in masked: + fill = fills[k] if isinstance(fills, dict) else fills + parts.append( + f'' + ) + parts.append("") + drawing.add("".join(parts)) + + +def nucleus_sprite(color, seed, n=64): + """Render a textured, shaded nucleus in 3D, tinted with the color of its track.""" + rng = np.random.default_rng(seed) + rows, cols = np.mgrid[:n, :n] + u, v = (cols + 0.5) / n * 2 - 1, (rows + 0.5) / n * 2 - 1 + angle = np.arctan2(v, u) + edge = 0.92 + 0.04 * np.sin(3 * angle + rng.uniform(0, 6)) + 0.03 * np.sin(5 * angle + rng.uniform(0, 6)) + radius = np.hypot(u, v) / edge + mask = radius < 1 + shading = np.sqrt(np.clip(1 - radius ** 2, 0, 1)) * (0.75 + 0.25 * np.clip(-u - v, -1, 1)) + texture = nucleus_texture(mask, rng, 0.62) * (0.45 + 0.55 * shading) + rgb = 0.45 * stain_rgb(1.5 * texture) + 0.55 * hex_to_rgb(color) * (0.6 + 0.9 * texture[..., None]) + alpha = np.clip((1 - radius) * 14, 0, 1) + rgba = np.concatenate([np.clip(rgb, 0, 1), alpha[..., None]], axis=-1) + buffer = io.BytesIO() + Image.fromarray((rgba * 255).astype("uint8"), mode="RGBA").save(buffer, format="PNG", optimize=True) + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +# the nuclei of the 3D+T example as (x, y, z), oldest time point first; the blue mother divides last +NUCLEI_3DT = { + "purple": [(0.06, 0.18, 0.2), (0.3, 0.2, 0.3), (0.54, 0.22, 0.4)], + "teal": [(0.84, 0.0, 0.55), (0.82, 0.24, 0.5), (0.8, 0.48, 0.45)], + "gold": [(0.04, 0.98, 0.15), (0.14, 0.78, 0.2), (0.24, 0.58, 0.25)], + "blue": [(0.36, 0.66, 0.75), (0.5, 0.74, 0.5)], + "pink": [(0.44, 0.98, 0.3)], + "orange": [(0.82, 0.9, 0.3)], +} + + +def automatic_3dt(drawing, x, cy, front=SIZE, depth=3.5): + """A dark volume with the segmented nuclei of the last time point and their tracks through earlier ones.""" + fx, fy, d = x, cy - (front + depth) / 2 + depth, depth + + def project(nx, ny, nz): + return fx + nx * front + nz * d, fy + ny * front - nz * d + + def polygon(points, fill): + return '' + + top = [(fx, fy), (fx + d, fy - d), (fx + front + d, fy - d), (fx + front, fy)] + side = [(fx + front, fy), (fx + front + d, fy - d), (fx + front + d, fy - d + front), (fx + front, fy + front)] + parts = [ + f'', + polygon(top, "#262536"), polygon(side, "#1a1925"), + f'', + ] + # earlier time points are smaller and faded, the last time point is solid; draw from back to front + nuclei = [] + for color, positions in NUCLEI_3DT.items(): + for k, position in enumerate(positions): + is_last = color != "blue" and k == len(positions) - 1 + nuclei.append((position, color, 0.13 if is_last else 0.075, 1.0 if is_last else 0.45)) + for (nx, ny, nz), color, r, opacity in sorted(nuclei, key=lambda n: -n[0][2]): + px, py = project(nx, ny, nz) + w, h = 2 * r * front, 1.75 * r * front + parts.append( + f'' + ) + drawing.add("".join(parts)) + + split = project(*NUCLEI_3DT["blue"][-1]) + tracks = [([project(*p) for p in NUCLEI_3DT[c]], NUCLEUS_COLORS[c]) for c in ("purple", "teal", "gold", "blue")] + tracks += [([split, project(*NUCLEI_3DT[c][0])], NUCLEUS_COLORS[c]) for c in ("pink", "orange")] + draw_tracks(drawing, tracks, split) + drawing.add( + f'' + ) + + +def batch_stack(drawing, x, cy, draw_slide): + """Draw a series of three slides, each processed automatically and marked as done; draw_slide draws slide k.""" + size, offset = STACK_FRAME, STACK_OFFSET + top = cy - (size + 2 * offset) / 2 + for k in reversed(range(3)): + fx, fy = x + offset * k, top + offset * (2 - k) + clip = drawing.clip_rect(fx, fy, size, size, 0.8) + draw_slide(drawing, k, fx, fy, size, clip) + cx, cyy = fx + size - 0.75, fy + 0.75 + drawing.add( + f'' + f'' + f'' + ) + + +def tissue_slide(drawing, k, x, y, size, clip): + """A slide classified into tissue types, with the colors of the pixel classification.""" + colors = np.array([hex_to_rgb(TEAL), hex_to_rgb(ORANGE), hex_to_rgb("#e8c547")]) + image, classes = he_tissue(seed=31 + 6 * k, n=160) + outlines = ndimage.binary_dilation(membranes(classes), iterations=1)[..., None] + predicted = np.where(outlines, colors[classes], 0.65 * image + 0.35 * colors[classes]) + drawing.add( + f'' + ) + + +def nuclei_slide(drawing, k, x, y, size, clip): + """A slide whose nuclei are classified, with the colors of the object classification.""" + seed = 41 + 5 * k + image, nuclei, classes = he_nuclei(seed) + parts = [ + f'', f'', + ] + for j, ((cx, cyy, r, aspect, angle), name) in enumerate(zip(nuclei, classes)): + path = blob_path(x + cx * size, y + cyy * size, r * size, aspect, angle, seed + j) + color = NUCLEUS_CLASSES[name] + parts.append(f'') + parts.append("") + drawing.add("".join(parts)) + + +def automatic_batch(drawing, x, cy): + """A series of histopathology slides, each classified into tissue types automatically.""" + batch_stack(drawing, x, cy, tissue_slide) + + +def automatic_batch_objects(drawing, x, cy): + """A series of histopathology slides, whose nuclei are each classified automatically.""" + batch_stack(drawing, x, cy, nuclei_slide) + + +ITEMS = { + "interactive_2d": (interactive_2d, SIZE, "2D"), + "interactive_3d": (interactive_3d, SIZE + 3.5, "3D"), + "interactive_2dt": (interactive_2dt, STACK_FRAME + 2 * STACK_OFFSET, "2D time-lapse"), + "pixel_classification": (pixel_classification, SIZE, "pixels"), + "object_classification": (object_classification, SIZE, "objects"), + "automatic_2d": (automatic_2d, SIZE, "2D"), + "automatic_3d": (automatic_3d, SIZE + 3.5, "3D"), + "automatic_2dt": (automatic_2dt, STACK_FRAME + 2 * STACK_OFFSET, "2D time-lapse"), + "automatic_3dt": (automatic_3dt, SIZE + 3.5, "3D time-lapse"), + "automatic_batch": (automatic_batch, STACK_FRAME + 2 * STACK_OFFSET, "pixels"), + "automatic_batch_objects": (automatic_batch_objects, STACK_FRAME + 2 * STACK_OFFSET, "objects"), +} + + +def group_width(names, scale=1.0): + return scale * (sum(ITEMS[name][1] for name in names) + ITEM_GAP * (len(names) - 1)) + + +def superellipse(cx, cy, a, b, power=4.0, samples=240): + """Get the points of a rounded rectangle-like superellipse.""" + t = np.linspace(0, 2 * np.pi, samples, endpoint=False) + c, s = np.cos(t), np.sin(t) + x = cx + a * np.sign(c) * np.abs(c) ** (2 / power) + y = cy + b * np.sign(s) * np.abs(s) ** (2 / power) + return np.stack([x, y], axis=1) + + +def loop_badge(drawing, center, r=2.7): + cx, cy = center + gid = drawing.linear_gradient(cx, cy + r, cx, cy - r) + drawing.add( + f'' + ) + + +def correction_icon(drawing, center): + """Two touching cells, where a positive and a negative point prompt separate the target from its neighbour.""" + cx, cy = center + target = blob_path(cx - 0.6, cy + 0.45, 1.15, 1.25, 0.5, seed=5) + neighbour = blob_path(cx + 0.75, cy - 0.6, 1.1, 1.2, 0.5, seed=8) + drawing.add( + f'' + f'' + ) + point_prompt(drawing, cx - 0.65, cy + 0.5, r=0.45) + px, py = cx + 0.75, cy - 0.6 + point_prompt(drawing, px, py, positive=False, r=0.45) + cursor = [(0, 0), (0, 0.9), (0.24, 0.69), (0.4, 1.03), (0.54, 0.97), (0.38, 0.63), (0.67, 0.61)] + d = "M" + "L".join(f"{fmt(px + 0.25 + x)},{fmt(py + 0.25 + y)}" for x, y in cursor) + "Z" + drawing.add(f'') + + +def logo_data_uri(path, width_px=480): + image = Image.open(path).convert("RGBA") + image = image.resize((width_px, round(image.height * width_px / image.width)), Image.LANCZOS) + buffer = io.BytesIO() + image.save(buffer, format="PNG", optimize=True) + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode(), image.width / image.height + + +def draw_interfaces(drawing, cx, cy, logo_path, napari_logo, python_logo, logo_h=7.0, height=8.2): + """Draw the μSAM2 ecosystem as one white pill centered at (cx, cy): the logo (an svg is kept as vectors), then a + round badge with the name to its right for each of its three interfaces. Returns the lower edge of the pill.""" + if logo_path.endswith(".svg"): + uri = svg_uri(logo_path) + with open(logo_path) as f: + _, _, w, h = map(float, re.search(r'viewBox="([^"]*)"', f.read()).group(1).split()) + aspect = w / h + else: + uri, aspect = logo_data_uri(logo_path, width_px=900) + logo_w = logo_h * aspect + interfaces = [("napari", svg_uri(napari_logo)), ("Python", svg_uri(python_logo)), ("CLI", None)] + # the napari icon looks larger than the Python icon at one size, so it is drawn smaller + badge, icon, gap, name_size = 2.5, 3.4, 0.8, 2.12 + icon_scale = {"napari": 0.86, "Python": 1.0} + widths = [2 * badge + gap + text_width(name, name_size) for name, _ in interfaces] + total = logo_w + sum(3.0 + w for w in widths) + x = cx - total / 2 + margin = height / 2 + drawing.add( + f'' + ) + drawing.add( + f'' + ) + x += logo_w + for (name, icon_uri), w in zip(interfaces, widths): + x += 3.0 + bx = x + badge + drawing.add( + f'' + ) + if icon_uri is None: + # the prompt of a terminal + s_ = 2.6 + drawing.add( + f'' + ) + else: + size = icon * icon_scale[name] + drawing.add( + f'' + ) + drawing.text(bx + badge + gap, cy + 0.75, name, name_size, color=TEXT, anchor="start") + x += w + return cy + height / 2 + + +def svg_uri(path): + with open(path, "rb") as f: + return "data:image/svg+xml;base64," + base64.b64encode(f.read()).decode() + + +A_W = 169.5 +# the panel is scaled to the usable width of an A4 page, and its main text is enlarged a little +A4_WIDTH, A_TEXT_SCALE = 180.0, 1.12 +A_HEADER_Y = 5.5 +# the inputs in a narrow column, the automatic band right of the encoder, the interactive band below +A_ENCODER_X = (18.0, 40.0) +A_BAND_X, A_BAND_GAP = 61.5, 3.0 +AUTO_BAND_H, INTER_H = 57.0, 45.0 +INTER_Y = AUTO_BAND_H + A_BAND_GAP +A_H = INTER_Y + INTER_H +A_CONTENT_X = 69.0 +# the embeddings enter the automatic head at the height of the 2D embedding, which is its first decoder stage +A_ENTRY = 19.0 +A_OUTPUT_X = 151.5 +A_SCALE = 1.45 +# the width of the inputs and of the automatic outputs, in mm of panel a +INPUT_SIZE = 14.6 +# the prompt encoder of the interactive band (left edge and width); APG leads into it from the decoder maps +PROMPT_ENCODER_X, PROMPT_ENCODER_W = 27.0, 17.0 +# the rows of the 2D and the 3D outputs, relative to the top of a band +A_ROW_2D, A_ROW_3D, AUTO_ROW_3D = 11.0, 31.0, 44.0 +# the 3D input and its embeddings, relative to the top of the panel +A_TRUNK_3D = 44.0 +# the Hiera stages as (tokens per side, size, layers, global attention), and the gap between them +HIERA_GAP = 3.0 +# the stages are 7% larger than first drawn, in the room that the lower attention key leaves +HIERA_STAGES = [(16, 7.9, 2, False), (8, 6.2, 3, False), (4, 4.8, 5, True), (2, 3.65, 3, True)] +# the decoder stages as (channels, height, depth) of their feature maps +STAGES = [(0.9, 22.0, 5.6), (1.5, 16.0, 4.2), (2.2, 11.5, 3.0), (3.0, 8.0, 2.2)] +ARROW = "#5f5a6b" +# the fills and outlines of the modules; the heads take the colors of the bands of panel c +ENCODER_STYLE = ("#d9d4ee", "#7c74ab") +INTERACTIVE_STYLE = ("#d7b9c3", "#8f6676") +AUTOMATIC_STYLE = ("#b9d1c8", "#6f9186") +GATE_STYLE = ("#b8e2e8", "#2a8f9c") +SLICE_STYLE = ("#dcdce1", "#8d8d96") + + +def scaled(drawing, scale, draw, x, cy): + """Draw an item of panel c at a larger scale, with its left edge at x and its center at cy.""" + start = len(drawing.body) + drawing.scale *= scale + draw(drawing, 0, 0) + drawing.scale /= scale + inner = "".join(drawing.body[start:]) + del drawing.body[start:] + drawing.add(f'{inner}') + + +def flow_arrow(drawing, points, head=0.9, width=0.25, color=ARROW, dash=None): + """Draw a thin polyline arrow with a filled head at its last point; 'dash' is an svg dash pattern of the line.""" + points = [np.asarray(p, dtype=float) for p in points] + direction = points[-1] - points[-2] + direction /= np.linalg.norm(direction) + normal = np.array([-direction[1], direction[0]]) + end, tip = points[-1] - direction * head * 0.8, points[-1] + left, right = tip - direction * head + normal * head * 0.45, tip - direction * head - normal * head * 0.45 + line = "M" + "L".join(f"{fmt(px)},{fmt(py)}" for px, py in points[:-1] + [end]) + drawing.add( + f'' + f'' + ) + + +def module(drawing, x, y, w, h, lines, style, size=1.94): + """Draw a network module as a flat rounded box with a thin outline and its name inside.""" + fill, edge = style + drawing.add( + f'' + ) + for k, line in enumerate(lines): + drawing.text(x + w / 2, y + h / 2 + (k - (len(lines) - 1) / 2) * 1.2 * size + 0.34 * size, line, size) + + +def feature_map(drawing, x, cy, w, h, d, style): + """Draw a feature map as a box seen at an angle: w channels deep, h high, and d into the page.""" + fill, edge = style + dx, dy = 0.7 * d, -0.5 * d + y = cy - (h + dy) / 2 + light, dark = mix(fill, "#ffffff", 0.45), mix(fill, edge, 0.35) + drawing.add( + f'' + f'' + f'' + f'' + ) + return x + w + dx + + +def mix(a, b, t): + rgb = (1 - t) * hex_to_rgb(a) + t * hex_to_rgb(b) + return "#" + "".join(f"{int(round(255 * c)):02x}" for c in rgb) + + +def embedding_slab(drawing, x, y, w, d, n=5, style=ENCODER_STYLE): + """Draw an embedding as a flat grid seen at an angle; (x, y) is its front left corner.""" + fill, edge = style + dx, dy = 0.75 * d, -0.55 * d + top = [(x, y), (x + w, y), (x + w + dx, y + dy), (x + dx, y + dy)] + grid = "".join( + f"M{fmt(x + w * k / n)},{fmt(y)}l{fmt(dx)},{fmt(dy)}" + f"M{fmt(x + dx * k / n)},{fmt(y + dy * k / n)}h{fmt(w)}" for k in range(1, n) + ) + drawing.add( + '' + f'' + ) + return x + w + dx + + +# the volume of panel a: a neuron among other cells in the unit cube, (x, y, z) with y down +VOLUME_N = 128 +NEURON_LABEL = 1 +# the neuron that is prompted: its cell body lies in the middle slice of the volume, which holds the prompt +NEURON_SOMA = ((0.33, 0.6, 0.5), 0.15) +NEURON_NUCLEUS = ((0.31, 0.62, 0.5), 0.085) +PROMPT = (0.42, 0.56, 0.5) +# the processes of the cells as (control points, radius, taper) +NEURON_PROCESSES = [ + ([(0.38, 0.54, 0.42), (0.47, 0.46, 0.26), (0.56, 0.42, 0.1), (0.6, 0.4, -0.1)], 0.055, (1.0, 0.75)), + ([(0.3, 0.5, 0.55), (0.27, 0.3, 0.62), (0.3, 0.1, 0.7), (0.33, -0.1, 0.76)], 0.05, (1.0, 0.7)), + ([(0.42, 0.68, 0.55), (0.62, 0.8, 0.62), (0.84, 0.9, 0.7), (1.1, 0.98, 0.76)], 0.034, (1.0, 1.0)), + ([(0.24, 0.64, 0.58), (0.12, 0.72, 0.76), (0.05, 0.76, 0.94), (-0.1, 0.8, 1.1)], 0.045, (1.0, 0.65)), +] +# a big cell body that the front face cuts through, and a smaller one that the side face cuts through +BIG_CELL_LABEL, SIDE_CELL_LABEL = 2, 3 +BIG_PROCESSES = [ + ([(0.74, 0.12, 0.2), (0.71, 0.02, 0.32), (0.69, -0.04, 0.42), (0.67, -0.1, 0.52)], 0.07, (1.0, 0.6)), + ([(0.86, 0.36, 0.18), (0.93, 0.52, 0.32), (0.99, 0.66, 0.46), (1.1, 0.76, 0.58)], 0.05, (1.0, 0.7)), +] +SIDE_PROCESSES = [ + ([(0.85, 0.45, 0.76), (0.7, 0.42, 0.71), (0.55, 0.4, 0.73), (0.4, 0.38, 0.79)], 0.045, (1.0, 0.7)), +] +# the cells as (label, soma, nucleus, processes) +CELLS = [ + (NEURON_LABEL, NEURON_SOMA, NEURON_NUCLEUS, NEURON_PROCESSES), + (BIG_CELL_LABEL, ((0.8, 0.22, 0.08), 0.2), ((0.79, 0.23, 0.07), 0.11), BIG_PROCESSES), + (SIDE_CELL_LABEL, ((0.96, 0.45, 0.8), 0.17), ((0.97, 0.46, 0.8), 0.095), SIDE_PROCESSES), +] +CAMERA_DIRECTION = (0.8, 0.62, 1.0) +EDGE = "#2d2b36" + + +def spline(points, samples=240): + points = np.asarray(points, dtype=float) + t = np.linspace(0, 1, len(points)) + return make_interp_spline(t, points, k=min(3, len(points) - 1))(np.linspace(0, 1, samples)) + + +def radius_profile(rng, base, samples, taper=(1.0, 1.0)): + """Let the thickness of a neurite swell and shrink smoothly along its path, with a few bead-like varicosities.""" + t = np.linspace(0, 1, samples) + radius = base * np.linspace(*taper, samples) + radius = radius * (1 + 0.3 * np.sin(2 * np.pi * rng.uniform(1, 3) * t + rng.uniform(0, 2 * np.pi))) + for center in rng.uniform(0.1, 0.9, rng.integers(0, 3)): + radius = radius * (1 + 0.6 * np.exp(-((t - center) / 0.03) ** 2)) + return radius + + +def stamp_path(labels, label, path, radius): + """Mark the voxels within the radius of a path in unit coordinates, without overwriting other objects.""" + n = labels.shape[0] + for (px, py, pz), r in zip(path * (n - 1), radius * (n - 1)): + low = np.maximum(np.floor([pz - r, py - r, px - r]).astype(int), 0) + high = np.minimum(np.ceil([pz + r, py + r, px + r]).astype(int) + 1, n) + if np.any(high <= low): + continue + zz, yy, xx = np.ogrid[low[0]:high[0], low[1]:high[1], low[2]:high[2]] + block = labels[low[0]:high[0], low[1]:high[1], low[2]:high[2]] + block[((zz - pz) ** 2 + (yy - py) ** 2 + (xx - px) ** 2 <= r ** 2) & (block == 0)] = label + + +def candidate_neurites(rng): + """Propose neurites: most run along z with a few bends, some along x or y, and some pairs twist around each + other.""" + while True: + kind = rng.uniform() + if kind < 0.2: + # two strands that spiral around a common axis along z + x0, y0 = rng.uniform(0.05, 0.95, 2) + drift, helix, turns = rng.normal(0, 0.1, 2), rng.uniform(0.05, 0.075), rng.uniform(1.0, 2.0) + t = np.linspace(0, 1, 240) + for phase in (0.0, np.pi): + angle = 2 * np.pi * turns * t + phase + path = np.stack([ + x0 + drift[0] * t + helix * np.cos(angle), y0 + drift[1] * t + helix * np.sin(angle), 1.2 * t - 0.1 + ], axis=1) + yield [(path, radius_profile(rng, rng.uniform(0.022, 0.032), len(path)))] + continue + axis = 2 if kind < 0.8 else (0 if kind < 0.92 else 1) + start = rng.uniform(0.0, 1.0, 3) + points = [] + for k, t in enumerate(np.linspace(-0.1, 1.1, 6)): + point = start + rng.normal(0, 0.06, 3) * (k > 0) + point[axis] = t + start = point + points.append(point.copy()) + path = spline(points) + yield [(path, radius_profile(rng, rng.uniform(0.016, 0.045), len(path)))] + + +def collides(path, radius, occupied, margin=0.004): + for other, other_radius in occupied: + gaps = cdist(path[::4], other[::4]) - radius[::4, None] - other_radius[None, ::4] + if gaps.min() < margin: + return True + return False + + +@lru_cache +def volume_data(count=220, grow=8): + """Label a volume with the cells and the other neurites, indexed as (z, y, x), and mark their mitochondria and + synaptic vesicles.""" + rng = np.random.default_rng(EM_SEED) + n = VOLUME_N + labels = np.zeros((n, n, n), dtype="uint32") + occupied = [] + for label, (center, radius), _, _ in CELLS: + soma = (np.array([center]), np.array([radius])) + stamp_path(labels, label, *soma) + occupied.append(soma) + for label, _, _, processes in CELLS: + for points, base, taper in processes: + path = spline(points) + process = (path, radius_profile(rng, base, len(path), taper)) + stamp_path(labels, label, *process) + occupied.append(process) + label, tries, neurites = len(CELLS) + 1, 0, [] + for group in candidate_neurites(rng): + tries += 1 + if label > count or tries > 20000: + break + # a pair of twisted strands is proposed one strand after the other + for path, radius in group: + if collides(path, radius, occupied): + continue + stamp_path(labels, label, path, radius) + occupied.append((path, radius)) + neurites.append((label, path, radius)) + label += 1 + # the neurites grow into the free space around them, like the densely packed neuropil of EM + distance, (zi, yi, xi) = ndimage.distance_transform_edt(labels == 0, return_indices=True) + labels = np.where(distance <= grow, labels[zi, yi, xi], 0).astype("uint32") + return labels, mitochondria(labels, neurites), vesicles(labels, neurites) + + +def mitochondria(labels, neurites): + """Place mitochondria as short capsules along the paths of some neurites, inside their neurite.""" + rng = np.random.default_rng(EM_SEED + 1) + marks = np.zeros(labels.shape, dtype="uint8") + for label, path, radius in neurites: + for _ in range(rng.integers(1, 4)): + length = rng.integers(25, 60) + start = rng.integers(0, len(path) - length) + segment = path[start:start + length] + rng.normal(0, 0.15, 3) * radius[start] + marks_label = np.zeros_like(marks) + stamp_path(marks_label, 1, segment, np.maximum(0.55 * radius[start:start + length], 0.012)) + marks |= marks_label & (labels == label) + return marks.astype(bool) + + +def vesicles(labels, neurites, per_bouton=40): + """Fill the varicosities of the neurites, their synaptic boutons, with clusters of small vesicles.""" + rng = np.random.default_rng(EM_SEED + 2) + marks = np.zeros(labels.shape, dtype="uint8") + for label, path, radius in neurites: + peaks = np.nonzero( + (radius[1:-1] > radius[:-2]) & (radius[1:-1] >= radius[2:]) & (radius[1:-1] > 1.3 * np.median(radius)) + )[0] + 1 + for index in peaks: + direction = rng.normal(0, 1, (per_bouton, 3)) + direction /= np.linalg.norm(direction, axis=1, keepdims=True) + points = path[index] + direction * 0.6 * radius[index] * rng.uniform(0, 1, (per_bouton, 1)) ** (1 / 3) + marks_label = np.zeros_like(marks) + stamp_path(marks_label, 1, points, np.full(per_bouton, 0.007)) + marks |= marks_label & (labels == label) + return marks.astype(bool) + + +def neuron_volume(): + return volume_data()[0] + + +def volume_faces(volume, axis=0): + """Get the front, top and side faces of a volume whose spatial axes start at axis; top and side have z as rows.""" + index = (slice(None),) * axis + front, top, side = volume[index + (0,)], volume[index + (slice(None), 0)], volume[index + (Ellipsis, -1)] + return {"front": front, "top": top, "side": side} + + +def face_coordinates_3d(face, n): + """Get the unit coordinates (x, y, z) of the pixels of a face from volume_faces.""" + a, b = np.meshgrid(np.linspace(0, 1, n), np.linspace(0, 1, n)) + if face == "front": + return a, b, np.zeros_like(a) + if face == "top": + return a, np.zeros_like(a), b + return np.ones_like(a), a, b + + +def em_section(labels, coordinates, mito, vesicle, seed): + """Simulate the EM section of a plane through the volume, with the nuclei and organelles of the cell bodies that it + cuts, and the sections of the mitochondria and vesicles.""" + soma, nucleus = np.zeros(labels.shape, dtype=bool), np.zeros(labels.shape, dtype=bool) + for label, cell_soma, cell_nucleus, _ in CELLS: + soma |= (labels == label) & in_sphere(*coordinates, cell_soma) + nucleus |= in_sphere(*coordinates, cell_nucleus) + gray = em_texture( + labels, membranes(labels), seed, soma=soma, nucleus=nucleus, mitochondria=mito & ~nucleus, vesicles=vesicle + ) + return np.repeat(gray[..., None], 3, axis=-1) + + +@lru_cache +def em_volume_faces(): + """Simulate EM sections on the visible faces of the volume.""" + n, (labels, mito, vesicle) = VOLUME_N, volume_data() + planes = zip(volume_faces(labels).items(), volume_faces(mito).values(), volume_faces(vesicle).values()) + return { + face: em_section(face_labels, face_coordinates_3d(face, n), face_mito, face_vesicle, EM_SEED + 17 * (k + 1)) + for k, ((face, face_labels), face_mito, face_vesicle) in enumerate(planes) + } + + +@lru_cache +def em_slice(index): + """Simulate the EM section of the slice at a z index.""" + n, (labels, mito, vesicle) = VOLUME_N, volume_data() + a, b = np.meshgrid(np.linspace(0, 1, n), np.linspace(0, 1, n)) + coordinates = (a, b, np.full_like(a, index / (n - 1))) + return em_section(labels[index], coordinates, mito[index], vesicle[index], EM_SEED + 5 * index) + + +# the face images of the volume are upsampled by this factor for the renders, and the renders use this window size +TEXTURE_FACTOR, RENDER_SIZE = 4, 800 + + +def to_world(points): + """Map unit coordinates (x, y, z) to the world coordinates of the renders, where y points up and z to the viewer.""" + points = np.asarray(points, dtype=float).reshape(-1, 3) + return np.stack([points[:, 0], 1 - points[:, 1], 1 - points[:, 2]], axis=1) * (VOLUME_N - 1) + + +def color_grid(dimensions, origin, data, factor=TEXTURE_FACTOR): + """Make a flat grid with the colors of an image. The image is upsampled by factor along its two axes with + 'upscale', so that the voxels of the volume do not show as steps in the renders.""" + flat = [k for k, d in enumerate(dimensions) if d == 1][0] + # the image has its axes in the reverse order of the dimensions + image = upscale(np.squeeze(np.asarray(data, dtype=float), axis=2 - flat), factor) + data = np.expand_dims(image, axis=2 - flat) + size = [d * factor if d > 1 else 1 for d in dimensions] + spacing = [(d - 1) / (m - 1) if d > 1 else 1.0 for d, m in zip(dimensions, size)] + grid = pv.ImageData(dimensions=size, origin=origin, spacing=spacing) + grid.point_data["rgb"] = (np.clip(data, 0, 1) * 255).astype("uint8").reshape(-1, 3) + return grid + + +def face_slabs(rgb): + """Turn the face images of volume_faces into flat grids at the faces of the volume, with their colors.""" + n = VOLUME_N + return [ + color_grid((n, n, 1), (0, 0, n - 1), rgb["front"][::-1][None]), + color_grid((n, 1, n), (0, n - 1, 0), rgb["top"][::-1][:, None]), + color_grid((1, n, n), (n - 1, 0, 0), rgb["side"][::-1, ::-1][:, :, None]), + ] + + +def slice_slab(rgb, index, spread=1.0): + """Place the image of the slice at a z index in the volume, with the slices pulled apart by spread.""" + n = VOLUME_N + return color_grid((n, n, 1), (0, 0, (n - 1) / 2 + spread * ((n - 1) / 2 - index)), rgb[::-1][None]) + + +@lru_cache +def label_surfaces(): + """Extract a smooth, closed surface for each object of the volume, slightly shrunk so that touching objects stay + apart, in the world coordinates of the renders.""" + labels = neuron_volume() + n = labels.shape[0] + meshes = {} + for label, box in enumerate(ndimage.find_objects(labels), start=1): + if box is None: + continue + # a margin of empty voxels closes the surfaces where the objects leave the volume + mask = np.pad(ndimage.binary_erosion(labels[box] == label), 3) + field = ndimage.gaussian_filter(mask.astype("float32"), 1.0) + origin = [box[2].start - 3, box[1].start - 3, box[0].start - 3] + grid = pv.ImageData(dimensions=field.shape[::-1], origin=origin) + grid.point_data["field"] = field.ravel() + mesh = grid.contour([0.5], scalars="field") + if mesh.n_points == 0: + continue + x, y, z = mesh.points.T + mesh.points = np.stack([x, n - 1 - y, n - 1 - z], axis=1) + meshes[label] = mesh + return meshes + + +def render(add, size=RENDER_SIZE, edges=True): + """Render a scene off screen from the viewpoint of the blocks, inside the edges of the volume, and crop it.""" + plotter = pv.Plotter(off_screen=True, window_size=(size, size)) + # translucent objects need depth peeling to be drawn in the right order + plotter.enable_depth_peeling() + add(plotter) + n = VOLUME_N - 1 + if edges: + plotter.add_mesh(pv.Box(bounds=(0, n, 0, n, 0, n)).extract_all_edges(), color=EDGE, line_width=3 * size / 900) + focal = np.full(3, n / 2) + direction = np.asarray(CAMERA_DIRECTION) / np.linalg.norm(CAMERA_DIRECTION) + plotter.camera.focal_point = focal + plotter.camera.position = focal + direction * n * 4.2 + plotter.camera.up = (0, 1, 0) + # the view must hold the whole cube, whose nearest corner reaches furthest out + plotter.camera.view_angle = 30 + plotter.reset_camera_clipping_range() + plotter.render() + image = plotter.screenshot(transparent_background=True, return_img=True) + plotter.close() + rows, cols = np.nonzero(image[..., 3]) + return image[rows.min():rows.max() + 1, cols.min():cols.max() + 1] + + +def add_colors(plotter, grids, opacity=1.0): + for grid in grids: + plotter.add_mesh(grid, scalars="rgb", rgb=True, ambient=0.35, diffuse=0.75, specular=0.05, opacity=opacity) + + +def add_point_prompt(plotter, point, radius=6.5, world=False): + """Lay a positive point prompt flat onto the slice that holds it, facing the viewer. 'point' is in unit + coordinates, or in world coordinates with 'world'.""" + center = (np.asarray(point, float) if world else to_world(point)[0]) + (0, 0, 0.6) + ring = pv.Disc(center=center, inner=0, outer=1.25 * radius, normal=(0, 0, 1), c_res=60) + disc = pv.Disc(center=center + (0, 0, 0.2), inner=0, outer=radius, normal=(0, 0, 1), c_res=60) + plotter.add_mesh(ring, color="white", ambient=0.6) + plotter.add_mesh(disc, color=TEAL, ambient=0.6) + arm, half = 0.55 * radius, 0.13 * radius + cx, cy, cz = center + (0, 0, 0.5) + for bounds in ((cx - arm, cx + arm, cy - half, cy + half), (cx - half, cx + half, cy - arm, cy + arm)): + plotter.add_mesh(pv.Box(bounds=bounds + (cz, cz + 0.3)), color="white", ambient=0.8) + + +@lru_cache +def render_em_block(): + return render(lambda plotter: add_colors(plotter, face_slabs(em_volume_faces()))) + + +# eight slices, like the eight embeddings that the encoder makes of them +SLICE_INDICES = (0, 18, 36, 54, 73, 91, 109, 127) + + +@lru_cache +@lru_cache +def render_prompted_slice(): + """The volume with its middle slice, on which the point prompt is placed.""" + middle = VOLUME_N // 2 + + def add(plotter): + # the volume in front of the prompted slice is a faint shell, so the prompt sits on the cut face + n, z, faces = VOLUME_N, VOLUME_N - 1 - middle, em_volume_faces() + back = n - middle + add_colors(plotter, [ + color_grid((n, 1, back), (0, n - 1, 0), faces["top"][middle:][::-1][:, None]), + color_grid((1, n, back), (n - 1, 0, 0), faces["side"][middle:][::-1, ::-1][:, :, None]), + ]) + front = pv.Box(bounds=(0, n - 1, 0, n - 1, z + 0.5, n - 1)) + plotter.add_mesh(front, color="#c9c3e3", opacity=0.12, ambient=0.5) + add_colors(plotter, [slice_slab(em_slice(middle), middle)]) + add_point_prompt(plotter, PROMPT, radius=9.0) + return render(add) + + +@lru_cache +def apg_points(): + """One positive point per object of the middle slice, at its most interior pixel, as (row, col): the prompts + that APG derives from the decoder maps. Every object gets one; only specks of fewer + than 20 pixels at the border are left out.""" + labels, points = slice_labels(), [] + for label, box in enumerate(ndimage.find_objects(labels), start=1): + if box is None: + continue + mask = labels == label + if mask.sum() < 20: + continue + distance = ndimage.distance_transform_edt(mask) + points.append(np.unravel_index(distance.argmax(), distance.shape)) + return points + + +def slice_apg(drawing, x, cy): + """The middle slice with a positive point on each object, as APG prompts it.""" + tx, ty = slice_tile(drawing, x, cy, em_slice(MIDDLE)) + for point in apg_points(): + # the points of objects at the border stay inside the tile + px, py = slice_point(tx, ty, point) + px, py = np.clip(px, tx + 0.6, tx + SIZE - 0.6), np.clip(py, ty + 0.6, ty + SIZE - 0.6) + point_prompt(drawing, px, py, r=0.32) + + +def slice_apg_sheets(drawing, x, cy, n_sheets=3, step=0.85): + """The APG plane with the neighbouring slices of the volume behind it, as translucent sheets of paper shifted up + and to the right.""" + for k in range(n_sheets, 0, -1): + sx, sy = x + k * step, cy - SIZE / 2 - k * step + clip = drawing.clip_rect(sx, sy, SIZE, SIZE, 0.9) + drawing.add( + f'' + f'' + ) + slice_apg(drawing, x, cy) + + +@lru_cache +def render_apg_slice(): + """The volume with its middle slice, with a positive point on each object of the slice, as APG prompts it.""" + middle = VOLUME_N // 2 + + def add(plotter): + n, z, faces = VOLUME_N, VOLUME_N - 1 - middle, em_volume_faces() + back = n - middle + add_colors(plotter, [ + color_grid((n, 1, back), (0, n - 1, 0), faces["top"][middle:][::-1][:, None]), + color_grid((1, n, back), (n - 1, 0, 0), faces["side"][middle:][::-1, ::-1][:, :, None]), + ]) + front = pv.Box(bounds=(0, n - 1, 0, n - 1, z + 0.5, n - 1)) + plotter.add_mesh(front, color="#c9c3e3", opacity=0.12, ambient=0.5) + add_colors(plotter, [slice_slab(em_slice(middle), middle)]) + for row, col in apg_points(): + add_point_prompt(plotter, (col / (n - 1), row / (n - 1), PROMPT[2]), radius=4.0) + return render(add) + + +@lru_cache +def render_map_block(name): + return render(lambda plotter: add_colors(plotter, face_slabs(decoder_maps()[1][name]))) + + +@lru_cache +def render_neuron(): + """The neuron that the prompt on the middle slice segments through the whole volume, among faint other objects.""" + n = VOLUME_N - 1 + + def add(plotter): + for label, mesh in label_surfaces().items(): + if label == NEURON_LABEL: + plotter.add_mesh(mesh, color=PURPLE, smooth_shading=True, specular=0.3, specular_power=20) + else: + plotter.add_mesh(mesh, color="#9a9aa2", opacity=0.1, smooth_shading=True) + plane = pv.Box(bounds=(0, n, 0, n, n / 2, n / 2)).extract_all_edges() + plotter.add_mesh(plane, color=TEAL, line_width=4 * RENDER_SIZE / 900) + return render(add) + + +@lru_cache +def render_instances(): + """All objects of the volume, each in its own color.""" + colors = neurite_colors(int(neuron_volume().max()) + 1) + + def add(plotter): + for label, mesh in label_surfaces().items(): + color = PURPLE if label == NEURON_LABEL else colors[label] + plotter.add_mesh(mesh, color=color, smooth_shading=True, specular=0.3, specular_power=20) + return render(add) + + +def colored_slice_rgb(index): + """A slice of the volume with its profiles in the colors of their 3d objects, as in 'automatic_slice_rgb', so + that an object keeps its color from slice to slice.""" + labels = neuron_volume()[index] + colors = neurite_colors(int(neuron_volume().max()) + 1) + base = em_slice(index) + image = np.where((labels > 0)[..., None], 0.45 * base + 0.55 * colors[labels], base) + image[membranes(labels)] *= 0.45 + return image + + +def slice_apg_output_sheets(drawing, x, cy, n_sheets=3, step=0.85): + """The APG output as a stack like the APG input: the middle slice in front with all objects and their points, + and the slices behind it as translucent sheets with the same objects in the same colors, so that the objects + chain through the volume.""" + for k in range(n_sheets, 0, -1): + sx, sy = x + k * step, cy - SIZE / 2 - k * step + clip = drawing.clip_rect(sx, sy, SIZE, SIZE, 0.9) + drawing.add( + f'' + f'' + ) + tx, ty = slice_tile(drawing, x, cy, colored_slice_rgb(MIDDLE)) + # the stack is drawn larger than the APG input, so its points are smaller in its own units + for point in apg_points(): + px, py = slice_point(tx, ty, point) + px, py = np.clip(px, tx + 0.45, tx + SIZE - 0.45), np.clip(py, ty + 0.45, ty + SIZE - 0.45) + point_prompt(drawing, px, py, r=0.25) + + +def rgba_uri(image): + buffer = io.BytesIO() + Image.fromarray(image).save(buffer, format="PNG", optimize=True) + return "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + + +def place_render(drawing, image, x, cy, width): + """Draw a render with its left edge at x and its center at cy, and return its height.""" + height = width * image.shape[0] / image.shape[1] + drawing.add( + f'' + ) + return height + + +def interactive_3d_render(drawing, x, cy): + place_render(drawing, render_neuron(), x, cy, SIZE + 3.5) + + +def automatic_3d_render(drawing, x, cy): + place_render(drawing, render_instances(), x, cy, SIZE + 3.5) + + +MIDDLE = VOLUME_N // 2 + + +@lru_cache +def slice_labels(): + """The labels of the middle slice of the volume, which is the 2D image of panel a.""" + return neuron_volume()[MIDDLE] + + +@lru_cache +def slice_prompts(): + """Find the prompts on the middle slice: a positive point on the cell body of the neuron, a negative point on a + neurite that touches it, and a box around another profile. Points are (row, col) and the box is (r0, c0, r1, c1) + in pixels.""" + labels, n = slice_labels(), VOLUME_N + positive = (PROMPT[1] * (n - 1), PROMPT[0] * (n - 1)) + ring = ndimage.binary_dilation(labels == NEURON_LABEL, iterations=3) & (labels != NEURON_LABEL) & (labels > 0) + neighbour = np.bincount(labels[ring]).argmax() + distance = ndimage.distance_transform_edt(labels == neighbour) + negative = np.unravel_index(distance.argmax(), distance.shape) + best, box_label = 0.0, None + for label, box in enumerate(ndimage.find_objects(labels), start=1): + if box is None or label in (NEURON_LABEL, neighbour): + continue + (r0, r1), (c0, c1) = (box[0].start, box[0].stop), (box[1].start, box[1].stop) + mask = labels == label + far = np.hypot((r0 + r1) / 2 - positive[0], (c0 + c1) / 2 - positive[1]) > 0.3 * n + if not far or min(r0, c0) < 3 or max(r1, c1) > n - 3 or not 250 < mask.sum() < 1200: + continue + # the roundest of the large profiles makes a clear box + thickness = ndimage.distance_transform_edt(mask).max() + if thickness > best: + best, box_label = thickness, (label, (r0, c0, r1, c1)) + return {"positive": positive, "negative": negative, "box": box_label} + + +def overlay(base, masks, colors=None): + """Lay masks with a blue to purple gradient (or the rgb of 'colors', one per mask) and a light rim over a gray + image.""" + n = base.shape[0] + rows, cols = np.mgrid[:n, :n] + g = ((rows + cols) / (2 * n))[..., None] + gradient = (1 - g) * hex_to_rgb(BLUE) + g * hex_to_rgb(PURPLE) + image = base.copy() + for k, mask in enumerate(masks): + rim = mask & ~ndimage.binary_erosion(mask, iterations=2) + fill = gradient if colors is None or colors[k] is None else hex_to_rgb(colors[k]) + image = np.where(mask[..., None], 0.35 * image + 0.65 * fill, image) + image[rim] = hex_to_rgb(RIM) + return image + + +def slice_tile(drawing, x, cy, rgb): + """Draw an image of the middle slice as a rounded tile with its left edge at x and its center at cy.""" + y = cy - SIZE / 2 + clip = drawing.clip_rect(x, y, SIZE, SIZE, 0.9) + drawing.add( + f'' + ) + return x, y + + +def slice_point(x, y, point): + return x + point[1] / VOLUME_N * SIZE, y + point[0] / VOLUME_N * SIZE + + +def draw_slice_prompts(drawing, x, y, negative=True): + prompts = slice_prompts() + r0, c0, r1, c1 = prompts["box"][1] + (x0, y0), (x1, y1) = slice_point(x, y, (r0 - 2, c0 - 2)), slice_point(x, y, (r1 + 2, c1 + 2)) + drawing.add( + f'' + ) + point_prompt(drawing, *slice_point(x, y, prompts["positive"]), r=0.55) + if negative: + point_prompt(drawing, *slice_point(x, y, prompts["negative"]), positive=False, r=0.55) + + +def slice_input(drawing, x, cy): + slice_tile(drawing, x, cy, em_slice(MIDDLE)) + + +def slice_prompted(drawing, x, cy): + """The middle slice with the prompts that μSAM2 is trained with: a box, and positive and negative points.""" + draw_slice_prompts(drawing, *slice_tile(drawing, x, cy, em_slice(MIDDLE))) + + +def slice_interactive(drawing, x, cy): + """The neuron and the boxed profile, segmented from their prompts on the middle slice.""" + labels, prompts = slice_labels(), slice_prompts() + masks = [labels == NEURON_LABEL, labels == prompts["box"][0]] + # the object of the point keeps the purple, the object of the box is amber + image = overlay(em_slice(MIDDLE), masks, colors=[None, BOX_OBJECT]) + draw_slice_prompts(drawing, *slice_tile(drawing, x, cy, image), negative=False) + + +def automatic_slice_rgb(): + """The middle slice with all its profiles, each in its own color.""" + labels = slice_labels() + colors = neurite_colors(int(labels.max()) + 1) + base = em_slice(MIDDLE) + image = np.where((labels > 0)[..., None], 0.45 * base + 0.55 * colors[labels], base) + image[membranes(labels)] *= 0.45 + return image + + +def slice_automatic(drawing, x, cy): + """All profiles of the middle slice, each in its own color.""" + slice_tile(drawing, x, cy, automatic_slice_rgb()) + + +# the fire lookup table of ImageJ, from dark purple over red and orange to pale yellow, as (position, rgb) +FIRE = [ + (0.0, (0.16, 0.0, 0.32)), (0.18, (0.55, 0.0, 0.55)), (0.38, (0.9, 0.12, 0.08)), (0.58, (1.0, 0.5, 0.0)), + (0.78, (1.0, 0.85, 0.1)), (1.0, (1.0, 0.98, 0.75)), +] + + +def fire(value): + """Map values in [0, 1] to the colors of the fire lookup table.""" + positions = [position for position, _ in FIRE] + return np.stack([np.interp(value, positions, [rgb[c] for _, rgb in FIRE]) for c in range(3)], axis=-1) + + +def colorize(channels): + """Map the foreground, directed distance and boundary channels of micro-sam to rgb images.""" + inside = channels[0][..., None] > 0 + background = hex_to_rgb("#2d2b36") + mid = hex_to_rgb("#f2eff7") + maps = { + # the object boundaries are dark, so that touching objects of dense data stay apart + "foreground": np.where(inside & (channels[4] == 0)[..., None], mid, background), + "boundary": np.where((channels[4] > 0)[..., None], mid, background), + } + shown = inside[..., 0] & (channels[4] == 0) + for name, channel in zip(("z-distance", "y-distance", "x-distance"), channels[1:4]): + # the distances crowd around zero, so they are stretched by their rank over the whole fire table + ranks = np.zeros(channel.shape) + if shown.any(): + values = channel[shown] + ordered = np.sort(values) + # equal values, like the many zeros of the z-distance, share their middle rank + low, high = np.searchsorted(ordered, values, side="left"), np.searchsorted(ordered, values, side="right") + ranks[shown] = (low + high) / 2 / len(values) + maps[name] = np.where(shown[..., None], fire(ranks), background) + return maps + + +@lru_cache +def decoder_maps(): + """Compute the targets of the automatic segmentation decoder with micro-sam, for the 2D image and the volume. + + The maps of the volume are given for its front, top and side faces. + """ + transform = DirectedPerObjectBoundaryDistanceTransform(with_boundaries=True) + maps_2d = colorize(transform(slice_labels())) + faces = {face: colorize(channels) for face, channels in volume_faces(transform(neuron_volume()), axis=1).items()} + maps_3d = {name: {face: faces[face][name] for face in faces} for name in maps_2d} + return maps_2d, maps_3d + + +def map_tile(drawing, x, y, size, rgb): + clip = drawing.clip_rect(x, y, size, size, 0.5) + drawing.add( + f'' + ) + + +def item_label(drawing, x, width, y, text, size=1.76): + drawing.text(x + width / 2, y, text, size, color=SUBTEXT) + + +def outputs(drawing, draw_2d, draw_3d, y, row_2d, row_3d, scale=A_SCALE): + """Draw the 2D output above the 3D output, and return the left edge of each.""" + # the outputs are as large as the inputs, centered in the column of the widest output + w2 = w3 = INPUT_SIZE + wide = (SIZE + 3.5) * A_SCALE + x2, x3 = A_OUTPUT_X + (wide - w2) / 2, A_OUTPUT_X + (wide - w3) / 2 + scaled(drawing, w2 / SIZE, draw_2d, x2, y + row_2d) + item_label(drawing, A_OUTPUT_X, wide, y + row_2d + w2 / 2 + 2.2, "2D") + scaled(drawing, w3 / (SIZE + 3.5), draw_3d, x3, y + row_3d) + item_label(drawing, A_OUTPUT_X, wide, y + row_3d + w3 / 2 + 2.3, "3D") + return x2, x3 + + +def a_band(drawing, x, y, height, title, background, title_y, horizontal=False, title_x=None): + drawing.add( + f'' + ) + # a little smaller than the headers, so that it fits the height of the thin interactive band + if horizontal: + # at the top center inside the band + drawing.text((x + A_W) / 2 if title_x is None else title_x, y + 3.0, title, 1.8, weight="bold", color=BLUE) + else: + drawing.text(x + 3.1, title_y, title, 1.8, weight="bold", color=BLUE, rotate=-90) + + +def token_grid(drawing, x, y, size, n, highlight=None, style=ENCODER_STYLE): + """Draw one transformer block as a square grid of n x n tokens; highlight is the side of an attention window + in tokens, or n for global attention.""" + fill, edge = style + lines = "".join( + f"M{fmt(x + size * k / n)},{fmt(y)}v{fmt(size)}M{fmt(x)},{fmt(y + size * k / n)}h{fmt(size)}" + for k in range(1, n) + ) + parts = [ + f'', + f'', + ] + if highlight: + w = size * highlight / n + offset = 0.0 if highlight == n else size / n + parts.append( + f'' + ) + drawing.add("".join(parts)) + + +def hiera(drawing, cx, top): + """Draw the Hiera stages from top to bottom: transformer blocks on token grids that pooling halves from stage to + stage (the arrows between the stages, without a label). The early stages attend within + local windows, the late stages globally, and a neck right next to the stages merges all of them. Return the bottom + of the stages.""" + shift, y, links = 0.5, top, [] + widest = max(size + (layers - 1) * shift for _, size, layers, _ in HIERA_STAGES) + neck_x = cx + widest / 2 + 2.0 + for k, (n, size, layers, is_global) in enumerate(HIERA_STAGES): + width = size + (layers - 1) * shift + x = cx - width / 2 + y += (layers - 1) * shift + for layer in range(layers): + last = layer == layers - 1 + window = (n if is_global else n // 4) if last else None + token_grid(drawing, x + layer * shift, y - layer * shift, size, n, highlight=window) + links.append((x + width, y + size / 2)) + y += size + if k < len(HIERA_STAGES) - 1: + flow_arrow(drawing, [(cx, y + 0.3), (cx, y + HIERA_GAP - 0.3)], head=0.55, width=0.2) + y += HIERA_GAP + neck_top, neck_bottom = top - 0.5, y + 0.5 + fill, edge = ENCODER_STYLE + drawing.add( + f'' + ) + # move the baseline of the rotated text by half the letter height to center it + drawing.text(neck_x + 1.8 + 0.36 * drawing.text_size(1.5), (neck_top + neck_bottom) / 2, "neck", 1.5, rotate=-90) + for x, y_link in links: + flow_arrow(drawing, [(x + 0.3, y_link), (neck_x - 0.2, y_link)], head=0.5, width=0.18) + return y + + +def attention_legend(drawing, x, y): + """Explain the two kinds of attention of the Hiera stages with small token grids.""" + for k, (highlight, text) in enumerate(((1, "local attention"), (4, "global attention"))): + gy = y + k * 2.8 + token_grid(drawing, x, gy - 1.5, 2.2, 4, highlight=highlight) + drawing.text(x + 2.8, gy + 0.1, text, 1.3, color=SUBTEXT, anchor="start") + + +def draw_trunk(drawing): + """Draw the inputs, the hierarchical image encoder and the embeddings, which both heads share. The image and the + volume stack in one column; the image encoder takes a volume slice by slice.""" + # the 2D embedding lines up with the entry of the automatic head, so that its line runs straight through + cy_2d, cy_3d = A_ENTRY, A_TRUNK_3D + # the inputs are as large as the column allows (larger images and volumes) + column = 15.4 + drawing.text(column / 2, A_HEADER_Y, "Input", 2.12, weight="bold") + scaled(drawing, INPUT_SIZE / SIZE, slice_input, 0.3, cy_2d) + item_label(drawing, 0.0, column, cy_2d + INPUT_SIZE / 2 + 2.4, "2D image") + height = place_render(drawing, render_em_block(), 0.4, cy_3d, INPUT_SIZE) + item_label(drawing, 0.0, column, cy_3d + height / 2 + 2.4, "3D volume") + # the Hiera encoder, a hierarchical vision transformer, downsamples each image or slice in stages + (x0, x1), top = A_ENCODER_X, 1.5 + box_index = len(drawing.body) + drawing.text((x0 + x1) / 2, A_HEADER_Y, "Image encoder", 2.12, weight="bold") + # the box fits the stages, its legend and its caption, and still takes the arrow of the volume + bottom = max(hiera(drawing, x0 + 7.8, top + 7.5) + 6.6, cy_3d + 6.0) + 2.0 + fill, edge = ENCODER_STYLE + drawing.body.insert(box_index, ( + f'' + )) + attention_legend(drawing, x0 + 1.4, bottom - 4.5) + # the name of the encoder stands under its box + item_label(drawing, x0, x1 - x0, bottom + 2.3, "Hierarchical ViT (Hiera)", size=1.5) + flow_arrow(drawing, [(column + 0.6, cy_2d), (x0 - 0.7, cy_2d)]) + flow_arrow(drawing, [(column + 0.6, cy_3d), (x0 - 0.7, cy_3d)]) + emb_x = x1 + 3.5 + for cy in (cy_2d, cy_3d): + flow_arrow(drawing, [(x1 + 0.6, cy), (emb_x - 0.6, cy)]) + drawing.text(emb_x + 5.9, A_HEADER_Y, "Embeddings", 2.12, weight="bold") + embedding_slab(drawing, emb_x, cy_2d + 1.5, 8.0, 5.0) + item_label(drawing, emb_x, 11.75, cy_2d + 6.0, "1 × embedding") + for k in range(len(SLICE_INDICES)): + embedding_slab(drawing, emb_x, cy_3d + 4.0 - 0.95 * k, 8.0, 5.0) + item_label(drawing, emb_x, 11.75, cy_3d + 8.6, "Z × embeddings") + # both embeddings go to the automatic head on the right and down to the interactive head + bus_x = emb_x + 15.0 + drawing.add( + f'' + ) + return bus_x, (top, bottom) + + +def recycle_icon(drawing, cx, cy, r, color): + """Two arrows that chase each other around a circle: the memory is used again and again.""" + for start in (20.0, 200.0): + a0, a1 = np.radians(start), np.radians(start + 130.0) + x0, y0 = cx + r * np.cos(a0), cy + r * np.sin(a0) + x1, y1 = cx + r * np.cos(a1), cy + r * np.sin(a1) + drawing.add( + f'' + ) + # the head points along the circle, clockwise + tangent = np.array([-np.sin(a1), np.cos(a1)]) + normal = np.array([np.cos(a1), np.sin(a1)]) + tip = np.array([x1, y1]) + 0.55 * tangent + left, right = tip - 0.75 * tangent + 0.38 * normal, tip - 0.75 * tangent - 0.38 * normal + drawing.add( + f'' + ) + + +def memory_bank(drawing, cx, cy, size=5.6, gap=0.7, step=14, squeeze=0.45, shear=-0.32): + """Draw the memory bank centered at (cx, cy) on a card with rounded corners: a recycle sign, the masks of the + slices that were already processed, standing side by side like books with a small gap and facing left, the way + the memory flows, and its label. Returns the left and right edges of the card.""" + labels = neuron_volume() + # the prompted middle slice stands in front, the slices that the memory reached from it behind + indices = [MIDDLE, MIDDLE + step, MIDDLE - step, MIDDLE + 2 * step] + w = size * squeeze + shift = w + gap + icon_r, pad = 1.5, 1.4 + label_w = max( + text_width("memory bank", drawing.text_size(1.25), bold=True), + text_width("masks of processed slices", drawing.text_size(1.2)) + ) + slices_w = (len(indices) - 1) * shift + w + total = pad + 2 * icon_r + 1.2 + slices_w + 1.2 + label_w + pad + x0, x1 = cx - total / 2, cx + total / 2 + y0, y1 = cy - size / 2 - 1.6, cy + size / 2 + 1.6 + fill, edge = INTERACTIVE_STYLE + drawing.add( + f'' + ) + recycle_icon(drawing, x0 + pad + icon_r, cy, icon_r, mix(edge, "#000000", 0.2)) + x = x0 + pad + 2 * icon_r + 1.2 + top = cy - size / 2 - shear * size * squeeze / 2 + # the slices further right stand behind, so they are drawn first + for k, index in reversed(list(enumerate(indices))): + # the memory keeps the masks themselves, shown in pastel red on a light background + mask = labels[index] == NEURON_LABEL + image = np.where(mask[..., None], hex_to_rgb("#ec8a84"), hex_to_rgb("#f4f1f6")) + matrix = ",".join(fmt(v) for v in (squeeze, shear * squeeze, 0, 1, x + k * shift, top)) + drawing.add( + f'' + ) + label_x = x + slices_w + 1.2 + drawing.text(label_x, cy - 0.3, "memory bank", 1.25, weight="bold", color=TEXT, anchor="start") + drawing.text(label_x, cy + 1.7, "masks of processed slices", 1.2, color=SUBTEXT, anchor="start") + return x0, x1 + + +def apg_frame(): + """The dashed frame of the APG stack (x0, y0, x1, y1), centered on the APG arrow above the prompt encoder, the + left edge of the stack and the center height of its front sheet.""" + line_x, shift = PROMPT_ENCODER_X + PROMPT_ENCODER_W / 2, 3 * 0.85 * 9.0 / SIZE + apg_cy = INTER_Y + 9.6 + width = 9.0 + shift + stack_x = line_x - width / 2 + frame = (stack_x - 0.7, apg_cy - 4.5 - shift - 0.7, stack_x + width + 0.7, apg_cy + 4.5 + 0.7) + return frame, stack_x, apg_cy + + +def draw_interactive_head(drawing, bus_x): + """Draw the interactive band below the trunk: the prompts below the inputs, the prompt encoder below the image + encoder, the memory attention that propagates a mask through volumes, and the mask decoder.""" + y0 = INTER_Y + a_band( + drawing, 0.0, y0, INTER_H, "Interactive segmentation", INTERACTIVE_BACKGROUND, y0 + INTER_H / 2, horizontal=True + ) + decoder_x = A_OUTPUT_X - 30.0 + decoder_right = decoder_x + 13.5 + module( + drawing, decoder_x, y0 + 1.5, decoder_right - decoder_x, INTER_H - 3.0, ["Mask", "decoder"], INTERACTIVE_STYLE + ) + # 2D embeddings go straight to the mask decoder, volumes pass the memory attention first + split, bypass, memory = bus_x, y0 + 0.14 * INTER_H, y0 + 0.34 * INTER_H + drawing.add( + f'' + ) + flow_arrow(drawing, [(split, bypass), (decoder_x - 0.7, bypass)]) + drawing.text(split + 1.5, bypass - 0.8, "2D", 1.3, color=SUBTEXT, anchor="start") + memory_x = split + 6.0 + flow_arrow(drawing, [(split, memory), (memory_x - 0.7, memory)]) + drawing.text(split + 1.5, memory - 0.8, "3D", 1.3, color=SUBTEXT, anchor="start") + module(drawing, memory_x, memory - 3.6, 26.0, 7.2, ["Memory attention"], INTERACTIVE_STYLE) + flow_arrow(drawing, [(memory_x + 26.7, memory), (decoder_x - 0.7, memory)]) + # the APG prompts of a volume take the memory path too: dashed lines next to it + apg = dict(color=TEXT, dash="0.9,0.5") + flow_arrow(drawing, [(memory_x + 26.7, memory + 1.0), (decoder_x - 0.7, memory + 1.0)], **apg) + # the loop from the mask decoder through the memory bank into the memory attention + loop_y, loop_x, bend = y0 + 0.72 * INTER_H, memory_x + 13.0, 2.0 + drawing.add( + f'' + ) + flow_arrow(drawing, [(loop_x, memory + 5.5), (loop_x, memory + 4.2)]) + # the dashed APG loop runs inside the solid loop, with the same round bend + dx, dy = loop_x + 1.0, loop_y - 1.0 + drawing.add( + f'' + ) + flow_arrow(drawing, [(dx, memory + 5.5), (dx, memory + 4.2)], color=TEXT) + memory_bank(drawing, (loop_x + decoder_x) / 2, loop_y) + # prompts are placed on a 2D image, or on one slice of a volume, at the left + row_2d, row_3d = y0 + A_ROW_2D, y0 + A_ROW_3D + # the prompted volume sits lower, in the room at the bottom left of the band + in_row_3d = row_3d + 3.0 + prompt = 12.5 # the size of the prompted image + # the prompted inputs move left into the room of the former vertical band title + in_2d, in_3d = 3.0, 2.0 + scaled(drawing, prompt / SIZE, slice_prompted, in_2d, row_2d) + place_render(drawing, render_prompted_slice(), in_3d, in_row_3d, 1.2 * prompt) + encoder_x, encoder_y, elbow = PROMPT_ENCODER_X, y0 + INTER_H / 2, 23.5 + # the APG prompts: a dashed frame on the APG arrow, so that both read as one unit + (fx0, fy0, fx1, fy1), stack_x, apg_cy = apg_frame() + scaled(drawing, 9.0 / SIZE, slice_apg_sheets, stack_x, apg_cy) + flow_arrow(drawing, [((fx0 + fx1) / 2, fy1), ((fx0 + fx1) / 2, encoder_y - 5.0 - 0.7)], color=TEXT, dash="0.9,0.5") + drawing.add( + f'' + ) + module(drawing, encoder_x, encoder_y - 5.0, PROMPT_ENCODER_W, 10.0, ["Prompt", "encoder"], INTERACTIVE_STYLE) + inputs = ((in_2d + 0.6 + prompt, row_2d, encoder_y - 2.0), (in_3d + 0.6 + 1.2 * prompt, in_row_3d, encoder_y + 2.0)) + for start, row, end in inputs: + flow_arrow(drawing, [(start, row), (elbow, row), (elbow, end), (encoder_x - 0.7, end)]) + # the prompt embeddings run below the memory to the mask decoder + lane = y0 + INTER_H - 2.5 + flow_arrow(drawing, [(encoder_x + 8.5, encoder_y + 5.6), (encoder_x + 8.5, lane), (decoder_x - 0.7, lane)]) + # the APG prompts take the same way to the mask decoder + apg_lane = lane - 1.0 + flow_arrow( + drawing, [(encoder_x + 9.5, encoder_y + 5.6), (encoder_x + 9.5, apg_lane), (decoder_x - 0.7, apg_lane)], + color=TEXT, dash="0.9,0.5" + ) + # the outputs in two rows: promptable segmentation on top, APG below + tile, width_3d = 10.5, 12.0 + # the outputs keep to the right edge, so that the arrows from the mask decoder get room + x_3d = A_W - width_3d - 0.3 + x_2d = x_3d - tile - 2.5 + # the interactive outputs: the 2D output left of the 3D output + row = row_2d + flow_arrow(drawing, [(decoder_right + 0.7, row), (x_2d - 0.6, row)]) + scaled(drawing, tile / SIZE, slice_interactive, x_2d, row) + height = place_render(drawing, render_neuron(), x_3d, row, width_3d) + top, bottom = row - max(tile, height) / 2, row + max(tile, height) / 2 + 2.0 + drawing.text((x_2d + x_3d + width_3d) / 2, top - 1.0, "Interactive segmentation", 1.3, weight="bold", color=SUBTEXT) + drawing.text(x_2d + tile / 2, bottom, "2D", 1.5, color=SUBTEXT) + drawing.text(x_3d + width_3d / 2, bottom, "3D", 1.5, color=SUBTEXT) + # the APG output: one stack of slices, as the APG input + row, stack = row_3d + 3.5, 13.0 + shift = 3 * 0.85 * stack / SIZE + x_stack = (x_2d + A_W - 0.3) / 2 - (stack + shift) / 2 + flow_arrow(drawing, [(decoder_right + 0.7, row), (x_stack - 0.6, row)], color=TEXT, dash="0.9,0.5") + scaled(drawing, stack / SIZE, slice_apg_output_sheets, x_stack, row) + drawing.text( + x_stack + (stack + shift) / 2, row - stack / 2 - shift - 1.0, "APG: automatic segmentation", 1.3, weight="bold", + color=SUBTEXT + ) + drawing.text(x_stack + stack / 2, row + stack / 2 + 2.0, "2D / 3D", 1.5, color=SUBTEXT) + + +def draw_automatic_head(drawing, bus_x, encoder_box): + """Draw the automatic band: the segmentation decoder, as tall as its stages, and the maps it predicts for an + image and for a volume, with the instance segmentation made from each: the 2D maps right of the decoder, the 3D + maps in a row below it.""" + # the title is centered in the room right of the decoder box + a_band( + drawing, A_BAND_X, 0.0, AUTO_BAND_H, "Automatic segmentation", AUTOMATIC_BACKGROUND, AUTO_BAND_H / 2 + 4.0, + horizontal=True, title_x=(A_CONTENT_X + 40.0 + A_OUTPUT_X) / 2 + 2.0 + ) + top = encoder_box[0] + # the box ends a little below the largest feature map (drawn at 0.85 of its height and 0.75 of its depth) + bottom = A_ENTRY + max(0.85 * h / 2 + 0.75 * d for _, h, d in STAGES) + 2.0 + x0, x1 = A_CONTENT_X, A_CONTENT_X + 40.0 + fill, edge = AUTOMATIC_STYLE + drawing.add( + f'' + ) + drawing.text((x0 + x1) / 2, A_HEADER_Y, "Segmentation decoder", 2.12, weight="bold") + flow_arrow(drawing, [(bus_x, A_ENTRY), (x0 + 1.3, A_ENTRY)]) + # an image passes from stage to stage, a volume takes the detour through DSP + cy, x, markers = A_ENTRY, x0 + 2.0, [] + # the DSP circles are wide enough for their name at the smallest print size + gap, drop, marker = 5.6, 3.8, 2.3 + for w, h, d in STAGES[::-1]: + x = feature_map(drawing, x, cy, w, 0.85 * h, 0.75 * d, AUTOMATIC_STYLE) + mx = x + gap / 2 + markers.append(mx) + flow_arrow(drawing, [(x + 0.25, cy), (x + gap - 0.3, cy)], head=0.5, width=0.2) + drawing.add( + f'' + ) + x += gap + # the key of the dashed path: DSP is applied only in 3D + for dash, text in (("0.35,0.3", "3D only: DSP = Depth Strip Pooling"),): + ky = bottom - 1.2 + dashed = f' stroke-dasharray="{dash}"' if dash else "" + drawing.add( + f'' + ) + drawing.text(x0 + 4.8, ky, text, 1.3, color=SUBTEXT, anchor="start") + for mx in markers: + drawing.add( + f'' + ) + # darker than the outline, for a contrast above 4.5 on white + drawing.text(mx, cy + drop + 0.55, "DSP", 0.85, weight="bold", color=mix(edge, "#000000", 0.4)) + # the outputs of the decoder, computed with the training targets of micro-sam; z only exists in volumes + maps_2d = decoder_maps()[0] + names = ["foreground", "boundary", "x-distance", "y-distance", "z-distance"] + front, depth, map_gap = 4.8, 1.6, 0.7 + pitch, mx = front + depth + map_gap, x1 + 4.0 + row_2d, row_3d = A_ENTRY, AUTO_ROW_3D + # the four 2D maps stretch over the whole row, the five 3D maps sit in their own rows; each map is named + width = len(names) * pitch - map_gap + tile = (width - 3 * map_gap) / 4 + for k, name in enumerate(names[:4]): + tx = mx + k * (tile + map_gap) + drawing.text(tx + tile / 2, row_2d - tile / 2 - 1.2, name, 1.3, color=SUBTEXT) + map_tile(drawing, tx, row_2d - tile / 2, tile, maps_2d[name]) + maps_end = mx + width + # the 3D maps in one row below the decoder, ending before the 2D maps so that the arrow has room + block, spacing = 12.4, 1.6 + row_width = len(names) * block + (len(names) - 1) * spacing + row_start = (x0 + 0.5 + maps_end - 4.0) / 2 - row_width / 2 + row_end = row_start + row_width + for k, name in enumerate(names): + tx = row_start + k * (block + spacing) + height = place_render(drawing, render_map_block(name), tx, row_3d, block) + drawing.text(tx + block / 2, row_3d + height / 2 + 2.0, name, 1.3, color=SUBTEXT) + flow_arrow(drawing, [(x - 0.3, row_2d), (mx - 0.7, row_2d)]) + branch = x1 + 1.6 + flow_arrow(drawing, [(branch, row_2d), (branch, row_3d - block / 2 - 0.6)]) + x2, x3 = outputs(drawing, slice_automatic, automatic_3d_render, 0.0, row_2d=row_2d, row_3d=row_3d) + # AIS post-processes the maps; APG makes prompts from them for the interactive branch + for row, start, end in ((row_2d, maps_end, x2), (row_3d, row_end, x3)): + flow_arrow(drawing, [(start + 0.5, row), (end - 1.0, row)]) + drawing.text((maps_end + x2) / 2 - 0.4, row_2d - 1.0, "AIS", 1.15, weight="bold", color=SUBTEXT) + apg_x = maps_end + 1.6 + gap_y = AUTO_BAND_H + A_BAND_GAP / 2 + target_x = PROMPT_ENCODER_X + PROMPT_ENCODER_W / 2 + # the arrow ends on the top of the dashed frame of the APG stack (see 'apg_frame') + target_y = apg_frame()[0][1] + # the APG path touches the arrows to both outputs; it is drawn after the interactive band + return [(apg_x, row_2d), (apg_x, gap_y), (target_x, gap_y), (target_x, target_y)], [row_2d, row_3d] + + +def draw_apg(drawing, points, taps): + """The dashed APG path from the decoder maps down to the prompt encoder, with its name along the line between + the arrows to the 2D and the 3D output.""" + # darker than the other arrows, without a head: it runs into the frame of the APG stack + line = "M" + "L".join(f"{fmt(px)},{fmt(py)}" for px, py in points) + drawing.add( + f'' + ) + apg_x = points[0][0] + for y in taps: + drawing.add(f'') + # the name runs along the dashed line between the two taps, as AIS names its arrow + drawing.text(apg_x - 0.7, (taps[0] + taps[1]) / 2, "APG", 1.3, weight="bold", color=SUBTEXT, rotate=-90) + + +def fetch_python_logo(path, url=PYTHON_LOGO_URL): + """Download the Python logo to path, unless it is there already.""" + if os.path.exists(path): + return + os.makedirs(os.path.dirname(path), exist_ok=True) + request = urllib.request.Request(url, headers={"User-Agent": "micro-sam-figures"}) + with urllib.request.urlopen(request) as response: + data = response.read() + with open(path, "wb") as f: + f.write(data) + print(f"Downloaded the Python logo to {path}") + + +# panel b: example crops of the training data, one per dataset or imaging technique, grouped by domain +DATA_ROOT = "/mnt/vast-nhr/projects/cidas/cca/data" +# the number of datasets of the generalist model; the four LM datasets with both modalities count once here +B_TRAIN_DATASETS = 91 +# the datasets per domain; a dataset with both LM modalities counts in the one of most of its images +B_DOMAIN_DATASETS = {"fluorescence": 34, "label-free": 18, "em": 21, "histopathology": 18} +# the annotated objects per domain (level C of figures/figure1/data_stats/full, counted 2026-11-05); LICONN +# (83,837 objects) and XPRESS (51) count with EM, as in the text; full_stats.csv puts LICONN in fluorescence +B_OBJECTS = { + "fluorescence": 21653662 - 83837, "label-free": 4443510, "em": 2083391 + 83837, "histopathology": 1707969, +} +# the pixels of all 2D images and the voxels of all 3D volumes of the same data +B_PIXELS, B_VOXELS = 57086325840, 230487473654 +# the domains with their titles, their quadrant (column, row) around the hub, and their fill and edge colors +B_DOMAINS = ( + ("fluorescence", "Fluorescence microscopy", (0, 0), ("#e2ece6", "#5f8a78")), + ("label-free", "Label-free microscopy", (1, 0), ("#efe9dc", "#9a8256")), + ("em", "Electron microscopy", (0, 1), ("#e6e4ee", "#736ca3")), + ("histopathology", "Histopathology", (1, 1), ("#f1e2e8", "#8f6676")), +) +# the crops are resized to this size in pixels before they are drawn +B_TILE_PX = 256 +# each quadrant holds a grid of tiles without the cell in its inner corner, which the pie chart covers +B_COLUMNS, B_ROWS = 3, 2 +B_GAP, B_PAD, B_QUADRANT_GAP = 2.2, 1.6, 1.4 +# the gap between the rows of tiles, larger than between the columns, for some air +B_ROW_GAP = 5.0 +# how far the cards behind a tile reach to its right +B_CARD_REACH = 1.5 +# the tiles are as large as needed for their captions at 5 pt; panel c takes the rest of the width +B_TILE = 12.6 +B_CHANNEL_COLORS = { + "gray": (1.0, 1.0, 1.0), "red": (1.0, 0.25, 0.2), "green": (0.2, 1.0, 0.35), "blue": (0.3, 0.45, 1.0), + "magenta": (1.0, 0.2, 1.0), "cyan": (0.1, 0.9, 1.0), "yellow": (1.0, 0.9, 0.1), +} +# the example crops: 'crop' is [y0, x0, size] in source pixels, 'index' the slice or frame +B_DATASETS = [ + dict( + dataset="Omnipose", domain="fluorescence", caption=("widefield", "bacteria"), + raw_path="omnipose/data/bact_fluor/train_sorted/cex/s_flex_cex_xy2_tile12_cyto.tif", raw_key=None, + label_path="omnipose/data/bact_fluor/train_sorted/cex/s_flex_cex_xy2_tile12_cyto_masks.tif", label_key=None, + crop=(0, 0, 508), + ), + dict( + dataset="TissueNet", domain="fluorescence", caption=("multiplexed", "tissue"), + raw_path="tissuenet/train/image_1100.zarr", raw_key="raw/rgb", label_key="labels/cell", + channels=[(1, "green"), (0, "magenta")], channel_axis=0, crop=(0, 0, 512), + ), + dict( + dataset="PlantSeg", domain="fluorescence", caption=("light-sheet", "plant root"), ndim=3, + # the volumes are the time points of three movies; the cards behind the tile show further slices + time_lapse=True, + raw_path="plantseg/root_train/Movie1_t00049_crop_gt.h5", raw_key="raw", label_key="label", index=139, + crop=(130, 280, 320), + ), + dict( + dataset="DynamicNuclearNet", domain="fluorescence", caption=("widefield", "nuclei"), time_lapse=True, + raw_path="dynamicnuclearnet/train/image_2520.zarr", raw_key="raw", label_key="labels", crop=(0, 0, 512), + # the frames of a movie are consecutive files; 2520, 2522 and 2524 belong to the same movie + frame_pattern="dynamicnuclearnet/train/image_{:04d}.zarr", frame=2520, + ), + # LICONN counts as EM-like, so a confocal C. elegans volume is the fluorescence example + dict( + dataset="C. elegans atlas", domain="fluorescence", caption=("confocal", "C. elegans"), ndim=3, + raw_path="celegans_atlas/c_elegans_nuclei/train/images/cnd1threeL1_1213061.tif", raw_key=None, + label_path="celegans_atlas/c_elegans_nuclei/train/masks/cnd1threeL1_1213061.tif", label_key=None, index=70, + crop=(25, 250, 90), + ), + dict( + dataset="OrgLine", domain="label-free", caption=("brightfield", "organoids"), + raw_path="orgline/data1/Intestine/train/20210727_38203_ckufhb5ev1aqr0y8e55bv30ld_img.jpg.h5", raw_key="image", + label_key="masks", crop=(1488, 1104, 384), + ), + dict( + dataset="NeurIPS CellSeg", domain="label-free", caption=("stained BF", "bone marrow"), + raw_path="neurips_cellseg/Training-labeled/images/cell_00011.bmp", raw_key=None, + label_path="neurips_cellseg/Training-labeled/labels/cell_00011_label.tiff", label_key=None, rgb=True, + crop=(360, 360, 960), + ), + dict( + dataset="NeurIPS CellSeg", domain="label-free", caption=("DIC", "cultured cells"), + raw_path="neurips_cellseg/Training-labeled/images/cell_00318.tif", raw_key=None, + label_path="neurips_cellseg/Training-labeled/labels/cell_00318_label.tiff", label_key=None, + crop=(704, 1188, 352), + ), + dict( + dataset="VICAR", domain="label-free", caption=("QPI", "cancer cells"), + raw_path="vicar/labelled/PC3/00231_PC3_img.tif", raw_key=None, + label_path="vicar/labelled/PC3/00231_PC3_mask.png", label_key=None, crop=(216, 0, 384), + ), + dict( + dataset="Cell-ACDC", domain="label-free", caption=("phase contrast", "budding yeast"), time_lapse=True, + raw_path=( + "cell_acdc/test_data_budding_yeast/TimeLapse_2D/MIA_KC_htb1_mCitrine_labeled/Position_4/Images/" + "19-03-2021_KCY050_SCGE_s04_phase_contr.tif" + ), + raw_key=None, + label_path=( + "cell_acdc/test_data_budding_yeast/TimeLapse_2D/MIA_KC_htb1_mCitrine_labeled/Position_4/Images/" + "19-03-2021_KCY050_SCGE_s04_segm.tif" + ), + label_key=None, index=168, crop=(0, 24, 192), + ), + dict( + dataset="CREMI", domain="em", caption=("ssTEM", "fly brain"), ndim=3, raw_path="cremi/sampleA.h5", + raw_key="volumes/raw", label_key="volumes/labels/neuron_ids", index=24, crop=(672, 0, 384), + ), + dict( + dataset="Platynereis cells", domain="em", caption=("SBEM", "platynereis"), ndim=3, + raw_path="platynereis/membrane/train_data_membrane_01.n5", raw_key="volumes/raw/s1", + label_key="volumes/labels/segmentation_corrected/v1/ignore_16777215/s1", index=64, crop=(96, 96, 384), + ), + dict( + dataset="Wafer4", domain="em", caption=("mSEM", "cortex"), ndim=3, + raw_path="wafer4/wafer4_inputs.h5", raw_key="main", label_path="wafer4/wafer4_labels.h5", label_key="main", + index=40, crop=(400, 400, 384), + ), + dict( + dataset="ASTIH", domain="em", caption=("SEM", "spinal cord"), + raw_path="astih/SEM1/train/sub-rat4_sample-data12_SEM.h5", raw_key="raw", label_key="labels", + instances=["class", 2], crop=(100, 100, 512), + ), + dict( + dataset="Tumor spheroid EM", domain="em", caption=("SBF-SEM", "spheroid"), + raw_path="tumor_spheroid_em/2d_manual/50-50-50/Au_01-vol_01-z_0034.h5", raw_key="raw", + label_key="labels/cells", crop=(0, 0, 2000), + ), + dict( + dataset="LyNSeC", domain="histopathology", caption=("IHC", "lymphoma"), + raw_path="lynsec/data/ihc/images/302_l1.tif", raw_key=None, label_path="lynsec/data/ihc/labels/302_l1.tif", + label_key=None, rgb=True, channel_axis=2, crop=(0, 0, 256), + ), + dict( + dataset="SRSA-Net", domain="histopathology", caption=("IHC", "lung TMA"), + raw_path="srsanet/IHC_TMA_dataset/images/fold1_122_CD20.png", raw_key=None, + label_path="srsanet/IHC_TMA_dataset/preprocessed_labels/fold1_122_CD20.tif", label_key=None, rgb=True, + channel_axis=2, crop=(0, 0, 256), + ), + dict( + dataset="CryoNuSeg", domain="histopathology", caption=("cryosection", "thymus"), + raw_path="cryonuseg/tissue images/Human_Thymus_01.tif", raw_key=None, + label_path="cryonuseg/Annotator 1 (biologist)/label masks modify/Human_Thymus_01.tif", label_key=None, rgb=True, + channel_axis=2, crop=(0, 256, 256), + ), + dict( + dataset="SPATCH", domain="histopathology", caption=("Visium HD", "colon"), + raw_path="spatch/visium_hd_coad/data/tile4.h5", raw_key="raw/rgb", label_key="labels/nuclei", rgb=True, + channel_axis=0, crop=(64, 192, 256), + ), + dict( + dataset="PUMA", domain="histopathology", caption=("H&E", "melanoma"), + raw_path="puma/train/preprocessed/training_set_primary_roi_081.h5", raw_key="raw", + label_key="labels/instances/nuclei", rgb=True, channel_axis=0, crop=(768, 288, 256), + ), +] + + +def open_array(path, key): + """Open an array lazily from h5, zarr, n5 or tif files, or read a small image file.""" + extension = os.path.splitext(path.rstrip("/"))[1].lower() + if extension in (".h5", ".hdf5", ".hdf", ".zarr", ".n5"): + data = open_file(path, "r") + return data if key is None else data[key] + if extension in (".tif", ".tiff"): + store = tifffile.imread(path, aszarr=True) + data = zarr.open(store, mode="r") + # pyramidal tifs open as a group of levels; the first level has the full resolution + return data if hasattr(data, "shape") else data["0"] + return imageio.imread(path) + + +def read_crop(path, key, index, crop, channel_axis=None): + """Read the crop [y0, x0, size] of an image, with its channels last if it has a channel axis. + + The channel axis counts in the full array. The last two other axes are the spatial axes, and 'index' selects along + the axes in front of them, such as the slice of a volume or the image of a stack. + """ + data = open_array(path, key) + ndim = len(data.shape) + index = [] if index is None else list(index) if isinstance(index, (list, tuple)) else [index] + # an axis beyond the index and the two spatial axes holds the channels: the last one if short, else the first one + if channel_axis is None and ndim == len(index) + 3: + channel_axis = -1 if data.shape[-1] <= 4 else 0 + if channel_axis is not None: + channel_axis %= ndim + others = [axis for axis in range(ndim) if axis != channel_axis] + if len(index) != len(others) - 2: + raise ValueError(f"The index {index} does not fit the shape {data.shape} of {path}.") + y0, x0, size = crop + box = [slice(None)] * ndim + for axis, value in zip(others[:-2], index): + box[axis] = value + box[others[-2]], box[others[-1]] = slice(y0, y0 + size), slice(x0, x0 + size) + crop = np.asarray(data[tuple(box)]) + if channel_axis is None: + return crop + # the indexed axes are gone, so the channel axis moves by the number of indexed axes in front of it + shift = sum(axis < channel_axis for axis in others[:-2]) + return np.moveaxis(crop, channel_axis - shift, -1) + + +def percentile_normalize(image, low=1.0, high=99.5): + image = image.astype("float32") + lo, hi = np.percentile(image, [low, high]) + return np.clip((image - lo) / max(hi - lo, 1e-6), 0, 1) + + +def to_display(raw, spec): + """Map a raw crop to rgb in [0, 1]: rgb images as they are, channels composited in their colors, gray otherwise.""" + if spec.get("rgb") and raw.ndim == 3: + rgb = raw[..., :3].astype("float32") + return rgb / 255.0 if 0 <= rgb.min() and rgb.max() <= 255 else percentile_normalize(rgb, 0.5, 99.8) + if spec.get("channels"): + rgb = np.zeros(raw.shape[:2] + (3,), dtype="float32") + for channel, color in spec["channels"]: + rgb += percentile_normalize(raw[..., channel])[..., None] * np.array(B_CHANNEL_COLORS[color]) + return np.clip(rgb, 0, 1) + if raw.ndim == 3: + raw = raw[..., 0] + return np.repeat(percentile_normalize(raw)[..., None], 3, axis=-1) + + +def resize(image, size, nearest=False): + if nearest: + rows = (np.arange(size) * image.shape[0] / size).astype(int) + cols = (np.arange(size) * image.shape[1] / size).astype(int) + return image[rows][:, cols] + channels = [ + np.asarray(Image.fromarray(image[..., c].astype("float32")).resize((size, size), Image.BILINEAR)) + for c in range(image.shape[-1]) + ] + return np.clip(np.stack(channels, axis=-1), 0, 1) + + +# the ignore labels of the training data +IGNORE_LABELS = (2 ** 64 - 1, 2 ** 32 - 1, 2 ** 24 - 1, 2 ** 16 - 1, -1) + + +def instance_labels(labels, instances=None): + """Get the instances of a label crop: as they are, as the connected components of the foreground ('components'), + or as the connected components of one class (['class', k]).""" + labels = labels.astype("int64") if labels.dtype != np.uint64 else labels.astype("float64") + labels[np.isin(labels, IGNORE_LABELS)] = 0 + labels = labels.astype("int64") + if instances == "components": + labels = ndimage.label(labels > 0)[0] + elif instances is not None: + labels = ndimage.label(labels == instances[1])[0] + # consecutive ids keep the color table of the overlay small + return np.unique(np.maximum(labels, 0), return_inverse=True)[1].reshape(labels.shape) + + +@lru_cache +def dataset_tile(k): + """Load the crop of the k-th example and return its rgb image and its instance labels at the tile size.""" + spec = B_DATASETS[k] + raw_path = os.path.join(DATA_ROOT, spec["raw_path"]) + label_path = os.path.join(DATA_ROOT, spec.get("label_path", spec["raw_path"])) + raw = read_crop(raw_path, spec.get("raw_key"), spec.get("index"), spec["crop"], spec.get("channel_axis")) + labels = read_crop(label_path, spec.get("label_key"), spec.get("label_index", spec.get("index")), spec["crop"]) + if labels.ndim == 3: + labels = labels[..., 0] + labels = instance_labels(labels, spec.get("instances")) + rgb = resize(to_display(raw, spec), B_TILE_PX) + labels = resize(labels, B_TILE_PX, nearest=True) + return rgb, labels + + +@lru_cache +def neighbor_slice(k, step, spacing=4): + """Load the image of the slice or frame 'step * spacing' away from the one of the k-th example, in the direction + that stays inside the volume or movie, at a smaller size and a little darker, for the cards behind its tile.""" + spec = B_DATASETS[k] + if "frame_pattern" in spec: + # a movie stored as one file per frame: the later frames are the files with the next numbers + path = os.path.join(DATA_ROOT, spec["frame_pattern"].format(spec["frame"] + step * spacing)) + raw = read_crop(path, spec.get("raw_key"), None, spec["crop"], spec.get("channel_axis")) + return 0.85 * resize(to_display(raw, spec), B_TILE_PX // 2) + path = os.path.join(DATA_ROOT, spec["raw_path"]) + depth = open_array(path, spec.get("raw_key")).shape[0] + index = spec["index"] + step * spacing + if index >= depth: + index = spec["index"] - step * spacing + raw = read_crop(path, spec.get("raw_key"), index, spec["crop"], spec.get("channel_axis")) + return 0.85 * resize(to_display(raw, spec), B_TILE_PX // 2) + + +def label_overlay(rgb, labels, seed): + """Lay the instances over the lower right half of the image, split by the diagonal, in the label colors.""" + n = rgb.shape[0] + ids = np.unique(labels[labels > 0]) + rng = np.random.default_rng(seed) + palette = neurite_colors(14) + colors = np.zeros((int(labels.max()) + 1, 3)) + colors[ids] = palette[rng.integers(0, len(palette), len(ids))] + rows, cols = np.mgrid[:n, :n] + half = (rows + cols >= n)[..., None] + gray = rgb.mean(axis=-1, keepdims=True) + inside = (labels > 0)[..., None] + boundary = (ndimage.grey_dilation(labels, size=3) != ndimage.grey_erosion(labels, size=3)) & (labels > 0) + filled = np.where(inside, 0.4 * gray + 0.6 * colors[labels], 0.75 * gray) + filled = np.where(boundary[..., None], 0.25 * gray + 0.75 * colors[labels] * 1.1, filled) + return np.where(half, np.clip(filled, 0, 1), rgb) + + +def b_tile(drawing, x, y, size, k, spec): + """Draw one example as a rounded tile, with a stack behind it for volumes and movies, and its caption on it in + small text: the modality on the upper left, the tissue or organism on the lower right.""" + if spec.get("ndim", 2) == 3 or spec.get("time_lapse"): + # the cards behind the tile show the slices further in the volume, or the frames later in the movie + spacing = 2 if spec.get("time_lapse") else 4 + for step in (2, 1): + o = 0.75 * step + clip = drawing.clip_rect(x + o, y - o, size, size, 0.6) + drawing.add( + f'' + f'' + ) + rgb, labels = dataset_tile(k) + clip = drawing.clip_rect(x, y, size, size, 0.6) + drawing.add( + f'' + f'' + ) + modality, tissue = spec["caption"] + tile_tag(drawing, x + 0.2, y + 0.2, modality, "start") + tile_tag(drawing, x + size - 0.2, y + size - 2.2, tissue, "end") + + +def tile_tag(drawing, x, y, text, anchor): + """Write a small text (5 pt) on a tile, white on a tight dark box with its top at y, from x to the right (start) + or to the left (end).""" + size = drawing.text_size(1.76) + w = text_width(text, size) + 0.4 + x0 = x if anchor == "start" else x - w + drawing.add( + f'' + ) + drawing.text(x0 + w / 2, y + 1.48, text, 1.76, color="#ffffff") + + +def format_count(value): + """Format a number of objects in millions or thousands.""" + return f"{value / 1e6:.1f} M" if value >= 1e6 else f"{value / 1e3:.0f} k" + + +def quadrant_cells(column, row): + """Get the grid cells of a quadrant in reading order, without the cell of its inner corner.""" + inner = (B_ROWS - 1 if row == 0 else 0, B_COLUMNS - 1 if column == 0 else 0) + return [(r, c) for r in range(B_ROWS) for c in range(B_COLUMNS) if (r, c) != inner] + + +def arc_path(cx, cy, radius, start, stop): + """Get the svg path of a circular arc, with the angles in degrees clockwise from the top.""" + a0, a1 = np.radians(start), np.radians(stop) + x0, y0 = cx + radius * np.sin(a0), cy - radius * np.cos(a0) + x1, y1 = cx + radius * np.sin(a1), cy - radius * np.cos(a1) + large = 1 if stop - start > 180 else 0 + return f"M{fmt(x0)},{fmt(y0)}A{fmt(radius)},{fmt(radius)} 0 {large} 1 {fmt(x1)},{fmt(y1)}" + + +def b_pie(drawing, cx, cy, radius): + """Draw the pie chart of the training data: a ring split into the domains by their number of datasets, in the + edge colors of their quadrants, with the totals of the datasets, objects, pixels and voxels inside it.""" + counts = B_DOMAIN_DATASETS + total = sum(counts.values()) + assert total == B_TRAIN_DATASETS + width = 2.6 + ring = radius - width / 2 - 0.2 + # the disc ends at the ring, so that the card outlines merge into it + drawing.add(f'') + # the slices run clockwise in the order of the quadrants, each towards its quadrant + order = ("label-free", "histopathology", "em", "fluorescence") + start = 20.0 + colors = {domain: mix(edge, "#000000", 0.15) for domain, _, _, (_, edge) in B_DOMAINS} + angle = start + for domain in order: + span = 360.0 * counts[domain] / total + drawing.add( + f'' + ) + # the number of datasets of each slice, along the ring + middle = angle + span / 2 + turn = middle if middle % 360 < 90 or middle % 360 > 270 else middle - 180 + # one letter at a time along the ring, since rsvg-convert does not draw svg text paths + size = drawing.text_size(1.5) + upper = turn == middle + radius = ring - 0.36 * size if upper else ring + 0.36 * size + label = f"{counts[domain]} datasets" + widths = [text_width(c, size, bold=True) for c in label] + position = -sum(widths) / 2 + for char, w in zip(label, widths): + along = np.degrees((position + w / 2) / radius) * (1 if upper else -1) + theta = middle + along + lx, ly = cx + radius * np.sin(np.radians(theta)), cy - radius * np.cos(np.radians(theta)) + rot = theta if upper else theta - 180 + drawing.add( + f'' + f'{char}' + ) + position += w + angle += span + # white gaps between the slices + angle = start + for domain in order: + t = np.radians(angle) + x0, y0 = cx + (ring - width / 2) * np.sin(t), cy - (ring - width / 2) * np.cos(t) + x1, y1 = cx + (ring + width / 2) * np.sin(t), cy - (ring + width / 2) * np.cos(t) + drawing.add(f'') + angle += 360.0 * counts[domain] / total + drawing.text(cx, cy - 3.0, "Training data", 2.12, weight="bold", color=BLUE) + drawing.text(cx, cy - 0.7, f"{B_TRAIN_DATASETS} datasets", 1.94, weight="bold") + drawing.text(cx, cy + 1.4, f"{format_count(sum(B_OBJECTS.values()))} objects", 1.76, color=SUBTEXT) + drawing.text(cx, cy + 3.3, f"{B_PIXELS / 1e9:.0f} G pixels (2D)", 1.76, color=SUBTEXT) + drawing.text(cx, cy + 5.2, f"{B_VOXELS / 1e9:.0f} G voxels (3D)", 1.76, color=SUBTEXT) + + +def b_size(): + """Get the size of a quadrant of panel b, the margin of the panel and the size of the panel. The domain names sit + on the outer edges, so the quadrants need no header; the panel keeps a margin for the outer halves of the names.""" + width_q = 2 * B_PAD + B_CARD_REACH + B_COLUMNS * B_TILE + (B_COLUMNS - 1) * B_GAP + height_q = 5.4 + B_ROWS * B_TILE + (B_ROWS - 1) * B_ROW_GAP + margin = 0.8 * Drawing().text_size(2.12) + return width_q, height_q, margin, 2 * width_q + B_QUADRANT_GAP, 2 * height_q + B_QUADRANT_GAP + 2 * margin + + +def build_panel_b(): + """Draw panel b in its own coordinates in mm: the training data as a world of four domains around a pie chart of + their training objects. Each domain fills a quadrant with example crops of its datasets; each tile shows the + image in its upper left half and the annotated instances in its lower right half. Returns the content and the + size of the panel.""" + drawing = Drawing() + pitch_x, pitch_y = B_TILE + B_GAP, B_TILE + B_ROW_GAP + width_q, height_q, margin, width, height = b_size() + gap = B_QUADRANT_GAP + cx, cy = width / 2, height / 2 + tiles, boxes = [], [] + for domain, title, (column, row), (fill, edge) in B_DOMAINS: + qx, qy = column * (width_q + gap), margin + row * (height_q + gap) + drawing.add( + f'' + ) + # white cards with colored outlines; the titles sit on their outer edges + top = qy + (2.7 if row == 0 else 1.9) + title_y = qy if row == 0 else qy + height_q + title_x, anchor = (qx + B_PAD, "start") if column == 0 else (qx + width_q - B_PAD - B_CARD_REACH, "end") + edge_label(drawing, title_x, title_y, title, 2.12, "#ffffff", edge, mix(edge, "#000000", 0.35), anchor=anchor) + members = [(k, spec) for k, spec in enumerate(B_DATASETS) if spec["domain"] == domain] + for (r, c), (k, spec) in zip(quadrant_cells(column, row), members): + x, y = qx + B_PAD + c * pitch_x, top + r * pitch_y + tiles.append((x, y, k, spec)) + boxes.append((x - 1.0, y - 1.6, x + B_TILE + 1.0, y + B_TILE + 0.4)) + # the pie chart reaches up to the nearest tile or caption + halo = min( + np.hypot(max(x0 - cx, 0, cx - x1), max(y0 - cy, 0, cy - y1)) for x0, y0, x1, y1 in boxes + ) - 0.5 + # no white halo: the pie chart draws its own disc + for x, y, k, spec in tiles: + b_tile(drawing, x, y, B_TILE, k, spec) + b_pie(drawing, cx, cy, halo - 0.6) + content = f'{shared_defs()}{"".join(drawing.defs)}{"".join(drawing.body)}' + # the ids must differ from the ones of panels a and c, since all panels end up in the same figure + return content.replace("f1c-", "f1b-"), width, height + + +def build_panel_a(width=A4_WIDTH): + """Draw panel a, the architecture of μSAM2, in its own coordinates, scaled to the given width in mm (the text + stays between MIN_TEXT and MAX_TEXT in print).""" + drawing = Drawing(text_scale=A_TEXT_SCALE, scale=width / A_W) + bus_x, encoder_box = draw_trunk(drawing) + apg_path, apg_taps = draw_automatic_head(drawing, bus_x, encoder_box) + draw_interactive_head(drawing, bus_x) + draw_apg(drawing, apg_path, apg_taps) + w3 = (SIZE + 3.5) * A_SCALE + drawing.text(A_OUTPUT_X + w3 / 2, A_HEADER_Y, "Output", 2.12, weight="bold") + body = f'{"".join(drawing.body)}' + content = f'{shared_defs()}{"".join(drawing.defs)}{body}' + # the ids must differ from the ones of panel c, since both panels end up in the same figure + return content.replace("f1c-", "f1a-") + + +def draw_items(drawing, cx, cy, names, scale): + """Draw the items of one cell of panel c centered at (cx, cy) and enlarged by scale, each with its label right + below it (the items are about as high as they are wide).""" + x = cx - group_width(names, scale) / 2 + for name in names: + draw, item_width, label = ITEMS[name] + scaled(drawing, scale, draw, x, cy) + drawing.text(x + scale * item_width / 2, cy + scale * item_width / 2 + 1.9, label, 1.6, color=SUBTEXT) + x += scale * (item_width + ITEM_GAP) + + +def edge_label(drawing, x, y, text, size, fill, edge, color, anchor="middle"): + """Draw a title stitched onto the edge of a card: a small pill of the card's colors, centered on the edge at y, + half inside and half outside the card.""" + size = drawing.text_size(size) + w, h = text_width(text, size, bold=True) + 2.6, 1.5 * size + x0 = x - w / 2 if anchor == "middle" else (x if anchor == "start" else x - w) + drawing.add( + f'' + ) + # the size is already the printed one, so it is passed on unchanged + drawing.add( + f'{text}' + ) + return w + + +def correction_badge(drawing, x, y, label_dy=4.2): + """Draw the correction badge at (x, y), with its name in italics below it (or above it, for a negative + label_dy).""" + # the badge is 35% larger than first drawn: it is drawn at the origin and scaled into place + start = len(drawing.body) + loop_badge(drawing, (0, 0), r=2.1) + correction_icon(drawing, (0, 0)) + inner = "".join(drawing.body[start:]) + del drawing.body[start:] + drawing.add(f'{inner}') + label_dy *= 1.25 + size = drawing.text_size(1.76) + # a white plate behind the name, so that the outlines of the bands do not run through it + w = text_width("correction", size, bold=True) + 1.0 + drawing.add( + f'' + ) + drawing.add( + f'correction' + ) + + +# the rows of panel c: the name, the interactive items and the automatic items +C_ROWS = [ + ("Segmentation", ["interactive_2d", "interactive_3d"], ["automatic_2d", "automatic_3d"]), + ("Tracking", ["interactive_2dt"], ["automatic_2dt", "automatic_3dt"]), + ("Classification", ["pixel_classification", "object_classification"], + ["automatic_batch", "automatic_batch_objects"]), +] + + +def build_panel_c(logo_path, napari_logo, python_logo, width, height): + """Draw panel c in mm as a table: the name of each row rotated in a narrow column on the left, the interactive + tasks in the left band and the automatic tasks in the right band, with the μSAM2 logo and its interfaces on top. + + Each row holds the same task in both bands, so the two modes of each task face each other; the correction runs + above all rows from the automatic to the interactive band, since every automatic result can be corrected + interactively. Returns the content, the size of the panel and its scale. + """ + drawing = Drawing() + title_size, label_size = 2.12, 2.12 + # the row names stand rotated on the left, so that the bands get the rest of the width + names_w, gap = 3.6, 1.6 + band = (width - names_w - gap) / 2 + lefts = (names_w, names_w + band + gap) + cx = names_w + band + gap / 2 + # the pill overlaps the upper edges of both bands; the band titles sit on the lower edges + pill_y, bands_top = 4.0, 5.2 + bands_bottom = height - 0.8 * drawing.text_size(title_size) + # the rows start a little below the pill, so that it has some air + rows_top, rows_bottom = bands_top + 5.2, bands_bottom - 1.6 + row_h = (rows_bottom - rows_top) / len(C_ROWS) + # the items take the largest scale at which the widest cell fits its band and the highest one its row + widest = max(group_width(names) for _, *cells in C_ROWS for names in cells) + scale = min((band - 2.5) / widest, (row_h - 3.6) / 12.0) + bands = ( + (lefts[0], INTERACTIVE_STYLE[1], "Interactive analysis"), (lefts[1], AUTOMATIC_STYLE[1], "Automatic analysis") + ) + # the bands have no fill color, only their colored outline + for x0, edge, title in bands: + drawing.add( + f'' + ) + edge_label(drawing, x0 + band / 2, bands_bottom, title, title_size, "#ffffff", edge, BLUE) + draw_interfaces(drawing, cx, pill_y, logo_path, napari_logo, python_logo) + # the correction badge between the bands, without an arrow: no direction is implied + correction_badge(drawing, cx, bands_bottom, label_dy=-3.0) + for k, (name, interactive, automatic) in enumerate(C_ROWS): + top = rows_top + k * row_h + # the items sit a little above the middle of the row, since their labels hang below them + cy = top + row_h / 2 - 1.6 + draw_items(drawing, lefts[0] + band / 2, cy, interactive, scale) + draw_items(drawing, lefts[1] + band / 2, cy, automatic, scale) + drawing.text( + names_w / 2 + 0.36 * drawing.text_size(label_size), cy, name, label_size, weight="bold", color=TEXT, + rotate=-90 + ) + content = f'{shared_defs()}{"".join(drawing.defs)}{"".join(drawing.body)}' + return content, width, height, 1.0 + + +def save_panel(content, output_folder, name, width, height): + """Save the panel as a standalone svg and export it to pdf and png with rsvg-convert.""" + svg_path = os.path.join(output_folder, f"{name}.svg") + with open(svg_path, "w") as f: + f.write( + '\n' + f'' + f"{content}\n" + ) + # a fixed date keeps the pdf the same between runs + env = dict(os.environ, SOURCE_DATE_EPOCH="0") + base = os.path.join(output_folder, name) + subprocess.run(["rsvg-convert", "-u", "-f", "pdf", "-o", f"{base}.pdf", svg_path], check=True, env=env) + dpi = str(PNG_DPI) + subprocess.run( + ["rsvg-convert", "-u", "-f", "png", "-d", dpi, "-p", dpi, "-b", "white", "-o", f"{base}.png", svg_path], + check=True, env=env, + ) + # rsvg-convert writes no resolution into the png + with Image.open(f"{base}.png") as image: + image.load() + image.save(f"{base}.png", dpi=(PNG_DPI, PNG_DPI)) + print(f"Saved {name} as svg, pdf and png in {output_folder}") + + +def build_figure(panels): + """Put the panels together: panel a on top, panels b and c next to each other below it, and panel d below them, + each with its letter. Returns the content and the size of the figure.""" + (a, _, a_h), (b, b_w, b_h), (c, c_w, c_h, c_scale) = panels["a"], panels["b"], panels["c"] + c_x, row_y = LETTER_W + b_w + PANEL_GAP + LETTER_W, a_h + ROW_GAP + if c_x + c_scale * c_w > FIGURE_W + 0.01: + raise ValueError(f"Panel c is {c_x + c_scale * c_w - FIGURE_W:.1f} mm too wide for the figure.") + parts = [ + f'{a}', + f'{b}', + f'{c}', + ] + d, _, d_h = panels["d"] + d_y = row_y + max(b_h, c_scale * c_h) + ROW_GAP + parts.append(f'{d}') + for letter, x, y in (("a", 0.0, 0.0), ("b", 0.0, row_y), ("c", c_x - LETTER_W, row_y), ("d", 0.0, d_y)): + parts.append( + f'{letter}' + ) + # a small margin keeps the outlines of the lowest panel inside the page + height = d_y + d_h + 0.6 + width = max(FIGURE_W, LETTER_W + panels["d"][1]) + return '' + "".join(parts), width, height + + +# Panel d: studio renders of the user study examples, made off screen on a GPU node. + +D_LINE = "#1a1a1a" +# the top margin of each example of panel d in mm: room for its number +D_TOP = {"platynereis": 2.5, "spheroid": 2.5, "gruen": 2.5, "histai": 2.5} +D_START = time.time() +# the expanded Platynereis (Huisken lab): nuclei of the whole animal (s1 grid: z, y, x) +PLATY = os.path.join(DATA_ROOT, "huisken_data/exm_lsm/high_res/stitched/Expanded_Platynereis.ome.zarr") +PLATY_SHAPE = (770, 477, 1197) +PLATY_VOXEL_UM = 2.5112882406416843 # s1 voxel size, expanded +PLATY_EXPANSION = 10.0 # assumed expansion factor, for the pre-expansion scale bars +PLATY_SLAB_UM = 11.0 # section thickness, pre-expansion +# the studio view; PLATY_RES scales the renders to print at more than 1200 dpi +PLATY_RES = 1.25 +PLATY_SIZE = (round(3300 * PLATY_RES), round(6000 * PLATY_RES)) +PLATY_BOUNDS = (0, PLATY_SHAPE[2], 0, PLATY_SHAPE[1], 0, PLATY_SHAPE[0]) +PLATY_CENTER = np.array([PLATY_SHAPE[2], PLATY_SHAPE[1], PLATY_SHAPE[0]], float) / 2 +PLATY_DIRECTION = np.array([-0.3, 1.0, -0.5]) / np.linalg.norm([-0.3, 1.0, -0.5]) +PLATY_UP = (-1, 0, 0) +PLATY_ZOOM = 1.55 +# the crop of the studio render (rows r0:r1, columns c0:c1): the animal, its shadows and its raw signal, with margins +PLATY_FRAME = tuple(round(v * PLATY_RES) for v in (334, 2639, 0, 1468)) +# the section is seen from its normal (z, the studio camera side), tilted a little like the studio view +PLATY_DIRECTION_B = np.array([-0.15, 0.25, -1.0]) / np.linalg.norm([-0.15, 0.25, -1.0]) +# the MCF7 spheroid with DAPI and E-cadherin stains, and its nucleus and cell segmentations +SPHEROID = os.path.join(DATA_ROOT, "beghin_data/mcf7_spheroids/dapi_ecadh_btub/Composite_s23") +SPHEROID_XY_UM, SPHEROID_Z_UM = 0.414, 1.5 +# the heart section (Gruen lab): the normalized volume (z, y, x, channel), three crops and their cell masks +GRUEN = os.path.join(DATA_ROOT, "gruen_lab_data") +# no voxel size in the files: z from the 10 um section in 16 planes, xy estimated from the size of the nuclei +GRUEN_XY_UM, GRUEN_Z_UM = 0.12, 10.0 / 16 +# the Ki67 core (HistAI, case_13673): image pyramid, instances and classes; 0.5 um per level 0 px +HISTAI = os.path.join(DATA_ROOT, "histai_specimen_omezarr_2026-11-06", "case_13673.ome.zarr.zip") +# one crop of 256 x 256 px (128 um), smaller than first drawn +HISTAI_UM, HISTAI_CROP = 0.5, 256 +# the classes of the Ki67 labels, from the label metadata of the OME-Zarr, as (name, rgb) +HISTAI_CLASSES = [("positive", (214, 39, 40)), ("negative", (31, 90, 214)), ("others", (240, 180, 0))] +HISTAI_CLASS_RGB = np.array([[200, 200, 200], [214, 39, 40], [31, 90, 214], [240, 180, 0]], np.uint8) + + +def d_log(*args): + print(*args, round(time.time() - D_START), flush=True) + + +def d_separate(labels): + """Remove the voxels where two objects touch, so that each object becomes its own surface; also return the + pairs of touching objects.""" + pairs = set() + cut = np.zeros(labels.shape, bool) + for axis in range(3): + a = np.moveaxis(labels, axis, 0) + lo, hi = a[:-1], a[1:] + touch = (lo != hi) & (lo > 0) & (hi > 0) + found = np.unique(np.stack([lo[touch], hi[touch]], 1), axis=0) + pairs.update(map(tuple, found)) + c = np.moveaxis(cut, axis, 0) + c[:-1] |= touch + c[1:] |= touch + labels = labels.copy() + labels[cut] = 0 + return labels, pairs + + +def d_near_pairs(coarse, radius=2): + """Pairs of objects closer than about radius voxels.""" + pairs = set() + offsets = [ + (dz, dy, dx) for dz in range(0, radius + 1) for dy in range(-radius, radius + 1) + for dx in range(-radius, radius + 1) if (dz, dy, dx) > (0, 0, 0) and dz * dz + dy * dy + + dx * dx <= radius * radius + 1 + ] + nz, ny, nx = coarse.shape + for dz, dy, dx in offsets: + a = coarse[0:nz - dz, max(0, -dy):ny - max(0, dy), max(0, -dx):nx - max(0, dx)] + b = coarse[dz:nz, max(0, dy):ny - max(0, -dy), max(0, dx):nx - max(0, -dx)] + hit = (a != b) & (a > 0) & (b > 0) + code = np.unique(np.minimum(a[hit], b[hit]).astype("int64") * 100000 + np.maximum(a[hit], b[hit])) + pairs.update(zip((code // 100000).tolist(), (code % 100000).tolist())) + return pairs + + +def d_distinct_colors(n_labels, pairs, seed=1): + """A palette of clear colours; each object gets a colour that is clearly different (in Lab) from the colours of + its neighbours, so that close objects never look alike.""" + palette = [] + for i in range(60): + hue = (i * 0.6180339887) % 1.0 + sat = (0.42, 0.58, 0.30)[i % 3] + val = (0.97, 0.88, 0.78)[(i // 3) % 3] + palette.append(colorsys.hsv_to_rgb(hue, sat, val)) + palette = np.array(palette) + lab = rgb2lab(palette[None])[0] + dist = np.linalg.norm(lab[:, None] - lab[None], axis=-1) + neighbours = [[] for _ in range(n_labels + 1)] + for a, b in pairs: + neighbours[a].append(b) + neighbours[b].append(a) + rng = np.random.default_rng(seed) + choice = np.full(n_labels + 1, -1) + # the most connected objects first + order = sorted(range(1, n_labels + 1), key=lambda i: (-len(neighbours[i]), rng.random())) + for label in order: + used = [choice[n] for n in neighbours[label] if choice[n] >= 0] + # a random colour among those clearly different from all neighbours, else the most different one + gap = dist[:, used].min(axis=1) if used else np.full(len(palette), np.inf) + clear = np.flatnonzero(gap >= 30) + choice[label] = int(rng.choice(clear)) if len(clear) else int(np.argmax(gap)) + choice[0] = 0 + return palette[choice] + + +def d_label_surfaces(labels): + """Smooth surfaces of all objects in one pass; the label of each face is in cell_data["label"].""" + grid = pv.ImageData(dimensions=labels.shape[::-1]) + grid.point_data["values"] = labels.ravel(order="C") + nets = vtk.vtkSurfaceNets3D() + nets.SetInputData(grid) + nets.SetInputArrayToProcess(0, 0, 0, 0, "values") + nets.GenerateLabels(int(labels.max()), 1, int(labels.max())) + nets.SetOutputMeshTypeToTriangles() + nets.SetOutputStyleToBoundary() + nets.SmoothingOn() + nets.SetNumberOfIterations(40) + nets.Update() + mesh = pv.wrap(nets.GetOutput()) + mesh.cell_data["label"] = np.asarray(mesh.cell_data["BoundaryLabels"]).max(axis=1) + return mesh + + +def d_colored_mesh(labels, colors, z_scale=1.0): + """The surfaces with one rgb colour per face; a colour map would be resampled to 256 colours and merge + neighbouring ids.""" + mesh = d_label_surfaces(labels) + mesh.points[:, 2] *= z_scale + mesh.cell_data["rgb"] = (colors[mesh.cell_data["label"]] * 255).astype("uint8") + return mesh + + +def d_add_objects(plotter, mesh): + plotter.add_mesh( + mesh, scalars="rgb", rgb=True, smooth_shading=True, specular=0.2, specular_power=20, ambient=0.35, diffuse=0.75, + show_scalar_bar=False + ) + + +def d_project(plotter, points): + """Image coordinates (column, row) of world points, at the saved (half) image size.""" + renderer = plotter.renderer + height = plotter.window_size[1] + out = [] + for p in points: + renderer.SetWorldPoint(*p, 1.0) + renderer.WorldToDisplay() + x, y, _ = renderer.GetDisplayPoint() + out.append((x / 2, (height - y) / 2)) + return out + + +def d_px_per_unit(plotter, centre, step=50.0): + """Pixels of the saved render per scene unit at the depth of the centre, along x.""" + a, b = d_project(plotter, [centre, np.asarray(centre) + np.array([step, 0, 0])]) + return float(np.hypot(b[0] - a[0], b[1] - a[1]) / step) + + +def d_screenshot(plotter, size, transparent=False): + """Render, and scale the double size window down to the saved size, for smooth edges.""" + plotter.render() + image = plotter.screenshot(return_img=True, transparent_background=transparent) + return np.asarray(Image.fromarray(image).resize(size, Image.LANCZOS)) + + +def d_view(elevation, azimuth): + """The direction from the scene to the camera, and its horizontal part.""" + flat = np.array([np.cos(np.radians(azimuth)), np.sin(np.radians(azimuth)), 0.0]) + return np.cos(np.radians(elevation)) * flat + np.array([0, 0, np.sin(np.radians(elevation))]), flat + + +def d_quad(corners): + """A rectangle from four corners, finely split so that the shadows look smooth.""" + return pv.PolyData(np.array(corners, float), faces=[4, 0, 1, 2, 3]).triangulate().subdivide(4) + + +def d_backdrop(plotter, bounds, view, margin, reach, height): + """A floor under the scene (z up) and two walls behind it, on the sides away from the camera.""" + x0, x1, y0, y1, z0, z1 = bounds + floor_z = z0 - margin + wx = x0 - margin if view[0] > 0 else x1 + margin + wy = y0 - margin if view[1] > 0 else y1 + margin + fx = x1 + reach if view[0] > 0 else x0 - reach + fy = y1 + reach if view[1] > 0 else y0 - reach + top = z1 + height + for plane in ( + d_quad([(wx, wy, floor_z), (fx, wy, floor_z), (fx, fy, floor_z), (wx, fy, floor_z)]), + d_quad([(wx, wy, floor_z), (wx, fy, floor_z), (wx, fy, top), (wx, wy, top)]), + d_quad([(wx, wy, floor_z), (fx, wy, floor_z), (fx, wy, top), (wx, wy, top)]) + ): + plotter.add_mesh(plane, color=(0.95, 0.95, 0.95), ambient=0.45, diffuse=0.6, specular=0.0) + + +def d_studio(size, bounds, view, distance, zoom): + """A plotter with the studio lights and shadows (z up), the camera from the view direction onto the bounds.""" + x0, x1, y0, y1, z0, z1 = bounds + middle = np.array([(x0 + x1) / 2, (y0 + y1) / 2, (z0 + z1) / 2]) + plotter = pv.Plotter(off_screen=True, window_size=(size[0] * 2, size[1] * 2), lighting="none") + plotter.set_background("white") + # a key light from above, a little to the side of the camera, and a soft fill light from the camera + side = np.cross(view, [0, 0, 1.0]) + key = middle + distance * (0.5 * view + 0.45 * side / np.linalg.norm(side) + np.array([0, 0, 1.0])) + plotter.add_light(pv.Light( + position=tuple(key), focal_point=tuple(middle), intensity=0.7, positional=True, cone_angle=50, + light_type="scene light" + )) + plotter.add_light(pv.Light( + position=tuple(middle + 2 * distance * view), focal_point=tuple(middle), intensity=0.55, + light_type="scene light" + )) + plotter.enable_shadows() + plotter.renderer._render_passes._shadow_map_pass.GetShadowMapBakerPass().SetResolution(8192) + plotter.camera.focal_point = middle + plotter.camera.position = middle + 3 * distance * view + plotter.camera.up = (0, 0, 1) + plotter.camera.view_angle = 20 + plotter.reset_camera(bounds=bounds) + plotter.camera.zoom(zoom) + return plotter + + +def d_section_shot(add, centre, direction, up, key, bounds, size, zoom, transparent=False, points=()): + """A render of a section from its camera direction, with ambient occlusion and no shadows; returns the image, + the image positions of the points and the pixels per scene unit at the centre.""" + plotter = pv.Plotter(off_screen=True, window_size=(size[0] * 2, size[1] * 2), lighting="none") + plotter.set_background("white") + add(plotter) + plotter.add_light( + pv.Light(position=tuple(centre + key), focal_point=tuple(centre), intensity=0.7, light_type="scene light") + ) + plotter.add_light(pv.Light( + position=tuple(centre + 3000 * direction), focal_point=tuple(centre), intensity=0.55, light_type="scene light" + )) + plotter.enable_ssao(radius=5, bias=0.05, kernel_size=128, blur=True) + plotter.camera.focal_point = centre + plotter.camera.position = centre + 3000 * direction + plotter.camera.up = tuple(up) + plotter.camera.view_angle = 20 + plotter.reset_camera(bounds=bounds) + plotter.camera.zoom(zoom) + image = d_screenshot(plotter, size, transparent) + projected, ppu = d_project(plotter, points), d_px_per_unit(plotter, centre) + plotter.close() + return image, projected, ppu + + +def d_card(plotter, image, origin, step, thickness): + """A raw card: a dark box with the image on its top face. Image row 0 is at the far (high y) edge; the box spans + x in [ox, ox + width * step] and y in [oy - height * step, oy].""" + h, w = image.shape[:2] + ox, oy, oz = origin + box = pv.Box(bounds=(ox, ox + w * step, oy - h * step, oy, oz, oz + thickness)) + plotter.add_mesh(box, color=(0.08, 0.08, 0.08), ambient=0.6, diffuse=0.4, specular=0.0) + grid = pv.ImageData( + dimensions=(w, h, 1), spacing=(step, step, 1.0), + origin=(ox + step / 2, oy - h * step + step / 2, oz + thickness + 0.5) + ) + grid.point_data["rgb"] = image[::-1].reshape(-1, 3) + plotter.add_mesh(grid, scalars="rgb", rgb=True, ambient=1.0, diffuse=0.0, specular=0.0, show_scalar_bar=False) + + +def d_rectangle(bounds): + """The four corners of a box in its middle z plane.""" + x0, x1, y0, y1, z0, z1 = bounds + z = (z0 + z1) / 2 + return [(x0, y0, z), (x1, y0, z), (x1, y1, z), (x0, y1, z)] + + +def d_upsample(volume, factor, shape): + """Nearest upsampling of a coarse volume into the given shape; a missing edge is filled from the last voxel.""" + up = volume + for axis in range(3): + up = np.repeat(up, factor, axis) + up = np.pad(up, [(0, max(0, s - u)) for s, u in zip(shape, up.shape)], mode="edge") + return up[:shape[0], :shape[1], :shape[2]] + + +def platy_raw_s0(cache): + """The Platynereis raw at s0, divided by its smooth local level, scaled to uint8 (20th to 99.9th percentile) and + masked to the body. It is large (3.5 GB), so it is cached.""" + if os.path.exists(cache): + return np.load(cache, mmap_mode="r") + group = zarr.open_group(PLATY, mode="r") + shape = tuple(2 * s for s in PLATY_SHAPE) + coarse = zarr.open_group(PLATY + "/labels/nuclei", mode="r")["s2"][:] > 0 + body = ndimage.binary_dilation(ndimage.binary_closing(coarse, iterations=3), iterations=3) + level = ndimage.gaussian_filter(group["s2"][:].astype("float32"), 12) + 1.0 + # the normalised volume in half precision, then the contrast limits from a sparse sample of it + norm = np.zeros(shape, "float16") + slab = 64 + for z0 in range(0, shape[0], slab): + z1 = min(z0 + slab, shape[0]) + c0, c1 = z0 // 4, (z1 + 3) // 4 + block = group["s0"][z0:z1, :shape[1], :shape[2]].astype("float32") + lev = d_upsample(level[c0:c1], 4, (c1 * 4 - c0 * 4,) + shape[1:])[z0 - c0 * 4:z1 - c0 * 4] + mask = d_upsample(body[c0:c1], 4, (c1 * 4 - c0 * 4,) + shape[1:])[z0 - c0 * 4:z1 - c0 * 4] + block = block / lev + block[~mask] = np.nan + norm[z0:z1, :block.shape[1], :block.shape[2]] = block + lo, hi = np.nanpercentile(norm[::3, ::3, ::3].astype("float32"), [20, 99.9]) + raw = np.zeros(shape, "uint8") + for z0 in range(0, shape[0], slab): + block = norm[z0:z0 + slab].astype("float32") + raw[z0:z0 + slab] = np.nan_to_num(np.clip((block - lo) / (hi - lo), 0, 1) * 255, nan=0).astype("uint8") + del norm + np.save(cache, raw) + d_log("raw s0", raw.shape) + return raw + + +def platy_backdrop(plotter): + """A floor below the tail end and two walls behind the animal, in light grey.""" + x1, y1, z1 = PLATY_SHAPE[2], PLATY_SHAPE[1], PLATY_SHAPE[0] + m, far, top = 60, 350, -250 + floor_x, wall_y, wall_z = x1 + m, -m, z1 + m + # the floor spans from the walls to the front; the walls from the floor to above the round end + front_y, front_z = y1 + far, -far + planes = ( + d_quad([ + (floor_x, wall_y, wall_z), (floor_x, front_y, wall_z), (floor_x, front_y, front_z), + (floor_x, wall_y, front_z), + ]), + d_quad([(top, wall_y, front_z), (top, wall_y, wall_z), (floor_x, wall_y, wall_z), (floor_x, wall_y, front_z)]), + d_quad([(top, wall_y, wall_z), (top, front_y, wall_z), (floor_x, front_y, wall_z), (floor_x, wall_y, wall_z)]), + ) + for plane in planes: + plotter.add_mesh(plane, color=(0.95, 0.95, 0.95), ambient=0.45, diffuse=0.6, specular=0.0) + + +def render_platynereis(out, raw_cache): + """The renders of the Platynereis panel: the nuclei in the studio, an 11 um longitudinal section (the nuclei with + their centre in a slab, and the raw of its middle plane), and close-ups of its anterior and posterior ends.""" + os.makedirs(out, exist_ok=True) + nuclei = zarr.open_group(PLATY + "/labels/nuclei", mode="r") + labels = nuclei["s1"][:] + n_labels = int(labels.max()) + separated, pairs = d_separate(labels) + pairs |= d_near_pairs(nuclei["s2"][:]) + colors = d_distinct_colors(n_labels, pairs) + d_log("colours") + # the centre of each nucleus, in s1 voxels (z, y, x) + counts = np.zeros(n_labels + 1) + sums = np.zeros((3, n_labels + 1)) + for z0 in range(0, labels.shape[0], 64): + block = labels[z0:z0 + 64] + zz, yy, xx = np.indices(block.shape, dtype="float32") + flat = block.ravel() + counts += np.bincount(flat, minlength=n_labels + 1) + for axis, c in enumerate((zz + z0, yy, xx)): + sums[axis] += np.bincount(flat, weights=c.ravel(), minlength=n_labels + 1) + del labels + cz, cy, cx = (sums / np.maximum(counts, 1))[:, 1:] + ids = np.arange(1, n_labels + 1) + # the slab: the nuclei with their centre near the middle of the body in z + half = PLATY_SLAB_UM * PLATY_EXPANSION / PLATY_VOXEL_UM / 2 + z_mid = float(np.median(cz)) + in_slab = np.abs(cz - z_mid) <= half + slab_ids = ids[in_slab] + layout = { + "nuclei": n_labels, "slab_nuclei": int(len(slab_ids)), "slab_um": PLATY_SLAB_UM, "expansion": PLATY_EXPANSION, + "voxel_um": PLATY_VOXEL_UM + } + + # the studio, cropped to the frame, and the outline of the slab plane + plotter = pv.Plotter(off_screen=True, window_size=PLATY_SIZE, lighting="none") + plotter.set_background("white") + d_add_objects(plotter, d_colored_mesh(separated, colors)) + platy_backdrop(plotter) + plotter.add_light(pv.Light( + position=tuple(PLATY_CENTER + np.array([-2500.0, 1200.0, -1600.0])), focal_point=tuple(PLATY_CENTER), + intensity=0.7, positional=True, cone_angle=40, light_type="scene light" + )) + plotter.add_light(pv.Light( + position=tuple(PLATY_CENTER + PLATY_DIRECTION * 3000 + np.array([0.0, 600.0, 0.0])), + focal_point=tuple(PLATY_CENTER), intensity=0.55, light_type="scene light" + )) + plotter.enable_shadows() + plotter.renderer._render_passes._shadow_map_pass.GetShadowMapBakerPass().SetResolution(8192) + plotter.camera.focal_point = PLATY_CENTER + plotter.camera.position = PLATY_CENTER + PLATY_DIRECTION * 4000 + plotter.camera.up = PLATY_UP + plotter.camera.view_angle = 20 + plotter.reset_camera(bounds=PLATY_BOUNDS) + plotter.camera.zoom(PLATY_ZOOM) + image = d_screenshot(plotter, (PLATY_SIZE[0] // 2, PLATY_SIZE[1] // 2)) + r0, r1, c0, c1 = PLATY_FRAME + Image.fromarray(image[r0:r1, c0:c1]).save(f"{out}/seg_studio.png") + sub = np.where(np.isin(separated, slab_ids), separated, 0) + del separated + mesh = d_colored_mesh(sub, colors) + del sub + x0, x1, y0, y1, _, _ = mesh.bounds + outline = d_project(plotter, d_rectangle((x0 - 40, x1 + 60, y0 - 25, y1 + 25, z_mid, z_mid))) + layout["studio"] = { + "slab_outline": [(c - c0, r - r0) for c, r in outline], "px_per_voxel": d_px_per_unit(plotter, PLATY_CENTER) + } + plotter.close() + d_log("studio") + + # the anterior (round end, low x) and posterior boxes, from the centres of the slab nuclei + sx, sy = cx[in_slab], cy[in_slab] + x_lo, x_hi = sx.min(), sx.max() + length = x_hi - x_lo + box_ids, box_points = {}, [] + for name, (b0, b1) in ( + ("anterior", (x_lo - 8, x_lo + 0.30 * length)), ("posterior", (x_hi - 0.22 * length, x_hi + 8)) + ): + pick = (sx >= b0) & (sx <= b1) + box_ids[name] = slab_ids[pick] + box_points += d_rectangle((b0, b1, sy[pick].min() - 14, sy[pick].max() + 14, z_mid - half, z_mid + half)) + + # the section on a transparent background, and the raw of its middle plane + slab_bounds = mesh.bounds + + def shot(add, bounds, size, zoom, transparent=False, points=()): + centre = np.array([(bounds[0] + bounds[1]) / 2, (bounds[2] + bounds[3]) / 2, (bounds[4] + bounds[5]) / 2]) + return d_section_shot( + add, centre, PLATY_DIRECTION_B, PLATY_UP, np.array([-2500.0, 1500.0, -1500.0]), bounds, size, zoom, + transparent, points + ) + + # the section prints narrow and tall, so it is rendered larger than the studio + section_size = (round(1100 * 2.5), round(2000 * 2.5)) + image, projected, ppv = shot(lambda p: d_add_objects(p, mesh), slab_bounds, section_size, 1.0, True, box_points) + Image.fromarray(image).save(f"{out}/section_rgba.png") + layout["section"] = { + "anterior_box": projected[:4], "posterior_box": projected[4:], "px_per_voxel": ppv, + "raw_slice_s0_z": int(round(2 * z_mid)) + } + raw = platy_raw_s0(raw_cache) + pad = 12 + ia, ib = int(max(0, 2 * (x0 - pad))), int(min(raw.shape[2], 2 * (x1 + pad))) + ja, jb = int(max(0, 2 * (y0 - pad))), int(min(raw.shape[1], 2 * (y1 + pad))) + plane = np.asarray(raw[int(round(2 * z_mid)), ja:jb, ia:ib], dtype="float32") + # the contrast of the single slice, which is dimmer than the projection the raw was scaled for + raw_hi = float(np.percentile(plane[plane > 0], 99.0)) + + def add_raw(plotter): + grid = pv.ImageData( + dimensions=(plane.shape[1], plane.shape[0], 1), spacing=(0.5, 0.5, 1.0), origin=(ia / 2, ja / 2, z_mid) + ) + grid.point_data["values"] = plane.ravel(order="C") + plotter.add_mesh( + grid, scalars="values", cmap="gray", clim=(0, raw_hi), ambient=1.0, diffuse=0.0, specular=0.0, + show_scalar_bar=False + ) + + Image.fromarray(shot(add_raw, slab_bounds, section_size, 1.0)[0]).save(f"{out}/section_raw.png") + d_log("section") + for name in ("anterior", "posterior"): + part = mesh.extract_cells(np.flatnonzero(np.isin(mesh.cell_data["label"], box_ids[name]))) + image, _, ppv = shot(lambda p, part=part: d_add_objects(p, part), part.bounds, (1300, 1300), 1.05) + Image.fromarray(image).save(f"{out}/{name}.png") + layout[name] = {"nuclei": int(len(box_ids[name])), "px_per_voxel": ppv} + json.dump(layout, open(f"{out}/layout.json", "w"), indent=1) + d_log("platynereis done") + + +# the colours of the cells and the nuclei on the plane of the spheroid example, and of their outlines +SPHEROID_CELL_RGB, SPHEROID_CELL_EDGE = (0.84, 0.42, 0.76), (0.52, 0.16, 0.46) +SPHEROID_NUCLEUS_RGB, SPHEROID_NUCLEUS_EDGE = (1.0, 1.0, 1.0), (0.62, 0.62, 0.68) + + +def spheroid_cells(seg): + """Keep only the cells of the spheroid: the objects that lie mostly in the largest connected foreground component, + that are at least a fifth of the median cell size and that fill at least a fifth of their bounding box. The loose + pieces around the spheroid and the flat sheets on its surface go.""" + components, _ = ndimage.label(seg > 0) + largest = np.argmax(np.bincount(components[components > 0])) + ids, sizes = np.unique(seg[seg > 0], return_counts=True) + inside = np.bincount(seg[components == largest], minlength=int(seg.max()) + 1)[ids] + boxes = ndimage.find_objects(seg) + fill = np.array([sizes[k] / np.prod([b.stop - b.start for b in boxes[i - 1]]) for k, i in enumerate(ids)]) + keep = ids[(inside > 0.5 * sizes) & (sizes >= 0.2 * np.median(sizes)) & (fill >= 0.2)] + return np.where(np.isin(seg, keep), seg, 0) + + +def render_spheroid(out): + """The renders of the spheroid example: its cells and its nuclei in the studio, side by side with one camera, a + nucleus in the colour of the cell that holds most of it; and the middle z plane of both channels (E-cadherin + magenta, DAPI cyan), the raw on its left half and the outlines of the cells (white) and the nuclei (yellow) on its + right half.""" + os.makedirs(out, exist_ok=True) + z_scale = SPHEROID_Z_UM / SPHEROID_XY_UM # the scene is in xy voxel units + raw = tifffile.imread(f"{SPHEROID}.tif") # z, c, y, x: DAPI, E-cadherin, beta-tubulin + + def relabel(labels): + ids, inverse = np.unique(labels, return_inverse=True) + return inverse.reshape(labels.shape).astype("uint16"), len(ids) - 1 + + cells, n_cells = relabel(spheroid_cells(tifffile.imread(f"{SPHEROID}_membrane_prediction.tif"))) + nuclei, n_nuclei = relabel(tifffile.imread(f"{SPHEROID}_nucleus_prediction.tif")) + cells_sep, pairs = d_separate(cells) + pairs |= d_near_pairs(cells, radius=3) + cell_colors = d_distinct_colors(n_cells, pairs) + both = (nuclei > 0) & (cells > 0) + votes = np.zeros((n_nuclei + 1, n_cells + 1)) + np.add.at(votes, (nuclei[both], cells[both]), 1) + # a nucleus outside every cell is grey + nucleus_colors = np.full((n_nuclei + 1, 3), 0.75) + has = votes[:, 1:].sum(1) > 0 + nucleus_colors[has] = cell_colors[votes[has, 1:].argmax(1) + 1] + meshes = { + "cells": d_colored_mesh(cells_sep, cell_colors, z_scale), + "nuclei": d_colored_mesh(d_separate(nuclei)[0], nucleus_colors, z_scale) + } + + # the nuclei are shifted across the line of sight, beside the cells + view, flat = d_view(30, -60) + across = np.array([-flat[1], flat[0], 0.0]) + zz, yy, xx = np.nonzero(cells) + span = max(np.ptp(xx), np.ptp(yy)) + # a little apart, closer than first drawn + shift = 0.95 * span * across + meshes["nuclei"].points += shift + z = int(np.median(zz)) + z_plane = (z + 0.5) * z_scale + # the plane as one rectangle around the footprints of both spheroids + footprint = np.c_[xx, yy].astype(float) + footprint = np.r_[footprint, footprint + shift[:2]] + u, v = footprint @ across[:2], footprint @ flat[:2] + pad = 8.0 + corners = [ + (u.min() - pad, v.max() + pad), (u.max() + pad, v.max() + pad), (u.max() + pad, v.min() - pad), + (u.min() - pad, v.min() - pad), + ] + rectangle = [tuple(cu * across[:2] + cv * flat[:2]) + (z_plane,) for cu, cv in corners] + points = np.r_[np.array([m.bounds for m in meshes.values()])] + bounds = ( + points[:, 0].min() - 20, points[:, 1].max() + 20, points[:, 2].min() - 20, points[:, 3].max() + 20, 0, + points[:, 5].max() + ) + layout = {"xy_um": SPHEROID_XY_UM, "cells": n_cells, "nuclei": n_nuclei} + plotter = d_studio((2400, 1600), bounds, view, 900, 1.0) + for mesh in meshes.values(): + d_add_objects(plotter, mesh) + d_backdrop(plotter, bounds, view, 10, 600, 40) + Image.fromarray(d_screenshot(plotter, (2400, 1600))).save(f"{out}/studio.png") + layout["rectangle"] = d_project(plotter, rectangle) + # the top centre of each spheroid, for its name + tops = { + name: ((m.bounds[0] + m.bounds[1]) / 2, (m.bounds[2] + m.bounds[3]) / 2, m.bounds[5]) for name, + m in meshes.items() + } + layout["labels"] = {name: d_project(plotter, [top])[0] for name, top in tops.items()} + centre = ((bounds[0] + bounds[1]) / 2, (bounds[2] + bounds[3]) / 2, bounds[5] / 2) + layout["px_per_voxel"] = d_px_per_unit(plotter, np.array(centre)) + plotter.close() + d_log("spheroid studio") + + norm = [np.clip(c / np.percentile(c, 99.5), 0, 1) for c in raw[z].astype("float32")] + rgb = np.clip( + norm[1][..., None] * np.array([1.0, 0.25, 0.8]) + norm[0][..., None] * np.array([0.2, 0.75, 1.0]), 0, 1 + ) + # the right half: cells in magenta, as the E-cadherin stain, nuclei in white + lined = rgb.copy() + lined[cells[z] > 0] = SPHEROID_CELL_RGB + lined[find_boundaries(cells[z], mode="inner")] = SPHEROID_CELL_EDGE + lined[nuclei[z] > 0] = SPHEROID_NUCLEUS_RGB + lined[find_boundaries(nuclei[z], mode="inner")] = SPHEROID_NUCLEUS_EDGE + rows, cols = np.nonzero(cells[z]) + r0, r1 = max(rows.min() - 15, 0), min(rows.max() + 15, rgb.shape[0]) + c0, c1 = max(cols.min() - 15, 0), min(cols.max() + 15, rgb.shape[1]) + half = (c0 + c1) // 2 + plane = np.concatenate([rgb[r0:r1, c0:half], lined[r0:r1, half:c1]], 1) + Image.fromarray((plane * 255).astype("uint8")).save(f"{out}/plane.png") + layout["plane_z"] = z + json.dump(layout, open(f"{out}/layout.json", "w"), indent=1) + d_log("spheroid done") + + +def render_gruen(out): + """The renders of the heart section panel: the volume as one card of its merged channels (on white) with the + outlines of the three crops on it, and for each crop its cells alone, in the studio, from the same direction.""" + os.makedirs(out, exist_ok=True) + z_scale = GRUEN_Z_UM / GRUEN_XY_UM # the scene is in xy pixel units + view, _ = d_view(35, -60) + down = 2 # the downscaling of the raw card of the volume + volume = tifffile.imread(glob.glob(f"{GRUEN}/*_normalized.tif")[0]) + height, width = volume.shape[1:3] + thickness = volume.shape[0] * z_scale + mip = volume.max(0) + # the nuclear channel (blue) is dim and blue looks dark: stretch it to its 99.5th percentile, with a gamma + nuclei = mip[..., 2].astype("float32") + mip[..., 2] = (np.clip(nuclei / np.percentile(nuclei, 99.5), 0, 1) ** 0.7 * 255).astype("uint8") + del volume + small = np.asarray(Image.fromarray(mip).resize((width // down, height // down), Image.LANCZOS)) + crops = [] + for path in sorted(glob.glob(f"{GRUEN}/RGB/*crop*.tif")): + name = os.path.basename(path) + crops.append({ + "name": f"crop{name.split('_crop')[1].split('_')[0]}", + "y": int(name.split("_y")[1].split("_")[0]), "x": int(name.split("_x")[1].split(".")[0]), + "mask": f"{GRUEN}/masks/{name.replace('.tif', '_mask.tif')}", + }) + + # one card of the merged channels + top_z = 0.0 + plotter = d_studio((2400, 2400), (0, width, -height, 0, 0, top_z + thickness), view, 4000, 1.0) + d_card(plotter, small, (0, 0, top_z), down, thickness) + Image.fromarray(d_screenshot(plotter, (2400, 2400))).save(f"{out}/volume.png") + layout = {"xy_um": GRUEN_XY_UM, "z_um": GRUEN_Z_UM, "volume": [int(height), int(width)], "crops": {}} + for crop in crops: + # the crop footprint on the top face of the top card: top left, top right, bottom right, bottom left + x, y, z = crop["x"], -crop["y"], top_z + thickness + 0.5 + corners = [(x, y, z), (x + 512, y, z), (x + 512, y - 512, z), (x, y - 512, z)] + layout["crops"][crop["name"]] = {"outline": d_project(plotter, corners), "y": crop["y"], "x": crop["x"]} + layout["volume_px_per_pixel"] = d_px_per_unit(plotter, (width / 2, -height / 2, top_z), 500.0) + plotter.close() + d_log("volume") + + # each crop: its cells alone, on the studio floor + for crop in crops: + labels = tifffile.imread(crop["mask"]) + ids, inverse = np.unique(labels, return_inverse=True) + labels = inverse.reshape(labels.shape).astype("uint16") + n_labels = len(ids) - 1 + separated, pairs = d_separate(labels) + pairs |= d_near_pairs(labels, radius=3) + colors = d_distinct_colors(n_labels, pairs) + # the cells fill all planes and reach the crop edges: pad with background, so that their surfaces close + mesh = d_label_surfaces(np.pad(separated, 1)) + mesh.points -= 1 + # y down the image rows as in the volume render; the mirror flips the faces, so turn them back + mesh.points[:, 1] *= -1 + mesh.points[:, 2] *= z_scale + mesh.flip_faces(inplace=True) + mesh.cell_data["rgb"] = (colors[mesh.cell_data["label"]] * 255).astype("uint8") + bounds = (0, 512, -512, 0, 0, labels.shape[0] * z_scale) + plotter = d_studio((1400, 1400), bounds, view, 700, 1.0) + d_add_objects(plotter, mesh) + d_backdrop(plotter, bounds, view, 30, 300, 180) + Image.fromarray(d_screenshot(plotter, (1400, 1400))).save(f"{out}/{crop['name']}.png") + # the corners of the top face of the cell block, in the order of the crop outline + z = bounds[5] + face = d_project(plotter, [(0, 0, z), (512, 0, z), (512, -512, z), (0, -512, z)]) + layout["crops"][crop["name"]].update( + {"top_face": face, "cells": n_labels, "px_per_pixel": d_px_per_unit(plotter, (256, -256, 0))} + ) + plotter.close() + # the exploded view: the channels at the bottom, the merged block, the cells floating on top + y, x = crop["y"], crop["x"] + merged = mip[y:y + 512, x:x + 512] + # the channel cards spread like a dealt deck, so that more of each shows + height_cells, card, gap, spread = bounds[5], 10.0, 110.0, 110.0 + levels = [k * (card + gap) for k in range(3)] + block_z = 3 * (card + gap) + cells_z = block_z + height_cells + 1.4 * gap + exploded = mesh.copy() + exploded.points[:, 2] += cells_z + scene = (-3 * spread, 512, -512, 0, 0, cells_z + height_cells) + plotter = d_studio((1400, 2000), scene, view, 900, 1.0) + # without the studio: no walls, floor or shadows + plotter.disable_shadows() + # each channel stretched and in a light tint, since pure blue looks dark + tints = np.array([(1.0, 0.25, 0.2), (0.25, 1.0, 0.35), (0.35, 0.6, 1.0)]) + for k, z0 in enumerate(levels): + value = merged[..., k].astype("float32") + value = np.clip(value / max(np.percentile(value, 99.5), 1.0), 0, 1) ** 0.8 + single = (value[..., None] * tints[k] * 255).astype("uint8") + d_card(plotter, single, (-(3 - k) * spread, 0, z0), 1.0, card) + d_card(plotter, merged, (0, 0, block_z), 1.0, height_cells) + d_add_objects(plotter, exploded) + Image.fromarray(d_screenshot(plotter, (1400, 2000))).save(f"{out}/{crop['name']}_exploded.png") + zb = block_z + height_cells + layout["crops"][crop["name"]]["block_face"] = d_project( + plotter, [(0, 0, zb), (512, 0, zb), (512, -512, zb), (0, -512, zb)] + ) + layout["crops"][crop["name"]]["exploded_px_per_pixel"] = d_px_per_unit(plotter, (256, -256, zb)) + # the top face of the lowest channel card, for the lines from the outline on the section + x0 = -3 * spread + layout["crops"][crop["name"]]["deck_face"] = d_project( + plotter, [(x0, 0, card), (x0 + 512, 0, card), (x0 + 512, -512, card), (x0, -512, card)] + ) + plotter.close() + d_log(crop["name"], n_labels, "cells") + json.dump(layout, open(f"{out}/layout.json", "w"), indent=1) + d_log("gruen done") + + +def render_histai(out): + """The render of the Ki67 core example: the whole core upright in the studio (scan orientation), and in a column on + its right one full resolution crop, the window with the most of both tumor classes, joined by lines from its box + on the core: the image data on top and the classification below it, every segmented nucleus with a light fill and + an outline in the colour of its class (one smaller zoom, image and classification).""" + os.makedirs(out, exist_ok=True) + group = zarr.open_group(zarr.storage.ZipStore(HISTAI, mode="r"), mode="r", zarr_format=2) + h0, w0 = group["0"].shape[1:] + size_px = HISTAI_CROP + # the crops, found on level 3 (1 px = 8 level 0 px), at least two crop sizes apart + classes3 = group["labels/classes/3"][:] + k = size_px // 8 + maps = { + "positive": ndimage.uniform_filter((classes3 == 1).astype("float32"), k), + "negative": ndimage.uniform_filter((classes3 == 2).astype("float32"), k), + "other": ndimage.uniform_filter((classes3 == 3).astype("float32"), k) + } + # the most diverse window: the one where the rarest of the three classes is densest + score = np.minimum(np.minimum(maps["positive"], maps["negative"]), maps["other"]) + cy, cx = np.unravel_index(np.argmax(score), score.shape) + crop = (int(np.clip(cy * 8 - size_px // 2, 0, h0 - size_px)), int(np.clip(cx * 8 - size_px // 2, 0, w0 - size_px))) + + def raw_crop(y0, x0): + return np.moveaxis(group["0"][:, y0:y0 + size_px, x0:x0 + size_px], 0, -1).copy() + + def outlined(y0, x0): + """The crop with each nucleus in the colour of its class (the class of most of its pixels): a light fill and a + thick outline. Also return the number of nuclei per class.""" + raw = np.moveaxis(group["0"][:, y0:y0 + size_px, x0:x0 + size_px], 0, -1).copy() + inst = group["labels/instances/0"][y0:y0 + size_px, x0:x0 + size_px] + cls = group["labels/classes/0"][y0:y0 + size_px, x0:x0 + size_px] + ids, inverse = np.unique(inst, return_inverse=True) + inst = inverse.reshape(inst.shape) + n = len(ids) - 1 + votes = np.stack([np.bincount(inst[cls == c].ravel(), minlength=n + 1) for c in range(4)], 1) + obj_class = votes[:, 1:].argmax(1) + 1 + obj_class[votes[:, 1:].sum(1) == 0] = 0 + inside = inst > 0 + raw[inside] = (0.6 * raw[inside] + 0.4 * HISTAI_CLASS_RGB[obj_class[inst[inside]]]).astype(np.uint8) + edge = ndimage.binary_dilation(find_boundaries(inst, mode="inner"), iterations=2) & inside + raw[edge] = HISTAI_CLASS_RGB[obj_class[inst[edge]]] + return raw, [int((obj_class[1:] == c).sum()) for c in (1, 2, 3)] + + def rect(x, y, w, h): + """Top left, top right, bottom right, bottom left; y is the top edge (scene y grows upwards).""" + return [(x, y), (x + w, y), (x + w, y - h), (x, y - h)] + + t, core_h, card, gap_x, gap_y = 20.0, 3000.0, 1100.0, 250.0, 120.0 + step0 = core_h / h0 # scene units per level 0 px on the core + core_w = w0 * step0 + column_x = core_w + gap_x + column_top = -(core_h - (2 * card + gap_y)) / 2 + view, _ = d_view(55, -90) + size = (3900, 3900) # large enough to print at more than 1200 dpi + plotter = d_studio(size, (0, column_x + card, -core_h, 0, 0, t), view, 8000, 1.0) + d_card(plotter, np.moveaxis(group["3"][:], 0, -1), (0, 0, 0), step0 * 8, t) + layout = {"um_per_px": HISTAI_UM, "crops": {}} + y0, x0 = crop + classified, counts = outlined(y0, x0) + box = rect(x0 * step0, -y0 * step0, size_px * step0, size_px * step0) + ring = [(x, y, t + 1) for x, y in box + box[:1]] + plotter.add_mesh(pv.lines_from_points(np.array(ring)), color=(0.1, 0.1, 0.1), line_width=8) + # the image data on top, the classification below it + for i, image in enumerate((raw_crop(y0, x0), classified)): + top = column_top - i * (card + gap_y) + d_card(plotter, image, (column_x, top, 0), card / size_px, t) + if i == 0: + # from the right corners of the box to the left corners of the image card + target = rect(column_x, top, card, card) + for a, b in ((box[1], target[0]), (box[2], target[3])): + plotter.add_mesh( + pv.Line((a[0], a[1], t + 1), (b[0], b[1], t + 1)), color=(0.15, 0.15, 0.15), line_width=5 + ) + layout["crops"]["mixed"] = {"y0": y0, "x0": x0, "positive": counts[0], "negative": counts[1], "other": counts[2]} + image = d_screenshot(plotter, size) + # pixels of the render per micrometer, at the centre of the core and of the middle crop + layout["core_px_per_um"] = d_px_per_unit(plotter, (core_w / 2, -core_h / 2, t)) * step0 / HISTAI_UM + middle = (column_x + card / 2, column_top - card / 2, t) + layout["crop_px_per_um"] = d_px_per_unit(plotter, middle) * card / size_px / HISTAI_UM + left, bottom_px = d_project(plotter, [(column_x, column_top - 2 * card - gap_y, t)])[0] + # the top of the image card, for the legend above it + column_top_px = d_project(plotter, [(column_x, column_top, t)])[0][1] + plotter.close() + rows, cols = np.nonzero((image < 245).any(-1)) + r0, c0 = max(rows.min() - 8, 0), max(cols.min() - 8, 0) + image = image[r0:rows.max() + 8, c0:cols.max() + 8] + layout["crop_column_left"] = (left - c0) / image.shape[1] + layout["crop_column_top"] = (column_top_px - r0) / image.shape[0] + layout["crop_column_bottom"] = (bottom_px - r0) / image.shape[0] + Image.fromarray(image).save(f"{out}/zoom.png") + json.dump(layout, open(f"{out}/layout.json", "w"), indent=1) + d_log("histai done") + + +def d_fonts(): + for name in os.listdir(FONT_DIR): + if name.endswith(".ttf"): + font_manager.fontManager.addfont(os.path.join(FONT_DIR, name)) + matplotlib.rcParams["font.family"] = "Liberation Sans" + matplotlib.rcParams["font.size"] = 7 + + +def d_load(path, mode="RGB"): + return np.asarray(Image.open(path).convert(mode)) + + +def d_crop_white(image, margin): + """Crop a white background image to its content plus a margin; also return the offset (column, row).""" + rows, cols = np.nonzero((image < 245).any(-1)) + r0, c0 = max(rows.min() - margin, 0), max(cols.min() - margin, 0) + return image[r0:rows.max() + margin, c0:cols.max() + margin], np.array([c0, r0]) + + +def d_crop_to(image, points, keep, margin): + """Crop to the pixels to keep and the given points, plus a margin; also return the offset (column, row).""" + rows, cols = np.nonzero(keep) + pts = np.asarray(points).reshape(-1, 2) + c0, c1 = min(cols.min(), pts[:, 0].min()), max(cols.max(), pts[:, 0].max()) + r0, r1 = min(rows.min(), pts[:, 1].min()), max(rows.max(), pts[:, 1].max()) + c0, r0 = int(max(c0 - margin, 0)), int(max(r0 - margin, 0)) + c1, r1 = int(min(c1 + margin, image.shape[1])), int(min(r1 + margin, image.shape[0])) + return image[r0:r1, c0:c1], np.array([c0, r0]) + + +def d_colourful(image): + """The scene without the grey studio: coloured or dark pixels.""" + rgb = image.astype(int) + return (np.ptp(rgb, -1) > 30) | (rgb.max(-1) < 60) + + +def d_card_corners(image): + """The corners of a dark tilted card on white: top left, top right, bottom right, bottom left.""" + rows, cols = np.nonzero((image < 250).any(-1)) + s, d = cols + rows, cols - rows + return np.array([(cols[i], rows[i]) for i in (np.argmin(s), np.argmax(d), np.argmax(s), np.argmin(d))], float) + + +def d_stretch(raw, on_card): + """More contrast in a raw card: stretch to its 98th percentile on the card, with a slight gamma.""" + grey = raw[..., 0].astype(float) + hi = np.percentile(grey[on_card & (grey > 8)], 98) + stretched = (np.clip(grey / hi, 0, 1) ** 0.8 * 255)[..., None].repeat(3, -1) + return np.where(on_card[..., None], stretched, raw).astype("uint8") + + +def d_layered(raw, seg, shift): + """A stack of two cards of the same tilted shape: the raw at the back with its edge drawn, and the objects (with + their own alpha, rendered on a transparent background) on a clear card in front, moved by the shift vector. + Returns the image, the card corners and the origins of the back and front layers.""" + h, w = raw.shape[:2] + dx, dy = int(round(shift[0])), int(round(shift[1])) + back = np.array([0, max(0, -dy)]) + front = back + [dx, dy] + out = np.full((h + abs(dy), w + dx, 3), 255, "uint8") + out[back[1]:back[1] + h, back[0]:back[0] + w] = raw + card = d_card_corners(raw) + edge = Image.fromarray(out) + ImageDraw.Draw(edge).polygon([tuple(p) for p in card + back], outline=(138, 138, 138), width=3) + out = np.asarray(edge).astype(float) + alpha = seg[..., 3:4].astype(float) / 255 + region = out[front[1]:front[1] + h, front[0]:front[0] + w] + out[front[1]:front[1] + h, front[0]:front[0] + w] = region * (1 - alpha) + seg[..., :3] * alpha + return np.clip(out, 0, 255).astype("uint8"), card, back, front + + +# the print resolutions of the images placed in panel d +D_DPI = [] + + +class DFigure: + """A matplotlib figure laid out in mm from the top left.""" + + def __init__(self, width, height): + self.width, self.height = width, height + self.fig = plt.figure(figsize=(width / 25.4, height / 25.4), dpi=300) + + def axes(self, x, y, w, h): + return self.fig.add_axes([x / self.width, 1 - (y + h) / self.height, w / self.width, h / self.height]) + + def image(self, image, x, y, w, h, frame=False): + """The image as large as fits in a box, aligned to the top left; with a thin frame or without axes.""" + ih, iw = image.shape[:2] + scale = min(w / iw, h / ih) + # the print resolution of the image (pixels per inch), for the check that panel d stays above 1200 dpi + D_DPI.append(25.4 / scale) + ax = self.axes(x, y, iw * scale, ih * scale) + # no interpolation: the svg keeps the full resolution of the render, instead of resampling it to 72 dpi + ax.imshow(image, interpolation="none") + if frame: + ax.set_xticks([]) + ax.set_yticks([]) + for spine in ax.spines.values(): + spine.set_linewidth(0.6) + spine.set_color(D_LINE) + else: + ax.axis("off") + return ax + + def title(self, ax, text): + ax.text( + 0.5, 1.0 + 1.6 / (ax.get_position().height * self.height), text, transform=ax.transAxes, ha="center", + va="baseline", fontsize=7, color=D_LINE + ) + + def connect(self, ax_a, point_a, ax_b, point_b): + self.fig.add_artist(ConnectionPatch( + xyA=point_a, coordsA=ax_a.transData, xyB=point_b, coordsB=ax_b.transData, lw=0.5, color=D_LINE + )) + + def svg(self): + """The figure as an svg element sized in mm, with editable text, to place in the svg of the figure.""" + buffer = io.StringIO() + # transparent, so that an example placed over the empty margin of another does not hide it + with matplotlib.rc_context({"svg.fonttype": "none", "image.composite_image": False}): + self.fig.savefig(buffer, format="svg", dpi=300, transparent=True) + plt.close(self.fig) + text = buffer.getvalue() + root = text[text.index("", 1) + head = re.sub(r'\s(width|height)="[^"]*"', "", head) + # the text asks for Arial first, as in the other panels; Liberation Sans has its metrics + body = re.sub(r"font-family:\s*'?Liberation Sans'?", "font-family:Arial,'Liberation Sans',sans-serif", body) + return f'{head} width="{fmt(self.width)}" height="{fmt(self.height)}">{body}' + + +def d_scale_bar(fig, ax, um, px_per_um, below=1.6, inset=0.03): + """A scale bar a fixed distance (in mm) below the image of the axes, at its left, with its label to its right, + so that neither covers the image at any size. Whole millimeters are labelled in mm.""" + h, w = ax.get_ylim()[0] + 0.5, ax.get_xlim()[1] + 0.5 + px_per_mm = h / (ax.get_position().height * fig.height) + x0, y = inset * w, h + below * px_per_mm + ax.plot([x0, x0 + um * px_per_um], [y, y], color=D_LINE, lw=1.6, solid_capstyle="butt", clip_on=False) + label = f"{um // 1000} mm" if um >= 1000 and um % 1000 == 0 else f"{um} µm" + ax.text( + x0 + um * px_per_um + 0.8 * px_per_mm, y, label, ha="left", va="center", color=D_LINE, fontsize=6.5, + clip_on=False + ) + ax.set_ylim(h - 0.5, -0.5) + + +def d_box(ax, corners, lw=0.7): + """A dark outline with a white halo under it, so that it shows on dark and on white.""" + ax.add_patch(Polygon(corners, closed=True, fill=False, edgecolor="white", lw=lw + 1.1, joinstyle="miter")) + ax.add_patch(Polygon(corners, closed=True, fill=False, edgecolor=D_LINE, lw=lw, joinstyle="miter")) + + +def d_right_edge(corners): + """The upper and the lower of the two rightmost corners.""" + right = corners[np.argsort(corners[:, 0])[-2:]] + return right[np.argmin(right[:, 1])], right[np.argmax(right[:, 1])] + + +def build_platynereis(folder, height, measure=False): + """The Platynereis example: a) the nuclei in the studio with the section plane, b) the section as a stack of the + raw card at the back and its nuclei on a clear card in front, joined to the plane by thin lines.""" + layout = json.load(open(f"{folder}/layout.json")) + um_pre = layout["voxel_um"] / layout["expansion"] # pre-expansion micrometers per s1 voxel + raw = d_load(f"{folder}/section_raw.png") + raw = d_stretch(raw, (raw < 250).any(-1)) + corners = d_card_corners(raw) + section, card, back, front = d_layered( + raw, d_load(f"{folder}/section_rgba.png", "RGBA"), 0.42 * (corners[1] - corners[0]) + ) + section, offset = d_crop_white(section, 8) + glass, back_card = card + front - offset, card + back - offset + # a: the studio cropped to the animal and the section plane, with a small margin + seg = d_load(f"{folder}/seg_studio.png") + outline = np.array(layout["studio"]["slab_outline"]) + cols = np.flatnonzero((np.ptp(seg.astype(int), -1) > 40).any(0)) + a0 = max(int(min(cols.min(), outline[:, 0].min())) - 4, 0) + a1 = min(int(max(cols.max(), outline[:, 0].max())) + 4, seg.shape[1]) + seg, outline = seg[:, a0:a1], outline - [a0, 0] + + # the widths follow from one common height of a and b + top, gap = D_TOP["platynereis"], 2.5 + row_h = height - top - 4 + wa = row_h * seg.shape[1] / seg.shape[0] + wb = row_h * section.shape[1] / section.shape[0] + xa, xb = 3.0, 3.0 + wa + gap + width = xb + wb + 1.5 + if measure: + return None, width + fig = DFigure(width, height) + ax_a = fig.image(seg, xa, top, wa, row_h) + ax_b = fig.image(section, xb, top, wb, row_h) + # lines from the right edge of the plane to the left corners of the raw card + ax_a.add_patch(Polygon(outline, closed=True, fill=False, edgecolor=D_LINE, lw=0.8, joinstyle="miter")) + upper, lower = d_right_edge(outline) + fig.connect(ax_a, upper, ax_b, back_card[0]) + fig.connect(ax_a, lower, ax_b, back_card[3]) + ax_b.add_patch(Polygon(glass, closed=True, fill=False, edgecolor="#8a8a8a", lw=0.5, joinstyle="miter")) + d_scale_bar(fig, ax_a, 50, layout["studio"]["px_per_voxel"] / um_pre) + d_scale_bar(fig, ax_b, 20, layout["section"]["px_per_voxel"] / um_pre) + return fig.svg(), width + + +# the height of the studio of the spheroid example, as a share of the height of its plane +SPHEROID_STUDIO_SHARE = 0.72 + + +def build_spheroid(folder, height, measure=False): + """The spheroid example: its cells and its nuclei in one studio, side by side in shared colours, with one + rectangle of the middle plane through both; and the plane of both channels, larger, with the outlines of both + segmentations on its right half, joined to the rectangle by lines.""" + layout = json.load(open(f"{folder}/layout.json")) + studio = d_load(f"{folder}/studio.png") + rectangle = np.array(layout["rectangle"]) + studio, offset = d_crop_to(studio, rectangle, np.ptp(studio.astype(int), -1) > 40, 12) + # a little more of the studio, so that both walls and the floor show + full = d_load(f"{folder}/studio.png") + h = studio.shape[0] + r0, c0 = max(int(offset[1] - 0.22 * h), 0), max(int(offset[0] - 0.08 * h), 0) + r1 = min(int(offset[1] + h + 0.06 * h), full.shape[0]) + c1 = min(int(offset[0] + studio.shape[1] + 0.08 * h), full.shape[1]) + studio, offset = full[r0:r1, c0:c1], np.array([c0, r0]) + rectangle = rectangle - offset + plane = d_load(f"{folder}/plane.png") + + # the plane takes the full height of the row, the studio a part of it, centred (a larger plane) + top, gap = D_TOP["spheroid"] + 1.5, 4.0 + row_h = height - top - 4.0 + ha = SPHEROID_STUDIO_SHARE * row_h + wa = ha * studio.shape[1] / studio.shape[0] + wb = row_h * plane.shape[1] / plane.shape[0] + width = 3.0 + wa + gap + wb + 1.5 + if measure: + return None, width + fig = DFigure(width, height) + ax_a = fig.image(studio, 3.0, top + (row_h - ha) / 2, wa, ha) + ax_b = fig.image(plane, 3.0 + wa + gap, top, wb, row_h) + ax_a.add_patch(Polygon(rectangle, closed=True, fill=False, edgecolor=D_LINE, lw=0.8, joinstyle="miter")) + # the right corners of the rectangle to the left corners of the plane + right = rectangle[np.argsort(rectangle[:, 0])[-2:]] + right = right[np.argsort(right[:, 1])] + ph = plane.shape[0] + fig.connect(ax_a, right[0], ax_b, (-0.5, -0.5)) + fig.connect(ax_a, right[1], ax_b, (-0.5, ph - 0.5)) + # the names stand above the studio, over their spheroids + px_per_mm = studio.shape[0] / ha + for name, (x, _) in layout["labels"].items(): + ax_a.text( + x - offset[0], -0.8 * px_per_mm, name, ha="center", va="bottom", fontsize=7, color=D_LINE, clip_on=False + ) + d_scale_bar(fig, ax_a, 50, layout["px_per_voxel"] / layout["xy_um"]) + d_scale_bar(fig, ax_b, 50, 1 / layout["xy_um"]) + return fig.svg(), width + + +def build_gruen(folder, height, measure=False): + """The heart section panel: the volume as one card of its merged channels with the outline of one crop on it + (the lowest), and on its right the exploded view of the crop: its three channels at the + bottom, the merged block above them and the segmented cells floating on top, joined to the outline by lines to + the top face of the block.""" + layout = json.load(open(f"{folder}/layout.json")) + volume = d_load(f"{folder}/volume.png") + crops = layout["crops"] + name = max(crops, key=lambda n: np.mean(np.array(crops[n]["outline"])[:, 1])) + # tight crops and a small gap, so that the example is as large as its box allows + volume, offset = d_crop_to(volume, [crops[name]["outline"]], d_colourful(volume), 2) + image = d_load(f"{folder}/{name}_exploded.png") + image, off = d_crop_to(image, crops[name]["block_face"], d_colourful(image), 2) + deck = np.array(crops[name]["deck_face"]) - off + + top, gap = D_TOP["gruen"], 2.5 + row_h = height - top - 4 + wa = row_h * volume.shape[1] / volume.shape[0] + wc = row_h * image.shape[1] / image.shape[0] + xa, xc = 3.0, 3.0 + wa + gap + width = xc + wc + 1.5 + if measure: + return None, width + fig = DFigure(width, height) + ax_a = fig.image(volume, xa, top, wa, row_h) + outline = np.array(crops[name]["outline"]) - offset + d_box(ax_a, outline) + ax = fig.image(image, xc, top, wc, row_h) + # the pixel size is estimated (GRUEN_XY_UM), not from the files + d_scale_bar(fig, ax, 20, crops[name]["exploded_px_per_pixel"] / layout["xy_um"]) + # lines between the lower edges of the outline and of the lowest channel card + low_a = outline[np.argsort(outline[:, 1])[-2:]] + low_b = deck[np.argsort(deck[:, 1])[-2:]] + for a, b in zip(low_a[np.argsort(low_a[:, 0])], low_b[np.argsort(low_b[:, 0])]): + fig.connect(ax_a, a, ax, b) + d_scale_bar(fig, ax_a, 100, layout["volume_px_per_pixel"] / layout["xy_um"]) + return fig.svg(), width + + +def build_histai(folder, height, measure=False): + """The Ki67 core example: its render with a scale bar for the core and one for the crop below it, and the legend of + the classes above the crop.""" + layout = json.load(open(f"{folder}/layout.json")) + image = d_load(f"{folder}/zoom.png") + top = D_TOP["histai"] + row_h = height - top - 4.0 + width = 3.0 + row_h * image.shape[1] / image.shape[0] + 1.5 + if measure: + return None, width + fig = DFigure(width, height) + ax = fig.image(image, 3.0, top, width - 4.5, row_h) + d_scale_bar(fig, ax, 1000, layout["core_px_per_um"], inset=0.02) + d_scale_bar(fig, ax, 50, layout["crop_px_per_um"], inset=layout["crop_column_left"]) + h, w = ax.get_ylim()[0] + 0.5, ax.get_xlim()[1] + 0.5 + px_per_mm = h / (ax.get_position().height * fig.height) + x0 = layout["crop_column_left"] * w + # the legend below the classification, in two rows + renderer = fig.fig.canvas.get_renderer() + data_per_display = w / ax.get_window_extent(renderer=renderer).width + swatch, inner, outer = 1.4 * px_per_mm, 0.6 * px_per_mm, 1.4 * px_per_mm + rows = (HISTAI_CLASSES[:2], HISTAI_CLASSES[2:]) + for r, entries in enumerate(rows): + y = layout["crop_column_bottom"] * h + (1.6 + 2.4 * r) * px_per_mm + texts, widths = [], [] + for name, _ in entries: + text = ax.text(0, y, name, ha="left", va="center", color=D_LINE, fontsize=6.5, clip_on=False) + texts.append(text) + widths.append(text.get_window_extent(renderer=renderer).width * data_per_display) + total = sum(swatch + inner + tw for tw in widths) + outer * (len(widths) - 1) + x = x0 + (w - x0 - total) / 2 + for (name, rgb), text, tw in zip(entries, texts, widths): + ax.add_patch(matplotlib.patches.Rectangle( + (x, y - 0.7 * px_per_mm), swatch, swatch, facecolor=np.array(rgb) / 255, edgecolor="none", + clip_on=False + )) + text.set_x(x + swatch + inner) + x += swatch + inner + tw + outer + return fig.svg(), width + + +D_RENDERS = { + "platynereis": render_platynereis, "spheroid": render_spheroid, "gruen": render_gruen, "histai": render_histai +} +# the columns of panel d; the examples of a column are stacked at one width +D_COLUMNS = (("platynereis",), ("spheroid", "gruen"), ("histai",)) +D_BUILDERS = { + "platynereis": build_platynereis, "spheroid": build_spheroid, "gruen": build_gruen, "histai": build_histai +} +D_EXAMPLE_GAP = 1.5 +# the gap between stacked examples (the one above already ends with its scale bar strip) +D_STACK_GAP = 0.0 + + +def build_panel_d(folder, width=A4_WIDTH, height=None): + """Panel d: the examples in one row of columns, from their renders in the folder, each with its number (i, ii, + ...) at the top left of its first image. All columns have the same height, at which the row fills the width (at + most the given height); the examples of a column are stacked at the same width. The width of an example grows + linearly with its height. Returns the svg content and the size of the panel.""" + d_fonts() + line = {} + for column in D_COLUMNS: + for name in column: + build, path = D_BUILDERS[name], os.path.join(folder, name) + low, high = build(path, 50.0, measure=True)[1], build(path, 100.0, measure=True)[1] + line[name] = ((high - low) / 50.0, low - (high - low)) # slope and offset + + def column_width(column, h): + """The width of a column of height h: its examples stacked at one width, filling the height.""" + inverse = sum(1 / line[n][0] for n in column) + return (h - D_STACK_GAP * (len(column) - 1) + sum(line[n][1] / line[n][0] for n in column)) / inverse + + def row_width(h): + return sum(column_width(c, h) for c in D_COLUMNS) + D_EXAMPLE_GAP * (len(D_COLUMNS) - 1) + + # the height at which the row fills the width (the row width grows linearly with the height) + h50, h100 = row_width(50.0), row_width(100.0) + panel_h = 50.0 + (width - h50) * 50.0 / (h100 - h50) + if height is not None: + panel_h = min(panel_h, height) + widths = [column_width(c, panel_h) for c in D_COLUMNS] + gap = (width - sum(widths)) / (len(D_COLUMNS) - 1) + parts, x, numbers = [], 0.0, iter(("i", "ii", "iii", "iv", "v", "vi")) + size = MAX_TEXT # 7 pt, below the 8 pt panel letters + for column, column_w in zip(D_COLUMNS, widths): + y = 0.0 + for name in column: + example_h = (column_w - line[name][1]) / line[name][0] + svg, _ = D_BUILDERS[name](os.path.join(folder, name), example_h) + parts.append(f'{svg}') + # right next to the first image of the example (which starts 3 mm in and D_TOP down), at its top + parts.append( + f'{next(numbers)}' + ) + y += example_h + D_STACK_GAP + x += column_w + gap + return "".join(parts), width, panel_h + + +def main(): + parser = argparse.ArgumentParser(description="Make the panels of Figure 1 and put them together.") + parser.add_argument("--output_folder", default="figures/figure1", help="The folder for the panel files.") + parser.add_argument("--final_folder", default="figures/final", help="The folder for the whole figure.") + parser.add_argument("--logo", default=LOGO, help="The micro-sam2 logo.") + parser.add_argument("--napari_logo", default=NAPARI_LOGO, help="The napari logo, by default the one of napari.") + parser.add_argument("--python_logo", default=PYTHON_LOGO, help="The Python logo.") + parser.add_argument("--panels", nargs="+", default=["a", "b", "c", "d"], help="The panels to make.") + parser.add_argument( + "--render", nargs="+", default=[], choices=sorted(D_RENDERS), + help="The panel d examples to render again, before panel d is put together (on a GPU node)." + ) + parser.add_argument( + "--platy_raw_cache", default=os.path.join(os.environ.get("TMPDIR", "/tmp"), "platynereis_raw_s0.npy"), + help="The cache of the normalised Platynereis raw at s0 (3.5 GB), for the section." + ) + args = parser.parse_args() + + os.makedirs(args.output_folder, exist_ok=True) + panels = {} + # panel d first: its examples may be rendered again + d_folder = os.path.join(args.output_folder, "panel_d") + for name in args.render: + if name == "platynereis": + render_platynereis(os.path.join(d_folder, name), args.platy_raw_cache) + else: + D_RENDERS[name](os.path.join(d_folder, name)) + d = build_panel_d(d_folder) + if "d" in args.panels: + panels["d"] = d + if "a" in args.panels: + panels["a"] = (build_panel_a(), A4_WIDTH, A_H * A4_WIDTH / A_W) + # panel c takes the size of panel b, so panel b comes first + if "b" in args.panels or "c" in args.panels: + panels["b"] = build_panel_b() + if "c" in args.panels: + fetch_python_logo(args.python_logo) + _, b_w, b_h = panels["b"] + c_w = FIGURE_W - 2 * LETTER_W - PANEL_GAP - b_w + panels["c"] = build_panel_c(args.logo, args.napari_logo, args.python_logo, c_w, b_h) + for name, (content, width, height, *_) in panels.items(): + if name in args.panels: + save_panel(content, args.output_folder, f"fig1_panel_{name}", width, height) + # the whole figure needs all panels + if {"a", "b", "c", "d"} <= set(args.panels): + os.makedirs(args.final_folder, exist_ok=True) + content, width, height = build_figure(panels) + save_panel(content, args.final_folder, "figure_1", width, height) + + +if __name__ == "__main__": + main() diff --git a/scripts/plotting/v2/for_paper/plots_for_figure2.py b/scripts/plotting/v2/for_paper/plots_for_figure2.py new file mode 100644 index 000000000..33eaa3cfb --- /dev/null +++ b/scripts/plotting/v2/for_paper/plots_for_figure2.py @@ -0,0 +1,1760 @@ +"""Plots for Figure 2 of the micro-sam2 paper. + +The figure compares 2d automatic and interactive instance segmentation of micro-sam2 (hvit_l) with the baselines. +The script writes these plots to 'figures/figure2//': +- 'automatic_2d': the mSA of each method per modality, for the in-domain and the out-of-domain datasets. +- 'interactive_2d': the mSA over the correction iterations, with a point or a box as the first prompt. +- 'automatic_2d_qualitative': one test image per modality and domain, with the reference and the predictions. +- 'interactive_2d_qualitative': the crops of one object per modality and domain, segmented from one prompt. +- 'figure_2' (in 'figures/final'): the four plots in one figure, as panels a to d. + +Usage: + + python plots_for_figure2.py + python plots_for_figure2.py --only automatic_2d_qualitative interactive_2d_qualitative + +The inputs are: +- RESULTS_CSV and INTERACTIVE_CSV: the scores from the scripts in 'finetuning/v2/evaluation'. +- QUALITATIVE_PREDICTIONS: the predictions for the automatic qualitative plot. To make them, write the script with + '--write_prediction_script run.sh' and run it on a GPU node. +- SINGLE_PROMPT_RUNS: the single-prompt predictions for the interactive qualitative plot, from + 'run_posthoc_scripts.py first_prompt'. + +The text uses Liberation Sans, which has the letter widths of Arial (see plot_style.py). +""" + +import io +import os +import re +import glob +import shutil +import argparse +import subprocess + +import tifffile +import numpy as np +import pandas as pd +from PIL import Image +from skimage.segmentation import find_boundaries +from scipy.ndimage import binary_dilation, grey_dilation + +import matplotlib.pyplot as plt +import matplotlib.colors as colors +from matplotlib import font_manager +import matplotlib.transforms as transforms +import matplotlib.patheffects as patheffects + +from elf.evaluation import mean_segmentation_accuracy + +from plot_style import MAX_FONT_SIZE, METHOD_COLORS, TEXT_COLOR, set_style + +# The labels that have little room (the values and ranks next to the bars, the mode tags in the bars and the scores +# on the images) are a half point smaller than all other text, which is MAX_FONT_SIZE (Constantin Pape, 2026-11-06: +# bigger fonts). +SMALL_FONT_SIZE = MAX_FONT_SIZE - 0.5 + + +ROOT = os.path.dirname(os.path.abspath(__file__)) +RESULTS_CSV = os.path.join(ROOT, "figures", "cells_automatic_segmentation_2d_results.csv") +INTERACTIVE_CSV = os.path.join(ROOT, "figures", "cells_interactive_segmentation_2d_results.csv") +OUTPUT_DIR = os.path.join(ROOT, "figures", "figure2") +EXPERIMENTS = "/mnt/vast-nhr/projects/cidas/cca/experiments/micro_sam2/experiments" +QUALITATIVE_PREDICTIONS = os.path.join(EXPERIMENTS, "qualitative_2d", "out") +SINGLE_PROMPT_RUNS = os.path.join(EXPERIMENTS, "qualitative_single_prompt_{dataset}") +EVALUATION = os.path.join(ROOT, "..", "..", "..", "..", "finetuning", "v2", "evaluation") + +# The 'method', 'model' and 'mode' values of each method in the results. A mode of None selects the rows without one. +METHODS = { + "StarDist": ("stardist", ("default",), None), + "SAM": ("sam_amg", ("vit_l",), None), + "micro-sam": ("micro-sam", ("vit_l_lm", "vit_l_histopathology"), "apg"), + "CellSAM": ("cellsam", ("default",), None), + "Cellpose3": ("cellpose", ("cyto3",), None), + "SAM2": ("sam2_amg", ("hvit_l",), None), + "InstanSeg": ("instanseg", ("fluorescence_nuclei_and_cells", "brightfield_nuclei"), None), + "CellposeSAM": ("cellpose", ("cpsam",), None), + "CellViT++": ("cellvitpp", ("sam_h_x40_amp",), None), + "SAM3": ("sam3", ("sam3.1",), None), + "micro-sam2": ("micro-sam2", ("hvit_l",), "apg"), +} + +PANELS = { + "label_free_cells": ("label-free", "cell", "Label-free (cell)"), + "fluorescence_cells": ("fluorescence", "cell", "Fluorescence (cell)"), + "fluorescence_nuclei": ("fluorescence", "nucleus", "Fluorescence (nucleus)"), + "histopathology_nuclei": ("histopathology", "nucleus", "Histopathology (nucleus)"), +} + +# The methods of each modality, in their order in the plots. +PANEL_METHODS = { + "label_free_cells": ["SAM", "SAM2", "SAM3", "Cellpose3", "micro-sam", "CellSAM", "CellposeSAM", "micro-sam2"], + "fluorescence_cells": [ + "SAM", "SAM2", "SAM3", "InstanSeg", "Cellpose3", "micro-sam", "CellSAM", "CellposeSAM", "micro-sam2", + ], + "fluorescence_nuclei": [ + "SAM", "SAM2", "SAM3", "StarDist", "InstanSeg", "Cellpose3", "micro-sam", "CellSAM", "CellposeSAM", + "micro-sam2", + ], + "histopathology_nuclei": [ + "SAM", "SAM2", "SAM3", "StarDist", "InstanSeg", "micro-sam", "CellSAM", "CellViT++", "CellposeSAM", + "micro-sam2", + ], +} + +# micro-sam shows only AIS, its default mode, in the automatic plot. +FIGURE_MODES = {"micro-sam": "ais"} + +MODES = ["ais", "apg"] +MODE_METHODS = ["micro-sam", "micro-sam2"] + +PANEL_MODELS = { + "fluorescence_nuclei": {"Cellpose3": ("nuclei",)}, +} + +# These methods run a different model for some modalities, so they get a star. +STARRED_METHODS = {"Cellpose3", "InstanSeg", "StarDist"} + +EXCLUDED_DATASETS = {"dic_hepg2", "xenium_nuclei", "xenium_cells"} + +# The in-domain and out-of-domain datasets of each modality. They set the domain of each dataset. +PANEL_DATASETS = { + "label_free_cells": { + "ID": ["livecell", "neurips_cellseg_label_free", "cell_acdc", "vicar", "omnipose_bact_phase", "orgline"], + # Only datasets kept out of the micro-sam2 training, see DATASETS_2D_LM_LABEL_FREE_OOD in 'common.py'. + # dic_hepg2 is left out: all methods score close to 0 on it. + "OOD": ["deepseas", "bbbc010", "yeastsam", "cellapp", "bac_mother", "ecoli_microcolony_lineage"], + }, + "fluorescence_nuclei": { + "ID": ["dsb", "dynamicnuclearnet", "bmgd", "ifnuclei", "u20s", "cellbindb"], + # spatch_dapi is left out: its tumor samples also give the sPATCH H&E training data. + "OOD": ["arvidsson", "hela_cytonuc", "covid_if_nuclei", "cardioblast_nuclei", "mif_tonsil", "micro_bench"], + }, + "histopathology_nuclei": { + "ID": ["glysac", "histo_miner", "monuseg", "cpm17", "puma", "pannuke"], + # Only datasets kept out of the micro-sam2 training and densely annotated by hand or fully proofread, see + # DATASETS_2D_HP_OOD in 'common.py'. cd47_ihcnusc is IHC, the others are H&E. + "OOD": ["cytodark0", "pcns", "khoshdeli", "beacon", "cd47_ihcnusc", "amonuseg"], + }, + "fluorescence_cells": { + "ID": [ + "tissuenet", "neurips_cellseg_fluorescence", "flywing", "omnipose_bact_fluor", "cvz_fluo_cell", + "deepbacs_fluorescence", + ], + "OOD": ["covid_if_cells", "hpa", "medussa", "mif_tonsil_cells", "fusionx", "wing_disc_timelapse"], + }, +} + +DOMAIN_TITLES = {"ID": "In-domain", "OOD": "Out-of-domain"} +# The domain titles in the style of the dataset names: in-domain in italics, out-of-domain in bold. +DOMAIN_STYLES = {"ID": {"style": "italic"}, "OOD": {"fontweight": "bold"}} + +# The value axis ends at the first multiple of VALUE_STEP that is at least VALUE_ROOM above the highest bar. +VALUE_STEP = 0.05 +VALUE_ROOM = 0.01 +# The room for the mean rank after the longest bar, and the number of bars that show their mean rank. +RANK_ROOM = 0.1 + +BAR_HEIGHT = 0.72 +FOUNDATION_MODELS = {"SAM", "SAM2", "SAM3"} +# The gap between the method groups, in bar rows. +GROUP_GAP = 0.5 +# This gray has a contrast of 5.3 on white. Nature asks for at least 4.5. +GROUP_COLOR = "#6b6b6b" + +DATASET_NAMES = { + "arvidsson": "Arvidsson", + "cellapp": "Cell-APP", + "cellbindb": "CellBinDB", + "cd47_ihcnusc": "CD47 IHC-NuSC", + "covid_if_nuclei": "Covid IF", + "cpm17": "CPM17", + "cytodark0": "CytoDArk0", + "histo_miner": "Histo-Miner", + "hpa": "HPA", + "mcellseg": "mCellSeg", + "medussa": "MEDUSSA", + "neurips_cellseg_label_free": "NeurIPS CellSeg", + "spatch_dapi": "SPATCH DAPI", + "srsanet": "SRSA-Net", + "tissuenet": "TissueNet", + "vicar": "VICAR", + "yeastsam": "YeastSAM", +} + + +def load_results(category, structure, mode, models=None, path=RESULTS_CSV, methods=None, metric="msa"): + """Load the scores of all methods for one modality, with the methods of MODE_METHODS in 'mode'. + + Args: + category: The category of the modality in the results. + structure: The structure of the modality in the results. + mode: The mode of the methods in MODE_METHODS. + models: The models that replace the models in METHODS, per method name. + path: The path to the results table. + methods: The method table. The default is METHODS. + metric: The score column. + + Returns: + The results in long format. + """ + df = pd.read_csv(path) + df = df[(df["category"] == category) & (df["structure"] == structure)] + df = df[~df["dataset"].isin(EXCLUDED_DATASETS)] + results = [] + for name, (method, method_models, method_mode) in (methods or METHODS).items(): + method_models = (models or {}).get(name, method_models) + if name in MODE_METHODS: + method_mode = mode + rows = df[(df["method"] == method) & (df["model"].isin(method_models))] + rows = rows[rows["mode"].isna()] if method_mode is None else rows[rows["mode"] == method_mode] + rows = rows[["dataset", "domain", metric]].rename(columns={metric: "score"}) + rows["method"] = name + results.append(rows) + return pd.concat(results, ignore_index=True) + + +def load_modes(category, structure, models=None, **kwargs): + """Load the results of one modality with the methods of MODE_METHODS in every mode of MODES. + + The column 'mode' holds the mode of these methods, and an empty string for the other methods. + """ + parts = [] + for mode in MODES: + results = load_results(category, structure, mode, models, **kwargs) + is_mode_method = results["method"].isin(MODE_METHODS) + # Keep the other methods only once. + if mode != MODES[-1]: + results, is_mode_method = results[is_mode_method], is_mode_method[is_mode_method] + parts.append(results.assign(mode=np.where(is_mode_method, mode, ""))) + return pd.concat(parts, ignore_index=True) + + +def one_mode(results, modes): + """Keep one mode per method in 'modes', and clear the mode of its rows.""" + for method, mode in modes.items(): + rows = results["method"] == method + results = results[~rows | (results["mode"] == mode)].copy() + results.loc[results["method"] == method, "mode"] = "" + return results + + +def names_of(methods, starred): + """Return the plot names of the methods, such as "μSAM2" and "CPSAM", with a mark for the methods in 'starred'. + + 'starred' is a set of methods, which get a star, or a dict from method to its mark (such as "†"). + """ + names = [method.replace("micro-sam", "μSAM").replace("Cellpose", "CP") for method in methods] + marks = starred if isinstance(starred, dict) else {method: "*" for method in starred} + return [f"{name}{marks.get(method, '')}" for name, method in zip(names, methods)] + + +def tick_formatter(step, zero=False): + """Return the formatter of the value ticks. It leaves the 0 empty, unless 'zero' is set.""" + decimals = 1 if round(step * 10, 6).is_integer() else 2 + return plt.FuncFormatter(lambda value, _: "" if abs(value) < 1e-9 and not zero else f"{value:.{decimals}f}") + + +def method_positions(methods, n_bars, group_gap): + """Return the bar position of each method, top to bottom, with a gap between the method groups. + + Args: + methods: The methods. + n_bars: The number of bars of each method with more than one bar. + group_gap: The gap between the groups, in bar rows. + + Returns: + The positions. + """ + groups = np.array([0 if method in FOUNDATION_MODELS else 2 if method == "micro-sam2" else 1 for method in methods]) + extra = np.array([BAR_HEIGHT * (n_bars.get(method, 1) - 1) for method in methods]) + # The position of a method is the middle of its bars. + offsets = np.cumsum(extra) - extra / 2 + return -(np.arange(len(methods)) + group_gap * groups + offsets) + + +def modes_per_method(results): + """Return the number of modes of each method with results in more than one mode.""" + modes = {} + for result in results: + for method, mode in result[["method", "mode"]].drop_duplicates().itertuples(index=False): + if mode: + modes.setdefault(method, set()).add(mode) + return {method: len(names) for method, names in modes.items()} + + +def mean_ranks(results, lower_is_better=False): + """Return the rank of each (method, mode) per dataset, averaged over the datasets. The best rank is 1.""" + table = results.pivot_table(index="dataset", columns=["method", "mode"], values="score") + return table.rank(axis=1, ascending=lower_is_better).mean() + + +def contrast(color_a, color_b): + """The WCAG contrast ratio of two colors.""" + def luminance(color): + r, g, b = [c / 12.92 if c <= 0.03928 else ((c + 0.055) / 1.055) ** 2.4 for c in colors.to_rgb(color)] + return 0.2126 * r + 0.7152 * g + 0.0722 * b + + low, high = sorted([luminance(color_a), luminance(color_b)]) + return (high + 0.05) / (low + 0.05) + + +def label_color(bar_color): + """White or the text color for a label on a bar, whichever has the higher contrast.""" + return max(["white", TEXT_COLOR], key=lambda color: contrast(color, bar_color)) + + +def plot_domain(ax, results, domain, methods, value_max, positions, tick_step, ranks, lower_is_better=False): + """Plot one horizontal bar per method with its mean score over the datasets of one domain. + + Args: + ax: The axes. + results: The results of the modality. + domain: The domain. + methods: The methods to plot. + value_max: The end of the value axis. + positions: The bar position of each method. + tick_step: The distance between the value ticks. + ranks: Whether the bars show their mean rank. + lower_is_better: Whether a lower score ranks better, as the CREMI score. + """ + results = results[results["domain"] == domain] + values = results.groupby(["method", "mode"])["score"].mean() + rank_of = mean_ranks(results, lower_is_better) if ranks else {} + y = np.asarray(positions) + + # The width of a mode name on a sub-bar, in data units. + axes_width = ax.get_position().width * FIGURE_WIDTH + tag_width = MODE_TAG_WIDTH / 72 / axes_width * value_max + for yi, method in zip(y, methods): + # A method with more than one mode gets one sub-bar per mode. The first modes get a lighter color. + modes = [mode for mode in [""] + MODES if (method, mode) in values.index] + for k, mode in enumerate(modes): + position = yi + BAR_HEIGHT * ((len(modes) - 1) / 2 - k) + color = METHOD_COLORS[method] + if mode and mode != MODES[-1]: + tint = MODE_TINTS.get(mode, SUB_BAR_TINT) + color = colors.to_hex([1 - tint * (1 - c) for c in colors.to_rgb(color)]) + bar = ax.barh(position, values[method, mode], height=BAR_HEIGHT, color=color, linewidth=0, zorder=2)[0] + if method == "micro-sam2": + bar.set_hatch("///") + bar.set_edgecolor("white") + short = mode and values[method, mode] < tag_width + if mode and not short: + # The patch in the bar color covers the hatch under the mode name. + ax.text( + 0, position, MODE_NAMES[mode], ha="left", va="center", fontsize=SMALL_FONT_SIZE, fontweight="bold", + color=label_color(color), zorder=4, + bbox={"boxstyle": "square,pad=0.08", "facecolor": color, "linewidth": 0}, + transform=transforms.offset_copy(ax.transData, fig=ax.figure, x=2, units="points"), + ) + elif short: + # The mode name does not fit on the bar, so it goes after the end of the bar. + ax.text( + values[method, mode] + 0.01, position, MODE_NAMES[mode], ha="left", va="center", + fontsize=SMALL_FONT_SIZE, fontweight="bold", color=GROUP_COLOR, + ) + # The value of the first mode is smaller and gray, so that the value of the last mode stands out. + secondary = mode not in ("", MODES[-1]) + end = values[method, mode] + (tag_width if short else 0) + value = ax.text( + end + 0.01, position, f"{values[method, mode]:.3f}", ha="left", va="center", + fontsize=SMALL_FONT_SIZE, color=GROUP_COLOR if secondary else TEXT_COLOR, + ) + if ranks: + ax.annotate( + f"({rank_of[method, mode]:.1f})", xy=(1, 0.5), xycoords=value, xytext=(1.5, 0), + textcoords="offset points", ha="left", va="center", fontsize=SMALL_FONT_SIZE, color=GROUP_COLOR, + annotation_clip=False, + ) + + ax.set_xlim(0, value_max) + ax.set_xticks(np.arange(0, value_max + 1e-6, tick_step)) + ax.xaxis.set_major_formatter(tick_formatter(tick_step)) + ax.grid(axis="x", color="#e6e5e0", linewidth=0.5, linestyle=(0, (3, 3)), zorder=0) + ax.set_axisbelow(True) + ax.tick_params(axis="y", length=0) + ax.tick_params(axis="x", labelsize=MAX_FONT_SIZE) + ax.set_title(DOMAIN_TITLES[domain], fontsize=MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR, pad=4) + + +# The layout of the automatic plot, in inches. +AUTOMATIC_ROWS = [["label_free_cells", "fluorescence_cells"], ["fluorescence_nuclei", "histopathology_nuclei"]] +# The double column width of 183 mm. +FIGURE_WIDTH = 7.2 +NAME_WIDTH = 0.6 +RIGHT_MARGIN = 0.1 +COLUMN_GAP = 0.3 +BAR_ROW_HEIGHT = 0.2 +ROW_BOTTOM = 0.45 +DOMAIN_LABEL_WIDTH = 0.45 +DOMAIN_LINE_OFFSET = 0.27 +DOMAIN_ROW_TOP = 0.3 +DOMAIN_ROW_SPACE = 0.1 +DOMAIN_ROW_GAP_NO_VALUES = 0.08 +# The value ticks use the smallest step of TICK_STEPS that leaves at least TICK_WIDTH between two ticks. +TICK_STEPS = [0.1, 0.2, 0.25, 0.5] +TICK_WIDTH = 0.3 +# The band over the row of a method that has no model for a modality. The text has a contrast of 4.56 on the band. +NOT_APPLICABLE_COLOR = "#f7f7f7" +NOT_APPLICABLE_TEXT_COLOR = "#717171" +SCORE_LABEL = "Mean segmentation accuracy ↑" +SUB_BAR_TINT = 0.55 +# The tint of the sub-bars of single modes, in place of SUB_BAR_TINT, e.g. one tint per model size. +MODE_TINTS = {} +MODE_NAMES = {"ais": "AIS", "apg": "APG"} +# The width of a mode name on its sub-bar, in points. +MODE_TAG_WIDTH = 15 + +# micro-sam also gets a star, because it runs its histopathology model (PathoSAM) on H&E. +AUTOMATIC_STARRED_METHODS = STARRED_METHODS | {"micro-sam"} + + +def merge_orders(orders): + """Merge method lists into one list that keeps the order of each list. + + A new method goes before the next method of its list that is already in the merged list. + """ + merged = [] + for methods in orders: + for i, method in enumerate(methods): + if method in merged: + continue + after = [other for other in methods[i + 1:] if other in merged] + before = [other for other in methods[:i] if other in merged] + if after: + merged.insert(merged.index(after[0]), method) + elif before: + merged.insert(merged.index(before[-1]) + 1, method) + else: + merged.append(method) + return merged + + +def new_figure(height, region=None): + """Return a new figure with the given height in inches, or a sub-figure from 'region' (see 'plot_figure').""" + return plt.figure(figsize=(FIGURE_WIDTH, height)) if region is None else region(height) + + +def height_of(fig): + """Return the height of a figure or a sub-figure, in inches.""" + return fig.bbox.height / fig.dpi + + +# The gaps of a modality title to the panels under its line and to the line, in inches. +TITLE_LINE_GAP = 0.06 +TITLE_TEXT_GAP = 0.03 + + +def column_grid(grid, n_columns): + """Return the left edges and the width of the columns of a grid (left margin, right margin, gap), in inches.""" + left, right, gap = grid + width = (FIGURE_WIDTH - left - right - gap * (n_columns - 1)) / n_columns + return [left + k * (width + gap) for k in range(n_columns)], width + + +def draw_title(fig, left, right, y, title): + """Write a modality title over a thin line from 'left' to 'right' at the height 'y', in inches.""" + height = height_of(fig) + x0, x1, line = left / FIGURE_WIDTH, right / FIGURE_WIDTH, y / height + fig.add_artist(plt.Line2D([x0, x1], [line, line], color=GROUP_COLOR, linewidth=0.6)) + fig.text( + (x0 + x1) / 2, line + TITLE_TEXT_GAP / height, title, ha="center", va="bottom", fontsize=MAX_FONT_SIZE, + fontweight="bold", color=TEXT_COLOR, + ) + + +def plot_row( + fig, results, cells, methods, panels, starred, bottom, height, n_columns, scope, right_margin, grid, title_line, + ranks, group_gap, +): + """Plot one row of the automatic plot: one panel per cell, all with the same method rows. + + Args: + fig: The figure. + results: The results of each modality. + cells: The (column, modality, domain, title) of each panel. + methods: The methods of the row. + panels: The (title, score label, methods) of each modality. + starred: The methods with a star. + bottom: The bottom of the bars, in inches. + height: The height of the bars, in inches. + n_columns: The number of columns. + scope: The modalities that can share a value axis. + right_margin: The room right of the panels, in inches. + grid: The column grid, see 'column_grid'. The default layout is used for None. + title_line: Whether the titles stand over a thin line. + ranks: Whether the bars show their mean rank, see 'plot_domain'. + group_gap: The gap between the method groups, in bar rows. + + Returns: + The axes of each cell. + """ + label_width = NAME_WIDTH + gap = COLUMN_GAP + if grid is not None: + label_width, right_margin, gap = grid + n_bars = modes_per_method([results[name] for name in scope]) + y = method_positions(methods, n_bars, group_gap) + rows = dict(zip(methods, y)) + width = (FIGURE_WIDTH - label_width - right_margin - gap * (n_columns - 1)) / n_columns + axes = [] + for column, name, domain, title in cells: + panel_methods = panels[name][2] + shown = [method for method in methods if method in panel_methods and method in set(results[name]["method"])] + # The modalities with the same score share the value axis. + shared = pd.concat([results[n].assign(modality=n) for n in scope if panels[n][1] == panels[name][1]]) + shared = shared[shared["method"].isin(shown)] + highest = shared.groupby(["modality", "domain", "method", "mode"])["score"].mean().max() + room = VALUE_ROOM + (RANK_ROOM if ranks else 0) + value_max = np.ceil(round((highest + room) / VALUE_STEP, 6)) * VALUE_STEP + left = label_width + column * (width + gap) + ax = fig.add_axes([ + left / FIGURE_WIDTH, bottom / height_of(fig), width / FIGURE_WIDTH, height / height_of(fig), + ], sharey=axes[0] if axes else None) + tick_step = next((step for step in TICK_STEPS if width * step / value_max >= TICK_WIDTH), TICK_STEPS[-1]) + plot_domain( + ax, results[name][results[name]["method"].isin(shown)], domain, shown, value_max, + [rows[method] for method in shown], tick_step, ranks, panels[name][1].endswith("↓"), + ) + if title_line: + ax.set_title("") + if title: + draw_title(fig, left, left + width, bottom + height + TITLE_LINE_GAP, title) + else: + ax.set_title(title, fontsize=MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR, pad=4) + # The band shows that an empty row is not a score of zero. + for method in methods: + if method not in panel_methods: + ax.barh( + rows[method], value_max, height=BAR_HEIGHT * n_bars.get(method, 1), color=NOT_APPLICABLE_COLOR, + linewidth=0, zorder=0, + ) + # A narrow panel uses a smaller label, so that the label fits the band. + ax.text( + value_max / 2, rows[method], "not applicable", ha="center", va="center", + fontsize=MAX_FONT_SIZE, style="italic", color=NOT_APPLICABLE_TEXT_COLOR, + ) + if axes: + ax.tick_params(axis="y", labelleft=False) + axes.append(ax) + + axes[0].set_yticks(y) + axes[0].set_yticklabels(names_of(methods, starred), fontsize=MAX_FONT_SIZE) + half = np.array([BAR_HEIGHT * n_bars.get(method, 1) / 2 for method in methods]) + axes[0].set_ylim((y - half).min() - 0.24, (y + half).max() + 0.24) + return axes + + +def group_labels(panels): + """Group neighbouring panels, given as (score label, bottom of the bars, axes), with the same label and bottom.""" + groups = [] + for label, bottom, ax in panels: + if groups and groups[-1][:2] == (label, bottom): + groups[-1][2].append(ax) + else: + groups.append((label, bottom, [ax])) + return groups + + +def plot_automatic( + results, rows, panels, starred=None, row_height=BAR_ROW_HEIGHT, region=None, domain_offset=DOMAIN_LINE_OFFSET, + grid=None, title_line=False, ranks=False, group_gap=GROUP_GAP, row_gap=None, +): + """Plot the automatic results of all modalities in one figure. + + The in-domain panels of all modalities are the first row, and the out-of-domain panels are the second row. + + Args: + results: The results of each modality. + rows: The modalities, in rows that are joined into one row. + panels: The (title, score label, methods) of each modality. The score label ends with an arrow. + starred: The methods with a star. The default is AUTOMATIC_STARRED_METHODS. + row_height: The height of one bar row, in inches. + region: The function that returns a sub-figure, see 'new_figure'. + domain_offset: The distance from the panels to the line of the domain titles, in inches. + grid: The column grid, see 'column_grid'. + title_line: Whether the modality titles stand over a thin line. + ranks: Whether the bars show their mean rank, see 'plot_domain'. + group_gap: The gap between the method groups, in bar rows. + row_gap: The room between the rows, in inches. + + Returns: + The figure. + """ + starred = AUTOMATIC_STARRED_METHODS if starred is None else starred + modalities = [name for row in rows for name in row] + layout = [] + for i, domain in enumerate(DOMAIN_TITLES): + # Only the first row shows the modality titles. + cells = [ + (k, name, domain, panels[name][0] if i == 0 else "") for k, name in enumerate(modalities) + if domain in set(results[name]["domain"]) + ] + layout.append((domain, cells)) + methods = merge_orders([panels[name][2] for name in modalities]) + n_bars = modes_per_method([results[name] for name in modalities]) + y = method_positions(methods, n_bars, group_gap) + half = np.array([BAR_HEIGHT * n_bars.get(method, 1) / 2 for method in methods]) + height = ((y + half).max() - (y - half).min() + 0.48) * row_height + heights = [height] * len(layout) + tops = [DOMAIN_ROW_TOP] + [DOMAIN_ROW_SPACE if row_gap is None else 0] * (len(layout) - 1) + gap = DOMAIN_ROW_GAP_NO_VALUES if row_gap is None else row_gap + bottoms = [gap] * (len(layout) - 1) + [ROW_BOTTOM] + right_margin = RIGHT_MARGIN + DOMAIN_LABEL_WIDTH - DOMAIN_LINE_OFFSET + domain_offset + if grid is not None: + right_margin = grid[1] + figure_height = sum(heights) + sum(tops) + sum(bottoms) + fig = new_figure(figure_height, region) + + # The lowest panel of each column, as (score label, bottom of the bars, axes). + lowest = {} + top = figure_height + for i, ((domain, cells), room_top, room_bottom) in enumerate(zip(layout, tops, bottoms)): + bottom = top - room_top - height + axes = plot_row( + fig, results, cells, methods, panels, starred, bottom, height, len(modalities), modalities, + right_margin=right_margin, grid=grid, title_line=title_line, ranks=ranks, + group_gap=group_gap, + ) + for ax in axes: + ticks = ax.get_xticks() + ax.xaxis.set_major_formatter(tick_formatter(ticks[1] - ticks[0], zero=True)) + # Only the last row labels its value ticks. + if i < len(layout) - 1: + for ax in axes: + ax.tick_params(axis="x", labelbottom=False) + x = (FIGURE_WIDTH - right_margin + domain_offset) / FIGURE_WIDTH + low, high = (bottom + 0.05) / figure_height, (bottom + height - 0.05) / figure_height + fig.add_artist(plt.Line2D([x, x], [low, high], color=GROUP_COLOR, linewidth=0.6)) + fig.text( + x + 0.05 / FIGURE_WIDTH, (low + high) / 2, DOMAIN_TITLES[domain], rotation=270, ha="left", va="center", + fontsize=MAX_FONT_SIZE, color=TEXT_COLOR, **DOMAIN_STYLES[domain], + ) + for cell, ax in zip(cells, axes): + lowest[cell[0]] = (panels[cell[1]][1], bottom, ax) + top = bottom - room_bottom + score_labels = group_labels([lowest[column] for column in sorted(lowest)]) + + for label, bottom, label_axes in score_labels: + left, right = label_axes[0].get_position().x0, label_axes[-1].get_position().x1 + text, arrow = label[:-2], label[-1] + score = fig.text( + (left + right) / 2, (bottom - 0.24) / figure_height, text, ha="center", va="top", fontsize=MAX_FONT_SIZE, + fontweight="bold", color=TEXT_COLOR, + ) + label_axes[0].annotate( + arrow, xy=(1, 0.5), xycoords=score, xytext=(1.5, 0), textcoords="offset points", ha="left", + va="center", fontsize=MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR, + annotation_clip=False, + ) + return fig + + +# The 'method', 'model' and optional 'mode' values of each interactive method in the results. Every method feeds its +# previous mask back with the correction clicks: SAM2, SAM3 and micro-sam2 always do, SAM and micro-sam use their +# 'with_masks' runs. +INTERACTIVE_METHODS = { + "SAM": ("sam", ("vit_l",), "with_masks"), + "SAM2": ("sam2", ("hvit_l",)), + "SAM3": ("sam3", ("default",)), + "micro-sam": ("micro-sam", ("vit_l_lm", "vit_l_histopathology"), "with_masks"), + "micro-sam2": ("micro-sam2", ("hvit_l",)), +} +PROMPT_NAMES = {"box": "Box", "point": "Point"} + + +def load_interactive(category, structure, datasets, path=INTERACTIVE_CSV, methods=None, metric="msa"): + """Load the scores of the interactive methods for one modality, per dataset, first prompt and iteration. + + Args: + category: The category of the modality in the results. + structure: The structure of the modality in the results. + datasets: The datasets of each domain. + path: The path to the results table. + methods: The method table. The default is INTERACTIVE_METHODS. + metric: The score column. + + Returns: + The results in long format. + """ + df = pd.read_csv(path) + df = df[(df["category"] == category) & (df["structure"] == structure)] + results = [] + for name, (method, models, *mode) in (methods or INTERACTIVE_METHODS).items(): + rows = df[(df["method"] == method) & (df["model"].isin(models))] + if mode: + # The table stores an empty mode as a missing value. + rows = rows[rows["mode"].fillna("") == mode[0]] + rows = rows[["dataset", "prompt", "iteration", metric]].rename(columns={metric: "score"}) + rows["method"] = name + results.append(rows) + return select_datasets(pd.concat(results, ignore_index=True), datasets) + + +# The layout of the interactive plot, in inches. The box runs are shifted by BOX_OFFSET iterations. +PANEL_HEIGHT = 1.5 +LEGEND_HEIGHT = 0.08 +LEGEND_BORDER_COLOR = "#cfcdc6" +# The first prompt and the first 4 corrections. +N_INTERACTIVE_ITERATIONS = 5 +BOX_OFFSET = N_INTERACTIVE_ITERATIONS + 1 +BOX_BAND_COLOR = "#f4f3ef" +AXIS_PRECISION = 0.05 +INTERACTIVE_TICK_STEPS = [0.05, 0.1, 0.2, 0.25, 0.5] +MAX_TICKS = 8 +INTERACTIVE_LABEL_WIDTH = 0.5 +INTERACTIVE_SCORE_X = 0.12 +INTERACTIVE_DOMAIN_WIDTH = 0.3 +INTERACTIVE_DOMAIN_OFFSET = 0.1 +# The share of a panel under its lowest point, for the panels in 'room_below'. +ROOM_BELOW = 0.12 +X_LABEL_OFFSET = 0.15 # the iteration label under the panels, in inches +ITERATION_LABEL = "Correction iteration" +OWN_LABEL_OFFSET = 0.27 # the own score label of a modality, left of its panels, in inches + + +def first_tick(low, step): + """Return the first value tick of an axis from 'low': a multiple of 'step', and not below 0.""" + return max(np.ceil(round(low / step, 6)) * step, 0.0) + + +def add_score_arrow(ax, label, arrow, fontsize=MAX_FONT_SIZE): + """Add the arrow over a vertical score label. The arrow stays upright, so that it points along the axis.""" + ax.annotate( + arrow, xy=(0.5, 1), xycoords=label, xytext=(0, 1.5), textcoords="offset points", ha="center", va="bottom", + fontsize=fontsize, fontweight="bold", color=TEXT_COLOR, annotation_clip=False, + ) + + +def plot_interactive_by_domain( + results, modalities, titles, starred=None, methods=None, score_labels=None, placeholder=False, + panel_height=PANEL_HEIGHT, region=None, legend_panel=None, grid=None, title_line=False, legend_left=False, + room_below=(), shared_rows=False, show_titles=True, score_gap=None, row_gap=None, band_color=BOX_BAND_COLOR, + legend_wide=False, prompts_on_top=False, x_label=None, +): + """Plot the mean score over the correction iterations of all modalities in one figure. + + The in-domain panels are the first row, and the out-of-domain panels are the second row. Each panel shows the + point runs and the box runs side by side. The value axis of a panel spans its curves. + + Args: + results: The interactive results of each modality. + modalities: The modalities, one column each. + titles: The title of each modality. + starred: The methods with a star. The default is AUTOMATIC_STARRED_METHODS. + methods: The methods, in their order. The default is INTERACTIVE_METHODS. + score_labels: The score label of each modality. The default is SCORE_LABEL. + placeholder: Whether all panels stay empty, with the note "Results pending". + panel_height: The height of a panel, in inches. + region: The function that returns a sub-figure, see 'new_figure'. + legend_panel: The (domain, modality) of the panel with the legend. + grid: The column grid, see 'column_grid'. + title_line: Whether the modality titles stand over a thin line. + legend_left: Whether the legend stands in the upper left corner, with the value axis raised for it. + legend_wide: Whether the legend spans the top of its panel, with the value axis raised for it. + prompts_on_top: Whether the prompt names stand once, over the halves of the first row, clear of the curves. + x_label: The label of the iteration axis, once under all panels, as the score label of the automatic plot. + room_below: The (domain, modality) of the panels with a lowered value axis. + shared_rows: Whether the panels of a row with the shared score label share one value range. + show_titles: Whether the panels show the modality titles. + score_gap: The distance from the shared score label to the panels, in inches. + row_gap: The room between the rows, in inches. + band_color: The color of the box half of a panel. + + Returns: + The figure. + """ + starred = AUTOMATIC_STARRED_METHODS if starred is None else starred + score_labels = {name: (score_labels or {}).get(name, SCORE_LABEL) for name in modalities} + shared_label = max(set(score_labels.values()), key=list(score_labels.values()).count) + methods = list(methods or INTERACTIVE_METHODS) + averages = { + name: results[name][results[name]["iteration"] < N_INTERACTIVE_ITERATIONS].groupby( + ["domain", "prompt", "method", "iteration"] + )["score"].mean() + for name in modalities + } + domains = list(DOMAIN_TITLES) + offsets = {"point": 0, "box": BOX_OFFSET} + n_iterations = N_INTERACTIVE_ITERATIONS + n_columns = len(modalities) + grid = grid or (INTERACTIVE_LABEL_WIDTH, RIGHT_MARGIN + INTERACTIVE_DOMAIN_WIDTH, COLUMN_GAP) + right_margin = grid[1] + lefts, width = column_grid(grid, n_columns) + tops = [DOMAIN_ROW_TOP if show_titles else DOMAIN_ROW_SPACE] + tops += [DOMAIN_ROW_SPACE if row_gap is None else 0] * (len(domains) - 1) + gap = DOMAIN_ROW_GAP_NO_VALUES if row_gap is None else row_gap + bottoms = [gap] * (len(domains) - 1) + [ROW_BOTTOM] + + def curves_of(name, domain): + return [ + averages[name][domain, prompt, method] for prompt in offsets for method in methods + if (domain, prompt, method) in averages[name].index.droplevel("iteration") + ] + + # The value range of each row over the panels with the shared score label, for 'shared_rows'. + row_ranges = {} + for domain in domains: + curves = [ + curve for name in modalities if score_labels[name] == shared_label and not placeholder + for curve in curves_of(name, domain) + ] + if curves: + row_ranges[domain] = (min(c.min() for c in curves), max(c.max() for c in curves)) + height = LEGEND_HEIGHT + len(domains) * panel_height + sum(tops) + sum(bottoms) + fig = new_figure(height, region) + + top = height - LEGEND_HEIGHT + titled = set() + own_label_axes = {} + for domain, room_top, room_bottom in zip(domains, tops, bottoms): + bottom = top - room_top - panel_height + shared_axes = [] + for column, name in enumerate(modalities): + pending = placeholder or domain not in set(results[name]["domain"]) + left = lefts[column] + ax = fig.add_axes([left / FIGURE_WIDTH, bottom / height, width / FIGURE_WIDTH, panel_height / height]) + last = BOX_OFFSET + n_iterations - 1 + ax.axvspan(BOX_OFFSET - 0.6, last + 0.6, color=band_color, linewidth=0, zorder=0) + average = averages[name] + for method in [] if pending else methods: + ours = method == "micro-sam2" + for prompt, offset in offsets.items(): + if (domain, prompt, method) not in average.index.droplevel("iteration"): + continue + curve = average[domain, prompt, method] + zorder = 3 if ours else 2 + ax.plot( + curve.index + offset, curve.values, color=METHOD_COLORS[method], linewidth=1.0, marker="o", + markersize=2.2, markeredgecolor="white", markeredgewidth=0.4, zorder=zorder, + ) + # The first prompt is an open ring. + ax.plot( + curve.index[:1] + offset, curve.values[:1], linestyle="none", marker="o", markersize=2.8, + markerfacecolor="white", markeredgecolor=METHOD_COLORS[method], markeredgewidth=0.7, + zorder=zorder + 0.5, + ) + # Each half labels the first prompt 0 in gray, and an arrow spans the corrections. + ticks = [offset + i for offset in offsets.values() for i in range(n_iterations)] + ax.set_xticks(ticks) + ax.set_xticklabels([]) + ax.tick_params(axis="x", length=1.5) + ax.tick_params(axis="both", labelsize=MAX_FONT_SIZE) + transform = ax.get_xaxis_transform() + # Only the last row labels its iterations. + labelled = domain == domains[-1] + for prompt, offset in offsets.items(): + for iteration in (0, 1, n_iterations - 1) if labelled else (): + ax.text( + offset + iteration, -0.065, str(iteration), ha="center", va="center", fontsize=MAX_FONT_SIZE, + color=GROUP_COLOR if iteration == 0 else TEXT_COLOR, transform=transform, + ) + if labelled: + ax.annotate( + "", xy=(offset + n_iterations - 1.55, -0.058), xytext=(offset + 1.45, -0.058), + xycoords=transform, textcoords=transform, annotation_clip=False, + arrowprops={ + "arrowstyle": "-|>,head_length=0.35,head_width=0.15", "color": GROUP_COLOR, + "linewidth": 0.6, "shrinkA": 0, "shrinkB": 0, + }, + ) + if not prompts_on_top: + ax.text( + offset + (n_iterations - 1) / 2, 0.015, PROMPT_NAMES[prompt], ha="center", va="bottom", + fontsize=MAX_FONT_SIZE, style="italic", color=GROUP_COLOR, transform=transform, zorder=4, + ) + elif domain == domains[0]: + ax.text( + offset + (n_iterations - 1) / 2, 1.02, PROMPT_NAMES[prompt], ha="center", va="bottom", + fontsize=MAX_FONT_SIZE, style="italic", color=GROUP_COLOR, transform=transform, + ) + ax.set_xlim(-0.6, last + 0.6) + if pending: + ax.set_yticks([]) + ax.text( + 0.5, 0.55, "Results pending", ha="center", va="center", fontsize=MAX_FONT_SIZE, style="italic", + color=GROUP_COLOR, transform=ax.transAxes, + ) + shown = [] if pending else curves_of(name, domain) + lowest, highest = (0.0, 1.0) if pending else (min(c.min() for c in shown), max(c.max() for c in shown)) + in_shared_row = shared_rows and score_labels[name] == shared_label and domain in row_ranges + if in_shared_row: + lowest, highest = row_ranges[domain] + # The axis ends are the nearest multiples of AXIS_PRECISION around the curves. + low = np.floor(round(lowest / AXIS_PRECISION, 6)) * AXIS_PRECISION + if (domain, name) in room_below and not pending: + # Keep the lowest point over the 'Point' and 'Box' labels. + axis_top = np.ceil(round((highest + 0.01) / AXIS_PRECISION, 6)) * AXIS_PRECISION + low = (lowest - ROOM_BELOW * axis_top) / (1 - ROOM_BELOW) + low = np.floor(round(low / AXIS_PRECISION, 6)) * AXIS_PRECISION + high = np.ceil(round((highest + 0.01) / AXIS_PRECISION, 6)) * AXIS_PRECISION + step = next(s for s in INTERACTIVE_TICK_STEPS if (high - low) / s <= MAX_TICKS) + ax.set_ylim(low, high) + if not pending: + ax.set_yticks(np.arange(first_tick(low, step), high + 1e-6, step)) + ax.yaxis.set_major_formatter(tick_formatter(step)) + ax.grid(axis="y", color="#e6e5e0", linewidth=0.5, linestyle=(0, (3, 3)), zorder=0) + ax.set_axisbelow(True) + if in_shared_row: + shared_axes.append((ax, step)) + if in_shared_row and column > 0: + ax.tick_params(axis="y", labelleft=False) + if name not in titled and show_titles: + if title_line: + draw_title(fig, left, left + width, bottom + panel_height + TITLE_LINE_GAP, titles[name]) + else: + # With the prompt names over the first row, the title stands above them. + pad = 12 if prompts_on_top else 4 + ax.set_title(titles[name], fontsize=MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR, pad=pad) + titled.add(name) + if score_labels[name] != shared_label: + own_label_axes.setdefault(name, []).append(ax) + if legend_panel == (domain, name): + legend = add_panel_legend(ax, methods, starred, left=legend_left, wide=legend_wide) + if (legend_left or legend_wide) and not pending: + make_room_above(ax, legend, highest, step) + # A panel raised for the legend raises the other panels of its shared row alike. + if shared_axes: + axis_low = min(ax.get_ylim()[0] for ax, _ in shared_axes) + axis_top = max(ax.get_ylim()[1] for ax, _ in shared_axes) + for ax, step in shared_axes: + ax.set_ylim(axis_low, axis_top) + ax.set_yticks(np.arange(first_tick(axis_low, step), axis_top - 0.5 * step + 1e-6, step)) + x = (FIGURE_WIDTH - right_margin + INTERACTIVE_DOMAIN_OFFSET) / FIGURE_WIDTH + low, high = (bottom + 0.05) / height, (bottom + panel_height - 0.05) / height + fig.add_artist(plt.Line2D([x, x], [low, high], color=GROUP_COLOR, linewidth=0.6)) + fig.text( + x + 0.05 / FIGURE_WIDTH, (low + high) / 2, DOMAIN_TITLES[domain], rotation=270, ha="left", va="center", + fontsize=MAX_FONT_SIZE, color=TEXT_COLOR, **DOMAIN_STYLES[domain], + ) + top = bottom - room_bottom + # A modality with its own score label gets it once, left of its column, centered on its panels. + for name, axes in own_label_axes.items(): + upper, lower = axes[0].get_position(), axes[-1].get_position() + ax = axes[-1] + ax.set_ylabel(score_labels[name][:-2], fontsize=MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR) + ax.yaxis.set_label_coords( + -OWN_LABEL_OFFSET / (lower.width * FIGURE_WIDTH), ((upper.y1 + lower.y0) / 2 - lower.y0) / lower.height + ) + add_score_arrow(ax, ax.yaxis.label, score_labels[name][-1], fontsize=MAX_FONT_SIZE) + middle = 0.5 * (height - LEGEND_HEIGHT) / height + score_x = INTERACTIVE_SCORE_X if score_gap is None else grid[0] - score_gap + score = fig.text( + score_x / FIGURE_WIDTH, middle, shared_label[:-2], rotation=90, ha="left", va="center", fontsize=MAX_FONT_SIZE, + fontweight="bold", color=TEXT_COLOR, + ) + add_score_arrow(fig.axes[0], score, shared_label[-1]) + if x_label is not None: + fig.text( + (lefts[0] + lefts[-1] + width) / 2 / FIGURE_WIDTH, (bottom - X_LABEL_OFFSET) / height, x_label, + ha="center", va="top", fontsize=MAX_FONT_SIZE, fontweight="bold", color=TEXT_COLOR, + ) + return fig + + +def add_panel_legend(ax, methods, starred, left=False, wide=False): + """Add the legend of the interactive plot to a panel, in two columns. + + Args: + ax: The axes of the panel. + methods: The methods. + starred: The methods with a star. + left: Whether the legend stands in the upper left corner. By default, it stands in the lower right corner. + wide: Whether the legend spans the top of the panel in three columns. + + Returns: + The legend. + """ + names = dict(zip(methods, names_of(methods, starred))) + handles = [ + plt.Line2D( + [], [], color=METHOD_COLORS[method], marker="o", markersize=2.2, linewidth=1.0, markeredgecolor="white", + markeredgewidth=0.4, + ) + for method in methods + ] + if wide: + place = dict(loc="upper left", bbox_to_anchor=(0.01, 0.985, 0.98, 0), mode="expand", ncol=3) + elif left: + place = dict(loc="upper left", bbox_to_anchor=(0.01, 0.985), ncol=2) + else: + place = dict(loc="lower right", bbox_to_anchor=(0.995, 0.09), ncol=2) + legend = ax.legend( + handles, [names[method] for method in methods], **place, + fontsize=MAX_FONT_SIZE, handlelength=1.1, handletextpad=0.3, columnspacing=0.5, labelspacing=0.15, + borderpad=0.25, + frameon=True, framealpha=1, facecolor="white", edgecolor=LEGEND_BORDER_COLOR, + ) + legend.get_frame().set_linewidth(0.6) + legend.set_zorder(5) + return legend + + +def make_room_above(ax, legend, highest, step): + """Raise the top of the value axis until the highest point 'highest' stays under the legend. + + The function also adds the value ticks of 'step' up to the new top. + """ + fig = ax.figure + fig.canvas.draw() + box = legend.get_window_extent().transformed(ax.transAxes.inverted()) + low, high = ax.get_ylim() + # Leave a small gap under the legend. + share = box.y0 - 0.03 + top = low + (highest - low) / share + if top > high: + top = np.ceil(round(top / AXIS_PRECISION, 6)) * AXIS_PRECISION + ax.set_ylim(low, top) + ax.set_yticks(np.arange(first_tick(low, step), top - 0.5 * step + 1e-6, step)) + + +def save_figure(fig, name, kind, style="quantitative", dpi=None): + """Save the figure as png, svg and pdf in the folder '/