From 542ee8d9588ad459522b9a9aed1f8b654f28c48b Mon Sep 17 00:00:00 2001 From: Paul Scotti Date: Sat, 24 Oct 2026 15:14:41 +0000 Subject: [PATCH 1/2] Add H-optimus-0-SPECTRA encoder --- README.md | 3 +- src/thunder/benchmark.py | 2 +- .../pretrained_model/hoptimus0_spectra.yaml | 9 ++ src/thunder/models/download.py | 11 ++- src/thunder/models/pretrained_models.py | 33 +++++-- src/thunder/utils/constants.py | 1 + tests/test_hoptimus0_spectra.py | 88 +++++++++++++++++++ 7 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 src/thunder/config/pretrained_model/hoptimus0_spectra.yaml create mode 100644 tests/test_hoptimus0_spectra.py diff --git a/README.md b/README.md index 588f3ae..d6b1b4d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ To learn more about how to use `thunder`, please visit our [documentation](https An API and command line interface (CLI) are provided to allow users to download datasets, models, and run benchmarks. The API is designed to be user-friendly and allows for easy integration into existing workflows. The CLI provides a convenient way to access the same functionality from the command line. > [!IMPORTANT] -> **Downloading supported foundation models**: you will have to visit the Huggingface URL of supported models you wish to use in order to accept usage conditions. +> **Downloading supported foundation models**: some models require accepting their usage conditions on Hugging Face before downloading.
List of Huggingface URLs @@ -65,6 +65,7 @@ An API and command line interface (CLI) are provided to allow users to download * Virchow2: https://huggingface.co/paige-ai/Virchow2 * H-optimus-0: https://huggingface.co/bioptimus/H-optimus-0 * H-optimus-1: https://huggingface.co/bioptimus/H-optimus-1 +* H-optimus-0-SPECTRA: https://huggingface.co/medarc/H-optimus-0-SPECTRA * GenBio-PathFM: https://huggingface.co/genbio-ai/genbio-pathfm * CONCH: https://huggingface.co/MahmoodLab/CONCH * TITAN/CONCHv1.5: https://huggingface.co/MahmoodLab/TITAN diff --git a/src/thunder/benchmark.py b/src/thunder/benchmark.py index 01f0e58..f01c1cd 100644 --- a/src/thunder/benchmark.py +++ b/src/thunder/benchmark.py @@ -21,7 +21,7 @@ def benchmark( where options are: - dataset: *bach*, *bracs*, *break_his*, *ccrcc*, *crc*, *esca*, *mhist*, *ocelot*, *pannuke*, *patch_camelyon*, *segpath_epithelial*, *segpath_lymphocytes*, *tcga_crc_msi*, *tcga_tils*, *tcga_uniform*, *wilds* - - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* + - model: *hiboub*, *hiboul*, *hoptimus0*, *hoptimus1*, *genbio-pathfm*, *hoptimus0_spectra*, *midnight*, *phikon*, *phikon2*, *uni*, *uni2h*, *virchow*, *virchow2*, *conch*, *titan*, *keep*, *musk*, *plip*, *quiltnetb32*, *dinov2base*, *dinov2large*, *vitbasepatch16224in21k*, *vitlargepatch16224in21k*, *clipvitbasepatch32*, *clipvitlargepatch14* - task: *adversarial_attack*, *alignment_scoring*, *image_retrieval*, *knn*, *linear_probing*, *pre_computing_embeddings*, *segmentation*, *simple_shot*, *transformation_invariance*, *zero_shot_vlm* - loading_mode: *online_loading*, *image_pre_loading*, *embedding_pre_loading* diff --git a/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml b/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml new file mode 100644 index 0000000..7839b6d --- /dev/null +++ b/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml @@ -0,0 +1,9 @@ +model_name: hoptimus0_spectra +type: timm +vlm: false +emb_dim: 1536 +emb_dim_seg: 1536 +ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/hoptimus0spectra/model.safetensors +hf_tag: hf-hub:medarc/H-optimus-0-SPECTRA + +timm_kwargs: {} diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index db7e381..9c7b53e 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -7,8 +7,8 @@ ) # Models -# Importantly, you will have to visit the Huggingface URL of all models to accept -# usage conditions to be able to download their associated checkpoints. +# Some models require accepting their usage conditions on Hugging Face +# before downloading their checkpoints. TAGS_FILENAMES = { "uni": ( "MahmoodLab/uni", @@ -26,6 +26,10 @@ "paige-ai/Virchow2", "pytorch_model.bin", ), # Virchow2 (https://huggingface.co/paige-ai/Virchow2) + "hoptimus0_spectra": ( + "medarc/H-optimus-0-SPECTRA", + "model.safetensors", + ), # H-optimus-0-SPECTRA (https://huggingface.co/medarc/H-optimus-0-SPECTRA) "hoptimus0": ( "bioptimus/H-optimus-0", "pytorch_model.bin", @@ -158,6 +162,7 @@ def download_models(models: Union[List[str], str]) -> None: * virchow * virchow2 * hoptimus0 + * hoptimus0_spectra * h0mini * hoptimus1 * genbio-pathfm @@ -262,7 +267,7 @@ def download_model(model: str) -> None: ] if local_dir_tag != "plip": extra_files.extend(["merges.txt", "vocab.json"]) - elif local_dir_tag == "midnight": + elif local_dir_tag in ["midnight", "hoptimus0spectra"]: extra_files = ["config.json"] elif local_dir_tag == "titan": extra_files = [ diff --git a/src/thunder/models/pretrained_models.py b/src/thunder/models/pretrained_models.py index f39af21..44b02d0 100644 --- a/src/thunder/models/pretrained_models.py +++ b/src/thunder/models/pretrained_models.py @@ -1,9 +1,11 @@ +import json from abc import ABC, abstractmethod from pathlib import Path import timm import torch from omegaconf import DictConfig, OmegaConf +from safetensors.torch import load_file from timm.data import resolve_data_config from timm.data.transforms_factory import create_transform from transformers import AutoImageProcessor, AutoModel, AutoTokenizer @@ -375,7 +377,11 @@ def extract_embedding(src, pretrained_model, task_type="linear_probing"): if model_cfg.model_name == "uni2h": nb_reg = 8 - elif model_cfg.model_name in ["hoptimus0", "hoptimus1"]: + elif model_cfg.model_name in [ + "hoptimus0", + "hoptimus1", + "hoptimus0_spectra", + ]: nb_reg = 4 else: nb_reg = 0 @@ -396,6 +402,7 @@ def get_model_from_name(model_name: str, device: str): * virchow * virchow2 * hoptimus0 + * hoptimus0_spectra * h0mini * hoptimus1 * genbio-pathfm @@ -470,16 +477,28 @@ def get_from_timm(hf_tag: str, timm_kwargs: dict, ckpt_path: str, device: str): :param hf_tag: Huggingface model tag. :param timm_kargs: dictionary of timm arguments. - :ckpt_path: path to the stored checkpoint. + :param ckpt_path: path to the stored checkpoint. Timm safetensors require + an adjacent config.json. :param device: device to use (cpu, cuda). """ # Model - model = timm.create_model(hf_tag, pretrained=False, **timm_kwargs) - model.load_state_dict( - torch.load(ckpt_path, weights_only=True, map_location=torch.device(device)), - strict=True, - ) + if Path(ckpt_path).suffix == ".safetensors": + # Read the downloaded timm configuration so loading also works offline. + config = json.loads(Path(ckpt_path).with_name("config.json").read_text()) + model = timm.create_model( + config["architecture"], + pretrained=False, + pretrained_cfg=config["pretrained_cfg"], + **(config["model_args"] | timm_kwargs), + ) + model.load_state_dict(load_file(ckpt_path, device=device), strict=True) + else: + model = timm.create_model(hf_tag, pretrained=False, **timm_kwargs) + model.load_state_dict( + torch.load(ckpt_path, weights_only=True, map_location=torch.device(device)), + strict=True, + ) # Transform transform = create_transform( diff --git a/src/thunder/utils/constants.py b/src/thunder/utils/constants.py index 5a71411..e5c3764 100644 --- a/src/thunder/utils/constants.py +++ b/src/thunder/utils/constants.py @@ -7,6 +7,7 @@ class ModelConstants(Enum): "hiboub", "hiboul", "hoptimus0", + "hoptimus0_spectra", "h0mini", "hoptimus1", "genbio-pathfm", diff --git a/tests/test_hoptimus0_spectra.py b/tests/test_hoptimus0_spectra.py new file mode 100644 index 0000000..39bbd67 --- /dev/null +++ b/tests/test_hoptimus0_spectra.py @@ -0,0 +1,88 @@ +import json +from pathlib import Path +from unittest.mock import patch + +import torch +from omegaconf import OmegaConf +from PIL import Image + +from thunder.models import pretrained_models +from thunder.models.download import download_models + + +def test_hoptimus0_spectra_download(tmp_path, monkeypatch): + monkeypatch.setenv("THUNDER_BASE_DATA_FOLDER", str(tmp_path)) + with patch("huggingface_hub.hf_hub_download") as download: + download_models("hoptimus0_spectra") + assert [ + (call.args[0], call.kwargs["filename"]) for call in download.call_args_list + ] == [ + ("medarc/H-optimus-0-SPECTRA", "model.safetensors"), + ("medarc/H-optimus-0-SPECTRA", "config.json"), + ] + assert all( + Path(call.kwargs["local_dir"]) == tmp_path / "pretrained_ckpts/hoptimus0spectra" + for call in download.call_args_list + ) + + +def test_hoptimus0_spectra_features(tmp_path, monkeypatch): + monkeypatch.setenv("THUNDER_BASE_DATA_FOLDER", str(tmp_path)) + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + cfg = OmegaConf.load( + Path(pretrained_models.__file__).parents[1] + / "config/pretrained_model/hoptimus0_spectra.yaml" + ) + import timm + from safetensors.torch import save_file + + args = dict( + img_size=224, + patch_size=14, + embed_dim=32, + depth=1, + num_heads=4, + num_classes=0, + reg_tokens=4, + global_pool="token", + ) + cfg.timm_kwargs = {} + reference = timm.create_model( + "vit_tiny_patch16_224", pretrained=False, **args + ).eval() + save_file(reference.state_dict(), tmp_path / "model.safetensors") + pretrained_cfg = dict( + input_size=[3, 224, 224], + mean=[0.5] * 3, + std=[0.5] * 3, + interpolation="bicubic", + crop_pct=1.0, + num_classes=0, + ) + (tmp_path / "config.json").write_text( + json.dumps( + { + "architecture": "vit_tiny_patch16_224", + "pretrained_cfg": pretrained_cfg, + "model_args": { + k: v for k, v in args.items() if k not in ["mlp_layer", "act_layer"] + }, + } + ) + ) + cfg.ckpt_path = str(tmp_path / "model.safetensors") + model, transform, extract = pretrained_models.get_model(cfg, "cpu") + model.eval().requires_grad_(False) + x = transform(Image.new("RGB", (257, 231), color=(127, 91, 164))).unsqueeze(0) + assert x.shape == (1, 3, 224, 224) + with torch.no_grad(): + tokens = reference.forward_features(x) + patches = tokens[:, 5:] + pooled = tokens[:, 0] + torch.testing.assert_close(extract(x, model, "linear_probing"), pooled) + dense = extract(x, model, "segmentation") + assert dense.shape == (1, 256, 32) + torch.testing.assert_close(dense, patches) + x.requires_grad_() + extract(x, model, "linear_probing")[0, 0].backward() + assert torch.isfinite(x.grad).all() and x.grad.abs().sum() > 0 From f507072926a8f1b6bc0e9c84afc72d6b8bfcc3b8 Mon Sep 17 00:00:00 2001 From: Paul Scotti Date: Sat, 24 Oct 2026 16:31:48 +0000 Subject: [PATCH 2/2] Keep submission focused on model integration --- README.md | 2 +- .../pretrained_model/hoptimus0_spectra.yaml | 1 - src/thunder/models/download.py | 4 +- tests/test_hoptimus0_spectra.py | 88 ------------------- 4 files changed, 3 insertions(+), 92 deletions(-) delete mode 100644 tests/test_hoptimus0_spectra.py diff --git a/README.md b/README.md index d6b1b4d..9aacc00 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ To learn more about how to use `thunder`, please visit our [documentation](https An API and command line interface (CLI) are provided to allow users to download datasets, models, and run benchmarks. The API is designed to be user-friendly and allows for easy integration into existing workflows. The CLI provides a convenient way to access the same functionality from the command line. > [!IMPORTANT] -> **Downloading supported foundation models**: some models require accepting their usage conditions on Hugging Face before downloading. +> **Downloading supported foundation models**: you will have to visit the Huggingface URL of supported models you wish to use in order to accept usage conditions.
List of Huggingface URLs diff --git a/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml b/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml index 7839b6d..c6581e1 100644 --- a/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml +++ b/src/thunder/config/pretrained_model/hoptimus0_spectra.yaml @@ -2,7 +2,6 @@ model_name: hoptimus0_spectra type: timm vlm: false emb_dim: 1536 -emb_dim_seg: 1536 ckpt_path: ${oc.env:THUNDER_BASE_DATA_FOLDER}/pretrained_ckpts/hoptimus0spectra/model.safetensors hf_tag: hf-hub:medarc/H-optimus-0-SPECTRA diff --git a/src/thunder/models/download.py b/src/thunder/models/download.py index 9c7b53e..0851413 100644 --- a/src/thunder/models/download.py +++ b/src/thunder/models/download.py @@ -7,8 +7,8 @@ ) # Models -# Some models require accepting their usage conditions on Hugging Face -# before downloading their checkpoints. +# Importantly, you will have to visit the Huggingface URL of all models to accept +# usage conditions to be able to download their associated checkpoints. TAGS_FILENAMES = { "uni": ( "MahmoodLab/uni", diff --git a/tests/test_hoptimus0_spectra.py b/tests/test_hoptimus0_spectra.py deleted file mode 100644 index 39bbd67..0000000 --- a/tests/test_hoptimus0_spectra.py +++ /dev/null @@ -1,88 +0,0 @@ -import json -from pathlib import Path -from unittest.mock import patch - -import torch -from omegaconf import OmegaConf -from PIL import Image - -from thunder.models import pretrained_models -from thunder.models.download import download_models - - -def test_hoptimus0_spectra_download(tmp_path, monkeypatch): - monkeypatch.setenv("THUNDER_BASE_DATA_FOLDER", str(tmp_path)) - with patch("huggingface_hub.hf_hub_download") as download: - download_models("hoptimus0_spectra") - assert [ - (call.args[0], call.kwargs["filename"]) for call in download.call_args_list - ] == [ - ("medarc/H-optimus-0-SPECTRA", "model.safetensors"), - ("medarc/H-optimus-0-SPECTRA", "config.json"), - ] - assert all( - Path(call.kwargs["local_dir"]) == tmp_path / "pretrained_ckpts/hoptimus0spectra" - for call in download.call_args_list - ) - - -def test_hoptimus0_spectra_features(tmp_path, monkeypatch): - monkeypatch.setenv("THUNDER_BASE_DATA_FOLDER", str(tmp_path)) - monkeypatch.setenv("HF_HUB_OFFLINE", "1") - cfg = OmegaConf.load( - Path(pretrained_models.__file__).parents[1] - / "config/pretrained_model/hoptimus0_spectra.yaml" - ) - import timm - from safetensors.torch import save_file - - args = dict( - img_size=224, - patch_size=14, - embed_dim=32, - depth=1, - num_heads=4, - num_classes=0, - reg_tokens=4, - global_pool="token", - ) - cfg.timm_kwargs = {} - reference = timm.create_model( - "vit_tiny_patch16_224", pretrained=False, **args - ).eval() - save_file(reference.state_dict(), tmp_path / "model.safetensors") - pretrained_cfg = dict( - input_size=[3, 224, 224], - mean=[0.5] * 3, - std=[0.5] * 3, - interpolation="bicubic", - crop_pct=1.0, - num_classes=0, - ) - (tmp_path / "config.json").write_text( - json.dumps( - { - "architecture": "vit_tiny_patch16_224", - "pretrained_cfg": pretrained_cfg, - "model_args": { - k: v for k, v in args.items() if k not in ["mlp_layer", "act_layer"] - }, - } - ) - ) - cfg.ckpt_path = str(tmp_path / "model.safetensors") - model, transform, extract = pretrained_models.get_model(cfg, "cpu") - model.eval().requires_grad_(False) - x = transform(Image.new("RGB", (257, 231), color=(127, 91, 164))).unsqueeze(0) - assert x.shape == (1, 3, 224, 224) - with torch.no_grad(): - tokens = reference.forward_features(x) - patches = tokens[:, 5:] - pooled = tokens[:, 0] - torch.testing.assert_close(extract(x, model, "linear_probing"), pooled) - dense = extract(x, model, "segmentation") - assert dense.shape == (1, 256, 32) - torch.testing.assert_close(dense, patches) - x.requires_grad_() - extract(x, model, "linear_probing")[0, 0].backward() - assert torch.isfinite(x.grad).all() and x.grad.abs().sum() > 0