From d180b9e9e8b900c2ee3c2ad98d8ffba96aa04cb6 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 01:21:15 +0200 Subject: [PATCH 01/15] refactor(misc): make_app_config returns a real struct-mode DictConfig (refs #352) --- src/raitap/testing/factories.py | 23 +++++++++++----- src/raitap/testing/tests/test_factories.py | 31 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/src/raitap/testing/factories.py b/src/raitap/testing/factories.py index bf4b2b7c..b63d5d06 100644 --- a/src/raitap/testing/factories.py +++ b/src/raitap/testing/factories.py @@ -6,11 +6,13 @@ from __future__ import annotations -from types import SimpleNamespace -from typing import Any +from typing import Any, cast import torch import torch.nn as nn +from omegaconf import DictConfig, OmegaConf + +from raitap.configs.schema import AppConfig def make_tiny_classifier( @@ -66,8 +68,15 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return model -def make_app_config(**overrides: Any) -> SimpleNamespace: - """Minimal AppConfig stand-in. Pass keyword overrides for any attribute.""" - base: dict[str, Any] = {"experiment_name": "test", "hardware": "cpu", "_output_root": "."} - base.update(overrides) - return SimpleNamespace(**base) +def make_app_config(**overrides: Any) -> DictConfig: + """A faithful ``AppConfig`` stand-in for tests. + + Returns a struct-mode OmegaConf config typed by ``AppConfig`` — the same + shape Hydra hands prod at runtime. Attribute access resolves declared + fields (with schema defaults) and raises on undeclared ones, so tests + model valid configs and prod can read fields directly without defensive + ``getattr``. Pass flat or nested keyword overrides + (``make_app_config(model={"source": "resnet18"})``). + """ + merged = OmegaConf.merge(OmegaConf.structured(AppConfig), overrides) + return cast("DictConfig", merged) diff --git a/src/raitap/testing/tests/test_factories.py b/src/raitap/testing/tests/test_factories.py index a611ab99..bbed556c 100644 --- a/src/raitap/testing/tests/test_factories.py +++ b/src/raitap/testing/tests/test_factories.py @@ -1,6 +1,8 @@ from __future__ import annotations +import pytest import torch +from omegaconf import DictConfig from raitap.testing import make_app_config, make_tiny_classifier @@ -16,3 +18,32 @@ def test_app_config_factory_overrides() -> None: cfg = make_app_config(experiment_name="demo", hardware="cpu") assert cfg.experiment_name == "demo" assert cfg.hardware == "cpu" + + +def test_make_app_config_returns_struct_dictconfig_with_schema_defaults() -> None: + cfg = make_app_config() + assert isinstance(cfg, DictConfig) + # Declared defaults are present without any override. + assert cfg.transparency == {} + assert cfg.robustness == {} + assert cfg.metrics is None + assert cfg.reporting is None + assert cfg.tracking is None + assert cfg.seed is None + assert str(cfg.hardware) == "gpu" + assert cfg.experiment_name == "Experiment" + + +def test_make_app_config_applies_flat_and_nested_overrides() -> None: + cfg = make_app_config(experiment_name="demo", hardware="cpu", model={"source": "resnet18"}) + assert cfg.experiment_name == "demo" + assert str(cfg.hardware) == "cpu" + assert cfg.model.source == "resnet18" + # Untouched nested defaults survive the merge. + assert cfg.model.pretrained is False + + +def test_make_app_config_reading_undeclared_field_raises() -> None: + cfg = make_app_config() + with pytest.raises(AttributeError): + _ = cfg.definitely_not_a_field From c7776516e8e7466a67d3565c9d1f7044efe40717 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 01:28:03 +0200 Subject: [PATCH 02/15] refactor(model): declare ModelBackend.device; add make_fake_backend (refs #352) --- src/raitap/models/base_backend.py | 4 +++ src/raitap/testing/__init__.py | 2 ++ src/raitap/testing/factories.py | 37 ++++++++++++++++++++++ src/raitap/testing/tests/test_factories.py | 13 +++++++- 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/raitap/models/base_backend.py b/src/raitap/models/base_backend.py index f12e73dd..0b053612 100644 --- a/src/raitap/models/base_backend.py +++ b/src/raitap/models/base_backend.py @@ -81,6 +81,10 @@ class ModelBackend(ABC): # Optional id->name table for the model's output classes (e.g. torchvision # detection ``weights.meta["categories"]``). ``None`` when unavailable. category_names: list[str] | None = None + # Runtime device for backends that place tensors (TorchBackend sets this in + # __init__). ``None`` for device-less backends (ONNX, tree). Declared here + # so callers read ``backend.device`` directly instead of defensive getattr. + device: torch.device | None = None @classmethod def from_path( diff --git a/src/raitap/testing/__init__.py b/src/raitap/testing/__init__.py index 8af90ec7..8583e5ce 100644 --- a/src/raitap/testing/__init__.py +++ b/src/raitap/testing/__init__.py @@ -5,6 +5,7 @@ from raitap.testing.deps import requires from raitap.testing.factories import ( make_app_config, + make_fake_backend, make_pixel_linear_classifier, make_tiny_classifier, make_tiny_mlp, @@ -12,6 +13,7 @@ __all__ = [ "make_app_config", + "make_fake_backend", "make_pixel_linear_classifier", "make_tiny_classifier", "make_tiny_mlp", diff --git a/src/raitap/testing/factories.py b/src/raitap/testing/factories.py index b63d5d06..59515d89 100644 --- a/src/raitap/testing/factories.py +++ b/src/raitap/testing/factories.py @@ -13,6 +13,8 @@ from omegaconf import DictConfig, OmegaConf from raitap.configs.schema import AppConfig +from raitap.models.base_backend import ModelBackend +from raitap.types import Capability, TaskKind def make_tiny_classifier( @@ -80,3 +82,38 @@ def make_app_config(**overrides: Any) -> DictConfig: """ merged = OmegaConf.merge(OmegaConf.structured(AppConfig), overrides) return cast("DictConfig", merged) + + +def make_fake_backend( + *, + provides: frozenset[Capability] = frozenset(), + task_kind: TaskKind = TaskKind.classification, + hardware_label: str = "cpu", +) -> ModelBackend: + """A minimal concrete ``ModelBackend`` for tests that feed a backend into + prod read paths (``backend.provides`` / ``.task_kind`` / ``.device``). + + Replaces ``SimpleNamespace`` backend fakes so those reads resolve against + the real ABC contract instead of defensive ``getattr``. + """ + # Aliased to avoid the class-body self-reference trap: `provides = provides` + # inside the class body would make `provides` a class-local name for the + # whole statement, so the RHS lookup raises NameError instead of closing + # over the function argument. + _provides = provides + + class _FakeBackend(ModelBackend): + provides = _provides # type: ignore[misc] + + @property + def hardware_label(self) -> str: + return hardware_label + + @property + def task_kind(self) -> TaskKind: + return task_kind + + def __call__(self, inputs: Any, **kwargs: Any) -> Any: + return inputs + + return _FakeBackend() diff --git a/src/raitap/testing/tests/test_factories.py b/src/raitap/testing/tests/test_factories.py index bbed556c..5586ce52 100644 --- a/src/raitap/testing/tests/test_factories.py +++ b/src/raitap/testing/tests/test_factories.py @@ -4,7 +4,9 @@ import torch from omegaconf import DictConfig -from raitap.testing import make_app_config, make_tiny_classifier +from raitap.models.base_backend import ModelBackend +from raitap.testing import make_app_config, make_fake_backend, make_tiny_classifier +from raitap.types import Capability, TaskKind def test_tiny_classifier_is_deterministic_and_runs() -> None: @@ -47,3 +49,12 @@ def test_make_app_config_reading_undeclared_field_raises() -> None: cfg = make_app_config() with pytest.raises(AttributeError): _ = cfg.definitely_not_a_field + + +def test_make_fake_backend_exposes_typed_attrs() -> None: + backend = make_fake_backend(provides=frozenset({Capability.AUTOGRAD})) + assert isinstance(backend, ModelBackend) + assert backend.provides == frozenset({Capability.AUTOGRAD}) + assert backend.task_kind is TaskKind.classification + assert backend.device is None + assert backend.hardware_label == "cpu" From 119d7920f78566885792f1172669b5ff544eaa9f Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:43 +0200 Subject: [PATCH 03/15] refactor(model): direct config/backend access; type check_backend_compat as ModelBackend|None (refs #352) --- src/raitap/_adapters.py | 7 ++- src/raitap/models/model.py | 10 ++--- src/raitap/models/tests/test_model_class.py | 32 ++++++-------- src/raitap/models/tests/test_models.py | 43 ++++++------------- src/raitap/models/tests/test_tree_backend.py | 13 +++--- src/raitap/models/torch_backend.py | 4 ++ .../tests/test_adapter_capability_gate.py | 12 ++---- .../tests/test_capability_migration_guard.py | 19 ++++---- 8 files changed, 57 insertions(+), 83 deletions(-) diff --git a/src/raitap/_adapters.py b/src/raitap/_adapters.py index b58b7c2b..6b231d41 100644 --- a/src/raitap/_adapters.py +++ b/src/raitap/_adapters.py @@ -50,6 +50,7 @@ from collections.abc import Iterator, Sequence from types import ModuleType + from raitap.models.base_backend import ModelBackend from raitap.types import Capability CtxT = TypeVar("CtxT", contravariant=True) @@ -195,15 +196,17 @@ def required_capabilities(self) -> frozenset[Capability]: hints = registry.get(getattr(self, "algorithm", "")) return getattr(hints, "requires", frozenset()) - def check_backend_compat(self, backend: object) -> None: + def check_backend_compat(self, backend: ModelBackend | None) -> None: """Default backend gate: every required capability must be provided. + An absent backend provides nothing (empty capability set). + Override only for a non-capability contract (e.g. ``MarabouAssessor`` uses the hook for per-call setup; ``AutoLiRPAAssessor`` adds an XPU warning). """ from raitap.utils.errors import BackendIncompatibilityError - provided = getattr(backend, "provides", frozenset()) + provided = backend.provides if backend is not None else frozenset() missing = self.required_capabilities() - provided if missing: raise BackendIncompatibilityError( diff --git a/src/raitap/models/model.py b/src/raitap/models/model.py index 8da06691..7370b199 100644 --- a/src/raitap/models/model.py +++ b/src/raitap/models/model.py @@ -73,7 +73,7 @@ def __init__( def _load_model(self, config: AppConfig, *, allow_unsafe_pickle: bool = False) -> ModelBackend: source = config.model.source - hardware = getattr(config, "hardware", "gpu") + hardware = config.hardware if not source: raise ValueError( "No model specified. Set model.source in your config.\n" @@ -81,7 +81,7 @@ def _load_model(self, config: AppConfig, *, allow_unsafe_pickle: bool = False) - " model.source: resnet50 (built-in demo model)" ) - tokenizer = getattr(config.model, "tokenizer", None) + tokenizer = config.model.tokenizer if tokenizer: return load_hf_text_backend( source, tokenizer=tokenizer, device=resolve_torch_device(hardware) @@ -100,9 +100,7 @@ def _load_model(self, config: AppConfig, *, allow_unsafe_pickle: bool = False) - name = str(source).lower() if _resolve_torchvision_factory(name) is not None: - return _load_pretrained( - name, hardware=hardware, task_kind=getattr(config.model, "task_kind", None) - ) + return _load_pretrained(name, hardware=hardware, task_kind=config.model.task_kind) from raitap.models.registration import supported_model_formats @@ -198,7 +196,7 @@ def _resolve_shape_override(config: Any) -> tuple[int | None, ...] | None: :func:`raitap.data.metadata.shape_tuple` so semantics stay consistent with the rest of the metadata pipeline. """ - data_cfg = getattr(config, "data", None) + data_cfg = config.data if data_cfg is None: return None input_metadata = getattr(data_cfg, "input_metadata", None) diff --git a/src/raitap/models/tests/test_model_class.py b/src/raitap/models/tests/test_model_class.py index c816a306..0404ee26 100644 --- a/src/raitap/models/tests/test_model_class.py +++ b/src/raitap/models/tests/test_model_class.py @@ -2,18 +2,17 @@ from __future__ import annotations -from types import SimpleNamespace -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any from unittest.mock import MagicMock import pytest import torch +from raitap.testing import make_app_config + if TYPE_CHECKING: from pathlib import Path - from raitap.configs.schema import AppConfig - from raitap.models import Model from raitap.models.torch_backend import TorchBackend @@ -25,22 +24,15 @@ def _make_config( num_classes: int | None = None, pretrained: bool = False, hardware: str = "cpu", -) -> AppConfig: - return cast( - "AppConfig", - SimpleNamespace( - model=SimpleNamespace( - source=source, - arch=arch, - num_classes=num_classes, - pretrained=pretrained, - ), - data=SimpleNamespace( - preprocessing=None, - input_metadata=None, - ), - hardware=hardware, - ), +) -> Any: # a faithful AppConfig stand-in for Model.__init__ + return make_app_config( + model={ + "source": source, + "arch": arch, + "num_classes": num_classes, + "pretrained": pretrained, + }, + hardware=hardware, ) diff --git a/src/raitap/models/tests/test_models.py b/src/raitap/models/tests/test_models.py index 0aaf5279..c8a521b3 100644 --- a/src/raitap/models/tests/test_models.py +++ b/src/raitap/models/tests/test_models.py @@ -8,6 +8,7 @@ import torch import torch.nn as nn +from raitap.configs.schema import AppConfig, DataConfig, ModelConfig from raitap.data.preprocessing import resolve_preprocessing from raitap.models import Model, runtime from raitap.models.base_backend import _adapt_input_shape @@ -22,8 +23,6 @@ import onnxruntime as ort - from raitap.configs.schema import AppConfig - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -84,34 +83,18 @@ def _make_config( preprocessing: str | None = None, model_input_transformation: str | None = None, ) -> AppConfig: - return cast( - "AppConfig", - type( - "AppConfig", - (), - { - "model": type( - "ModelConfig", - (), - { - "source": source, - "arch": arch, - "num_classes": num_classes, - "pretrained": pretrained, - }, - )(), - "data": type( - "DataConfig", - (), - { - "preprocessing": preprocessing, - "model_input_transformation": model_input_transformation, - "input_metadata": None, - }, - )(), - "hardware": hardware, - }, - )(), + # Real schema dataclasses (not a hand-rolled stand-in): every field + # ``model.py`` reads directly (``tokenizer``, ``task_kind``, ...) already + # has the right default here, and ``hardware`` accepts a plain string + # (including invalid values, for ``test_invalid_hardware_raises_value_error``) + # since dataclass construction does no enum validation. + return AppConfig( + model=ModelConfig(source=source, arch=arch, num_classes=num_classes, pretrained=pretrained), + data=DataConfig( + preprocessing=preprocessing, + model_input_transformation=model_input_transformation, + ), + hardware=cast("Hardware", hardware), ) diff --git a/src/raitap/models/tests/test_tree_backend.py b/src/raitap/models/tests/test_tree_backend.py index 6399fcda..f69fe6fa 100644 --- a/src/raitap/models/tests/test_tree_backend.py +++ b/src/raitap/models/tests/test_tree_backend.py @@ -6,6 +6,7 @@ from raitap.models.access import EstimatorProvider from raitap.models.tree_backend import TabularTreeBackend +from raitap.testing import make_fake_backend from raitap.types import Capability, TaskKind @@ -97,11 +98,10 @@ def test_tree_explainer_gates_out_on_torch_backend() -> None: explainer = ShapExplainer("TreeExplainer") assert explainer.required_capabilities() == frozenset({Capability.TREE_MODEL}) - class _AutogradOnly: - provides = frozenset({Capability.AUTOGRAD}) + backend = make_fake_backend(provides=frozenset({Capability.AUTOGRAD})) with pytest.raises(BackendIncompatibilityError) as excinfo: - explainer.check_backend_compat(_AutogradOnly()) + explainer.check_backend_compat(backend) assert "tree_model" in str(excinfo.value) @@ -110,7 +110,8 @@ def test_tree_explainer_passes_on_tree_backend() -> None: explainer = ShapExplainer("TreeExplainer") - class _TreeOnly: - provides = frozenset({Capability.TREE_MODEL, Capability.PREDICT_PROBA}) + backend = make_fake_backend( + provides=frozenset({Capability.TREE_MODEL, Capability.PREDICT_PROBA}) + ) - explainer.check_backend_compat(_TreeOnly()) # no raise + explainer.check_backend_compat(backend) # no raise diff --git a/src/raitap/models/torch_backend.py b/src/raitap/models/torch_backend.py index 58d1afb7..bdcb0867 100644 --- a/src/raitap/models/torch_backend.py +++ b/src/raitap/models/torch_backend.py @@ -39,6 +39,10 @@ def _is_torchvision_detection_model(model: nn.Module) -> bool: class TorchBackend(ModelBackend): """PyTorch-backed model runtime.""" + # Narrows the ``ModelBackend.device`` ABC field (``torch.device | None``): + # a TorchBackend always resolves a concrete device in ``__init__``. + device: torch.device + def __init__( self, model: nn.Module, diff --git a/src/raitap/tests/test_adapter_capability_gate.py b/src/raitap/tests/test_adapter_capability_gate.py index d5a5ad56..49688554 100644 --- a/src/raitap/tests/test_adapter_capability_gate.py +++ b/src/raitap/tests/test_adapter_capability_gate.py @@ -3,15 +3,11 @@ import pytest from raitap._adapters import AdapterMixin +from raitap.testing import make_fake_backend from raitap.types import Capability from raitap.utils.errors import BackendIncompatibilityError -class _Backend: - def __init__(self, provides: frozenset[Capability]) -> None: - self.provides = provides - - class _Adapter(AdapterMixin): algorithm_registry: ClassVar = { "needs_grad": type("H", (), {"requires": frozenset({Capability.AUTOGRAD})})() @@ -23,13 +19,13 @@ def __init__(self, algorithm: str) -> None: def test_gate_passes_when_capabilities_met() -> None: _Adapter("needs_grad").check_backend_compat( - _Backend(frozenset({Capability.AUTOGRAD})) + make_fake_backend(provides=frozenset({Capability.AUTOGRAD})) ) # no raise def test_gate_rejects_when_capability_missing() -> None: with pytest.raises(BackendIncompatibilityError, match="autograd"): - _Adapter("needs_grad").check_backend_compat(_Backend(frozenset())) + _Adapter("needs_grad").check_backend_compat(make_fake_backend()) def test_empty_requires_runs_anywhere() -> None: @@ -39,4 +35,4 @@ class _Agnostic(AdapterMixin): def __init__(self) -> None: self.algorithm = "x" - _Agnostic().check_backend_compat(_Backend(frozenset())) # no raise + _Agnostic().check_backend_compat(make_fake_backend()) # no raise diff --git a/src/raitap/tests/test_capability_migration_guard.py b/src/raitap/tests/test_capability_migration_guard.py index 85e60e2f..c2446427 100644 --- a/src/raitap/tests/test_capability_migration_guard.py +++ b/src/raitap/tests/test_capability_migration_guard.py @@ -2,18 +2,11 @@ import pytest +from raitap.testing import make_fake_backend from raitap.types import Capability from raitap.utils.errors import BackendIncompatibilityError -class _NoCapBackend: - provides = frozenset() - - -class _AutogradBackend: - provides = frozenset({Capability.AUTOGRAD}) - - @pytest.mark.parametrize( ("module_path", "cls_name", "algorithm"), [ @@ -32,11 +25,15 @@ def test_known_gradient_algorithm_is_rejected_on_no_capability_backend( cls = getattr(importlib.import_module(module_path), cls_name) adapter = cls(algorithm=algorithm) with pytest.raises(BackendIncompatibilityError, match="autograd"): - adapter.check_backend_compat(_NoCapBackend()) - adapter.check_backend_compat(_AutogradBackend()) # gradient backend -> no raise + adapter.check_backend_compat(make_fake_backend(provides=frozenset())) + adapter.check_backend_compat( + make_fake_backend(provides=frozenset({Capability.AUTOGRAD})) + ) # gradient backend -> no raise def test_model_agnostic_algorithm_runs_on_no_capability_backend() -> None: from raitap.transparency.explainers.captum_explainer import CaptumExplainer - CaptumExplainer(algorithm="Occlusion").check_backend_compat(_NoCapBackend()) # no raise + CaptumExplainer(algorithm="Occlusion").check_backend_compat( + make_fake_backend(provides=frozenset()) + ) # no raise From 7ac9cb743038ffe2586c3db58304985fd2e4bae0 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:44 +0200 Subject: [PATCH 04/15] refactor(misc): direct config access in pipeline; drop dead config.run read (refs #352) --- src/raitap/pipeline/orchestrator.py | 13 ++++---- src/raitap/pipeline/phases/forward_pass.py | 6 ++-- src/raitap/pipeline/phases/input_metadata.py | 2 +- .../phases/tests/test_forward_pass.py | 5 +-- .../tests/test_forward_output_kind.py | 31 ++++++------------- src/raitap/pipeline/ui.py | 4 +-- 6 files changed, 21 insertions(+), 40 deletions(-) diff --git a/src/raitap/pipeline/orchestrator.py b/src/raitap/pipeline/orchestrator.py index 2db2979d..3b700df6 100644 --- a/src/raitap/pipeline/orchestrator.py +++ b/src/raitap/pipeline/orchestrator.py @@ -92,7 +92,7 @@ def _run_pipeline( if verbose: print_summary(config, model) - seed = getattr(config, "seed", None) + seed = config.seed if seed is not None: pin_global_seed(seed) outputs = run_without_tracking( @@ -107,7 +107,7 @@ def _run_pipeline( # whenever there is something to warn about — independent of reporting (the # run dir + console exist regardless); only the report banner (inside # build_report) is gated on reporting. - repro = assess_reproducibility(outputs, getattr(config, "seed", None)) + repro = assess_reproducibility(outputs, config.seed) # Write the run-level reproducibility artefact when there is something to # warn about OR a seed was pinned (record the seed run-wide even if fully # reproducible). Not duplicated into per-module metadata. @@ -128,20 +128,19 @@ def _run_pipeline( # warned, which the guard above rules out. raitap_log.warn(cast("str", reproducibility_caveat(repro))) - tracking_config = getattr(config, "tracking", None) - has_tracker = bool(tracking_config and getattr(tracking_config, "_target_", None)) - if not has_tracker: + tracking_config = config.tracking + if tracking_config is None or not getattr(tracking_config, "_target_", None): return outputs with BaseTracker.create_tracker(config) as tracker: tracker.log_config() - if getattr(config.tracking, "log_model", False): + if tracking_config.log_model: model.log(tracker) data.log(tracker) # Each phase result owns how it logs itself (artifacts + subdirectories). for phase_result in outputs.phase_results.values(): phase_result.log(tracker) - reporting_cfg = getattr(config, "reporting", None) + reporting_cfg = config.reporting if report_generation is not None and reporting_cfg is not None: report_generation.log(tracker) diff --git a/src/raitap/pipeline/phases/forward_pass.py b/src/raitap/pipeline/phases/forward_pass.py index 90a99786..55e49d1e 100644 --- a/src/raitap/pipeline/phases/forward_pass.py +++ b/src/raitap/pipeline/phases/forward_pass.py @@ -78,9 +78,7 @@ def extract_primary_tensor(model_output: object) -> torch.Tensor: def resolve_forward_batch_size(config: AppConfig) -> int: """Resolve prediction/metrics forward batch size from config, falling back to 32.""" - configured = getattr(getattr(config, "run", None), "forward_batch_size", None) - if configured is None: - configured = getattr(getattr(config, "data", None), "forward_batch_size", None) + configured = config.data.forward_batch_size if configured is None: return _DEFAULT_FORWARD_BATCH_SIZE if not isinstance(configured, int): @@ -149,7 +147,7 @@ def forward_pass( payload = family.extract_forward(ctx, batch_size=batch_size) output_kind = ( OutputKind.PROBABILITIES - if Capability.PREDICT_PROBA in getattr(backend, "provides", frozenset()) + if Capability.PREDICT_PROBA in backend.provides else OutputKind.LOGITS ) return ForwardOutput( diff --git a/src/raitap/pipeline/phases/input_metadata.py b/src/raitap/pipeline/phases/input_metadata.py index 200f87a6..6d83f8db 100644 --- a/src/raitap/pipeline/phases/input_metadata.py +++ b/src/raitap/pipeline/phases/input_metadata.py @@ -21,7 +21,7 @@ def input_metadata_for_data(config: AppConfig, data: Data) -> InputSpec | None: explicit = getattr(data, "input_metadata", None) if isinstance(explicit, InputSpec): return explicit - config_explicit = getattr(getattr(config, "data", None), "input_metadata", None) + config_explicit = config.data.input_metadata if isinstance(config_explicit, InputSpec): return config_explicit metadata = infer_data_input_metadata(config, data) diff --git a/src/raitap/pipeline/phases/tests/test_forward_pass.py b/src/raitap/pipeline/phases/tests/test_forward_pass.py index e743b73b..eb7c7732 100644 --- a/src/raitap/pipeline/phases/tests/test_forward_pass.py +++ b/src/raitap/pipeline/phases/tests/test_forward_pass.py @@ -26,10 +26,7 @@ def _make_config(batch_size: int = 32) -> Any: """Return a minimal config stub accepted by resolve_forward_batch_size.""" - return SimpleNamespace( - run=SimpleNamespace(forward_batch_size=batch_size), - data=SimpleNamespace(forward_batch_size=batch_size), - ) + return SimpleNamespace(data=SimpleNamespace(forward_batch_size=batch_size)) # --------------------------------------------------------------------------- diff --git a/src/raitap/pipeline/tests/test_forward_output_kind.py b/src/raitap/pipeline/tests/test_forward_output_kind.py index c497aace..ef958ee0 100644 --- a/src/raitap/pipeline/tests/test_forward_output_kind.py +++ b/src/raitap/pipeline/tests/test_forward_output_kind.py @@ -1,34 +1,17 @@ from __future__ import annotations -from types import SimpleNamespace from typing import Any import torch from raitap.pipeline.outputs import ForwardOutput, OutputKind from raitap.pipeline.phases.forward_pass import forward_pass +from raitap.testing import make_app_config, make_fake_backend from raitap.types import Capability, TaskKind -class _ProbaBackend: - provides = frozenset({Capability.TREE_MODEL, Capability.PREDICT_PROBA}) - task_kind = TaskKind.classification - expected_input_shape = None - - def _prepare_inputs(self, inputs: torch.Tensor) -> torch.Tensor: - return inputs - - def __call__(self, inputs: torch.Tensor) -> torch.Tensor: - n = inputs.shape[0] - return torch.full((n, 2), 0.5) - - -class _LogitBackend(_ProbaBackend): - provides = frozenset({Capability.AUTOGRAD}) - - -def _config() -> Any: # a fake AppConfig-shaped object for forward_pass - return SimpleNamespace(run=SimpleNamespace(forward_batch_size=8), data=None) +def _config() -> Any: # a faithful AppConfig stand-in for forward_pass + return make_app_config() def test_default_output_kind_is_logits() -> None: @@ -37,10 +20,14 @@ def test_default_output_kind_is_logits() -> None: def test_forward_pass_stamps_probabilities_for_predict_proba_backend() -> None: - out = forward_pass(_config(), _ProbaBackend(), torch.zeros(4, 3)) + backend = make_fake_backend( + provides=frozenset({Capability.TREE_MODEL, Capability.PREDICT_PROBA}) + ) + out = forward_pass(_config(), backend, torch.zeros(4, 3)) assert out.output_kind == OutputKind.PROBABILITIES def test_forward_pass_stamps_logits_for_autograd_backend() -> None: - out = forward_pass(_config(), _LogitBackend(), torch.zeros(4, 3)) + backend = make_fake_backend(provides=frozenset({Capability.AUTOGRAD})) + out = forward_pass(_config(), backend, torch.zeros(4, 3)) assert out.output_kind == OutputKind.LOGITS diff --git a/src/raitap/pipeline/ui.py b/src/raitap/pipeline/ui.py index 7f7ea9d1..14c0b204 100644 --- a/src/raitap/pipeline/ui.py +++ b/src/raitap/pipeline/ui.py @@ -31,8 +31,8 @@ def print_summary(config: AppConfig, model: Model) -> None: """Render the assessment-summary banner. Defensive against missing fields.""" - transparency = getattr(config, "transparency", None) or {} - robustness = getattr(config, "robustness", None) or {} + transparency = config.transparency or {} + robustness = config.robustness or {} try: from raitap.metrics import metrics_run_enabled From e71ff90d8b1be5d9f8182eeee53ce980c7261d64 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:45 +0200 Subject: [PATCH 05/15] refactor(transparency): direct config/backend access; ModelBackend|None gate protocol (refs #352) --- src/raitap/transparency/contracts.py | 7 +++- .../transparency/explainers/base_explainer.py | 16 ++++---- .../explainers/tests/test_base_explainer.py | 38 ++++++++++++++++--- src/raitap/transparency/phase.py | 16 ++++---- .../transparency/tests/e2e_case_matrix.py | 6 +++ .../test_assess_transparency_detection.py | 27 +++++-------- .../tests/test_e2e_integration.py | 6 +++ src/raitap/transparency/tests/test_factory.py | 7 ++++ 8 files changed, 82 insertions(+), 41 deletions(-) diff --git a/src/raitap/transparency/contracts.py b/src/raitap/transparency/contracts.py index 067d7805..54f02b2b 100644 --- a/src/raitap/transparency/contracts.py +++ b/src/raitap/transparency/contracts.py @@ -18,7 +18,7 @@ from dataclasses import dataclass, field from enum import StrEnum from pathlib import Path # noqa: TC003 -from typing import Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable # Runtime imports (not TYPE_CHECKING): ``Capability`` and ``Seeding`` appear in # public ``ExplainerAlgorithmSpec`` / ``ExplanationSemantics`` annotations, so @@ -27,6 +27,9 @@ from raitap.reproducibility import Seeding # noqa: TC001 from raitap.types import Capability # noqa: TC001 +if TYPE_CHECKING: + from raitap.models.base_backend import ModelBackend + ConfiguredVisualiser = Any ExplanationResult = Any Module = Any @@ -461,7 +464,7 @@ class ExplainerAdapter(Protocol): ``ATTRIBUTIONS`` when absent). """ # noqa: E501 - def check_backend_compat(self, backend: object) -> None: + def check_backend_compat(self, backend: ModelBackend | None) -> None: pass def explain( diff --git a/src/raitap/transparency/explainers/base_explainer.py b/src/raitap/transparency/explainers/base_explainer.py index 526d8d3a..bf311671 100644 --- a/src/raitap/transparency/explainers/base_explainer.py +++ b/src/raitap/transparency/explainers/base_explainer.py @@ -18,6 +18,7 @@ import torch from raitap.models.access import ExplanationModel + from raitap.models.base_backend import ModelBackend else: torch = lazy_import("torch") @@ -90,7 +91,7 @@ def explain( model: ExplanationModel, inputs: torch.Tensor, *, - backend: object | None = None, + backend: ModelBackend | None = None, run_dir: str | Path | None = None, output_root: str | Path | None = None, experiment_name: str | None = None, @@ -137,7 +138,7 @@ def explain( explainer=self, method_families=method_families, layer_path=_layer_path_for_explainer(self), - task_kind=getattr(backend, "task_kind", None), + task_kind=None if backend is None else backend.task_kind, ) _validate_output_space_shape(input_spec=input_spec, output_space=output_space) semantics = ExplanationSemantics( @@ -209,7 +210,7 @@ def _compute_with_optional_batches( model: ExplanationModel, inputs: torch.Tensor, attribution_kwargs: dict[str, Any], - backend: object | None, + backend: ModelBackend | None, *, input_spec: object | None = None, batch_size: int | None = None, @@ -274,13 +275,12 @@ def _compute_with_optional_batches( return torch.cat(principal_chunks, dim=0), extras @staticmethod - def _prepare_inputs_for_backend(inputs: torch.Tensor, backend: object | None) -> torch.Tensor: + def _prepare_inputs_for_backend( + inputs: torch.Tensor, backend: ModelBackend | None + ) -> torch.Tensor: if backend is None: return inputs - prepare_inputs = getattr(backend, "_prepare_inputs", None) - if not callable(prepare_inputs): - return inputs - return cast("torch.Tensor", prepare_inputs(inputs)) + return backend._prepare_inputs(inputs) @staticmethod def _normalise_attributions(attributions: torch.Tensor) -> torch.Tensor: diff --git a/src/raitap/transparency/explainers/tests/test_base_explainer.py b/src/raitap/transparency/explainers/tests/test_base_explainer.py index 122de147..39639010 100644 --- a/src/raitap/transparency/explainers/tests/test_base_explainer.py +++ b/src/raitap/transparency/explainers/tests/test_base_explainer.py @@ -12,6 +12,7 @@ from raitap.models.access import ExplanationModel import raitap.transparency.explainers.base_explainer as base_explainer_module +from raitap.testing import make_fake_backend from raitap.transparency.contracts import ( BaselineMode, ExplainerAlgorithmSpec, @@ -222,6 +223,7 @@ def test_explain_stores_raitap_visualisation_metadata_in_kwargs() -> None: model, inputs, target=7, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for( inputs, sample_names=["ISIC_1", "ISIC_2"], @@ -246,6 +248,7 @@ def test_explain_builds_semantics_with_sample_ids_separate_from_display_names() model, inputs, target=7, + backend=make_fake_backend(), raitap_kwargs={ "input_metadata": {"kind": "image", "shape": inputs.shape, "layout": "NCHW"}, "sample_ids": ["stable-1", "stable-2"], @@ -276,6 +279,7 @@ def test_explain_uses_declared_output_scope() -> None: result = explainer.explain( model, inputs, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs), ) @@ -293,6 +297,7 @@ def test_explain_uses_optional_batching_and_slices_per_sample_kwargs() -> None: inputs, target=[0, 1, 2, 3, 4], background_data=background, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs, batch_size=2), ) @@ -302,14 +307,23 @@ def test_explain_uses_optional_batching_and_slices_per_sample_kwargs() -> None: def test_explain_prepares_each_batch_with_backend() -> None: - class _PreparingBackend: + from raitap.models.base_backend import ModelBackend + + class _PreparingBackend(ModelBackend): def __init__(self) -> None: self.prepared_batch_sizes: list[int] = [] + @property + def hardware_label(self) -> str: + return "cpu" + def _prepare_inputs(self, inputs: torch.Tensor) -> torch.Tensor: self.prepared_batch_sizes.append(int(inputs.shape[0])) return inputs + 10 + def __call__(self, inputs: Any, **kwargs: Any) -> Any: + return inputs + explainer = _BatchRecordingExplainer() backend = _PreparingBackend() model = torch.nn.Identity() @@ -336,6 +350,7 @@ def test_explain_accepts_progress_kwargs_without_forwarding_to_compute() -> None model, inputs, target=7, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for( inputs, batch_size=1, @@ -368,6 +383,7 @@ def _record_wrap(starts: range, *, total_batches: int, progress_desc: str | None model, inputs, target=[0, 1, 2, 3, 4], + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs, batch_size=2), ) @@ -397,6 +413,7 @@ def _record_wrap(starts: range, *, total_batches: int, progress_desc: str | None model, inputs, target=[0, 1, 2, 3, 4], + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs, batch_size=2, show_progress=False), ) @@ -430,6 +447,7 @@ def _record_empty_cache() -> None: model, inputs, target=[0, 1, 2, 3, 4], + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs, batch_size=2, show_progress=False), ) @@ -456,6 +474,7 @@ def _record_cat(tensors: list[torch.Tensor], dim: int = 0) -> torch.Tensor: result = explainer.explain( model, inputs, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs, batch_size=2, show_progress=False), ) @@ -471,7 +490,9 @@ def test_explain_normalises_unbatched_attributions_to_detached_cpu() -> None: model = torch.nn.Identity() inputs = torch.randn(2, 3) - result = explainer.explain(model, inputs, raitap_kwargs=_raitap_kwargs_for(inputs)) + result = explainer.explain( + model, inputs, backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs) + ) assert result.attributions.device.type == "cpu" assert not result.attributions.requires_grad @@ -483,7 +504,9 @@ def test_explain_accepts_tuple_output_as_structured_payload() -> None: model = torch.nn.Identity() inputs = torch.randn(2, 3) - result = explainer.explain(model, inputs, raitap_kwargs=_raitap_kwargs_for(inputs)) + result = explainer.explain( + model, inputs, backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs) + ) assert [p.name for p in result.structured_payloads] == ["convergence_delta"] payload = result.structured_payloads[0] @@ -499,6 +522,7 @@ def test_explain_accepts_list_output_in_batched_path() -> None: result = explainer.explain( model, inputs, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs, batch_size=2, show_progress=False), ) @@ -575,8 +599,8 @@ def test_explain_threads_backend_task_kind_into_infer_output_space() -> None: def test_explain_classification_unchanged_when_backend_has_no_task_kind() -> None: - """No backend → task_kind defaults to None → classification path stays - untouched (INPUT_FEATURES for an NCHW image input).""" + """Backend's ``task_kind`` defaults to classification → classification path + stays untouched (INPUT_FEATURES for an NCHW image input).""" from raitap.transparency.contracts import ExplanationOutputSpace explainer = _StrictExplainer() @@ -586,6 +610,7 @@ def test_explain_classification_unchanged_when_backend_has_no_task_kind() -> Non result = explainer.explain( model, inputs, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs), ) assert result.semantics.output_space.space is ExplanationOutputSpace.INPUT_FEATURES @@ -632,6 +657,7 @@ def test_explain_attaches_zero_baseline_when_absent(tmp_path: Path) -> None: inputs, run_dir=tmp_path / "exp", target=0, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs), ) @@ -661,6 +687,7 @@ def _boom(**_kwargs: Any) -> None: inputs, run_dir=tmp_path / "exp", target=0, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs), ) @@ -683,6 +710,7 @@ def test_explain_records_configured_baseline_with_provenance(tmp_path: Path) -> target=0, baselines=baseline_tensor, call_provenance={"baselines": {"source": "zeros_cfg", "n_samples": 1}}, + backend=make_fake_backend(), raitap_kwargs=_raitap_kwargs_for(inputs), ) diff --git a/src/raitap/transparency/phase.py b/src/raitap/transparency/phase.py index 3b459959..f7e9a143 100644 --- a/src/raitap/transparency/phase.py +++ b/src/raitap/transparency/phase.py @@ -67,7 +67,7 @@ class TransparencyPhase(AssessmentPhase): name = "transparency" def is_configured(self, config: AppConfig) -> bool: - return bool(getattr(config, "transparency", None)) + return bool(config.transparency) def run(self, ctx: PhaseContext) -> PhaseResult | None: output = assess_transparency( @@ -236,13 +236,11 @@ def prepare_explainer( call_provenance=call_provenance, base_run_dir=base_run_dir, backend=backend, - experiment_name=str(getattr(config, "experiment_name", "")), + experiment_name=str(config.experiment_name), explainer_config=explainer_config, - # ``config.model`` is a struct-mode DictConfig; when the YAML omits the - # optional ``class_names`` key an unconditional read raises - # ``ConfigAttributeError``. Read defensively so the optional field + - # backend fallback in ``resolve_category_names`` work as designed (#240). - class_names=getattr(config.model, "class_names", None), + # ``class_names`` is an optional field (default ``None``); the backend + # fallback in ``resolve_category_names`` handles the unset case (#240). + class_names=config.model.class_names, ) @@ -264,7 +262,7 @@ def assess_transparency( per-adapter ``evaluation`` block is configured, its explanations are graded as a post-step (issue #341); otherwise ``evaluations`` stays empty. """ - explainer_names = list((getattr(config, "transparency", None) or {}).keys()) + explainer_names = list((config.transparency or {}).keys()) if not explainer_names: return TransparencyOutput(explanations=[], evaluations=[]) @@ -294,6 +292,6 @@ def assess_transparency( ExplainContext(prepared=prepared, forward_output=forward_output, data=data) ) results += expls - evaluation_cfg = getattr(config.transparency[name], "evaluation", None) + evaluation_cfg = config.transparency[name].evaluation evaluations += grade_explanations(evaluation_cfg, expls, prepared) return TransparencyOutput(explanations=results, evaluations=evaluations) diff --git a/src/raitap/transparency/tests/e2e_case_matrix.py b/src/raitap/transparency/tests/e2e_case_matrix.py index e67fe902..f069b09d 100644 --- a/src/raitap/transparency/tests/e2e_case_matrix.py +++ b/src/raitap/transparency/tests/e2e_case_matrix.py @@ -9,6 +9,7 @@ import torch.nn as nn from omegaconf import OmegaConf +from raitap.configs.schema import DataConfig from raitap.models.access import ExplanationModel, explanation_model from raitap.models.torch_backend import TorchBackend from raitap.transparency import ExplanationResult, VisualisationResult @@ -247,6 +248,11 @@ def _build_factory_config(case: MatrixCase, tmp_path: Path) -> AppConfig: } ) }, + # ``resolve_per_image_transform``'s no-``resolved_preprocessing`` + # fallback reads ``config.data`` directly; a real (defaulted) + # ``DataConfig`` keeps that read honest instead of re-adding a + # ``getattr`` default here. + data=DataConfig(), ), ), ) diff --git a/src/raitap/transparency/tests/test_assess_transparency_detection.py b/src/raitap/transparency/tests/test_assess_transparency_detection.py index 1979ae90..91252af1 100644 --- a/src/raitap/transparency/tests/test_assess_transparency_detection.py +++ b/src/raitap/transparency/tests/test_assess_transparency_detection.py @@ -23,7 +23,7 @@ from raitap.transparency.results import ExplanationResult from raitap.configs import set_output_root -from raitap.configs.schema import AppConfig, TransparencyConfig +from raitap.configs.schema import AppConfig, ModelConfig, TransparencyConfig from raitap.models.base_backend import ModelBackend from raitap.pipeline.outputs import ForwardOutput from raitap.transparency.phase import assess_transparency @@ -551,25 +551,18 @@ def test_assess_transparency_detection_skips_when_no_explainers(tmp_path: Path) def test_assess_transparency_detection_handles_model_block_without_class_names( tmp_path: Path, ) -> None: - """Regression for #240: a struct-mode ``model:`` block that omits - ``class_names`` must not crash the transparency phase. - - The real pipeline hands ``config.model`` as a YAML-loaded struct-mode - ``DictConfig`` (``object_type=dict``). When the block never declares the - optional ``class_names`` key, the unconditional attribute read raised - ``ConfigAttributeError`` before ``resolve_category_names`` could fall back - to ``backend.category_names``. The read must be defensive so the optional - field + backend fallback work as designed. + """Regression for #240: a ``model:`` block that omits ``class_names`` in + its YAML source (leaving the schema default ``None``) must not crash the + transparency phase. + + ``class_names`` is a declared, defaulted (``None``) field on + ``ModelConfig``, so a schema-typed model block that never sets it reads + back as ``None`` directly (no ``getattr`` needed) and + ``resolve_category_names`` falls back to ``backend.category_names``. """ - from omegaconf import OmegaConf - config = AppConfig(experiment_name="no-class-names") set_output_root(config, tmp_path) - # Mirror a YAML-loaded model block: a plain struct-mode DictConfig that - # never declares ``class_names`` (object_type=dict, struct=True). - model_block = OmegaConf.create({"source": "fasterrcnn_resnet50_fpn_v2"}) - OmegaConf.set_struct(model_block, True) - config.model = model_block # type: ignore[assignment] + config.model = ModelConfig(source="fasterrcnn_resnet50_fpn_v2") config.transparency = { "ig_det": TransparencyConfig( _target_="CaptumExplainer", diff --git a/src/raitap/transparency/tests/test_e2e_integration.py b/src/raitap/transparency/tests/test_e2e_integration.py index 26167e9f..c9db397d 100644 --- a/src/raitap/transparency/tests/test_e2e_integration.py +++ b/src/raitap/transparency/tests/test_e2e_integration.py @@ -11,6 +11,7 @@ from omegaconf import OmegaConf from raitap.configs import set_output_root +from raitap.configs.schema import DataConfig from raitap.models.access import explanation_model from raitap.models.torch_backend import TorchBackend from raitap.transparency import ExplanationResult, VisualisationResult @@ -96,6 +97,11 @@ def _captum_config() -> AppConfig: } ) }, + # ``resolve_per_image_transform``'s no-``resolved_preprocessing`` + # fallback reads ``config.data`` directly; a real (defaulted) + # ``DataConfig`` keeps that read honest instead of re-adding a + # ``getattr`` default here. + data=DataConfig(), ), ), ) diff --git a/src/raitap/transparency/tests/test_factory.py b/src/raitap/transparency/tests/test_factory.py index 2cdfbdb3..d2d01e74 100644 --- a/src/raitap/transparency/tests/test_factory.py +++ b/src/raitap/transparency/tests/test_factory.py @@ -10,6 +10,7 @@ from omegaconf import OmegaConf from raitap.configs.adapter_factory import resolve_call_data_sources +from raitap.configs.schema import DataConfig from raitap.models.base_backend import ModelBackend from raitap.transparency import ( PayloadVisualiserIncompatibilityError, @@ -111,6 +112,11 @@ def _make_config(tmp_path: Path, transparency_config: Any) -> AppConfig: experiment_name="test", _output_root=str(tmp_path), transparency={"test_explainer": transparency_config}, + # ``resolve_per_image_transform``'s no-``resolved_preprocessing`` + # fallback reads ``config.data`` directly; a real (defaulted) + # ``DataConfig`` keeps that read honest instead of re-adding a + # ``getattr`` default here. + data=DataConfig(), ), ) @@ -1194,6 +1200,7 @@ def explain(self, *_args: Any, **kwargs: Any) -> ExplanationResult: experiment_name="test", _output_root=str(tmp_path), transparency={"shap_gradient": _load_transparency_preset("shap_gradient")}, + data=DataConfig(), ), ), ) From 473b2b6fde4af6bc01d66f31bfd1f359d5319a3d Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:46 +0200 Subject: [PATCH 06/15] refactor(robustness): direct config/backend access; backend.device + None-safe gate (refs #352) --- .../assessors/auto_lirpa_assessor.py | 10 ++++- .../robustness/assessors/base_assessor.py | 12 +++--- .../robustness/assessors/marabou_assessor.py | 4 +- .../tests/test_auto_lirpa_assessor.py | 19 +++++++- .../assessors/tests/test_foolbox_assessor.py | 13 +----- .../assessors/tests/test_marabou_assessor.py | 39 +++++++---------- .../tests/test_torchattacks_assessor.py | 16 +++---- src/raitap/robustness/contracts.py | 9 ++-- src/raitap/robustness/factory.py | 2 +- src/raitap/robustness/phase.py | 4 +- .../tests/test_e2e_marabou_acas_xu.py | 20 ++++++++- .../robustness/tests/test_e2e_real_data.py | 43 +++++++++++++------ .../tests/test_e2e_robustness_matrix.py | 5 +++ .../robustness/tests/test_marabou_parity.py | 10 ++++- 14 files changed, 127 insertions(+), 79 deletions(-) diff --git a/src/raitap/robustness/assessors/auto_lirpa_assessor.py b/src/raitap/robustness/assessors/auto_lirpa_assessor.py index d75195d1..2d8b10c0 100644 --- a/src/raitap/robustness/assessors/auto_lirpa_assessor.py +++ b/src/raitap/robustness/assessors/auto_lirpa_assessor.py @@ -43,6 +43,8 @@ if TYPE_CHECKING: import torch + + from raitap.models.base_backend import ModelBackend else: torch = lazy_import("torch") @@ -131,7 +133,7 @@ def __init__( # Backend acceptance # ------------------------------------------------------------------ - def check_backend_compat(self, backend: object) -> None: + def check_backend_compat(self, backend: ModelBackend | None) -> None: """Inherit the autograd gate, then warn on Intel XPU. auto-LiRPA needs the live ``nn.Module`` + autograd, so non-autograd @@ -140,7 +142,11 @@ def check_backend_compat(self, backend: object) -> None: back to a CPU backend themselves. """ super().check_backend_compat(backend) - device = getattr(backend, "device", None) + if backend is None: + # The AUTOGRAD requirement above rejects a missing backend before + # reaching here; this only narrows the type for the device check. + return + device = backend.device if getattr(device, "type", None) == "xpu": from raitap import raitap_log diff --git a/src/raitap/robustness/assessors/base_assessor.py b/src/raitap/robustness/assessors/base_assessor.py index 99563636..6f8b2030 100644 --- a/src/raitap/robustness/assessors/base_assessor.py +++ b/src/raitap/robustness/assessors/base_assessor.py @@ -25,6 +25,7 @@ from raitap import raitap_log from raitap._adapters import AdapterMixin from raitap.configs import resolve_run_dir +from raitap.models.base_backend import ModelBackend from raitap.utils.lazy import lazy_import if TYPE_CHECKING: @@ -180,7 +181,7 @@ def assess( inputs: torch.Tensor, targets: torch.Tensor, *, - backend: object | None = None, + backend: ModelBackend | None = None, run_dir: str | Path | None = None, output_root: str | Path | None = None, experiment_name: str | None = None, @@ -361,7 +362,7 @@ def assess( inputs: torch.Tensor, targets: torch.Tensor, *, - backend: object | None = None, + backend: ModelBackend | None = None, run_dir: str | Path | None = None, output_root: str | Path | None = None, experiment_name: str | None = None, @@ -525,7 +526,7 @@ def assess( inputs: torch.Tensor, targets: torch.Tensor, *, - backend: object | None = None, + backend: ModelBackend | None = None, run_dir: str | Path | None = None, output_root: str | Path | None = None, experiment_name: str | None = None, @@ -679,9 +680,8 @@ def _prepare_inputs_for_forward( used by transparency); (2) ``next(model.parameters()).device`` fallback for parameter-bearing modules; (3) leave inputs untouched. """ - prepare = getattr(backend, "_prepare_inputs", None) - if callable(prepare): - prepared = prepare(inputs) + if isinstance(backend, ModelBackend): + prepared = backend._prepare_inputs(inputs) if not isinstance(prepared, torch.Tensor): raise TypeError( f"backend._prepare_inputs returned {type(prepared).__name__}, expected Tensor." diff --git a/src/raitap/robustness/assessors/marabou_assessor.py b/src/raitap/robustness/assessors/marabou_assessor.py index a13cbc9d..7e4e12e4 100644 --- a/src/raitap/robustness/assessors/marabou_assessor.py +++ b/src/raitap/robustness/assessors/marabou_assessor.py @@ -70,6 +70,8 @@ if TYPE_CHECKING: from torch import nn + from raitap.models.base_backend import ModelBackend + logger = logging.getLogger(__name__) @@ -148,7 +150,7 @@ def __init__( # Backend acceptance # ------------------------------------------------------------------ - def check_backend_compat(self, backend: object) -> None: + def check_backend_compat(self, backend: ModelBackend | None) -> None: # Reset per-assess() state. ``check_backend_compat`` is called by # ``FormalVerificationAssessor.assess`` exactly once before the # per-sample loop, so it doubles as a per-call setup hook. Any temp diff --git a/src/raitap/robustness/assessors/tests/test_auto_lirpa_assessor.py b/src/raitap/robustness/assessors/tests/test_auto_lirpa_assessor.py index df0e9f7a..76fabe57 100644 --- a/src/raitap/robustness/assessors/tests/test_auto_lirpa_assessor.py +++ b/src/raitap/robustness/assessors/tests/test_auto_lirpa_assessor.py @@ -10,6 +10,7 @@ import pytest import torch +from raitap.models.base_backend import ModelBackend from raitap.robustness.assessors import AutoLiRPAAssessor from raitap.robustness.contracts import ( PerturbationBudget, @@ -79,16 +80,30 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return self.linear(x.flatten(1) if x.ndim > 2 else x) -class _TorchBackend: +class _TorchBackend(ModelBackend): provides = frozenset({Capability.AUTOGRAD}) def __init__(self, device_type: str = "cpu") -> None: self.device = torch.device(device_type) + @property + def hardware_label(self) -> str: + return self.device.type if self.device is not None else "cpu" -class _OnnxBackend: + def __call__(self, inputs: torch.Tensor, **kwargs: Any) -> torch.Tensor: + return inputs + + +class _OnnxBackend(ModelBackend): provides = frozenset() + @property + def hardware_label(self) -> str: + return "onnx" + + def __call__(self, inputs: torch.Tensor, **kwargs: Any) -> torch.Tensor: + return inputs + def _linf_budget(eps: float = 0.05) -> PerturbationBudget: return PerturbationBudget(norm=PerturbationNorm.LINF, epsilon=eps) diff --git a/src/raitap/robustness/assessors/tests/test_foolbox_assessor.py b/src/raitap/robustness/assessors/tests/test_foolbox_assessor.py index fa4450b9..7050323e 100644 --- a/src/raitap/robustness/assessors/tests/test_foolbox_assessor.py +++ b/src/raitap/robustness/assessors/tests/test_foolbox_assessor.py @@ -4,25 +4,16 @@ import torch from raitap.robustness.assessors import FoolboxAssessor -from raitap.testing import make_pixel_linear_classifier -from raitap.types import Capability +from raitap.testing import make_fake_backend, make_pixel_linear_classifier from raitap.utils.errors import BackendIncompatibilityError foolbox = pytest.importorskip("foolbox") -class _AutogradBackend: - provides = frozenset({Capability.AUTOGRAD}) - - -class _OnnxLikeBackend: - provides = frozenset() - - def test_foolbox_rejects_non_autograd_backend() -> None: assessor = FoolboxAssessor(algorithm="LinfPGD") with pytest.raises(BackendIncompatibilityError): - assessor.check_backend_compat(_OnnxLikeBackend()) + assessor.check_backend_compat(make_fake_backend(provides=frozenset())) def test_foolbox_rejects_list_epsilons() -> None: diff --git a/src/raitap/robustness/assessors/tests/test_marabou_assessor.py b/src/raitap/robustness/assessors/tests/test_marabou_assessor.py index 2ae64dee..7275da7c 100644 --- a/src/raitap/robustness/assessors/tests/test_marabou_assessor.py +++ b/src/raitap/robustness/assessors/tests/test_marabou_assessor.py @@ -12,6 +12,7 @@ import pytest import torch +from raitap.models.base_backend import ModelBackend from raitap.robustness.assessors import MarabouAssessor from raitap.robustness.assessors.marabou_assessor import _bisect_output_bound from raitap.robustness.contracts import ( @@ -20,6 +21,7 @@ RobustnessVerdict, ) from raitap.robustness.exceptions import AssessorBackendIncompatibilityError +from raitap.testing import make_fake_backend if TYPE_CHECKING: from pathlib import Path @@ -133,12 +135,19 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return self.linear(x.flatten(1) if x.ndim > 2 else x) -class _OnnxBackend: +class _OnnxBackend(ModelBackend): """Backend stub that pretends to expose a pre-built ONNX file.""" - def __init__(self, onnx_path: str) -> None: + def __init__(self, onnx_path: Any) -> None: self.onnx_path = onnx_path + @property + def hardware_label(self) -> str: + return "test-onnx" + + def __call__(self, inputs: Any, **kwargs: Any) -> Any: + raise NotImplementedError("test stub: onnx-path resolution only") + # --------------------------------------------------------------------------- # Registry / construction @@ -377,7 +386,7 @@ def test_check_backend_compat_accepts_torch_and_onnx(tmp_path: Any) -> None: assessor = MarabouAssessor() # Either backend type accepted — no raise. assessor.check_backend_compat(_OnnxBackend(_onnx_path(tmp_path))) - assessor.check_backend_compat(object()) + assessor.check_backend_compat(make_fake_backend()) assessor.check_backend_compat(None) @@ -559,17 +568,13 @@ def test_verify_sample_returns_none_bounds_when_flag_disabled( onnx_path.write_bytes(b"\x00") fake_maraboupy.solve_result = ("unsat", {}, _FakeStats(0.0)) - class _Backend: - def __init__(self, p: Any) -> None: - self.onnx_path = p - assessor = MarabouAssessor(compute_output_bounds=False) outcome = assessor.verify_sample( model=_IdentityModel(), sample=torch.zeros(1, 5), target=torch.tensor([0]), budget=PerturbationBudget(norm=PerturbationNorm.LINF, epsilon=0.05), - backend=_Backend(onnx_path), + backend=_OnnxBackend(onnx_path), ) assert outcome.verdict == RobustnessVerdict.VERIFIED assert outcome.lower_bounds is None @@ -596,17 +601,13 @@ def fake_bisect(**kwargs: Any) -> float: fake_bisect, ) - class _Backend: - def __init__(self, p: Any) -> None: - self.onnx_path = p - assessor = MarabouAssessor(compute_output_bounds=True) outcome = assessor.verify_sample( model=_IdentityModel(), sample=torch.zeros(1, 5), target=torch.tensor([0]), budget=PerturbationBudget(norm=PerturbationNorm.LINF, epsilon=0.05), - backend=_Backend(onnx_path), + backend=_OnnxBackend(onnx_path), ) assert outcome.lower_bounds is not None and outcome.upper_bounds is not None assert outcome.lower_bounds.shape == (5,) @@ -635,17 +636,13 @@ def boom(**kwargs: Any) -> float: monkeypatch.setattr("raitap.robustness.assessors.marabou_assessor._bisect_output_bound", boom) - class _Backend: - def __init__(self, p: Any) -> None: - self.onnx_path = p - assessor = MarabouAssessor(compute_output_bounds=True) outcome = assessor.verify_sample( model=_IdentityModel(), sample=torch.zeros(1, 5), target=torch.tensor([0]), budget=PerturbationBudget(norm=PerturbationNorm.LINF, epsilon=0.05), - backend=_Backend(onnx_path), + backend=_OnnxBackend(onnx_path), ) assert outcome.verdict == RobustnessVerdict.FALSIFIED assert outcome.lower_bounds is None @@ -668,16 +665,12 @@ def test_assess_propagates_output_bounds_to_result( lambda **kw: -1.0 if kw["mode"] == "lower" else 1.0, ) - class _Backend: - def __init__(self, p: Any) -> None: - self.onnx_path = p - assessor = MarabouAssessor(compute_output_bounds=True) result = assessor.assess( model=_IdentityModel(), inputs=torch.zeros(2, 5), targets=torch.tensor([0, 0]), - backend=_Backend(onnx_path), + backend=_OnnxBackend(onnx_path), ) assert result.output_bounds is not None lower = result.output_bounds["lower"] diff --git a/src/raitap/robustness/assessors/tests/test_torchattacks_assessor.py b/src/raitap/robustness/assessors/tests/test_torchattacks_assessor.py index b898ef18..1d074c0d 100644 --- a/src/raitap/robustness/assessors/tests/test_torchattacks_assessor.py +++ b/src/raitap/robustness/assessors/tests/test_torchattacks_assessor.py @@ -4,30 +4,24 @@ import torch from raitap.robustness.assessors import TorchattacksAssessor -from raitap.testing import make_pixel_linear_classifier +from raitap.testing import make_fake_backend, make_pixel_linear_classifier from raitap.types import Capability from raitap.utils.errors import BackendIncompatibilityError torchattacks = pytest.importorskip("torchattacks") -class _AutogradBackend: - provides = frozenset({Capability.AUTOGRAD}) - - -class _OnnxLikeBackend: - provides = frozenset() - - def test_check_backend_compat_rejects_non_autograd_backend() -> None: assessor = TorchattacksAssessor(algorithm="FGSM", eps=0.03) with pytest.raises(BackendIncompatibilityError): - assessor.check_backend_compat(_OnnxLikeBackend()) + assessor.check_backend_compat(make_fake_backend(provides=frozenset())) def test_check_backend_compat_accepts_autograd_backend() -> None: assessor = TorchattacksAssessor(algorithm="FGSM", eps=0.03) - assessor.check_backend_compat(_AutogradBackend()) # no raise + assessor.check_backend_compat( + make_fake_backend(provides=frozenset({Capability.AUTOGRAD})) + ) # no raise def test_generate_adversarial_runs_with_fgsm() -> None: diff --git a/src/raitap/robustness/contracts.py b/src/raitap/robustness/contracts.py index 64a22af9..4fe54000 100644 --- a/src/raitap/robustness/contracts.py +++ b/src/raitap/robustness/contracts.py @@ -27,11 +27,14 @@ from dataclasses import dataclass, field from enum import StrEnum from pathlib import Path # noqa: TC003 -from typing import Any, ClassVar, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, ClassVar, Protocol, runtime_checkable from raitap.reproducibility import Seeding # noqa: TC001 from raitap.transparency.contracts import InputSpec, SampleSelection # noqa: TC001 +if TYPE_CHECKING: + from raitap.models.base_backend import ModelBackend + ConfiguredRobustnessVisualiser = Any RobustnessResult = Any Module = Any @@ -239,7 +242,7 @@ class AssessorAdapter(Protocol): assessment_kind: ClassVar[AssessmentKind] - def check_backend_compat(self, backend: object) -> None: + def check_backend_compat(self, backend: ModelBackend | None) -> None: pass def assess( @@ -248,7 +251,7 @@ def assess( inputs: Tensor, targets: Tensor, *, - backend: object | None = None, + backend: ModelBackend | None = None, run_dir: str | Path | None = None, output_root: str | Path | None = None, experiment_name: str | None = None, diff --git a/src/raitap/robustness/factory.py b/src/raitap/robustness/factory.py index 30e63d33..02c33d5a 100644 --- a/src/raitap/robustness/factory.py +++ b/src/raitap/robustness/factory.py @@ -172,7 +172,7 @@ def __new__( targets, backend=backend, run_dir=resolve_run_dir(config, subdir=f"robustness/{assessor_name}"), - experiment_name=str(getattr(config, "experiment_name", "")), + experiment_name=str(config.experiment_name), assessor_target=assessor_target, assessor_name=assessor_name, visualisers=visualisers, diff --git a/src/raitap/robustness/phase.py b/src/raitap/robustness/phase.py index ca978e3d..29bd2a45 100644 --- a/src/raitap/robustness/phase.py +++ b/src/raitap/robustness/phase.py @@ -34,7 +34,7 @@ class RobustnessPhase(AssessmentPhase): name = "robustness" def is_configured(self, config: AppConfig) -> bool: - return bool(getattr(config, "robustness", None)) + return bool(config.robustness) def run(self, ctx: PhaseContext) -> PhaseResult | None: results = assess_robustness( @@ -104,7 +104,7 @@ def assess_robustness( # (e.g. detection DetectionAdversarialLoss); the family opts out here. return [] - assessors = getattr(config, "robustness", None) or {} + assessors = config.robustness or {} classification_labels = labels if not isinstance(labels, list) else None targets = resolve_robustness_targets( labels=classification_labels, forward_output=forward_output diff --git a/src/raitap/robustness/tests/test_e2e_marabou_acas_xu.py b/src/raitap/robustness/tests/test_e2e_marabou_acas_xu.py index 6f025772..6d4a2715 100644 --- a/src/raitap/robustness/tests/test_e2e_marabou_acas_xu.py +++ b/src/raitap/robustness/tests/test_e2e_marabou_acas_xu.py @@ -20,6 +20,8 @@ import pytest import torch +from raitap.models.base_backend import ModelBackend + if TYPE_CHECKING: from pathlib import Path @@ -79,9 +81,16 @@ def test_acas_xu_tiny_eps_verified(acas_xu_fixture: tuple[Path, torch.Tensor, in model_path, sample, target_class = acas_xu_fixture - class _OnnxBackend: + class _OnnxBackend(ModelBackend): onnx_path = str(model_path) + @property + def hardware_label(self) -> str: + return "test-onnx" + + def __call__(self, inputs: object, **kwargs: object) -> object: + raise NotImplementedError("test stub: onnx-path resolution only") + assessor = MarabouAssessor(epsilon=1e-5, timeout_s=120.0) assessor.check_backend_compat(_OnnxBackend()) outcome = assessor.verify_sample( @@ -104,9 +113,16 @@ def test_acas_xu_huge_eps_falsified(acas_xu_fixture: tuple[Path, torch.Tensor, i model_path, sample, target_class = acas_xu_fixture - class _OnnxBackend: + class _OnnxBackend(ModelBackend): onnx_path = str(model_path) + @property + def hardware_label(self) -> str: + return "test-onnx" + + def __call__(self, inputs: object, **kwargs: object) -> object: + raise NotImplementedError("test stub: onnx-path resolution only") + huge_eps = 1e6 # whole input domain — adversarial example must exist. assessor = MarabouAssessor(epsilon=huge_eps, timeout_s=120.0) assessor.check_backend_compat(_OnnxBackend()) diff --git a/src/raitap/robustness/tests/test_e2e_real_data.py b/src/raitap/robustness/tests/test_e2e_real_data.py index 6b09f373..ba932383 100644 --- a/src/raitap/robustness/tests/test_e2e_real_data.py +++ b/src/raitap/robustness/tests/test_e2e_real_data.py @@ -24,10 +24,11 @@ import torch from omegaconf import OmegaConf +from raitap.configs import set_output_root from raitap.models.base_backend import ModelBackend from raitap.pipeline.orchestrator import run_without_tracking as _run_without_tracking from raitap.robustness import RobustnessAssessment, RobustnessResult -from raitap.testing import make_pixel_linear_classifier +from raitap.testing import make_app_config, make_pixel_linear_classifier from raitap.types import Capability if TYPE_CHECKING: @@ -82,6 +83,13 @@ def _make_robustness_config(tmp_path: Path, assessor_cfg: Any) -> AppConfig: _output_root=str(tmp_path), transparency={}, robustness={"pgd": assessor_cfg}, + # ``resolve_per_image_transform`` reads ``config.model`` directly and + # short-circuits to "no preprocessing" on ``None`` (these tests + # exercise the attack path, not preprocessing). ``config.data`` must + # stay a real object: ``resolve_forward_batch_size`` (pipeline) reads + # ``config.data.forward_batch_size`` directly too. + model=None, + data=SimpleNamespace(forward_batch_size=None), ), ) @@ -203,20 +211,27 @@ def test_pipeline_allows_robustness_only_runs(tmp_path: Path) -> None: targets = torch.tensor([0, 1]) model = make_pixel_linear_classifier(hw=8) - config = _make_robustness_config( - tmp_path, - OmegaConf.create( - { - "_target_": "raitap.robustness.TorchattacksAssessor", - "algorithm": "FGSM", - "constructor": {"eps": 0.05}, - "visualisers": [], - } - ), + # ``_run_without_tracking`` walks the full pipeline (forward pass, metrics, + # reporting gates) and reads several ``config.data``/``config.model`` + # fields directly, so this test needs a real ``AppConfig`` shape rather + # than the lighter ``_make_robustness_config`` namespace the other tests + # in this module use. + config = make_app_config( + experiment_name="test", + robustness={ + "pgd": OmegaConf.create( + { + "_target_": "raitap.robustness.TorchattacksAssessor", + "algorithm": "FGSM", + "constructor": {"eps": 0.05}, + "visualisers": [], + } + ) + }, + metrics=None, + reporting=None, ) - # Add the bits ``_run_without_tracking`` reads. - config.metrics = SimpleNamespace(_target_=None, num_classes=3) # type: ignore[attr-defined] - config.reporting = None # type: ignore[attr-defined] + set_output_root(config, tmp_path) raitap_model = cast("Model", SimpleNamespace(backend=_BackendStub(model))) data_stub = SimpleNamespace( diff --git a/src/raitap/robustness/tests/test_e2e_robustness_matrix.py b/src/raitap/robustness/tests/test_e2e_robustness_matrix.py index 6b53fd3b..28cf4234 100644 --- a/src/raitap/robustness/tests/test_e2e_robustness_matrix.py +++ b/src/raitap/robustness/tests/test_e2e_robustness_matrix.py @@ -98,6 +98,11 @@ def _make_robustness_config(tmp_path: Path, family: str, case: AssessorMatrixCas } ) }, + # ``resolve_per_image_transform`` reads ``config.model``/``config.data`` + # directly; explicit ``None`` (not a missing attribute) tells it there + # is no real preprocessing config to resolve. + model=None, + data=None, ), ) diff --git a/src/raitap/robustness/tests/test_marabou_parity.py b/src/raitap/robustness/tests/test_marabou_parity.py index e3d0ad2a..0d7e982d 100644 --- a/src/raitap/robustness/tests/test_marabou_parity.py +++ b/src/raitap/robustness/tests/test_marabou_parity.py @@ -20,6 +20,7 @@ import pytest import torch +from raitap.models.base_backend import ModelBackend from raitap.robustness.tests.test_e2e_marabou_acas_xu import ( _load_acas_xu_onnx, _run_onnx, @@ -109,9 +110,16 @@ def test_marabou_verdict_matches_direct_solve( ) # raitap path. - class _OnnxBackend: + class _OnnxBackend(ModelBackend): onnx_path = str(model_path) + @property + def hardware_label(self) -> str: + return "test-onnx" + + def __call__(self, inputs: object, **kwargs: object) -> object: + raise NotImplementedError("test stub: onnx-path resolution only") + assessor = MarabouAssessor(epsilon=eps, timeout_s=120.0) assessor.check_backend_compat(_OnnxBackend()) outcome = assessor.verify_sample( From 538585162cdb9640086e0c043eb554d3890ac103 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:47 +0200 Subject: [PATCH 07/15] refactor(data): direct config access in loaders + metadata (refs #352) --- src/raitap/data/data.py | 13 +- src/raitap/data/metadata.py | 18 ++- src/raitap/data/tests/test_data.py | 144 +++--------------- src/raitap/data/tests/test_data_class.py | 6 +- .../data/tests/test_detection_ragged.py | 3 +- src/raitap/data/tests/test_metadata.py | 48 ++++-- src/raitap/data/tests/test_text_metadata.py | 11 +- 7 files changed, 84 insertions(+), 159 deletions(-) diff --git a/src/raitap/data/data.py b/src/raitap/data/data.py index 3c03dde1..ffefa27a 100644 --- a/src/raitap/data/data.py +++ b/src/raitap/data/data.py @@ -123,7 +123,7 @@ def _load_data( ) inputs_cfg = _get_optional_config_value(cfg.data, "inputs") - model_cfg_for_text = getattr(cfg, "model", None) + model_cfg_for_text = cfg.model tokenizer_id = _get_optional_config_value(model_cfg_for_text, "tokenizer") if inputs_cfg is not None and tokenizer_id is not None: try: @@ -153,12 +153,9 @@ def _load_data( if resolved_preprocessing is not None: data_module = resolved_preprocessing.data_module else: - model_cfg = getattr(cfg, "model", None) - if model_cfg is None: - data_module = None - else: - resolved = resolve_preprocessing(model_cfg, cfg.data) - data_module = resolved.data_module + model_cfg = cfg.model + resolved = resolve_preprocessing(model_cfg, cfg.data) + data_module = resolved.data_module per_image_transform = module_as_per_image_callable(data_module) is_detection = self.task_kind is TaskKind.detection @@ -327,7 +324,7 @@ def _resolve_and_parse_labels( ) data_source = _get_optional_config_value(cfg.data, "source") - model = getattr(cfg, "model", None) + model = cfg.model class_names = _get_optional_config_value(model, "class_names") return parser.parse( diff --git a/src/raitap/data/metadata.py b/src/raitap/data/metadata.py index 36a18ddc..e34db1c2 100644 --- a/src/raitap/data/metadata.py +++ b/src/raitap/data/metadata.py @@ -2,13 +2,16 @@ from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from raitap.configs import cfg_to_dict from raitap.data.types import MODALITY_EXTENSIONS, InputModality from .samples import SAMPLE_SOURCES +if TYPE_CHECKING: + from raitap.configs.schema import AppConfig + _IMAGE_EXTENSIONS = MODALITY_EXTENSIONS[InputModality.image] _TABULAR_EXTENSIONS = MODALITY_EXTENSIONS[InputModality.tabular] @@ -22,10 +25,10 @@ class DataInputMetadata: metadata: dict[str, Any] | None = None -def infer_data_input_metadata(config: object, data: object) -> DataInputMetadata: +def infer_data_input_metadata(config: AppConfig, data: object) -> DataInputMetadata: explicit = getattr(data, "input_metadata", None) if explicit is None: - explicit = getattr(getattr(config, "data", None), "input_metadata", None) + explicit = config.data.input_metadata if explicit is not None: explicit_mapping = cfg_to_dict(explicit) kind = explicit_mapping.get("kind") @@ -51,10 +54,11 @@ def infer_data_input_metadata(config: object, data: object) -> DataInputMetadata return DataInputMetadata(kind="text", shape=shape, layout="TOKENS") # Fallback for callers with no recorded modality (e.g. direct/test use): - # sniff the source path as before. - source = str( - getattr(data, "source", None) or getattr(getattr(config, "data", None), "source", "") - ) + # sniff the source path as before. ``data`` is a loosely-typed runtime + # object here (tests may pass fakes without a ``source`` attribute), so + # that half stays defensive; ``config.data.source`` is a real schema + # field and always resolves. + source = str(getattr(data, "source", None) or config.data.source) if is_image_source(source): return DataInputMetadata(kind="image", shape=shape, layout="NCHW") if is_tabular_source(source): diff --git a/src/raitap/data/tests/test_data.py b/src/raitap/data/tests/test_data.py index 907778df..c4152426 100644 --- a/src/raitap/data/tests/test_data.py +++ b/src/raitap/data/tests/test_data.py @@ -47,6 +47,13 @@ def _write_csv(path: Path, rows: int = 4, cols: int = 5) -> None: path.write_text(f"{header}\n{data_rows}") +def _simple_cfg(source: str, name: str = "isic2018") -> AppConfig: + """A minimal real ``AppConfig`` (``model`` uses its schema defaults).""" + from raitap.configs.schema import AppConfig, DataConfig + + return AppConfig(data=DataConfig(source=source, name=name)) + + # --------------------------------------------------------------------------- # get_source_path # --------------------------------------------------------------------------- @@ -489,14 +496,7 @@ class TestLoadData: def test_local_image_file(self, tmp_path: Path) -> None: p = tmp_path / "img.png" _write_image(p) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(p), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(p)) data = Data(cfg) assert isinstance(data.tensor, torch.Tensor) assert data.tensor.shape == (1, 3, 32, 32) @@ -504,14 +504,7 @@ def test_local_image_file(self, tmp_path: Path) -> None: def test_local_image_directory(self, tmp_path: Path) -> None: for i in range(2): _write_image(tmp_path / f"img{i}.jpg") - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(tmp_path), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(tmp_path)) data = Data(cfg) assert isinstance(data.tensor, torch.Tensor) assert data.tensor.shape[0] == 2 @@ -519,14 +512,7 @@ def test_local_image_directory(self, tmp_path: Path) -> None: def test_local_csv_file(self, tmp_path: Path) -> None: p = tmp_path / "data.csv" _write_csv(p, rows=6, cols=3) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(p), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(p)) data = Data(cfg) assert isinstance(data.tensor, torch.Tensor) assert data.tensor.shape == (6, 3) @@ -534,14 +520,7 @@ def test_local_csv_file(self, tmp_path: Path) -> None: def test_local_tabular_directory(self, tmp_path: Path) -> None: _write_csv(tmp_path / "a.csv", rows=2, cols=3) _write_csv(tmp_path / "b.csv", rows=4, cols=3) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(tmp_path), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(tmp_path)) data = Data(cfg) assert isinstance(data.tensor, torch.Tensor) assert data.tensor.shape == (6, 3) @@ -549,40 +528,19 @@ def test_local_tabular_directory(self, tmp_path: Path) -> None: def test_mixed_directory_raises(self, tmp_path: Path) -> None: _write_image(tmp_path / "img.jpg") _write_csv(tmp_path / "data.csv") - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(tmp_path), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(tmp_path)) with pytest.raises(ValueError, match="both image and tabular"): Data(cfg) def test_empty_directory_raises(self, tmp_path: Path) -> None: - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(tmp_path), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(tmp_path)) with pytest.raises(FileNotFoundError, match="No supported files"): Data(cfg) def test_unknown_extension_raises(self, tmp_path: Path) -> None: p = tmp_path / "data.xyz" p.write_text("something") - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(p), "name": "isic2018"})}, - )(), - ) + cfg = _simple_cfg(str(p)) with pytest.raises(ValueError, match="Cannot infer data type"): Data(cfg) @@ -629,50 +587,19 @@ def forward(self, batch: torch.Tensor) -> torch.Tensor: def test_tabular_no_data_module_passes_through(self, tmp_path: Path) -> None: p = tmp_path / "rows.csv" _write_csv(p, rows=2, cols=3) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(p), "name": "tab"})}, - )(), - ) + cfg = _simple_cfg(str(p), name="tab") data = Data(cfg) torch.testing.assert_close(data.tensor, _load_tabular(p)) def test_invalid_source_raises(self) -> None: - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - { - "data": type( - "DataConfig", (), {"source": "/no/such/path/file.csv", "name": "isic2018"} - ) - }, - )(), - ) + cfg = _simple_cfg("/no/such/path/file.csv") with pytest.raises(ValueError, match="could not be resolved"): Data(cfg) def test_url_source_loads_csv_via_get_source_path(self, tmp_path: Path) -> None: p = tmp_path / "remote.csv" _write_csv(p, rows=2, cols=3) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - { - "data": type( - "DataConfig", - (), - {"source": "https://example.com/remote.csv", "name": "isic2018"}, - ) - }, - )(), - ) + cfg = _simple_cfg("https://example.com/remote.csv") with patch("raitap.data.data.get_source_path", return_value=p): data = Data(cfg) assert isinstance(data.tensor, torch.Tensor) @@ -681,20 +608,7 @@ def test_url_source_loads_csv_via_get_source_path(self, tmp_path: Path) -> None: def test_url_source_loads_image_via_get_source_path(self, tmp_path: Path) -> None: p = tmp_path / "remote.png" _write_image(p) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - { - "data": type( - "DataConfig", - (), - {"source": "https://example.com/remote.png", "name": "isic2018"}, - ) - }, - )(), - ) + cfg = _simple_cfg("https://example.com/remote.png") with patch("raitap.data.data.get_source_path", return_value=p): data = Data(cfg) assert isinstance(data.tensor, torch.Tensor) @@ -742,18 +656,7 @@ class TestDescribeData: def test_describe_data_includes_shape_dtype_and_sample_shape(self, tmp_path: Path) -> None: img_file = tmp_path / "img.jpg" _write_image(img_file, 32, 32) - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - { - "data": type( - "DataConfig", (), {"source": str(img_file), "name": "imagenet_samples"} - ) - }, - )(), - ) + cfg = _simple_cfg(str(img_file), name="imagenet_samples") data = Data(cfg) data.source = "/tmp/imagenet" @@ -769,14 +672,7 @@ def test_describe_data_includes_shape_dtype_and_sample_shape(self, tmp_path: Pat def test_describe_data_without_sample_shape_for_1d_input(self, tmp_path: Path) -> None: csv_file = tmp_path / "data.csv" csv_file.write_text("a\n1\n2\n3\n4\n5\n6\n7\n8") - cfg = cast( - "AppConfig", - type( - "AppConfig", - (), - {"data": type("DataConfig", (), {"source": str(csv_file), "name": "vector_data"})}, - )(), - ) + cfg = _simple_cfg(str(csv_file), name="vector_data") data = Data(cfg) info = data.describe() diff --git a/src/raitap/data/tests/test_data_class.py b/src/raitap/data/tests/test_data_class.py index 158a9e07..e8608203 100644 --- a/src/raitap/data/tests/test_data_class.py +++ b/src/raitap/data/tests/test_data_class.py @@ -57,7 +57,8 @@ def _make_config( source=source, name=name, labels=labels, - ) + ), + model=SimpleNamespace(class_names=None), ), ) @@ -561,7 +562,8 @@ def test_directory_source_derives_labels_from_layout(self, tmp_path: Path) -> No source=str(img_dir), name="test_dir", labels=DirectoryLabelsConfig(), - ) + ), + model=SimpleNamespace(class_names=None), ), ) data = Data(cfg) diff --git a/src/raitap/data/tests/test_detection_ragged.py b/src/raitap/data/tests/test_detection_ragged.py index bbad5a7b..8f058363 100644 --- a/src/raitap/data/tests/test_detection_ragged.py +++ b/src/raitap/data/tests/test_detection_ragged.py @@ -49,7 +49,8 @@ def _make_config(source: str, name: str = "test_det") -> AppConfig: source=source, name=name, labels=None, - ) + ), + model=SimpleNamespace(class_names=None), ), ) diff --git a/src/raitap/data/tests/test_metadata.py b/src/raitap/data/tests/test_metadata.py index b52cdd09..5d91ae1d 100644 --- a/src/raitap/data/tests/test_metadata.py +++ b/src/raitap/data/tests/test_metadata.py @@ -1,7 +1,7 @@ from __future__ import annotations from types import SimpleNamespace -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from raitap.data.metadata import ( DataInputMetadata, @@ -15,6 +15,8 @@ if TYPE_CHECKING: from pathlib import Path + from raitap.configs.schema import AppConfig + class _FakeTensor: shape = (2, 3, 8, 8) @@ -29,9 +31,14 @@ def _fake_data(modality: InputModality, source: str) -> SimpleNamespace: ) +def _config_without_explicit_metadata() -> AppConfig: + """A config whose ``data`` has no explicit ``input_metadata`` set.""" + return cast("AppConfig", SimpleNamespace(data=SimpleNamespace(input_metadata=None))) + + def test_recorded_image_modality_beats_unsniffable_source() -> None: data = _fake_data(InputModality.image, "opaque://not-a-path") - md = infer_data_input_metadata(SimpleNamespace(data=None), data) + md = infer_data_input_metadata(_config_without_explicit_metadata(), data) assert md.kind == "image" assert md.layout == "NCHW" assert md.shape == (2, 3, 8, 8) @@ -39,7 +46,7 @@ def test_recorded_image_modality_beats_unsniffable_source() -> None: def test_recorded_tabular_modality_beats_unsniffable_source() -> None: data = _fake_data(InputModality.tabular, "opaque://not-a-path") - md = infer_data_input_metadata(SimpleNamespace(data=None), data) + md = infer_data_input_metadata(_config_without_explicit_metadata(), data) assert md.kind == "tabular" assert md.layout == "(B,F)" @@ -48,22 +55,25 @@ def test_falls_back_to_source_sniff_when_modality_absent(tmp_path: Path) -> None csv = tmp_path / "rows.csv" csv.write_text("a,b\n1,2\n", encoding="utf-8") data = SimpleNamespace(input_metadata=None, tensor=_FakeTensor(), source=str(csv)) - md = infer_data_input_metadata(SimpleNamespace(data=None), data) + md = infer_data_input_metadata(_config_without_explicit_metadata(), data) assert md.kind == "tabular" assert md.layout == "(B,F)" def test_infer_data_input_metadata_prefers_explicit_config_metadata() -> None: - config = SimpleNamespace( - data=SimpleNamespace( - input_metadata={ - "kind": "text", - "shape": [2, 5], - "layout": "(B,T)", - "feature_names": ["token_1", "token_2"], - "tokenizer": "demo", - } - ) + config = cast( + "AppConfig", + SimpleNamespace( + data=SimpleNamespace( + input_metadata={ + "kind": "text", + "shape": [2, 5], + "layout": "(B,T)", + "feature_names": ["token_1", "token_2"], + "tokenizer": "demo", + } + ) + ), ) data = SimpleNamespace(tensor=SimpleNamespace(shape=(2, 3)), source="ignored.csv") @@ -87,7 +97,10 @@ def test_infer_data_input_metadata_prefers_explicit_config_metadata() -> None: def test_infer_data_input_metadata_detects_image_source(tmp_path: Path) -> None: image_path = tmp_path / "sample.png" image_path.write_bytes(b"not-an-image-for-this-test") - config = SimpleNamespace(data=SimpleNamespace(source=str(image_path))) + config = cast( + "AppConfig", + SimpleNamespace(data=SimpleNamespace(source=str(image_path), input_metadata=None)), + ) data = SimpleNamespace(tensor=SimpleNamespace(shape=(1, 3, 8, 8))) metadata = infer_data_input_metadata(config, data) @@ -98,7 +111,10 @@ def test_infer_data_input_metadata_detects_image_source(tmp_path: Path) -> None: def test_infer_data_input_metadata_detects_tabular_source(tmp_path: Path) -> None: csv_path = tmp_path / "features.csv" csv_path.write_text("a,b\n1,2\n") - config = SimpleNamespace(data=SimpleNamespace(source=str(csv_path))) + config = cast( + "AppConfig", + SimpleNamespace(data=SimpleNamespace(source=str(csv_path), input_metadata=None)), + ) data = SimpleNamespace(tensor=SimpleNamespace(shape=(1, 2))) metadata = infer_data_input_metadata(config, data) diff --git a/src/raitap/data/tests/test_text_metadata.py b/src/raitap/data/tests/test_text_metadata.py index 3c9a2f69..62f848bb 100644 --- a/src/raitap/data/tests/test_text_metadata.py +++ b/src/raitap/data/tests/test_text_metadata.py @@ -1,8 +1,16 @@ +from __future__ import annotations + +from types import SimpleNamespace +from typing import TYPE_CHECKING, cast + import torch from raitap.data.metadata import infer_data_input_metadata from raitap.data.types import InputModality +if TYPE_CHECKING: + from raitap.configs.schema import AppConfig + class _Data: input_metadata = None @@ -12,6 +20,7 @@ class _Data: def test_text_modality_infers_text_kind_and_tokens_layout() -> None: - md = infer_data_input_metadata(config=object(), data=_Data()) + config = cast("AppConfig", SimpleNamespace(data=SimpleNamespace(input_metadata=None))) + md = infer_data_input_metadata(config=config, data=_Data()) assert md.kind == "text" assert md.layout == "TOKENS" From 3327f974f8401c82f71c577787ccbeb49f904a29 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:48 +0200 Subject: [PATCH 08/15] refactor(reporting): direct config access, preserve None fallbacks (refs #352) --- src/raitap/reporting/builder.py | 22 +++++++++------- src/raitap/reporting/factory.py | 2 +- src/raitap/reporting/html_reporter.py | 11 +++++--- src/raitap/reporting/hydra_callback.py | 2 +- src/raitap/reporting/pdf_reporter.py | 7 +++-- src/raitap/reporting/tests/test_builder.py | 30 +++++++++++++++------- 6 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/raitap/reporting/builder.py b/src/raitap/reporting/builder.py index ce40af81..61a97d69 100644 --- a/src/raitap/reporting/builder.py +++ b/src/raitap/reporting/builder.py @@ -95,9 +95,9 @@ def build_report(config: AppConfig, outputs: RunOutputs) -> BuiltReport: kind="run", sections=tuple(sections), metadata={ - "experiment_name": getattr(config, "experiment_name", None), - "model_source": getattr(getattr(config, "model", None), "source", None), - "data_name": getattr(getattr(config, "data", None), "name", None), + "experiment_name": config.experiment_name, + "model_source": config.model.source, + "data_name": config.data.name, "selected_samples": [ _selected_sample_manifest_entry(sample) for sample in selected_samples ], @@ -178,9 +178,9 @@ def build_merged_report( kind="multirun", sections=ordered_sections, metadata={ - "experiment_name": getattr(config, "experiment_name", None), - "model_source": getattr(getattr(config, "model", None), "source", None), - "data_name": getattr(getattr(config, "data", None), "name", None), + "experiment_name": config.experiment_name, + "model_source": config.model.source, + "data_name": config.data.name, "children": [ {"job_label": job_label, "override_summary": override_summary} for job_label, override_summary, _manifest in child_manifests @@ -201,7 +201,7 @@ def _reproducibility_banner(config: AppConfig, outputs: RunOutputs) -> ReportSec only channel both the HTML and PDF reporters consume. Prepended so it renders first. See issue #251. """ - report = assess_reproducibility(outputs, getattr(config, "seed", None)) + report = assess_reproducibility(outputs, config.seed) caveat = reproducibility_caveat(report) if caveat is None: return None @@ -213,15 +213,19 @@ def _reproducibility_banner(config: AppConfig, outputs: RunOutputs) -> ReportSec def _manifest_filename(config: AppConfig) -> str: + # Both call sites (``build_report``, ``build_merged_report``) run only when + # ``reporting_enabled(config)`` is true, so ``config.reporting`` is guaranteed + # non-None here. + assert config.reporting is not None target = _reporting_target(config) - filename = str(getattr(getattr(config, "reporting", None), "filename", "report")) + filename = str(config.reporting.filename) if target in {"HTMLReporter", "raitap.reporting.HTMLReporter"}: return report_output_filename(filename, ".html") return report_output_filename(filename, ".pdf") def _reporting_target(config: AppConfig) -> str: - reporting = getattr(config, "reporting", None) + reporting = config.reporting return str(getattr(reporting, "_target_", "")) diff --git a/src/raitap/reporting/factory.py b/src/raitap/reporting/factory.py index bf6abbdc..993027ae 100644 --- a/src/raitap/reporting/factory.py +++ b/src/raitap/reporting/factory.py @@ -24,7 +24,7 @@ def reporting_enabled(config: AppConfig) -> bool: """Check if reporting is enabled in config.""" - reporting_cfg = getattr(config, "reporting", None) + reporting_cfg = config.reporting if reporting_cfg is None: return False target = getattr(reporting_cfg, "_target_", None) diff --git a/src/raitap/reporting/html_reporter.py b/src/raitap/reporting/html_reporter.py index 6d1f7bc5..d3dfa7ea 100644 --- a/src/raitap/reporting/html_reporter.py +++ b/src/raitap/reporting/html_reporter.py @@ -34,14 +34,17 @@ def generate( ) run_dir.mkdir(parents=True, exist_ok=True) + # HTMLReporter is only instantiated for reporting-enabled configs (see + # ``factory.create_report``), so ``self.config.reporting`` is non-None here. + assert self.config.reporting is not None html_path = run_dir / report_output_filename( - getattr(self.config.reporting, "filename", "report"), + self.config.reporting.filename, ".html", ) metadata = { - "experiment_name": getattr(self.config, "experiment_name", None), - "model_source": getattr(getattr(self.config, "model", None), "source", None), - "data_name": getattr(getattr(self.config, "data", None), "name", None), + "experiment_name": self.config.experiment_name, + "model_source": self.config.model.source, + "data_name": self.config.data.name, } view = build_view(sections, metadata, version=__about__.__version__) env = _jinja_environment() diff --git a/src/raitap/reporting/hydra_callback.py b/src/raitap/reporting/hydra_callback.py index b12aa01e..31840e44 100644 --- a/src/raitap/reporting/hydra_callback.py +++ b/src/raitap/reporting/hydra_callback.py @@ -22,7 +22,7 @@ def on_multirun_end(self, config: Any, **kwargs: Any) -> None: raitap_log.debug("Reporting disabled; skipping merged sweep report.") return - reporting_cfg = getattr(config, "reporting", None) + reporting_cfg = config.reporting if not bool(getattr(reporting_cfg, "multirun_report", True)): # The callback can still be registered for reporting-enabled configs; this flag # disables only the merged multirun report, not normal per-run reports. diff --git a/src/raitap/reporting/pdf_reporter.py b/src/raitap/reporting/pdf_reporter.py index cda37dee..842ce51d 100644 --- a/src/raitap/reporting/pdf_reporter.py +++ b/src/raitap/reporting/pdf_reporter.py @@ -230,8 +230,11 @@ def generate( ) run_dir.mkdir(parents=True, exist_ok=True) + # PDFReporter is only instantiated for reporting-enabled configs (see + # ``factory.create_report``), so ``self.config.reporting`` is non-None here. + assert self.config.reporting is not None filename = report_output_filename( - getattr(self.config.reporting, "filename", "report"), + self.config.reporting.filename, ".pdf", ) output_path = run_dir / filename @@ -270,7 +273,7 @@ def _add_cover_page(self, doc: Any, b: SimpleNamespace) -> None: ) ) layout.append_layout_element( - b.Paragraph(_pdf_display_text(f"Model: {getattr(self.config.model, 'source', 'N/A')}")) + b.Paragraph(_pdf_display_text(f"Model: {self.config.model.source or 'N/A'}")) ) layout.append_layout_element( b.Paragraph(_pdf_display_text(f"Dataset: {self.config.data.name}")) diff --git a/src/raitap/reporting/tests/test_builder.py b/src/raitap/reporting/tests/test_builder.py index 8eaea9b5..fc81e664 100644 --- a/src/raitap/reporting/tests/test_builder.py +++ b/src/raitap/reporting/tests/test_builder.py @@ -50,6 +50,7 @@ ) from raitap.robustness.visualisers import ImagePairVisualiser, PerturbationHeatmapVisualiser from raitap.robustness.visualisers.base_visualiser import BaseRobustnessVisualiser +from raitap.testing import make_app_config from raitap.transparency.contracts import ( DetectionBox, ExplanationOutputSpace, @@ -1950,13 +1951,15 @@ def _capture_report(cfg: Any, report: Any) -> Any: monkeypatch.setattr("raitap.reporting.hydra_callback.create_report", _capture_report) callback = ReportingSweepCallback() - config = OmegaConf.create( - { - "experiment_name": "demo", - "reporting": {"_target_": "PDFReporter", "filename": "report.pdf"}, - "hydra": {"sweep": {"dir": str(sweep_dir)}}, - } + config = make_app_config( + experiment_name="demo", + reporting={"_target_": "PDFReporter", "filename": "report.pdf"}, ) + # ``hydra`` is a Hydra runtime node, not an ``AppConfig`` field; attach it + # outside the struct schema, matching the shape Hydra hands callbacks. + OmegaConf.set_struct(config, False) + config.hydra = OmegaConf.create({"sweep": {"dir": str(sweep_dir)}}) + OmegaConf.set_struct(config, True) callback.on_multirun_end(config) report = captured["report"] @@ -1997,7 +2000,10 @@ def test_build_merged_report_deduplicates_identical_metrics_only(tmp_path: Path) ] report = build_merged_report( - AppConfig(experiment_name="demo"), + AppConfig( + experiment_name="demo", + reporting=ReportingConfig(_target_="PDFReporter", filename="report.pdf"), + ), sweep_dir=sweep_dir, child_manifests=child_manifests, skipped_children=[], @@ -2031,7 +2037,10 @@ def test_build_merged_report_preserves_present_section_order_with_aggregated( ] report = build_merged_report( - AppConfig(experiment_name="demo"), + AppConfig( + experiment_name="demo", + reporting=ReportingConfig(_target_="PDFReporter", filename="report.pdf"), + ), sweep_dir=sweep_dir, child_manifests=child_manifests, skipped_children=[], @@ -2059,7 +2068,10 @@ def test_build_merged_report_keeps_empty_metrics_groups(tmp_path: Path) -> None: ] report = build_merged_report( - AppConfig(experiment_name="demo"), + AppConfig( + experiment_name="demo", + reporting=ReportingConfig(_target_="PDFReporter", filename="report.pdf"), + ), sweep_dir=sweep_dir, child_manifests=child_manifests, skipped_children=[], From cf36c7a8b94f54006073a5544506061ad2c560b0 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:16:49 +0200 Subject: [PATCH 09/15] refactor(metrics): direct config access in metrics/task-family/config factories (refs #352) --- src/raitap/configs/adapter_factory.py | 4 ++-- src/raitap/metrics/factory.py | 2 +- src/raitap/task_families/classification.py | 3 ++- .../task_families/tests/test_classification_forward_mask.py | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/raitap/configs/adapter_factory.py b/src/raitap/configs/adapter_factory.py index f7d22340..94c6e002 100644 --- a/src/raitap/configs/adapter_factory.py +++ b/src/raitap/configs/adapter_factory.py @@ -413,8 +413,8 @@ def resolve_per_image_transform( if resolved_preprocessing is not None: return module_as_per_image_callable(resolved_preprocessing.data_module) - model_cfg = getattr(config, "model", None) - data_cfg = getattr(config, "data", None) + model_cfg = config.model + data_cfg = config.data if model_cfg is None or data_cfg is None: return None resolved = resolve_preprocessing(model_cfg, data_cfg) diff --git a/src/raitap/metrics/factory.py b/src/raitap/metrics/factory.py index 26d1ca70..ab31a8fb 100644 --- a/src/raitap/metrics/factory.py +++ b/src/raitap/metrics/factory.py @@ -32,7 +32,7 @@ def metrics_run_enabled(config: AppConfig) -> bool: """True when ``metrics`` is present and ``_target_`` is a non-empty string.""" - metrics_cfg = getattr(config, "metrics", None) + metrics_cfg = config.metrics if metrics_cfg is None: return False target = getattr(metrics_cfg, "_target_", None) diff --git a/src/raitap/task_families/classification.py b/src/raitap/task_families/classification.py index 140adfb9..229dfa14 100644 --- a/src/raitap/task_families/classification.py +++ b/src/raitap/task_families/classification.py @@ -173,7 +173,8 @@ def metrics_inputs(self, config: Any, forward_output: Any, labels: Any) -> Any: # distribution. No transform needed here. predictions_tensor = forward_output.as_classification() if ( - getattr(config.metrics, "num_classes", None) is None + config.metrics is not None + and config.metrics.num_classes is None and predictions_tensor.ndim == 2 and predictions_tensor.shape[1] >= 2 ): diff --git a/src/raitap/task_families/tests/test_classification_forward_mask.py b/src/raitap/task_families/tests/test_classification_forward_mask.py index 44769504..728d2d9e 100644 --- a/src/raitap/task_families/tests/test_classification_forward_mask.py +++ b/src/raitap/task_families/tests/test_classification_forward_mask.py @@ -2,7 +2,6 @@ from __future__ import annotations -from types import SimpleNamespace from typing import Any import torch @@ -10,6 +9,7 @@ from raitap.models.torch_backend import TorchBackend from raitap.pipeline.phases.forward_pass import forward_pass +from raitap.testing import make_app_config class _MaskAware(nn.Module): @@ -20,8 +20,8 @@ def forward( return ids.float() @ torch.ones((ids.shape[1], 2)) -def _config() -> Any: # a fake AppConfig-shaped object for forward_pass - return SimpleNamespace(run=SimpleNamespace(forward_batch_size=2), data=None) +def _config() -> Any: # a faithful AppConfig stand-in for forward_pass + return make_app_config(data={"forward_batch_size": 2}) def test_mask_threads_to_forward() -> None: From ffb8e200d216051502ded320045c541600cffdec Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:17:50 +0200 Subject: [PATCH 10/15] ci(infra): guard against defensive getattr on typed config/backend (refs #352) --- .github/workflows/code-quality.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/code-quality.yaml b/.github/workflows/code-quality.yaml index 4374b4a8..1110f98f 100644 --- a/.github/workflows/code-quality.yaml +++ b/.github/workflows/code-quality.yaml @@ -106,6 +106,13 @@ jobs: - name: Ruff Format run: uv run ruff format --check . + - name: Guard against defensive getattr on typed config/backend + run: | + if grep -rnE 'getattr\((config|cfg|self\.config|self\.cfg|backend|self\.backend|self\._backend)\s*,\s*"[a-zA-Z_][a-zA-Z0-9_]*"\s*,' src/raitap --include='*.py' | grep -v '/tests/'; then + echo "::error::Defensive getattr on typed config/backend. AppConfig fields and ModelBackend attrs are declared — read them directly (config.x / backend.x) so a missing field fails loud. See docs/contributor/adr.md." + exit 1 + fi + - name: Pyright run: uv run pyright From 2681450926702e22b6db62e9985d3a02c3e1611c Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:19:36 +0200 Subject: [PATCH 11/15] docs: ADR record for typed config access (refs #352) --- docs/contributor/adr.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/contributor/adr.md b/docs/contributor/adr.md index 4265be8a..5b784198 100644 --- a/docs/contributor/adr.md +++ b/docs/contributor/adr.md @@ -115,6 +115,16 @@ The *why* behind the layout in {doc}`architecture`. Each record has the same fou **Consequences:** Zero-config for plugin authors. Each plugin loads under its own try/except so one failure logs and continues. The version check reuses the plugin's pip `Requires-Dist` raitap specifier against `raitap.__about__`; a mismatch warns and skips. See {doc}`writing-a-plugin`. +## Config access is typed, not defensive + +**Context:** `AppConfig` is a Hydra structured config; every field has a declared type and default, and `ModelBackend` declares `provides` / `task_kind` / `device` on the ABC. Yet prod code read them through `getattr(config, "x", default)` at ~56 sites, driven by tests injecting `SimpleNamespace` fakes that lacked fields. A real missing-field bug returned a silent default instead of failing loud. + +**Options:** Keep defensive `getattr`; commit to direct typed access. + +**Decision:** Direct attribute access (`config.transparency`, `backend.provides`). Test configs build via `make_app_config` (a real struct-mode `DictConfig`, the same shape Hydra hands prod), never `SimpleNamespace`; backend fakes build via `make_fake_backend`. Where a value is *genuinely* optional (an assessor may run with no backend), handle the `None` explicitly at the call site — `backend.provides if backend is not None else frozenset()` — not with a `getattr` default that also masks a truly missing attribute. + +**Consequences:** A missing field fails loud instead of returning a silent default. A CI guard (`Quality Gate`) bans defensive `getattr` on config/backend in `src/**`. Genuinely dynamic reflection (`getattr(obj, variable)`) and explicit `None`-handling stay in scope of normal code — the ban targets only the string-literal-plus-default shape. The type system becomes load-bearing instead of decoration. + ## Cross-cutting principles The threads that recur across the records above: From e3753a605a90ef5ae5166f3cd42f588b1cdb3bca Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:37:16 +0200 Subject: [PATCH 12/15] fix(metrics): guard polymorphic num_classes read; direct data_cfg.input_metadata (refs #352) --- src/raitap/models/model.py | 2 +- src/raitap/task_families/classification.py | 17 ++++++++++-- .../tests/test_classification.py | 27 ++++++++++++++++++- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/src/raitap/models/model.py b/src/raitap/models/model.py index 7370b199..ce9c382b 100644 --- a/src/raitap/models/model.py +++ b/src/raitap/models/model.py @@ -199,7 +199,7 @@ def _resolve_shape_override(config: Any) -> tuple[int | None, ...] | None: data_cfg = config.data if data_cfg is None: return None - input_metadata = getattr(data_cfg, "input_metadata", None) + input_metadata = data_cfg.input_metadata if input_metadata is None: return None try: diff --git a/src/raitap/task_families/classification.py b/src/raitap/task_families/classification.py index 229dfa14..7860ba69 100644 --- a/src/raitap/task_families/classification.py +++ b/src/raitap/task_families/classification.py @@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, Any, cast +from omegaconf import OmegaConf + from raitap.task_families.base import ATTENTION_MASK_KEY from raitap.task_families.registry import task_family from raitap.types import TaskKind @@ -172,9 +174,20 @@ def metrics_inputs(self, config: Any, forward_output: Any, labels: Any) -> Any: # label: argmax is identical, and proba-consuming metrics now get a true # distribution. No transform needed here. predictions_tensor = forward_output.as_classification() + # ``num_classes`` is declared only on the multiclass typed subclass; + # binary/multilabel ``MetricsConfig`` variants have no such key. + # ``OmegaConf.select`` reads it without crashing on those variants, + # while ``throw_on_missing=True`` preserves the pre-existing crash + # when a multiclass config leaves ``num_classes`` as ``MISSING``. + num_classes = ( + OmegaConf.select(config.metrics, "num_classes", default=None, throw_on_missing=True) + if config.metrics is not None + else None + ) if ( - config.metrics is not None - and config.metrics.num_classes is None + num_classes is None + and config.metrics is not None + and "num_classes" in config.metrics and predictions_tensor.ndim == 2 and predictions_tensor.shape[1] >= 2 ): diff --git a/src/raitap/task_families/tests/test_classification.py b/src/raitap/task_families/tests/test_classification.py index 67a17ff8..061b5543 100644 --- a/src/raitap/task_families/tests/test_classification.py +++ b/src/raitap/task_families/tests/test_classification.py @@ -2,10 +2,13 @@ import pytest import torch +from omegaconf import OmegaConf -from raitap.pipeline.outputs import OutputKind +from raitap.configs.schema import BinaryClassificationMetricsConfig +from raitap.pipeline.outputs import ForwardOutput, OutputKind from raitap.task_families.classification import ClassificationFamily from raitap.task_families.registry import resolve_task_family +from raitap.testing import make_app_config from raitap.types import TaskKind @@ -65,3 +68,25 @@ def test_omitted_output_kind_defaults_to_softmax() -> None: assert rows is not None expected = float(torch.softmax(logits, dim=1).max()) assert abs(rows[0].confidence - expected) < 1e-6 + + +def test_metrics_inputs_with_binary_config_does_not_crash() -> None: + # Regression (issue #352): ``num_classes`` is declared only on + # ``MulticlassClassificationMetricsConfig``. A binary config's struct-mode + # DictConfig has no such key, so ``metrics_inputs`` must not read it via a + # bare attribute access — that raises ``ConfigAttributeError``. + family = ClassificationFamily() + config = make_app_config() + config.metrics = OmegaConf.structured(BinaryClassificationMetricsConfig(threshold=0.5)) + + predictions = torch.tensor([[0.2], [0.8], [0.6]]) + forward_output = ForwardOutput( + task_kind=TaskKind.classification, + batch_size=3, + payload=predictions, + output_kind=OutputKind.PROBABILITIES, + ) + + preds, targs = family.metrics_inputs(config, forward_output, torch.tensor([0, 1, 1])) + assert torch.equal(preds, predictions.squeeze(1)) + assert torch.equal(targs, torch.tensor([0, 1, 1])) From 5194760619c535e4cc53ae21cd41ee5a2c43cecc Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:44:34 +0200 Subject: [PATCH 13/15] fix(model): suppress invariant-override on TorchBackend.device narrowing (refs #352) --- src/raitap/models/torch_backend.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/raitap/models/torch_backend.py b/src/raitap/models/torch_backend.py index bdcb0867..77c74329 100644 --- a/src/raitap/models/torch_backend.py +++ b/src/raitap/models/torch_backend.py @@ -40,8 +40,10 @@ class TorchBackend(ModelBackend): """PyTorch-backed model runtime.""" # Narrows the ``ModelBackend.device`` ABC field (``torch.device | None``): - # a TorchBackend always resolves a concrete device in ``__init__``. - device: torch.device + # a TorchBackend always resolves a concrete device in ``__init__``. The + # attribute is mutable so pyright treats its type as invariant and flags the + # narrowing; it is safe here (device-less backends keep the base ``None``). + device: torch.device # pyright: ignore[reportIncompatibleVariableOverride] def __init__( self, From 9da1bbc1a658e9f0cea684a9fa6df076b110d3f2 Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 02:56:25 +0200 Subject: [PATCH 14/15] fix(model): device via widened helpers not attr-narrowing; restore polymorphic num_classes getattr (refs #352) --- src/raitap/models/base_backend.py | 6 +++--- src/raitap/models/tests/test_models.py | 1 + src/raitap/models/torch_backend.py | 12 ++++------- src/raitap/task_families/classification.py | 23 ++++++++-------------- 4 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/raitap/models/base_backend.py b/src/raitap/models/base_backend.py index 0b053612..02434f60 100644 --- a/src/raitap/models/base_backend.py +++ b/src/raitap/models/base_backend.py @@ -81,9 +81,9 @@ class ModelBackend(ABC): # Optional id->name table for the model's output classes (e.g. torchvision # detection ``weights.meta["categories"]``). ``None`` when unavailable. category_names: list[str] | None = None - # Runtime device for backends that place tensors (TorchBackend sets this in - # __init__). ``None`` for device-less backends (ONNX, tree). Declared here - # so callers read ``backend.device`` directly instead of defensive getattr. + # Runtime device for backends that place tensors (``TorchBackend`` sets this + # in ``__init__``). ``None`` for device-less backends (ONNX, tree). Declared + # here so callers read ``backend.device`` directly instead of via getattr. device: torch.device | None = None @classmethod diff --git a/src/raitap/models/tests/test_models.py b/src/raitap/models/tests/test_models.py index c8a521b3..9e91b8a5 100644 --- a/src/raitap/models/tests/test_models.py +++ b/src/raitap/models/tests/test_models.py @@ -538,6 +538,7 @@ def test_torch_gpu_uses_real_cuda_backend(self, saved_pth: Path) -> None: model = Model(cfg, allow_unsafe_pickle=True) assert isinstance(model.backend, TorchBackend) + assert model.backend.device is not None assert model.backend.device.type == "cuda" prepared_inputs = model.backend._prepare_inputs(torch.randn(2, 4)) diff --git a/src/raitap/models/torch_backend.py b/src/raitap/models/torch_backend.py index 77c74329..2a1fe81c 100644 --- a/src/raitap/models/torch_backend.py +++ b/src/raitap/models/torch_backend.py @@ -39,12 +39,6 @@ def _is_torchvision_detection_model(model: nn.Module) -> bool: class TorchBackend(ModelBackend): """PyTorch-backed model runtime.""" - # Narrows the ``ModelBackend.device`` ABC field (``torch.device | None``): - # a TorchBackend always resolves a concrete device in ``__init__``. The - # attribute is mutable so pyright treats its type as invariant and flags the - # narrowing; it is safe here (device-less backends keep the base ``None``). - device: torch.device # pyright: ignore[reportIncompatibleVariableOverride] - def __init__( self, model: nn.Module, @@ -126,7 +120,7 @@ def autograd_module(self) -> nn.Module: return self.model -def _move_tensors_to_device(value: Any, device: torch.device) -> Any: +def _move_tensors_to_device(value: Any, device: torch.device | None) -> Any: if isinstance(value, torch.Tensor): return value.to(device) if isinstance(value, list): @@ -138,7 +132,9 @@ def _move_tensors_to_device(value: Any, device: torch.device) -> Any: return value -def _torch_hardware_label(device: torch.device) -> str: +def _torch_hardware_label(device: torch.device | None) -> str: + if device is None: + return "CPU" label_by_type = { "cpu": "CPU", "cuda": "CUDA", diff --git a/src/raitap/task_families/classification.py b/src/raitap/task_families/classification.py index 7860ba69..86bffe5f 100644 --- a/src/raitap/task_families/classification.py +++ b/src/raitap/task_families/classification.py @@ -10,8 +10,6 @@ from typing import TYPE_CHECKING, Any, cast -from omegaconf import OmegaConf - from raitap.task_families.base import ATTENTION_MASK_KEY from raitap.task_families.registry import task_family from raitap.types import TaskKind @@ -174,20 +172,15 @@ def metrics_inputs(self, config: Any, forward_output: Any, labels: Any) -> Any: # label: argmax is identical, and proba-consuming metrics now get a true # distribution. No transform needed here. predictions_tensor = forward_output.as_classification() - # ``num_classes`` is declared only on the multiclass typed subclass; - # binary/multilabel ``MetricsConfig`` variants have no such key. - # ``OmegaConf.select`` reads it without crashing on those variants, - # while ``throw_on_missing=True`` preserves the pre-existing crash - # when a multiclass config leaves ``num_classes`` as ``MISSING``. - num_classes = ( - OmegaConf.select(config.metrics, "num_classes", default=None, throw_on_missing=True) - if config.metrics is not None - else None - ) + # ``num_classes`` is a genuinely polymorphic field — it lives only on the + # multiclass typed ``MetricsConfig`` subclass; binary/multilabel variants + # lack the key. ``getattr`` reads it across both the dataclass-instance + # and OmegaConf shapes the config takes at runtime, returning ``None`` + # where absent. This is not the typed-config defensiveness the #352 + # sweep removed, and the CI guard's regex does not match + # ``getattr(config.metrics, ...)`` (first arg must be bare ``config``). if ( - num_classes is None - and config.metrics is not None - and "num_classes" in config.metrics + getattr(config.metrics, "num_classes", None) is None and predictions_tensor.ndim == 2 and predictions_tensor.shape[1] >= 2 ): From 5a1e52036697adf7005b1d52e49476a4018d0a5a Mon Sep 17 00:00:00 2001 From: Stanislas Laurent Date: Tue, 7 Jul 2026 03:23:17 +0200 Subject: [PATCH 15/15] test(misc): promote integration-test config/backend fakes to real builders (refs #352) --- src/raitap/tests/test_memory_leaks.py | 26 ++-- .../test_reproducibility_orchestrator.py | 16 ++- src/raitap/tests/test_run_labels.py | 25 ++-- src/raitap/tests/test_run_main.py | 117 +++++++++--------- 4 files changed, 96 insertions(+), 88 deletions(-) diff --git a/src/raitap/tests/test_memory_leaks.py b/src/raitap/tests/test_memory_leaks.py index 7af82369..c2d0d995 100644 --- a/src/raitap/tests/test_memory_leaks.py +++ b/src/raitap/tests/test_memory_leaks.py @@ -4,7 +4,7 @@ from pathlib import Path from types import SimpleNamespace -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest.mock import MagicMock, patch import pytest @@ -23,6 +23,7 @@ from raitap.transparency.results import ExplanationResult if TYPE_CHECKING: + from raitap.configs.schema import AppConfig from raitap.data import Data from raitap.models import Model @@ -141,23 +142,28 @@ def test_run_without_tracking_forward_output_is_cpu_and_detached() -> None: """The forward output stored in RunOutputs must always be CPU-resident and detached.""" import torch.nn as nn + from raitap.models.base_backend import ModelBackend from raitap.pipeline.orchestrator import run_without_tracking as _run_without_tracking + from raitap.testing import make_app_config from raitap.types import TaskKind net = nn.Linear(4, 2, bias=False) - class _Backend: - hardware_label = "cpu" + class _Backend(ModelBackend): + @property + def hardware_label(self) -> str: + return "cpu" @property def task_kind(self) -> TaskKind: return TaskKind.classification - def _prepare_inputs(self, x: torch.Tensor) -> torch.Tensor: - return x + def _prepare_inputs(self, inputs: torch.Tensor) -> torch.Tensor: + return inputs - def __call__(self, x: torch.Tensor) -> torch.Tensor: - return net(x) + def __call__(self, inputs: torch.Tensor, **kwargs: Any) -> torch.Tensor: + del kwargs + return net(inputs) def autograd_module(self) -> torch.nn.Module: return net @@ -166,11 +172,7 @@ def autograd_module(self) -> torch.nn.Module: model = cast("Model", SimpleNamespace(backend=_Backend())) data = cast("Data", SimpleNamespace(tensor=torch.randn(2, 4), labels=None, sample_ids=None)) - # Transparency must be dict-like (supports .items() and [key] access) - config = MagicMock() - config.transparency = {"explainer1": SimpleNamespace()} - config.run.forward_batch_size = None - config.data.forward_batch_size = None + config = cast("AppConfig", make_app_config(transparency={"explainer1": {}})) fake_explanation = MagicMock() fake_explanation._visualise.return_value = [] diff --git a/src/raitap/tests/test_reproducibility_orchestrator.py b/src/raitap/tests/test_reproducibility_orchestrator.py index 73e07f01..f7dad152 100644 --- a/src/raitap/tests/test_reproducibility_orchestrator.py +++ b/src/raitap/tests/test_reproducibility_orchestrator.py @@ -16,6 +16,7 @@ from raitap.pipeline import orchestrator from raitap.pipeline.outputs import ForwardOutput, RunOutputs from raitap.reproducibility import REPRODUCIBILITY_FILENAME +from raitap.testing import make_app_config from raitap.types import TaskKind if TYPE_CHECKING: @@ -59,10 +60,9 @@ def _patch_pipeline(monkeypatch: MonkeyPatch, run_dir: Path, outputs: RunOutputs def test_stochastic_run_writes_md_and_warns_with_reporting_off( tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: - config = SimpleNamespace( + config = make_app_config( experiment_name="repro", - model=SimpleNamespace(source="resnet50"), - data=SimpleNamespace(), + model={"source": "resnet50"}, seed=None, ) _patch_pipeline(monkeypatch, tmp_path, _outputs(stochastic=True)) @@ -76,10 +76,9 @@ def test_stochastic_run_writes_md_and_warns_with_reporting_off( def test_deterministic_run_emits_nothing(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: - config = SimpleNamespace( + config = make_app_config( experiment_name="repro", - model=SimpleNamespace(source="resnet50"), - data=SimpleNamespace(), + model={"source": "resnet50"}, seed=None, ) _patch_pipeline(monkeypatch, tmp_path, _outputs(stochastic=False)) @@ -95,10 +94,9 @@ def test_deterministic_run_emits_nothing(tmp_path: Path, monkeypatch: MonkeyPatc def test_seed_config_pins_global_seed(tmp_path: Path, monkeypatch: MonkeyPatch) -> None: calls: list[int] = [] monkeypatch.setattr(orchestrator, "pin_global_seed", lambda s: calls.append(s)) - config = SimpleNamespace( + config = make_app_config( experiment_name="repro", - model=SimpleNamespace(source="resnet50"), - data=SimpleNamespace(), + model={"source": "resnet50"}, seed=99, ) _patch_pipeline(monkeypatch, tmp_path, _outputs(stochastic=False)) diff --git a/src/raitap/tests/test_run_labels.py b/src/raitap/tests/test_run_labels.py index 2e4f170e..6b8c34f8 100644 --- a/src/raitap/tests/test_run_labels.py +++ b/src/raitap/tests/test_run_labels.py @@ -1,7 +1,7 @@ from __future__ import annotations from types import SimpleNamespace -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast import pytest import torch @@ -11,24 +11,33 @@ # Bind the phase submodules as objects so ``monkeypatch.setattr`` patches the module # directly — the lazy adapter ``__getattr__`` on ``raitap.metrics`` / ``raitap.transparency`` # makes dotted-string monkeypatch targets unreliable once another test unbinds them. +from raitap.configs.schema import MulticlassClassificationMetricsConfig from raitap.metrics import phase as _metrics_phase from raitap.metrics import resolve_metric_targets +from raitap.models.base_backend import ModelBackend +from raitap.testing import make_app_config from raitap.transparency import phase as _transparency_phase from raitap.types import Capability, TaskKind if TYPE_CHECKING: from pathlib import Path + from omegaconf import DictConfig + from raitap.configs.schema import AppConfig from raitap.data import Data from raitap.models import Model from raitap.transparency.contracts import ExplainerAdapter -class _BackendStub: +class _BackendStub(ModelBackend): def __init__(self, output: torch.Tensor) -> None: self._output = output + @property + def hardware_label(self) -> str: + return "CPU" + @property def task_kind(self) -> TaskKind: return TaskKind.classification @@ -36,18 +45,18 @@ def task_kind(self) -> TaskKind: def _prepare_inputs(self, inputs: torch.Tensor) -> torch.Tensor: return inputs - def __call__(self, inputs: torch.Tensor) -> torch.Tensor: - del inputs + def __call__(self, inputs: torch.Tensor, **kwargs: Any) -> torch.Tensor: + del inputs, kwargs return self._output def autograd_module(self) -> torch.nn.Module: return torch.nn.Identity() -def _minimal_run_config() -> SimpleNamespace: - return SimpleNamespace( +def _minimal_run_config() -> DictConfig: + return make_app_config( transparency={"default": {}}, - metrics=SimpleNamespace(_target_="MulticlassClassificationMetrics", num_classes=3), + metrics=MulticlassClassificationMetricsConfig(num_classes=3), ) @@ -138,7 +147,7 @@ def fake_metrics( monkeypatch.setattr(_transparency_phase, "prepare_explainer", _fake_prepare) outputs = run_pipeline.run_without_tracking( - cast("AppConfig", cast("object", config)), + cast("AppConfig", config), cast("Model", cast("object", model)), cast("Data", cast("object", data)), ) diff --git a/src/raitap/tests/test_run_main.py b/src/raitap/tests/test_run_main.py index 4f1b5a47..dd2c18eb 100644 --- a/src/raitap/tests/test_run_main.py +++ b/src/raitap/tests/test_run_main.py @@ -2,11 +2,12 @@ import sys from types import SimpleNamespace -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast from unittest.mock import MagicMock import pytest import torch +from omegaconf import DictConfig, OmegaConf if TYPE_CHECKING: from pathlib import Path @@ -14,12 +15,12 @@ from _pytest.monkeypatch import MonkeyPatch from raitap.configs.schema import AppConfig - from raitap.models.base_backend import ModelBackend from raitap.transparency.contracts import ExplainerAdapter, InputSpec import raitap.pipeline.orchestrator as run_pipeline from raitap import pipeline as run_module +from raitap.configs.schema import MulticlassClassificationMetricsConfig from raitap.data.preprocessing import ResolvedPreprocessing from raitap.metrics import metrics_prediction_pair @@ -28,11 +29,13 @@ # another test unbinds them from ``sys.modules`` — so ``monkeypatch.setattr`` with a # dotted string path is unreliable. Patching the module object sidesteps that. from raitap.metrics import phase as _metrics_phase +from raitap.models.base_backend import ModelBackend from raitap.pipeline import __main__ as run_entry from raitap.pipeline import extract_primary_tensor from raitap.pipeline.outputs import ForwardOutput as _ForwardOutput from raitap.pipeline.phases import forward_pass as run_pipeline_forward from raitap.robustness.report import RobustnessPhaseResult +from raitap.testing import make_app_config from raitap.tracking import BaseTracker from raitap.transparency import phase as _transparency_phase from raitap.transparency.report import TransparencyPhaseResult @@ -97,7 +100,7 @@ def _fake_run_outputs( ) -class _BackendStub: +class _BackendStub(ModelBackend): def __init__(self, model: torch.nn.Module) -> None: self._model = model @@ -115,21 +118,22 @@ def _prepare_inputs(self, inputs: torch.Tensor) -> torch.Tensor: def _prepare_kwargs(self, kwargs: dict[str, object]) -> dict[str, object]: return kwargs - def __call__(self, inputs: torch.Tensor) -> torch.Tensor: + def __call__(self, inputs: torch.Tensor, **kwargs: Any) -> torch.Tensor: + del kwargs return self._model(inputs) def autograd_module(self) -> torch.nn.Module: return self._model -def _run_config(**kwargs: object) -> SimpleNamespace: +def _run_config(**overrides: object) -> DictConfig: defaults: dict[str, object] = { - "model": SimpleNamespace(source="resnet50"), - "data": SimpleNamespace(preprocessing=None), + "model": {"source": "resnet50"}, + "data": {"preprocessing": None}, "experiment_name": "test", } - defaults.update(kwargs) - return SimpleNamespace(**defaults) + defaults.update(overrides) + return make_app_config(**defaults) def test_extract_primary_tensor_tensor() -> None: @@ -176,10 +180,14 @@ def test_metrics_prediction_pair_multiclass_and_vector() -> None: def test_forward_primary_tensor_batches_backend_calls() -> None: - class _RecordingBackend: + class _RecordingBackend(ModelBackend): def __init__(self) -> None: self.prepared_batch_sizes: list[int] = [] + @property + def hardware_label(self) -> str: + return "CPU" + @property def task_kind(self) -> _TaskKind: return _TaskKind.classification @@ -188,18 +196,16 @@ def _prepare_inputs(self, inputs: torch.Tensor) -> torch.Tensor: self.prepared_batch_sizes.append(int(inputs.shape[0])) return inputs - def __call__(self, inputs: torch.Tensor) -> torch.Tensor: + def __call__(self, inputs: torch.Tensor, **kwargs: Any) -> torch.Tensor: + del kwargs return inputs.sum(dim=1, keepdim=True) - config = SimpleNamespace( - run=SimpleNamespace(forward_batch_size=2), - data=SimpleNamespace(), - ) + config = make_app_config(data={"forward_batch_size": 2}) backend = _RecordingBackend() inputs = torch.arange(10, dtype=torch.float32).reshape(5, 2) output = run_pipeline_forward.forward_pass( - cast("AppConfig", cast("object", config)), + cast("AppConfig", config), backend, inputs, ) @@ -453,12 +459,11 @@ class _HydraConfig: "raitap.utils.console.metrics_run_enabled", lambda _cfg: False, raising=False ) - config = SimpleNamespace( + config = make_app_config( experiment_name="demo", - model=SimpleNamespace(source="resnet50"), - data=SimpleNamespace(name="imagenet_samples"), + model={"source": "resnet50"}, + data={"name": "imagenet_samples"}, transparency={"captum_ig": {}}, - _output_root="fallback-output", ) model = SimpleNamespace(backend=_BackendStub(torch.nn.Identity())) @@ -526,9 +531,9 @@ def test_run_resolves_preprocessing_once_for_model_and_data(monkeypatch: MonkeyP monkeypatch.setattr(run_pipeline, "print_summary", lambda _cfg, _model: None) monkeypatch.setattr(BaseTracker, "create_tracker", tracker_factory) - config = SimpleNamespace( - model=SimpleNamespace(source="resnet50"), - data=SimpleNamespace(preprocessing="model-bundled"), + config = make_app_config( + model={"source": "resnet50"}, + data={"preprocessing": "model-bundled"}, tracking=None, experiment_name="test", ) @@ -589,7 +594,7 @@ def test_run_invalid_report_sample_selection_fails_before_pipeline_work( monkeypatch.setattr(BaseTracker, "create_tracker", tracker_factory) config = _run_config( - reporting=SimpleNamespace(_target_="PDFReporter", sample_selection=["missing.png"]), + reporting={"_target_": "PDFReporter", "sample_selection": ["missing.png"]}, tracking=None, ) @@ -636,7 +641,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: monkeypatch.setattr(BaseTracker, "create_tracker", lambda _cfg: _TrackerContext()) config = _run_config( - tracking=SimpleNamespace(_target_="MLFlowTracker", log_model=True), + tracking={"_target_": "MLFlowTracker", "log_model": True}, ) run_module.run(config) # type: ignore[arg-type] @@ -681,7 +686,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: monkeypatch.setattr(BaseTracker, "create_tracker", lambda _cfg: _TrackerContext()) config = _run_config( - tracking=SimpleNamespace(_target_="MLFlowTracker", log_model=False), + tracking={"_target_": "MLFlowTracker", "log_model": False}, ) run_module.run(config) # type: ignore[arg-type] @@ -724,7 +729,7 @@ def __exit__(self, exc_type: object, exc: object, tb: object) -> bool: monkeypatch.setattr(BaseTracker, "create_tracker", lambda _cfg: _TrackerContext()) config = _run_config( - tracking=SimpleNamespace(_target_="MLFlowTracker", log_model=False), + tracking={"_target_": "MLFlowTracker", "log_model": False}, ) run_module.run(config) # type: ignore[arg-type] @@ -757,7 +762,7 @@ def test_run_with_tracking_config_but_no_target_skips_tracking(monkeypatch: Monk monkeypatch.setattr(BaseTracker, "create_tracker", tracker_factory) # tracking config exists but _target_ is None or empty - config = _run_config(tracking=SimpleNamespace(_target_=None)) + config = _run_config(tracking={"_target_": ""}) result = run_module.run(config) # type: ignore[arg-type] assert result is fake_output @@ -788,11 +793,15 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: data = SimpleNamespace(tensor=torch.randn(2, 4), sample_ids=None, labels=None) # A real (non-empty) _target_ is exactly what metrics_run_enabled checks; no # transparency/robustness configured -> genuine metrics-only run. - config = SimpleNamespace( + config = make_app_config( transparency={}, robustness={}, - metrics=SimpleNamespace(_target_="MulticlassClassificationMetrics", num_classes=None), + metrics={"_target_": "MulticlassClassificationMetrics"}, ) + # ``num_classes`` deliberately absent here (unlike the typed subclass, where + # it is mandatory): exercises the auto-infer branch in + # ``ClassificationFamily.metrics_inputs``, which assigns it onto this node. + OmegaConf.set_struct(config.metrics, False) monkeypatch.setattr(_metrics_phase, "Metrics", lambda _c, _p, _t: SimpleNamespace()) outputs = run_pipeline.run_without_tracking(config, model, data) # type: ignore[arg-type] @@ -816,10 +825,11 @@ def _fake_metrics(cfg: object, preds: torch.Tensor, targs: torch.Tensor) -> obje metrics_calls.append((cfg, preds, targs)) return SimpleNamespace() - config = SimpleNamespace( + config = make_app_config( transparency={"one": {}}, - metrics=SimpleNamespace(_target_="MulticlassClassificationMetrics", num_classes=None), + metrics={"_target_": "MulticlassClassificationMetrics"}, ) + OmegaConf.set_struct(config.metrics, False) _patch_prepare_explainer( monkeypatch, explainer=_CapturingExplainer(explanation, {}), @@ -849,9 +859,9 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: data = SimpleNamespace(tensor=torch.randn(1, 4), sample_ids=None, labels=None) explanation = _FakeExplainerResult("exp") - config = SimpleNamespace( + config = make_app_config( transparency={"one": {}}, - metrics=SimpleNamespace(_target_="MulticlassClassificationMetrics", num_classes=10), + metrics=MulticlassClassificationMetricsConfig(num_classes=10), ) _patch_prepare_explainer( monkeypatch, @@ -875,10 +885,7 @@ def test_run_without_tracking_passes_sample_names_to_explanation(monkeypatch: Mo ) explanation = _FakeExplainerResult("exp") - config = SimpleNamespace( - transparency={"one": {}}, - metrics=SimpleNamespace(num_classes=None), - ) + config = make_app_config(transparency={"one": {}}) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) captured = _patch_prepare_explainer( monkeypatch, @@ -909,10 +916,7 @@ def test_run_without_tracking_passes_resolved_preprocessing_to_explanation( description="test", ) - config = SimpleNamespace( - transparency={"one": {}}, - metrics=SimpleNamespace(num_classes=None), - ) + config = make_app_config(transparency={"one": {}}) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) captured = _patch_prepare_explainer( monkeypatch, @@ -946,10 +950,9 @@ def test_run_without_tracking_threads_sample_ids_and_image_metadata_to_explanati ) explanation = _FakeExplainerResult("exp") - config = SimpleNamespace( - data=SimpleNamespace(source=str(image_path)), + config = make_app_config( + data={"source": str(image_path)}, transparency={"one": {}}, - metrics=SimpleNamespace(num_classes=None), ) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) captured = _patch_prepare_explainer( @@ -983,10 +986,9 @@ def test_run_without_tracking_threads_tabular_metadata_to_explanation( ) explanation = _FakeExplainerResult("exp") - config = SimpleNamespace( - data=SimpleNamespace(source=str(csv_path)), + config = make_app_config( + data={"source": str(csv_path)}, transparency={"one": {}}, - metrics=SimpleNamespace(num_classes=None), ) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) captured = _patch_prepare_explainer( @@ -1023,10 +1025,9 @@ def test_run_without_tracking_passes_none_when_inference_cant_determine_kind( ) explanation = _FakeExplainerResult("exp") - config = SimpleNamespace( - data=SimpleNamespace(source=str(empty_dir)), + config = make_app_config( + data={"source": str(empty_dir)}, transparency={"one": {}}, - metrics=SimpleNamespace(num_classes=None), ) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) captured = _patch_prepare_explainer( @@ -1059,13 +1060,12 @@ def test_run_without_tracking_preserves_layout_only_input_metadata( ) explanation = _FakeExplainerResult("exp") - config = SimpleNamespace( - data=SimpleNamespace( - source=str(empty_dir), - input_metadata={"layout": "(B,T,C)"}, - ), + config = make_app_config( + data={ + "source": str(empty_dir), + "input_metadata": {"layout": "(B,T,C)"}, + }, transparency={"one": {}}, - metrics=SimpleNamespace(num_classes=None), ) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) captured = _patch_prepare_explainer( @@ -1094,9 +1094,8 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: explanation = _FakeExplainerResult("exp") captured_kwargs: dict[str, object] = {} - config = SimpleNamespace( + config = make_app_config( transparency={"one": {"call": {"target": "auto_pred"}}}, - metrics=SimpleNamespace(num_classes=None), ) monkeypatch.setattr(_metrics_phase, "metrics_run_enabled", lambda _cfg: False) # Real ``ClassificationFamily.explain`` runs against this prepared object and