From 70c2073a87631e0b1a799f17443ce6149fb7d79f Mon Sep 17 00:00:00 2001 From: Robin Gutzen Date: Wed, 9 Sep 2026 12:38:52 +0200 Subject: [PATCH] fix: re-raise on test results CSV save failure save_results() caught exceptions from writing test_outputs.csv and only logged them, without re-raising. This let the test rule's shell command exit 0 with a missing or corrupted (e.g. empty/truncated) test_outputs.csv on disk. Snakemake then treated the output as successfully produced and never reran the job, so the corrupted CSV was silently picked up by downstream processing (process_single_test), failing later with a confusing EmptyDataError. Mirrors the existing error handling in the response-tensor save path below, which already re-raises so Snakemake can detect the failure via a missing output file. --- dynvision/runtime/test_model.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dynvision/runtime/test_model.py b/dynvision/runtime/test_model.py index b2cea20..600d211 100644 --- a/dynvision/runtime/test_model.py +++ b/dynvision/runtime/test_model.py @@ -290,6 +290,11 @@ def save_results(self, model: pl.LightningModule, precision: int = 16) -> None: del results_df except Exception as e: logger.error(f"Failed to save test results: {e}") + # Don't silently swallow the error - a partial/corrupt or missing + # test_outputs.csv must fail the job so Snakemake reruns it, + # instead of being picked up as valid input by downstream + # processing steps (mirrors the response-saving error handling below). + raise # Save model responses (tensors) try: