Fixes

scripts/run_benchmark/param_sweep/moscot_params.yaml

@@ -0,0 +1,61 @@
+# Parameter sweep for the moscot (MOSCOT, OTT-jax fused Gromov-Wasserstein optimal-
+# transport reference mapping) cell-type-annotation method. GPU method.
+#
+# Committed source of truth for run_test_moscot_nebius.sh, read from GitHub via a raw
+# URL (the Nebius compute env pulls the repo but cannot see the launch host's files).
+#
+# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting
+# (src/workflows/run_benchmark/main.nf). For every method the workflow builds:
+#   * one "default" variant using the `default:` args below, and
+#   * one extra variant per value in each `sweep:` list, with that ONE arg overridden.
+# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not
+# a full grid), so total moscot variants = 1 default + sum(sweep list lengths) = 9.
+#
+# See src/methods_cell_type_annotation/moscot/NOTES.md ("Optimization / tuning") for
+# the tiers, tool defaults, and rationale.
+#
+# Only ALREADY-EXPOSED args are swept (alpha, epsilon, mapping_mode), so this is
+# SUBMITTABLE against the build/main container as-is (no rebuild). Un-exposed knobs
+# (n_comps, scale_cost, initializer, convergence thresholds) are documented in NOTES
+# as future work.
+#
+# !!! SMALL-DATA CLAMP — why tau and rank are NOT swept here !!!
+#   script.py:62-66 forces rank=-1 and tau=1.0 whenever the spatial AnnData has
+#   < 10000 cells. The test resource (mouse_brain_combined) has n_obs=306, so on this
+#   run rank and tau are OVERRIDDEN regardless of any value passed -- sweeping them
+#   would only add identical no-op variants. They deviate from moscot's tool defaults
+#   (tau 0.3 vs 1.0, rank 500 vs -1) ONLY for >10k-cell datasets, and are documented in
+#   NOTES.md as the Tier-1 knobs for a real (>10k) dataset sweep.
+parameters:
+  moscot:
+    # Baseline == the component's shipped defaults (config.vsh.yaml).
+    default:
+      alpha: 0.8
+      epsilon: 0.01
+      tau: 0.3          # clamped to 1.0 on this <10k test data (script.py:65-66)
+      rank: 500         # clamped to -1  on this <10k test data (script.py:63-64)
+      batch_size: 1024  # pure memory knob (online GW-cost batching); held fixed
+      mapping_mode: max
+    sweep:
+      # ---- Tier 1: FGW interpolation weight (quality) ----
+      # alpha in (0,1] balances the quadratic Gromov-Wasserstein term (intra-domain
+      # structure: SC 30-dim PCA vs spatial local-pca coords) against the linear term
+      # (shared-gene expression). alpha->1 = pure GW/structure; alpha->0 = pure
+      # expression matching. Config default 0.8 DEVIATES from moscot's tool default 0.5
+      # (structure-heavy) -> promoted to a sweep axis. 0.8 omitted (the default variant
+      # covers it); range straddles the tool default (0.5) up through 0.9.
+      alpha: [0.5, 0.7, 0.9]
+      # ---- Tier 1: entropic regularization (quality vs. convergence) ----
+      # epsilon sets the entropy/stochasticity of the transport plan: higher = blurrier
+      # map, faster & more stable convergence; lower = sharper, more confident mapping
+      # but harder to converge. Config default 0.01 == moscot's tool default (not a
+      # deviation), swept as a legitimate Tier-1 quality axis. 0.01 omitted; two lower
+      # (sharper) + two higher (diffuse) probe an order of magnitude each way.
+      epsilon: [0.001, 0.005, 0.05, 0.1]
+      # ---- Tier 2: label read-out from the transport plan ----
+      # mapping_mode: how a per-cell label is pulled from the plan. 'max' (default) takes
+      # the label of the single highest-mass source cell; 'sum' aggregates the plan mass
+      # per cell type first, then takes the argmax (smoother, uses the full coupling).
+      # moscot has NO tool default here (required arg); 'sum' is the one meaningful
+      # non-default value.
+      mapping_mode: [sum]

scripts/run_benchmark/param_sweep/rctd_params.yaml

@@ -0,0 +1,52 @@
+# Parameter sweep for the rctd (spacexr RCTD, R) cell type annotation method.
+# Committed source of truth, read at runtime from GitHub via a raw URL by
+# run_test_rctd_nebius.sh (the Nebius compute env pulls the repo but cannot see the
+# launch host's local files).
+#
+# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting
+# (src/workflows/run_benchmark/main.nf). For every method the workflow builds:
+#   * one "default" variant using the `default:` args below, and
+#   * one extra variant per value in each `sweep:` list, with that ONE arg overridden.
+# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not
+# a full grid), so total rctd variants = 1 default + sum(sweep list lengths) = 13.
+#
+# See src/methods_cell_type_annotation/rctd/NOTES.md ("Optimization / tuning") for the
+# tiers and rationale.
+#
+# All six args are create.RCTD DE-gene / UMI thresholds. Every config default is a
+# DELIBERATELY RELAXED value (walked away from the spacexr default) because iST panels
+# are small (~100-500 genes) and low-count; the stock spacexr defaults strand too few DE
+# genes and abort with "fewer than 10 regression differentially expressed genes". Each
+# sweep list therefore walks that ONE arg back TOWARD its spacexr default (the meaningful
+# direction, since the relaxed default already sits at the permissive extreme, and the
+# default variant already covers the relaxed point).
+#
+# CAVEAT: on a very small panel, the high-threshold variants (values at/near the spacexr
+# defaults) can drop back under the 10-DE-gene floor and legitimately fail — that failure
+# boundary is part of what this sweep measures.
+parameters:
+  rctd:
+    # Baseline == the component's shipped (relaxed-for-iST) defaults.
+    default:
+      gene_cutoff: 0.0
+      fc_cutoff: 0.1
+      gene_cutoff_reg: 0.0
+      fc_cutoff_reg: 0.1
+      umi_min: 20
+      umi_min_sigma: 20
+    sweep:
+      # ---- Tier 1: DE-gene fold-change thresholds (sharpest levers) ----
+      # fc_cutoff: min log-FC for a PLATFORM-EFFECT DE gene. Relaxed 0.1 -> spacexr 0.5.
+      fc_cutoff: [0.25, 0.5]
+      # fc_cutoff_reg: min log-FC for a REGRESSION DE gene. Relaxed 0.1 -> spacexr 0.75.
+      fc_cutoff_reg: [0.4, 0.75]
+      # ---- Tier 1: DE-gene expression-mean thresholds (companion filters) ----
+      # gene_cutoff: min normalized mean expr, platform-effect. 0.0 -> spacexr 0.000125.
+      gene_cutoff: [0.0000625, 0.000125]
+      # gene_cutoff_reg: min normalized mean expr, regression. 0.0 -> spacexr 0.0002.
+      gene_cutoff_reg: [0.0001, 0.0002]
+      # ---- Tier 1: UMI thresholds (which cells get annotated / fit variance) ----
+      # umi_min: min total UMI per spatial cell to annotate it. 20 -> spacexr 100.
+      umi_min: [50, 100]
+      # umi_min_sigma: min UMI for cells fitting the platform-effect variance. 20 -> 300.
+      umi_min_sigma: [100, 300]

scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml

@@ -0,0 +1,45 @@
+# Parameter sweep for the resolvi_correction (resolVI, scvi-tools) expression-correction method.
+# Committed source of truth read by run_test_resolvi_correction_nebius.sh from GitHub via a
+# raw URL (the Nebius compute env pulls the repo but cannot see the launch host's local files).
+#
+# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting
+# (src/workflows/run_benchmark/main.nf). For every method the workflow builds:
+#   * one "default" variant using the `default:` args below, and
+#   * one extra variant per value in each `sweep:` list, with that ONE arg overridden.
+# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not a full
+# grid), so total resolvi_correction variants = 1 default + sum(sweep list lengths) = 5.
+#
+# SUBMITTABLE-NOW CONSTRAINT: the Nebius launch runs `--revision build/main`, i.e. the
+# build/main container as it exists today. It only honours arguments ALREADY EXPOSED in
+# src/methods_expression_correction/resolvi_correction/config.vsh.yaml. That container exposes
+# exactly four args -- celltype_key, n_hidden, encode_covariates, downsample_counts -- so the
+# sweep touches ONLY those (celltype_key is an input mapping, not a tuning knob). The high-value
+# resolVI levers num_samples (posterior samples), max_epochs, and mixture_k are HARD-CODED in
+# script.py, NOT exposed; sweeping them would need a config change + container rebuild, so they
+# are documented in NOTES.md ("Optimization / tuning") as future work and are NOT swept here.
+#
+# See src/methods_expression_correction/resolvi_correction/NOTES.md for tiers, rationale, and
+# the verified upstream defaults. Trim or widen the lists to taste.
+parameters:
+  resolvi_correction:
+    # Baseline == the component's shipped defaults, which (verified) all match the scvi-tools
+    # RESOLVI upstream defaults (n_hidden=32, encode_covariates=False, downsample_counts=True).
+    default:
+      celltype_key: cell_type
+      n_hidden: 32
+      encode_covariates: false
+      downsample_counts: true
+    sweep:
+      # ---- Tier 2: quality / capacity trade-offs (the only exposed knobs) ----
+      # n_hidden: width of the VAE hidden layers. 32 is the scvi RESOLVI default (small, fast);
+      # raising it gives the encoder/decoder more capacity to model diffusion + background at a
+      # time cost. script.py's own grid-search note lists 64 and 128 (default 32 omitted).
+      n_hidden: [64, 128]
+      # encode_covariates: BOOLEAN, default false. Flip to true to also feed batch/covariates
+      # through the encoder (helps when integrating across batches). Forwarded to the RESOLVI
+      # module via **model_kwargs; only one meaningful non-default value exists.
+      encode_covariates: [true]
+      # downsample_counts: BOOLEAN, default true (subsamples per-cell counts to equalise depth
+      # during training). Flip to false to train on full observed counts; only one meaningful
+      # non-default value exists.
+      downsample_counts: [false]

scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+# Nebius test run: all default methods + MOSCOT (moscot) cell-type annotation, with a
+# parameter sweep over the optimization levers identified for moscot.
+# See src/methods_cell_type_annotation/moscot/NOTES.md ("Optimization / tuning").
+#
+# moscot is an OTT-jax fused Gromov-Wasserstein optimal-transport mapping method that
+# runs on GPU (jax[cuda12], gpuh100), hence the `gpu` label and the Nebius GPU compute
+# env (mirrors run_gpu_nebius.sh).
+#
+# The annotation stage keeps its default method `tacco` alongside `moscot` so the run
+# has a baseline to compare the moscot variants against. Every other stage is left on
+# its single default (the swept method must be the only non-default thing).
+#
+# SUBMITTABLE-NOW: only already-exposed args are swept (alpha, epsilon, mapping_mode),
+# so this launches against the build/main container with no rebuild. tau and rank are
+# NOT swept because script.py clamps them to tool defaults on the <10k-cell test data
+# (see moscot_params.yaml + NOTES.md).
+#
+# PARAMS-FILE CAVEAT (why this reads from GitHub):
+#   `tw launch --params-file` is read client-side, but `method_parameters_yaml`
+#   is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow
+#   file()). A local /tmp path does not exist there, and /scratch (where results
+#   publish) is READ-ONLY from the launch host. file() does stage http(s):// though,
+#   and this repo is public, so we keep the sweep in a COMMITTED file
+#   (scripts/run_benchmark/param_sweep/moscot_params.yaml) and read it from GitHub
+#   via its raw URL. => the params file must be committed AND PUSHED to
+#   $params_branch before launching (edit the file there, not here, to change the
+#   sweep). This is independent of --revision below, which selects the pipeline CODE.
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing
+# Results publish to /scratch — created and written by the cloud compute env, so
+# the launcher does NOT create it here (it is read-only from the launch host).
+publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_moscot"
+
+# The sweep lives in a committed file, read from GitHub at runtime. $params_branch
+# defaults to the branch you are on; the file must be pushed there on GitHub. (This
+# is independent of --revision below, which selects the pipeline CODE to run.)
+params_repo="openproblems-bio/task_ist_preprocessing"
+params_branch="$(git rev-parse --abbrev-ref HEAD)"
+params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/moscot_params.yaml"
+
+cat > /tmp/params_settings.yaml << HERE
+default_methods:
+  - custom_segmentation
+  - basic_transcript_assignment
+  - basic_count_aggregation
+  - basic_qc_filter
+  - alpha_shapes
+  - normalize_by_volume
+  - tacco
+  - no_correction
+segmentation_methods:
+  - custom_segmentation
+transcript_assignment_methods:
+  - basic_transcript_assignment
+count_aggregation_methods:
+  - basic_count_aggregation
+qc_filtering_methods:
+  - basic_qc_filter
+volume_calculation_methods:
+  - alpha_shapes
+normalization_methods:
+  - normalize_by_volume
+celltype_annotation_methods:
+  - tacco
+  - moscot
+expression_correction_methods:
+  - no_correction
+gene_efficiency_correction_methods:
+  - no_correction
+method_parameters_yaml: $params_url
+HERE
+
+# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this)
+cat > /tmp/params.yaml << HERE
+input_states: $resources_test_s3/**/state.yaml
+rename_keys: 'input_sc:output_sc;input_sp:output_sp'
+save_spatial_data: false
+settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+# Fail early with a clear message if the params file isn't reachable on GitHub yet.
+if ! curl -fsSL -o /dev/null "$params_url"; then
+  echo "ERROR: params file not reachable at:" >&2
+  echo "  $params_url" >&2
+  echo "Commit and push scripts/run_benchmark/param_sweep/moscot_params.yaml to '$params_branch' first." >&2
+  exit 1
+fi
+
+tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 167877437119966 \
+  --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \
+  --params-file /tmp/params.yaml \
+  --entry-name auto \
+  --config src/base/labels_nebius.config \
+  --labels task_ist_preprocessing,test,moscot,gpu

scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+# Nebius test run: all default methods + RCTD (rctd) cell type annotation, with a
+# parameter sweep over RCTD's create.RCTD DE-gene / UMI thresholds.
+# See src/methods_cell_type_annotation/rctd/NOTES.md ("Optimization / tuning").
+#
+# rctd is an R (spacexr) CPU-only method, so it runs on the standard (non-GPU)
+# compute env with NO `gpu` label.
+#
+# The stage default (tacco) is kept enabled alongside rctd so the run also produces
+# the baseline annotation — the workflow allows at most ONE non-default variant at a
+# time, and rctd is that one non-default method here.
+#
+# SUBMITTABLE AS-IS: every swept arg (the six create.RCTD thresholds) is ALREADY
+# exposed in src/methods_cell_type_annotation/rctd/config.vsh.yaml, so the build/main
+# container run by `--revision build/main` accepts them with NO rebuild.
+#
+# PARAMS-FILE CAVEAT (why this differs from a local script):
+#   `tw launch --params-file` is read client-side, but `method_parameters_yaml`
+#   is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow
+#   file()). A local /tmp path does not exist there, and /scratch (where results
+#   publish) is READ-ONLY from the launch host — which is why the binning
+#   method_params block is commented out in run_test_nebius.sh. file() does stage
+#   http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED
+#   file (scripts/run_benchmark/param_sweep/rctd_params.yaml) and read it from GitHub
+#   via its raw URL. => the params file must be committed AND PUSHED to $params_branch
+#   before launching (edit the file there, not here, to change the sweep).
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing
+# Results publish to /scratch — created and written by the cloud compute env, so
+# the launcher does NOT create it here (it is read-only from the launch host).
+publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_rctd"
+
+# The sweep lives in a committed file, read from GitHub at runtime. $params_branch
+# defaults to the branch you are on; the file must be pushed there on GitHub. (This
+# is independent of --revision below, which selects the pipeline CODE to run.)
+params_repo="openproblems-bio/task_ist_preprocessing"
+params_branch="$(git rev-parse --abbrev-ref HEAD)"
+params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/rctd_params.yaml"
+
+cat > /tmp/params_settings.yaml << HERE
+default_methods:
+  - custom_segmentation
+  - basic_transcript_assignment
+  - basic_count_aggregation
+  - basic_qc_filter
+  - alpha_shapes
+  - normalize_by_volume
+  - tacco
+  - no_correction
+segmentation_methods:
+  - custom_segmentation
+transcript_assignment_methods:
+  - basic_transcript_assignment
+count_aggregation_methods:
+  - basic_count_aggregation
+qc_filtering_methods:
+  - basic_qc_filter
+volume_calculation_methods:
+  - alpha_shapes
+normalization_methods:
+  - normalize_by_volume
+celltype_annotation_methods:
+  - tacco
+  - rctd
+#  - ssam
+#  - moscot
+#  - mapmycells
+#  - tangram
+#  - singler
+expression_correction_methods:
+  - no_correction
+gene_efficiency_correction_methods:
+  - no_correction
+method_parameters_yaml: $params_url
+HERE
+
+# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this)
+cat > /tmp/params.yaml << HERE
+input_states: $resources_test_s3/**/state.yaml
+rename_keys: 'input_sc:output_sc;input_sp:output_sp'
+save_spatial_data: false
+settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)'
+output_state: "state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+# Fail early with a clear message if the params file isn't reachable on GitHub yet.
+if ! curl -fsSL -o /dev/null "$params_url"; then
+  echo "ERROR: params file not reachable at:" >&2
+  echo "  $params_url" >&2
+  echo "Commit and push scripts/run_benchmark/param_sweep/rctd_params.yaml to '$params_branch' first." >&2
+  exit 1
+fi
+
+tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/workflows/run_benchmark/main.nf \
+  --workspace 167877437119966 \
+  --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \
+  --params-file /tmp/params.yaml \
+  --entry-name auto \
+  --config src/base/labels_nebius.config \
+  --labels task_ist_preprocessing,test,rctd