diff --git a/scripts/run_benchmark/param_sweep/moscot_params.yaml b/scripts/run_benchmark/param_sweep/moscot_params.yaml new file mode 100644 index 000000000..e615b0f9e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/moscot_params.yaml @@ -0,0 +1,61 @@ +# Parameter sweep for the moscot (MOSCOT, OTT-jax fused Gromov-Wasserstein optimal- +# transport reference mapping) cell-type-annotation method. GPU method. +# +# Committed source of truth for run_test_moscot_nebius.sh, read from GitHub via a raw +# URL (the Nebius compute env pulls the repo but cannot see the launch host's files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total moscot variants = 1 default + sum(sweep list lengths) = 9. +# +# See src/methods_cell_type_annotation/moscot/NOTES.md ("Optimization / tuning") for +# the tiers, tool defaults, and rationale. +# +# Only ALREADY-EXPOSED args are swept (alpha, epsilon, mapping_mode), so this is +# SUBMITTABLE against the build/main container as-is (no rebuild). Un-exposed knobs +# (n_comps, scale_cost, initializer, convergence thresholds) are documented in NOTES +# as future work. +# +# !!! SMALL-DATA CLAMP — why tau and rank are NOT swept here !!! +# script.py:62-66 forces rank=-1 and tau=1.0 whenever the spatial AnnData has +# < 10000 cells. The test resource (mouse_brain_combined) has n_obs=306, so on this +# run rank and tau are OVERRIDDEN regardless of any value passed -- sweeping them +# would only add identical no-op variants. They deviate from moscot's tool defaults +# (tau 0.3 vs 1.0, rank 500 vs -1) ONLY for >10k-cell datasets, and are documented in +# NOTES.md as the Tier-1 knobs for a real (>10k) dataset sweep. +parameters: + moscot: + # Baseline == the component's shipped defaults (config.vsh.yaml). + default: + alpha: 0.8 + epsilon: 0.01 + tau: 0.3 # clamped to 1.0 on this <10k test data (script.py:65-66) + rank: 500 # clamped to -1 on this <10k test data (script.py:63-64) + batch_size: 1024 # pure memory knob (online GW-cost batching); held fixed + mapping_mode: max + sweep: + # ---- Tier 1: FGW interpolation weight (quality) ---- + # alpha in (0,1] balances the quadratic Gromov-Wasserstein term (intra-domain + # structure: SC 30-dim PCA vs spatial local-pca coords) against the linear term + # (shared-gene expression). alpha->1 = pure GW/structure; alpha->0 = pure + # expression matching. Config default 0.8 DEVIATES from moscot's tool default 0.5 + # (structure-heavy) -> promoted to a sweep axis. 0.8 omitted (the default variant + # covers it); range straddles the tool default (0.5) up through 0.9. + alpha: [0.5, 0.7, 0.9] + # ---- Tier 1: entropic regularization (quality vs. convergence) ---- + # epsilon sets the entropy/stochasticity of the transport plan: higher = blurrier + # map, faster & more stable convergence; lower = sharper, more confident mapping + # but harder to converge. Config default 0.01 == moscot's tool default (not a + # deviation), swept as a legitimate Tier-1 quality axis. 0.01 omitted; two lower + # (sharper) + two higher (diffuse) probe an order of magnitude each way. + epsilon: [0.001, 0.005, 0.05, 0.1] + # ---- Tier 2: label read-out from the transport plan ---- + # mapping_mode: how a per-cell label is pulled from the plan. 'max' (default) takes + # the label of the single highest-mass source cell; 'sum' aggregates the plan mass + # per cell type first, then takes the argmax (smoother, uses the full coupling). + # moscot has NO tool default here (required arg); 'sum' is the one meaningful + # non-default value. + mapping_mode: [sum] diff --git a/scripts/run_benchmark/param_sweep/rctd_params.yaml b/scripts/run_benchmark/param_sweep/rctd_params.yaml new file mode 100644 index 000000000..c0d6404ce --- /dev/null +++ b/scripts/run_benchmark/param_sweep/rctd_params.yaml @@ -0,0 +1,52 @@ +# Parameter sweep for the rctd (spacexr RCTD, R) cell type annotation method. +# Committed source of truth, read at runtime from GitHub via a raw URL by +# run_test_rctd_nebius.sh (the Nebius compute env pulls the repo but cannot see the +# launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total rctd variants = 1 default + sum(sweep list lengths) = 13. +# +# See src/methods_cell_type_annotation/rctd/NOTES.md ("Optimization / tuning") for the +# tiers and rationale. +# +# All six args are create.RCTD DE-gene / UMI thresholds. Every config default is a +# DELIBERATELY RELAXED value (walked away from the spacexr default) because iST panels +# are small (~100-500 genes) and low-count; the stock spacexr defaults strand too few DE +# genes and abort with "fewer than 10 regression differentially expressed genes". Each +# sweep list therefore walks that ONE arg back TOWARD its spacexr default (the meaningful +# direction, since the relaxed default already sits at the permissive extreme, and the +# default variant already covers the relaxed point). +# +# CAVEAT: on a very small panel, the high-threshold variants (values at/near the spacexr +# defaults) can drop back under the 10-DE-gene floor and legitimately fail — that failure +# boundary is part of what this sweep measures. +parameters: + rctd: + # Baseline == the component's shipped (relaxed-for-iST) defaults. + default: + gene_cutoff: 0.0 + fc_cutoff: 0.1 + gene_cutoff_reg: 0.0 + fc_cutoff_reg: 0.1 + umi_min: 20 + umi_min_sigma: 20 + sweep: + # ---- Tier 1: DE-gene fold-change thresholds (sharpest levers) ---- + # fc_cutoff: min log-FC for a PLATFORM-EFFECT DE gene. Relaxed 0.1 -> spacexr 0.5. + fc_cutoff: [0.25, 0.5] + # fc_cutoff_reg: min log-FC for a REGRESSION DE gene. Relaxed 0.1 -> spacexr 0.75. + fc_cutoff_reg: [0.4, 0.75] + # ---- Tier 1: DE-gene expression-mean thresholds (companion filters) ---- + # gene_cutoff: min normalized mean expr, platform-effect. 0.0 -> spacexr 0.000125. + gene_cutoff: [0.0000625, 0.000125] + # gene_cutoff_reg: min normalized mean expr, regression. 0.0 -> spacexr 0.0002. + gene_cutoff_reg: [0.0001, 0.0002] + # ---- Tier 1: UMI thresholds (which cells get annotated / fit variance) ---- + # umi_min: min total UMI per spatial cell to annotate it. 20 -> spacexr 100. + umi_min: [50, 100] + # umi_min_sigma: min UMI for cells fitting the platform-effect variance. 20 -> 300. + umi_min_sigma: [100, 300] diff --git a/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml b/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml new file mode 100644 index 000000000..303dcdefd --- /dev/null +++ b/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml @@ -0,0 +1,45 @@ +# Parameter sweep for the resolvi_correction (resolVI, scvi-tools) expression-correction method. +# Committed source of truth read by run_test_resolvi_correction_nebius.sh from GitHub via a +# raw URL (the Nebius compute env pulls the repo but cannot see the launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not a full +# grid), so total resolvi_correction variants = 1 default + sum(sweep list lengths) = 5. +# +# SUBMITTABLE-NOW CONSTRAINT: the Nebius launch runs `--revision build/main`, i.e. the +# build/main container as it exists today. It only honours arguments ALREADY EXPOSED in +# src/methods_expression_correction/resolvi_correction/config.vsh.yaml. That container exposes +# exactly four args -- celltype_key, n_hidden, encode_covariates, downsample_counts -- so the +# sweep touches ONLY those (celltype_key is an input mapping, not a tuning knob). The high-value +# resolVI levers num_samples (posterior samples), max_epochs, and mixture_k are HARD-CODED in +# script.py, NOT exposed; sweeping them would need a config change + container rebuild, so they +# are documented in NOTES.md ("Optimization / tuning") as future work and are NOT swept here. +# +# See src/methods_expression_correction/resolvi_correction/NOTES.md for tiers, rationale, and +# the verified upstream defaults. Trim or widen the lists to taste. +parameters: + resolvi_correction: + # Baseline == the component's shipped defaults, which (verified) all match the scvi-tools + # RESOLVI upstream defaults (n_hidden=32, encode_covariates=False, downsample_counts=True). + default: + celltype_key: cell_type + n_hidden: 32 + encode_covariates: false + downsample_counts: true + sweep: + # ---- Tier 2: quality / capacity trade-offs (the only exposed knobs) ---- + # n_hidden: width of the VAE hidden layers. 32 is the scvi RESOLVI default (small, fast); + # raising it gives the encoder/decoder more capacity to model diffusion + background at a + # time cost. script.py's own grid-search note lists 64 and 128 (default 32 omitted). + n_hidden: [64, 128] + # encode_covariates: BOOLEAN, default false. Flip to true to also feed batch/covariates + # through the encoder (helps when integrating across batches). Forwarded to the RESOLVI + # module via **model_kwargs; only one meaningful non-default value exists. + encode_covariates: [true] + # downsample_counts: BOOLEAN, default true (subsamples per-cell counts to equalise depth + # during training). Flip to false to train on full observed counts; only one meaningful + # non-default value exists. + downsample_counts: [false] diff --git a/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh new file mode 100644 index 000000000..7319cfb74 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_moscot_nebius.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# Nebius test run: all default methods + MOSCOT (moscot) cell-type annotation, with a +# parameter sweep over the optimization levers identified for moscot. +# See src/methods_cell_type_annotation/moscot/NOTES.md ("Optimization / tuning"). +# +# moscot is an OTT-jax fused Gromov-Wasserstein optimal-transport mapping method that +# runs on GPU (jax[cuda12], gpuh100), hence the `gpu` label and the Nebius GPU compute +# env (mirrors run_gpu_nebius.sh). +# +# The annotation stage keeps its default method `tacco` alongside `moscot` so the run +# has a baseline to compare the moscot variants against. Every other stage is left on +# its single default (the swept method must be the only non-default thing). +# +# SUBMITTABLE-NOW: only already-exposed args are swept (alpha, epsilon, mapping_mode), +# so this launches against the build/main container with no rebuild. tau and rank are +# NOT swept because script.py clamps them to tool defaults on the <10k-cell test data +# (see moscot_params.yaml + NOTES.md). +# +# PARAMS-FILE CAVEAT (why this reads from GitHub): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host. file() does stage http(s):// though, +# and this repo is public, so we keep the sweep in a COMMITTED file +# (scripts/run_benchmark/param_sweep/moscot_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the +# sweep). This is independent of --revision below, which selects the pipeline CODE. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_moscot" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/moscot_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - moscot +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/moscot_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,moscot,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh new file mode 100644 index 000000000..438c3a14e --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_rctd_nebius.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +# Nebius test run: all default methods + RCTD (rctd) cell type annotation, with a +# parameter sweep over RCTD's create.RCTD DE-gene / UMI thresholds. +# See src/methods_cell_type_annotation/rctd/NOTES.md ("Optimization / tuning"). +# +# rctd is an R (spacexr) CPU-only method, so it runs on the standard (non-GPU) +# compute env with NO `gpu` label. +# +# The stage default (tacco) is kept enabled alongside rctd so the run also produces +# the baseline annotation — the workflow allows at most ONE non-default variant at a +# time, and rctd is that one non-default method here. +# +# SUBMITTABLE AS-IS: every swept arg (the six create.RCTD thresholds) is ALREADY +# exposed in src/methods_cell_type_annotation/rctd/config.vsh.yaml, so the build/main +# container run by `--revision build/main` accepts them with NO rebuild. +# +# PARAMS-FILE CAVEAT (why this differs from a local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host — which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/param_sweep/rctd_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_rctd" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/rctd_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - rctd +# - ssam +# - moscot +# - mapmycells +# - tangram +# - singler +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/rctd_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,rctd diff --git a/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh new file mode 100644 index 000000000..a96ab96f0 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_resolvi_correction_nebius.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# Nebius test run: all default methods + resolVI (resolvi_correction) expression correction, +# with a parameter sweep over the exposed optimization knobs identified for resolVI. +# See src/methods_expression_correction/resolvi_correction/NOTES.md ("Optimization / tuning"). +# +# resolVI is a scvi-tools VAE (deep model). It carries the `gpu` label (mirroring +# run_gpu_nebius.sh) and runs on the Nebius GPU compute env. The expression-correction stage +# lists BOTH the stage default (no_correction) AND resolvi_correction so the sweep is scored +# against the uncorrected baseline; every OTHER stage stays on its single default (the one +# non-default-variant-at-a-time constraint means resolvi_correction must be the only non-default +# thing in the pipeline). +# +# PARAMS-FILE CAVEAT (why this differs from a local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` is a path the +# WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow file()). A local /tmp path does +# not exist there, and /scratch (where results publish) is READ-ONLY from the launch host. +# file() DOES stage http(s):// though, and this repo is public, so we keep the sweep in a +# COMMITTED file (scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml) and read it +# from GitHub via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the sweep). This is +# independent of --revision below, which selects the pipeline CODE to run. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so the launcher +# does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_resolvi_correction" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there on GitHub. +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction + - resolvi_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/resolvi_correction_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,resolvi_correction,gpu diff --git a/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh new file mode 100644 index 000000000..0a20da769 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_split_nebius.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Nebius test run: all default methods + SPLIT (split) expression correction, with a +# parameter sweep over the six create.RCTD DE-gene / UMI thresholds SPLIT exposes. +# See src/methods_expression_correction/split/NOTES.md ("Optimization / tuning"). +# +# split is an R method wrapping spacexr (RCTD) + the SPLIT R package. It is CPU-only, so it +# runs on the standard (non-GPU) compute env with NO `gpu` label (modelled on run_test_nebius.sh). +# +# The stage default (no_correction) is kept enabled alongside split so the run also produces +# the uncorrected baseline the sweep is scored against — the workflow allows at most ONE +# non-default variant at a time, and split is that one non-default method here; every OTHER +# stage stays on its single default. +# +# SUBMITTABLE AS-IS: every swept arg (the six create.RCTD thresholds) is ALREADY exposed in +# src/methods_expression_correction/split/config.vsh.yaml, so the build/main container run by +# `--revision build/main` accepts them with NO rebuild. SPLIT's own core levers +# (DO_purify_singlets, DO_remove_residual_contamination, RCTD doublet_mode) are hard-coded in +# script.R and are documented in NOTES.md as future work, NOT swept here. +# +# PARAMS-FILE CAVEAT (why this differs from a local script): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` is a path the +# WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow file()). A local /tmp path does +# not exist there, and /scratch (where results publish) is READ-ONLY from the launch host — +# which is why the binning method_params block is commented out in run_test_nebius.sh. file() +# DOES stage http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/param_sweep/split_params.yaml) and read it from GitHub via its +# raw URL. => the params file must be committed AND PUSHED to $params_branch before launching +# (edit the file there, not here, to change the sweep). This is independent of --revision +# below, which selects the pipeline CODE to run. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_split" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch defaults to +# the branch you are on; the file must be pushed there on GitHub. (This is independent of +# --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/split_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco +expression_correction_methods: + - no_correction + - split +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/split_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,split diff --git a/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh new file mode 100644 index 000000000..7ac988081 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_ssam_nebius.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Nebius test run: all default methods + SSAM (ssam) cell-type annotation, with a +# parameter sweep over the one exposed ssam knob (um_per_pixel). +# See src/methods_cell_type_annotation/ssam/NOTES.md ("Optimization / tuning"). +# +# ssam is a CPU-only method (txsim -> plankton density estimation, no GPU), so it +# runs on the standard (non-GPU) compute env with no `gpu` label. +# +# SUBMITTABLE AS-IS: the swept arg (um_per_pixel) is ALREADY EXPOSED in the current +# config.vsh.yaml, so the build/main container run by --revision below already +# understands it -- no viash ns build / container rebuild is required. +# +# PARAMS-FILE CAVEAT (why the sweep is not an inline heredoc): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host -- which is why the binning +# method_params block is commented out in run_test_nebius.sh. file() does stage +# http(s):// though, and this repo is public, so we keep the sweep in a COMMITTED +# file (scripts/run_benchmark/param_sweep/ssam_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to $params_branch +# before launching (edit the file there, not here, to change the sweep). + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch -- created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_ssam" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/ssam_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - ssam +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/ssam_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,ssam diff --git a/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh b/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh new file mode 100644 index 000000000..be7c40618 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/run_test_tangram_nebius.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +# Nebius test run: all default methods + Tangram (tangram) cell-type annotation, +# with a parameter sweep over the optimization levers identified for Tangram. +# See src/methods_cell_type_annotation/tangram/NOTES.md ("Optimization / tuning"). +# +# Tangram is a torch deep-learning mapping method (GPU), hence the `gpu` label and +# the Nebius GPU compute env (mirrors run_gpu_nebius.sh). +# +# The annotation stage keeps its default method `tacco` alongside `tangram` so the +# run has a baseline to compare the tangram variants against. Every other stage is +# left on its single default (the swept method must be the only non-default thing). +# +# PARAMS-FILE CAVEAT (why this reads from GitHub): +# `tw launch --params-file` is read client-side, but `method_parameters_yaml` +# is a path the WORKFLOW opens at runtime on the cloud (readYaml -> Nextflow +# file()). A local /tmp path does not exist there, and /scratch (where results +# publish) is READ-ONLY from the launch host. file() does stage http(s):// though, +# and this repo is public, so we keep the sweep in a COMMITTED file +# (scripts/run_benchmark/param_sweep/tangram_params.yaml) and read it from GitHub +# via its raw URL. => the params file must be committed AND PUSHED to +# $params_branch before launching (edit the file there, not here, to change the +# sweep). This is independent of --revision below, which selects the pipeline CODE. + +# get the root of the directory +REPO_ROOT=$(git rev-parse --show-toplevel) + +# ensure that the command below is run from the root of the repository +cd "$REPO_ROOT" + +set -e + +resources_test_s3=s3://openproblems-data/resources_test/task_ist_preprocessing +# Results publish to /scratch — created and written by the cloud compute env, so +# the launcher does NOT create it here (it is read-only from the launch host). +publish_dir="/scratch/results/runs/$(date +%Y-%m-%d_%H-%M-%S)_tangram" + +# The sweep lives in a committed file, read from GitHub at runtime. $params_branch +# defaults to the branch you are on; the file must be pushed there on GitHub. (This +# is independent of --revision below, which selects the pipeline CODE to run.) +params_repo="openproblems-bio/task_ist_preprocessing" +params_branch="$(git rev-parse --abbrev-ref HEAD)" +params_url="https://raw.githubusercontent.com/${params_repo}/${params_branch}/scripts/run_benchmark/param_sweep/tangram_params.yaml" + +cat > /tmp/params_settings.yaml << HERE +default_methods: + - custom_segmentation + - basic_transcript_assignment + - basic_count_aggregation + - basic_qc_filter + - alpha_shapes + - normalize_by_volume + - tacco + - no_correction +segmentation_methods: + - custom_segmentation +transcript_assignment_methods: + - basic_transcript_assignment +count_aggregation_methods: + - basic_count_aggregation +qc_filtering_methods: + - basic_qc_filter +volume_calculation_methods: + - alpha_shapes +normalization_methods: + - normalize_by_volume +celltype_annotation_methods: + - tacco + - tangram +expression_correction_methods: + - no_correction +gene_efficiency_correction_methods: + - no_correction +method_parameters_yaml: $params_url +HERE + +# Write the parameters to file (input_states version, NOTE: enable `-entry_name auto` for this) +cat > /tmp/params.yaml << HERE +input_states: $resources_test_s3/**/state.yaml +rename_keys: 'input_sc:output_sc;input_sp:output_sp' +save_spatial_data: false +settings: '$(yq -o json /tmp/params_settings.yaml | jq -c .)' +output_state: "state.yaml" +publish_dir: "$publish_dir" +HERE + +# Fail early with a clear message if the params file isn't reachable on GitHub yet. +if ! curl -fsSL -o /dev/null "$params_url"; then + echo "ERROR: params file not reachable at:" >&2 + echo " $params_url" >&2 + echo "Commit and push scripts/run_benchmark/param_sweep/tangram_params.yaml to '$params_branch' first." >&2 + exit 1 +fi + +tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \ + --revision build/main \ + --pull-latest \ + --main-script target/nextflow/workflows/run_benchmark/main.nf \ + --workspace 167877437119966 \ + --compute-env 5hfmdCBxMRd4nHZaJKYEQZ \ + --params-file /tmp/params.yaml \ + --entry-name auto \ + --config src/base/labels_nebius.config \ + --labels task_ist_preprocessing,test,tangram,gpu diff --git a/scripts/run_benchmark/param_sweep/split_params.yaml b/scripts/run_benchmark/param_sweep/split_params.yaml new file mode 100644 index 000000000..228a0447d --- /dev/null +++ b/scripts/run_benchmark/param_sweep/split_params.yaml @@ -0,0 +1,63 @@ +# Parameter sweep for the split (SPLIT, R) expression-correction method. +# Committed source of truth, read at runtime from GitHub via a raw URL by +# run_test_split_nebius.sh (the Nebius compute env pulls the repo but cannot see the +# launch host's local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total split variants = 1 default + sum(sweep list lengths) = 13. +# +# See src/methods_expression_correction/split/NOTES.md ("Optimization / tuning") for the +# tiers and rationale. +# +# WHAT SPLIT ACTUALLY VARIES HERE: SPLIT runs RCTD (spacexr) INTERNALLY to deconvolve each +# cell, then purifies counts from that deconvolution. The only knobs the component exposes +# are the six create.RCTD DE-gene / UMI thresholds (plus keep_all_cells, a zero-count-cell +# guard that is NOT a tuning lever and can crash if flipped -> left fixed). Every one of the +# six config defaults is a DELIBERATELY RELAXED value walked away from the spacexr default, +# because iST panels are small (~100-500 genes) and low-count; the stock spacexr defaults +# strand too few DE genes and abort with "fewer than 10 regression differentially expressed +# genes". NON-DEFAULT AUDIT: because all six deviate from the spacexr defaults and none is a +# performance/resource knob, each is forced onto a sweep axis. Each list therefore walks that +# ONE arg back TOWARD its spacexr default (the meaningful direction, since the relaxed default +# already sits at the permissive extreme, and the default variant already covers it). +# +# SUBMITTABLE-NOW: the Nebius launch runs `--revision build/main`, so the sweep touches ONLY +# args already exposed in the build/main container -- the six thresholds below all exist in +# src/methods_expression_correction/split/config.vsh.yaml today, so no rebuild is needed. +# SPLIT's OWN core levers (DO_purify_singlets, DO_remove_residual_contamination, RCTD +# doublet_mode) are HARD-CODED in script.R, NOT exposed; sweeping them would need a config +# arg + container rebuild, so they are documented in NOTES.md as future work and NOT here. +# +# CAVEAT: on a very small panel, the high-threshold variants (values at/near the spacexr +# defaults) can drop back under the 10-DE-gene floor and legitimately fail inside create.RCTD +# -- that failure boundary is part of what this sweep measures. +parameters: + split: + # Baseline == the component's shipped (relaxed-for-iST) defaults. + default: + gene_cutoff: 0.0 + fc_cutoff: 0.1 + gene_cutoff_reg: 0.0 + fc_cutoff_reg: 0.1 + umi_min: 20 + umi_min_sigma: 20 + sweep: + # ---- Tier 1: DE-gene fold-change thresholds (sharpest levers) ---- + # fc_cutoff: min log-FC for a PLATFORM-EFFECT DE gene. Relaxed 0.1 -> spacexr 0.5. + fc_cutoff: [0.25, 0.5] + # fc_cutoff_reg: min log-FC for a REGRESSION DE gene. Relaxed 0.1 -> spacexr 0.75. + fc_cutoff_reg: [0.4, 0.75] + # ---- Tier 1: DE-gene expression-mean thresholds (companion filters) ---- + # gene_cutoff: min normalized mean expr, platform-effect. 0.0 -> spacexr 0.000125. + gene_cutoff: [0.0000625, 0.000125] + # gene_cutoff_reg: min normalized mean expr, regression. 0.0 -> spacexr 0.0002. + gene_cutoff_reg: [0.0001, 0.0002] + # ---- Tier 1: UMI thresholds (which cells get deconvolved / fit variance) ---- + # umi_min: min total UMI per spatial cell to include it. 20 -> spacexr 100. + umi_min: [50, 100] + # umi_min_sigma: min UMI for cells fitting the platform-effect variance. 20 -> 300. + umi_min_sigma: [100, 300] diff --git a/scripts/run_benchmark/param_sweep/ssam_params.yaml b/scripts/run_benchmark/param_sweep/ssam_params.yaml new file mode 100644 index 000000000..acb08d5ad --- /dev/null +++ b/scripts/run_benchmark/param_sweep/ssam_params.yaml @@ -0,0 +1,34 @@ +# Parameter sweep for the ssam (SSAM signature-based) cell-type-annotation method. +# Committed source of truth for run_test_ssam_nebius.sh (read from GitHub via a raw +# URL, since the Nebius compute env pulls the repo but cannot see the launch host's +# local files). +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total ssam variants = 1 default + sum(sweep list lengths) = 5. +# +# See src/methods_cell_type_annotation/ssam/NOTES.md ("Optimization / tuning") for the +# rationale. NOTE: the component exposes exactly ONE tunable knob (`um_per_pixel`); the +# real SSAM levers (kernel_bandwidth, threshold_cor, threshold_exp, patch_length) are +# HARDCODED inside txsim's run_ssam wrapper and cannot be reached without patching +# txsim + rebuilding the image — see NOTES.md "Optimization / tuning" (future work). +parameters: + ssam: + # Baseline == the component's shipped default. NB the txsim tool default for + # um_p_px is 0.325; the component ships 0.5 (a non-default deviation), so 0.325 + # is included on the sweep axis below (non-default audit). + default: + um_per_pixel: 0.5 + sweep: + # um_per_pixel: micrometre-per-pixel factor. txsim's run_ssam multiplies the + # transcript x/y coordinates by this before building the mRNA-density map with + # a FIXED Gaussian kernel_bandwidth of 4 (hardcoded in txsim). It is therefore + # the only reachable dial on the effective KDE smoothing scale: smaller values + # compress coordinates -> coarser smoothing relative to the kernel; larger + # values spread them -> finer effective resolution. Range straddles the txsim + # tool default (0.325) and the shipped default (0.5), which is omitted here + # because the "default" variant above already covers it. + um_per_pixel: [0.1, 0.2, 0.325, 1.0] diff --git a/scripts/run_benchmark/param_sweep/tangram_params.yaml b/scripts/run_benchmark/param_sweep/tangram_params.yaml new file mode 100644 index 000000000..3638b69b1 --- /dev/null +++ b/scripts/run_benchmark/param_sweep/tangram_params.yaml @@ -0,0 +1,39 @@ +# Parameter sweep for the tangram (Tangram, torch deep-learning mapping) +# cell-type-annotation method. +# +# Read from GitHub via a raw URL by run_test_tangram_nebius.sh, since the Nebius +# compute env pulls the repo but cannot see the launch host's local files. +# +# Consumed by the run_benchmark workflow via the `method_parameters_yaml` setting +# (src/workflows/run_benchmark/main.nf). For every method the workflow builds: +# * one "default" variant using the `default:` args below, and +# * one extra variant per value in each `sweep:` list, with that ONE arg overridden. +# The benchmark varies a SINGLE parameter at a time (a "star" around the default, not +# a full grid), so total tangram variants = 1 default + sum(sweep list lengths) = 6. +# +# See src/methods_cell_type_annotation/tangram/NOTES.md ("Optimization / tuning") for +# the tiers, defaults, and rationale. Only ALREADY-EXPOSED args are swept here +# (mode, num_epochs) so this is submittable against the build/main container as-is; +# density_prior / learning_rate / loss lambdas are documented in NOTES as future work +# (they would need config expose + a container rebuild first). +parameters: + tangram: + # Baseline == the component's shipped defaults. + default: + mode: "clusters" + num_epochs: 1000 + sweep: + # ---- Tier 1: highest impact on quality (accuracy vs. runtime) ---- + # num_epochs: gradient-descent steps for the mapping. 1000 == Tangram's own + # default (our baseline). Fewer -> faster but under-converged/noisier labels; + # more -> better-converged mapping at a time cost. Sweep below and above the + # default to find the knee. Default 1000 is omitted (the default variant covers it). + num_epochs: [100, 500, 2000, 3000] + # ---- Tier 2: mapping granularity ---- + # mode: our default 'clusters' maps per-cell-type clusters (fits GPU VRAM on + # large references); 'cells' maps individual reference cells (finer, can be + # more accurate on SMALL references, but its matrix scales with n_sc_cells and + # OOMs on large ones). 'cells' is the one meaningful non-default value; feasible + # on the small test reference. ('constrained' needs target_count -> not a + # drop-in axis for label projection, left out.) + mode: ["cells"] diff --git a/src/datasets/loaders/bruker_cosmx/script.py b/src/datasets/loaders/bruker_cosmx/script.py index 9317fbe8a..8cb45a846 100644 --- a/src/datasets/loaders/bruker_cosmx/script.py +++ b/src/datasets/loaders/bruker_cosmx/script.py @@ -536,8 +536,13 @@ def _fixed_global_cell_id(self, df): # Add info to metadata table # ############################## log("Add metadata to table") -# The common iST API requires a per-cell `cell_id` in obs; sopa keys the table by the global -# cell id (as the index) but doesn't add it as a column. +# The common iST API requires a per-cell unique `cell_id` in obs. sopa keys the table by the +# global cell id (the index), so expose that as `cell_id`. But the CosMx metadata already +# carries a per-FOV-*local* `cell_ID` column, and obs cannot hold two keys that differ only in +# case ('cell_ID' vs 'cell_id') — spatialdata's zarr writer rejects that as an invalid name. +# Rename the vendor local id out of the way first. +if "cell_ID" in sdata["table"].obs.columns: + sdata["table"].obs.rename(columns={"cell_ID": "cell_ID_local"}, inplace=True) sdata["table"].obs["cell_id"] = sdata["table"].obs.index.astype(str) for key in ["dataset_id", "dataset_name", "dataset_url", "dataset_reference", "dataset_summary", "dataset_description", "dataset_organism", "segmentation_id"]: sdata["table"].uns[key] = par[key] diff --git a/src/methods_cell_type_annotation/singler/script.py b/src/methods_cell_type_annotation/singler/script.py index c6d539283..921238451 100644 --- a/src/methods_cell_type_annotation/singler/script.py +++ b/src/methods_cell_type_annotation/singler/script.py @@ -38,8 +38,8 @@ mat_ref = mat_ref.sorted_indices() ## magic line to make sure the matrix is in the right format for SingleR ## create the reference from our sc data -built = singler.train_single(ref_data = mat_ref, - ref_labels = sce_ref.get_column_data().column("cell_type"), +built = singler.train_single(ref_data = mat_ref, + ref_labels = sce_ref.get_column_data().column(par['celltype_key']), ref_features = sce_ref.get_row_names(), test_features = features,) diff --git a/src/methods_transcript_assignment/fastreseg/input.py b/src/methods_transcript_assignment/fastreseg/input.py index 207952ee2..09e59ab37 100644 --- a/src/methods_transcript_assignment/fastreseg/input.py +++ b/src/methods_transcript_assignment/fastreseg/input.py @@ -149,6 +149,16 @@ def parse_arguments(): # below would otherwise corrupt it and any later compute of `transcripts` would yield # renamed columns that no longer match the dask _meta -> "Metadata mismatch". transcripts_df = transcripts.compute().copy() + +# Some datasets carry no per-molecule transcript id: e.g. the Vizgen MERSCOPE loader +# renames the vendor's `transcript_id` (a gene/ensembl id) to `ensembl_id`, leaving the +# transcripts without a `transcript_id` column. FastReseg itself does not use one +# (transID_coln = NULL in script.R), but output.py needs a `transcript_id` in the final +# output, so synthesise a unique one from the row position when it is absent. +if 'transcript_id' not in transcripts_df.columns: + print('No transcript_id column found; synthesising a unique transcript_id', flush=True) + transcripts_df['transcript_id'] = np.arange(len(transcripts_df), dtype=np.int64) + transcripts_df.rename(columns = {'feature_name': 'target', 'transcript_id': 'UMI_transID', 'cell_id': 'UMI_cellID'}, inplace = True)