From bdd8c6ba2e853209e114f1306ced2e18b735cf2e Mon Sep 17 00:00:00 2001
From: Owen Price Skelly <21372141+OwenPriceSkelly@users.noreply.github.com>
Date: Fri, 11 Sep 2026 11:45:54 -0500
Subject: [PATCH] smoke-test recipes: one-shot RESCHEDULE_BEGIN, sync-style HF
 token lookup

Port two fixes from the deployed Delta/Perlmutter copies (patched 2026-09-11)
into the generic SLURM recipe, plus the token fix into the PBS recipe.

1. RESCHEDULE_BEGIN propagation (SLURM only). The in-job resubmit used bare
   `sbatch --begin=...`, which defaults to --export=ALL, so the documented
   test kickoff `RESCHEDULE_BEGIN=now+3minutes sbatch ...` rode down the
   chain and made every generation reschedule 3 minutes ahead forever.
   Resubmit with `env -u RESCHEDULE_BEGIN` so the next generation recomputes
   from CADENCE_DAYS/RUN_HOUR, which still persist. The PBS recipe already
   passes only SELF_PATH/CLUSTER via `qsub -v`, so it is unaffected.

2. HF token lookup (both recipes). Only ~/.cache/huggingface/token was read.
   Match the sync scripts: respect an existing HF_TOKEN, else the first
   readable of HF_TOKEN_FILE, ~/.hf_token, ~/.cache/huggingface/token
   (whitespace stripped); export HUGGING_FACE_HUB_TOKEN too; warn if none.

Docs: note the override is one-shot and document the token knobs.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 docs/nightly-smoke-testing.md     | 17 ++++++++++++++++-
 scripts/nightly_smoke_test.pbs    | 16 +++++++++++++---
 scripts/nightly_smoke_test.sbatch | 26 +++++++++++++++++++++-----
 3 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/docs/nightly-smoke-testing.md b/docs/nightly-smoke-testing.md
index f37af1e..4a2215b 100644
--- a/docs/nightly-smoke-testing.md
+++ b/docs/nightly-smoke-testing.md
@@ -94,6 +94,13 @@ Both override when generation 1 schedules generation 2, which is the handoff
 worth testing. (On PBS, `qsub -a <time>` would only delay generation 1's own
 start and exercises nothing about the chain.)
 
+The override is one-shot: generation 2 recomputes its own resubmit time from
+`CADENCE_DAYS`/`RUN_HOUR`. On SLURM the recipe resubmits with
+`env -u RESCHEDULE_BEGIN` because `sbatch` defaults to `--export=ALL` and would
+otherwise carry the testing value down the chain forever; on PBS `qsub -v`
+passes only the variables it names, so `RESCHEDULE_AT` never propagates.
+`CADENCE_DAYS` and `RUN_HOUR` *do* persist on SLURM via the same `--export=ALL`.
+
 ## Knobs
 
 Override via env at submit time (SLURM: inline `VAR=val sbatch …`; PBS:
@@ -105,7 +112,15 @@ Override via env at submit time (SLURM: inline `VAR=val sbatch …`; PBS:
 | `CADENCE_DAYS` | `7` | days between runs (`1` = nightly) |
 | `RUN_HOUR` | `02:00` | wall-clock time of day for scheduled runs |
 | `STOP_FILE` | `~/.rootstock-nightly-stop` | presence ends the chain |
-| `RESCHEDULE_BEGIN` / `RESCHEDULE_AT` | computed | override the next-run time directly (testing) |
+| `RESCHEDULE_BEGIN` / `RESCHEDULE_AT` | computed | override the next-run time directly (testing); one-shot, not inherited by the next generation |
+| `HF_TOKEN` / `HF_TOKEN_FILE` | (see below) | Hugging Face token for gated checkpoints |
+
+The token lookup matches the sync scripts: an `HF_TOKEN` already in the
+environment wins; otherwise the first readable of `HF_TOKEN_FILE`,
+`~/.hf_token`, and `~/.cache/huggingface/token` is used (whitespace stripped)
+and exported as both `HF_TOKEN` and `HUGGING_FACE_HUB_TOKEN`. A missing token is
+a warning, not an error: smoke-test never downloads, but some checkpoints
+re-check gating in `setup()` even with weights cached.
 
 ## Per-cluster notes
 
diff --git a/scripts/nightly_smoke_test.pbs b/scripts/nightly_smoke_test.pbs
index ed5f3bf..4a542a2 100755
--- a/scripts/nightly_smoke_test.pbs
+++ b/scripts/nightly_smoke_test.pbs
@@ -96,11 +96,21 @@ export HTTPS_PROXY="${HTTPS_PROXY:-http://proxy.alcf.anl.gov:3128}"
 export http_proxy="$HTTP_PROXY"
 export https_proxy="$HTTPS_PROXY"
 
-if [[ -f ~/.cache/huggingface/token ]]; then
-  export HF_TOKEN="$(cat ~/.cache/huggingface/token)"
+# Lookup order matches the sync scripts: an explicit HF_TOKEN_FILE, then a
+# 0600 ~/.hf_token, then the huggingface-cli login file. An HF_TOKEN already
+# in the environment (handed in via `qsub -v HF_TOKEN=...`) wins over all three.
+if [[ -z "${HF_TOKEN:-}" ]]; then
+  for tf in "${HF_TOKEN_FILE:-}" "$HOME/.hf_token" "$HOME/.cache/huggingface/token"; do
+    if [[ -n "$tf" && -r "$tf" ]]; then
+      export HF_TOKEN="$(tr -d '[:space:]' < "$tf")"
+      break
+    fi
+  done
+fi
+if [[ -n "${HF_TOKEN:-}" ]]; then
   export HUGGING_FACE_HUB_TOKEN="$HF_TOKEN"
 else
-  echo "WARNING: no HF token at ~/.cache/huggingface/token; gated checkpoints may fail." >&2
+  echo "WARNING: no HF token (HF_TOKEN env, ~/.hf_token, or ~/.cache/huggingface/token); gated checkpoints may fail." >&2
 fi
 
 # Ensure `rootstock` is on PATH here (activate the venv / module load as your
diff --git a/scripts/nightly_smoke_test.sbatch b/scripts/nightly_smoke_test.sbatch
index af21a9b..861d778 100755
--- a/scripts/nightly_smoke_test.sbatch
+++ b/scripts/nightly_smoke_test.sbatch
@@ -45,7 +45,8 @@ CLUSTER="${CLUSTER:-}"
 # Cadence. Default weekly: drift-free wall-clock time CADENCE_DAYS ahead at
 # RUN_HOUR (not "now+24h", which would creep later every run). Set
 # CADENCE_DAYS=1 for nightly. RESCHEDULE_BEGIN overrides the whole computation
-# with any `sbatch --begin` spec (e.g. now+3minutes) for testing.
+# with any `sbatch --begin` spec (e.g. now+3minutes) for testing; it applies to
+# this generation's resubmit only and is not propagated down the chain.
 CADENCE_DAYS="${CADENCE_DAYS:-7}"
 RUN_HOUR="${RUN_HOUR:-02:00}"
 RESCHEDULE_BEGIN="${RESCHEDULE_BEGIN:-$(date -d "+${CADENCE_DAYS} days ${RUN_HOUR}" +%Y-%m-%dT%H:%M:%S)}"
@@ -77,7 +78,12 @@ elif squeue --me --name="$JOB_NAME" --states=PENDING --noheader 2>/dev/null | gr
   # while a run is already queued).
   echo "a '$JOB_NAME' job is already PENDING; not scheduling another."
 else
-  next_id="$(sbatch --begin="$RESCHEDULE_BEGIN" --parsable "$SELF_PATH")"
+  # env -u: RESCHEDULE_BEGIN is a one-shot testing knob. sbatch defaults to
+  # --export=ALL, so it would otherwise ride down the chain and a `now+3minutes`
+  # kickoff would recur every 3 minutes forever. The next generation must
+  # recompute it from CADENCE_DAYS/RUN_HOUR (those knobs are meant to persist
+  # and still do).
+  next_id="$(env -u RESCHEDULE_BEGIN sbatch --begin="$RESCHEDULE_BEGIN" --parsable "$SELF_PATH")"
   echo "scheduled next run: job $next_id (begins $RESCHEDULE_BEGIN)"
 fi
 
@@ -89,11 +95,21 @@ umask 0002
 # HF token for any checkpoint whose setup() re-checks gating even with weights
 # cached. smoke-test never downloads, so a missing token is a warning, not a
 # hard error.
-if [[ -f ~/.cache/huggingface/token ]]; then
-  export HF_TOKEN="$(cat ~/.cache/huggingface/token)"
+# Lookup order matches the sync scripts: an explicit HF_TOKEN_FILE, then a
+# 0600 ~/.hf_token, then the huggingface-cli login file. An HF_TOKEN already
+# in the environment (propagated from the submit shell) wins over all three.
+if [[ -z "${HF_TOKEN:-}" ]]; then
+  for tf in "${HF_TOKEN_FILE:-}" "$HOME/.hf_token" "$HOME/.cache/huggingface/token"; do
+    if [[ -n "$tf" && -r "$tf" ]]; then
+      export HF_TOKEN="$(tr -d '[:space:]' < "$tf")"
+      break
+    fi
+  done
+fi
+if [[ -n "${HF_TOKEN:-}" ]]; then
   export HUGGING_FACE_HUB_TOKEN="$HF_TOKEN"
 else
-  echo "WARNING: no HF token at ~/.cache/huggingface/token; gated checkpoints may fail." >&2
+  echo "WARNING: no HF token (HF_TOKEN env, ~/.hf_token, or ~/.cache/huggingface/token); gated checkpoints may fail." >&2
 fi
 
 # Ensure `rootstock` is on PATH here (activate the venv / module load as your