v2.0.1: download traces from GitHub release assets, and release notes

.claude/rules/ci-stats.md

@@ -41,6 +41,10 @@ Read `docs/ci-stats.md` before redesigning any of this.
   per repo at a time (`concurrency`). No A100.
 - `--array` (with `--per-kernel`) logs tasks as `<array>_<task>`; keep job ids
   strings and `squeue -r` in job_status. Without `--array` nothing changes.
+- Every CI SASS run (main-sass, main-tracer, weekly) launches with
+  `SIM_PER_KERNEL` (lib/common.sh: `--per-kernel --array -M 10G`) and verifies
+  with `--per-kernel-dirs`. Not main-docker: inside the image the launcher is
+  procman, and `--array` needs sbatch.
 - A trace depot is replaced only by `publish_depot` of a finished run.
 - `CI_SANDBOX=<dir>` is the only path override; do not add per-path env knobs.
 - Every simulated app needs hardware stats: `NO-HW-REFERENCE` fails

.github/ci-notes.md

@@ -13,14 +13,16 @@
   on a remote H200 over VPN) then SASS-H200 (`H200-SASS`: rodinia, ubench, TMA,
   cutlass_tma_small); Tracer-V100 (on tgrogers-dgx) then SASS-V100
   (`QV100-SASS`: rodinia, ubench); SASS-H200-ML (`H200-SASS` on the hosted
-  `H200_ml_traces`, never traced here; `--per-kernel --array`: one Slurm array
-  task per kernel, job ids `<array>_<task>`). Each SASS job simulates,
+  ML traces of the static H200 depot, never traced here). Each SASS job simulates,
   correlates, archives.
 - gpgpu-sim is cloned at build time (`clone_gpgpusim` in `lib/common.sh`: the
   same-named branch of the code's gpgpu-sim fork if it has one, else upstream
   `dev`, also when there is no such fork; any other clone or fetch failure
   fails the build): a simulator failure may come from that repo, not this one.
-- Simulations run as slurm jobs; the monitor prints each job's status.
+- Simulations run as slurm jobs; the monitor prints each job's status. Every
+  SASS run except SASS-Simulation-Docker is per kernel (`--per-kernel --array
+  -M 10G`): one Slurm array task per kernel, job ids `<array>_<task>`, run dirs
+  `.../<config>/per-kernel/<kernel>/`.
   `FUNC_TEST_FAILED` = the simulation ran but its output was wrong;
   `FUNC_TEST_FAILED, ASSERT` = the simulator hit an assertion;
   `COMPLETE_ERR_FILE_HAS_CONTENTS` = completed but wrote to stderr (a warning

.github/scripts/lib/common.sh

@@ -186,7 +186,11 @@ build_accelsim() {
 #   $2 - Config
 #   $3 - Trace path
 #   $4 - Job name suffix (default: $$)
-#   $5.. - Extra run_simulations.py flags (e.g. --per-kernel)
+#   $5.. - Extra run_simulations.py flags (e.g. -M)
+# Every CI run is per kernel: one Slurm array task per kernel (get_stats
+# --per-kernel-dirs collects them), so a long kernel neither serialises the
+# app's other kernels nor delays its own failure.
+SIM_PER_KERNEL=(--per-kernel --array -M 10G)
 run_simulations() {
     local benchmarks="$1"
     local config="$2"

.github/scripts/main-sass.sh

@@ -108,11 +108,11 @@ stage_simulate() {
     for run in "${RUNS[@]}"; do
         config="${run%%:*}"
         trace=$(trace_dir "$config")
-        run_simulations "${run#*:}" "$config" "$trace" "$JOB_NAME"
+        run_simulations "${run#*:}" "$config" "$trace" "$JOB_NAME" "${SIM_PER_KERNEL[@]}"
     done
 
     monitor_simulations "$JOB_NAME" 12 300 "stats-per-app-sass.csv" -v
-    verify_simulations "$JOB_NAME" "${RUNS[@]}"
+    verify_simulations --per-kernel-dirs "$JOB_NAME" "${RUNS[@]}"
 }
 
 stage_correlate() {

.github/scripts/main-tracer.sh

@@ -115,7 +115,8 @@ stage_test() {
         -B "$BENCHMARKS" \
         -C "$CONFIG" \
         -T ./hw_run/traces/ \
-        -N "$JOB_NAME"
+        -N "$JOB_NAME" \
+        "${SIM_PER_KERNEL[@]}"
 
     monitor_simulations "$JOB_NAME" 12 300 "rodinia-stats-per-app.csv" -v
 }

.github/scripts/weekly.sh

@@ -8,8 +8,8 @@
 #   trace-h200                         VPN, trace on the remote H200, publish (one Slurm job)
 #   build-tracer-v100, trace-v100,     tracer and apps on the runner; traces and
 #   hw-stats-v100, publish-v100        hardware stats on gpu01 (gpu-lock sruns); publish
-#   simulate|correlate|archive <run>   run: h200, v100, or h200-ml (the hosted
-#                                      ML traces of H200_ml_traces, never traced here)
+#   simulate|correlate|archive <run>   run: h200, v100, or h200-ml (the ML traces
+#                                      in the static H200 depot, never traced here)
 #
 # Requires GITHUB_REPOSITORY and BRANCH_NAME (set by weekly.yml); the trace-h200
 # stage also VPN_HOST, VPN_CERT and REMOTE_HOST (org secrets).
@@ -23,7 +23,6 @@ source "$SCRIPT_DIR/lib/common.sh"
 H200_SUITES="rodinia_2.0-ft,rodinia-3.1,GPU_Microbenchmark,GPU_Microbenchmark_TMA,cutlass_tma_small"
 V100_SUITES="rodinia_2.0-ft,rodinia-3.1,GPU_Microbenchmark"
 ML_SUITES="llama_inference,llama_inference_2048,resnet_inference,bert_inference,dlrmv2_inference,unet3d_inference,retinanet_inference,sdxl_inference,mixtral_inference,deepseek_r1_inference,whisper_inference,qwen25_inference,qwen25_inference_2048,llama_training"
-ML_DEPOT="$DEPOT/H200_ml_traces"
 TRACE_DEVICE=7
 
 export OPENCONNECT_PATH="/home/tgrogers-raid/a/common/openconnect/openconnect"
@@ -51,20 +50,16 @@ depot_dir() {
     echo "$SHARED_ROOT/nightly-$1/${GITHUB_REPOSITORY//\//_}"
 }
 
-# Sets CONFIG, SUITES, TRACES, HW, SIM_FLAGS, STATS_FLAGS, MONITOR_HOURS and
-# GROUP for run $1
+# Sets CONFIG, SUITES, TRACES, HW, MONITOR_HOURS and GROUP for run $1
 gpu_vars() {
-    SIM_FLAGS=() STATS_FLAGS=() MONITOR_HOURS=14 GROUP=ubench
+    MONITOR_HOURS=14 GROUP=ubench
     case "$1" in
         h200) CONFIG="H200-SASS"; SUITES="$H200_SUITES" ;;
         v100) CONFIG="QV100-SASS"; SUITES="$V100_SUITES" ;;
         h200-ml)
-            # ~9K kernels, one array task each (accelsim-layout.sh in the depot
-            # adds the traces/ and device-0/ layout these paths read)
+            # ~9K kernels, in the static CI depot (docs/ci-stats.md)
             CONFIG="H200-SASS" SUITES="$ML_SUITES" GROUP=ml MONITOR_HOURS=40
-            # one kernel needs far less than its app's accel-sim-mem; 10G ran all of them
-            SIM_FLAGS=(--per-kernel --array -M 10G) STATS_FLAGS=(--per-kernel-dirs)
-            TRACES="$ML_DEPOT/traces/device-0/12.8/" HW="$ML_DEPOT/"
+            HW="$DEPOT/H200/hw_run/" TRACES="$DEPOT/H200/hw_run/traces/device-7/12.8/"
             return
             ;;
         *)
@@ -165,10 +160,10 @@ stage_simulate() {
     source ./env-setup/12.8_env_setup.sh
     source ./gpu-simulator/setup_environment.sh
     local job_name="weekly-$run-$(ci_job_name)"
-    run_simulations "$SUITES" "$CONFIG" "$TRACES" "$job_name" "${SIM_FLAGS[@]}"
+    run_simulations "$SUITES" "$CONFIG" "$TRACES" "$job_name" "${SIM_PER_KERNEL[@]}"
     # Weekly: one summary line per poll; a job the node's OOM killer took is not a failure
     monitor_simulations "$job_name" "$MONITOR_HOURS" 300 "weekly-$run-stats-per-app.csv" --allow-oom
-    verify_simulations "${STATS_FLAGS[@]}" --allow-oom "$job_name" "$CONFIG:$SUITES"
+    verify_simulations --per-kernel-dirs --allow-oom "$job_name" "$CONFIG:$SUITES"
 }
 
 stage_correlate() {

.github/workflows/weekly.yml

@@ -156,7 +156,7 @@ jobs:
           CI_ANALYSIS_FILES: "ci-stats/*/correl.txt ci-stats/*/*.kernel.raw.csv util/job_launching/logfiles/failed_job_log_*.txt"
         run: $AI/ci-analysis.sh
 
-  # The hosted ML traces of H200_ml_traces (never traced here), one Slurm array
+  # The ML traces of the static H200 depot (never traced here), one Slurm array
   # task per kernel; independent of the tracers
   SASS-H200-ML:
     runs-on: tgrogers-raid

CITATION.cff

@@ -26,7 +26,7 @@ message: >-
 type: software
 url: "https://accel-sim.github.io/"
 repository-code: "https://github.com/accel-sim/accel-sim-framework"
-version: "2.0"
+version: "2.0.1"
 # CFF requires a top-level `authors` key, so Accel-Sim is credited here as a single
 # entity rather than an enumerated list -- the project has many contributors beyond any
 # one paper's author list. See the GitHub contributors page for individual credit.

README.md

@@ -396,10 +396,16 @@ To disable it entirely, pass `--spinlock_handling none`:
 The detection tool lives in [`./util/tracer_nvbit/others/spinlock_tool/`](./util/tracer_nvbit/others/spinlock_tool/).
 
 **Pre-traced applications.**
-To fetch a repository of pre-collected traces:
+Pre-collected traces are published as GitHub release assets, one tar per GPU / workload / argument set, split into parts under 2 GB with SHA-256 checksums in the release's `manifest.json`.
+To list, download, verify and extract them under `./hw_run/<GPU>/` in the standard layout (`traces/device-0/<cuda>/<workload>/<args>/traces/` plus the Nsight Compute stats in `device-0/<cuda>/<workload>/<args>/`):
 ```bash
-./get-accel-sim-traces.py
+./get-accel-sim-traces.py -l                                   # list what is available
+./get-accel-sim-traces.py -a H200/inference_bert.sh/32_256     # one argument set
+./get-accel-sim-traces.py -a H200/inference_sdxl.sh,H200/all   # GPU[/workload[/args]], 'all' at any level
 ```
+Pass `-T ./hw_run/<GPU>/traces/device-0/<cuda>` to `run_simulations.py` and `-H ./hw_run/<GPU>` to `plot-correlation.py`.
+The release's `README.md` lists every set and how to download one by hand.
+Downloading from a private repository needs `GITHUB_TOKEN` or a `gh auth login` session.
 
 ### SASS Frontend and Simulation Engine
 

docs/ci-stats.md

@@ -6,7 +6,7 @@
 |---|---|---|---|
 | pull_request (smoke) | QV100/H200: GPU_Microbenchmark_CI, rodinia_2.0-ft; H200: GPU_Microbenchmark_TMA_CI, cutlass_tma_CI, hopper_features_CI | no | no |
 | push to dev (full) | QV100/H200: GPU_Microbenchmark, rodinia_2.0-ft; H200: cutlass_tma_small, GPU_Microbenchmark_TMA | yes | yes |
-| weekly | H200 (this week's traces): rodinia, ubench, TMA, cutlass_tma_small; QV100 (this week's V100 traces): rodinia, ubench; H200 ML (hosted `H200_ml_traces`, 14 suites, 20 runs, ~9K kernels) | all | all |
+| weekly | H200 (this week's traces): rodinia, ubench, TMA, cutlass_tma_small; QV100 (this week's V100 traces): rodinia, ubench; H200 ML (static H200 depot, 14 suites, 20 runs, ~9K kernels) | all | all |
 
 Each SASS job: simulate (`run_simulations`), monitor, `verify_simulations`
 (`get_stats --strict` per config into `ci-stats/<config>/run.csv`),
@@ -68,17 +68,17 @@ CSVs can be diffed by hand when needed.
   `weekly.sh` stage per step. No A100, no static-depot regression runs.
 - `concurrency: weekly-<repo>`: the depots are per repo, not per run, so a
   second run's tracer would replace the traces the first run's sims read.
-- SASS-H200-ML replays `accel-sim/H200_ml_traces` (owned by another user;
-  its layout is `<workload>/<args>/{processed_traces,ncu/full,ncu/cycles}`).
-  `accelsim-layout.sh` in that depot adds, without touching its files,
-  `traces/device-0/12.8/<w>/<a>/traces` (links to `processed_traces`, with
-  its own `kernelslist.g`, which the depot's layout script may filter; the
-  hardware rows of a left-out kernel go too, so both sides pair one-to-one) and
-  `device-0/12.8/<w>/<a>/` with
-  `ncu.csv.nsight` (-> ncu_stats.csv) and `ncu.csv.gpc__cycles_elapsed.0`
-  (ncu_cycles.0.csv converted to the long one-metric format). Not in it:
-  `llama_inference_32k` (no traces), FA3 (`benchmark_hopper_forward_varlen_prefill.py`:
-  its 18 dirs hold only legacy ncu backups, no traces).
+- SASS-H200-ML replays the ML traces in the static depot
+  `accel-sim/H200/hw_run` (`traces/device-7/12.8/<w>/<a>/traces`, HW in
+  `device-7/12.8/<w>/<a>/{ncu.csv.nsight,ncu.csv.gpc__cycles_elapsed.0}`),
+  copied 2026-10-20 from `accel-sim/H200_ml_traces` (another user's depot:
+  `<w>/<a>/{processed_traces,ncu/full,ncu/cycles}`) by that depot's
+  `accelsim-layout.sh` layer: the kernels in `H200/ML-DROPPED-KERNELS.md` are
+  left out of `kernelslist.g` and their HW rows dropped too, so both sides pair
+  one-to-one; `ncu.csv.gpc__cycles_elapsed.0` is ncu_cycles.0.csv in the long
+  one-metric format. Not in it: `llama_inference_32k` (no traces), FA3
+  (`benchmark_hopper_forward_varlen_prefill.py`: no traces). Like every
+  static-depot trace, nothing updates them automatically.
 - ~9K per-kernel jobs: `run_simulations.py --per-kernel --array` submits each
   app's kernels as Slurm job arrays (<= MaxArraySize = 1001 tasks), one queue
   record while pending, instead of ~9K sbatch calls against the cluster-wide