From 6ec3a0f58627fb5b29d85b9709d0b42dae5161f7 Mon Sep 17 00:00:00 2001 From: JRPAN <25518778+JRPan@users.noreply.github.com> Date: Sat, 17 Oct 2026 19:54:50 -0400 Subject: [PATCH 1/5] Download pre-collected traces from GitHub release assets instead of the FTP server get-accel-sim-traces.py now reads manifest.json from a GitHub release, lists the available GPU/workload/argument sets, downloads the selected tar parts (each under 2 GB), checks every part and the whole tar against SHA-256, and extracts them to hw_run/traces////. Private repositories use GITHUB_TOKEN or the gh login. The old FTP trace listing is removed. Co-Authored-By: Claude Opus 5.5 --- README.md | 10 +- get-accel-sim-traces.py | 412 +++++++++++++++++++++------------------- 2 files changed, 224 insertions(+), 198 deletions(-) diff --git a/README.md b/README.md index bbcbeaf06b..510bd8e8e4 100644 --- a/README.md +++ b/README.md @@ -396,10 +396,16 @@ To disable it entirely, pass `--spinlock_handling none`: The detection tool lives in [`./util/tracer_nvbit/others/spinlock_tool/`](./util/tracer_nvbit/others/spinlock_tool/). **Pre-traced applications.** -To fetch a repository of pre-collected traces: +Pre-collected traces are published as GitHub release assets, one tar per GPU / workload / argument set, split into parts under 2 GB with SHA-256 checksums in the release's `manifest.json`. +To list, download, verify and extract them into `./hw_run/traces////`: ```bash -./get-accel-sim-traces.py +./get-accel-sim-traces.py -l # list what is available +./get-accel-sim-traces.py -a H200/inference_bert.sh/32_256 # one argument set +./get-accel-sim-traces.py -a H200/inference_sdxl.sh,H200/all # GPU[/workload[/args]], 'all' at any level ``` +Each set also carries the matching Nsight Compute CSVs under `ncu/`. +Pass `-T ./hw_run/traces/` to `run_simulations.py`. +Downloading from a private repository needs `GITHUB_TOKEN` or a `gh auth login` session. ### SASS Frontend and Simulation Engine diff --git a/get-accel-sim-traces.py b/get-accel-sim-traces.py index e2350f8fb7..3dc8e6829e 100755 --- a/get-accel-sim-traces.py +++ b/get-accel-sim-traces.py @@ -1,213 +1,233 @@ #!/usr/bin/env python3 +"""Download pre-collected Accel-Sim SASS traces from a GitHub release. -from optparse import OptionParser -import re +Each trace set (one GPU / workload / argument set) is a tar split into parts +smaller than 2 GB. The release's manifest.json lists every part with its +SHA-256; this script downloads the selected parts, verifies them, and extracts +the tar under /traces////. + +Private repositories need a token: set GITHUB_TOKEN, or log in with `gh`. +""" + +import argparse +import hashlib +import json import os -import math +import shutil +import subprocess import sys -from subprocess import Popen, STDOUT +import tarfile +import urllib.error +import urllib.request + +DEFAULT_REPO = "purdue-aalp/accel-sim-framework-private" +DEFAULT_TAG = "h200-ml-traces-test" +API = "https://api.github.com" +CHUNK = 1 << 22 +EXTRACT_KW = {"filter": "data"} if hasattr(tarfile, "data_filter") else {} +this_directory = os.path.dirname(os.path.realpath(__file__)) + + +def human(n): + for unit in ("B", "KB", "MB", "GB", "TB"): + if abs(n) < 1000 or unit == "TB": + return f"{n:.1f} {unit}" if unit != "B" else f"{n} B" + n /= 1000.0 + + +def get_token(): + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + return token + if shutil.which("gh"): + r = subprocess.run(["gh", "auth", "token"], capture_output=True, text=True) + if r.returncode == 0 and r.stdout.strip(): + return r.stdout.strip() + return None + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def open_url(url, token=None, accept=None): + """Open url; with a token, follow the asset redirect without forwarding Authorization.""" + headers = {"User-Agent": "accel-sim-get-traces"} + if accept: + headers["Accept"] = accept + if token: + headers["Authorization"] = f"Bearer {token}" + opener = urllib.request.build_opener(NoRedirect) + try: + return opener.open(urllib.request.Request(url, headers=headers)) + except urllib.error.HTTPError as e: + if e.code in (301, 302, 303, 307, 308): + loc = e.headers["Location"] + return urllib.request.urlopen(urllib.request.Request(loc, headers={"User-Agent": headers["User-Agent"]})) + raise -VERSION = "1.1.0" -WEB_DIRECTORY = "ftp://ftp.ecn.purdue.edu/tgrogers/accel-sim/traces/" -this_directory = os.path.dirname(os.path.realpath(__file__)) + "/" -sys.path.insert(0, os.path.join(this_directory, "util", "job_launching")) -import common -millnames = ["", " K", " M", " G", " T"] +class Release: + def __init__(self, repo, tag, token): + self.repo, self.tag, self.token = repo, tag, token + try: + with open_url(f"{API}/repos/{repo}/releases/tags/{tag}", token, "application/vnd.github+json") as r: + rel = json.load(r) + except urllib.error.HTTPError as e: + hint = "" if token else " (private repositories need GITHUB_TOKEN or `gh auth login`)" + sys.exit(f"Cannot read release {tag} of {repo}: HTTP {e.code}{hint}") + self.assets = {a["name"]: a for a in rel["assets"]} + + def open_asset(self, name): + if name not in self.assets: + sys.exit(f"Asset {name} is not in release {self.tag} of {self.repo}") + a = self.assets[name] + if self.token: + return open_url(a["url"], self.token, "application/octet-stream") + return open_url(a["browser_download_url"]) + + +def sha256_file(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for b in iter(lambda: f.read(CHUNK), b""): + h.update(b) + return h.hexdigest() + + +def download_part(release, part, dest): + """Download one part to dest, skipping it if a verified copy is already there.""" + if os.path.exists(dest) and os.path.getsize(dest) == part["size"] and sha256_file(dest) == part["sha256"]: + print(f" {part['name']}: already downloaded") + return + tmp = dest + ".tmp" + h, done = hashlib.sha256(), 0 + with release.open_asset(part["name"]) as r, open(tmp, "wb") as f: + for b in iter(lambda: r.read(CHUNK), b""): + f.write(b) + h.update(b) + done += len(b) + print(f"\r {part['name']}: {human(done)} / {human(part['size'])}", end="", flush=True) + print() + if done != part["size"] or h.hexdigest() != part["sha256"]: + os.remove(tmp) + sys.exit(f"{part['name']}: size/SHA-256 mismatch (got {done} bytes, expected {part['size']})") + os.replace(tmp, dest) + + +class Concat: + """Read a list of files as one stream, hashing everything read.""" + + def __init__(self, paths): + self.paths, self.f, self.h = list(paths), None, hashlib.sha256() + + def read(self, n=-1): + out = b"" + while n < 0 or len(out) < n: + if self.f is None: + if not self.paths: + break + self.f = open(self.paths.pop(0), "rb") + b = self.f.read(-1 if n < 0 else n - len(out)) + if not b: + self.f.close() + self.f = None + continue + out += b + self.h.update(out) + return out + + +def extract(entry, part_paths, traces_dir): + stream = Concat(part_paths) + with tarfile.open(fileobj=stream, mode="r|") as tar: + root = os.path.realpath(traces_dir) + for m in tar: + target = os.path.realpath(os.path.join(traces_dir, m.name)) + if not (target == root or target.startswith(root + os.sep)) or not (m.isfile() or m.isdir()): + sys.exit(f"Refusing unexpected tar member {m.name}") + if not m.name.startswith(entry["path"] + "/"): + sys.exit(f"Tar member {m.name} is outside {entry['path']}") + tar.extract(m, traces_dir, **EXTRACT_KW) + while stream.read(CHUNK): # hash trailing padding so the whole-tar checksum covers every byte + pass + if stream.h.hexdigest() != entry["sha256"]: + sys.exit(f"{entry['path']}: whole-tar SHA-256 mismatch after extraction") -def getNumRaw(n): - try: - return float(n) - except ValueError: - count = 0 - for name in millnames: - if n[-1].strip() == name.strip(): - n = float(n[:-1].strip()) * 10 ** (3 * count) - break - count += 1 - return float(n) - - -def millify(n): - n = getNumRaw(n) - if math.isnan(n): - return "NaN" - if math.isinf(n): - return "inf" - millidx = max( - 0, - min( - len(millnames) - 1, int(math.floor(0 if n == 0 else math.log10(abs(n)) / 3)) - ), - ) - return "{:.2f}{}".format(n / 10 ** (3 * millidx), millnames[millidx]) - - -class Suite: - def __init__(self, name): - self.name = name - self.uncompressedSize = None - self.compressedSize = None - - -class Card: - def __init__(self, name): - self.name = name - self.suites = {} - - def getTotalCompressed(self): - total = 0.0 - for name, suite in self.suites.items(): - try: - total += suite.compressedSize - except TypeError: - sys.exit("Problem with compressed size in suite {0}".format(name)) - return total - - def getTotalUncompressed(self): - total = 0.0 - for name, suite in self.suites.items(): - total += suite.uncompressedSize - return total - - -def downloadTrace(cardName, suiteName): - webFile = os.path.join( - WEB_DIRECTORY, cardName, VERSION + ".latest", suiteName + ".tgz" - ) - print("\n\nDownloading {0}".format(webFile)) - wget = Popen(["wget " + webFile], stderr=STDOUT, shell=True) - wget.communicate() - if wget.returncode != 0: - sys.exit("wget {0} returned {1}".format(webFile, wget.returncode)) +def select(manifest, selection): + """selection: comma list of GPU[/workload[/args]], each level may be 'all'.""" + chosen = [] + for item in selection.split(","): + want = [x for x in item.strip().split("/") if x] + if not want: + continue + hits = [e for e in manifest["traces"] + if all(w in ("all", v) for w, v in zip(want, (e["gpu"], e["workload"], e["args"])))] + if not hits: + raise ValueError(f"nothing matches '{item}'") + chosen += [e for e in hits if e not in chosen] + return chosen def main(): - parser = OptionParser() - parser.add_option( - "-a", - "--apps", - dest="apps", - default=None, - help="Pass the comma seperated input list instead of asking " - "from stdin. Example: -a tesla-v100/rodinia-3.1,tesla-v100/cudasdk", - ) - parser.add_option( - "-d", - "--download_dir", - dest="download_dir", - default="", - help="Directory to download the traces to.", - ) - (options, args) = parser.parse_args() - try: - hw_run_dir = common.dir_option_test( - options.download_dir, "hw_run", this_directory - ) - except common.PathMissing: - hw_run_dir = os.path.join(this_directory, "hw_run") - - if not os.path.exists(hw_run_dir): - os.makedirs(hw_run_dir) - os.chdir(hw_run_dir) - # Parse the trace summary - trace_summary = os.path.join(hw_run_dir, VERSION + ".trace.summary.txt") - try: - os.remove(trace_summary) - except OSError: - pass - - Popen( - ["wget " + " " + WEB_DIRECTORY + VERSION + ".trace.summary.txt"], - stderr=STDOUT, - shell=True, - ).communicate() - - lineFormat = re.compile(r"(.*)\t(.*)/" + VERSION + ".latest/(.*)") - sizeDict = {} - for line in open(trace_summary): - lineMatch = lineFormat.match(line) - if lineMatch != None: - size = lineMatch.group(1) - cardName = lineMatch.group(2) - fileName = lineMatch.group(3) - suiteName = re.sub(r"(.*)\.tgz", r"\1", fileName) - if cardName not in sizeDict: - sizeDict[cardName] = Card(cardName) - card = sizeDict[cardName] - if suiteName not in card.suites: - newSuite = Suite(suiteName) - card.suites[suiteName] = Suite(suiteName) - - suite = sizeDict[cardName].suites[suiteName] - if suiteName != fileName: - suite.compressedSize = getNumRaw(size) - else: - suite.uncompressedSize = getNumRaw(size) - - # Infor the user what is available - ask them what they want to do - print("\n\nCurrently Available Traces:") - for cardName, card in sizeDict.items(): - print( - "GPU Name: {0}. All Apps Compressed (Download size): {1}, All Apps Uncompressed (Size on disk when used): {2}".format( - card.name, - millify(card.getTotalCompressed()), - millify(card.getTotalUncompressed()), - ) - ) - for name, suite in card.suites.items(): - print( - "\t{0}: Compressed = {1}, Uncompressed = {2}".format( - suite.name, - millify(suite.compressedSize), - millify(suite.uncompressedSize), - ) - ) - - selectionValid = False - while not selectionValid: - if options.apps == None: - selection = input( - "\n-------\nWhat do you want to download?" - "\n, (i.e. tesla-v100/rodinia-3.1,tesla-v100/cudasdk)" - "\n(Default=all/all) : " - ) - if selection == "" or selection == None: - selection = "all/all" - else: - selection = options.apps + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("-a", "--apps", help="Comma separated GPU[/workload[/args]] list, 'all' allowed at any level. " + "Example: -a H200/inference_bert.sh/32_256,H200/inference_sdxl.sh") + p.add_argument("-d", "--download_dir", default=os.path.join(this_directory, "hw_run"), + help="Traces go to /traces/ (default: ./hw_run)") + p.add_argument("-l", "--list", action="store_true", help="List the available traces and exit") + p.add_argument("--repo", default=DEFAULT_REPO, help=f"GitHub repository (default: {DEFAULT_REPO})") + p.add_argument("--tag", default=DEFAULT_TAG, help=f"Release tag (default: {DEFAULT_TAG})") + p.add_argument("--keep_parts", action="store_true", help="Keep the downloaded tar parts after extraction") + o = p.parse_args() + + release = Release(o.repo, o.tag, get_token()) + with release.open_asset("manifest.json") as r: + manifest = json.load(r) + assert manifest.get("format") == 1, f"unsupported manifest format {manifest.get('format')}" + + print(f"Available traces in {o.repo} release {o.tag}:") + for e in sorted(manifest["traces"], key=lambda e: (e["gpu"], e["workload"], e["args"])): + print(f" {e['gpu']}/{e['workload']}/{e['args']}: {e['kernels']} kernels, {human(e['size'])}") + total = sum(e["size"] for e in manifest["traces"]) + print(f" total: {len(manifest['traces'])} trace sets, {human(total)}") + if o.list: + return + + while True: + selection = o.apps or input("\nWhat do you want to download? GPU[/workload[/args]],... (default all): ") or "all" try: - for item in selection.split(","): - cardName = item.split(r"/")[0] - suiteName = item.split(r"/")[1] - if cardName == "all": - for cardName, card in sizeDict.items(): - if suiteName == "all": - for suiteName, suite in card.suites.items(): - downloadTrace(cardName, suiteName) - else: - downloadTrace(cardName, suiteName) - else: - card = sizeDict[cardName] - if suiteName == "all": - for suiteName, suite in card.suites.items(): - downloadTrace(cardName, suiteName) - else: - downloadTrace(cardName, suiteName) - - selectionValid = True - except Exception as e: - selectionValid = False - print("Invalid Input: {0}".format(e)) - if options.apps != None: + chosen = select(manifest, selection) + break + except ValueError as e: + print(f"Invalid input: {e}") + if o.apps: sys.exit(1) - print( - "\n\nDownload successful to {0}.\nFiles must be uncompressed with tar -xzvf to be usable by accel-sim".format( - hw_run_dir - ) - ) + traces_dir = os.path.join(os.path.abspath(o.download_dir), "traces") + parts_dir = os.path.join(os.path.abspath(o.download_dir), ".trace-parts") + os.makedirs(parts_dir, exist_ok=True) + print(f"\nDownloading {len(chosen)} trace sets, {human(sum(e['size'] for e in chosen))}, into {traces_dir}") + for e in chosen: + print(f"\n{e['gpu']}/{e['workload']}/{e['args']} ({len(e['parts'])} parts)") + paths = [] + for part in e["parts"]: + dest = os.path.join(parts_dir, part["name"]) + download_part(release, part, dest) + paths.append(dest) + extract(e, paths, traces_dir) + if not o.keep_parts: + for path in paths: + os.remove(path) + print(f" extracted to {os.path.join(traces_dir, e['path'])}") + if not o.keep_parts: + try: + os.rmdir(parts_dir) + except OSError: + pass + print(f"\nDone. Pass -T {traces_dir}/ to util/job_launching/run_simulations.py.") if __name__ == "__main__": From 5e22170feb96a2d53122a20e8ea2ecb60952a021 Mon Sep 17 00:00:00 2001 From: JRPAN <25518778+JRPan@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:37:17 -0400 Subject: [PATCH 2/5] Add a trace publishing script and put the NCU CSV next to each trace set publish-trace-release.py packs one tar per GPU/workload/argument set, splits it under GitHub's 2 GiB asset limit, uploads the parts and writes manifest.json plus a README.md with download instructions. ncu_stats.csv now sits beside traces/ so the extracted directory works with both run_simulations.py -T and plot-correlation.py -H. get-accel-sim-traces.py defaults to the v2.0.1 release of accel-sim-framework. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- get-accel-sim-traces.py | 12 +- util/tracer_nvbit/publish-trace-release.py | 167 +++++++++++++++++++++ util/tracer_nvbit/trace-release-README.md | 46 ++++++ 4 files changed, 224 insertions(+), 6 deletions(-) create mode 100755 util/tracer_nvbit/publish-trace-release.py create mode 100644 util/tracer_nvbit/trace-release-README.md diff --git a/README.md b/README.md index 510bd8e8e4..e387469cd4 100644 --- a/README.md +++ b/README.md @@ -403,8 +403,9 @@ To list, download, verify and extract them into `./hw_run/traces//` to `run_simulations.py`. +Each set also carries the matching Nsight Compute CSV, `ncu_stats.csv`, next to its `traces/`. +Pass `-T ./hw_run/traces/` to `run_simulations.py` and `-H ./hw_run/traces/` to `plot-correlation.py`. +Maintainers publish new trace sets with `./util/tracer_nvbit/publish-trace-release.py`. Downloading from a private repository needs `GITHUB_TOKEN` or a `gh auth login` session. ### SASS Frontend and Simulation Engine diff --git a/get-accel-sim-traces.py b/get-accel-sim-traces.py index 3dc8e6829e..d9d9251700 100755 --- a/get-accel-sim-traces.py +++ b/get-accel-sim-traces.py @@ -4,7 +4,9 @@ Each trace set (one GPU / workload / argument set) is a tar split into parts smaller than 2 GB. The release's manifest.json lists every part with its SHA-256; this script downloads the selected parts, verifies them, and extracts -the tar under /traces////. +the tar under /traces////: traces/ for +run_simulations.py and ncu_stats.csv for plot-correlation.py. +Traces are published with util/tracer_nvbit/publish-trace-release.py. Private repositories need a token: set GITHUB_TOKEN, or log in with `gh`. """ @@ -20,8 +22,9 @@ import urllib.error import urllib.request -DEFAULT_REPO = "purdue-aalp/accel-sim-framework-private" -DEFAULT_TAG = "h200-ml-traces-test" +DEFAULT_REPO = "accel-sim/accel-sim-framework" +# Traces ship as assets of the code release they were validated with. +DEFAULT_TAG = "v2.0.1" API = "https://api.github.com" CHUNK = 1 << 22 EXTRACT_KW = {"filter": "data"} if hasattr(tarfile, "data_filter") else {} @@ -227,7 +230,8 @@ def main(): os.rmdir(parts_dir) except OSError: pass - print(f"\nDone. Pass -T {traces_dir}/ to util/job_launching/run_simulations.py.") + print(f"\nDone. Use -T {traces_dir}/ with util/job_launching/run_simulations.py" + f" and -H {traces_dir}/ with util/plotting/plot-correlation.py.") if __name__ == "__main__": diff --git a/util/tracer_nvbit/publish-trace-release.py b/util/tracer_nvbit/publish-trace-release.py new file mode 100755 index 0000000000..619182986b --- /dev/null +++ b/util/tracer_nvbit/publish-trace-release.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Publish pre-collected traces as GitHub release assets for get-accel-sim-traces.py. + +Input, one directory per argument set: + ////kernelslist.g + kernel-*.trace(z|g...) + /// (optional Nsight Compute CSV) + +Each argument set becomes one uncompressed tar (traces are already compressed) +laid out as + ///traces/... + ///ncu_stats.csv +so that run_simulations.py -T / and plot-correlation.py -H / +both work after extraction. Tars are split into parts below GitHub's 2 GiB asset +limit, uploaded with `gh release upload`, and described in manifest.json and a\nREADME.md with download instructions. + +Re-running resumes: argument sets already recorded in /meta are skipped. +""" + +import argparse +import glob +import hashlib +import json +import os +import shlex +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor + +PART_BYTES = 1900 * 1024 * 1024 +assert PART_BYTES < 2 * 1024**3, "GitHub release assets must be smaller than 2 GiB" + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for b in iter(lambda: f.read(1 << 24), b""): + h.update(b) + return h.hexdigest() + + +class Publisher: + def __init__(self, o): + self.o = o + self.lock = threading.Lock() + os.makedirs(os.path.join(o.work, "meta"), exist_ok=True) + os.makedirs(os.path.join(o.work, "stage"), exist_ok=True) + + def gh_upload(self, paths): + for attempt in range(5): + r = subprocess.run( + ["gh", "release", "upload", self.o.tag, "-R", self.o.repo, "--clobber", *paths], + capture_output=True, + text=True, + ) + if r.returncode == 0: + return + print(f"upload retry {attempt}: {r.stderr.strip()[:200]}", flush=True) + raise RuntimeError(f"upload failed: {paths}") + + def write_manifest(self): + entries = [json.load(open(f)) for f in sorted(glob.glob(os.path.join(self.o.work, "meta", "*.json")))] + path = os.path.join(self.o.work, "manifest.json") + with open(path, "w") as f: + json.dump(dict(format=1, repo=self.o.repo, tag=self.o.tag, traces=entries), f, indent=1) + readme = os.path.join(self.o.work, "README.md") + with open(readme, "w") as f: + f.write(open(self.o.readme).read().rstrip("\n") + "\n\n") + f.write("| GPU | Workload | Arguments | Kernels | Size | Parts |\n|---|---|---|---|---|---|\n") + for e in sorted(entries, key=lambda e: (e["gpu"], e["workload"], e["args"])): + f.write(f"| {e['gpu']} | {e['workload']} | {e['args']} | {e['kernels']} | {e['size'] / 1e9:.2f} GB | {len(e['parts'])} |\n") + self.gh_upload([path, readme]) + return entries + + def one(self, workload, args): + o = self.o + base = f"{o.gpu}__{workload}__{args}.tar" + meta = os.path.join(o.work, "meta", base + ".json") + if os.path.exists(meta): + print(f"skip {base} (already published)", flush=True) + return + src = os.path.join(o.src, workload, args) + tdir = os.path.join(src, o.traces_subdir) + kernels = [l for l in open(os.path.join(tdir, "kernelslist.g")).read().split("\n") if l.startswith("kernel-")] + files = sorted(f for f in os.listdir(tdir) if f.startswith("kernel-")) + assert sorted(kernels) == files, f"{workload}/{args}: kernelslist.g lists {len(kernels)} kernels, directory has {len(files)} trace files" + pre = f"{o.gpu}/{workload}/{args}" + members = [f"{workload}/{args}/{o.traces_subdir}"] + transforms = [f"s|^{workload}/{args}/{o.traces_subdir}|{pre}/traces|"] + ncu = os.path.join(src, o.ncu_csv) if o.ncu_csv else None + if ncu and os.path.exists(ncu): + members.append(f"{workload}/{args}/{o.ncu_csv}") + transforms.append(f"s|^{workload}/{args}/{o.ncu_csv}|{pre}/ncu_stats.csv|") + elif ncu and o.require_ncu: + sys.exit(f"{workload}/{args}: missing {ncu}") + stage = os.path.join(o.work, "stage", base) + os.makedirs(stage, exist_ok=True) + for p in glob.glob(stage + "/*"): + os.remove(p) + tar = ["tar", "-C", o.src, "--dereference", "--sort=name", "-cf", "-"] + tar += [f"--transform={t}" for t in transforms] + members + cmd = ( + "set -o pipefail; " + + " ".join(map(shlex.quote, tar)) + + f" | tee >(sha256sum | cut -d' ' -f1 > {shlex.quote(stage)}/full.sha256)" + + f" | split -b {PART_BYTES} -d -a 3 - {shlex.quote(os.path.join(stage, base))}.part" + ) + subprocess.run(["bash", "-c", cmd], check=True) + parts = sorted(glob.glob(os.path.join(stage, base + ".part*"))) + assert parts and all(os.path.getsize(p) <= PART_BYTES for p in parts), f"{base}: bad split" + plist = [dict(name=os.path.basename(p), size=os.path.getsize(p), sha256=sha256(p)) for p in parts] + self.gh_upload(parts) + entry = dict( + gpu=o.gpu, + workload=workload, + args=args, + path=pre, + kernels=len(kernels), + size=sum(p["size"] for p in plist), + sha256=open(os.path.join(stage, "full.sha256")).read().strip(), + parts=plist, + ) + with open(meta, "w") as f: + json.dump(entry, f, indent=1) + for p in parts: + os.remove(p) + print(f"published {base}: {len(plist)} parts, {entry['size'] / 1e9:.2f} GB", flush=True) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--src", required=True, help="Root holding //") + p.add_argument("--gpu", required=True, help="GPU name used as the top-level directory, e.g. H200") + p.add_argument("--repo", required=True, help="GitHub repository, e.g. accel-sim/accel-sim-framework") + p.add_argument("--tag", required=True, help="Release tag; the release must already exist") + p.add_argument("--work", required=True, help="Scratch directory for tar parts and per-set metadata") + p.add_argument("--traces_subdir", default="traces", help="Trace directory inside each / (default: traces)") + p.add_argument("--ncu_csv", default="", help="Nsight Compute CSV inside each /, shipped as ncu_stats.csv") + p.add_argument("--require_ncu", action="store_true", help="Fail if an argument set has no NCU CSV") + p.add_argument("--jobs", type=int, default=3, help="Argument sets packed and uploaded in parallel") + p.add_argument( + "--readme", + default=os.path.join(os.path.dirname(os.path.realpath(__file__)), "trace-release-README.md"), + help="Template uploaded as the release's README.md, with a table of the sets appended", + ) + p.add_argument("only", nargs="*", help="Optional / subset") + o = p.parse_args() + + sets = sorted( + (k.split("/")[-4], k.split("/")[-3]) + for k in glob.glob(os.path.join(o.src, "*", "*", o.traces_subdir, "kernelslist.g")) + ) + if o.only: + sets = [s for s in sets if f"{s[0]}/{s[1]}" in o.only] + missing = set(o.only) - {f"{a}/{b}" for a, b in sets} + assert not missing, f"not found under {o.src}: {sorted(missing)}" + assert sets, f"no //{o.traces_subdir}/kernelslist.g under {o.src}" + pub = Publisher(o) + print(f"{len(sets)} argument sets -> {o.repo} release {o.tag}", flush=True) + with ThreadPoolExecutor(o.jobs) as ex: + list(ex.map(lambda s: pub.one(*s), sets)) + entries = pub.write_manifest() + print(f"manifest.json: {len(entries)} argument sets, {sum(e['size'] for e in entries) / 1e9:.1f} GB", flush=True) + + +if __name__ == "__main__": + main() diff --git a/util/tracer_nvbit/trace-release-README.md b/util/tracer_nvbit/trace-release-README.md new file mode 100644 index 0000000000..e13f5bb7f1 --- /dev/null +++ b/util/tracer_nvbit/trace-release-README.md @@ -0,0 +1,46 @@ +# Accel-Sim pre-collected traces + +SASS traces for [Accel-Sim](https://github.com/accel-sim/accel-sim-framework), one set per +GPU / workload / argument set. Each set holds the traces (`traces/kernelslist.g` plus one +`kernel-*.tracez` per kernel) and, where available, the Nsight Compute CSV measured on the +same GPU (`ncu_stats.csv`). + +Every set is one tar, split into parts smaller than 2 GB +(`____.tar.partNNN`). `manifest.json` lists every part with its size +and SHA-256, plus the SHA-256 of the whole tar. + +## Download with the script (recommended) + +From the root of an Accel-Sim checkout: + +```bash +./get-accel-sim-traces.py -l # list the sets in this release +./get-accel-sim-traces.py -a H200/inference_bert.sh/32_256 # one argument set +./get-accel-sim-traces.py -a H200/inference_sdxl.sh # every argument set of a workload +./get-accel-sim-traces.py -a H200/all # everything for one GPU +``` + +The script checks every part and the whole tar against `manifest.json` and extracts to +`./hw_run/traces////`. Use `-d ` for another location, `--keep_parts` +to keep the downloaded parts, and `--repo` / `--tag` to read a different release. + +Then simulate and correlate: + +```bash +./util/job_launching/run_simulations.py -B bert_inference -C H200-SASS -T ./hw_run/traces/H200 -N bert +./util/job_launching/get_stats.py -R -k -K -N bert | tee bert.stats.csv +./util/plotting/plot-correlation.py -c bert.stats.csv -H ./hw_run/traces/H200 +``` + +## Download by hand + +```bash +SET=H200__inference_bert.sh__32_256.tar +# download every $SET.partNNN asset of this release, then: +sha256sum $SET.part* # compare with "parts" in manifest.json +cat $SET.part* > $SET # the parts are plain byte splits, concatenated in order +sha256sum $SET # compare with "sha256" of the set in manifest.json +tar -xf $SET -C ./hw_run/traces +``` + +## Available sets From 97b33526f4c3cceef535c207a236bea0a484719d Mon Sep 17 00:00:00 2001 From: JRPAN <25518778+JRPan@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:37:18 -0400 Subject: [PATCH 3/5] Add v2.0.1 release notes and bump the citation version Co-Authored-By: Claude Opus 5.5 --- CITATION.cff | 2 +- release.notes.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 07516b973c..679060a0c7 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -26,7 +26,7 @@ message: >- type: software url: "https://accel-sim.github.io/" repository-code: "https://github.com/accel-sim/accel-sim-framework" -version: "2.0" +version: "2.0.1" # CFF requires a top-level `authors` key, so Accel-Sim is credited here as a single # entity rather than an enumerated list -- the project has many contributors beyond any # one paper's author list. See the GitHub contributors page for individual credit. diff --git a/release.notes.md b/release.notes.md index cd097d0ec9..17aef027e8 100644 --- a/release.notes.md +++ b/release.notes.md @@ -1,4 +1,51 @@ +# Release v2.0.1 + +_October 2026._ + +A maintenance release on top of 2.0.0: pre-collected traces move from the FTP server to +GitHub release assets, a trace post-processing fix for multi-lane `mbarrier` instructions, +and a reworked CI. + +> **Requires the matching GPGPU-Sim:** `gpgpu-sim_distribution` tag `v2.0.1`. + +## Pre-collected traces on GitHub releases +- Traces are now published as assets of the code release they were validated with, starting + with this one, so code and traces stay consistent. Each GPU / workload / argument set is one + tar, split into parts under 2 GB, and `manifest.json` records the SHA-256 of every part and + of the whole tar. The release also carries a `README.md` with download instructions. +- The first set is the **H200 ML suite**: 20 argument sets of BERT, ResNet-50, DLRMv2, + 3D U-Net, RetinaNet, Whisper, SDXL, Llama 3.1 8B (inference and training), Qwen 2.5 7B, + DeepSeek R1 and Mixtral 8x7B, traced with NVBit 1.8 as `.tracez`. Each argument set comes + with its Nsight Compute CSV (`ncu_stats.csv`). +- `get-accel-sim-traces.py` is rewritten. It lists the available sets (`-l`), downloads a + selection (`-a H200/inference_bert.sh/32_256`, with `all` at any level), checks every + checksum, and extracts to `hw_run/traces////`. That directory works + directly with `run_simulations.py -T` and `plot-correlation.py -H`. The FTP download path + and the pre-2.0 trace listing are removed. +- New `util/tracer_nvbit/publish-trace-release.py` packs and uploads trace sets for + maintainers. + +## Tracer +- `post-traces-processing` decodes every lane's `mbarrier` address when dropping unused + `TRYWAIT`s (#564). It used to read only the first lane of `base_stride` / `base_delta` + lines, so a `TRYWAIT` on an `mbarrier` reached only through another lane could be dropped + and the kernel could deadlock (#561). + +## Job launching +- `slurm.sim` is back to the pre-2.0 template (#556). + +## CI +- PRs run a smoke tier, every `dev` push runs correlated QV100 and H200 suites, and a + single Weekly workflow replaces the Volta and Hopper weeklies, adding the H200 ML suite + (#566). +- Issue triage and PR review automation moved to `purdue-aalp/github-ci`, with Slurm + cleanup and fork-aware fixes (#559). + +## GPGPU-Sim v2.0.1 +- Fixes 47 build warnings and two statistics that read uninitialized memory (#147). +- CI picks its runner pool by repository visibility (#144); README badge cleanup (#146). + # Release v2.0.0 — Accel-Sim 2.0 (full Hopper support) _Aug 25, 2026._ From 1a3cbae474dfa5b3af5b3e74cf9712d09ff23b43 Mon Sep 17 00:00:00 2001 From: JRPAN <25518778+JRPan@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:48:09 -0400 Subject: [PATCH 4/5] Publish and download traces in the standard Accel-Sim layout Trace sets now ship as /traces/device-0////traces/ plus the Nsight Compute stats in /device-0////, extracted under hw_run/, so run_simulations.py -T and plot-correlation.py -H use them directly. Only the kernels listed in each kernelslist.g are packed. The manifest moves to format 2 with per-set trace and hardware directories, and the download checks that every member stays inside them and that the kernel count matches. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +- get-accel-sim-traces.py | 51 ++++++++----- release.notes.md | 7 +- util/tracer_nvbit/publish-trace-release.py | 85 ++++++++++++---------- util/tracer_nvbit/trace-release-README.md | 24 +++--- 5 files changed, 101 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index e387469cd4..745bb10fda 100644 --- a/README.md +++ b/README.md @@ -397,14 +397,14 @@ The detection tool lives in [`./util/tracer_nvbit/others/spinlock_tool/`](./util **Pre-traced applications.** Pre-collected traces are published as GitHub release assets, one tar per GPU / workload / argument set, split into parts under 2 GB with SHA-256 checksums in the release's `manifest.json`. -To list, download, verify and extract them into `./hw_run/traces////`: +To list, download, verify and extract them under `./hw_run//` in the standard layout (`traces/device-0////traces/` plus the Nsight Compute stats in `device-0////`): ```bash ./get-accel-sim-traces.py -l # list what is available ./get-accel-sim-traces.py -a H200/inference_bert.sh/32_256 # one argument set ./get-accel-sim-traces.py -a H200/inference_sdxl.sh,H200/all # GPU[/workload[/args]], 'all' at any level ``` -Each set also carries the matching Nsight Compute CSV, `ncu_stats.csv`, next to its `traces/`. -Pass `-T ./hw_run/traces/` to `run_simulations.py` and `-H ./hw_run/traces/` to `plot-correlation.py`. +Pass `-T ./hw_run//traces/device-0/` to `run_simulations.py` and `-H ./hw_run/` to `plot-correlation.py`. +The release's `README.md` lists every set and how to download one by hand. Maintainers publish new trace sets with `./util/tracer_nvbit/publish-trace-release.py`. Downloading from a private repository needs `GITHUB_TOKEN` or a `gh auth login` session. diff --git a/get-accel-sim-traces.py b/get-accel-sim-traces.py index d9d9251700..206139f77a 100755 --- a/get-accel-sim-traces.py +++ b/get-accel-sim-traces.py @@ -4,8 +4,9 @@ Each trace set (one GPU / workload / argument set) is a tar split into parts smaller than 2 GB. The release's manifest.json lists every part with its SHA-256; this script downloads the selected parts, verifies them, and extracts -the tar under /traces////: traces/ for -run_simulations.py and ncu_stats.csv for plot-correlation.py. +the tar under (default ./hw_run) in the standard Accel-Sim layout: + /traces/device-0////traces/ run_simulations.py -T + /device-0////ncu.csv.* plot-correlation.py -H Traces are published with util/tracer_nvbit/publish-trace-release.py. Private repositories need a token: set GITHUB_TOKEN, or log in with `gh`. @@ -142,21 +143,30 @@ def read(self, n=-1): return out -def extract(entry, part_paths, traces_dir): +def set_name(e): + return f"{e['gpu']}/{e['workload']}/{e['args']}" + + +def extract(entry, part_paths, out_dir): + """Extract one set; every member must be a file or directory inside its trace or hw directory.""" + allowed = [d for d in (entry["trace_dir"], entry["hw_dir"]) if d] stream = Concat(part_paths) with tarfile.open(fileobj=stream, mode="r|") as tar: - root = os.path.realpath(traces_dir) + root = os.path.realpath(out_dir) for m in tar: - target = os.path.realpath(os.path.join(traces_dir, m.name)) - if not (target == root or target.startswith(root + os.sep)) or not (m.isfile() or m.isdir()): + target = os.path.realpath(os.path.join(out_dir, m.name)) + if not target.startswith(root + os.sep) or not (m.isfile() or m.isdir()): sys.exit(f"Refusing unexpected tar member {m.name}") - if not m.name.startswith(entry["path"] + "/"): - sys.exit(f"Tar member {m.name} is outside {entry['path']}") - tar.extract(m, traces_dir, **EXTRACT_KW) + if not any(m.name == d or m.name.startswith(d + "/") for d in allowed): + sys.exit(f"Tar member {m.name} is outside {allowed}") + tar.extract(m, out_dir, **EXTRACT_KW) while stream.read(CHUNK): # hash trailing padding so the whole-tar checksum covers every byte pass if stream.h.hexdigest() != entry["sha256"]: - sys.exit(f"{entry['path']}: whole-tar SHA-256 mismatch after extraction") + sys.exit(f"{set_name(entry)}: whole-tar SHA-256 mismatch after extraction") + listed = [l for l in open(os.path.join(out_dir, entry["trace_dir"], "kernelslist.g")) if l.startswith("kernel-")] + if len(listed) != entry["kernels"]: + sys.exit(f"{set_name(entry)}: kernelslist.g has {len(listed)} kernels, manifest says {entry['kernels']}") def select(manifest, selection): @@ -179,7 +189,7 @@ def main(): p.add_argument("-a", "--apps", help="Comma separated GPU[/workload[/args]] list, 'all' allowed at any level. " "Example: -a H200/inference_bert.sh/32_256,H200/inference_sdxl.sh") p.add_argument("-d", "--download_dir", default=os.path.join(this_directory, "hw_run"), - help="Traces go to /traces/ (default: ./hw_run)") + help="Extract into // (default: ./hw_run)") p.add_argument("-l", "--list", action="store_true", help="List the available traces and exit") p.add_argument("--repo", default=DEFAULT_REPO, help=f"GitHub repository (default: {DEFAULT_REPO})") p.add_argument("--tag", default=DEFAULT_TAG, help=f"Release tag (default: {DEFAULT_TAG})") @@ -189,7 +199,7 @@ def main(): release = Release(o.repo, o.tag, get_token()) with release.open_asset("manifest.json") as r: manifest = json.load(r) - assert manifest.get("format") == 1, f"unsupported manifest format {manifest.get('format')}" + assert manifest.get("format") == 2, f"unsupported manifest format {manifest.get('format')}, expected 2" print(f"Available traces in {o.repo} release {o.tag}:") for e in sorted(manifest["traces"], key=lambda e: (e["gpu"], e["workload"], e["args"])): @@ -209,10 +219,10 @@ def main(): if o.apps: sys.exit(1) - traces_dir = os.path.join(os.path.abspath(o.download_dir), "traces") - parts_dir = os.path.join(os.path.abspath(o.download_dir), ".trace-parts") + out_dir = os.path.abspath(o.download_dir) + parts_dir = os.path.join(out_dir, ".trace-parts") os.makedirs(parts_dir, exist_ok=True) - print(f"\nDownloading {len(chosen)} trace sets, {human(sum(e['size'] for e in chosen))}, into {traces_dir}") + print(f"\nDownloading {len(chosen)} trace sets, {human(sum(e['size'] for e in chosen))}, into {out_dir}") for e in chosen: print(f"\n{e['gpu']}/{e['workload']}/{e['args']} ({len(e['parts'])} parts)") paths = [] @@ -220,18 +230,21 @@ def main(): dest = os.path.join(parts_dir, part["name"]) download_part(release, part, dest) paths.append(dest) - extract(e, paths, traces_dir) + extract(e, paths, out_dir) if not o.keep_parts: for path in paths: os.remove(path) - print(f" extracted to {os.path.join(traces_dir, e['path'])}") + print(f" extracted to {os.path.join(out_dir, e['trace_dir'])}") if not o.keep_parts: try: os.rmdir(parts_dir) except OSError: pass - print(f"\nDone. Use -T {traces_dir}/ with util/job_launching/run_simulations.py" - f" and -H {traces_dir}/ with util/plotting/plot-correlation.py.") + print("\nDone.") + for t, h in sorted({(e["trace_root"], e["hw_root"]) for e in chosen}): + gpu_dir = os.path.join(out_dir, t.split("/")[0]) + print(f" run_simulations.py -T {os.path.join(out_dir, t)}") + print(f" plot-correlation.py -H {gpu_dir}") if __name__ == "__main__": diff --git a/release.notes.md b/release.notes.md index 17aef027e8..c066f7cb18 100644 --- a/release.notes.md +++ b/release.notes.md @@ -17,11 +17,12 @@ and a reworked CI. - The first set is the **H200 ML suite**: 20 argument sets of BERT, ResNet-50, DLRMv2, 3D U-Net, RetinaNet, Whisper, SDXL, Llama 3.1 8B (inference and training), Qwen 2.5 7B, DeepSeek R1 and Mixtral 8x7B, traced with NVBit 1.8 as `.tracez`. Each argument set comes - with its Nsight Compute CSV (`ncu_stats.csv`). + with its Nsight Compute stats, row-aligned with `kernelslist.g`. A few kernels that do not + complete in simulation are left out of both. - `get-accel-sim-traces.py` is rewritten. It lists the available sets (`-l`), downloads a selection (`-a H200/inference_bert.sh/32_256`, with `all` at any level), checks every - checksum, and extracts to `hw_run/traces////`. That directory works - directly with `run_simulations.py -T` and `plot-correlation.py -H`. The FTP download path + checksum, and extracts under `hw_run//` in the standard layout, ready for + `run_simulations.py -T hw_run//traces/device-0/` and `plot-correlation.py -H hw_run/`. The FTP download path and the pre-2.0 trace listing are removed. - New `util/tracer_nvbit/publish-trace-release.py` packs and uploads trace sets for maintainers. diff --git a/util/tracer_nvbit/publish-trace-release.py b/util/tracer_nvbit/publish-trace-release.py index 619182986b..d6ab494d5e 100755 --- a/util/tracer_nvbit/publish-trace-release.py +++ b/util/tracer_nvbit/publish-trace-release.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 """Publish pre-collected traces as GitHub release assets for get-accel-sim-traces.py. -Input, one directory per argument set: - ////kernelslist.g + kernel-*.trace(z|g...) - /// (optional Nsight Compute CSV) +Input is the standard Accel-Sim layout under --src: + ////traces/kernelslist.g + kernel-* trace files + ////*.csv* (Nsight Compute stats, optional) +with --trace_root traces/device-0/ and --hw_root device-0/ by default. Each argument set becomes one uncompressed tar (traces are already compressed) -laid out as - ///traces/... - ///ncu_stats.csv -so that run_simulations.py -T / and plot-correlation.py -H / -both work after extraction. Tars are split into parts below GitHub's 2 GiB asset -limit, uploaded with `gh release upload`, and described in manifest.json and a\nREADME.md with download instructions. +holding exactly the kernels listed in its kernelslist.g, prefixed with the GPU +name, e.g. + H200/traces/device-0/12.8///traces/... + H200/device-0/12.8///ncu.csv.nsight +Extracted under hw_run/, that is run_simulations.py -T hw_run/H200/traces/device-0/12.8 +and plot-correlation.py -H hw_run/H200. Tars are split below GitHub's 2 GiB asset +limit, uploaded with `gh release upload`, and described in manifest.json and a +README.md with download instructions. Re-running resumes: argument sets already recorded in /meta are skipped. """ @@ -23,7 +26,6 @@ import os import shlex import subprocess -import sys import threading from concurrent.futures import ThreadPoolExecutor @@ -62,13 +64,15 @@ def write_manifest(self): entries = [json.load(open(f)) for f in sorted(glob.glob(os.path.join(self.o.work, "meta", "*.json")))] path = os.path.join(self.o.work, "manifest.json") with open(path, "w") as f: - json.dump(dict(format=1, repo=self.o.repo, tag=self.o.tag, traces=entries), f, indent=1) + json.dump(dict(format=2, repo=self.o.repo, tag=self.o.tag, traces=entries), f, indent=1) readme = os.path.join(self.o.work, "README.md") with open(readme, "w") as f: f.write(open(self.o.readme).read().rstrip("\n") + "\n\n") f.write("| GPU | Workload | Arguments | Kernels | Size | Parts |\n|---|---|---|---|---|---|\n") for e in sorted(entries, key=lambda e: (e["gpu"], e["workload"], e["args"])): - f.write(f"| {e['gpu']} | {e['workload']} | {e['args']} | {e['kernels']} | {e['size'] / 1e9:.2f} GB | {len(e['parts'])} |\n") + f.write( + f"| {e['gpu']} | {e['workload']} | {e['args']} | {e['kernels']} | {e['size'] / 1e9:.2f} GB | {len(e['parts'])} |\n" + ) self.gh_upload([path, readme]) return entries @@ -79,26 +83,27 @@ def one(self, workload, args): if os.path.exists(meta): print(f"skip {base} (already published)", flush=True) return - src = os.path.join(o.src, workload, args) - tdir = os.path.join(src, o.traces_subdir) - kernels = [l for l in open(os.path.join(tdir, "kernelslist.g")).read().split("\n") if l.startswith("kernel-")] - files = sorted(f for f in os.listdir(tdir) if f.startswith("kernel-")) - assert sorted(kernels) == files, f"{workload}/{args}: kernelslist.g lists {len(kernels)} kernels, directory has {len(files)} trace files" - pre = f"{o.gpu}/{workload}/{args}" - members = [f"{workload}/{args}/{o.traces_subdir}"] - transforms = [f"s|^{workload}/{args}/{o.traces_subdir}|{pre}/traces|"] - ncu = os.path.join(src, o.ncu_csv) if o.ncu_csv else None - if ncu and os.path.exists(ncu): - members.append(f"{workload}/{args}/{o.ncu_csv}") - transforms.append(f"s|^{workload}/{args}/{o.ncu_csv}|{pre}/ncu_stats.csv|") - elif ncu and o.require_ncu: - sys.exit(f"{workload}/{args}: missing {ncu}") + tdir = os.path.join(o.trace_root, workload, args, "traces") + hdir = os.path.join(o.hw_root, workload, args) + klist = open(os.path.join(o.src, tdir, "kernelslist.g")).read().split("\n") + kernels = [l.strip() for l in klist if l.startswith("kernel-")] + assert len(set(kernels)) == len(kernels), f"{workload}/{args}: duplicate kernel in kernelslist.g" + members = [os.path.join(tdir, "kernelslist.g")] + for k in kernels: + assert os.path.isfile(os.path.join(o.src, tdir, k)), f"{workload}/{args}: kernelslist.g lists {k}, file missing" + members.append(os.path.join(tdir, k)) + hw = sorted(glob.glob(os.path.join(o.src, hdir, "*.csv*"))) + if o.require_hw: + assert hw, f"{workload}/{args}: no hardware CSV in {os.path.join(o.src, hdir)}" + members += [os.path.relpath(h, o.src) for h in hw] stage = os.path.join(o.work, "stage", base) os.makedirs(stage, exist_ok=True) for p in glob.glob(stage + "/*"): os.remove(p) - tar = ["tar", "-C", o.src, "--dereference", "--sort=name", "-cf", "-"] - tar += [f"--transform={t}" for t in transforms] + members + listing = os.path.join(stage, "members.txt") + with open(listing, "w") as f: + f.write("\n".join(members) + "\n") + tar = ["tar", "-C", o.src, "--dereference", "-cf", "-", f"--transform=s|^|{o.gpu}/|", "-T", listing] cmd = ( "set -o pipefail; " + " ".join(map(shlex.quote, tar)) @@ -114,7 +119,10 @@ def one(self, workload, args): gpu=o.gpu, workload=workload, args=args, - path=pre, + trace_dir=f"{o.gpu}/{tdir}", + hw_dir=f"{o.gpu}/{hdir}" if hw else None, + trace_root=f"{o.gpu}/{o.trace_root}", + hw_root=f"{o.gpu}/{o.hw_root}", kernels=len(kernels), size=sum(p["size"] for p in plist), sha256=open(os.path.join(stage, "full.sha256")).read().strip(), @@ -124,19 +132,19 @@ def one(self, workload, args): json.dump(entry, f, indent=1) for p in parts: os.remove(p) - print(f"published {base}: {len(plist)} parts, {entry['size'] / 1e9:.2f} GB", flush=True) + print(f"published {base}: {len(kernels)} kernels, {len(plist)} parts, {entry['size'] / 1e9:.2f} GB", flush=True) def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("--src", required=True, help="Root holding //") + p.add_argument("--src", required=True, help="Root holding / and /") p.add_argument("--gpu", required=True, help="GPU name used as the top-level directory, e.g. H200") p.add_argument("--repo", required=True, help="GitHub repository, e.g. accel-sim/accel-sim-framework") p.add_argument("--tag", required=True, help="Release tag; the release must already exist") p.add_argument("--work", required=True, help="Scratch directory for tar parts and per-set metadata") - p.add_argument("--traces_subdir", default="traces", help="Trace directory inside each / (default: traces)") - p.add_argument("--ncu_csv", default="", help="Nsight Compute CSV inside each /, shipped as ncu_stats.csv") - p.add_argument("--require_ncu", action="store_true", help="Fail if an argument set has no NCU CSV") + p.add_argument("--trace_root", default="traces/device-0/12.8", help="Trace tree under --src") + p.add_argument("--hw_root", default="device-0/12.8", help="Hardware stats tree under --src") + p.add_argument("--require_hw", action="store_true", help="Fail if an argument set has no hardware CSV") p.add_argument("--jobs", type=int, default=3, help="Argument sets packed and uploaded in parallel") p.add_argument( "--readme", @@ -146,15 +154,16 @@ def main(): p.add_argument("only", nargs="*", help="Optional / subset") o = p.parse_args() + root = os.path.join(o.src, o.trace_root) sets = sorted( - (k.split("/")[-4], k.split("/")[-3]) - for k in glob.glob(os.path.join(o.src, "*", "*", o.traces_subdir, "kernelslist.g")) + tuple(os.path.relpath(k, root).split(os.sep)[:2]) + for k in glob.glob(os.path.join(root, "*", "*", "traces", "kernelslist.g")) ) if o.only: sets = [s for s in sets if f"{s[0]}/{s[1]}" in o.only] missing = set(o.only) - {f"{a}/{b}" for a, b in sets} - assert not missing, f"not found under {o.src}: {sorted(missing)}" - assert sets, f"no //{o.traces_subdir}/kernelslist.g under {o.src}" + assert not missing, f"not found under {root}: {sorted(missing)}" + assert sets, f"no //traces/kernelslist.g under {root}" pub = Publisher(o) print(f"{len(sets)} argument sets -> {o.repo} release {o.tag}", flush=True) with ThreadPoolExecutor(o.jobs) as ex: diff --git a/util/tracer_nvbit/trace-release-README.md b/util/tracer_nvbit/trace-release-README.md index e13f5bb7f1..ab355d3dbb 100644 --- a/util/tracer_nvbit/trace-release-README.md +++ b/util/tracer_nvbit/trace-release-README.md @@ -1,9 +1,15 @@ # Accel-Sim pre-collected traces SASS traces for [Accel-Sim](https://github.com/accel-sim/accel-sim-framework), one set per -GPU / workload / argument set. Each set holds the traces (`traces/kernelslist.g` plus one -`kernel-*.tracez` per kernel) and, where available, the Nsight Compute CSV measured on the -same GPU (`ncu_stats.csv`). +GPU / workload / argument set, in the standard Accel-Sim layout: + +``` +/traces/device-0////traces/ kernelslist.g + kernel-*.tracez +/device-0//// Nsight Compute stats (ncu.csv.*) +``` + +The Nsight Compute rows are in the same order as the kernels in `kernelslist.g`. A few +kernels that do not complete in simulation are left out of both. Every set is one tar, split into parts smaller than 2 GB (`____.tar.partNNN`). `manifest.json` lists every part with its size @@ -20,16 +26,16 @@ From the root of an Accel-Sim checkout: ./get-accel-sim-traces.py -a H200/all # everything for one GPU ``` -The script checks every part and the whole tar against `manifest.json` and extracts to -`./hw_run/traces////`. Use `-d ` for another location, `--keep_parts` -to keep the downloaded parts, and `--repo` / `--tag` to read a different release. +The script checks every part and the whole tar against `manifest.json` and extracts under +`./hw_run/`. Use `-d ` for another location, `--keep_parts` to keep the downloaded +parts, and `--repo` / `--tag` to read a different release. Then simulate and correlate: ```bash -./util/job_launching/run_simulations.py -B bert_inference -C H200-SASS -T ./hw_run/traces/H200 -N bert +./util/job_launching/run_simulations.py -B bert_inference -C H200-SASS -T ./hw_run/H200/traces/device-0/12.8 -N bert ./util/job_launching/get_stats.py -R -k -K -N bert | tee bert.stats.csv -./util/plotting/plot-correlation.py -c bert.stats.csv -H ./hw_run/traces/H200 +./util/plotting/plot-correlation.py -c bert.stats.csv -H ./hw_run/H200 ``` ## Download by hand @@ -40,7 +46,7 @@ SET=H200__inference_bert.sh__32_256.tar sha256sum $SET.part* # compare with "parts" in manifest.json cat $SET.part* > $SET # the parts are plain byte splits, concatenated in order sha256sum $SET # compare with "sha256" of the set in manifest.json -tar -xf $SET -C ./hw_run/traces +tar -xf $SET -C ./hw_run ``` ## Available sets From 75f17b51b855e588e29cfbf7f6fe2bb8ac7fa512 Mon Sep 17 00:00:00 2001 From: JRPAN <25518778+JRPan@users.noreply.github.com> Date: Tue, 20 Oct 2026 13:51:20 -0400 Subject: [PATCH 5/5] Move the trace publishing script out of the repository Publishing now lives in the publish-trace-release skill of the AALP accel-sim-research plugin; the repository keeps only the download side. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 - get-accel-sim-traces.py | 1 - release.notes.md | 2 - util/tracer_nvbit/publish-trace-release.py | 176 --------------------- util/tracer_nvbit/trace-release-README.md | 52 ------ 5 files changed, 232 deletions(-) delete mode 100755 util/tracer_nvbit/publish-trace-release.py delete mode 100644 util/tracer_nvbit/trace-release-README.md diff --git a/README.md b/README.md index 745bb10fda..dc1a23d797 100644 --- a/README.md +++ b/README.md @@ -405,7 +405,6 @@ To list, download, verify and extract them under `./hw_run//` in the standa ``` Pass `-T ./hw_run//traces/device-0/` to `run_simulations.py` and `-H ./hw_run/` to `plot-correlation.py`. The release's `README.md` lists every set and how to download one by hand. -Maintainers publish new trace sets with `./util/tracer_nvbit/publish-trace-release.py`. Downloading from a private repository needs `GITHUB_TOKEN` or a `gh auth login` session. ### SASS Frontend and Simulation Engine diff --git a/get-accel-sim-traces.py b/get-accel-sim-traces.py index 206139f77a..85046539d7 100755 --- a/get-accel-sim-traces.py +++ b/get-accel-sim-traces.py @@ -7,7 +7,6 @@ the tar under (default ./hw_run) in the standard Accel-Sim layout: /traces/device-0////traces/ run_simulations.py -T /device-0////ncu.csv.* plot-correlation.py -H -Traces are published with util/tracer_nvbit/publish-trace-release.py. Private repositories need a token: set GITHUB_TOKEN, or log in with `gh`. """ diff --git a/release.notes.md b/release.notes.md index c066f7cb18..da518a9cb6 100644 --- a/release.notes.md +++ b/release.notes.md @@ -24,8 +24,6 @@ and a reworked CI. checksum, and extracts under `hw_run//` in the standard layout, ready for `run_simulations.py -T hw_run//traces/device-0/` and `plot-correlation.py -H hw_run/`. The FTP download path and the pre-2.0 trace listing are removed. -- New `util/tracer_nvbit/publish-trace-release.py` packs and uploads trace sets for - maintainers. ## Tracer - `post-traces-processing` decodes every lane's `mbarrier` address when dropping unused diff --git a/util/tracer_nvbit/publish-trace-release.py b/util/tracer_nvbit/publish-trace-release.py deleted file mode 100755 index d6ab494d5e..0000000000 --- a/util/tracer_nvbit/publish-trace-release.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/env python3 -"""Publish pre-collected traces as GitHub release assets for get-accel-sim-traces.py. - -Input is the standard Accel-Sim layout under --src: - ////traces/kernelslist.g + kernel-* trace files - ////*.csv* (Nsight Compute stats, optional) -with --trace_root traces/device-0/ and --hw_root device-0/ by default. - -Each argument set becomes one uncompressed tar (traces are already compressed) -holding exactly the kernels listed in its kernelslist.g, prefixed with the GPU -name, e.g. - H200/traces/device-0/12.8///traces/... - H200/device-0/12.8///ncu.csv.nsight -Extracted under hw_run/, that is run_simulations.py -T hw_run/H200/traces/device-0/12.8 -and plot-correlation.py -H hw_run/H200. Tars are split below GitHub's 2 GiB asset -limit, uploaded with `gh release upload`, and described in manifest.json and a -README.md with download instructions. - -Re-running resumes: argument sets already recorded in /meta are skipped. -""" - -import argparse -import glob -import hashlib -import json -import os -import shlex -import subprocess -import threading -from concurrent.futures import ThreadPoolExecutor - -PART_BYTES = 1900 * 1024 * 1024 -assert PART_BYTES < 2 * 1024**3, "GitHub release assets must be smaller than 2 GiB" - - -def sha256(path): - h = hashlib.sha256() - with open(path, "rb") as f: - for b in iter(lambda: f.read(1 << 24), b""): - h.update(b) - return h.hexdigest() - - -class Publisher: - def __init__(self, o): - self.o = o - self.lock = threading.Lock() - os.makedirs(os.path.join(o.work, "meta"), exist_ok=True) - os.makedirs(os.path.join(o.work, "stage"), exist_ok=True) - - def gh_upload(self, paths): - for attempt in range(5): - r = subprocess.run( - ["gh", "release", "upload", self.o.tag, "-R", self.o.repo, "--clobber", *paths], - capture_output=True, - text=True, - ) - if r.returncode == 0: - return - print(f"upload retry {attempt}: {r.stderr.strip()[:200]}", flush=True) - raise RuntimeError(f"upload failed: {paths}") - - def write_manifest(self): - entries = [json.load(open(f)) for f in sorted(glob.glob(os.path.join(self.o.work, "meta", "*.json")))] - path = os.path.join(self.o.work, "manifest.json") - with open(path, "w") as f: - json.dump(dict(format=2, repo=self.o.repo, tag=self.o.tag, traces=entries), f, indent=1) - readme = os.path.join(self.o.work, "README.md") - with open(readme, "w") as f: - f.write(open(self.o.readme).read().rstrip("\n") + "\n\n") - f.write("| GPU | Workload | Arguments | Kernels | Size | Parts |\n|---|---|---|---|---|---|\n") - for e in sorted(entries, key=lambda e: (e["gpu"], e["workload"], e["args"])): - f.write( - f"| {e['gpu']} | {e['workload']} | {e['args']} | {e['kernels']} | {e['size'] / 1e9:.2f} GB | {len(e['parts'])} |\n" - ) - self.gh_upload([path, readme]) - return entries - - def one(self, workload, args): - o = self.o - base = f"{o.gpu}__{workload}__{args}.tar" - meta = os.path.join(o.work, "meta", base + ".json") - if os.path.exists(meta): - print(f"skip {base} (already published)", flush=True) - return - tdir = os.path.join(o.trace_root, workload, args, "traces") - hdir = os.path.join(o.hw_root, workload, args) - klist = open(os.path.join(o.src, tdir, "kernelslist.g")).read().split("\n") - kernels = [l.strip() for l in klist if l.startswith("kernel-")] - assert len(set(kernels)) == len(kernels), f"{workload}/{args}: duplicate kernel in kernelslist.g" - members = [os.path.join(tdir, "kernelslist.g")] - for k in kernels: - assert os.path.isfile(os.path.join(o.src, tdir, k)), f"{workload}/{args}: kernelslist.g lists {k}, file missing" - members.append(os.path.join(tdir, k)) - hw = sorted(glob.glob(os.path.join(o.src, hdir, "*.csv*"))) - if o.require_hw: - assert hw, f"{workload}/{args}: no hardware CSV in {os.path.join(o.src, hdir)}" - members += [os.path.relpath(h, o.src) for h in hw] - stage = os.path.join(o.work, "stage", base) - os.makedirs(stage, exist_ok=True) - for p in glob.glob(stage + "/*"): - os.remove(p) - listing = os.path.join(stage, "members.txt") - with open(listing, "w") as f: - f.write("\n".join(members) + "\n") - tar = ["tar", "-C", o.src, "--dereference", "-cf", "-", f"--transform=s|^|{o.gpu}/|", "-T", listing] - cmd = ( - "set -o pipefail; " - + " ".join(map(shlex.quote, tar)) - + f" | tee >(sha256sum | cut -d' ' -f1 > {shlex.quote(stage)}/full.sha256)" - + f" | split -b {PART_BYTES} -d -a 3 - {shlex.quote(os.path.join(stage, base))}.part" - ) - subprocess.run(["bash", "-c", cmd], check=True) - parts = sorted(glob.glob(os.path.join(stage, base + ".part*"))) - assert parts and all(os.path.getsize(p) <= PART_BYTES for p in parts), f"{base}: bad split" - plist = [dict(name=os.path.basename(p), size=os.path.getsize(p), sha256=sha256(p)) for p in parts] - self.gh_upload(parts) - entry = dict( - gpu=o.gpu, - workload=workload, - args=args, - trace_dir=f"{o.gpu}/{tdir}", - hw_dir=f"{o.gpu}/{hdir}" if hw else None, - trace_root=f"{o.gpu}/{o.trace_root}", - hw_root=f"{o.gpu}/{o.hw_root}", - kernels=len(kernels), - size=sum(p["size"] for p in plist), - sha256=open(os.path.join(stage, "full.sha256")).read().strip(), - parts=plist, - ) - with open(meta, "w") as f: - json.dump(entry, f, indent=1) - for p in parts: - os.remove(p) - print(f"published {base}: {len(kernels)} kernels, {len(plist)} parts, {entry['size'] / 1e9:.2f} GB", flush=True) - - -def main(): - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("--src", required=True, help="Root holding / and /") - p.add_argument("--gpu", required=True, help="GPU name used as the top-level directory, e.g. H200") - p.add_argument("--repo", required=True, help="GitHub repository, e.g. accel-sim/accel-sim-framework") - p.add_argument("--tag", required=True, help="Release tag; the release must already exist") - p.add_argument("--work", required=True, help="Scratch directory for tar parts and per-set metadata") - p.add_argument("--trace_root", default="traces/device-0/12.8", help="Trace tree under --src") - p.add_argument("--hw_root", default="device-0/12.8", help="Hardware stats tree under --src") - p.add_argument("--require_hw", action="store_true", help="Fail if an argument set has no hardware CSV") - p.add_argument("--jobs", type=int, default=3, help="Argument sets packed and uploaded in parallel") - p.add_argument( - "--readme", - default=os.path.join(os.path.dirname(os.path.realpath(__file__)), "trace-release-README.md"), - help="Template uploaded as the release's README.md, with a table of the sets appended", - ) - p.add_argument("only", nargs="*", help="Optional / subset") - o = p.parse_args() - - root = os.path.join(o.src, o.trace_root) - sets = sorted( - tuple(os.path.relpath(k, root).split(os.sep)[:2]) - for k in glob.glob(os.path.join(root, "*", "*", "traces", "kernelslist.g")) - ) - if o.only: - sets = [s for s in sets if f"{s[0]}/{s[1]}" in o.only] - missing = set(o.only) - {f"{a}/{b}" for a, b in sets} - assert not missing, f"not found under {root}: {sorted(missing)}" - assert sets, f"no //traces/kernelslist.g under {root}" - pub = Publisher(o) - print(f"{len(sets)} argument sets -> {o.repo} release {o.tag}", flush=True) - with ThreadPoolExecutor(o.jobs) as ex: - list(ex.map(lambda s: pub.one(*s), sets)) - entries = pub.write_manifest() - print(f"manifest.json: {len(entries)} argument sets, {sum(e['size'] for e in entries) / 1e9:.1f} GB", flush=True) - - -if __name__ == "__main__": - main() diff --git a/util/tracer_nvbit/trace-release-README.md b/util/tracer_nvbit/trace-release-README.md deleted file mode 100644 index ab355d3dbb..0000000000 --- a/util/tracer_nvbit/trace-release-README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Accel-Sim pre-collected traces - -SASS traces for [Accel-Sim](https://github.com/accel-sim/accel-sim-framework), one set per -GPU / workload / argument set, in the standard Accel-Sim layout: - -``` -/traces/device-0////traces/ kernelslist.g + kernel-*.tracez -/device-0//// Nsight Compute stats (ncu.csv.*) -``` - -The Nsight Compute rows are in the same order as the kernels in `kernelslist.g`. A few -kernels that do not complete in simulation are left out of both. - -Every set is one tar, split into parts smaller than 2 GB -(`____.tar.partNNN`). `manifest.json` lists every part with its size -and SHA-256, plus the SHA-256 of the whole tar. - -## Download with the script (recommended) - -From the root of an Accel-Sim checkout: - -```bash -./get-accel-sim-traces.py -l # list the sets in this release -./get-accel-sim-traces.py -a H200/inference_bert.sh/32_256 # one argument set -./get-accel-sim-traces.py -a H200/inference_sdxl.sh # every argument set of a workload -./get-accel-sim-traces.py -a H200/all # everything for one GPU -``` - -The script checks every part and the whole tar against `manifest.json` and extracts under -`./hw_run/`. Use `-d ` for another location, `--keep_parts` to keep the downloaded -parts, and `--repo` / `--tag` to read a different release. - -Then simulate and correlate: - -```bash -./util/job_launching/run_simulations.py -B bert_inference -C H200-SASS -T ./hw_run/H200/traces/device-0/12.8 -N bert -./util/job_launching/get_stats.py -R -k -K -N bert | tee bert.stats.csv -./util/plotting/plot-correlation.py -c bert.stats.csv -H ./hw_run/H200 -``` - -## Download by hand - -```bash -SET=H200__inference_bert.sh__32_256.tar -# download every $SET.partNNN asset of this release, then: -sha256sum $SET.part* # compare with "parts" in manifest.json -cat $SET.part* > $SET # the parts are plain byte splits, concatenated in order -sha256sum $SET # compare with "sha256" of the set in manifest.json -tar -xf $SET -C ./hw_run -``` - -## Available sets