Stop missing binaries aborting hw runs; dedupe the runner scripts; aarch64 NVBit

util/hw_stats/pm_cupti_tools/run_cupti.py

@@ -102,31 +102,15 @@
             cuda_version,
             run_name,
         )
-        if not os.path.exists(this_run_dir):
-            os.makedirs(this_run_dir)
-
-        # link the data directory
-        try:
-            benchmark_data_dir = common.dir_option_test(
-                os.path.join(ddir, exe, "data"), "", this_directory
-            )
-            if os.path.lexists(os.path.join(this_run_dir, "data")):
-                os.remove(os.path.join(this_run_dir, "data"))
-            os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data"))
-        except common.PathMissing:
-            pass
-
-        all_data_link = os.path.join(this_run_dir, "data_dirs")
-        if os.path.lexists(all_data_link):
-            os.remove(all_data_link)
-        top_data_dir_path = common.dir_option_test(ddir, "", this_directory)
-        os.symlink(top_data_dir_path, all_data_link)
+        exec_path = common.prepare_run_dir(
+            this_run_dir, ddir, edir, exe, this_directory
+        )
+        if exec_path is None:
+            continue
 
         if args is None:
             args = ""
 
-        exec_path = common.file_option_test(os.path.join(edir, exe), "", this_directory)
-
         # Build the run script with CUPTI injection
         sh_contents = f"""#!/bin/bash
 
@@ -145,10 +129,7 @@
 {exec_path} {args}
 """
 
-        run_script = os.path.join(this_run_dir, "run.sh")
-        with open(run_script, "w") as f:
-            f.write(sh_contents)
-        subprocess.call(["chmod", "u+x", run_script])
+        run_script = common.write_run_script(this_run_dir, "run.sh", sh_contents)
 
         if not options.norun:
             saved_dir = os.getcwd()

util/hw_stats/run_hw.py

@@ -190,30 +190,14 @@
             cuda_version,
             run_name,
         )
-        if not os.path.exists(this_run_dir):
-            os.makedirs(this_run_dir)
-
-        # link the data directory
-        try:
-            benchmark_data_dir = common.dir_option_test(
-                os.path.join(ddir, exe, "data"), "", this_directory
-            )
-            if os.path.lexists(os.path.join(this_run_dir, "data")):
-                os.remove(os.path.join(this_run_dir, "data"))
-            os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data"))
-        except common.PathMissing:
-            pass
-
-        all_data_link = os.path.join(this_run_dir, "data_dirs")
-        if os.path.lexists(all_data_link):
-            os.remove(all_data_link)
-        top_data_dir_path = common.dir_option_test(ddir, "", this_directory)
-        os.symlink(top_data_dir_path, all_data_link)
+        exec_path = common.prepare_run_dir(
+            this_run_dir, ddir, edir, exe, this_directory
+        )
+        if exec_path is None:
+            continue
 
         if args == None:
             args = ""
-
-        exec_path = common.file_option_test(os.path.join(edir, exe), "", this_directory)
         sh_contents = ""
         kernel_number = ""
         if "mlperf" in exec_path:
@@ -379,7 +363,9 @@
                     + str(args)
                     + "; nsys stats -f csv --report cuda_gpu_trace "
                     + os.path.join(this_run_dir, "out.nsys-rep")
-                    + " | tee cycles.csv; python "
+                    + " | tee cycles.csv; "
+                    + sys.executable
+                    + " "
                     + os.path.join(this_directory, "postprocess-nsys-csv.py")
                     + " --path "
                     + this_run_dir
@@ -419,9 +405,7 @@
                     )
                 )
 
-        open(os.path.join(this_run_dir, "run.sh"), "w").write(sh_contents)
-        if subprocess.call(["chmod", "u+x", os.path.join(this_run_dir, "run.sh")]) != 0:
-            exit("Error chmod runfile")
+        common.write_run_script(this_run_dir, "run.sh", sh_contents)
 
         if not options.norun:
             saved_dir = os.getcwd()

util/job_launching/apps/define-all-apps.yml

@@ -645,27 +645,28 @@ Deepbench_nvidia_normal:
     exec_dir: "$GPUAPPS_ROOT/bin/$CUDA_VERSION/release/"
     data_dirs: "$GPUAPPS_ROOT/data_dirs/"
     execs:
-        - conv_bench:
-            - args: train half 7 7 832 16 128 5 5 2 2 1 1
-              accel-sim-mem: 16G
-            - args: train half 7 7 512 8 512 1 1 0 0 1 1
-              accel-sim-mem: 3G
-            - args: train half 7 7 832 16 256 1 1 0 0 1 1
-              accel-sim-mem: 4G
-            - args: train half 7 7 512 8 512 3 3 1 1 1 1
-              accel-sim-mem: 9G
-            - args: train half 14 14 512 16 512 3 3 1 1 1 1
-              accel-sim-mem: 17G
-            - args: inference half 700 161 1 1 32 20 5 0 0 2 2
-              accel-sim-mem: 2G
-            - args: inference half 700 161 1 2 32 20 5 0 0 2 2
-              accel-sim-mem: 2G
-            - args: inference half 341 79 32 1 32 10 5 0 0 2 2
-              accel-sim-mem: 6G
-            - args: inference half 341 79 32 2 32 10 5 0 0 2 2
-              accel-sim-mem: 80G
-            - args: inference half 480 48 1 1 16 3 3 1 1 1 1
-              accel-sim-mem: 2G
+        # not built: they use cuDNN APIs removed in cuDNN 9.
+#        - conv_bench:
+#            - args: train half 7 7 832 16 128 5 5 2 2 1 1
+#              accel-sim-mem: 16G
+#            - args: train half 7 7 512 8 512 1 1 0 0 1 1
+#              accel-sim-mem: 3G
+#            - args: train half 7 7 832 16 256 1 1 0 0 1 1
+#              accel-sim-mem: 4G
+#            - args: train half 7 7 512 8 512 3 3 1 1 1 1
+#              accel-sim-mem: 9G
+#            - args: train half 14 14 512 16 512 3 3 1 1 1 1
+#              accel-sim-mem: 17G
+#            - args: inference half 700 161 1 1 32 20 5 0 0 2 2
+#              accel-sim-mem: 2G
+#            - args: inference half 700 161 1 2 32 20 5 0 0 2 2
+#              accel-sim-mem: 2G
+#            - args: inference half 341 79 32 1 32 10 5 0 0 2 2
+#              accel-sim-mem: 6G
+#            - args: inference half 341 79 32 2 32 10 5 0 0 2 2
+#              accel-sim-mem: 80G
+#            - args: inference half 480 48 1 1 16 3 3 1 1 1 1
+#              accel-sim-mem: 2G
         - gemm_bench:
             - args: train half 1760 7000 1760 0 0
               accel-sim-mem: 18G
@@ -687,35 +688,35 @@ Deepbench_nvidia_normal:
               accel-sim-mem: 7G
             - args: inference half 35 1500 2560 0 0
               accel-sim-mem: 2G
-        - rnn_bench:
-            - args: inference half 512 1 25 lstm
-              accel-sim-mem: 12G
-            - args: inference half 512 2 25 lstm
-              accel-sim-mem: 12G
-            - args: inference half 1024 1 25 lstm
-              accel-sim-mem: 12G
-            - args: inference half 256 4 150 lstm
-              accel-sim-mem: 12G
-            - args: inference half 1536 1 50 lstm
-              accel-sim-mem: 12G
-            - args: inference half 512 1 1 gru
-              accel-sim-mem: 12G
-            - args: inference half 512 2 1 gru
-              accel-sim-mem: 12G
-            - args: inference half 512 4 1 gru
-              accel-sim-mem: 12G
-            - args: inference half 1536 1 750 gru
-              accel-sim-mem: 12G
-            - args: train half 128 32 15 lstm
-              accel-sim-mem: 20G
-            - args: train half 128 64 15 lstm
-              accel-sim-mem: 20G
-            - args: train half 128 32 1 lstm
-              accel-sim-mem: 20G
-            - args: train half 128 64 1 gru
-              accel-sim-mem: 20G
-            - args: train half 128 32 1 gru
-              accel-sim-mem: 20G
+#        - rnn_bench:
+#            - args: inference half 512 1 25 lstm
+#              accel-sim-mem: 12G
+#            - args: inference half 512 2 25 lstm
+#              accel-sim-mem: 12G
+#            - args: inference half 1024 1 25 lstm
+#              accel-sim-mem: 12G
+#            - args: inference half 256 4 150 lstm
+#              accel-sim-mem: 12G
+#            - args: inference half 1536 1 50 lstm
+#              accel-sim-mem: 12G
+#            - args: inference half 512 1 1 gru
+#              accel-sim-mem: 12G
+#            - args: inference half 512 2 1 gru
+#              accel-sim-mem: 12G
+#            - args: inference half 512 4 1 gru
+#              accel-sim-mem: 12G
+#            - args: inference half 1536 1 750 gru
+#              accel-sim-mem: 12G
+#            - args: train half 128 32 15 lstm
+#              accel-sim-mem: 20G
+#            - args: train half 128 64 15 lstm
+#              accel-sim-mem: 20G
+#            - args: train half 128 32 1 lstm
+#              accel-sim-mem: 20G
+#            - args: train half 128 64 1 gru
+#              accel-sim-mem: 20G
+#            - args: train half 128 32 1 gru
+#              accel-sim-mem: 20G
 
 sdk-4.2-scaled:
     exec_dir: "$GPUAPPS_ROOT/bin/$CUDA_VERSION/release/"
@@ -1135,13 +1136,13 @@ dragon-cdp:
 
 
 proxy-apps-doe:
-    exec_dir: "$GPUAPPS_ROOT/src/cuda/proxy-apps-doe/"
+    exec_dir: "$GPUAPPS_ROOT/bin/$CUDA_VERSION/release/"
     data_dirs: "$GPUAPPS_ROOT/data_dirs/proxy-apps-doe/"
     execs:
             # Does not compile
 #         - comd/CoMDCUDA:
 #            - args: p ag -e -x 20 -y 20 -z 20 -n 0 -m thread_atom
-         - cns/cns_all:
+         - cns_all:
             - args: 64 32 32 1
          - XSBench:
             - args: 12 small 0

util/job_launching/common.py

@@ -232,6 +232,57 @@ def dir_option_test(name, default, this_directory):
     return name
 
 
+def prepare_run_dir(this_run_dir, ddir, edir, exe, this_directory):
+    """Set up one benchmark's run directory and resolve its binary.
+
+    Creates the directory, links in the benchmark's own data dir (as "data")
+    and the suite-wide one (as "data_dirs"), then locates the executable.
+
+    Returns the executable path, or None when the benchmark is not built in
+    this tree -- suites differ between branches, so callers skip it rather
+    than aborting every app listed after it.
+    """
+    # Resolve the binary first: a benchmark that is not built should not leave
+    # an empty run directory behind.
+    try:
+        exec_path = file_option_test(os.path.join(edir, exe), "", this_directory)
+    except PathMissing:
+        print("SKIP - no binary for {0}".format(exe))
+        return None
+
+    if not os.path.exists(this_run_dir):
+        os.makedirs(this_run_dir)
+
+    # link the data directory
+    try:
+        benchmark_data_dir = dir_option_test(
+            os.path.join(ddir, exe, "data"), "", this_directory
+        )
+        if os.path.lexists(os.path.join(this_run_dir, "data")):
+            os.remove(os.path.join(this_run_dir, "data"))
+        os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data"))
+    except PathMissing:
+        pass
+
+    all_data_link = os.path.join(this_run_dir, "data_dirs")
+    if os.path.lexists(all_data_link):
+        os.remove(all_data_link)
+    top_data_dir_path = dir_option_test(ddir, "", this_directory)
+    os.symlink(top_data_dir_path, all_data_link)
+
+    return exec_path
+
+
+def write_run_script(this_run_dir, name, contents):
+    """Write an executable run script into a run directory."""
+    path = os.path.join(this_run_dir, name)
+    with open(path, "w") as f:
+        f.write(contents)
+    if subprocess.call(["chmod", "u+x", path]) != 0:
+        exit("Error chmod {0}".format(path))
+    return path
+
+
 def parse_run_simulations_options():
     parser = OptionParser()
     parser.add_option(

util/tracer_nvbit/install_nvbit.sh

@@ -1,8 +1,19 @@
 #!/bin/bash
 export BASH_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )"
 
+# NVBit ships a separate tarball per host architecture; pick the one matching
+# this machine rather than assuming x86_64, so the tracer builds on aarch64
+# (Grace-Hopper, Grace-Blackwell) too.
+NVBIT_VERSION=${NVBIT_VERSION:-1.8}
+NVBIT_ARCH=$(uname -m)
+case "$NVBIT_ARCH" in
+    x86_64|aarch64) ;;
+    *) echo "No NVBit release for $NVBIT_ARCH" >&2; exit 1 ;;
+esac
+NVBIT_TARBALL="nvbit-Linux-${NVBIT_ARCH}-${NVBIT_VERSION}.tar.bz2"
+
 rm -rf $BASH_ROOT/nvbit_release
 mkdir -p "$BASH_ROOT/nvbit_release"
-wget https://github.com/NVlabs/NVBit/releases/download/v1.8/nvbit-Linux-x86_64-1.8.tar.bz2
-tar -xf nvbit-Linux-x86_64-1.8.tar.bz2 -C "$BASH_ROOT/nvbit_release" --strip-components=1
-rm nvbit-Linux-x86_64-1.8.tar.bz2
+wget "https://github.com/NVlabs/NVBit/releases/download/v${NVBIT_VERSION}/${NVBIT_TARBALL}"
+tar -xf "$NVBIT_TARBALL" -C "$BASH_ROOT/nvbit_release" --strip-components=1
+rm "$NVBIT_TARBALL"

util/tracer_nvbit/run_hw_trace.py

@@ -247,13 +247,7 @@ def write_trace_scripts(item, gpu):
         ("run_spinlock_detection.sh", spinlock_contents),
         ("run.sh", run_contents),
     ]:
-        write_script(item["run_dir"], path, content)
-
-
-def write_script(run_dir, path, content):
-    open(os.path.join(run_dir, path), "w").write(content)
-    if subprocess.call(["chmod", "u+x", os.path.join(run_dir, path)]) != 0:
-        exit(f"Error chmod {path} runfile")
+        common.write_run_script(item["run_dir"], path, content)
 
 
 # Set up every app's directory before any GPU work, so path errors show up first
@@ -281,31 +275,16 @@ def write_script(run_dir, path, content):
             )
         )
         this_trace_folder = os.path.join(this_run_dir, "traces")
-        if not os.path.exists(this_run_dir):
-            os.makedirs(this_run_dir)
+        exec_path = common.prepare_run_dir(
+            this_run_dir, ddir, edir, exe, this_directory
+        )
+        if exec_path is None:
+            continue
         if not os.path.exists(this_trace_folder):
             os.makedirs(this_trace_folder)
 
-        # link the data directory
-        try:
-            benchmark_data_dir = common.dir_option_test(
-                os.path.join(ddir, exe, "data"), "", this_directory
-            )
-            if os.path.lexists(os.path.join(this_run_dir, "data")):
-                os.remove(os.path.join(this_run_dir, "data"))
-            os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data"))
-        except common.PathMissing:
-            pass
-
-        all_data_link = os.path.join(this_run_dir, "data_dirs")
-        if os.path.lexists(all_data_link):
-            os.remove(all_data_link)
-        top_data_dir_path = common.dir_option_test(ddir, "", this_directory)
-        os.symlink(top_data_dir_path, all_data_link)
-
         if args == None:
             args = ""
-        exec_path = common.file_option_test(os.path.join(edir, exe), "", this_directory)
         sh_contents = "set -e\n"
 
         if options.terminate_upon_limit:
@@ -357,7 +336,7 @@ def write_script(run_dir, path, content):
             + this_trace_folder
             + "/kernelslist "
         )
-        write_script(this_run_dir, "run_postprocess.sh", postprocess_contents)
+        common.write_run_script(this_run_dir, "run_postprocess.sh", postprocess_contents)
 
         item = {
             "name": run_name,