diff --git a/util/hw_stats/pm_cupti_tools/run_cupti.py b/util/hw_stats/pm_cupti_tools/run_cupti.py index 5ae1d76a7f..443e288fda 100755 --- a/util/hw_stats/pm_cupti_tools/run_cupti.py +++ b/util/hw_stats/pm_cupti_tools/run_cupti.py @@ -102,31 +102,15 @@ cuda_version, run_name, ) - if not os.path.exists(this_run_dir): - os.makedirs(this_run_dir) - - # link the data directory - try: - benchmark_data_dir = common.dir_option_test( - os.path.join(ddir, exe, "data"), "", this_directory - ) - if os.path.lexists(os.path.join(this_run_dir, "data")): - os.remove(os.path.join(this_run_dir, "data")) - os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data")) - except common.PathMissing: - pass - - all_data_link = os.path.join(this_run_dir, "data_dirs") - if os.path.lexists(all_data_link): - os.remove(all_data_link) - top_data_dir_path = common.dir_option_test(ddir, "", this_directory) - os.symlink(top_data_dir_path, all_data_link) + exec_path = common.prepare_run_dir( + this_run_dir, ddir, edir, exe, this_directory + ) + if exec_path is None: + continue if args is None: args = "" - exec_path = common.file_option_test(os.path.join(edir, exe), "", this_directory) - # Build the run script with CUPTI injection sh_contents = f"""#!/bin/bash @@ -145,10 +129,7 @@ {exec_path} {args} """ - run_script = os.path.join(this_run_dir, "run.sh") - with open(run_script, "w") as f: - f.write(sh_contents) - subprocess.call(["chmod", "u+x", run_script]) + run_script = common.write_run_script(this_run_dir, "run.sh", sh_contents) if not options.norun: saved_dir = os.getcwd() diff --git a/util/hw_stats/run_hw.py b/util/hw_stats/run_hw.py index 8131fdf393..91439d60f8 100755 --- a/util/hw_stats/run_hw.py +++ b/util/hw_stats/run_hw.py @@ -190,30 +190,14 @@ cuda_version, run_name, ) - if not os.path.exists(this_run_dir): - os.makedirs(this_run_dir) - - # link the data directory - try: - benchmark_data_dir = common.dir_option_test( - os.path.join(ddir, exe, "data"), "", this_directory - ) - if os.path.lexists(os.path.join(this_run_dir, "data")): - os.remove(os.path.join(this_run_dir, "data")) - os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data")) - except common.PathMissing: - pass - - all_data_link = os.path.join(this_run_dir, "data_dirs") - if os.path.lexists(all_data_link): - os.remove(all_data_link) - top_data_dir_path = common.dir_option_test(ddir, "", this_directory) - os.symlink(top_data_dir_path, all_data_link) + exec_path = common.prepare_run_dir( + this_run_dir, ddir, edir, exe, this_directory + ) + if exec_path is None: + continue if args == None: args = "" - - exec_path = common.file_option_test(os.path.join(edir, exe), "", this_directory) sh_contents = "" kernel_number = "" if "mlperf" in exec_path: @@ -379,7 +363,9 @@ + str(args) + "; nsys stats -f csv --report cuda_gpu_trace " + os.path.join(this_run_dir, "out.nsys-rep") - + " | tee cycles.csv; python " + + " | tee cycles.csv; " + + sys.executable + + " " + os.path.join(this_directory, "postprocess-nsys-csv.py") + " --path " + this_run_dir @@ -419,9 +405,7 @@ ) ) - open(os.path.join(this_run_dir, "run.sh"), "w").write(sh_contents) - if subprocess.call(["chmod", "u+x", os.path.join(this_run_dir, "run.sh")]) != 0: - exit("Error chmod runfile") + common.write_run_script(this_run_dir, "run.sh", sh_contents) if not options.norun: saved_dir = os.getcwd() diff --git a/util/job_launching/apps/define-all-apps.yml b/util/job_launching/apps/define-all-apps.yml index 5aea018462..d0cdb62d90 100644 --- a/util/job_launching/apps/define-all-apps.yml +++ b/util/job_launching/apps/define-all-apps.yml @@ -645,27 +645,28 @@ Deepbench_nvidia_normal: exec_dir: "$GPUAPPS_ROOT/bin/$CUDA_VERSION/release/" data_dirs: "$GPUAPPS_ROOT/data_dirs/" execs: - - conv_bench: - - args: train half 7 7 832 16 128 5 5 2 2 1 1 - accel-sim-mem: 16G - - args: train half 7 7 512 8 512 1 1 0 0 1 1 - accel-sim-mem: 3G - - args: train half 7 7 832 16 256 1 1 0 0 1 1 - accel-sim-mem: 4G - - args: train half 7 7 512 8 512 3 3 1 1 1 1 - accel-sim-mem: 9G - - args: train half 14 14 512 16 512 3 3 1 1 1 1 - accel-sim-mem: 17G - - args: inference half 700 161 1 1 32 20 5 0 0 2 2 - accel-sim-mem: 2G - - args: inference half 700 161 1 2 32 20 5 0 0 2 2 - accel-sim-mem: 2G - - args: inference half 341 79 32 1 32 10 5 0 0 2 2 - accel-sim-mem: 6G - - args: inference half 341 79 32 2 32 10 5 0 0 2 2 - accel-sim-mem: 80G - - args: inference half 480 48 1 1 16 3 3 1 1 1 1 - accel-sim-mem: 2G + # not built: they use cuDNN APIs removed in cuDNN 9. +# - conv_bench: +# - args: train half 7 7 832 16 128 5 5 2 2 1 1 +# accel-sim-mem: 16G +# - args: train half 7 7 512 8 512 1 1 0 0 1 1 +# accel-sim-mem: 3G +# - args: train half 7 7 832 16 256 1 1 0 0 1 1 +# accel-sim-mem: 4G +# - args: train half 7 7 512 8 512 3 3 1 1 1 1 +# accel-sim-mem: 9G +# - args: train half 14 14 512 16 512 3 3 1 1 1 1 +# accel-sim-mem: 17G +# - args: inference half 700 161 1 1 32 20 5 0 0 2 2 +# accel-sim-mem: 2G +# - args: inference half 700 161 1 2 32 20 5 0 0 2 2 +# accel-sim-mem: 2G +# - args: inference half 341 79 32 1 32 10 5 0 0 2 2 +# accel-sim-mem: 6G +# - args: inference half 341 79 32 2 32 10 5 0 0 2 2 +# accel-sim-mem: 80G +# - args: inference half 480 48 1 1 16 3 3 1 1 1 1 +# accel-sim-mem: 2G - gemm_bench: - args: train half 1760 7000 1760 0 0 accel-sim-mem: 18G @@ -687,35 +688,35 @@ Deepbench_nvidia_normal: accel-sim-mem: 7G - args: inference half 35 1500 2560 0 0 accel-sim-mem: 2G - - rnn_bench: - - args: inference half 512 1 25 lstm - accel-sim-mem: 12G - - args: inference half 512 2 25 lstm - accel-sim-mem: 12G - - args: inference half 1024 1 25 lstm - accel-sim-mem: 12G - - args: inference half 256 4 150 lstm - accel-sim-mem: 12G - - args: inference half 1536 1 50 lstm - accel-sim-mem: 12G - - args: inference half 512 1 1 gru - accel-sim-mem: 12G - - args: inference half 512 2 1 gru - accel-sim-mem: 12G - - args: inference half 512 4 1 gru - accel-sim-mem: 12G - - args: inference half 1536 1 750 gru - accel-sim-mem: 12G - - args: train half 128 32 15 lstm - accel-sim-mem: 20G - - args: train half 128 64 15 lstm - accel-sim-mem: 20G - - args: train half 128 32 1 lstm - accel-sim-mem: 20G - - args: train half 128 64 1 gru - accel-sim-mem: 20G - - args: train half 128 32 1 gru - accel-sim-mem: 20G +# - rnn_bench: +# - args: inference half 512 1 25 lstm +# accel-sim-mem: 12G +# - args: inference half 512 2 25 lstm +# accel-sim-mem: 12G +# - args: inference half 1024 1 25 lstm +# accel-sim-mem: 12G +# - args: inference half 256 4 150 lstm +# accel-sim-mem: 12G +# - args: inference half 1536 1 50 lstm +# accel-sim-mem: 12G +# - args: inference half 512 1 1 gru +# accel-sim-mem: 12G +# - args: inference half 512 2 1 gru +# accel-sim-mem: 12G +# - args: inference half 512 4 1 gru +# accel-sim-mem: 12G +# - args: inference half 1536 1 750 gru +# accel-sim-mem: 12G +# - args: train half 128 32 15 lstm +# accel-sim-mem: 20G +# - args: train half 128 64 15 lstm +# accel-sim-mem: 20G +# - args: train half 128 32 1 lstm +# accel-sim-mem: 20G +# - args: train half 128 64 1 gru +# accel-sim-mem: 20G +# - args: train half 128 32 1 gru +# accel-sim-mem: 20G sdk-4.2-scaled: exec_dir: "$GPUAPPS_ROOT/bin/$CUDA_VERSION/release/" @@ -1135,13 +1136,13 @@ dragon-cdp: proxy-apps-doe: - exec_dir: "$GPUAPPS_ROOT/src/cuda/proxy-apps-doe/" + exec_dir: "$GPUAPPS_ROOT/bin/$CUDA_VERSION/release/" data_dirs: "$GPUAPPS_ROOT/data_dirs/proxy-apps-doe/" execs: # Does not compile # - comd/CoMDCUDA: # - args: p ag -e -x 20 -y 20 -z 20 -n 0 -m thread_atom - - cns/cns_all: + - cns_all: - args: 64 32 32 1 - XSBench: - args: 12 small 0 diff --git a/util/job_launching/common.py b/util/job_launching/common.py index ff8d029478..ccd56b86b6 100644 --- a/util/job_launching/common.py +++ b/util/job_launching/common.py @@ -232,6 +232,57 @@ def dir_option_test(name, default, this_directory): return name +def prepare_run_dir(this_run_dir, ddir, edir, exe, this_directory): + """Set up one benchmark's run directory and resolve its binary. + + Creates the directory, links in the benchmark's own data dir (as "data") + and the suite-wide one (as "data_dirs"), then locates the executable. + + Returns the executable path, or None when the benchmark is not built in + this tree -- suites differ between branches, so callers skip it rather + than aborting every app listed after it. + """ + # Resolve the binary first: a benchmark that is not built should not leave + # an empty run directory behind. + try: + exec_path = file_option_test(os.path.join(edir, exe), "", this_directory) + except PathMissing: + print("SKIP - no binary for {0}".format(exe)) + return None + + if not os.path.exists(this_run_dir): + os.makedirs(this_run_dir) + + # link the data directory + try: + benchmark_data_dir = dir_option_test( + os.path.join(ddir, exe, "data"), "", this_directory + ) + if os.path.lexists(os.path.join(this_run_dir, "data")): + os.remove(os.path.join(this_run_dir, "data")) + os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data")) + except PathMissing: + pass + + all_data_link = os.path.join(this_run_dir, "data_dirs") + if os.path.lexists(all_data_link): + os.remove(all_data_link) + top_data_dir_path = dir_option_test(ddir, "", this_directory) + os.symlink(top_data_dir_path, all_data_link) + + return exec_path + + +def write_run_script(this_run_dir, name, contents): + """Write an executable run script into a run directory.""" + path = os.path.join(this_run_dir, name) + with open(path, "w") as f: + f.write(contents) + if subprocess.call(["chmod", "u+x", path]) != 0: + exit("Error chmod {0}".format(path)) + return path + + def parse_run_simulations_options(): parser = OptionParser() parser.add_option( diff --git a/util/tracer_nvbit/install_nvbit.sh b/util/tracer_nvbit/install_nvbit.sh index 0b489a56ae..26fcde4413 100755 --- a/util/tracer_nvbit/install_nvbit.sh +++ b/util/tracer_nvbit/install_nvbit.sh @@ -1,8 +1,19 @@ #!/bin/bash export BASH_ROOT="$( cd "$( dirname "$BASH_SOURCE" )" && pwd )" +# NVBit ships a separate tarball per host architecture; pick the one matching +# this machine rather than assuming x86_64, so the tracer builds on aarch64 +# (Grace-Hopper, Grace-Blackwell) too. +NVBIT_VERSION=${NVBIT_VERSION:-1.8} +NVBIT_ARCH=$(uname -m) +case "$NVBIT_ARCH" in + x86_64|aarch64) ;; + *) echo "No NVBit release for $NVBIT_ARCH" >&2; exit 1 ;; +esac +NVBIT_TARBALL="nvbit-Linux-${NVBIT_ARCH}-${NVBIT_VERSION}.tar.bz2" + rm -rf $BASH_ROOT/nvbit_release mkdir -p "$BASH_ROOT/nvbit_release" -wget https://github.com/NVlabs/NVBit/releases/download/v1.8/nvbit-Linux-x86_64-1.8.tar.bz2 -tar -xf nvbit-Linux-x86_64-1.8.tar.bz2 -C "$BASH_ROOT/nvbit_release" --strip-components=1 -rm nvbit-Linux-x86_64-1.8.tar.bz2 +wget "https://github.com/NVlabs/NVBit/releases/download/v${NVBIT_VERSION}/${NVBIT_TARBALL}" +tar -xf "$NVBIT_TARBALL" -C "$BASH_ROOT/nvbit_release" --strip-components=1 +rm "$NVBIT_TARBALL" diff --git a/util/tracer_nvbit/run_hw_trace.py b/util/tracer_nvbit/run_hw_trace.py index c7a3c5e44c..eed27643f8 100755 --- a/util/tracer_nvbit/run_hw_trace.py +++ b/util/tracer_nvbit/run_hw_trace.py @@ -247,13 +247,7 @@ def write_trace_scripts(item, gpu): ("run_spinlock_detection.sh", spinlock_contents), ("run.sh", run_contents), ]: - write_script(item["run_dir"], path, content) - - -def write_script(run_dir, path, content): - open(os.path.join(run_dir, path), "w").write(content) - if subprocess.call(["chmod", "u+x", os.path.join(run_dir, path)]) != 0: - exit(f"Error chmod {path} runfile") + common.write_run_script(item["run_dir"], path, content) # Set up every app's directory before any GPU work, so path errors show up first @@ -281,31 +275,16 @@ def write_script(run_dir, path, content): ) ) this_trace_folder = os.path.join(this_run_dir, "traces") - if not os.path.exists(this_run_dir): - os.makedirs(this_run_dir) + exec_path = common.prepare_run_dir( + this_run_dir, ddir, edir, exe, this_directory + ) + if exec_path is None: + continue if not os.path.exists(this_trace_folder): os.makedirs(this_trace_folder) - # link the data directory - try: - benchmark_data_dir = common.dir_option_test( - os.path.join(ddir, exe, "data"), "", this_directory - ) - if os.path.lexists(os.path.join(this_run_dir, "data")): - os.remove(os.path.join(this_run_dir, "data")) - os.symlink(benchmark_data_dir, os.path.join(this_run_dir, "data")) - except common.PathMissing: - pass - - all_data_link = os.path.join(this_run_dir, "data_dirs") - if os.path.lexists(all_data_link): - os.remove(all_data_link) - top_data_dir_path = common.dir_option_test(ddir, "", this_directory) - os.symlink(top_data_dir_path, all_data_link) - if args == None: args = "" - exec_path = common.file_option_test(os.path.join(edir, exe), "", this_directory) sh_contents = "set -e\n" if options.terminate_upon_limit: @@ -357,7 +336,7 @@ def write_script(run_dir, path, content): + this_trace_folder + "/kernelslist " ) - write_script(this_run_dir, "run_postprocess.sh", postprocess_contents) + common.write_run_script(this_run_dir, "run_postprocess.sh", postprocess_contents) item = { "name": run_name,