diff --git a/src/sst/elements/balar/.gitignore b/src/sst/elements/balar/.gitignore index 8ff1532cfc..fad02978d1 100644 --- a/src/sst/elements/balar/.gitignore +++ b/src/sst/elements/balar/.gitignore @@ -7,6 +7,7 @@ _cuobjdump_list_ptx_* _app_cuda_version_* .gdbinit *.trace +!tests/traces/*.trace *.data gpgpu_inst_stats.txt *.dot diff --git a/src/sst/elements/balar/Makefile.am b/src/sst/elements/balar/Makefile.am index a7d65c0fc0..0a8bf772dc 100644 --- a/src/sst/elements/balar/Makefile.am +++ b/src/sst/elements/balar/Makefile.am @@ -75,6 +75,13 @@ EXTRA_DIST = \ tests/testsuite_default_balar_wire.py \ tests/testBalar-testcpu.py \ tests/testBalar-vanadis.py \ + tests/testBalar-doorbell.py \ + tests/testBalar-malloc-free.py \ + tests/testBalar-wide-packet.py \ + tests/balar_test_topology.py \ + tests/traces/doorbell_malloc.trace \ + tests/traces/malloc_free.trace \ + tests/traces/wide_memcpy_d2h.trace \ tests/testsuite_default_balar.py \ tests/utils.py \ tests/vanadisBlock.py \ diff --git a/src/sst/elements/balar/testcpu/balarTestCPU.cc b/src/sst/elements/balar/testcpu/balarTestCPU.cc index 479e8536ed..4a7af9a684 100644 --- a/src/sst/elements/balar/testcpu/balarTestCPU.cc +++ b/src/sst/elements/balar/testcpu/balarTestCPU.cc @@ -686,7 +686,16 @@ Interfaces::StandardMem::Request* BalarTestCPU::CudaAPITraceParser::getNextCall( Interfaces::StandardMem::Request* launch_req = cpu->createGPUReqFromPacket(launch_pack); initReqs->push(launch_req); } else if (cudaCallType.find("free") != std::string::npos) { - // Future: Remove device pointer from hashmap as cleanup task? + pack.cuda_call_id = CUDA_FREE; + + std::string dptr_name = trim(params_map.find(std::string("dptr"))->second); + auto pair = dptr_map->find(dptr_name); + if (pair == dptr_map->end()) { + out->fatal(CALL_INFO, -1,"Error: device pointer: '%s' not exist in hashmap\n", dptr_name.c_str()); + } + pack.cuda_free.devPtr = (void*) *(pair->second); + out->verbose(CALL_INFO, 2, 0, "Free Device pointer (%s) val: 0x%llx\n", dptr_name.c_str(), (unsigned long long)*(pair->second)); + req = cpu->createGPUReqFromPacket(pack); } } return req; diff --git a/src/sst/elements/balar/tests/balar_test_topology.py b/src/sst/elements/balar/tests/balar_test_topology.py new file mode 100644 index 0000000000..c7a3b38ada --- /dev/null +++ b/src/sst/elements/balar/tests/balar_test_topology.py @@ -0,0 +1,102 @@ +""" +Shared SST topology builder for Balar contract tests. +""" + +import sst + +DEBUG_PARAMS = {"debug": 0, "debug_level": 10} +NETWORK_BW = "25GB/s" +CLOCK = "2GHz" + +CORE_GROUP = 0 +MMIO_GROUP = 1 +MEMORY_GROUP = 2 + +CORE_DST = [MEMORY_GROUP, MMIO_GROUP] +MMIO_SRC = [CORE_GROUP] +MMIO_DST = [MEMORY_GROUP] +MEMORY_SRC = [CORE_GROUP, MMIO_GROUP] + + +def build_testcpu_router(balar_builder, cfg_file, balar_verbosity=0, dma_verbosity=0): + """Return (cpu, balar_mmio_addr) for merlin router + BalarTestCPU.""" + balar_mmio_iface, balar_mmio_addr, dma_mem_if, dma_mmio_if = balar_builder.buildTestCPU( + cfg_file, balar_verbosity, dma_verbosity + ) + + mmio_nic = balar_mmio_iface.setSubComponent("lowlink", "memHierarchy.MemNIC") + mmio_nic.addParams({ + "group": MMIO_GROUP, + "sources": MMIO_SRC, + "destinations": MMIO_DST, + "network_bw": NETWORK_BW, + }) + + dma_mem_nic = dma_mem_if.setSubComponent("lowlink", "memHierarchy.MemNIC") + dma_mem_nic.addParams({ + "group": CORE_GROUP, + "destinations": CORE_DST, + "network_bw": NETWORK_BW, + }) + + dma_mmio_nic = dma_mmio_if.setSubComponent("lowlink", "memHierarchy.MemNIC") + dma_mmio_nic.addParams({ + "group": MEMORY_GROUP, + "sources": MEMORY_SRC, + "network_bw": NETWORK_BW, + }) + + cpu = sst.Component("cpu", "balar.BalarTestCPU") + cpu.addParams({ + "clock": CLOCK, + "verbose": balar_verbosity, + "scratch_mem_addr": 0, + "gpu_addr": balar_mmio_addr, + "enable_memcpy_dump": False, + }) + iface = cpu.setSubComponent("memory", "memHierarchy.standardInterface") + iface.addParams(DEBUG_PARAMS) + cpu_nic = iface.setSubComponent("lowlink", "memHierarchy.MemNIC") + cpu_nic.addParams({ + "group": CORE_GROUP, + "destinations": CORE_DST, + "network_bw": NETWORK_BW, + }) + + chiprtr = sst.Component("chiprtr", "merlin.hr_router") + chiprtr.addParams({ + "xbar_bw": "1GB/s", + "id": "0", + "input_buf_size": "1KB", + "num_ports": "5", + "flit_size": "72B", + "output_buf_size": "1KB", + "link_bw": "1GB/s", + "topology": "merlin.singlerouter", + }) + chiprtr.setSubComponent("topology", "merlin.singlerouter") + + memctrl = sst.Component("memory", "memHierarchy.MemController") + memctrl.addParams({ + "clock": "1GHz", + "addr_range_end": balar_mmio_addr - 1, + }) + mem_nic = memctrl.setSubComponent("highlink", "memHierarchy.MemNIC") + mem_nic.addParams({ + "group": MEMORY_GROUP, + "sources": MEMORY_SRC, + "network_bw": NETWORK_BW, + }) + memory = memctrl.setSubComponent("backend", "memHierarchy.simpleMem") + memory.addParams({ + "access_time": "100 ns", + "mem_size": "512MiB", + }) + + sst.Link("link_cpu").connect((cpu_nic, "port", "1000ps"), (chiprtr, "port0", "1000ps")) + sst.Link("link_mmio").connect((mmio_nic, "port", "500ps"), (chiprtr, "port1", "500ps")) + sst.Link("link_dma_mem").connect((dma_mem_nic, "port", "500ps"), (chiprtr, "port2", "500ps")) + sst.Link("link_dma_mmio").connect((dma_mmio_nic, "port", "500ps"), (chiprtr, "port3", "500ps")) + sst.Link("link_mem").connect((mem_nic, "port", "1000ps"), (chiprtr, "port4", "1000ps")) + + return cpu, balar_mmio_addr diff --git a/src/sst/elements/balar/tests/testBalar-doorbell.py b/src/sst/elements/balar/tests/testBalar-doorbell.py new file mode 100644 index 0000000000..4a14701d8f --- /dev/null +++ b/src/sst/elements/balar/tests/testBalar-doorbell.py @@ -0,0 +1,22 @@ +# Contract test: minimal CUDA API (malloc) exercises scratch write + MMIO doorbell + DMA read. +# Run: sst testBalar-doorbell.py --model-options='-c gpu-v100-mem.cfg -s stats.out -x ./balar_trace/vectorAdd -t traces/doorbell_malloc.trace' + +import sst +import argparse +from utils import * +import balarBlock +import balar_test_topology + +args = vars(balarTestParser.parse_args()) +balarBuilder = balarBlock.Builder(args) +cpu, _ = balar_test_topology.build_testcpu_router( + balarBuilder, args["config"], args["balar_verbosity"], args["dma_verbosity"] +) +cpu.addParams({ + "trace_file": args["trace"], + "cuda_executable": args["cuda_binary"], +}) + +sst.setStatisticLoadLevel(args["statlevel"]) +sst.enableAllStatisticsForAllComponents({"type": "sst.AccumulatorStatistic"}) +sst.setStatisticOutput("sst.statOutputTXT", {"filepath": args["statfile"]}) diff --git a/src/sst/elements/balar/tests/testBalar-malloc-free.py b/src/sst/elements/balar/tests/testBalar-malloc-free.py new file mode 100644 index 0000000000..376efaa315 --- /dev/null +++ b/src/sst/elements/balar/tests/testBalar-malloc-free.py @@ -0,0 +1,21 @@ +# Contract test: cudaMalloc + cudaFree round-trip via scratch + doorbell + DMA. +# Run: sst testBalar-malloc-free.py --model-options='-c gpu-v100-mem.cfg -s stats.out -x ./balar_trace/vectorAdd -t traces/malloc_free.trace' + +import sst +from utils import * +import balarBlock +import balar_test_topology + +args = vars(balarTestParser.parse_args()) +balarBuilder = balarBlock.Builder(args) +cpu, _ = balar_test_topology.build_testcpu_router( + balarBuilder, args["config"], args["balar_verbosity"], args["dma_verbosity"] +) +cpu.addParams({ + "trace_file": args["trace"], + "cuda_executable": args["cuda_binary"], +}) + +sst.setStatisticLoadLevel(args["statlevel"]) +sst.enableAllStatisticsForAllComponents({"type": "sst.AccumulatorStatistic"}) +sst.setStatisticOutput("sst.statOutputTXT", {"filepath": args["statfile"]}) diff --git a/src/sst/elements/balar/tests/testBalar-wide-packet.py b/src/sst/elements/balar/tests/testBalar-wide-packet.py new file mode 100644 index 0000000000..56f8da56ac --- /dev/null +++ b/src/sst/elements/balar/tests/testBalar-wide-packet.py @@ -0,0 +1,21 @@ +# Contract test: 4 KiB cudaMemcpy H2D+D2H validates multi-cache-line scratch packet path. +# Run: sst testBalar-wide-packet.py --model-options='-c gpu-v100-mem.cfg -s stats.out -x ./balar_trace/vectorAdd -t traces/wide_memcpy_d2h.trace' + +import sst +from utils import * +import balarBlock +import balar_test_topology + +args = vars(balarTestParser.parse_args()) +balarBuilder = balarBlock.Builder(args) +cpu, _ = balar_test_topology.build_testcpu_router( + balarBuilder, args["config"], args["balar_verbosity"], args["dma_verbosity"] +) +cpu.addParams({ + "trace_file": args["trace"], + "cuda_executable": args["cuda_binary"], +}) + +sst.setStatisticLoadLevel(args["statlevel"]) +sst.enableAllStatisticsForAllComponents({"type": "sst.AccumulatorStatistic"}) +sst.setStatisticOutput("sst.statOutputTXT", {"filepath": args["statfile"]}) diff --git a/src/sst/elements/balar/tests/testBalar_testsuite_util.py b/src/sst/elements/balar/tests/testBalar_testsuite_util.py index f94e629437..f448016a1d 100644 --- a/src/sst/elements/balar/tests/testBalar_testsuite_util.py +++ b/src/sst/elements/balar/tests/testBalar_testsuite_util.py @@ -58,25 +58,45 @@ def composed(f): return f return composed - def balar_basic_unittest(test): - def wrapper(*args): - self = args[0] - return self.compose_decorators( - unittest.skipIf(self.missing_cuda_install_path, "test_balar_runvecadd test: Requires missing environment variable CUDA_INSTALL_PATH."), - unittest.skipIf(self.missing_gpgpusim_root, "test_balar_runvecadd test: Requires missing environment variable GPGPUSIM_ROOT."), - test(*args) + @staticmethod + def _sst_element_registered(name): + """Return True iff sst-info reports `name` as a known element library.""" + try: + import subprocess + out = subprocess.run( + ["sst-info", name], + capture_output=True, text=True, timeout=15).stdout + upper = out.upper() + return ( + "UNABLE TO PROCESS LIBRARY" not in upper + and "ELEMENT LIBRARY" in upper ) - return wrapper + except Exception: + return False + @staticmethod + def balar_basic_unittest(test): + """Skip when CUDA or GPGPU-Sim env vars are missing (must return a test method, not call it).""" + test = unittest.skipIf( + os.getenv("CUDA_INSTALL_PATH") is None, + "Requires CUDA_INSTALL_PATH")(test) + test = unittest.skipIf( + os.getenv("GPGPUSIM_ROOT") is None, + "Requires GPGPUSIM_ROOT")(test) + # The balar GPU memory hierarchy uses shogun (XBar/NIC). Skip when + # shogun isn't built into the local SST install — these tests need it. + test = unittest.skipUnless( + BalarTestCase._sst_element_registered("shogun"), + "Requires shogun element (full balar GPU mem hierarchy)")(test) + return test + + @staticmethod def balar_gpuapp_unittest(test): - def wrapper(*args): - self = args[0] - return self.compose_decorators( - self.balar_basic_unittest, - unittest.skipIf(self.missing_gpu_app_collection_root, "test_balar_clang test: Requires missing environment variable GPUAPPS_ROOT."), - test(*args) - ) - return wrapper + test = BalarTestCase.balar_basic_unittest(test) + test = unittest.skipIf( + os.getenv("GPUAPPS_ROOT") is None, + "Requires GPUAPPS_ROOT")(test) + return test #### @@ -98,9 +118,11 @@ def balar_testcpu_template(self, testcase, testtimeout=400): tmpdir = self.get_test_output_tmp_dir() self.balarElementDir = os.path.abspath("{0}/../".format(test_path)) - self.balarElementVectorAddTestDir = os.path.abspath("{0}/vectorAdd".format(test_path)) + self.balarElementVectorAddTestDir = os.path.abspath("{0}/balar_trace".format(test_path)) + self.balarElementTracesTestDir = os.path.abspath("{0}/traces".format(test_path)) self.testbalarDir = "{0}/test_balar".format(tmpdir) - self.testbalarVectorAddDir = "{0}/vectorAdd".format(self.testbalarDir) + self.testbalarVectorAddDir = "{0}/balar_trace".format(self.testbalarDir) + self.testbalarTracesDir = "{0}/traces".format(self.testbalarDir) # Set the various file paths testDataFileName="test_gpgpu_{0}".format(testcase) @@ -122,7 +144,7 @@ def balar_testcpu_template(self, testcase, testtimeout=400): log_debug("testbalarDir = {0}".format(self.testbalarDir)) gpuMemCfgfile = "{0}/gpu-v100-mem.cfg".format(self.testbalarDir) - otherargs = '--model-options=\"-c {0} -s {1} -x {2} -t {3}"'.format(gpuMemCfgfile, statsfile, vecAddBinary, vecAddTrace) + otherargs = '--model-options=\"-c {0} -s {1} -x {2} -t {3} -v 1"'.format(gpuMemCfgfile, statsfile, vecAddBinary, vecAddTrace) # Run SST # Unset BALAR_CUDA_EXE_PATH so that we will use the path @@ -151,6 +173,108 @@ def balar_testcpu_template(self, testcase, testtimeout=400): if line_count_diff > 15: self.assertFalse(line_count_diff > 15, "Line count between Stats file {0} does not match Reference File {1}; They contain {2} different lines".format(statsfile, reffile, line_count_diff)) + def balar_contract_testcpu_template( + self, testcase, sdl_name, trace_name, testtimeout=600, min_d2h_ratio=0.0): + """Contract test: trace-driven testcpu with completion + optional gold line count.""" + missing_nvcc_path = os.getenv("NVCC_PATH") == None + self.assertFalse(missing_nvcc_path, "balar contract test: Requires NVCC_PATH.") + + test_path = self.get_testsuite_dir() + outdir = self.get_test_output_run_dir() + testDataFileName = "test_gpgpu_{0}".format(testcase) + reffile = "{0}/refFiles/{1}.out".format(test_path, testDataFileName) + outfile = "{0}/{1}.out".format(outdir, testDataFileName) + errfile = "{0}/{1}.err".format(outdir, testDataFileName) + statsfile = "{0}/{1}.stats_out".format(outdir, testDataFileName) + sdlfile = "{0}/{1}".format(test_path, sdl_name) + vecAddBinary = "{0}/balar_trace/vectorAdd".format(self.testbalarDir) + trace_path = "{0}/traces/{1}".format(self.testbalarDir, trace_name) + gpuMemCfgfile = "{0}/gpu-v100-mem.cfg".format(self.testbalarDir) + otherargs = '--model-options=\"-c {0} -s {1} -x {2} -t {3} -v 1"'.format( + gpuMemCfgfile, statsfile, vecAddBinary, trace_path) + + cmd = self.run_sst(sdlfile, outfile, errfile, set_cwd=self.testbalarDir, + other_args=otherargs, timeout_sec=testtimeout) + log_debug("cmd = {0}".format(cmd)) + + self.assertTrue(os_test_file(statsfile, "-e"), + "balar contract {0}: missing stats file {1}".format(testcase, statsfile)) + with open(outfile, "r", errors="replace") as fh: + out_text = fh.read() + self.assertIn("Test Completed Successfuly", out_text, + "balar contract {0}: BalarTestCPU did not report success".format(testcase)) + with open(errfile, "r", errors="replace") as fh: + err_text = fh.read() + self.assertNotIn("FATAL", err_text, + "balar contract {0}: simulation fatal in stderr".format(testcase)) + if min_d2h_ratio > 0.0: + self._assert_min_d2h_ratio(statsfile, testcase, "balar contract", min_d2h_ratio) + + if os.path.isfile(reffile): + num_stat_lines = int(os_wc(statsfile, [0])) + num_ref_lines = int(os_wc(reffile, [0])) + line_count_diff = abs(num_ref_lines - num_stat_lines) + if line_count_diff > 15: + self.assertFalse( + line_count_diff > 15, + "Line count between Stats file {0} and Reference {1} differ by {2}".format( + statsfile, reffile, line_count_diff)) + + def _stat_values(self, statsfile, stat_name, field_name): + values = [] + with open(statsfile, "r", errors="replace") as fh: + for line in fh: + if ".{0}".format(stat_name) not in line: + continue + parts = line.replace("=", " ").replace(";", " ").split() + for i, p in enumerate(parts): + if p == field_name and i + 1 < len(parts): + values.append(parts[i + 1]) + break + return values + + def _stat_sum_u64(self, statsfile, stat_name): + total = 0 + for value in self._stat_values(statsfile, stat_name, "Sum.u64"): + try: + total += int(value) + except ValueError: + pass + return total + + def _stat_min_f64(self, statsfile, stat_name): + values = [] + for value in self._stat_values(statsfile, stat_name, "Min.f64"): + try: + values.append(float(value)) + except ValueError: + pass + return min(values) if values else None + + def _stat_count_u64(self, statsfile, stat_name): + total = 0 + for value in self._stat_values(statsfile, stat_name, "Count.u64"): + try: + total += int(value) + except ValueError: + pass + return total + + def _assert_min_d2h_ratio(self, statsfile, testcase, test_kind, min_d2h_ratio): + stat_name = "correct_memD2H_ratio" + ratio_count = self._stat_count_u64(statsfile, stat_name) + self.assertGreater( + ratio_count, 0, + "{0} {1}: {2} was not recorded".format(test_kind, testcase, stat_name)) + min_ratio = self._stat_min_f64(statsfile, stat_name) + self.assertIsNotNone( + min_ratio, + "{0} {1}: missing Min.f64 for {2}".format(test_kind, testcase, stat_name)) + self.assertGreaterEqual( + min_ratio + 1e-9, min_d2h_ratio, + "{0} {1}: {2} Min.f64 {3} < {4}".format( + test_kind, testcase, stat_name, min_ratio, min_d2h_ratio)) + def balar_vanadis_template(self, testcase, testtimeout=400): """Balar testcase template with vanadis with explicit CUDA api calls @@ -380,6 +504,7 @@ def _setupbalarTestFiles(self): shutil.rmtree(self.testbalarDir, True) os.makedirs(self.testbalarDir) os.makedirs(self.testbalarVectorAddDir) + os.makedirs(self.testbalarTracesDir) os.makedirs(self.testbalarVanadisHandshakeDir) os.makedirs(self.testbalarLLVMVanadisDir) os.makedirs(self.cudartDir) @@ -392,6 +517,13 @@ def _setupbalarTestFiles(self): os_symlink_file(test_path, self.testbalarDir, "memory.py") os_symlink_file(test_path, self.testbalarDir, "vanadisBlock.py") os_symlink_file(test_path, self.testbalarDir, "vanadisOS.py") + os_symlink_file(test_path, self.testbalarDir, "balar_test_topology.py") + for sdl in ( + "testBalar-doorbell.py", + "testBalar-malloc-free.py", + "testBalar-wide-packet.py", + ): + os_symlink_file(test_path, self.testbalarDir, sdl) # Copy the shared packet definition files from balar src os_symlink_file(self.balarElementDir, tmpdir, "balar_packet.h") os_symlink_file(self.balarElementDir, tmpdir, "balar_consts.h") @@ -405,6 +537,18 @@ def _setupbalarTestFiles(self): for f in os.listdir(self.balarElementVectorAddTestDir): os_symlink_file(self.balarElementVectorAddTestDir, self.testbalarVectorAddDir, f) + for f in os.listdir(self.balarElementTracesTestDir): + os_symlink_file(self.balarElementTracesTestDir, self.testbalarTracesDir, f) + + # 4 KiB memcpy payload for wide-packet contract test + wide_data = bytes([(i % 256) for i in range(4096)]) + wide_payload = os.path.join(self.testbalarTracesDir, "cuMemcpyH2D-0-4096.data") + with open(wide_payload, "wb") as wf: + wf.write(wide_data) + wide_d2h = os.path.join(self.testbalarTracesDir, "cuMemcpyD2H-0-4096.data") + with open(wide_d2h, "wb") as wf: + wf.write(wide_data) + # Create a simlink of each file in the balar/tests/vanadisHandshake directory for f in os.listdir(self.balarElementVanadisHandshakeTestDir): os_symlink_file(self.balarElementVanadisHandshakeTestDir, self.testbalarVanadisHandshakeDir, f) @@ -413,11 +557,14 @@ def _setupbalarTestFiles(self): for f in os.listdir(self.balarElementLLVMVanadisTestDir): os_symlink_file(self.balarElementLLVMVanadisTestDir, self.testbalarLLVMVanadisDir, f) - # Now build libcudart - cmd = "make" - rtn = os_command(cmd, set_cwd=self.cudartDir).run() - log_debug("Balar cudart Make result = {0}; output =\n{1}".format(rtn.result(), rtn.output())) - self.assertTrue(rtn.result() == 0, "libcudart failed to compile") + # Build Vanadis libcudart only when the RISC-V toolchain is available. + if self.missing_riscv_gcc_path: + log_debug("Skipping libcudart build (RISCV_TOOLCHAIN_INSTALL_PATH missing)") + else: + cmd = "make" + rtn = os_command(cmd, set_cwd=self.cudartDir).run() + log_debug("Balar cudart Make result = {0}; output =\n{1}".format(rtn.result(), rtn.output())) + self.assertTrue(rtn.result() == 0, "libcudart failed to compile") # Now build the vectorAdd example cmd = "make" @@ -431,11 +578,14 @@ def _setupbalarTestFiles(self): # log_debug("Balar vanadisHandshake Make result = {0}; output =\n{1}".format(rtn.result(), rtn.output())) # self.assertTrue(rtn.result() == 0, "vanadisHandshake.c failed to compile") - # Build vanadis_llvm_rv64, contains helloworld, vecadd, and the custom CUDA lib - cmd = "make" - rtn = os_command(cmd, set_cwd=self.testbalarLLVMVanadisDir).run() - log_debug("Balar vanadisLLVM Make result = {0}; output =\n{1}".format(rtn.result(), rtn.output())) - self.assertTrue(rtn.result() == 0, "vanadisLLVM executables and lib failed to compile") + # Build vanadis_llvm_rv64 when LLVM/RISC-V toolchain is available (optional in CI) + if self.missing_riscv_cuda_clang_path or self.missing_riscv_gcc_path: + log_debug("Skipping vanadis_llvm_rv64 build (LLVM_INSTALL_PATH or RISCV toolchain missing)") + else: + cmd = "make" + rtn = os_command(cmd, set_cwd=self.testbalarLLVMVanadisDir).run() + log_debug("Balar vanadisLLVM Make result = {0}; output =\n{1}".format(rtn.result(), rtn.output())) + self.assertTrue(rtn.result() == 0, "vanadisLLVM executables and lib failed to compile") # Let GPU app knows we are building for SST integration if not self.missing_gpu_app_collection_root: @@ -469,11 +619,13 @@ def _setupbalarVarEnv(self): tmpdir = self.get_test_output_tmp_dir() self.balarElementDir = os.path.abspath("{0}/../".format(test_path)) - self.balarElementVectorAddTestDir = os.path.abspath("{0}/vectorAdd".format(test_path)) + self.balarElementVectorAddTestDir = os.path.abspath("{0}/balar_trace".format(test_path)) + self.balarElementTracesTestDir = os.path.abspath("{0}/traces".format(test_path)) self.balarElementVanadisHandshakeTestDir = os.path.abspath("{0}/vanadisHandshake".format(test_path)) self.balarElementLLVMVanadisTestDir = os.path.abspath("{0}/vanadis_llvm_rv64".format(test_path)) self.testbalarDir = "{0}/test_balar".format(tmpdir) - self.testbalarVectorAddDir = "{0}/vectorAdd".format(self.testbalarDir) + self.testbalarVectorAddDir = "{0}/balar_trace".format(self.testbalarDir) + self.testbalarTracesDir = "{0}/traces".format(self.testbalarDir) self.testbalarVanadisHandshakeDir = "{0}/vanadisHandshake".format(self.testbalarDir) self.testbalarLLVMVanadisDir = "{0}/vanadis_llvm_rv64".format(self.testbalarDir) self.cudartDir = os.path.abspath("{0}/libcudart".format(self.testbalarDir)) @@ -490,7 +642,11 @@ def _setupbalarVarEnv(self): print(f"GPU_ARCH was not set — assigning GPU_ARCH = {fallback}") if "GPGPUSIM_ROOT" not in os.environ: - self.fail("Environment variable GPGPUSIM_ROOT not set — aborting test.") + gpgpu = os.environ.get("GPGPUSIM_INSTALL_PATH") + if gpgpu: + os.environ["GPGPUSIM_ROOT"] = gpgpu + else: + self.fail("Environment variable GPGPUSIM_ROOT not set — aborting test.") # Get nvcc cmd = "which nvcc" diff --git a/src/sst/elements/balar/tests/testsuite_default_balar.py b/src/sst/elements/balar/tests/testsuite_default_balar.py index 5f3a101fd8..2dd3ada01d 100644 --- a/src/sst/elements/balar/tests/testsuite_default_balar.py +++ b/src/sst/elements/balar/tests/testsuite_default_balar.py @@ -8,33 +8,63 @@ from testBalar_testsuite_util import * -## TODO -## Commenting-out some of these tests temporarily -## Should re-enable once the basic vectorAdd test is integrated -## +## Balar integration tests +## Requires: CUDA_INSTALL_PATH, GPGPUSIM_ROOT, NVCC + + +class testcase_balar_smoke(SSTTestCase): + """Tier-0: no GPGPU-Sim required.""" + + def setUp(self): + super().setUp() + self._setupbalarVarEnv_smoke() + + def _setupbalarVarEnv_smoke(self): + test_path = self.get_testsuite_dir() + self.balarElementDir = os.path.abspath("{0}/../".format(test_path)) + self.testbalarDir = "{0}/test_balar_smoke".format(self.get_test_output_tmp_dir()) + if os.path.isdir(self.testbalarDir): + shutil.rmtree(self.testbalarDir, True) + os.makedirs(self.testbalarDir) + for f in ("balarBlock.py", "utils.py", "gpu-v100-mem.cfg"): + os_symlink_file(test_path, self.testbalarDir, f) + + def _sst_info_text(self): + import subprocess + rtn = subprocess.run(["sst-info", "balar"], capture_output=True, text=True, timeout=30) + return rtn.stdout + rtn.stderr, rtn.returncode + + def test_balar_sst_info_registers_components(self): + """sst-info balar lists balarMMIO when the element is built.""" + text, rc = self._sst_info_text() + if "UNABLE TO PROCESS LIBRARY" in text.upper() or rc != 0: + self.skipTest("balar element not built (configure without --with-gpgpusim)") + self.assertIn("balarMMIO", text, "sst-info balar should list balar.balarMMIO") + class testcase_balar_simple(BalarTestCase): - # @BalarTestCase.balar_basic_unittest - # def test_balar_runvecadd_testcpu(self): - # self.balar_testcpu_template("vectorAdd") - # - # @BalarTestCase.balar_basic_unittest - # def test_balar_runvecadd_vanadis(self): - # self.balar_vanadis_template("vanadisHandshake") - # - # @BalarTestCase.balar_basic_unittest - # def test_balar_vanadis_clang_helloworld(self): - # self.balar_vanadis_clang_template("helloworld") + @BalarTestCase.balar_basic_unittest + def test_balar_runvecadd_testcpu(self): + self.balar_testcpu_template("vectorAdd", testtimeout=60 * 20) @BalarTestCase.balar_basic_unittest - # @categorize("nightly") + def test_balar_contract_doorbell(self): + self.balar_contract_testcpu_template( + "doorbell", "testBalar-doorbell.py", "doorbell_malloc.trace", testtimeout=60 * 10) + + @BalarTestCase.balar_basic_unittest + def test_balar_contract_malloc_free(self): + self.balar_contract_testcpu_template( + "malloc_free", "testBalar-malloc-free.py", "malloc_free.trace", testtimeout=60 * 15) + + @BalarTestCase.balar_basic_unittest + def test_balar_contract_wide_packet(self): + self.balar_contract_testcpu_template( + "wide_packet", "testBalar-wide-packet.py", "wide_memcpy_d2h.trace", + testtimeout=60 * 20, min_d2h_ratio=1.0) + + @unittest.skipIf( + os.getenv("LLVM_INSTALL_PATH") is None or os.getenv("RISCV_TOOLCHAIN_INSTALL_PATH") is None, + "Vanadis clang vecadd requires LLVM_INSTALL_PATH and RISCV_TOOLCHAIN_INSTALL_PATH") + @BalarTestCase.balar_basic_unittest def test_balar_vanadis_clang_vecadd(self): self.balar_vanadis_clang_template("vecadd", 60 * 30) - # - # @BalarTestCase.balar_gpuapp_unittest - # def test_balar_vanadis_clang_rodinia_20_backprop_short(self): - # self.balar_vanadis_clang_template("rodinia-2.0-backprop-short", 60 * 30) - # - # @BalarTestCase.balar_gpuapp_unittest - # def test_balar_vanadis_clang_rodinia_20_bfs_SampleGraph(self): - # self.balar_vanadis_clang_template("rodinia-2.0-bfs-SampleGraph") - diff --git a/src/sst/elements/balar/tests/traces/doorbell_malloc.trace b/src/sst/elements/balar/tests/traces/doorbell_malloc.trace new file mode 100644 index 0000000000..62bb81d5d0 --- /dev/null +++ b/src/sst/elements/balar/tests/traces/doorbell_malloc.trace @@ -0,0 +1 @@ +CUDA memalloc: dptr: dptr-0, size: 64 diff --git a/src/sst/elements/balar/tests/traces/malloc_free.trace b/src/sst/elements/balar/tests/traces/malloc_free.trace new file mode 100644 index 0000000000..223972811f --- /dev/null +++ b/src/sst/elements/balar/tests/traces/malloc_free.trace @@ -0,0 +1,2 @@ +CUDA memalloc: dptr: dptr-0, size: 256 +CUDA free detected: dptr: dptr-0 diff --git a/src/sst/elements/balar/tests/traces/wide_memcpy_d2h.trace b/src/sst/elements/balar/tests/traces/wide_memcpy_d2h.trace new file mode 100644 index 0000000000..0ed8f1001e --- /dev/null +++ b/src/sst/elements/balar/tests/traces/wide_memcpy_d2h.trace @@ -0,0 +1,4 @@ +CUDA memalloc: dptr: dptr-0, size: 4096 +CUDA memcpyH2D detected: device_ptr: dptr-0, host_ptr: 0x1000, size: 4096, data_file: cuMemcpyH2D-0-4096.data, addr: 0x0 +CUDA memcpyD2H detected: host_ptr: 0x2000, device_ptr: dptr-0, size: 4096, data_file: cuMemcpyD2H-0-4096.data +CUDA free detected: dptr: dptr-0