diff --git a/requirements-ftest.txt b/requirements-ftest.txt index c4ae044a38d..9f2f86d1cf9 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,9 +1,9 @@ -avocado-framework==82 -avocado-framework-plugin-result-html==82 -avocado-framework-plugin-varianter-yaml-to-mux==82 +avocado-framework==103.0 +avocado-framework-plugin-result-html==103.0 +avocado-framework-plugin-varianter-yaml-to-mux==103.0 clustershell distro paramiko -# Newer setuptools does not have pkg_resources, which avocado 82 uses +# Newer setuptools does not have pkg_resources, which avocado uses setuptools<82 torch diff --git a/src/tests/ftest/cart/util/cart_utils.py b/src/tests/ftest/cart/util/cart_utils.py index de1a35ebf92..65c2bbad602 100644 --- a/src/tests/ftest/cart/util/cart_utils.py +++ b/src/tests/ftest/cart/util/cart_utils.py @@ -17,7 +17,7 @@ from ClusterShell.NodeSet import NodeSet from host_utils import get_local_host from job_manager_utils import Orterun -from run_utils import stop_processes +from run_utils import run_local, stop_processes from write_host_file import write_host_file @@ -33,6 +33,7 @@ def __init__(self, *args, **kwargs): self.src_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname( os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))))) self.attach_dir = None + self.cleanup_commands = [] def setUp(self): """Set up the test case.""" @@ -147,10 +148,15 @@ def check_files(self, glob_pattern, count=1, retries=10): def cleanup_processes(self): """Clean up cart processes, in case avocado/apricot does not.""" + self.log.info("Cleaning up cart commands: %s", self.cleanup_commands) error_list = [] + if not self.cleanup_commands: + self.log.info("No cart commands to cleanup.") + return error_list + cleanup_commands = "|".join(self.cleanup_commands) localhost = get_local_host() - processes = r"'\<(crt_launch|orterun)\>'" - negative_filter = r"'\<(grep|defunct)\>'" + processes = rf"'\<({cleanup_commands})\>'" + negative_filter = r"'\<(grep|defunct|avocado-runner-avocado-instrumented)\>'" running = True for _ in range(2): _, running = stop_processes(self.log, localhost, processes, exclude=negative_filter) @@ -271,28 +277,10 @@ def build_cmd(self, env, host, **kwargs): # Write memcheck result file(s) to $HOME or DAOS_TEST_SHARED_DIR. daos_test_shared_dir = os.getenv('DAOS_TEST_SHARED_DIR', os.getenv('HOME')) - memcheck_xml = r"{}/valgrind.%q\{{PMIX_ID\}}_{}.memcheck".format( - daos_test_shared_dir, - test_name) + tst_cmd = "" tst_cont = None - index = kwargs.get('index', None) - daos_test_shared_dir = os.getenv('DAOS_TEST_SHARED_DIR', os.getenv('HOME')) - - # Return 0 on memory leaks while suppression file is completed - # (CART-975 and CART-977) - memcheck_error_code = 0 - - tst_vgd = " valgrind --xml=yes " + \ - "--xml-file={}".format(memcheck_xml) + " " + \ - "--fair-sched=yes --partial-loads-ok=yes " + \ - "--leak-check=full --show-leak-kinds=all " + \ - " --gen-suppressions=all " + \ - "--suppressions=" + self.supp_file + " " + \ - "--track-origins=yes " + \ - "--error-exitcode=" + str(memcheck_error_code) + " " \ - "--show-reachable=yes --trace-children=yes" _tst_bin = self.params.get("{}_bin".format(host), "/run/tests/*/") _tst_arg = self.params.get("{}_arg".format(host), "/run/tests/*/") @@ -339,10 +327,27 @@ def build_cmd(self, env, host, **kwargs): tst_mod = os.getenv("WITH_VALGRIND", "native") if tst_mod == "memcheck": - tst_cmd += tst_vgd + # Return 0 on memory leaks while suppression file is completed (CART-975 and CART-977) + memcheck_error_code = 0 + memcheck_xml = fr"{daos_test_shared_dir}/valgrind.%q\{{PMIX_ID\}}_{test_name}.memcheck" + tst_cmd += " ".join([ + " valgrind", + "--xml=yes", + f"--xml-file={memcheck_xml}", + "--fair-sched=yes", + "--partial-loads-ok=yes", + "--leak-check=full --show-leak-kinds=all", + " --gen-suppressions=all", + f"--suppressions={self.supp_file}", + "--track-origins=yes", + f"--error-exitcode={memcheck_error_code}", + "--show-reachable=yes", + "--trace-children=yes"]) + self.cleanup_commands.append("valgrind") if tst_bin is not None: tst_cmd += " " + tst_bin + self.cleanup_commands.append(tst_bin) if tst_arg is not None: tst_cmd += " " + tst_arg @@ -353,6 +358,7 @@ def build_cmd(self, env, host, **kwargs): job.hostfile.update(hostfile) job.pprnode.update(tst_ppn) job.processes.update(tst_processes) + self.cleanup_commands.append("orterun") return str(job) @@ -413,19 +419,17 @@ def launch_test(self, cmd, srv1=None, srv2=None): self.log.info("CMD : %s", cmd) self.log.info("ENV : %s", os.environ) - cmd = shlex.split(cmd) - rtn = subprocess.call(cmd) - - if rtn: + result = run_local(self.log, cmd) + if not result.passed: if srv1 is not None: self.stop_process(srv1) if srv2 is not None: self.stop_process(srv2) - self.fail(f"Failed, return codes {rtn}") + self.fail(f"Failed, return codes {result.return_code}") self.convert_xml_files() - return rtn + return result.return_code def launch_cmd_bg(self, cmd): """Launch the given cmd in background.""" diff --git a/src/tests/ftest/erasurecode/space_usage.py b/src/tests/ftest/erasurecode/space_usage.py index 4998c9e4f92..81cda2d158a 100644 --- a/src/tests/ftest/erasurecode/space_usage.py +++ b/src/tests/ftest/erasurecode/space_usage.py @@ -1,5 +1,6 @@ """ (C) Copyright 2023 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -34,7 +35,7 @@ def test_ec_space_balanced_ec_4p1gx(self): :avocado: tags=ec,ior,pool,query_targets :avocado: tags=EcodSpaceUsage,test_ec_space_balanced_ec_4p1gx """ - self._run_test(ior_namespace='/run/ior_ec_4p1gx/*') + self.__run_test(ior_namespace='/run/ior_ec_4p1gx/*') def test_ec_space_balanced_ec_4p2gx(self): """Jira ID: DAOS-10912. @@ -46,9 +47,9 @@ def test_ec_space_balanced_ec_4p2gx(self): :avocado: tags=ec,ior,pool,query_targets :avocado: tags=EcodSpaceUsage,test_ec_space_balanced_ec_4p2gx """ - self._run_test(ior_namespace='/run/ior_ec_4p2gx/*') + self.__run_test(ior_namespace='/run/ior_ec_4p2gx/*') - def _run_test(self, ior_namespace): + def __run_test(self, ior_namespace): """Run the test. Args: diff --git a/src/tests/ftest/harness/basic.py b/src/tests/ftest/harness/basic.py index 1db15555921..f940e79c703 100644 --- a/src/tests/ftest/harness/basic.py +++ b/src/tests/ftest/harness/basic.py @@ -160,20 +160,16 @@ def test_sub_process_command(self): test_command = ["ls", "-al", os.path.dirname(__file__)] command = SubProcessCommand("/run/command/*", " ".join(test_command)) for sub_process in (False, True): - for check in ("both", "combined"): - self.log.info("-" * 80) - self.log.info( - "Running '%s' with output_check='%s' and run_as_subprocess=%s", - str(command), check, sub_process) - command.output_check = check - command.run_as_subprocess = sub_process - try: - command.run() - except CommandFailure: - self.log.error("The '%s' command failed", str(command), exc_info=True) - failed = True - finally: - command.stop() + self.log.info("-" * 80) + self.log.info("Running '%s' with run_as_subprocess=%s", str(command), sub_process) + command.run_as_subprocess = sub_process + try: + command.run() + except CommandFailure: + self.log.error("The '%s' command failed", str(command), exc_info=True) + failed = True + finally: + command.stop() if failed: self.fail("The '{}' command failed".format(command)) self.log.info("Test passed") diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 9dc9c221b5b..70bb326ca2f 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -17,6 +17,7 @@ from avocado import Test as avocadoTest from avocado import TestFail, fail_on, skip from avocado.core import exceptions +from avocado.core.teststatus import STATUSES_NOT_OK from cart_ctl_utils import CartCtl from ClusterShell.NodeSet import NodeSet from command_utils_base import EnvironmentVariables @@ -1382,16 +1383,17 @@ def report_timeout(self): # dump engines ULT stacks upon test timeout self.__dump_engines_stacks("Test has timed-out") - def fail(self, message=None): + def fail(self, msg=None): + # pylint: disable=arguments-renamed """Dump engines ULT stacks upon test failure.""" self.__dump_engines_stacks("Test has failed") - super().fail(message) + super().fail(msg) - def error(self, message=None): + def error(self, msg=None): # pylint: disable=arguments-renamed """Dump engines ULT stacks upon test error.""" self.__dump_engines_stacks("Test has errored") - super().error(message) + super().error(msg) def tearDown(self): """Tear down after each test case.""" @@ -1402,7 +1404,7 @@ def tearDown(self): # error() method above, to overload the methods of Avocado base Test # class (see DAOS-1452/DAOS-9941 and Avocado issue #5217 with # associated PR-5224) - if self.status is not None and self.status != 'PASS' and self.status != 'SKIP': + if self.status is not None and self.status in STATUSES_NOT_OK: self.__dump_engines_stacks(f"Test status is {self.status}") # Report whether or not the timeout has expired diff --git a/src/tests/ftest/util/avocado_utils.py b/src/tests/ftest/util/avocado_utils.py index 4038fc2c168..b24f561a398 100644 --- a/src/tests/ftest/util/avocado_utils.py +++ b/src/tests/ftest/util/avocado_utils.py @@ -1,5 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. + (C) Copyright 2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -81,6 +82,8 @@ def set_config(overwrite=False): "[sysinfo.collectibles]\n", f"files = {sysinfo_files_file}\n", f"commands = {sysinfo_commands_file}\n", + "[run]\n", + "max_parallel_tasks = 1\n", ] try: diff --git a/src/tests/ftest/util/collection_utils.py b/src/tests/ftest/util/collection_utils.py index 7c0d3ccf08d..eb18d31ae57 100644 --- a/src/tests/ftest/util/collection_utils.py +++ b/src/tests/ftest/util/collection_utils.py @@ -1,6 +1,6 @@ """ (C) Copyright 2022-2024 Intel Corporation. - (C) Copyright 2025 Hewlett Packard Enterprise Development LP + (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -622,8 +622,14 @@ def create_steps_log(logger, job_results_dir, test_result): test_logs_lnk = os.path.join(job_results_dir, "latest") test_logs_dir = os.path.realpath(test_logs_lnk) - job_log = os.path.join(test_logs_dir, 'job.log') step_log = os.path.join(test_logs_dir, 'steps.log') + job_log = os.path.join(test_logs_dir, 'full.log') + if not os.path.exists(job_log): + job_log = os.path.join(test_logs_dir, 'job.log') + if not os.path.exists(job_log): + message = f"Unable to find a full.log or job.log file for creating {step_log}" + test_result.fail_test(logger, "Process", message, sys.exc_info()) + return 8192 command = rf"grep -E '(INFO |ERROR)\| (==> Step|START|PASS|FAIL|ERROR)' {job_log}" result = run_local(logger, command) if not result.passed: diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 483626e7a3d..8856c02d66a 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -58,7 +58,6 @@ def __init__(self, namespace, command, path="", subprocess=False, check_results= self.run_as_subprocess = subprocess self.timeout = None self.exit_status_exception = True - self.output_check = "both" self.verbose = True self.env = EnvironmentVariables() @@ -249,8 +248,7 @@ def _run_process(self, raise_exception=None): # Block until the command is complete or times out self.result = run_command( - command, self.timeout, self.verbose, raise_exception, - self.output_check, env=self.env) + command, self.timeout, self.verbose, raise_exception, env=self.env) except DaosTestError as error: # Command failed or possibly timed out @@ -320,7 +318,6 @@ def _run_subprocess(self): kwargs = { "cmd": str(self), "verbose": self.verbose, - "allow_output_check": "combined", "shell": False, "env": self.env, "sudo": self.sudo, @@ -732,8 +729,6 @@ def _get_json_result(self, sub_command_list=None, json_err=False, f"The {self.command} command doesn't have json option defined!") prev_json_val = self.json.value self.json.update(True) - prev_output_check = self.output_check - self.output_check = "both" if json_err: prev_exit_exception = self.exit_status_exception self.exit_status_exception = False @@ -742,7 +737,6 @@ def _get_json_result(self, sub_command_list=None, json_err=False, self._get_result(sub_command_list, raise_exception=raise_exception, **kwargs) finally: self.json.update(prev_json_val) - self.output_check = prev_output_check if json_err: self.exit_status_exception = prev_exit_exception if not self.result.stdout: diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 02261e82d95..de6d5b76744 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -88,8 +88,7 @@ def bytes_to_human(size, digits=2, binary=True): return "".join([str(round(value[0], digits)), value[1]]) -def run_command(command, timeout=60, verbose=True, raise_exception=True, - output_check="both", env=None): +def run_command(command, timeout=60, verbose=True, raise_exception=True, env=None): """Run the command on the local host. This method uses the avocado.utils.process.run() method to run the specified @@ -106,15 +105,6 @@ def run_command(command, timeout=60, verbose=True, raise_exception=True, stdout/stderr. Defaults to True. raise_exception (bool, optional): whether to raise an exception if the command returns a non-zero exit status. Defaults to True. - output_check (str, optional): whether to record the output from the - command (from stdout and stderr) in the test output record files. - Valid values: - "stdout" - standard output *only* - "stderr" - standard error *only* - "both" - both standard output and error in separate files - "combined" - standard output and error in a single file - "none" - disable all recording - Defaults to "both". env (dict, optional): dictionary of environment variable names and values to set when running the command. Defaults to None. @@ -143,7 +133,6 @@ def run_command(command, timeout=60, verbose=True, raise_exception=True, "timeout": timeout, "verbose": verbose, "ignore_status": not raise_exception, - "allow_output_check": output_check, "shell": False, "env": env, } diff --git a/src/tests/ftest/util/ior_utils.py b/src/tests/ftest/util/ior_utils.py index ea0d1a02516..2da092f504e 100644 --- a/src/tests/ftest/util/ior_utils.py +++ b/src/tests/ftest/util/ior_utils.py @@ -517,7 +517,6 @@ def __init__(self, test, manager, hosts, path=None, slots=None, namespace="/run/ self.manager.assign_hosts(hosts, path, slots) self.manager.job = IorCommand(test.test_env.log_dir, namespace) self.manager.job.get_params(test) - self.manager.output_check = "both" self.timeout = test.params.get("timeout", namespace, None) self.label_generator = test.label_generator self.test_id = test.test_id diff --git a/src/tests/ftest/util/mdtest_utils.py b/src/tests/ftest/util/mdtest_utils.py index 9aca97feae1..ccc51b67baa 100644 --- a/src/tests/ftest/util/mdtest_utils.py +++ b/src/tests/ftest/util/mdtest_utils.py @@ -255,7 +255,6 @@ def __init__(self, test, hosts, manager=None, path=None, slots=None, self.manager.assign_hosts(hosts, path, slots) self.manager.job = MdtestCommand(test.test_env.log_dir, namespace) self.manager.job.get_params(test) - self.manager.output_check = "both" self.timeout = test.params.get("timeout", namespace, None) self.label_generator = test.label_generator self.test_id = test.test_id diff --git a/src/tests/ftest/util/run_utils.py b/src/tests/ftest/util/run_utils.py index e4785743912..5de1e5c4b47 100644 --- a/src/tests/ftest/util/run_utils.py +++ b/src/tests/ftest/util/run_utils.py @@ -321,6 +321,15 @@ def joined_stderr(self): all_stderr = self.all_stderr return '\n'.join(filter(None, [all_stderr[key] for key in sorted(all_stderr)])) + @property + def return_code(self): + """Get the highest return code from the issued command. + + Returns: + int: the highest return code from the issued command; -1 if no return codes were found + """ + return max((data.returncode for data in self.output), default=-1) + def log_output(self, log): """Log the command result.