DAOS-15234 test: Move to Avocado 103 LTS

requirements-ftest.txt

@@ -1,9 +1,9 @@
-avocado-framework==82
-avocado-framework-plugin-result-html==82
-avocado-framework-plugin-varianter-yaml-to-mux==82
+avocado-framework==103.0
+avocado-framework-plugin-result-html==103.0
+avocado-framework-plugin-varianter-yaml-to-mux==103.0
 clustershell
 distro
 paramiko
-# Newer setuptools does not have pkg_resources, which avocado 82 uses
+# Newer setuptools does not have pkg_resources, which avocado uses
 setuptools<82
 torch

src/tests/ftest/cart/util/cart_utils.py

@@ -17,7 +17,7 @@
 from ClusterShell.NodeSet import NodeSet
 from host_utils import get_local_host
 from job_manager_utils import Orterun
-from run_utils import stop_processes
+from run_utils import run_local, stop_processes
 from write_host_file import write_host_file
 
 
@@ -33,6 +33,7 @@ def __init__(self, *args, **kwargs):
         self.src_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(
             os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))))
         self.attach_dir = None
+        self.cleanup_commands = []
 
     def setUp(self):
         """Set up the test case."""
@@ -147,10 +148,15 @@ def check_files(self, glob_pattern, count=1, retries=10):
 
     def cleanup_processes(self):
         """Clean up cart processes, in case avocado/apricot does not."""
+        self.log.info("Cleaning up cart commands: %s", self.cleanup_commands)
         error_list = []
+        if not self.cleanup_commands:
+            self.log.info("No cart commands to cleanup.")
+            return error_list
+        cleanup_commands = "|".join(self.cleanup_commands)
         localhost = get_local_host()
-        processes = r"'\<(crt_launch|orterun)\>'"
-        negative_filter = r"'\<(grep|defunct)\>'"
+        processes = rf"'\<({cleanup_commands})\>'"
+        negative_filter = r"'\<(grep|defunct|avocado-runner-avocado-instrumented)\>'"
         running = True
         for _ in range(2):
             _, running = stop_processes(self.log, localhost, processes, exclude=negative_filter)
@@ -271,28 +277,10 @@ def build_cmd(self, env, host, **kwargs):
 
         # Write memcheck result file(s) to $HOME or DAOS_TEST_SHARED_DIR.
         daos_test_shared_dir = os.getenv('DAOS_TEST_SHARED_DIR', os.getenv('HOME'))
-        memcheck_xml = r"{}/valgrind.%q\{{PMIX_ID\}}_{}.memcheck".format(
-            daos_test_shared_dir,
-            test_name)
+
         tst_cmd = ""
         tst_cont = None
-
         index = kwargs.get('index', None)
-        daos_test_shared_dir = os.getenv('DAOS_TEST_SHARED_DIR', os.getenv('HOME'))
-
-        # Return 0 on memory leaks while suppression file is completed
-        # (CART-975 and CART-977)
-        memcheck_error_code = 0
-
-        tst_vgd = " valgrind --xml=yes " + \
-                  "--xml-file={}".format(memcheck_xml) + " " + \
-                  "--fair-sched=yes --partial-loads-ok=yes " + \
-                  "--leak-check=full --show-leak-kinds=all " + \
-                  " --gen-suppressions=all " + \
-                  "--suppressions=" + self.supp_file + " " + \
-                  "--track-origins=yes " + \
-                  "--error-exitcode=" + str(memcheck_error_code) + " " \
-                  "--show-reachable=yes --trace-children=yes"
 
         _tst_bin = self.params.get("{}_bin".format(host), "/run/tests/*/")
         _tst_arg = self.params.get("{}_arg".format(host), "/run/tests/*/")
@@ -339,10 +327,27 @@ def build_cmd(self, env, host, **kwargs):
         tst_mod = os.getenv("WITH_VALGRIND", "native")
 
         if tst_mod == "memcheck":
-            tst_cmd += tst_vgd
+            # Return 0 on memory leaks while suppression file is completed (CART-975 and CART-977)
+            memcheck_error_code = 0
+            memcheck_xml = fr"{daos_test_shared_dir}/valgrind.%q\{{PMIX_ID\}}_{test_name}.memcheck"
+            tst_cmd += " ".join([
+                " valgrind",
+                "--xml=yes",
+                f"--xml-file={memcheck_xml}",
+                "--fair-sched=yes",
+                "--partial-loads-ok=yes",
+                "--leak-check=full --show-leak-kinds=all",
+                " --gen-suppressions=all",
+                f"--suppressions={self.supp_file}",
+                "--track-origins=yes",
+                f"--error-exitcode={memcheck_error_code}",
+                "--show-reachable=yes",
+                "--trace-children=yes"])
+            self.cleanup_commands.append("valgrind")
 
         if tst_bin is not None:
             tst_cmd += " " + tst_bin
+            self.cleanup_commands.append(tst_bin)
 
         if tst_arg is not None:
             tst_cmd += " " + tst_arg
@@ -353,6 +358,7 @@ def build_cmd(self, env, host, **kwargs):
         job.hostfile.update(hostfile)
         job.pprnode.update(tst_ppn)
         job.processes.update(tst_processes)
+        self.cleanup_commands.append("orterun")
 
         return str(job)
 
@@ -413,19 +419,17 @@ def launch_test(self, cmd, srv1=None, srv2=None):
         self.log.info("CMD : %s", cmd)
         self.log.info("ENV : %s", os.environ)
 
-        cmd = shlex.split(cmd)
-        rtn = subprocess.call(cmd)
-
-        if rtn:
+        result = run_local(self.log, cmd)
+        if not result.passed:
             if srv1 is not None:
                 self.stop_process(srv1)
             if srv2 is not None:
                 self.stop_process(srv2)
-            self.fail(f"Failed, return codes {rtn}")
+            self.fail(f"Failed, return codes {result.return_code}")
 
         self.convert_xml_files()
 
-        return rtn
+        return result.return_code
 
     def launch_cmd_bg(self, cmd):
         """Launch the given cmd in background."""

src/tests/ftest/erasurecode/space_usage.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2023 Intel Corporation.
+  (C) Copyright 2026 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -34,7 +35,7 @@ def test_ec_space_balanced_ec_4p1gx(self):
         :avocado: tags=ec,ior,pool,query_targets
         :avocado: tags=EcodSpaceUsage,test_ec_space_balanced_ec_4p1gx
         """
-        self._run_test(ior_namespace='/run/ior_ec_4p1gx/*')
+        self.__run_test(ior_namespace='/run/ior_ec_4p1gx/*')
 
     def test_ec_space_balanced_ec_4p2gx(self):
         """Jira ID: DAOS-10912.
@@ -46,9 +47,9 @@ def test_ec_space_balanced_ec_4p2gx(self):
         :avocado: tags=ec,ior,pool,query_targets
         :avocado: tags=EcodSpaceUsage,test_ec_space_balanced_ec_4p2gx
         """
-        self._run_test(ior_namespace='/run/ior_ec_4p2gx/*')
+        self.__run_test(ior_namespace='/run/ior_ec_4p2gx/*')
 
-    def _run_test(self, ior_namespace):
+    def __run_test(self, ior_namespace):
         """Run the test.
 
         Args:

src/tests/ftest/harness/basic.py

@@ -160,20 +160,16 @@ def test_sub_process_command(self):
         test_command = ["ls", "-al", os.path.dirname(__file__)]
         command = SubProcessCommand("/run/command/*", " ".join(test_command))
         for sub_process in (False, True):
-            for check in ("both", "combined"):
-                self.log.info("-" * 80)
-                self.log.info(
-                    "Running '%s' with output_check='%s' and run_as_subprocess=%s",
-                    str(command), check, sub_process)
-                command.output_check = check
-                command.run_as_subprocess = sub_process
-                try:
-                    command.run()
-                except CommandFailure:
-                    self.log.error("The '%s' command failed", str(command), exc_info=True)
-                    failed = True
-                finally:
-                    command.stop()
+            self.log.info("-" * 80)
+            self.log.info("Running '%s' with run_as_subprocess=%s", str(command), sub_process)
+            command.run_as_subprocess = sub_process
+            try:
+                command.run()
+            except CommandFailure:
+                self.log.error("The '%s' command failed", str(command), exc_info=True)
+                failed = True
+            finally:
+                command.stop()
         if failed:
             self.fail("The '{}' command failed".format(command))
         self.log.info("Test passed")

src/tests/ftest/util/apricot/apricot/test.py

@@ -17,6 +17,7 @@
 from avocado import Test as avocadoTest
 from avocado import TestFail, fail_on, skip
 from avocado.core import exceptions
+from avocado.core.teststatus import STATUSES_NOT_OK
 from cart_ctl_utils import CartCtl
 from ClusterShell.NodeSet import NodeSet
 from command_utils_base import EnvironmentVariables
@@ -1382,16 +1383,17 @@ def report_timeout(self):
             # dump engines ULT stacks upon test timeout
             self.__dump_engines_stacks("Test has timed-out")
 
-    def fail(self, message=None):
+    def fail(self, msg=None):
+        # pylint: disable=arguments-renamed
         """Dump engines ULT stacks upon test failure."""
         self.__dump_engines_stacks("Test has failed")
-        super().fail(message)
+        super().fail(msg)
 
-    def error(self, message=None):
+    def error(self, msg=None):
         # pylint: disable=arguments-renamed
         """Dump engines ULT stacks upon test error."""
         self.__dump_engines_stacks("Test has errored")
-        super().error(message)
+        super().error(msg)
 
     def tearDown(self):
         """Tear down after each test case."""
@@ -1402,7 +1404,7 @@ def tearDown(self):
         # error() method above, to overload the methods of Avocado base Test
         # class (see DAOS-1452/DAOS-9941 and Avocado issue #5217 with
         # associated PR-5224)
-        if self.status is not None and self.status != 'PASS' and self.status != 'SKIP':
+        if self.status is not None and self.status in STATUSES_NOT_OK:
             self.__dump_engines_stacks(f"Test status is {self.status}")
 
         # Report whether or not the timeout has expired

src/tests/ftest/util/avocado_utils.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2022-2024 Intel Corporation.
+  (C) Copyright 2026 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -81,6 +82,8 @@ def set_config(overwrite=False):
                 "[sysinfo.collectibles]\n",
                 f"files = {sysinfo_files_file}\n",
                 f"commands = {sysinfo_commands_file}\n",
+                "[run]\n",
+                "max_parallel_tasks = 1\n",
             ]
 
             try:

src/tests/ftest/util/collection_utils.py

@@ -1,6 +1,6 @@
 """
   (C) Copyright 2022-2024 Intel Corporation.
-  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
+  (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -622,8 +622,14 @@ def create_steps_log(logger, job_results_dir, test_result):
 
     test_logs_lnk = os.path.join(job_results_dir, "latest")
     test_logs_dir = os.path.realpath(test_logs_lnk)
-    job_log = os.path.join(test_logs_dir, 'job.log')
     step_log = os.path.join(test_logs_dir, 'steps.log')
+    job_log = os.path.join(test_logs_dir, 'full.log')
+    if not os.path.exists(job_log):
+        job_log = os.path.join(test_logs_dir, 'job.log')
+        if not os.path.exists(job_log):
+            message = f"Unable to find a full.log or job.log file for creating {step_log}"
+            test_result.fail_test(logger, "Process", message, sys.exc_info())
+            return 8192
     command = rf"grep -E '(INFO |ERROR)\| (==> Step|START|PASS|FAIL|ERROR)' {job_log}"
     result = run_local(logger, command)
     if not result.passed:

src/tests/ftest/util/command_utils.py

@@ -58,7 +58,6 @@ def __init__(self, namespace, command, path="", subprocess=False, check_results=
         self.run_as_subprocess = subprocess
         self.timeout = None
         self.exit_status_exception = True
-        self.output_check = "both"
         self.verbose = True
         self.env = EnvironmentVariables()
 
@@ -249,8 +248,7 @@ def _run_process(self, raise_exception=None):
             # Block until the command is complete or times out
 
             self.result = run_command(
-                command, self.timeout, self.verbose, raise_exception,
-                self.output_check, env=self.env)
+                command, self.timeout, self.verbose, raise_exception, env=self.env)
 
         except DaosTestError as error:
             # Command failed or possibly timed out
@@ -320,7 +318,6 @@ def _run_subprocess(self):
             kwargs = {
                 "cmd": str(self),
                 "verbose": self.verbose,
-                "allow_output_check": "combined",
                 "shell": False,
                 "env": self.env,
                 "sudo": self.sudo,
@@ -732,8 +729,6 @@ def _get_json_result(self, sub_command_list=None, json_err=False,
                 f"The {self.command} command doesn't have json option defined!")
         prev_json_val = self.json.value
         self.json.update(True)
-        prev_output_check = self.output_check
-        self.output_check = "both"
         if json_err:
             prev_exit_exception = self.exit_status_exception
             self.exit_status_exception = False
@@ -742,7 +737,6 @@ def _get_json_result(self, sub_command_list=None, json_err=False,
             self._get_result(sub_command_list, raise_exception=raise_exception, **kwargs)
         finally:
             self.json.update(prev_json_val)
-            self.output_check = prev_output_check
             if json_err:
                 self.exit_status_exception = prev_exit_exception
         if not self.result.stdout:

src/tests/ftest/util/general_utils.py

@@ -88,8 +88,7 @@ def bytes_to_human(size, digits=2, binary=True):
     return "".join([str(round(value[0], digits)), value[1]])
 
 
-def run_command(command, timeout=60, verbose=True, raise_exception=True,
-                output_check="both", env=None):
+def run_command(command, timeout=60, verbose=True, raise_exception=True, env=None):
     """Run the command on the local host.
 
     This method uses the avocado.utils.process.run() method to run the specified
@@ -106,15 +105,6 @@ def run_command(command, timeout=60, verbose=True, raise_exception=True,
             stdout/stderr. Defaults to True.
         raise_exception (bool, optional): whether to raise an exception if the
             command returns a non-zero exit status. Defaults to True.
-        output_check (str, optional): whether to record the output from the
-            command (from stdout and stderr) in the test output record files.
-            Valid values:
-                "stdout"    - standard output *only*
-                "stderr"    - standard error *only*
-                "both"      - both standard output and error in separate files
-                "combined"  - standard output and error in a single file
-                "none"      - disable all recording
-            Defaults to "both".
         env (dict, optional): dictionary of environment variable names and
             values to set when running the command. Defaults to None.
 
@@ -143,7 +133,6 @@ def run_command(command, timeout=60, verbose=True, raise_exception=True,
         "timeout": timeout,
         "verbose": verbose,
         "ignore_status": not raise_exception,
-        "allow_output_check": output_check,
         "shell": False,
         "env": env,
     }

src/tests/ftest/util/ior_utils.py

@@ -517,7 +517,6 @@ def __init__(self, test, manager, hosts, path=None, slots=None, namespace="/run/
         self.manager.assign_hosts(hosts, path, slots)
         self.manager.job = IorCommand(test.test_env.log_dir, namespace)
         self.manager.job.get_params(test)
-        self.manager.output_check = "both"
         self.timeout = test.params.get("timeout", namespace, None)
         self.label_generator = test.label_generator
         self.test_id = test.test_id

src/tests/ftest/util/mdtest_utils.py

@@ -255,7 +255,6 @@ def __init__(self, test, hosts, manager=None, path=None, slots=None,
         self.manager.assign_hosts(hosts, path, slots)
         self.manager.job = MdtestCommand(test.test_env.log_dir, namespace)
         self.manager.job.get_params(test)
-        self.manager.output_check = "both"
         self.timeout = test.params.get("timeout", namespace, None)
         self.label_generator = test.label_generator
         self.test_id = test.test_id

src/tests/ftest/util/run_utils.py

@@ -321,6 +321,15 @@ def joined_stderr(self):
         all_stderr = self.all_stderr
         return '\n'.join(filter(None, [all_stderr[key] for key in sorted(all_stderr)]))
 
+    @property
+    def return_code(self):
+        """Get the highest return code from the issued command.
+
+        Returns:
+            int: the highest return code from the issued command; -1 if no return codes were found
+        """
+        return max((data.returncode for data in self.output), default=-1)
+
     def log_output(self, log):
         """Log the command result.