From d16d2b53405d7863720b3e40178a6a4b56837a8f Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 10 Apr 2024 08:57:41 -0400 Subject: [PATCH 01/12] DAOS-15234 test: Moving to Avocado 103 LTS Moving to the latest avocado LTS release. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Doc-only: false Required-githooks: true Signed-off-by: Phil Henderson --- requirements-ftest.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-ftest.txt b/requirements-ftest.txt index c0c0f2b8eb1..5c157b84094 100644 --- a/requirements-ftest.txt +++ b/requirements-ftest.txt @@ -1,6 +1,6 @@ -avocado-framework==82 -avocado-framework-plugin-result-html==82 -avocado-framework-plugin-varianter-yaml-to-mux==82 +avocado-framework>=103.0,<104.0 +avocado-framework-plugin-result-html>=103.0,<104.0 +avocado-framework-plugin-varianter-yaml-to-mux>=103.0,<104.0 clustershell paramiko distro From 0b58ae11e029f18ba7501287040cc421e8472676 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 10 Apr 2024 17:22:35 -0400 Subject: [PATCH 02/12] Avocado 103 LTS needs a minimum of python 3.7. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Doc-only: false Required-githooks: true Signed-off-by: Phil Henderson --- src/tests/ftest/scripts/main.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index e6079637064..2f57131fc39 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,7 +22,10 @@ then rm -rf venv fi -python3 -m venv venv +# Install python 3.12 for running avocado in the virtual environment +dnf -y install python3.12 python3.12-devel + +python3.12 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate From fef1bd3812b4225915391b726364785a28527a38 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 10 Apr 2024 19:17:50 -0400 Subject: [PATCH 03/12] Move python3.12 install command. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-el9: false Skip-func-test-leap15: false Doc-only: false Test-tag: pr,-rebuild Required-githooks: true Signed-off-by: Phil Henderson --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 1 + src/tests/ftest/scripts/main.sh | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 2160df55e34..0d81375dc4c 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -21,6 +21,7 @@ distro_custom() { # pydaos into virtual environments. dnf -y install python39 python39-devel dnf -y install python3.11 python3.11-devel + dnf -y install python3.12 python3.12-devel } install_mofed() { diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 2f57131fc39..271bef18548 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,9 +22,6 @@ then rm -rf venv fi -# Install python 3.12 for running avocado in the virtual environment -dnf -y install python3.12 python3.12-devel - python3.12 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate From 01c5b20a3fda3333463aa732aa277d97bab3e069 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Apr 2024 09:00:40 -0400 Subject: [PATCH 04/12] Run the virtual env with the latest available python version installed. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-el9: false Skip-func-test-leap15: false Doc-only: false Test-tag: pr,-rebuild Required-githooks: true Signed-off-by: Phil Henderson --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 1 - src/tests/ftest/scripts/main.sh | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 0d81375dc4c..2160df55e34 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -21,7 +21,6 @@ distro_custom() { # pydaos into virtual environments. dnf -y install python39 python39-devel dnf -y install python3.11 python3.11-devel - dnf -y install python3.12 python3.12-devel } install_mofed() { diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 271bef18548..b81bbf10a60 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,7 +22,8 @@ then rm -rf venv fi -python3.12 -m venv venv +latest_python = $(find /usr/bin -name 'python*' -type f -executable | grep -E '[0-9]$' | sort -r | head -1) +${latest_python} -m venv venv # shellcheck disable=SC1091 source venv/bin/activate From b75c87dc3dfca166a7abbbde7ef37fa17622a2b4 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Apr 2024 11:31:07 -0400 Subject: [PATCH 05/12] Fix typo. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-el9: false Skip-func-test-leap15: false Doc-only: false Test-tag: pr,-rebuild Required-githooks: true Signed-off-by: Phil Henderson --- src/tests/ftest/scripts/main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index b81bbf10a60..96d58a0b766 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,7 +22,7 @@ then rm -rf venv fi -latest_python = $(find /usr/bin -name 'python*' -type f -executable | grep -E '[0-9]$' | sort -r | head -1) +latest_python=$(find /usr/bin -name 'python*' -type f -executable | grep -E '[0-9]$' | sort -r | head -1) ${latest_python} -m venv venv # shellcheck disable=SC1091 source venv/bin/activate From 8d06a9d210d1b325a2729b439383a946597f4502 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Apr 2024 17:55:50 -0400 Subject: [PATCH 06/12] Code updates for 103 LTS. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-el9: false Skip-func-test-leap15: false Doc-only: false Test-tag: pr,-rebuild Required-githooks: true Signed-off-by: Phil Henderson --- src/tests/ftest/erasurecode/space_usage.py | 6 +++--- src/tests/ftest/util/apricot/apricot/test.py | 4 ++-- src/tests/ftest/util/command_utils.py | 3 +-- src/tests/ftest/util/general_utils.py | 13 +------------ 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/tests/ftest/erasurecode/space_usage.py b/src/tests/ftest/erasurecode/space_usage.py index 7fa40365e76..3182a880a6d 100644 --- a/src/tests/ftest/erasurecode/space_usage.py +++ b/src/tests/ftest/erasurecode/space_usage.py @@ -34,7 +34,7 @@ def test_ec_space_balanced_ec_4p1gx(self): :avocado: tags=ec,ior,pool,query_targets :avocado: tags=EcodSpaceUsage,test_ec_space_balanced_ec_4p1gx """ - self._run_test(ior_namespace='/run/ior_ec_4p1gx/*') + self._execute_test(ior_namespace='/run/ior_ec_4p1gx/*') def test_ec_space_balanced_ec_4p2gx(self): """Jira ID: DAOS-10912. @@ -46,9 +46,9 @@ def test_ec_space_balanced_ec_4p2gx(self): :avocado: tags=ec,ior,pool,query_targets :avocado: tags=EcodSpaceUsage,test_ec_space_balanced_ec_4p2gx """ - self._run_test(ior_namespace='/run/ior_ec_4p2gx/*') + self._execute_test(ior_namespace='/run/ior_ec_4p2gx/*') - def _run_test(self, ior_namespace): + def _execute_test(self, ior_namespace): """Run the test. Args: diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index 641036f3917..dd35fbe093f 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -1385,13 +1385,13 @@ def report_timeout(self): def fail(self, message=None): """Dump engines ULT stacks upon test failure.""" self.dump_engines_stacks("Test has failed") - super().fail(message) + self.fail(message) def error(self, message=None): # pylint: disable=arguments-renamed """Dump engines ULT stacks upon test error.""" self.dump_engines_stacks("Test has errored") - super().error(message) + self.error(message) def tearDown(self): """Tear down after each test case.""" diff --git a/src/tests/ftest/util/command_utils.py b/src/tests/ftest/util/command_utils.py index 9c21c21feeb..77c0767f13d 100644 --- a/src/tests/ftest/util/command_utils.py +++ b/src/tests/ftest/util/command_utils.py @@ -230,7 +230,7 @@ def _run_process(self, raise_exception=None): self.result = run_command( command, self.timeout, self.verbose, raise_exception, - self.output_check, env=self.env) + env=self.env) except DaosTestError as error: # Command failed or possibly timed out @@ -284,7 +284,6 @@ def _run_subprocess(self): kwargs = { "cmd": str(self), "verbose": self.verbose, - "allow_output_check": "combined", "shell": False, "env": self.env, "sudo": self.sudo, diff --git a/src/tests/ftest/util/general_utils.py b/src/tests/ftest/util/general_utils.py index 599854fc788..aaea6d3361b 100644 --- a/src/tests/ftest/util/general_utils.py +++ b/src/tests/ftest/util/general_utils.py @@ -194,8 +194,7 @@ def bytes_to_human(size, digits=2, binary=True): return "".join([str(round(value[0], digits)), value[1]]) -def run_command(command, timeout=60, verbose=True, raise_exception=True, - output_check="both", env=None): +def run_command(command, timeout=60, verbose=True, raise_exception=True, env=None): """Run the command on the local host. This method uses the avocado.utils.process.run() method to run the specified @@ -212,15 +211,6 @@ def run_command(command, timeout=60, verbose=True, raise_exception=True, stdout/stderr. Defaults to True. raise_exception (bool, optional): whether to raise an exception if the command returns a non-zero exit status. Defaults to True. - output_check (str, optional): whether to record the output from the - command (from stdout and stderr) in the test output record files. - Valid values: - "stdout" - standard output *only* - "stderr" - standard error *only* - "both" - both standard output and error in separate files - "combined" - standard output and error in a single file - "none" - disable all recording - Defaults to "both". env (dict, optional): dictionary of environment variable names and values to set when running the command. Defaults to None. @@ -249,7 +239,6 @@ def run_command(command, timeout=60, verbose=True, raise_exception=True, "timeout": timeout, "verbose": verbose, "ignore_status": not raise_exception, - "allow_output_check": output_check, "shell": False, "env": env, } From 117482c1592e14499d84417ca7c7de9d527df900 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Apr 2024 18:32:14 -0400 Subject: [PATCH 07/12] Fix missed pylint issue. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-el9: false Skip-func-test-leap15: false Doc-only: false Test-tag: pr,-rebuild PosixSimul Required-githooks: true Signed-off-by: Phil Henderson --- src/tests/ftest/dfuse/simul.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/ftest/dfuse/simul.py b/src/tests/ftest/dfuse/simul.py index aef80b5d651..0c59d40f912 100644 --- a/src/tests/ftest/dfuse/simul.py +++ b/src/tests/ftest/dfuse/simul.py @@ -119,7 +119,7 @@ def run_simul(self, include=None, exclude=None, raise_exception=True): self.log.info("Running simul on %s", mpi_type) try: - result = run_command(cmd, output_check="combined", raise_exception=raise_exception) + result = run_command(cmd, raise_exception=raise_exception) finally: self.stop_dfuse() From 23f325951bef31793712c443ef2e3472ad9b3500 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 11 Apr 2024 19:04:22 -0400 Subject: [PATCH 08/12] Fix arguments-renamed error. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-el9: false Skip-func-test-leap15: false Doc-only: false Test-tag: pr,-rebuild PosixSimul Required-githooks: true Signed-off-by: Phil Henderson --- src/tests/ftest/util/apricot/apricot/test.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index dd35fbe093f..0fd573e3bf6 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -17,6 +17,7 @@ from avocado import Test as avocadoTest from avocado import TestFail, fail_on, skip from avocado.core import exceptions +from avocado.core.teststatus import STATUSES_NOT_OK from cart_ctl_utils import CartCtl from ClusterShell.NodeSet import NodeSet from command_utils_base import EnvironmentVariables @@ -1382,27 +1383,11 @@ def report_timeout(self): # dump engines ULT stacks upon test timeout self.dump_engines_stacks("Test has timed-out") - def fail(self, message=None): - """Dump engines ULT stacks upon test failure.""" - self.dump_engines_stacks("Test has failed") - self.fail(message) - - def error(self, message=None): - # pylint: disable=arguments-renamed - """Dump engines ULT stacks upon test error.""" - self.dump_engines_stacks("Test has errored") - self.error(message) - def tearDown(self): """Tear down after each test case.""" # dump engines ULT stacks upon test failure - # check of Avocado test status during teardown is presently useless - # and about same behavior has been implemented by adding both fail() - # error() method above, to overload the methods of Avocado base Test - # class (see DAOS-1452/DAOS-9941 and Avocado issue #5217 with - # associated PR-5224) - if self.status is not None and self.status != 'PASS' and self.status != 'SKIP': + if self.status is not None and self.status in STATUSES_NOT_OK: self.dump_engines_stacks("Test status is {}".format(self.status)) # Report whether or not the timeout has expired From f6d4c653ee0b3dcafa4bf3fb08a0e5fd12d77003 Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Wed, 29 May 2024 16:55:35 -0400 Subject: [PATCH 09/12] Use python3.11. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-leap15: false Skip-func-test-el9: false Test-tag: always_passes test_setup_vm test_core_files PosixSimul EcodSpaceUsage Required-githooks: true Signed-off-by: Phil Henderson --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 3 ++- ci/provisioning/post_provision_config_nodes_LEAP_15.sh | 6 ++++++ src/tests/ftest/scripts/main.sh | 3 +-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 2160df55e34..1adb0e400ee 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -19,8 +19,9 @@ distro_custom() { # Use a more recent python version for unit testing, this allows us to also test installing # pydaos into virtual environments. - dnf -y install python39 python39-devel dnf -y install python3.11 python3.11-devel + sudo update-alternatives --set python3 /usr/bin/python3.11 + update-alternatives --list } install_mofed() { diff --git a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh index 2c7c66da133..589f663c7e4 100755 --- a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh @@ -16,4 +16,10 @@ distro_custom() { sed -e '/MODULEPATH=/s/$/:\/usr\/share\/modules/' \ /etc/profile.d/lmod.sh; \ fi + + # Use a more recent python version for unit testing, this allows us to also test installing + # pydaos into virtual environments. + dnf -y install python3.11 python3.11-devel + sudo update-alternatives --set python3 /usr/bin/python3.11 + update-alternatives --list } diff --git a/src/tests/ftest/scripts/main.sh b/src/tests/ftest/scripts/main.sh index 96d58a0b766..e6079637064 100755 --- a/src/tests/ftest/scripts/main.sh +++ b/src/tests/ftest/scripts/main.sh @@ -22,8 +22,7 @@ then rm -rf venv fi -latest_python=$(find /usr/bin -name 'python*' -type f -executable | grep -E '[0-9]$' | sort -r | head -1) -${latest_python} -m venv venv +python3 -m venv venv # shellcheck disable=SC1091 source venv/bin/activate From d43b684af028565282f9d9fa6b0fc0efd5e4bfdc Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Thu, 30 May 2024 13:26:48 -0400 Subject: [PATCH 10/12] More testing. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-leap15: false Skip-func-test-el9: false Test-tag: pr,-cart Required-githooks: true Signed-off-by: Phil Henderson --- ci/provisioning/post_provision_config_nodes_EL_8.sh | 6 ++++-- ci/provisioning/post_provision_config_nodes_LEAP_15.sh | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/provisioning/post_provision_config_nodes_EL_8.sh b/ci/provisioning/post_provision_config_nodes_EL_8.sh index 1adb0e400ee..6f2316facbb 100644 --- a/ci/provisioning/post_provision_config_nodes_EL_8.sh +++ b/ci/provisioning/post_provision_config_nodes_EL_8.sh @@ -4,6 +4,8 @@ # # SPDX-License-Identifier: BSD-2-Clause-Patent +PYTHON3_VERSION="3.11" + bootstrap_dnf() { systemctl enable postfix.service systemctl start postfix.service @@ -19,8 +21,8 @@ distro_custom() { # Use a more recent python version for unit testing, this allows us to also test installing # pydaos into virtual environments. - dnf -y install python3.11 python3.11-devel - sudo update-alternatives --set python3 /usr/bin/python3.11 + dnf -y install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-devel + sudo update-alternatives --set python3 /usr/bin/python${PYTHON3_VERSION} update-alternatives --list } diff --git a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh index 589f663c7e4..7212cacbb03 100755 --- a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh @@ -1,5 +1,7 @@ #!/bin/bash +PYTHON3_VERSION="3.9" + bootstrap_dnf() { rm -rf "$REPOS_DIR" ln -s ../zypp/repos.d "$REPOS_DIR" @@ -19,7 +21,7 @@ distro_custom() { # Use a more recent python version for unit testing, this allows us to also test installing # pydaos into virtual environments. - dnf -y install python3.11 python3.11-devel - sudo update-alternatives --set python3 /usr/bin/python3.11 - update-alternatives --list + dnf -y install python${PYTHON3_VERSION} python${PYTHON3_VERSION}-devel + sudo update-alternatives --set python3 /usr/bin/python${PYTHON3_VERSION} + update-alternatives --list python3 } From e14207f90eef67405884400d332edd813a6c96bf Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 31 May 2024 17:27:50 -0400 Subject: [PATCH 11/12] Fix leap Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-leap15: false Skip-func-test-el9: false Test-tag: pr,-cart Required-githooks: true Signed-off-by: Phil Henderson --- ci/provisioning/post_provision_config_nodes_LEAP_15.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh index 7212cacbb03..b8a2eab6468 100755 --- a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh @@ -1,6 +1,6 @@ #!/bin/bash -PYTHON3_VERSION="3.9" +PYTHON3_VERSION="3.11" bootstrap_dnf() { rm -rf "$REPOS_DIR" From 5446200d89833bf1fee6d24b7f2fa7cf584df66e Mon Sep 17 00:00:00 2001 From: Phil Henderson Date: Fri, 31 May 2024 23:17:08 -0400 Subject: [PATCH 12/12] Adding missed change. Skip-unit-tests: true Skip-fault-injection-test: true Skip-func-hw-test: true Skip-func-test-leap15: false Skip-func-test-el9: false Test-tag: pr,-cart Required-githooks: true Signed-off-by: Phil Henderson --- ci/provisioning/post_provision_config_nodes_LEAP_15.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh index b8a2eab6468..9f36cb00edb 100755 --- a/ci/provisioning/post_provision_config_nodes_LEAP_15.sh +++ b/ci/provisioning/post_provision_config_nodes_LEAP_15.sh @@ -1,6 +1,6 @@ #!/bin/bash -PYTHON3_VERSION="3.11" +PYTHON3_VERSION="311" bootstrap_dnf() { rm -rf "$REPOS_DIR"