DAOS-15603: Support of FIO with ansible ftest script

utils/ansible/ftest/README.md

@@ -67,15 +67,15 @@ The inventory should also contain a set of mandatory and optional variables.
   sub-cluster.
 - **daos\_source\_dir**: mandatory variable only used by the node of the *daos\_dev* group defining
   the path of the directory containing the DAOS source code.
+- **daos\_ofi\_provider**: optional variable (default value: ofi+tcp;ofi_rxm) defining the network
+  provider to be used by the DAOS engines.  It also defines which network rpms driver to install
+  (e.g. ib mellanox driver).
 - **daos\_ofi\_interface**: optional variable only used by the node of the daos\_dev group defining
   the network interface to use.  When this variable is not defined, the network interface is
   arbitrarily selected by DAOS.
 - **daos\_hugepages\_nb**: optional variable (default value: 4096) only used by the nodes of the
-  *daos\_servers* group defining the number of hugepages to be allocated by the linux kernel.
-- **daos\_avocado\_version**: optional variable (default value: "2.4.3") only used by the node of
-  the *daos\_dev* group defining the version of *avocado* to install.
-- **daos\_avocado\_framework\_version**: optional variable (default value: "82.1") only used by the
-  node of the *daos\_dev* group defining the version of *avocado\_framework* to install.
+  *daos\_servers* group.  This variable defines the number of hugepages to be allocated by the linux
+  kernel.
 
 Different file format (e.g. YAML, INI, etc.) and file tree structure are supported to define an
 ansible inventory.  The following simple ansible inventory describe for example in one YAML file
@@ -155,31 +155,11 @@ defined in the inventory. Usage of these two scripts will be detailed in the fol
 ## Installing DAOS Binaries
 
 Building and installing binaries could be done thanks to the generated bash script `daos-make.sh`.
-This last script supports several options and the two sub-commands `install` and `update`.
+This last script supports several options alllowing such as building dependencies or not.
 
 More details on the supported options could be found with running the command with the `--help`
 option.
 
-The `install` sub-command should be used to install/reinstall from scratch all the DAOS binaries and
-their dependencies such as the *spdk* or the *mercury* libraries. For example, the *foo* user could
-use the following command line to install or reinstall the DAOS binaries and its dependencides into
-the `/home/foo/daos/install` directory.
-
-```bash
-/home/foo/daos/daos-make.sh -v -j 32 -f install
-```
-
-When the previous *install* step has been done and the DAOS source code tests have been updated, the
-DAOS binaries could be build and reinstalled thanks to the `update` sub-command.  This last one
-should be far quicker than a full reinstall.
-
-```bash
-/home/foo/daos/daos-make.sh -v -j 32 update
-```
-
-> :bulb: The option `-j 32` (i.e. one per core) seems to be a good compromise between compilation
-> reliability and speed.
-
 ## Launching Functional Tests
 
 Launching the DAOS functional tests could be done thanks to the generated bash script
@@ -192,7 +172,6 @@ with running the command with the `--help` option.
 After successfully installing the DAOS binaries and dependencies, the user *foo* could run all the
 test(s) with the tag `hello_world` thanks to the following command line.
 
-
 ```bash
 /home/foo/daos/daos-launch.sh -v -- --nvme=auto  hello_world
 ```

utils/ansible/ftest/file/configure-nofile.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# set -x
+set -u -e -o pipefail
+
+if [[ $(id -u) -ne 0 ]] ; then
+	echo "[ERROR] Could only be used by root"
+	exit 1
+fi
+
+sed -i -E -e "/DefaultLimitNOFILE/s/.*/DefaultLimitNOFILE=infinity:infinity/" /etc/systemd/system.conf
+
+cat > /etc/security/limits.d/daos-nofile.conf << EOF
+# /etc/security/limits.d/daos-nofile.conf
+
+# <domain>	<type>	<item>	<value>
+*		hard	nofile	unlimited
+*		soft	nofile	unlimited
+EOF
+
+systemctl daemon-reexec

utils/ansible/ftest/file/enable-coredumps.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# set -x
+set -u -e -o pipefail
+
+if [[ $(id -u) -ne 0 ]] ; then
+	echo "[ERROR] Could only be used by root"
+	exit 1
+fi
+
+sed -i -E -e "/DefaultLimitCORE/s/.*/DefaultLimitCORE=infinity:infinity/" /etc/systemd/system.conf
+
+cat > /etc/security/limits.d/daos-coredumps.conf << EOF
+# /etc/security/limits.d/daos-coredumps.conf
+
+# <domain>	<type>	<item>	<value>
+*		hard	core	unlimited
+*		soft	core	unlimited
+EOF
+
+cat > /etc/sysctl.d/daos-coredumps.conf << EOF
+# /etc/sysctl.d/daos-coredumps.conf
+
+fs.suid_dumpable=1
+EOF
+sysctl -q -w "fs.suid_dumpable=1"
+sysctl -p
+
+cat > /etc/profile.d/daos-coredumps.sh  << EOF
+# Enable core dump
+ulimit -c unlimited
+EOF
+
+mkdir -p /etc/systemd/coredump.conf.d/
+cat > /etc/systemd/coredump.conf.d/daos-coredumps.conf << EOF
+# /etc/systemd/coredump.conf.d/daos-coredumps.conf
+
+[Coredump]
+Storage=external
+Compress=yes
+ProcessSizeMax=8G
+ExternalSizeMax=8G
+KeepFree=2G
+EOF
+systemctl daemon-reexec

utils/ansible/ftest/inventory-sample.yml

@@ -1,21 +1,40 @@
 # Sample inventory of a DAOS functional platform running on the wolf cluster
+#
+# Detailed information on the content of this ansible inventory file is available in the README.md
+# file of the directory containing this first file.
 
-all:
-  vars:
-    daos_runtime_dir: /home/foo/daos
-  children:
-    daos_dev:
-      vars:
-        daos_source_dir: /home/foo/work/daos
-        daos_ofi_interface: eth0
-      hosts:
-        wolf-666:
-    daos_servers:
-      vars:
-        daos_hugepages_nb: 8182
-      hosts:
-        wolf-666:
-        wolf-999:
-    daos_clients:
-      hosts:
-        wolf-999:
+# all:
+#   vars:
+#     #
+#     # Mandatory variable defining the shared directory used to install DAOS.
+#     daos_runtime_dir: /home/foo/daos
+#     #
+#     # Optional variable defining the network provider to use by the DAOS engines.  This variable
+#     # is also used to define which network driver rpms have to be install.
+#     #  default: ofi+tcp;ofi_rxm
+#     daos_ofi_provider: ofi+tcp;ofi_rxm
+#   children:
+#     daos_dev:
+#       vars:
+#         #
+#         # Mandatory variable defining the path of the directory containing the DAOS source code.
+#         daos_source_dir: /home/foo/work/daos
+#         #
+#         # Optional variable defining the network interface to use with the launch.py python
+#         # script.
+#         #  default: None
+#         daos_ofi_interface: eth0
+#       hosts:
+#         wolf-666:
+#     daos_servers:
+#       vars:
+#         #
+#         # Optional variable defining the number of hugepages to be allocated by the linux kernel.
+#         #  default: 4096
+#         daos_hugepages_nb: 8182
+#       hosts:
+#         wolf-666:
+#         wolf-999:
+#     daos_clients:
+#       hosts:
+#         wolf-999:

utils/ansible/ftest/tasks/daos_base.yml

@@ -37,13 +37,28 @@
     mode: '0755'
     recurse: true
 
-- name: Create MPI module configuration file
+- name: Copy bash script enable-coredumps.sh
   remote_user: root
-  file:
-    src: "{{ daos_openmpi_module_path }}"
-    path: /etc/modulefiles/mpi/mlnx_openmpi-x86_64
-    state: link
+  copy:
+    src: file/enable-coredumps.sh
+    dest: /root/enable-coredumps.sh
     owner: root
     group: root
-    mode: '0755'
-    force: true
+    mode: '0644'
+
+- name: Enable CORE dump
+  remote_user: root
+  command: /bin/bash /root/enable-coredumps.sh
+
+- name: Copy bash script configure-nofile.sh
+  remote_user: root
+  copy:
+    src: file/configure-nofile.sh
+    dest: /root/configure-nofile.sh
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Configure Number of Open Files
+  remote_user: root
+  command: /bin/bash /root/configure-nofile.sh

utils/ansible/ftest/tasks/daos_clients.yml

@@ -69,3 +69,12 @@
     group: root
     mode: '0644'
     backup: true
+
+- name: Update search path commands
+  remote_user: root
+  lineinfile:
+    path: /etc/security/pam_env.conf
+    state: present
+    regexp: '^PATH='
+    line: "PATH={{ daos_runtime_dir }}/install/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/bin/syscfg:/root/bin"
+    backup: true

utils/ansible/ftest/tasks/daos_dev.yml

@@ -44,17 +44,16 @@
     group: daos_server
     mode: 0700
 
-- name: Install dependencies packages
-  include_tasks: "{{ item }}"
-  with_first_found:
-    - "{{ ansible_distribution | replace(' ', '_') }}{{ ansible_distribution_major_version}}/devs_dependencies.yml"
-
 - name: Create DAOS runtime directory
   file:
     path: "{{ daos_runtime_dir }}"
     state: directory
     mode: '0755'
-    recurse: true
+
+- name: Install dependencies packages
+  include_tasks: "{{ item }}"
+  with_first_found:
+    - "{{ ansible_distribution | replace(' ', '_') }}{{ ansible_distribution_major_version}}/devs_dependencies.yml"
 
 - name: Install DAOS make helper script
   template:

utils/ansible/ftest/tasks/daos_servers.yml

@@ -19,7 +19,7 @@
 - name: Check Huge Pages
   remote_user: root
   daos_hugepages:
-    size: "{{ daos_hugepages_nb | default(4096) }}"
+    size: "{{ daos_hugepages_nb | default(daos_hugepages_nb_default) }}"
 
 - name: Add DAOS server groups
   remote_user: root
@@ -84,4 +84,4 @@
     owner: root
     group: root
     mode: 0755
-  loop: "[''] + {{ range(0, 4, 1) | list }}"
+  loop: "{{ range(0, 4, 1) | list + [''] }}"

utils/ansible/ftest/tasks/el8/clients_dependencies.yml

@@ -6,3 +6,11 @@
     state: latest
     update_cache: true
     name: "{{ daos_base_deps.rpms + daos_client_deps.rpms }}"
+
+- name: Install MOFED rpms
+  remote_user: root
+  dnf:
+    state: latest
+    update_cache: true
+    name: "{{ mlnx_deps.rpms }}"
+  when: daos_ofi_provider | default(daos_ofi_provider_default) == "ofi+verbs;ofi_rxm"

utils/ansible/ftest/tasks/el8/devs_dependencies.yml

@@ -6,19 +6,3 @@
     state: latest
     update_cache: true
     name: "{{ daos_base_deps.rpms + daos_dev_deps.rpms }}"
-
-- name: Install DAOS building dependencies
-  remote_user: root
-  command:
-    dnf builddep --nobest --assumeyes "{{ playbook_dir }}/file/daos.spec"
-  args:
-    warn: false
-  register: dnf_result
-  become: true
-  changed_when: '"Nothing to do" not in dnf_result.stdout'
-
-- name: Install python development dependencies
-  remote_user: root
-  pip:
-    state: present
-    name: "{{ daos_dev_deps.pips }}"

utils/ansible/ftest/tasks/el8/servers_dependencies.yml

@@ -6,3 +6,11 @@
     state: latest
     update_cache: true
     name: "{{ daos_base_deps.rpms + daos_server_deps.rpms }}"
+
+- name: Install MOFED rpms
+  remote_user: root
+  dnf:
+    state: latest
+    update_cache: true
+    name: "{{ mlnx_deps.rpms }}"
+  when: daos_ofi_provider | default(daos_ofi_provider_default) == "ofi+verbs;ofi_rxm"