feat(conf): support slurm-quota v3

CHANGELOG.md

@@ -33,6 +33,11 @@ and this project adheres to
     setup with https as dependency of other role before certificate and key are
     deployed. This should be handled by handlers only.
   - Move HTTP certificate boostrap and deployment from slurm-web to nginx role.
+  - Refactor _slurm_ role to create system user/group in the first place,
+    before slurmquota is possibly run.
+  - Move _mariadb_ dependency from _slurm_ metadata to _server_ tasks in order
+    to avoid this dependency from being triggered with system user/group
+    creation.
 - core: Cache base OS image locally to avoid systematic download on cluster
   deployment.
 - load: Submit jobs with GPU types when GPU GRES are declared with types on

conf/bootstrap.yml

@@ -24,6 +24,10 @@
     include_role:
       name: nginx
       tasks_from: bootstrap
+  - name: Generate slurm-quota API TLS certificate and session key
+    include_role:
+      name: slurmquota
+      tasks_from: bootstrap
 
 - hosts: all
   connection: machinectl

conf/group_vars/all.yml

@@ -103,3 +103,9 @@ slurmquota_web_tls_key_file: "{{ fhpc_web_tls_key_file }}"
 slurmquota_web_http_server_names:
 - "{{ slurmquota_web_hostname }}"
 - "{{ slurmquota_web_hostname }}.{{ fhpc_namespace }}"
+slurmquota_ldap_uri: "ldaps://{{ fhpc_admin_server }}/"
+slurmquota_ldap_user_base: "ou=people,{{ fhpc_ldap_base }}"
+slurmquota_ldap_group_base: "ou=groups,{{ fhpc_ldap_base }}"
+slurmquota_admins: "{{ fhpc_users | map(attribute='login') | list }}"
+slurmquota_web_api_url: "https://{{ fhpc_admin_server }}:9911/"
+slurmquota_local_web_session_key_file: "{{ fhpc_cluster_state_dir }}/slurm-quota/web-session.key"

conf/roles/slurm/tasks/main.yml

@@ -1,24 +1,6 @@
 ---
-- name: Gather OS specific variables
-  ansible.builtin.include_vars:
-    file: "os/{{ ansible_facts.os_family | lower }}.yml"
-
-# On Debian, the packages create the slurm system user and group by default. On
-# redhat based distributions, the user must be created manually.
-- name: Create slurm system group
-  ansible.builtin.group:
-    name: slurm
-    gid: "{{ slurm_gid }}"
-
-- name: Create slurm system user
-  ansible.builtin.user:
-    name: slurm
-    uid: "{{ slurm_uid }}"
-    group: slurm
-    system: yes
-    shell: /sbin/nologin
-    home: /var/spool/slurm
-    create_home: no
+- name: Ensure slurm system user and group exist
+  ansible.builtin.include_tasks: user.yml
 
 - name: Install slurm common packages
   ansible.builtin.package:

conf/roles/slurm/tasks/server.yml

@@ -1,4 +1,19 @@
 ---
+- name: Install MariaDB for Slurm accounting
+  ansible.builtin.include_role:
+    name: mariadb
+    apply:
+      tags:
+      - mariadb
+      - dependencies
+  vars:
+    mariadb_users:
+    - name: slurm
+      host: localhost
+      password: "{{ slurm_db_password }}"
+      priv: slurm_acct_db.*:ALL
+  when: slurm_with_accounting
+
 - name: Create slurmctld state directory
   ansible.builtin.file:
     path: "{{ slurm_state_save_loc }}"

conf/roles/slurm/tasks/user.yml

@@ -0,0 +1,24 @@
+---
+# On Debian, slurm packages create the system user and group. On Red Hat based
+# distributions, they must be created before installing Slurm or components
+# such as slurm-quota that run services as the slurm user.
+- name: Gather OS specific variables
+  ansible.builtin.include_vars:
+    file: "os/{{ ansible_facts.os_family | lower }}.yml"
+
+- name: Create slurm system group
+  ansible.builtin.group:
+    name: slurm
+    gid: "{{ slurm_gid }}"
+  when: ansible_facts.os_family == 'RedHat'
+
+- name: Create slurm system user
+  ansible.builtin.user:
+    name: slurm
+    uid: "{{ slurm_uid }}"
+    group: slurm
+    system: yes
+    shell: /sbin/nologin
+    home: /var/spool/slurm
+    create_home: no
+  when: ansible_facts.os_family == 'RedHat'

conf/roles/slurmquota/defaults/main.yml

@@ -12,14 +12,37 @@ slurmquota_uwsgi_packages:
 - uwsgi-plugin-python3
 
 slurmquota_client_env_file: /etc/profile.d/slurm-quota.sh
-slurmquota_client_env_url: "http://{{ groups[slurmquota_controller_group][0] }}:9911/"
+slurmquota_client_env_url: "https://{{ groups[slurmquota_controller_group][0] }}:9911/"
 slurmquota_web_hostname: "{{ inventory_hostname }}"
 slurmquota_web_http_server_names:
 - "{{ slurmquota_web_hostname }}"
 slurmquota_web_subdir: quota
-slurmquota_web_static_dir: /usr/share/slurm-quota-web/static
+slurmquota_web_static_dir: /usr/share/slurm-quota/web/static
 slurmquota_web_tls_cert_file: /etc/nginx/tls/cert.crt
 slurmquota_web_tls_key_file: /etc/nginx/tls/key.pem
-slurmquota_web_app_file: /usr/libexec/slurm-quota/slurm-quota-web
+slurmquota_web_app_file: /usr/share/slurm-quota/web/wsgi/slurm-quota-web.wsgi
 slurmquota_web_uwsgi_socket: /run/slurm-quota-web/uwsgi.sock
 slurmquota_web_uwsgi_service_name: slurm-quota-web-uwsgi
+
+slurmquota_serve_ini: /etc/slurm-quota/serve.ini
+slurmquota_api_tls_dir: /etc/slurm-quota/tls
+slurmquota_api_tls_cert: "{{ slurmquota_api_tls_dir }}/cert.pem"
+slurmquota_api_tls_key: "{{ slurmquota_api_tls_dir }}/key.pem"
+slurmquota_api_hostname: "{{ fhpc_admin_server }}"
+slurmquota_local_ca_dir: "{{ common_local_ca_dir }}"
+slurmquota_local_tls_cert_file: "{{ slurmquota_local_ca_dir }}/cert-slurm-quota.crt"
+slurmquota_local_tls_key_file: "{{ slurmquota_local_ca_dir }}/key-slurm-quota.pem"
+slurmquota_local_ca_cert_file: "{{ slurmquota_local_ca_dir }}/ca.crt"
+slurmquota_local_ca_key_file: "{{ slurmquota_local_ca_dir }}/key.pem"
+slurmquota_local_ca_password_file: "{{ slurmquota_local_ca_dir }}/ca.password"
+
+slurmquota_web_env_file: /etc/default/slurm-quota-web
+slurmquota_web_session_key_file: /etc/slurm-quota/web-session.key
+slurmquota_local_web_session_key_file: slurm-quota/web-session.key  # dummy
+slurmquota_web_api_url: "https://{{ groups[slurmquota_controller_group][0] }}:9911/"
+
+slurmquota_auth_method: ldap
+slurmquota_ldap_uri: "ldaps://{{ fhpc_admin_server }}/"
+slurmquota_ldap_user_base: "ou=people,{{ fhpc_ldap_base }}"
+slurmquota_ldap_group_base: "ou=groups,{{ fhpc_ldap_base }}"
+slurmquota_admins: []

conf/roles/slurmquota/handlers/main.yml

@@ -1,4 +1,10 @@
 ---
+- name: Restart slurm-quota API
+  ansible.builtin.systemd_service:
+    name: slurm-quota.socket
+    state: restarted
+    daemon_reload: true
+
 - name: Restart slurm-quota-web uWSGI
   ansible.builtin.systemd_service:
     name: "{{ slurmquota_web_uwsgi_service_name }}"

conf/roles/slurmquota/tasks/bootstrap.yml

@@ -0,0 +1,36 @@
+---
+- name: Create private key for slurm-quota API TLS certificate
+  community.crypto.openssl_privatekey:
+    path: "{{ slurmquota_local_tls_key_file }}"
+
+- name: Create certificate signing request (CSR) for slurm-quota API certificate
+  community.crypto.openssl_csr_pipe:
+    privatekey_path: "{{ slurmquota_local_tls_key_file }}"
+    common_name: "{{ slurmquota_api_hostname }}"
+    subject_alt_name:
+    - "DNS:{{ slurmquota_api_hostname }}"
+    - "DNS:{{ slurmquota_api_hostname }}.{{ fhpc_namespace }}"
+  register: slurmquota_api_tls_csr
+
+- name: Sign slurm-quota API certificate with internal CA
+  community.crypto.x509_certificate:
+    csr_content: "{{ slurmquota_api_tls_csr.csr }}"
+    provider: ownca
+    ownca_path: "{{ slurmquota_local_ca_cert_file }}"
+    ownca_privatekey_path: "{{ slurmquota_local_ca_key_file }}"
+    ownca_privatekey_passphrase: "{{ lookup('ansible.builtin.file', slurmquota_local_ca_password_file) }}"
+    ownca_not_after: +365d  # valid for one year
+    ownca_not_before: "-1d"  # valid since yesterday
+    path: "{{ slurmquota_local_tls_cert_file }}"
+    force: true  # override possibly existing certificate
+
+- name: Create local slurm-quota directory
+  ansible.builtin.file:
+    path: "{{ slurmquota_local_web_session_key_file | dirname }}"
+    state: directory
+    recurse: true
+
+- name: Generate slurm-quota web session key file
+  ansible.builtin.shell:
+    cmd: "openssl rand -hex 32 > {{ slurmquota_local_web_session_key_file }}"
+    creates: "{{ slurmquota_local_web_session_key_file }}"

conf/roles/slurmquota/tasks/main.yml

@@ -26,37 +26,89 @@
     name: "{{ slurmquota_packages }}"
     state: present
 
-- name: Start and enable slurm-quota socket activation
-  ansible.builtin.systemd_service:
-    name: slurm-quota.socket
-    state: started
-    enabled: true
-    daemon_reload: true
+- name: Configure slurm-quota controller
   when: slurmquota_is_controller
+  block:
+  - name: Create slurm-quota API TLS directory
+    ansible.builtin.file:
+      path: "{{ slurmquota_api_tls_dir }}"
+      state: directory
+      owner: root
+      group: slurm
+      mode: "0750"
 
-- name: Install uWSGI packages for slurm-quota-web
-  ansible.builtin.dnf:
-    name: "{{ slurmquota_uwsgi_packages }}"
-    state: present
-  when: slurmquota_is_controller
+  - name: Deploy slurm-quota API TLS certificate
+    ansible.builtin.copy:
+      src: "{{ slurmquota_local_tls_cert_file }}"
+      dest: "{{ slurmquota_api_tls_cert }}"
+      owner: slurm
+      group: slurm
+      mode: "0644"
+    notify: Restart slurm-quota API
 
-- name: Deploy slurm-quota web uWSGI systemd service
-  ansible.builtin.template:
-    src: slurm-quota-web-uwsgi.service.j2
-    dest: "/etc/systemd/system/{{ slurmquota_web_uwsgi_service_name }}.service"
-    owner: root
-    group: root
-    mode: "0644"
-  notify: Restart slurm-quota-web uWSGI
-  when: slurmquota_is_controller
+  - name: Deploy slurm-quota API TLS private key
+    ansible.builtin.copy:
+      src: "{{ slurmquota_local_tls_key_file }}"
+      dest: "{{ slurmquota_api_tls_key }}"
+      owner: slurm
+      group: slurm
+      mode: "0640"
+    notify: Restart slurm-quota API
 
-- name: Start and enable slurm-quota-web uWSGI service
-  ansible.builtin.systemd_service:
-    name: "{{ slurmquota_web_uwsgi_service_name }}"
-    state: started
-    enabled: true
-    daemon_reload: true
-  when: slurmquota_is_controller
+  - name: Deploy slurm-quota web session key
+    ansible.builtin.copy:
+      src: "{{ slurmquota_local_web_session_key_file }}"
+      dest: "{{ slurmquota_web_session_key_file }}"
+      owner: root
+      group: root
+      mode: "0400"
+    notify: Restart slurm-quota-web uWSGI
+
+  - name: Deploy slurm-quota serve.ini
+    ansible.builtin.template:
+      src: serve.ini.j2
+      dest: "{{ slurmquota_serve_ini }}"
+      owner: root
+      group: root
+      mode: "0644"
+    notify: Restart slurm-quota API
+
+  - name: Deploy slurm-quota web environment file
+    ansible.builtin.template:
+      src: slurm-quota-web.default.j2
+      dest: "{{ slurmquota_web_env_file }}"
+      owner: root
+      group: root
+      mode: "0644"
+    notify: Restart slurm-quota-web uWSGI
+
+  - name: Start and enable slurm-quota socket activation
+    ansible.builtin.systemd_service:
+      name: slurm-quota.socket
+      state: started
+      enabled: true
+      daemon_reload: true
+
+  - name: Install uWSGI packages for slurm-quota-web
+    ansible.builtin.dnf:
+      name: "{{ slurmquota_uwsgi_packages }}"
+      state: present
+
+  - name: Deploy slurm-quota web uWSGI systemd service
+    ansible.builtin.template:
+      src: slurm-quota-web-uwsgi.service.j2
+      dest: "/etc/systemd/system/{{ slurmquota_web_uwsgi_service_name }}.service"
+      owner: root
+      group: root
+      mode: "0644"
+    notify: Restart slurm-quota-web uWSGI
+
+  - name: Start and enable slurm-quota-web uWSGI service
+    ansible.builtin.systemd_service:
+      name: "{{ slurmquota_web_uwsgi_service_name }}"
+      state: started
+      enabled: true
+      daemon_reload: true
 
 - name: Configure SLURM_QUOTA_URL on non-controller nodes
   ansible.builtin.template:

conf/roles/slurmquota/templates/serve.ini.j2

@@ -0,0 +1,19 @@
+# Managed by Ansible (slurmquota role)
+[authentication]
+method={{ slurmquota_auth_method }}
+
+[ldap]
+uri={{ slurmquota_ldap_uri }}
+user_base={{ slurmquota_ldap_user_base }}
+group_base={{ slurmquota_ldap_group_base }}
+
+[authorization]
+admins=
+{% for admin in slurmquota_admins %}
+  {{ admin }}
+{% endfor %}
+
+[tls]
+enabled=true
+cert={{ slurmquota_api_tls_cert }}
+key={{ slurmquota_api_tls_key }}

conf/roles/slurmquota/templates/slurm-quota-web.default.j2

@@ -0,0 +1,4 @@
+# Managed by Ansible (slurmquota role)
+SLURM_QUOTA_URL={{ slurmquota_web_api_url }}
+SLURM_QUOTA_WEB_SESSION_KEY_FILE={{ slurmquota_web_session_key_file }}
+SLURM_QUOTA_WEB_SECURE_COOKIES=1

conf/site.yml

@@ -29,6 +29,15 @@
 
 - hosts: all
   remote_user: root
+  pre_tasks:
+    # Create slurm system user/group before slurm-quota because the controller
+    # configures files owned by slurm. pre_tasks run before roles; listing slurm
+    # twice under roles would deduplicate the role.
+    - name: Create slurm system user and group
+      ansible.builtin.include_role:
+        name: slurm
+        tasks_from: user
+      tags: [ slurm, slurmquota ]
   roles:
     # Deploy slurm-quota before slurm because slurm needs job_submit.lua script
     # from slurm-quota.