Merge remote-tracking branch 'origin/main' into production-end-to-end-deployment-docs

Author: MoteHue

.gitignore

@@ -5,3 +5,4 @@ venv
 *.pyc
 packer/openhpc2
 .vscode
+requirements.yml.last

README.md

@@ -25,8 +25,6 @@ The default configuration in this repository may be used to create a cluster to
 - Persistent state backed by an OpenStack volume.
 - NFS-based shared file system backed by another OpenStack volume.
 
-Note that the Open OnDemand portal and its remote apps are not usable with this default configuration.
-
 It requires an OpenStack cloud, and an Ansible "deploy host" with access to that cloud.
 
 Before starting ensure that:

ansible/bootstrap.yml

@@ -143,7 +143,7 @@
       - appliances_mode == 'configure'
       - not (dnf_repos_allow_insecure_creds | default(false)) # useful for development
 
-- hosts: cacerts:!builder
+- hosts: cacerts
   tags: cacerts
   gather_facts: false
   tasks:

ansible/roles/cacerts/README.md

@@ -4,7 +4,7 @@ Configure CA certificates and trusts.
 
 ## Role variables
 
-- `ca-certificates`: Optional str. Path to directory containing certificates
+- `cacerts_cert_dir`: Optional str. Path to directory containing certificates
   in PEM or DER format. Any files here will be added to the list of CAs trusted
   by the system.
 

ansible/roles/cuda/README.md

@@ -2,14 +2,10 @@
 
 Install NVIDIA drivers and optionally CUDA packages. CUDA binaries are added to the `$PATH` for all users, and the [NVIDIA persistence daemon](https://docs.nvidia.com/deploy/driver-persistence/index.html#persistence-daemon) is enabled.
 
-## Prerequisites
-
-Requires OFED to be installed to provide required kernel-* packages.
-
 ## Role Variables
 
 - `cuda_repo_url`: Optional. URL of `.repo` file. Default is upstream for appropriate OS/architecture.
 - `cuda_nvidia_driver_stream`: Optional. Version of `nvidia-driver` stream to enable. This controls whether the open or proprietary drivers are installed and the major version. Changing this once the drivers are installed does not change the version.
-- `cuda_packages`: Optional. Default: `['cuda', 'nvidia-gds', 'cmake', 'cuda-toolkit-12-8']`.
+- `cuda_packages`: Optional. Default provides CUDA Toolkit and GPUDirect Storage (GDS).
 - `cuda_package_version`: Optional. Default `latest` which will install the latest packages if not installed but won't upgrade already-installed packages. Use `'none'` to skip installing CUDA.
 - `cuda_persistenced_state`: Optional. State of systemd `nvidia-persistenced` service. Values as [ansible.builtin.systemd:state](https://docs.ansible.com/ansible/latest/collections/ansible/builtin/systemd_module.html#parameter-state). Default `started`.

ansible/roles/cuda/defaults/main.yml

@@ -1,12 +1,12 @@
 cuda_repo_url: "https://developer.download.nvidia.com/compute/cuda/repos/rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/cuda-rhel{{ ansible_distribution_major_version }}.repo"
-cuda_nvidia_driver_stream: '570-open'
-cuda_package_version: '12.8.1-1'
-cuda_version_short: '12.8'
+cuda_nvidia_driver_stream: '575-open'
+cuda_nvidia_driver_pkg: "nvidia-open-3:575.57.08-1.el{{ ansible_distribution_major_version }}"
+cuda_package_version: '12.9.1-1'
+cuda_version_short: "{{ (cuda_package_version | split('.'))[0:2] | join('.') }}" # major.minor
 cuda_packages:
-  - "cuda{{ ('-' + cuda_package_version) if cuda_package_version != 'latest' else '' }}"
+  - "cuda-toolkit-{{ cuda_package_version }}"
   - nvidia-gds
   - cmake
-  - cuda-toolkit-12-8
 cuda_samples_release_url: "https://github.com/NVIDIA/cuda-samples/archive/refs/tags/v{{ cuda_version_short }}.tar.gz"
 cuda_samples_path: "/var/lib/{{ ansible_user }}/cuda_samples"
 cuda_samples_programs:

ansible/roles/cuda/tasks/install.yml

@@ -1,16 +1,5 @@
 
-# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#redhat8-installation
-
-- name: Check for OFED/DOCA
-  command:
-    cmd: dnf list --installed rdma-core
-  register: _dnf_rdma_core
-  changed_when: false
-
-- name: Assert OFED installed
-  assert: 
-    that: "'mlnx' in _dnf_rdma_core.stdout"
-    fail_msg: "Did not find 'mlnx' in installed rdma-core package, is OFED/DOCA installed?"
+# Based on https://docs.nvidia.com/datacenter/tesla/driver-installation-guide/
 
 - name: Install cuda repo
   get_url:
@@ -29,23 +18,18 @@
   when: "'No matching Modules to list' in _cuda_driver_module_enabled.stderr"
   changed_when: "'Nothing to do' not in _cuda_driver_module_enable.stdout"
 
-- name: Check if nvidia driver module is installed
-  ansible.builtin.command: dnf module list --installed nvidia-driver
-  changed_when: false
-  failed_when: false
-  register: _cuda_driver_module_installed
-
 - name: Install nvidia drivers
-  ansible.builtin.command: dnf module install -y nvidia-driver
+  ansible.builtin.dnf:
+    name: "{{ cuda_nvidia_driver_pkg }}"
   register: _cuda_driver_install
-  when: "'No matching Modules to list' in _cuda_driver_module_installed.stderr"
-  changed_when: "'Nothing to do' not in _cuda_driver_install.stdout"
 
 - name: Check kernel has not been modified
   assert:
     that: "'kernel ' not in _cuda_driver_install.stdout | default('')" # space ensures we don't flag e.g. kernel-devel-matched
     fail_msg: "{{ _cuda_driver_install.stdout_lines | default([]) | select('search', 'kernel ') }}"
 
+# Based on https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
+
 - name: Install cuda packages
   ansible.builtin.dnf:
     name: "{{ cuda_packages }}"

ansible/roles/cuda/tasks/samples.yml

@@ -25,36 +25,3 @@
     cmd: . /etc/profile.d/sh.local && cmake .. && make -j {{ ansible_processor_vcpus }}
     chdir: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build"
     creates: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build/Samples/1_Utilities/deviceQuery/deviceQuery"
-
-- name: Run CUDA deviceQuery
-  command:
-    cmd: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build/Samples/1_Utilities/deviceQuery/deviceQuery"
-  register: _cuda_devicequery
-
-- name: Set fact for CUDA devices
-  set_fact:
-    cuda_devices: "{{ _cuda_devicequery.stdout | regex_findall('Device (\\d+):') }}"
-
-- name: Run CUDA bandwidth test
-  command:
-    cmd: "{{ cuda_samples_path }}/cuda-samples-{{ cuda_version_short }}/build/Samples/1_Utilities/bandwidthTest/bandwidthTest --device={{ item }}"
-  register: _cuda_bandwidthtest
-  loop: "{{ cuda_devices }}"
-  loop_control:
-    label: "Device {{ item }}" # e.g '0'
-
-- name: Summarise bandwidth test output
-  debug:
-    msg: |
-      {{ _parts[1].splitlines()[0] | trim }}
-      Bandwidths: (Gb/s)
-      Host to Device: {{ _parts[2].split()[-1] }}
-      Device to Host: {{ _parts[3].split()[-1] }}
-      Device to Device: {{ _parts[4].split()[-1] }}
-      {{ ': '.join(_parts[5].split('=') | map('trim')) }}
-      {{ _parts[6] }}
-  loop: "{{ _cuda_bandwidthtest.results }}"
-  vars:
-    _parts: "{{ item.stdout.split('\n\n') }}"
-  loop_control:
-    label: "Device {{ item.item }}" # e.g '0'

ansible/roles/lustre/README.md

@@ -7,8 +7,7 @@ Install and configure a Lustre client. This builds RPM packages from source.
 **NB:** Currently this only supports RockyLinux 9.
 
 ## Role Variables
-
-- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6` which is the first version with EL9.5 support
+The following variables control configuration of Lustre clients.
 - `lustre_lnet_label`: Optional str. The "lnet label" part of the host's NID, e.g. `tcp0`. Only the `tcp` protocol type is currently supported. Default `tcp`.
 - `lustre_mgs_nid`: Required str. The NID(s) for the MGS, e.g. `192.168.227.11@tcp1` (separate mutiple MGS NIDs using `:`).
 - `lustre_mounts`: Required list. Define Lustre filesystems and mountpoints as a list of dicts with keys:
@@ -19,7 +18,11 @@ Install and configure a Lustre client. This builds RPM packages from source.
 - `lustre_mount_state`. Optional default mount state for all mounts, as for [ansible.posix.mount](https://docs.ansible.com/ansible/latest/collections/ansible/posix/mount_module.html#parameter-state). Default is `mounted`.
 - `lustre_mount_options`. Optional default mount options. Default values are systemd defaults from [Lustre client docs](http://wiki.lustre.org/Mounting_a_Lustre_File_System_on_Client_Nodes).
 
-The following variables control the package build and and install and should not generally be required:
+The following variables control the package build and and install:
+- `lustre_version`: Optional str. Version of lustre to build, default `2.15.6/lu-18085`
+  which is the first version with EL9.5 support, plus a fix for https://jira.whamcloud.com/browse/LU-18085.
+- `lustre_repo`: Optional str. URL for Lustre repo. Default is a StackHPC repo
+  incorporating the above fix.
 - `lustre_build_packages`: Optional list. Prerequisite packages required to build Lustre. See `defaults/main.yml`.
 - `lustre_build_dir`: Optional str. Path to build lustre at, default `/tmp/lustre-release`.
 - `lustre_configure_opts`: Optional list. Options to `./configure` command. Default builds client rpms supporting Mellanox OFED, without support for GSS keys.

ansible/roles/lustre/defaults/main.yml

@@ -1,4 +1,5 @@
-lustre_version: '2.15.6' # https://www.lustre.org/lustre-2-15-6-released/
+lustre_repo: https://github.com/stackhpc/lustre-release.git
+lustre_version: '2.15.6/lu-18085' # Fixes https://jira.whamcloud.com/browse/LU-18085
 lustre_lnet_label: tcp
 #lustre_mgs_nid:
 lustre_mounts: []