Merge pull request #60 from oracle-quickstart/2.11.2

What's Changed

- Improved monitoring and alerting
- Lustre Support
- B200 support
- Add Slurm Rest API support
- Fix home on existing FSS
- New Healthchecks
- Bug fixes
- ...
Release notes: https://github.com/oracle-quickstart/oci-hpc/releases/tag/v2.11.2
Readme: https://github.com/oracle-quickstart/oci-hpc/blob/v2.11.2/README.md

Author: arnaudfroidmont

README.md

@@ -2,6 +2,8 @@
 
 [![Deploy to Oracle Cloud](https://oci-resourcemanager-plugin.plugins.oci.oraclecloud.com/latest/deploy-to-oracle-cloud.svg)](https://cloud.oracle.com/resourcemanager/stacks/create?zipUrl=https://github.com/oracle-quickstart/oci-hpc/archive/refs/heads/master.zip)
 
+## Create a dynamic group
+For customer tenancies, check or create dynamic group. In OCI Console, navigate to Identity->Domains->Default domain->Dynamic groups, create a dynamic group e.g. instance_principal with `Any {instance.compartment.id = 'ocid1.compartment.oc1.example-ocid'}`. Though this provides a liberal access for the instance principal, it is recommended to narrow the scope depending on customer's security posture requirements. If you change the name of policy, be sure make the change in the policy examples that follow this section. Currently they all assume that you named the dynamic group as `instance_principal`.
 
 ## Policies to deploy the stack: 
 ```
@@ -24,6 +26,7 @@ Allow dynamic-group instance_principal to manage instance-family in compartment
 Allow dynamic-group instance_principal to use virtual-network-family in compartment compartmentName
 Allow dynamic-group instance_principal to use volumes in compartment compartmentName
 Allow dynamic-group instance_principal to manage dns in compartment compartmentName
+Allow dynamic-group instance_principal to read metrics in compartment compartmentName
 ```
 or:
 

autoscaling/tf_init/controller_update.tf

@@ -40,13 +40,21 @@ resource "local_file" "inventory" {
     home_nfs = var.home_nfs,
     create_fss = var.create_fss,
     home_fss = var.home_fss,
+    mount_target_count = var.mount_target_count,
+    nfs_list_of_mount_target_IPs = var.nfs_list_of_mount_target_IPs,
+    manual_multiple_mount_target = var.manual_multiple_mount_target,
     add_nfs = var.add_nfs,
     slurm_nfs_path = var.slurm_nfs_path,
     rack_aware = var.rack_aware,
     nfs_target_path = var.nfs_target_path,
     nfs_source_IP = var.nfs_source_IP,
     nfs_source_path = var.nfs_source_path,
     nfs_options = var.nfs_options,
+    add_lfs = var.add_lfs,
+    lfs_target_path = var.lfs_target_path,
+    lfs_source_IP = var.lfs_source_IP,
+    lfs_source_path = var.lfs_source_path,
+    lfs_options = var.lfs_options,
     localdisk = var.localdisk,
     log_vol = var.log_vol,
     redundancy = var.redundancy,

autoscaling/tf_init/inventory.tpl

@@ -32,6 +32,9 @@ scratch_nfs = ${scratch_nfs}
 home_nfs = ${home_nfs} 
 create_fss = ${create_fss} 
 home_fss = ${home_fss} 
+mount_target_count = ${mount_target_count}
+nfs_list_of_mount_target_IPs = ${nfs_list_of_mount_target_IPs}
+manual_multiple_mount_target = ${manual_multiple_mount_target}
 cluster_nfs = ${cluster_nfs}
 cluster_nfs_path = ${cluster_nfs_path}
 slurm_nfs_path = ${slurm_nfs_path}
@@ -81,3 +84,8 @@ healthchecks=${healthchecks}
 change_hostname=${change_hostname}
 hostname_convention=${hostname_convention}
 ons_topic_ocid=${ons_topic_ocid}
+add_lfs=${add_lfs}
+lfs_target_path=${lfs_target_path}
+lfs_source_IP=${lfs_source_IP}
+lfs_source_path=${lfs_source_path}
+lfs_options=${lfs_options}

autoscaling/tf_init/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
      oci = {
          source = "oracle/oci"
-         version = "6.9.0"
+         version = "7.1.0"
      }
   }
 }

bin/controller.sh

@@ -171,7 +171,7 @@ fi
 ansible-galaxy collection install ansible.netcommon:=2.5.1 --force > /dev/null
 ansible-galaxy collection install community.general:=4.8.1 --force > /dev/null
 ansible-galaxy collection install ansible.posix --force > /dev/null
-ansible-galaxy collection install community.crypto --force > /dev/null
+ansible-galaxy collection install community.crypto:=2.26.3 --force > /dev/null
 
 threads=$(nproc)
 forks=$(($threads * 8))

conf/variables.tpl

@@ -53,11 +53,10 @@ variable "marketplace_listing" {
 variable "marketplace_version_id" { 
   type = map(string) 
   default = { 
-    "HPC_OL8"       = "Oracle-Linux-8.10-2025.02.28-0-OCA-RHCK-OFED-24.10-1.1.4.0-2025.03.27-0"
-    "GPU_OL8_NV550" = "Oracle-Linux-8.10-2025.02.28-0-OCA-RHCK-OFED-24.10-1.1.4.0-GPU-550-CUDA-12.4-2025.03.27-0"
-    "GPU_OL8_NV560" = "Oracle-Linux-8.10-2025.02.28-0-OCA-RHCK-OFED-24.10-1.1.4.0-GPU-550-CUDA-12.4-2025.03.27-0"
-    "GPU_OL8_NV570" = "Oracle-Linux-8.10-2025.02.28-0-OCA-RHCK-OFED-24.10-1.1.4.0-GPU-550-CUDA-12.4-2025.03.27-0"
-    "GPU_OL8_AMD632" = "Oracle-Linux-8.10-2025.02.28-0-OCA-RHCK-OFED-24.10-1.1.4.0-AMD-ROCM-632-2025.03.28-0"
+    "HPC_OL8"       = "Oracle-Linux-8.10-2025.06.17-0-RHCK-OFED-24.10-1.1.4.0-2025.07.19-0"
+    "GPU_OL8_NV550" = "Oracle-Linux-8.10-2025.06.17-0-RHCK-OFED-24.10-1.1.4.0-GPU-550-CUDA-12.4-2025.07.19-0"
+    "GPU_OL8_NV570" = "Oracle-Linux-8.10-2025.06.17-0-RHCK-OFED-24.10-1.1.4.0-GPU-570-OPEN-CUDA-12.8-2025.07.18-0"
+    "GPU_OL8_AMD632" = "Oracle-Linux-8.10-2025.06.17-0-RHCK-OFED-24.10-1.1.4.0-AMD-ROCM-632-2025.07.20-0"
   }
 }
 
@@ -106,6 +105,9 @@ variable "controller_mount_ip" {default = "${controller_mount_ip}"}
 variable "login_mount_ip" {default = "${login_mount_ip}"}
 variable "home_nfs" { default = ${home_nfs} } 
 variable "home_fss" { default = ${home_fss} } 
+variable "mount_target_count" { default = ${mount_target_count} }
+variable "nfs_list_of_mount_target_IPs" { default = "${nfs_list_of_mount_target_IPs}" }
+variable "manual_multiple_mount_target" { default = ${manual_multiple_mount_target} }
 variable "latency_check" { default = ${latency_check} } 
 variable "create_fss" { default = ${create_fss} } 
 variable "configure" { default = true }
@@ -166,4 +168,24 @@ variable "hostname_convention" {
 }
 variable "ons_topic_ocid" {
   default = "${ons_topic_ocid}"
-}
+}
+variable "add_lfs" {
+  default = "${add_lfs}"
+  type = bool
+}
+variable "lfs_target_path" {
+  default = "${lfs_target_path}"
+  type = string
+}
+variable "lfs_source_IP" {
+  default = "${lfs_source_IP}"
+  type = string
+}
+variable "lfs_source_path" {
+  default = "${lfs_source_path}"
+  type = string
+}
+variable "lfs_options" {
+  default = "${lfs_options}"
+  type = string
+}

controller.tf

@@ -258,6 +258,9 @@ resource "null_resource" "cluster" {
       home_nfs                  = var.home_nfs,
       create_fss                = var.create_fss,
       home_fss                  = var.home_fss,
+      mount_target_count        = var.mount_target_count,
+      nfs_list_of_mount_target_IPs = local.nfs_list_of_mount_target_IPs,
+      manual_multiple_mount_target = var.manual_multiple_mount_target,
       scratch_nfs               = var.use_scratch_nfs && var.node_count > 0,
       cluster_nfs               = var.use_cluster_nfs,
       cluster_nfs_path          = var.cluster_nfs_path,
@@ -267,6 +270,11 @@ resource "null_resource" "cluster" {
       nfs_source_IP             = local.nfs_source_IP,
       nfs_source_path           = var.nfs_source_path,
       nfs_options               = var.nfs_options,
+      add_lfs                   = var.add_lfs,
+      lfs_target_path           = var.lfs_target_path,
+      lfs_source_IP             = local.luster_IP,
+      lfs_source_path           = var.lfs_source_path,
+      lfs_options               = var.lfs_options,
       localdisk                 = var.localdisk,
       log_vol                   = var.log_vol,
       redundancy                = var.redundancy,
@@ -436,11 +444,19 @@ resource "null_resource" "cluster" {
       home_nfs                            = var.home_nfs,
       create_fss                          = var.create_fss,
       home_fss                            = var.home_fss,
+      mount_target_count                  = var.mount_target_count,
+      nfs_list_of_mount_target_IPs        = local.nfs_list_of_mount_target_IPs,
+      manual_multiple_mount_target        = var.manual_multiple_mount_target,
       add_nfs                             = var.add_nfs,
       nfs_target_path                     = var.nfs_target_path,
       nfs_source_IP                       = local.nfs_source_IP,
       nfs_source_path                     = var.nfs_source_path,
       nfs_options                         = var.nfs_options,
+      add_lfs                             = var.add_lfs,
+      lfs_target_path                     = var.lfs_target_path,
+      lfs_source_IP                       = local.luster_IP,
+      lfs_source_path                     = var.lfs_source_path,
+      lfs_options                         = var.lfs_options,
       localdisk                           = var.localdisk,
       log_vol                             = var.log_vol,
       redundancy                          = var.redundancy,
@@ -596,4 +612,4 @@ resource "oci_dns_rrset" "rrset-controller" {
   }
   scope   = "PRIVATE"
   view_id = data.oci_dns_views.dns_views.views[0].id
-}
+}

inventory.tpl

@@ -28,6 +28,9 @@ scratch_nfs = ${scratch_nfs}
 home_nfs = ${home_nfs} 
 create_fss = ${create_fss} 
 home_fss = ${home_fss} 
+mount_target_count = ${mount_target_count}
+nfs_list_of_mount_target_IPs = ${nfs_list_of_mount_target_IPs}
+manual_multiple_mount_target = ${manual_multiple_mount_target}
 cluster_nfs = ${cluster_nfs}
 cluster_nfs_path = ${cluster_nfs_path}
 slurm_nfs_path = ${slurm_nfs_path}
@@ -86,4 +89,8 @@ healthchecks=${healthchecks}
 change_hostname=${change_hostname}
 hostname_convention=${hostname_convention}
 ons_topic_ocid=${ons_topic_ocid}
-
+add_lfs=${add_lfs}
+lfs_target_path=${lfs_target_path}
+lfs_source_IP=${lfs_source_IP}
+lfs_source_path=${lfs_source_path}
+lfs_options=${lfs_options}

locals.tf

@@ -27,7 +27,7 @@ locals {
   subnet_id = var.private_deployment ? var.use_existing_vcn ? var.private_subnet_id : element(concat(oci_core_subnet.private-subnet.*.id, [""]), 1) : var.use_existing_vcn ? var.private_subnet_id : element(concat(oci_core_subnet.private-subnet.*.id, [""]), 0)
 
   nfs_source_IP = var.create_fss ? oci_dns_rrset.fss-dns-round-robin[0].domain : var.nfs_source_IP  
-  nfs_list_of_mount_target_IPs = var.create_fss ? "[\"${join("\",\"",oci_file_storage_mount_target.FSSMountTarget.*.ip_address)}\"]" : var.nfs_source_IP  
+  nfs_list_of_mount_target_IPs = var.create_fss ? join(",",oci_file_storage_mount_target.FSSMountTarget.*.ip_address) : var.nfs_source_IP
 
   // subnet id derived either from created subnet or existing if specified
   // controller_subnet_id = var.use_existing_vcn ? var.public_subnet_id : element(concat(oci_core_subnet.public-subnet.*.id, [""]), 0)
@@ -83,4 +83,7 @@ locals {
   platform_type = local.shape == "BM.GPU4.8" ? "AMD_ROME_BM_GPU" : local.shape == "BM.GPU.B4.8" || local.shape == "BM.GPU.A100-v2.8" ? "AMD_MILAN_BM_GPU" : local.shape == "BM.Standard.E3.128" ? "AMD_ROME_BM" : local.shape == "BM.Standard.E4.128" || local.shape == "BM.DenseIO.E4.128" ? "AMD_MILAN_BM" : "GENERIC_BM"
 
   topic_id = var.alerting ? oci_ons_notification_topic.grafana_alerts[0].id : ""
+
+  // Lustre IP.
+  luster_IP = var.create_lfs ? oci_lustre_file_storage_lustre_file_system.lustre_file_system[0].management_service_address : var.lfs_source_IP
 }

lustre.tf

@@ -0,0 +1,30 @@
+resource "oci_lustre_file_storage_lustre_file_system" "lustre_file_system" {
+  #Required
+  count               = var.create_lfs ? 1 : 0
+  availability_domain = var.ad
+  capacity_in_gbs     = var.lfs_capacity_in_gbs
+  compartment_id      = var.lfs_compartment
+  file_system_name    = var.lfs_source_path # Mount name
+  performance_tier    = var.lfs_perf_tier
+
+  root_squash_configuration {
+
+    # #Optional
+    # client_exceptions = var.lfs_root_squash_configuration_client_exceptions
+    identity_squash = "NONE"
+    # squash_gid = var.lfs_root_squash_configuration_squash_gid
+    # squash_uid = var.lfs_root_squash_configuration_squash_uid
+  }
+  subnet_id = local.subnet_id
+
+  #Optional
+  # cluster_placement_group_id = oci_cluster_placement_groups_cluster_placement_group.test_cluster_placement_group.id
+  # defined_tags = {"Operations.CostCenter"= "42"}
+  display_name            = "${local.cluster_name}-lfs"            	# File system name
+  #file_system_description = var.lfs_file_system_description # File system description Optional
+  freeform_tags = {
+    "lfs_cluster_name"   = local.cluster_name
+  }
+  # kms_key_id = oci_kms_key.test_key.id
+  # nsg_ids = var.lustre_file_system_nsg_ids
+}