Add storage pool blueprints and integration tests for VM, GKE, and Slurm

examples/README.md

@@ -31,7 +31,7 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /"
   * [batch-mpi.yaml](#batch-mpiyaml-) ![core-badge]
   * [pfs-managed-lustre-vm.yaml](#pfs-managed-lustre-vmyaml-) ![core-badge]
   * [pfs-managed-lustre-slurm.yaml](#pfs-managed-lustre-slurmyaml-) ![core-badge]
-  * [rapid-storage-slurm.yaml](#rapid-storage-slurmyaml-) ![core-badge]
+  * [storage-slurm.yaml](#storage-slurmyaml-) ![core-badge]
   * [gke-managed-lustre.yaml](#gke-managed-lustreyaml-) ![core-badge]
   * [cae-slurm.yaml](#cae-slurmyaml-) ![core-badge]
   * [hpc-build-slurm-image.yaml](#hpc-build-slurm-imageyaml--) ![community-badge] ![experimental-badge]
@@ -47,7 +47,8 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /"
   * [af3-slurm.yaml](#af3-slurmyaml--) ![core-badge] ![experimental-badge]
   * [hpc-gke.yaml](#hpc-gkeyaml-) ![core-badge]
   * [ml-gke](#ml-gkeyaml-) ![core-badge]
-  * [storage-gke](#storage-gkeyaml-) ![core-badge]
+  * [storage-gke.yaml](#storage-gkeyaml-) ![core-badge]
+  * [storage-vm.yaml](#storage-vmyaml-) ![core-badge]
   * [gke-managed-hyperdisk.yaml](#gke-managed-hyperdiskyaml--) ![core-badge] ![experimental-badge]
   * [gke-a3-ultragpu.yaml](#gke-a3-ultragpuyaml-) ![core-badge]
   * [gke-a3-megagpu](#gke-a3-megagpuyaml-) ![core-badge]
@@ -697,7 +698,7 @@ To destroy the cluster,Run below command:
 ```
 
 [pfs-managed-lustre-slurm.yaml]: ./pfs-managed-lustre-slurm.yaml
-### [rapid-storage-slurm.yaml] ![core-badge]
+### [storage-slurm.yaml] ![core-badge]
 
 This blueprint showcases the integration of several storage solutions:
 
@@ -707,7 +708,10 @@ This blueprint showcases the integration of several storage solutions:
     * Note: A maximum of one cache per zone can be created for each bucket. For example, a bucket in `us-east1` can have caches in `us-east1-b` and `us-east1-c`.
     * Refer to [Create a Cache](https://docs.cloud.google.com/storage/docs/anywhere-cache#create_a_cache) for more parameter details.
 
-[rapid-storage-slurm.yaml]: ./rapid-storage-slurm.yaml
+* **Hyperdisk Storage Pools:**
+  * The `schedmd-slurm-gcp-v6-controller`, `schedmd-slurm-gcp-v6-login`, and `schedmd-slurm-gcp-v6-nodeset` modules attach persistent disks directly from pre-provisioned `hyperdisk-balanced` and `hyperdisk-throughput` Storage Pools, allowing you to share IOPS and throughput capacity across the cluster.
+
+[storage-slurm.yaml]: ./storage-slurm.yaml
 
 ### [gke-managed-lustre.yaml] ![core-badge]
 
@@ -1296,6 +1300,12 @@ credentials for the created cluster_ and _submit a job calling `nvidia_smi`_.
 
 [ml-gke.yaml]: ../examples/ml-gke.yaml
 
+### [storage-vm.yaml] ![core-badge]
+
+Creates a standalone VM instance and securely attaches persistent disks that are provisioned directly into specified `hyperdisk-balanced` and `hyperdisk-throughput` Storage Pools. This allows the VM to share IOPS and throughput capacity from the pre-provisioned pools.
+
+[storage-vm.yaml]: ./storage-vm.yaml
+
 ### [storage-gke.yaml] ![core-badge]
 
 This blueprint showcases the integration of several storage solutions:
@@ -1319,6 +1329,9 @@ This blueprint showcases the integration of several storage solutions:
     * **Balanced Persistent Disk (`pd-balanced`) ephemeral volume**: Similar to `pd-ssd`, a Persistent Disk is created and cleaned up with the job.
   * When using `pd-ssd` or `pd-balanced`, a persistent disk is automatically created upon job submission and cleaned up when the job is deleted.
 
+* **Hyperdisk Storage Pools:**
+  * The `gke-persistent-volume` module dynamically provisions Persistent Volumes (PVs) that draw directly from `hyperdisk-balanced` and `hyperdisk-throughput` Storage Pools, allowing workloads to share aggregate disk performance.
+
 > [!Note]
 > The Kubernetes API server will only allow requests from authorized networks.
 > The `gke-persistent-volume` module needs access to the Kubernetes API server

examples/storage-gke.yaml

@@ -23,6 +23,8 @@ vars:
   # The following line must be updated for this example to work.
   authorized_cidr: <your-ip-address>/32
   gcp_public_cidrs_access_enabled: false
+  hyperdisk_balanced_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-balanced"
+  hyperdisk_throughput_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-throughput"
 
 deployment_groups:
 - group: primary
@@ -210,3 +212,35 @@ deployment_groups:
       - --iodepth_batch_submit=64
       - --iodepth_batch_complete_max=64
     outputs: [instructions]
+
+  ### Storage Pools ###
+
+  - id: gke-pool-hp
+    source: modules/compute/gke-node-pool
+    use: [gke_cluster, node_pool_service_account]
+    settings:
+      name: hp-pool
+      zones: [$(vars.zone)]
+      machine_type: c3d-standard-4
+      disk_type: hyperdisk-balanced
+      disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool)
+
+  - id: gke-storage-hp
+    source: modules/file-system/gke-storage
+    use: [gke_cluster]
+    settings:
+      storage_type: hyperdisk-balanced
+      access_mode: ReadWriteOnce
+      capacity_gb: 20
+      sc_reclaim_policy: Delete
+      disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool)
+
+  - id: gke-storage-hp-thr
+    source: modules/file-system/gke-storage
+    use: [gke_cluster]
+    settings:
+      storage_type: hyperdisk-throughput
+      access_mode: ReadWriteOnce
+      capacity_gb: 5000
+      sc_reclaim_policy: Delete
+      disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool)

examples/rapid-storage-slurm.yaml → examples/storage-slurm.yaml

@@ -13,14 +13,16 @@
 # limitations under the License.
 
 ---
-blueprint_name: rapid-storage-slurm
+blueprint_name: storage-slurm
 
 vars:
   project_id:  # Set GCP Project ID Here
-  deployment_name: zonal-bucket-ac-slurm
+  deployment_name: storage-slurm
   region: us-central1
   zone: us-central1-b
   gcs_bucket_local_mount: /data
+  hyperdisk_balanced_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-balanced"
+  hyperdisk_throughput_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-throughput"
 
 deployment_groups:
 - group: primary
@@ -52,7 +54,18 @@ deployment_groups:
     use: [network]
     settings:
       node_count_dynamic_max: 2
-      machine_type: n2-standard-4
+      machine_type: c3d-standard-4
+      disk_type: hyperdisk-balanced
+      disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool)
+      additional_disks:
+      - disk_name: nodeset-data
+        device_name: nodeset-data
+        disk_type: hyperdisk-throughput
+        disk_size_gb: 2048
+        disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool)
+        auto_delete: true
+        boot: false
+        disk_labels: {}
       allow_automatic_updates: false
       metadata:
         gcs-bucket-name: $(zonal-gcs-bucket.gcs_bucket_name)
@@ -71,7 +84,18 @@ deployment_groups:
     use:
     - network
     settings:
-      machine_type: n2-standard-4
+      machine_type: c3d-standard-4
+      disk_type: hyperdisk-balanced
+      disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool)
+      additional_disks:
+      - disk_name: login-data
+        device_name: login-data
+        disk_type: hyperdisk-throughput
+        disk_size_gb: 2048
+        disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool)
+        auto_delete: true
+        boot: false
+        disk_labels: {}
       enable_login_public_ips: true
       metadata:
         gcs-bucket-name: $(zonal-gcs-bucket.gcs_bucket_name)
@@ -85,5 +109,16 @@ deployment_groups:
     - zonal-gcs-bucket
     - slurm_login
     settings:
-      machine_type: n2-standard-4
+      machine_type: c3d-standard-4
+      disk_type: hyperdisk-balanced
+      disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool)
+      additional_disks:
+      - disk_name: controller-data
+        device_name: controller-data
+        disk_type: hyperdisk-throughput
+        disk_size_gb: 2048
+        disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool)
+        auto_delete: true
+        boot: false
+        disk_labels: {}
       enable_controller_public_ips: true

examples/storage-vm.yaml

@@ -0,0 +1,48 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+blueprint_name: storage-vm
+vars:
+  project_id:  ## Set GCP Project ID Here ##
+  deployment_name: vm-storage
+  region: us-central1
+  zone: us-central1-b
+  test_name: vm-storage
+  network_name: default-net
+  hyperdisk_balanced_storage_pool: projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-balanced
+  hyperdisk_throughput_storage_pool: projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-throughput
+
+deployment_groups:
+- group: primary
+  modules:
+  - id: network
+    source: modules/network/vpc
+    settings:
+      network_name: $(vars.network_name)
+      network_description: "Test network for $(vars.test_name) in $(vars.region)"
+  - id: vm
+    source: modules/compute/vm-instance
+    use:
+    - network
+    settings:
+      machine_type: c3d-standard-4
+      instance_count: 1
+      disk_type: hyperdisk-balanced
+      disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool)
+      additional_persistent_disks:
+        count: 1
+        type: hyperdisk-throughput
+        size: 2048
+        storage_pool: $(vars.hyperdisk_throughput_storage_pool)

tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-storage-pool.yml

@@ -0,0 +1,163 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+- name: Verify GKE Node Pool Storage Pool
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    # The cluster name defaults to deployment_name in our GKE module
+    gcloud container node-pools describe hp-pool --cluster {{ deployment_name }} --region {{ region }} --project {{ project }} --format="value(config.storagePools)"
+  register: gke_node_pool
+  changed_when: false
+  # Ignore errors in case the cluster is zonal instead of regional, and try zonal
+  ignore_errors: true
+
+- name: Verify GKE Node Pool Storage Pool (Zonal Fallback)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    gcloud container node-pools describe hp-pool --cluster {{ deployment_name }} --zone {{ zone }} --project {{ project }} --format="value(config.storagePools)"
+  register: gke_node_pool_zonal
+  changed_when: false
+  when: gke_node_pool.failed
+
+- name: Set GKE Node Pool Result
+  delegate_to: localhost
+  ansible.builtin.set_fact:
+    gke_node_pool_output: "{{ gke_node_pool.stdout if not gke_node_pool.failed else gke_node_pool_zonal.stdout }}"
+
+- name: Assert GKE Node Pool uses Balanced Storage Pool
+  delegate_to: localhost
+  ansible.builtin.assert:
+    that:
+    - hyperdisk_balanced_pool in gke_node_pool_output
+
+
+- name: Authenticate to GKE Cluster (Regional)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    gcloud container clusters get-credentials {{ deployment_name }} --region {{ region }} --project {{ project }}
+  register: gke_auth
+  changed_when: false
+  ignore_errors: true
+
+- name: Authenticate to GKE Cluster (Zonal Fallback)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    gcloud container clusters get-credentials {{ deployment_name }} --zone {{ zone }} --project {{ project }}
+  changed_when: false
+  when: gke_auth.failed
+
+- name: Verify StorageClass contains storage-pools (Balanced)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl get sc hyperdisk-balanced-sc -o yaml
+  register: sc_balanced
+  changed_when: false
+
+- name: Assert StorageClass contains storage-pools (Balanced)
+  delegate_to: localhost
+  ansible.builtin.assert:
+    that:
+    - "'storage-pools: projects/{{ project }}' in sc_balanced.stdout"
+    - hyperdisk_balanced_pool in sc_balanced.stdout
+
+- name: Verify StorageClass contains storage-pools (Throughput)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl get sc hyperdisk-throughput-sc -o yaml
+  register: sc_throughput
+  changed_when: false
+
+- name: Assert StorageClass contains storage-pools (Throughput)
+  delegate_to: localhost
+  ansible.builtin.assert:
+    that:
+    - "'storage-pools: projects/{{ project }}' in sc_throughput.stdout"
+    - hyperdisk_throughput_pool in sc_throughput.stdout
+
+- name: Deploy Test Pod to trigger PVC binding
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    cat <<'POD' | kubectl apply -f -
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: test-hd-pod
+    spec:
+      containers:
+      - name: test
+        image: busybox
+        command: ["sleep", "3600"]
+        volumeMounts:
+        - mountPath: "/data-balanced"
+          name: vol-balanced
+        - mountPath: "/data-throughput"
+          name: vol-throughput
+      nodeSelector:
+        cloud.google.com/gke-nodepool: hp-pool
+      volumes:
+      - name: vol-balanced
+        persistentVolumeClaim:
+          claimName: hyperdisk-balanced-pvc-0
+      - name: vol-throughput
+        persistentVolumeClaim:
+          claimName: hyperdisk-throughput-pvc-0
+    POD
+  changed_when: true
+
+- name: Wait for Test Pod to be Running (Binding PVCs)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl wait --for=condition=Ready pod/test-hd-pod --timeout=600s
+  changed_when: false
+
+- name: Get Persistent Volume names (Balanced)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl get pvc hyperdisk-balanced-pvc-0 -o jsonpath='{.spec.volumeName}'
+  register: pv_balanced_name
+  changed_when: false
+
+- name: Get Persistent Volume names (Throughput)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    kubectl get pvc hyperdisk-throughput-pvc-0 -o jsonpath='{.spec.volumeName}'
+  register: pv_throughput_name
+  changed_when: false
+
+- name: Verify GCP Persistent Disk Placement (Balanced)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    gcloud compute disks describe {{ pv_balanced_name.stdout }} --zone {{ zone }} --project {{ project }} --format="value(storagePool)"
+  register: disk_pool_balanced
+  changed_when: false
+
+- name: Assert Balanced Disk is in Storage Pool
+  delegate_to: localhost
+  ansible.builtin.assert:
+    that:
+    - hyperdisk_balanced_pool in disk_pool_balanced.stdout
+
+- name: Verify GCP Persistent Disk Placement (Throughput)
+  delegate_to: localhost
+  ansible.builtin.shell: |
+    gcloud compute disks describe {{ pv_throughput_name.stdout }} --zone {{ zone }} --project {{ project }} --format="value(storagePool)"
+  register: disk_pool_throughput
+  changed_when: false
+
+- name: Assert Throughput Disk is in Storage Pool
+  delegate_to: localhost
+  ansible.builtin.assert:
+    that:
+    - hyperdisk_throughput_pool in disk_pool_throughput.stdout