diff --git a/examples/README.md b/examples/README.md index 5de8b89f0a..86f135ef0c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -31,7 +31,7 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /" * [batch-mpi.yaml](#batch-mpiyaml-) ![core-badge] * [pfs-managed-lustre-vm.yaml](#pfs-managed-lustre-vmyaml-) ![core-badge] * [pfs-managed-lustre-slurm.yaml](#pfs-managed-lustre-slurmyaml-) ![core-badge] - * [rapid-storage-slurm.yaml](#rapid-storage-slurmyaml-) ![core-badge] + * [storage-slurm.yaml](#storage-slurmyaml-) ![core-badge] * [gke-managed-lustre.yaml](#gke-managed-lustreyaml-) ![core-badge] * [cae-slurm.yaml](#cae-slurmyaml-) ![core-badge] * [hpc-build-slurm-image.yaml](#hpc-build-slurm-imageyaml--) ![community-badge] ![experimental-badge] @@ -47,7 +47,8 @@ md_toc github examples/README.md | sed -e "s/\s-\s/ * /" * [af3-slurm.yaml](#af3-slurmyaml--) ![core-badge] ![experimental-badge] * [hpc-gke.yaml](#hpc-gkeyaml-) ![core-badge] * [ml-gke](#ml-gkeyaml-) ![core-badge] - * [storage-gke](#storage-gkeyaml-) ![core-badge] + * [storage-gke.yaml](#storage-gkeyaml-) ![core-badge] + * [storage-vm.yaml](#storage-vmyaml-) ![core-badge] * [gke-managed-hyperdisk.yaml](#gke-managed-hyperdiskyaml--) ![core-badge] ![experimental-badge] * [gke-a3-ultragpu.yaml](#gke-a3-ultragpuyaml-) ![core-badge] * [gke-a3-megagpu](#gke-a3-megagpuyaml-) ![core-badge] @@ -697,7 +698,7 @@ To destroy the cluster,Run below command: ``` [pfs-managed-lustre-slurm.yaml]: ./pfs-managed-lustre-slurm.yaml -### [rapid-storage-slurm.yaml] ![core-badge] +### [storage-slurm.yaml] ![core-badge] This blueprint showcases the integration of several storage solutions: @@ -707,7 +708,10 @@ This blueprint showcases the integration of several storage solutions: * Note: A maximum of one cache per zone can be created for each bucket. For example, a bucket in `us-east1` can have caches in `us-east1-b` and `us-east1-c`. * Refer to [Create a Cache](https://docs.cloud.google.com/storage/docs/anywhere-cache#create_a_cache) for more parameter details. -[rapid-storage-slurm.yaml]: ./rapid-storage-slurm.yaml +* **Hyperdisk Storage Pools:** + * The `schedmd-slurm-gcp-v6-controller`, `schedmd-slurm-gcp-v6-login`, and `schedmd-slurm-gcp-v6-nodeset` modules attach persistent disks directly from pre-provisioned `hyperdisk-balanced` and `hyperdisk-throughput` Storage Pools, allowing you to share IOPS and throughput capacity across the cluster. + +[storage-slurm.yaml]: ./storage-slurm.yaml ### [gke-managed-lustre.yaml] ![core-badge] @@ -1296,6 +1300,12 @@ credentials for the created cluster_ and _submit a job calling `nvidia_smi`_. [ml-gke.yaml]: ../examples/ml-gke.yaml +### [storage-vm.yaml] ![core-badge] + +Creates a standalone VM instance and securely attaches persistent disks that are provisioned directly into specified `hyperdisk-balanced` and `hyperdisk-throughput` Storage Pools. This allows the VM to share IOPS and throughput capacity from the pre-provisioned pools. + +[storage-vm.yaml]: ./storage-vm.yaml + ### [storage-gke.yaml] ![core-badge] This blueprint showcases the integration of several storage solutions: @@ -1319,6 +1329,9 @@ This blueprint showcases the integration of several storage solutions: * **Balanced Persistent Disk (`pd-balanced`) ephemeral volume**: Similar to `pd-ssd`, a Persistent Disk is created and cleaned up with the job. * When using `pd-ssd` or `pd-balanced`, a persistent disk is automatically created upon job submission and cleaned up when the job is deleted. +* **Hyperdisk Storage Pools:** + * The `gke-persistent-volume` module dynamically provisions Persistent Volumes (PVs) that draw directly from `hyperdisk-balanced` and `hyperdisk-throughput` Storage Pools, allowing workloads to share aggregate disk performance. + > [!Note] > The Kubernetes API server will only allow requests from authorized networks. > The `gke-persistent-volume` module needs access to the Kubernetes API server diff --git a/examples/storage-gke.yaml b/examples/storage-gke.yaml index 575af2ed1d..6a0639bf8a 100644 --- a/examples/storage-gke.yaml +++ b/examples/storage-gke.yaml @@ -23,6 +23,8 @@ vars: # The following line must be updated for this example to work. authorized_cidr: /32 gcp_public_cidrs_access_enabled: false + hyperdisk_balanced_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-balanced" + hyperdisk_throughput_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-throughput" deployment_groups: - group: primary @@ -210,3 +212,35 @@ deployment_groups: - --iodepth_batch_submit=64 - --iodepth_batch_complete_max=64 outputs: [instructions] + + ### Storage Pools ### + + - id: gke-pool-hp + source: modules/compute/gke-node-pool + use: [gke_cluster, node_pool_service_account] + settings: + name: hp-pool + zones: [$(vars.zone)] + machine_type: c3d-standard-4 + disk_type: hyperdisk-balanced + disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool) + + - id: gke-storage-hp + source: modules/file-system/gke-storage + use: [gke_cluster] + settings: + storage_type: hyperdisk-balanced + access_mode: ReadWriteOnce + capacity_gb: 20 + sc_reclaim_policy: Delete + disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool) + + - id: gke-storage-hp-thr + source: modules/file-system/gke-storage + use: [gke_cluster] + settings: + storage_type: hyperdisk-throughput + access_mode: ReadWriteOnce + capacity_gb: 5000 + sc_reclaim_policy: Delete + disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool) diff --git a/examples/rapid-storage-slurm.yaml b/examples/storage-slurm.yaml similarity index 61% rename from examples/rapid-storage-slurm.yaml rename to examples/storage-slurm.yaml index dec96754ee..ba6f4b5b9b 100644 --- a/examples/rapid-storage-slurm.yaml +++ b/examples/storage-slurm.yaml @@ -13,14 +13,16 @@ # limitations under the License. --- -blueprint_name: rapid-storage-slurm +blueprint_name: storage-slurm vars: project_id: # Set GCP Project ID Here - deployment_name: zonal-bucket-ac-slurm + deployment_name: storage-slurm region: us-central1 zone: us-central1-b gcs_bucket_local_mount: /data + hyperdisk_balanced_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-balanced" + hyperdisk_throughput_storage_pool: "projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-throughput" deployment_groups: - group: primary @@ -52,7 +54,18 @@ deployment_groups: use: [network] settings: node_count_dynamic_max: 2 - machine_type: n2-standard-4 + machine_type: c3d-standard-4 + disk_type: hyperdisk-balanced + disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool) + additional_disks: + - disk_name: nodeset-data + device_name: nodeset-data + disk_type: hyperdisk-throughput + disk_size_gb: 2048 + disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool) + auto_delete: true + boot: false + disk_labels: {} allow_automatic_updates: false metadata: gcs-bucket-name: $(zonal-gcs-bucket.gcs_bucket_name) @@ -71,7 +84,18 @@ deployment_groups: use: - network settings: - machine_type: n2-standard-4 + machine_type: c3d-standard-4 + disk_type: hyperdisk-balanced + disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool) + additional_disks: + - disk_name: login-data + device_name: login-data + disk_type: hyperdisk-throughput + disk_size_gb: 2048 + disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool) + auto_delete: true + boot: false + disk_labels: {} enable_login_public_ips: true metadata: gcs-bucket-name: $(zonal-gcs-bucket.gcs_bucket_name) @@ -85,5 +109,16 @@ deployment_groups: - zonal-gcs-bucket - slurm_login settings: - machine_type: n2-standard-4 + machine_type: c3d-standard-4 + disk_type: hyperdisk-balanced + disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool) + additional_disks: + - disk_name: controller-data + device_name: controller-data + disk_type: hyperdisk-throughput + disk_size_gb: 2048 + disk_storage_pool: $(vars.hyperdisk_throughput_storage_pool) + auto_delete: true + boot: false + disk_labels: {} enable_controller_public_ips: true diff --git a/examples/storage-vm.yaml b/examples/storage-vm.yaml new file mode 100644 index 0000000000..c5d8e99b6e --- /dev/null +++ b/examples/storage-vm.yaml @@ -0,0 +1,48 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +blueprint_name: storage-vm +vars: + project_id: ## Set GCP Project ID Here ## + deployment_name: vm-storage + region: us-central1 + zone: us-central1-b + test_name: vm-storage + network_name: default-net + hyperdisk_balanced_storage_pool: projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-balanced + hyperdisk_throughput_storage_pool: projects/$(vars.project_id)/zones/$(vars.zone)/storagePools/your-pool-throughput + +deployment_groups: +- group: primary + modules: + - id: network + source: modules/network/vpc + settings: + network_name: $(vars.network_name) + network_description: "Test network for $(vars.test_name) in $(vars.region)" + - id: vm + source: modules/compute/vm-instance + use: + - network + settings: + machine_type: c3d-standard-4 + instance_count: 1 + disk_type: hyperdisk-balanced + disk_storage_pool: $(vars.hyperdisk_balanced_storage_pool) + additional_persistent_disks: + count: 1 + type: hyperdisk-throughput + size: 2048 + storage_pool: $(vars.hyperdisk_throughput_storage_pool) diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-storage-pool.yml b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-storage-pool.yml new file mode 100644 index 0000000000..ab556743fe --- /dev/null +++ b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-gke-storage-pool.yml @@ -0,0 +1,163 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify GKE Node Pool Storage Pool + delegate_to: localhost + ansible.builtin.shell: | + # The cluster name defaults to deployment_name in our GKE module + gcloud container node-pools describe hp-pool --cluster {{ deployment_name }} --region {{ region }} --project {{ project }} --format="value(config.storagePools)" + register: gke_node_pool + changed_when: false + # Ignore errors in case the cluster is zonal instead of regional, and try zonal + ignore_errors: true + +- name: Verify GKE Node Pool Storage Pool (Zonal Fallback) + delegate_to: localhost + ansible.builtin.shell: | + gcloud container node-pools describe hp-pool --cluster {{ deployment_name }} --zone {{ zone }} --project {{ project }} --format="value(config.storagePools)" + register: gke_node_pool_zonal + changed_when: false + when: gke_node_pool.failed + +- name: Set GKE Node Pool Result + delegate_to: localhost + ansible.builtin.set_fact: + gke_node_pool_output: "{{ gke_node_pool.stdout if not gke_node_pool.failed else gke_node_pool_zonal.stdout }}" + +- name: Assert GKE Node Pool uses Balanced Storage Pool + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_balanced_pool in gke_node_pool_output + + +- name: Authenticate to GKE Cluster (Regional) + delegate_to: localhost + ansible.builtin.shell: | + gcloud container clusters get-credentials {{ deployment_name }} --region {{ region }} --project {{ project }} + register: gke_auth + changed_when: false + ignore_errors: true + +- name: Authenticate to GKE Cluster (Zonal Fallback) + delegate_to: localhost + ansible.builtin.shell: | + gcloud container clusters get-credentials {{ deployment_name }} --zone {{ zone }} --project {{ project }} + changed_when: false + when: gke_auth.failed + +- name: Verify StorageClass contains storage-pools (Balanced) + delegate_to: localhost + ansible.builtin.shell: | + kubectl get sc hyperdisk-balanced-sc -o yaml + register: sc_balanced + changed_when: false + +- name: Assert StorageClass contains storage-pools (Balanced) + delegate_to: localhost + ansible.builtin.assert: + that: + - "'storage-pools: projects/{{ project }}' in sc_balanced.stdout" + - hyperdisk_balanced_pool in sc_balanced.stdout + +- name: Verify StorageClass contains storage-pools (Throughput) + delegate_to: localhost + ansible.builtin.shell: | + kubectl get sc hyperdisk-throughput-sc -o yaml + register: sc_throughput + changed_when: false + +- name: Assert StorageClass contains storage-pools (Throughput) + delegate_to: localhost + ansible.builtin.assert: + that: + - "'storage-pools: projects/{{ project }}' in sc_throughput.stdout" + - hyperdisk_throughput_pool in sc_throughput.stdout + +- name: Deploy Test Pod to trigger PVC binding + delegate_to: localhost + ansible.builtin.shell: | + cat <<'POD' | kubectl apply -f - + apiVersion: v1 + kind: Pod + metadata: + name: test-hd-pod + spec: + containers: + - name: test + image: busybox + command: ["sleep", "3600"] + volumeMounts: + - mountPath: "/data-balanced" + name: vol-balanced + - mountPath: "/data-throughput" + name: vol-throughput + nodeSelector: + cloud.google.com/gke-nodepool: hp-pool + volumes: + - name: vol-balanced + persistentVolumeClaim: + claimName: hyperdisk-balanced-pvc-0 + - name: vol-throughput + persistentVolumeClaim: + claimName: hyperdisk-throughput-pvc-0 + POD + changed_when: true + +- name: Wait for Test Pod to be Running (Binding PVCs) + delegate_to: localhost + ansible.builtin.shell: | + kubectl wait --for=condition=Ready pod/test-hd-pod --timeout=600s + changed_when: false + +- name: Get Persistent Volume names (Balanced) + delegate_to: localhost + ansible.builtin.shell: | + kubectl get pvc hyperdisk-balanced-pvc-0 -o jsonpath='{.spec.volumeName}' + register: pv_balanced_name + changed_when: false + +- name: Get Persistent Volume names (Throughput) + delegate_to: localhost + ansible.builtin.shell: | + kubectl get pvc hyperdisk-throughput-pvc-0 -o jsonpath='{.spec.volumeName}' + register: pv_throughput_name + changed_when: false + +- name: Verify GCP Persistent Disk Placement (Balanced) + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute disks describe {{ pv_balanced_name.stdout }} --zone {{ zone }} --project {{ project }} --format="value(storagePool)" + register: disk_pool_balanced + changed_when: false + +- name: Assert Balanced Disk is in Storage Pool + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_balanced_pool in disk_pool_balanced.stdout + +- name: Verify GCP Persistent Disk Placement (Throughput) + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute disks describe {{ pv_throughput_name.stdout }} --zone {{ zone }} --project {{ project }} --format="value(storagePool)" + register: disk_pool_throughput + changed_when: false + +- name: Assert Throughput Disk is in Storage Pool + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_throughput_pool in disk_pool_throughput.stdout diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-slurm-rapid-storage.yml b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-slurm-storage.yml similarity index 67% rename from tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-slurm-rapid-storage.yml rename to tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-slurm-storage.yml index c797539308..4818f6aa61 100644 --- a/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-slurm-rapid-storage.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-slurm-storage.yml @@ -101,3 +101,45 @@ - fio_results.jobs[0].read.iops_mean >= 300 fail_msg: "FIO read performance thresholds not met. Bandwidth: {{ fio_results.jobs[0].read.bw_mean }} KB/s (expected >= 1000 KB/s), IOPS: {{ fio_results.jobs[0].read.iops_mean }} (expected >= 300)." success_msg: "FIO read performance thresholds met. Bandwidth: {{ fio_results.jobs[0].read.bw_mean }} KB/s, IOPS: {{ fio_results.jobs[0].read.iops_mean }}." + +- name: Verify Controller Instance Template Storage Pools + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute instance-templates list --filter="name~{{ slurm_cluster_name }}-controller" --project={{ project }} --format="value(properties.disks[0].initializeParams.storagePool, properties.disks[1].initializeParams.storagePool)" + register: controller_template_pools + changed_when: false + +- name: Assert Controller Template uses correct Storage Pools + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_balanced_pool in controller_template_pools.stdout + - hyperdisk_throughput_pool in controller_template_pools.stdout + +- name: Verify Login Nodes Instance Template Storage Pools + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute instance-templates list --filter="name~{{ slurm_cluster_name }}-login" --project={{ project }} --format="value(properties.disks[0].initializeParams.storagePool, properties.disks[1].initializeParams.storagePool)" + register: login_template_pools + changed_when: false + +- name: Assert Login Template uses correct Storage Pools + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_balanced_pool in login_template_pools.stdout + - hyperdisk_throughput_pool in login_template_pools.stdout + +- name: Verify Nodeset Instance Template Storage Pools + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute instance-templates list --filter="name~{{ slurm_cluster_name }}-compute-slurmnodeset" --project={{ project }} --format="value(properties.disks[0].initializeParams.storagePool, properties.disks[1].initializeParams.storagePool)" + register: nodeset_template_pools + changed_when: false + +- name: Assert Nodeset Template uses correct Storage Pools + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_balanced_pool in nodeset_template_pools.stdout + - hyperdisk_throughput_pool in nodeset_template_pools.stdout diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-vm-storage.yml b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-vm-storage.yml new file mode 100644 index 0000000000..44876c368f --- /dev/null +++ b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-vm-storage.yml @@ -0,0 +1,40 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Verify VM Boot Disk Storage Pool + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute disks describe {{ deployment_name }}-0 --project={{ project }} --zone={{ zone }} --format="value(storagePool)" + register: vm_boot_pool + changed_when: false + +- name: Assert VM Boot Disk is in Balanced Pool + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_balanced_pool in vm_boot_pool.stdout + +- name: Verify VM Additional Disk Storage Pool + delegate_to: localhost + ansible.builtin.shell: | + gcloud compute disks describe {{ deployment_name }}-disk-0 --project={{ project }} --zone={{ zone }} --format="value(storagePool)" + register: vm_data_pool + changed_when: false + +- name: Assert VM Additional Disk is in Throughput Pool + delegate_to: localhost + ansible.builtin.assert: + that: + - hyperdisk_throughput_pool in vm_data_pool.stdout diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-zonal-bucket.yml b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-zonal-bucket.yml index c40f88dea1..2a0363129e 100644 --- a/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-zonal-bucket.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/test-validation/test-zonal-bucket.yml @@ -54,7 +54,7 @@ delegate_to: localhost - name: Wait for FIO Pod to complete - ansible.builtin.command: kubectl wait --for=condition=Succeeded pod/fio --namespace default --timeout=900s + ansible.builtin.command: kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/fio --namespace default --timeout=900s delegate_to: localhost ignore_errors: true # Even if we timeout, continue to next steps diff --git a/tools/cloud-build/daily-tests/builds/gke-storage.yaml b/tools/cloud-build/daily-tests/builds/gke-storage.yaml index 1842941026..521f333d35 100644 --- a/tools/cloud-build/daily-tests/builds/gke-storage.yaml +++ b/tools/cloud-build/daily-tests/builds/gke-storage.yaml @@ -23,6 +23,7 @@ tags: - m.gke-job-template - m.gke-node-pool - m.gke-persistent-volume +- m.gke-storage - m.vpc - gke @@ -180,11 +181,36 @@ steps: echo ' machine_type: e2-standard-2' >> \$$SG_EXAMPLE echo ' zone: us-central1-b' >> \$$SG_EXAMPLE + echo "Checking if required storage pools exist..." + if ! gcloud compute storage-pools describe test-pool-balanced --project=$PROJECT_ID --zone=us-central1-b >/dev/null 2>&1; then + echo "ERROR: Storage pool 'test-pool-balanced' does not exist in us-central1-b." + echo "Please create it by running the following command:" + echo "gcloud compute storage-pools create test-pool-balanced" + echo " --project=$PROJECT_ID" + echo " --zone=us-central1-b" + echo " --provisioned-capacity=10240GB" + echo " --provisioned-iops=10000" + echo " --provisioned-throughput=1024" + echo " --storage-pool-type=hyperdisk-balanced" + exit 1 + fi + if ! gcloud compute storage-pools describe test-pool-throughput --project=$PROJECT_ID --zone=us-central1-b >/dev/null 2>&1; then + echo "ERROR: Storage pool 'test-pool-throughput' does not exist in us-central1-b." + echo "Please create it by running the following command:" + echo "gcloud compute storage-pools create test-pool-throughput" + echo " --project=$PROJECT_ID" + echo " --zone=us-central1-b" + echo " --provisioned-capacity=28672GB" + echo " --provisioned-throughput=500" + echo " --storage-pool-type=hyperdisk-throughput" + exit 1 + fi + bash tools/add_ttl_label.sh "\$$SG_EXAMPLE" python3 tools/fix_vpc_name.py \$$SG_EXAMPLE "${_TEST_PREFIX}" ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \ - --extra-vars="test_prefix=${_TEST_PREFIX}" --extra-vars="use_fixed_vpc=true" \ + --extra-vars="hyperdisk_balanced_pool=test-pool-balanced hyperdisk_throughput_pool=test-pool-throughput" --extra-vars="test_prefix=${_TEST_PREFIX}" --extra-vars="use_fixed_vpc=true" \ --user="$$OSLOGIN_USER" --extra-vars="project=$PROJECT_ID build=$$BUILD_ID_SHORT full_build_id=$BUILD_ID" \ --extra-vars="@tools/cloud-build/daily-tests/tests/gke-storage.yml" \ --extra-vars="triage_gcs_bucket_override=$$TRIAGE_GCS_BUCKET" \ diff --git a/tools/cloud-build/daily-tests/builds/slurm-rapid-storage.yaml b/tools/cloud-build/daily-tests/builds/slurm-storage.yaml similarity index 80% rename from tools/cloud-build/daily-tests/builds/slurm-rapid-storage.yaml rename to tools/cloud-build/daily-tests/builds/slurm-storage.yaml index 05ecd476e1..91c99e8b69 100644 --- a/tools/cloud-build/daily-tests/builds/slurm-rapid-storage.yaml +++ b/tools/cloud-build/daily-tests/builds/slurm-storage.yaml @@ -71,7 +71,7 @@ steps: apiVersion: batch/v1 kind: Job metadata: - name: slurm-rapid-storage-$$BUILD_ID_SHORT + name: slurm-storage-$$BUILD_ID_SHORT namespace: default labels: kueue.x-k8s.io/queue-name: local-queue-test-locks @@ -117,7 +117,7 @@ steps: args: - | set -x -e - echo "\"test_file_path\": tools/cloud-build/daily-tests/builds/slurm-rapid-storage.yaml" + echo "\"test_file_path\": tools/cloud-build/daily-tests/builds/slurm-storage.yaml" cd /workspace DEPLOYMENT_NAME="rapid-$$BUILD_ID_SHORT" @@ -126,52 +126,47 @@ steps: gsutil rm "gs://daily-tests-tf-state/*/\$$DEPLOYMENT_NAME/**.tflock" 2>/dev/null || true RUN_CLEANUP=true - # Trap function that runs the rescue playbook (terraform destroy) - # if the pod is terminated by Kueue, fails, or is cancelled. - cleanup_pod() { - local exit_code=\$$? - trap - EXIT SIGTERM SIGINT ERR - set +e - - if [ "\$$RUN_CLEANUP" = "false" ]; then - exit 0 - fi - if [ \$$exit_code -eq 0 ]; then exit_code=1; fi - - echo "" - echo "==========================================================================" - echo "CAUGHT SIGTERM OR SCRIPT ERROR!" - echo "Halting primary Ansible execution..." - echo "==========================================================================" - if [ -n "\$${ANSIBLE_PID:-}" ]; then - kill -TERM \$$ANSIBLE_PID 2>/dev/null || true - wait \$$ANSIBLE_PID 2>/dev/null || true - echo "Waiting 15s for Terraform to release GCS backend state locks..." - sleep 15 - fi - - echo "" - echo "INITIATING RESCUE PLAYBOOK: Destroying leaked infrastructure for \$$DEPLOYMENT_NAME..." - echo "- hosts: localhost" > /workspace/cleanup-playbook.yml - echo " tasks:" >> /workspace/cleanup-playbook.yml - echo " - ansible.builtin.include_tasks:" >> /workspace/cleanup-playbook.yml - echo " file: tools/cloud-build/daily-tests/ansible_playbooks/tasks/rescue_gcluster_failure.yml" >> /workspace/cleanup-playbook.yml - - ansible-playbook /workspace/cleanup-playbook.yml -e deployment_name="\$$DEPLOYMENT_NAME" -e workspace="/workspace" || true - - echo "Graceful cleanup finished." - exit \$$exit_code + source /workspace/tools/cloud-build/kueue_cleanup_pod.sh + cleanup() { + echo "Cleaning up storage pools..." + cleanup_pod } - trap cleanup_pod EXIT SIGTERM SIGINT + trap cleanup EXIT SIGTERM SIGINT bash tools/get_binary.sh "${_TEST_PREFIX}" - BLUEPRINT="examples/rapid-storage-slurm.yaml" + BLUEPRINT="examples/storage-slurm.yaml" bash tools/add_ttl_label.sh \$$BLUEPRINT + echo "Checking if required storage pools exist..." + if ! gcloud compute storage-pools describe test-pool-balanced --project=$PROJECT_ID --zone=us-central1-b >/dev/null 2>&1; then + echo "ERROR: Storage pool 'test-pool-balanced' does not exist in us-central1-b." + echo "Please create it by running the following command:" + echo "gcloud compute storage-pools create test-pool-balanced" + echo " --project=$PROJECT_ID" + echo " --zone=us-central1-b" + echo " --provisioned-capacity=10240GB" + echo " --provisioned-iops=10000" + echo " --provisioned-throughput=1024" + echo " --storage-pool-type=hyperdisk-balanced" + exit 1 + fi + if ! gcloud compute storage-pools describe test-pool-throughput --project=$PROJECT_ID --zone=us-central1-b >/dev/null 2>&1; then + echo "ERROR: Storage pool 'test-pool-throughput' does not exist in us-central1-b." + echo "Please create it by running the following command:" + echo "gcloud compute storage-pools create test-pool-throughput" + echo " --project=$PROJECT_ID" + echo " --zone=us-central1-b" + echo " --provisioned-capacity=28672GB" + echo " --provisioned-throughput=500" + echo " --storage-pool-type=hyperdisk-throughput" + exit 1 + fi + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/slurm-integration-test.yml \ - --user="$$OSLOGIN_USER" --extra-vars="project=$PROJECT_ID build=$$BUILD_ID_SHORT full_build_id=$BUILD_ID" \ - --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-rapid-storage.yaml" \ + --user="$$OSLOGIN_USER" --extra-vars="project=$PROJECT_ID build=$$BUILD_ID_SHORT full_build_id=$BUILD_ID hyperdisk_balanced_pool=test-pool-balanced hyperdisk_throughput_pool=test-pool-throughput" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/slurm-storage.yml" \ --extra-vars="triage_gcs_bucket_override=$$TRIAGE_GCS_BUCKET" \ --extra-vars="triage_project_number_override=$$TRIAGE_PROJECT_NUMBER" \ --extra-vars="triage_invoker_sa_override=$$TRIAGE_INVOKER_SA" \ @@ -184,11 +179,11 @@ steps: requests: cpu: 200m memory: "2Gi" - test-locks/slurm-rapid-storage: 1 + test-locks/slurm-storage: 1 limits: cpu: 1 memory: "2Gi" - test-locks/slurm-rapid-storage: 1 + test-locks/slurm-storage: 1 EOF # 3. Submit and Monitor GKE Kueue Job with Retry @@ -201,7 +196,7 @@ steps: set -eo pipefail gcloud container clusters get-credentials test-kueue-cluster --region=us-central1 BUILD_ID_SHORT=$$(echo "$BUILD_ID" | cut -c1-6) - JOB_NAME="slurm-rapid-storage-$$BUILD_ID_SHORT" + JOB_NAME="slurm-storage-$$BUILD_ID_SHORT" # Cloud Build trap: If the Cloud Build step itself receives a cancellation signal, # delete the GKE job so the pod receives a SIGTERM and performs infrastructure cleanup. diff --git a/tools/cloud-build/daily-tests/builds/vm-storage.yaml b/tools/cloud-build/daily-tests/builds/vm-storage.yaml new file mode 100644 index 0000000000..f934495a17 --- /dev/null +++ b/tools/cloud-build/daily-tests/builds/vm-storage.yaml @@ -0,0 +1,295 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# limitations under the License. + +--- +tags: +- vm +- m.vpc +- m.vm-instance + +substitutions: + _TEST_PREFIX: "" + _ZONE: "us-central1-b" + +timeout: 86400s +queueTtl: 86400s # 24hr +logsBucket: "gs://$PROJECT_NUMBER.cloudbuild-logs.googleusercontent.com" + +steps: +# 1. Build and push workspace container containing the current code layer +- id: build-workspace-image + name: gcr.io/cloud-builders/docker + entrypoint: /bin/bash + args: + - -c + - | + set -eo pipefail + docker build -t us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner:$BUILD_ID -f- . < job.yaml + apiVersion: batch/v1 + kind: Job + metadata: + name: vm-storage-$$BUILD_ID_SHORT + namespace: default + labels: + kueue.x-k8s.io/queue-name: local-queue-test-locks + build-id: "$BUILD_ID" + spec: + suspend: true # Required for Kueue: the job must be created in suspended state for Kueue to manage it + backoffLimit: 0 + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "false" + spec: + $$PRIORITY_LINE + serviceAccountName: test-kueue-cluster-runner-ksa + restartPolicy: Never + containers: + - name: runner + image: us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner:$BUILD_ID + env: + - name: ANSIBLE_HOST_KEY_CHECKING + value: "false" + - name: ANSIBLE_CONFIG + value: "/workspace/tools/cloud-build/ansible.cfg" + - name: _TEST_PREFIX + value: "${_TEST_PREFIX}" + - name: ZONE + value: "${_ZONE}" + - name: BUILD_ID + value: "$BUILD_ID" + - name: PROJECT_ID + value: "$PROJECT_ID" + - name: GCLUSTER_GCS_PATH + value: "$$GCLUSTER_GCS_PATH" + - name: TRIAGE_GCS_BUCKET + value: "$$TRIAGE_GCS_BUCKET" + - name: TRIAGE_PROJECT_NUMBER + value: "$$TRIAGE_PROJECT_NUMBER" + - name: TRIAGE_INVOKER_SA + value: "$$TRIAGE_INVOKER_SA" + - name: TRIAGE_CLOUD_RUN_URL + value: "$$TRIAGE_CLOUD_RUN_URL" + command: ["/bin/bash", "-c"] + args: + - | + + set -x -e + cd /workspace + + DEPLOYMENT_NAME="vm-storage-$$BUILD_ID_SHORT" + + RUN_CLEANUP=true + source /workspace/tools/cloud-build/kueue_cleanup_pod.sh + trap cleanup_pod EXIT SIGTERM SIGINT + + REGION="\$${ZONE%-*}" + bash tools/get_binary.sh "\$$_TEST_PREFIX" + + BLUEPRINT="examples/storage-vm.yaml" + bash tools/add_ttl_label.sh \$$BLUEPRINT + + + echo "Checking if required storage pools exist..." + if ! gcloud compute storage-pools describe test-pool-balanced --project=$PROJECT_ID --zone=\$$ZONE >/dev/null 2>&1; then + echo "ERROR: Storage pool 'test-pool-balanced' does not exist." + echo "Please create it by running the following command:" + echo "gcloud compute storage-pools create test-pool-balanced" + echo " --project=$PROJECT_ID" + echo " --zone=\$$ZONE" + echo " --provisioned-capacity=10240GB" + echo " --provisioned-iops=10000" + echo " --provisioned-throughput=1024" + echo " --storage-pool-type=hyperdisk-balanced" + exit 1 + fi + if ! gcloud compute storage-pools describe test-pool-throughput --project=$PROJECT_ID --zone=\$$ZONE >/dev/null 2>&1; then + echo "ERROR: Storage pool 'test-pool-throughput' does not exist." + echo "Please create it by running the following command:" + echo "gcloud compute storage-pools create test-pool-throughput" + echo " --project=$PROJECT_ID" + echo " --zone=\$$ZONE" + echo " --provisioned-capacity=28672GB" + echo " --provisioned-throughput=500" + echo " --storage-pool-type=hyperdisk-throughput" + exit 1 + fi + + + ansible-playbook tools/cloud-build/daily-tests/ansible_playbooks/base-integration-test.yml \ + --user="$$OSLOGIN_USER" --extra-vars="project=\$$PROJECT_ID build=$$BUILD_ID_SHORT full_build_id=$BUILD_ID" \ + --extra-vars="region=\$$REGION zone=\$$ZONE" \ + --extra-vars="hyperdisk_balanced_pool=test-pool-balanced hyperdisk_throughput_pool=test-pool-throughput" \ + --extra-vars="@tools/cloud-build/daily-tests/tests/vm-storage.yml" \ + --extra-vars="triage_gcs_bucket_override=$$TRIAGE_GCS_BUCKET" \ + --extra-vars="triage_project_number_override=$$TRIAGE_PROJECT_NUMBER" \ + --extra-vars="triage_invoker_sa_override=$$TRIAGE_INVOKER_SA" \ + --extra-vars="triage_cloud_run_url_override=$$TRIAGE_CLOUD_RUN_URL" & + ANSIBLE_PID=\$$! + + wait \$$ANSIBLE_PID + RUN_CLEANUP=false + + resources: + requests: + cpu: 200m + memory: "2Gi" + test-locks/vm-storage: 1 + limits: + cpu: 1 + memory: "2Gi" + test-locks/vm-storage: 1 + EOF + +# 3. Submit and Monitor GKE Kueue Job with Retry +- id: submit-and-monitor-gke-job + name: gcr.io/cloud-builders/gcloud + entrypoint: /bin/bash + args: + - -c + - | + set -eo pipefail + gcloud container clusters get-credentials test-kueue-cluster --region=us-central1 + BUILD_ID_SHORT=$$(echo "$BUILD_ID" | cut -c1-6) + JOB_NAME="vm-storage-$$BUILD_ID_SHORT" + + # Cloud Build trap: If the Cloud Build step itself receives a cancellation signal, + # delete the GKE job so the pod receives a SIGTERM and performs infrastructure cleanup. + cleanup_cb() { + echo "" + echo "==========================================================================" + echo "PIPELINE CANCELLED: The Cloud Build step received a termination signal." + echo "Deleting GKE Kueue Job ($$JOB_NAME) to force the Pod to clean itself up!" + echo "==========================================================================" + kubectl delete job "$$JOB_NAME" -n default || true + exit 1 + } + trap cleanup_cb SIGTERM SIGINT + + MAX_RETRIES=10 + RETRY_DELAY=300 + ATTEMPT=1 + + while true; do + echo "=== ATTEMPT $$ATTEMPT: Submitting Kueue Job ===" + kubectl apply -f /workspace/job.yaml + + set +e + ( bash tools/cloud-build/monitor_kueue_job.sh \ + test-kueue-cluster \ + us-central1 \ + "$$JOB_NAME" \ + default | tee /workspace/job_logs.txt; exit $${PIPESTATUS[0]} ) & + MONITOR_PID=$$! + wait $$MONITOR_PID + EXIT_CODE=$$? + set -e + + if [ $$EXIT_CODE -eq 0 ]; then + echo "Job succeeded!" + break + fi + + # If so, retry. If it's a real error (like a terraform syntax error), fail immediately. + if bash tools/cloud-build/check_retriable_error.sh /workspace/job_logs.txt; then + echo "WARNING: Retriable error detected. Kueue Job has already been deleted. Retrying in $$RETRY_DELAY seconds..." + else + echo "ERROR: Test failed due to an actual error (not zone capacity). Failing pipeline." >&2 + exit 1 + fi + + if [ $$ATTEMPT -ge $$MAX_RETRIES ]; then + exit 1 + fi + + sleep $$RETRY_DELAY + ATTEMPT=$$((ATTEMPT + 1)) + done + +# 4. Cleanup the built image from Registry to save costs +- id: cleanup-image + name: gcr.io/cloud-builders/gcloud + entrypoint: /bin/bash + allowFailure: true + args: + - -c + - | + set -eo pipefail + IMAGE="us-central1-docker.pkg.dev/$PROJECT_ID/hpc-toolkit-repo/test-runner:$BUILD_ID" + MAX_RETRIES=5 + RETRY_DELAY=10 + ATTEMPT=1 + + echo "Starting cleanup for image: $$IMAGE" + + while [ "$$ATTEMPT" -le "$$MAX_RETRIES" ]; do + echo "Attempt $$ATTEMPT of $$MAX_RETRIES..." + + if gcloud artifacts docker images delete "$$IMAGE" --quiet; then + echo "Image successfully deleted." + exit 0 + fi + + if [ "$$ATTEMPT" -lt "$$MAX_RETRIES" ]; then + echo "Deletion failed. Retrying in $$RETRY_DELAY seconds..." + sleep $$RETRY_DELAY + fi + + ATTEMPT=$$((ATTEMPT + 1)) + done + + echo "Failed to delete image after $$MAX_RETRIES attempts." + echo "Image will be handled by Artifact Registry background cleanup policies." + exit 1 + +availableSecrets: + secretManager: + - versionName: projects/${PROJECT_ID}/secrets/sa-email/versions/latest + env: 'SA_EMAIL' + - versionName: projects/${PROJECT_ID}/secrets/gcluster-develop-release-bucket/versions/latest + env: 'GCLUSTER_GCS_PATH' + - versionName: projects/${PROJECT_ID}/secrets/triage-gcs-bucket/versions/latest + env: 'TRIAGE_GCS_BUCKET' + - versionName: projects/${PROJECT_ID}/secrets/triage-project-number/versions/latest + env: 'TRIAGE_PROJECT_NUMBER' + - versionName: projects/${PROJECT_ID}/secrets/triage-invoker-sa/versions/latest + env: 'TRIAGE_INVOKER_SA' + - versionName: projects/${PROJECT_ID}/secrets/triage-cloud-run-url/versions/latest + env: 'TRIAGE_CLOUD_RUN_URL' diff --git a/tools/cloud-build/daily-tests/tests/gke-storage.yml b/tools/cloud-build/daily-tests/tests/gke-storage.yml index dd9c947e0c..8f59c07c9a 100644 --- a/tools/cloud-build/daily-tests/tests/gke-storage.yml +++ b/tools/cloud-build/daily-tests/tests/gke-storage.yml @@ -23,11 +23,15 @@ remote_node: "{{ deployment_name }}-0" post_deploy_tests: - test-validation/test-zonal-bucket.yml - test-validation/test-anywhere-cache.yml +- test-validation/test-gke-storage-pool.yml cli_deployment_vars: test_name: "{{ test_name }}" region: "{{ region }}" zone: "{{ zone }}" authorized_cidr: "{{ build_ip.stdout }}/32" gcp_public_cidrs_access_enabled: true + project_id: "{{ project }}" + hyperdisk_balanced_storage_pool: "projects/{{ project }}/zones/us-central1-b/storagePools/{{ hyperdisk_balanced_pool }}" + hyperdisk_throughput_storage_pool: "projects/{{ project }}/zones/us-central1-b/storagePools/{{ hyperdisk_throughput_pool }}" custom_vars: project: "{{ project }}" diff --git a/tools/cloud-build/daily-tests/tests/slurm-rapid-storage.yaml b/tools/cloud-build/daily-tests/tests/slurm-storage.yml similarity index 77% rename from tools/cloud-build/daily-tests/tests/slurm-rapid-storage.yaml rename to tools/cloud-build/daily-tests/tests/slurm-storage.yml index dbbc0672bb..d881a81e01 100644 --- a/tools/cloud-build/daily-tests/tests/slurm-rapid-storage.yaml +++ b/tools/cloud-build/daily-tests/tests/slurm-storage.yml @@ -18,16 +18,19 @@ slurm_cluster_name: "rapid{{ build[0:4] }}" region: us-central1 zone: us-central1-b workspace: /workspace -blueprint_yaml: "{{ workspace }}/examples/rapid-storage-slurm.yaml" +blueprint_yaml: "{{ workspace }}/examples/storage-slurm.yaml" network: "{{ test_name }}-net" login_node: "{{ slurm_cluster_name }}-slurm-login-*" controller_node: "{{ slurm_cluster_name }}-controller" post_deploy_tests: -- test-validation/test-slurm-rapid-storage.yml +- test-validation/test-slurm-storage.yml cli_deployment_vars: region: "{{ region }}" network_name: "{{ network }}" slurm_cluster_name: "{{ slurm_cluster_name }}" + project_id: "{{ project }}" + hyperdisk_balanced_storage_pool: "projects/{{ project }}/zones/{{ zone }}/storagePools/{{ hyperdisk_balanced_pool }}" + hyperdisk_throughput_storage_pool: "projects/{{ project }}/zones/{{ zone }}/storagePools/{{ hyperdisk_throughput_pool }}" custom_vars: project_id: "{{ project }}" zone: "{{ zone }}" diff --git a/tools/cloud-build/daily-tests/tests/vm-storage.yml b/tools/cloud-build/daily-tests/tests/vm-storage.yml new file mode 100644 index 0000000000..78bebfad99 --- /dev/null +++ b/tools/cloud-build/daily-tests/tests/vm-storage.yml @@ -0,0 +1,32 @@ +# Copyright 2026 "Google LLC" +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +# region, zone must be defined in build file with --extra-vars flag! +test_name: vm-storage +deployment_name: vmstorage-{{ build }} +workspace: /workspace +blueprint_yaml: "{{ workspace }}/examples/storage-vm.yaml" +network: "{{ test_name }}-net" +remote_node: "{{ deployment_name }}-0" +post_deploy_tests: +- test-validation/test-vm-storage.yml +cli_deployment_vars: + project_id: "{{ project }}" + test_name: "{{ test_name }}" + network_name: "{{ network }}" + region: "{{ region }}" + zone: "{{ zone }}" + hyperdisk_balanced_storage_pool: "projects/{{ project }}/zones/{{ zone }}/storagePools/{{ hyperdisk_balanced_pool }}" + hyperdisk_throughput_storage_pool: "projects/{{ project }}/zones/{{ zone }}/storagePools/{{ hyperdisk_throughput_pool }}" diff --git a/tools/cloud-build/daily-tests/validate_tests_metadata.py b/tools/cloud-build/daily-tests/validate_tests_metadata.py index 6eadde3cc7..d398574400 100644 --- a/tools/cloud-build/daily-tests/validate_tests_metadata.py +++ b/tools/cloud-build/daily-tests/validate_tests_metadata.py @@ -78,7 +78,7 @@ def get_blueprint(build_path: str) -> Optional[str]: f"{BUILDS_DIR}/slurm-gcp-v6-simple-job-completion.yaml": "tools/python-integration-tests/blueprints/slurm-simple.yaml", f"{BUILDS_DIR}/slurm-flex.yaml": "tools/python-integration-tests/blueprints/slurm-flex.yaml", f"{BUILDS_DIR}/slurm-gcp-v6-topology.yaml": "tools/python-integration-tests/blueprints/topology-test.yaml", - f"{BUILDS_DIR}/slurm-rapid-storage.yaml": "examples/rapid-storage-slurm.yaml", + f"{BUILDS_DIR}/slurm-storage.yaml": "examples/storage-slurm.yaml", } if build_path in SPECIAL_CASES: return SPECIAL_CASES[build_path]