From 7c345ecbabd3b7ef80ef6ae1a17ece68faf452f3 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Wed, 16 Jul 2025 11:34:56 -0400 Subject: [PATCH 1/9] [Isolated] Install cfn-dependencies only for AL2 --- .../recipes/install/cfn_bootstrap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index cfe95c4ed5..7b5d8f0815 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -33,7 +33,7 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end -if aws_region.start_with?("us-iso") +if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" mode '0644' From a9724796b2e5139a437e94ce7d0ef7d0a96c9ad9 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 21 Jul 2025 09:51:55 -0400 Subject: [PATCH 2/9] Revert "[Isolated] Install cfn-dependencies only for AL2" This reverts commit 976d4799313634b3274be1da55ff768f1bd3503f. --- .../recipes/install/cfn_bootstrap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 7b5d8f0815..cfe95c4ed5 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -33,7 +33,7 @@ not_if { ::File.exist?("#{virtualenv_path}/bin/activate") } end -if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" +if aws_region.start_with?("us-iso") remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" mode '0644' From 857638cde2e30084bfdabc25e1f0dd4dbcbcc93f Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 21 Jul 2025 09:52:42 -0400 Subject: [PATCH 3/9] [Isolated] USe latest cfn-dependencies --- .../recipes/install/cfn_bootstrap.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index cfe95c4ed5..ea4aa8fbf9 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -35,7 +35,7 @@ if aws_region.start_with?("us-iso") remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cfn-dependencies.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/pypi-cfn-dependencies-3.12-x86_64.tgz" mode '0644' retries 3 retry_delay 5 @@ -48,8 +48,8 @@ cwd "#{node['cluster']['base_dir']}" code <<-REQ set -e - tar xzf cfn-dependencies.tgz - cd cfn + tar xzf pypi-cfn-dependencies-3.12-x86_64.tgz + cd dependencies #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end From b56bb2cad26b2fd482050a8ac8b30e74c45e0b98 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Mon, 21 Jul 2025 09:56:17 -0400 Subject: [PATCH 4/9] [Isolated] Using Git REF for Uploading cookbook --- util/upload-cookbook.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/util/upload-cookbook.sh b/util/upload-cookbook.sh index c2169a4c3e..864ce56959 100755 --- a/util/upload-cookbook.sh +++ b/util/upload-cookbook.sh @@ -93,21 +93,22 @@ main() { # Create archive and md5 _cwd=$(pwd) pushd "${_srcdir}" > /dev/null || exit + GIT_REF=$(git rev-parse HEAD) _stashName=$(git stash create) - git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" + git archive --format tar --prefix="aws-parallelcluster-cookbook-${_version}/" "${_stashName:-HEAD}" | gzip > "${_cwd}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz" #tar zcvf "${_cwd}/aws-parallelcluster-cookbook-${_version}.tgz" --transform "s,^aws-parallelcluster-cookbook/,aws-parallelcluster-cookbook-${_version}/," ../aws-parallelcluster-cookbook popd > /dev/null || exit - md5sum aws-parallelcluster-cookbook-${_version}.tgz > aws-parallelcluster-cookbook-${_version}.md5 + md5sum aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 # upload packages _key_path="parallelcluster/${_version}/cookbooks" if [ -n "${_scope}" ]; then _key_path="${_key_path}/${_scope}" fi - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz || _error_exit 'Failed to push cookbook to S3' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.md5 || _error_exit 'Failed to push cookbook md5 to S3' - aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to fetch LastModified date' - aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz.date || _error_exit 'Failed to push cookbook date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz || _error_exit 'Failed to push cookbook to S3' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.md5 || _error_exit 'Failed to push cookbook md5 to S3' + aws ${_profile} --region "${_region}" s3api head-object --bucket ${_bucket} --key ${_key_path}/aws-parallelcluster-cookbook-${_version}.tgz --output text --query LastModified > aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to fetch LastModified date' + aws ${_profile} --region "${_region}" s3 cp aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date s3://${_bucket}/${_key_path}/aws-parallelcluster-cookbook-${_version}-${GIT_REF}.tgz.date || _error_exit 'Failed to push cookbook date' _bucket_region=$(aws ${_profile} s3api get-bucket-location --bucket ${_bucket} --output text) if [ ${_bucket_region} = "None" ]; then From e49d4a17ec4ada4aefb9f8a1f5b66f6d695e6d86 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Tue, 22 Jul 2025 14:02:30 -0400 Subject: [PATCH 5/9] [Isolated] Rename the cfn-dependecies files --- .../recipes/install/cfn_bootstrap.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index ea4aa8fbf9..dad69ad964 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -48,7 +48,7 @@ cwd "#{node['cluster']['base_dir']}" code <<-REQ set -e - tar xzf pypi-cfn-dependencies-3.12-x86_64.tgz + tar xzf cfn-dependencies.tgz cd dependencies #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ From dd33ac100d207a1ca78e193972cda384d59ccc74 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 24 Jul 2025 10:37:38 -0400 Subject: [PATCH 6/9] [Isolated] Chnage the name of Cookbook Dependencies and the folder name inside the Tar --- .../recipes/install/cookbook_virtualenv.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb index cdc557df5f..2845ec9c29 100644 --- a/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb +++ b/cookbooks/aws-parallelcluster-platform/recipes/install/cookbook_virtualenv.rb @@ -12,8 +12,10 @@ # limitations under the License. virtualenv_path = cookbook_virtualenv_path -pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/pypi-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}.tgz" +dependency_package_name = "pypi-cookbook-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" +pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{dependency_package_name}.tgz" if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "dependencies" pypi_s3_uri = "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/cookbook-dependencies.tgz" end @@ -46,7 +48,7 @@ code <<-REQ set -e tar xzf cookbook-dependencies.tgz - cd dependencies + cd #{dependency_package_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end From 506bb335e71f6676535f1f5491e98a2b37007362 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Thu, 24 Jul 2025 12:33:12 -0400 Subject: [PATCH 7/9] [Isolated] Chnage the name of CFN Dependencies and the folder name inside the Tar --- .../recipes/install/cfn_bootstrap.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index dad69ad964..0eee866fee 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -34,8 +34,14 @@ end if aws_region.start_with?("us-iso") + dependency_package_name = "pypi-cfn-dependencies-#{node['cluster']['python-major-minor-version']}-#{node['kernel']['machine']}" + dependency_folder_name = dependency_package_name + if platform?('amazon') && node['platform_version'] == "2" + dependency_package_name = "cfn-dependencies" + dependency_folder_name = "cfn" + end remote_file "#{node['cluster']['base_dir']}/cfn-dependencies.tgz" do - source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/pypi-cfn-dependencies-3.12-x86_64.tgz" + source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/#{dependency_package_name}.tgz" mode '0644' retries 3 retry_delay 5 @@ -49,7 +55,7 @@ code <<-REQ set -e tar xzf cfn-dependencies.tgz - cd dependencies + cd #{dependency_folder_name} #{virtualenv_path}/bin/pip install * -f ./ --no-index REQ end From 5a30c1a3ebb131cde77fe2391d286d90ae57781f Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Tue, 29 Jul 2025 16:31:12 -0400 Subject: [PATCH 8/9] [Isolated] Installing Cfn-bootstrap using `--no-build-isolation` as 3.12.8 uses setup.py based installation where it uses a isolated build instead of looking at existing site-packages --- .../recipes/install/custom_parallelcluster_node.rb | 8 +++++++- .../recipes/install/cfn_bootstrap.rb | 7 ++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb index 247823c173..0cc5e36bed 100644 --- a/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb +++ b/cookbooks/aws-parallelcluster-computefleet/recipes/install/custom_parallelcluster_node.rb @@ -19,6 +19,12 @@ # TODO: once the pyenv Chef resource supports installing packages from a path (e.g. `pip install .`), convert the # bash block to a recipe that uses the pyenv resource. +if aws_region.start_with?("us-iso") + command = "pip install . --no-build-isolation" +else + command = "pip install ." +end + if aws_region.start_with?("us-iso") && platform?('amazon') && node['platform_version'] == "2" remote_file "#{node['cluster']['base_dir']}/node-dependencies.tgz" do source "#{node['cluster']['artifacts_s3_url']}/dependencies/PyPi/#{node['kernel']['machine']}/node-dependencies.tgz" @@ -59,7 +65,7 @@ mkdir aws-parallelcluster-custom-node tar -xzf aws-parallelcluster-node.tgz --directory aws-parallelcluster-custom-node cd aws-parallelcluster-custom-node/*aws-parallelcluster-node* - pip install . + #{command} deactivate NODE end diff --git a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb index 0eee866fee..8270a62ff7 100644 --- a/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb +++ b/cookbooks/aws-parallelcluster-environment/recipes/install/cfn_bootstrap.rb @@ -79,11 +79,16 @@ retry_delay 5 end +if aws_region.start_with?("us-iso") + command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package} --no-build-isolation" +else + command = "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" +end bash "Install CloudFormation helpers from #{cfnbootstrap_package}" do user 'root' group 'root' cwd '/tmp' - code "#{virtualenv_path}/bin/pip install #{cfnbootstrap_package}" + code command creates "#{virtualenv_path}/bin/cfn-hup" end From fc056aa8ac7e27dc90fc96877f74f621f52b0850 Mon Sep 17 00:00:00 2001 From: Himani Anil Deshpande Date: Fri, 1 Aug 2025 16:03:50 -0400 Subject: [PATCH 9/9] [Isolated] Install efs-proxy cargo dependecies for isolated environment --- .../resources/efs/efs_redhat8.rb | 20 ++++++++++++++++++- .../efs/partial/_install_from_tar.rb | 14 +++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb index b2ab45bc1b..67747e8859 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/efs_redhat8.rb @@ -18,6 +18,24 @@ use 'partial/_get_package_version_rpm' use 'partial/_common' -use 'partial/_redhat_based' +# use 'partial/_redhat_based' use 'partial/_install_from_tar' use 'partial/_mount_umount' + +def install_script_code(efs_utils_tarball, efs_utils_package, efs_utils_version) + <<-EFSUTILSINSTALL + set -e + tar xf #{efs_utils_tarball} + mv efs-proxy-dependencies-#{efs_utils_version}.tar.gz efs-utils-#{efs_utils_version}/src/proxy/ + cd efs-utils-#{efs_utils_version}/src/proxy/ + tar -xf efs-proxy-dependencies-#{efs_utils_version}.tar.gz + cargo build --offline + cd ../.. + make rpm + yum -y install ./build/#{efs_utils_package}*rpm + EFSUTILSINSTALL +end + +def prerequisites + %w(rpm-build make rust cargo openssl-devel) +end diff --git a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb index cf65440cf0..112a6ce094 100644 --- a/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb +++ b/cookbooks/aws-parallelcluster-environment/resources/efs/partial/_install_from_tar.rb @@ -50,6 +50,20 @@ action :create_if_missing end + if aws_region.start_with?("us-iso") && platform?('redhat') && node['platform_version'] == "8" + efs_proxy_deps = "efs-proxy-dependencies-#{package_version}.tar.gz" + efs_proxy_deps_tarball = "#{node['cluster']['sources_dir']}/#{efs_proxy_deps}" + efs_proxy_deps_url = "#{node['cluster']['artifacts_s3_url']}/dependencies/efs/#{efs_proxy_deps}" + remote_file efs_proxy_deps_tarball do + source efs_proxy_deps_url + mode '0644' + retries 3 + retry_delay 5 + checksum new_resource.efs_utils_checksum + action :create_if_missing + end + end + # Install EFS Utils following https://docs.aws.amazon.com/efs/latest/ug/installing-amazon-efs-utils.html bash "install efs utils" do cwd node['cluster']['sources_dir']