diff --git a/modules/resource_provisioner/locals.tf b/modules/resource_provisioner/locals.tf index dff9136c..5b09c239 100644 --- a/modules/resource_provisioner/locals.tf +++ b/modules/resource_provisioner/locals.tf @@ -5,7 +5,7 @@ locals { remote_terraform_path = format("%s/terraform-ibm-hpc", local.deployer_path) # da_hpc_repo_url = "github.ibm.com/workload-eng-services/HPCaaS.git" da_hpc_repo_url = "github.com/terraform-ibm-modules/terraform-ibm-hpc.git" - da_hpc_repo_tag = "29-sept" ###### change it to main in future + da_hpc_repo_tag = "main" ###### change it to main in future remote_ansible_path = format("%s/ibm-spectrumscale-cloud-deploy", local.deployer_path) scale_cloud_infra_repo_url = "https://github.com/jayeshh123/ibm-spectrum-scale-install-infra" scale_cloud_infra_repo_name = "ibm-spectrum-scale-install-infra" diff --git a/tools/access-management/README.md b/tools/access-management/README.md index 9431e3fa..f5db94af 100644 --- a/tools/access-management/README.md +++ b/tools/access-management/README.md @@ -8,7 +8,7 @@ For example, for the App configuration service, the user requires Administrator ### Benefits of the scripts: -#### Interactive input collection - The script prompts for the IBMid (admin email), Resource Group ID, Account ID, and target (User or Access Group). +#### Interactive input collection - The script prompts for the IBMid (admin email), Account ID, and target (User or Access Group). #### Permission check - The script verifies that the admin has account-level Administrator rights which is required to assign policies. @@ -34,7 +34,7 @@ chmod +x permissions.sh 3. Enter the admin email or IBMid. -4. Enter the Resource group and Account ID. +4. Enter the Account ID. For the Account ID, login to the IBM Cloud account by using your unique credentials. Go to Manage > Account > Account settings. You will find the Account ID. @@ -50,10 +50,9 @@ Select the required option. 7. User policy is successfully created. -If the user skips to enter the RESOURCE_GROUP_ID or the ACCOUNT_ID, then script displays the error message: +If the user skips to enter the ACCOUNT_ID, then script displays the error message: ``` -:x: RESOURCE_GROUP_ID is required. :x: ACCOUNT_ID is required. ``` diff --git a/tools/access-management/permissions.sh b/tools/access-management/permissions.sh index dd5946ad..82dc2052 100755 --- a/tools/access-management/permissions.sh +++ b/tools/access-management/permissions.sh @@ -12,12 +12,6 @@ if [ -z "$ADMIN_EMAIL" ]; then exit 1 fi -read -rp "Enter Resource Group ID: " RESOURCE_GROUP_ID -if [ -z "$RESOURCE_GROUP_ID" ]; then - echo "❌ RESOURCE_GROUP_ID is required." - exit 1 -fi - read -rp "Enter Account ID: " ACCOUNT_ID if [ -z "$ACCOUNT_ID" ]; then echo "❌ ACCOUNT_ID is required." @@ -71,6 +65,16 @@ check_policies() { select(any(.resources[].attributes[]?; .name == "serviceType" and .value == "platform_service")) ' >/dev/null 2>&1 && echo "true" || echo "false") + # Check role for IAM Identity service (Administrator) + local has_identity_role + has_identity_role=$(echo "$policies" | jq -e ' + .[] | + select(.roles? != null) | + select(any(.roles[]?.display_name; . == "Administrator")) | + select(any(.resources[].attributes[]?; .name == "accountId")) | + select(any(.resources[].attributes[]?; .name == "serviceName" and .value == "iam-identity")) + ' >/dev/null 2>&1 && echo "true" || echo "false") + # Debug printing if [ "$has_admin" = "true" ]; then echo "✅ At $scope policy level: Has Administrator for All Identity and Access enabled service" @@ -78,13 +82,19 @@ check_policies() { echo "❌ At $scope policy level: Missing Administrator for All Identity and Access enabled service" fi + if [ "$has_identity_role" = "true" ]; then + echo "✅ At $scope policy level: Has Administrator for IAM Identity services" + else + echo "❌ At $scope policy level: Missing Administrator for IAM Identity service" + fi + if [ "$has_platform_role" = "true" ]; then echo "✅ At $scope policy level: Has Viewer/Editor/Administrator for All Account Management services" else echo "❌ At $scope policy level: Missing Viewer/Editor/Administrator for All Account Management services" fi - [[ "$has_admin" == "true" && "$has_platform_role" == "true" ]] + [[ "$has_admin" == "true" && "$has_platform_role" == "true" && "$has_identity_role" == "true" ]] } USER_POLICIES=$(ibmcloud iam user-policies "$ADMIN_EMAIL" --output json 2>/dev/null || echo "[]") @@ -125,7 +135,11 @@ sysdig-monitor|Administrator|Manager kms|Service Configuration Reader|Manager secrets-manager|Administrator|Manager sysdig-secure|Administrator| -is|Editor|" +is|Editor| +iam-identity|Administrator| +atracker|Administrator| +logs-router|Administrator| +metrics-router|Administrator|" FRIENDLY_NAMES="apprapp|App Configuration cloud-object-storage|Cloud Object Storage @@ -134,7 +148,11 @@ sysdig-monitor|Cloud Monitoring kms|Key Protect secrets-manager|Secrets Manager sysdig-secure|Security and Compliance Center Workload Protection -is|VPC Infrastructure Services" +is|VPC Infrastructure Services +iam-identity|IAM Identity +atracker|Activity tracker event routing +logs-router|Cloud logs routing +metrics-router|Metrics routing" get_friendly_name() { local service="$1" @@ -166,11 +184,8 @@ if [ -n "$ACCESS_GROUP" ] && [ -z "$USER_EMAIL" ]; then existing_policies=$(ibmcloud iam access-group-policies "$ACCESS_GROUP" --output json 2>/dev/null || echo "[]") POLICY_ID=$(echo "$existing_policies" | jq -r \ - --arg service "$SERVICE_NAME" \ - --arg rg_id "$RESOURCE_GROUP_ID" ' + --arg service "$SERVICE_NAME" ' .[] | select(any(.resources[].attributes[]?; - .name == "resourceGroupId" and .value == $rg_id)) | - select(any(.resources[].attributes[]?; .name == "serviceName" and .value == $service)) | .id' | head -n1) @@ -194,24 +209,20 @@ if [ -n "$ACCESS_GROUP" ] && [ -z "$USER_EMAIL" ]; then ibmcloud iam access-group-policy-update "$ACCESS_GROUP" "$POLICY_ID" \ --roles "$MERGED_SORTED" \ - --resource-group-id "$RESOURCE_GROUP_ID" \ --service-name "$SERVICE_NAME" || echo "⚠️ Failed to update roles for $DISPLAY_NAME" fi else echo "➕ Creating new policy for $DISPLAY_NAME" ibmcloud iam access-group-policy-create "$ACCESS_GROUP" \ --roles "$ROLES" \ - --service-name "$SERVICE_NAME" \ - --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to assign $ROLES for $DISPLAY_NAME" + --service-name "$SERVICE_NAME" || echo "⚠️ Failed to assign $ROLES for $DISPLAY_NAME" fi done echo "🔍 Checking global Administrator/Manager policy for access group: $ACCESS_GROUP" existing_policies=$(ibmcloud iam access-group-policies "$ACCESS_GROUP" --output json 2>/dev/null || echo "[]") - POLICY_ID=$(echo "$existing_policies" | jq -r --arg rg_id "$RESOURCE_GROUP_ID" ' + POLICY_ID=$(echo "$existing_policies" | jq -r ' .[] | - select(any(.resources[].attributes[]?; - .name == "resourceGroupId" and .value == $rg_id)) | select(all(.resources[].attributes[]?.name; . != "serviceName")) | .id' | head -n1) @@ -234,14 +245,12 @@ if [ -n "$ACCESS_GROUP" ] && [ -z "$USER_EMAIL" ]; then echo " • Adding roles : $NEW_ROLES" ibmcloud iam access-group-policy-update "$ACCESS_GROUP" "$POLICY_ID" \ - --roles "$MERGED_SORTED" \ - --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to update Administrator,Manager roles for All Identity and Access enabled services to access group: $ACCESS_GROUP" + --roles "$MERGED_SORTED" || echo "⚠️ Failed to update Administrator,Manager roles for All Identity and Access enabled services to access group: $ACCESS_GROUP" fi else echo "➕ Creating new global Administrator/Manager policy for access group: $ACCESS_GROUP" ibmcloud iam access-group-policy-create "$ACCESS_GROUP" \ - --roles "Administrator,Manager" \ - --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to assign Administrator,Manager roles for All Identity and Access enabled services to access group: $ACCESS_GROUP" + --roles "Administrator,Manager" || echo "⚠️ Failed to assign Administrator,Manager roles for All Identity and Access enabled services to access group: $ACCESS_GROUP" fi elif [ -z "$ACCESS_GROUP" ] && [ -n "$USER_EMAIL" ]; then @@ -254,13 +263,11 @@ elif [ -z "$ACCESS_GROUP" ] && [ -n "$USER_EMAIL" ]; then existing_policies=$(ibmcloud iam user-policies "$USER_EMAIL" --output json 2>/dev/null || echo "[]") POLICY_ID=$(echo "$existing_policies" | jq -r \ - --arg service "$SERVICE_NAME" \ - --arg rg_id "$RESOURCE_GROUP_ID" ' - .[] | select(any(.resources[].attributes[]?; - .name == "resourceGroupId" and .value == $rg_id)) | - select(any(.resources[].attributes[]?; - .name == "serviceName" and .value == $service)) | - .id' | head -n1) + --arg service "$SERVICE_NAME" ' + .[] + | select(any(.resources[].attributes[]?; + .name == "serviceName" and .value == $service)) + | .id' | head -n1) if [ -n "$POLICY_ID" ] && [ "$POLICY_ID" != "null" ]; then EXISTING_ROLES=$(echo "$existing_policies" | jq -r --arg id "$POLICY_ID" ' @@ -282,24 +289,20 @@ elif [ -z "$ACCESS_GROUP" ] && [ -n "$USER_EMAIL" ]; then ibmcloud iam user-policy-update "$USER_EMAIL" "$POLICY_ID" \ --roles "$MERGED_SORTED" \ - --resource-group-id "$RESOURCE_GROUP_ID" \ --service-name "$SERVICE_NAME" || echo "⚠️ Failed to update roles for $DISPLAY_NAME" fi else echo "➕ Creating new policy for $DISPLAY_NAME" ibmcloud iam user-policy-create "$USER_EMAIL" \ --roles "$ROLES" \ - --service-name "$SERVICE_NAME" \ - --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to assign $ROLES for $DISPLAY_NAME" + --service-name "$SERVICE_NAME" || echo "⚠️ Failed to assign $ROLES for $DISPLAY_NAME" fi done echo "🔍 Checking global Administrator/Manager policy for $USER_EMAIL" existing_policies=$(ibmcloud iam user-policies "$USER_EMAIL" --output json 2>/dev/null || echo "[]") - POLICY_ID=$(echo "$existing_policies" | jq -r --arg rg_id "$RESOURCE_GROUP_ID" ' + POLICY_ID=$(echo "$existing_policies" | jq -r ' .[] | - select(any(.resources[].attributes[]?; - .name == "resourceGroupId" and .value == $rg_id)) | select(all(.resources[].attributes[]?.name; . != "serviceName")) | .id' | head -n1) @@ -322,14 +325,12 @@ elif [ -z "$ACCESS_GROUP" ] && [ -n "$USER_EMAIL" ]; then echo " • Adding roles : $NEW_ROLES" ibmcloud iam user-policy-update "$USER_EMAIL" "$POLICY_ID" \ - --roles "$MERGED_SORTED" \ - --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to update Administrator,Manager roles for All Identity and Access enabled services to user: $USER_EMAIL" + --roles "$MERGED_SORTED" || echo "⚠️ Failed to update Administrator,Manager roles for All Identity and Access enabled services to user: $USER_EMAIL" fi else echo "➕ Creating new global Administrator/Manager policy for $USER_EMAIL" ibmcloud iam user-policy-create "$USER_EMAIL" \ - --roles "Administrator,Manager" \ - --resource-group-id "$RESOURCE_GROUP_ID" || echo "⚠️ Failed to assign Administrator,Manager roles for All Identity and Access enabled services to user: $USER_EMAIL" + --roles "Administrator,Manager" || echo "⚠️ Failed to assign Administrator,Manager roles for All Identity and Access enabled services to user: $USER_EMAIL" fi else diff --git a/tools/image-builder/packer/hpcaas/compute/script.sh b/tools/image-builder/packer/hpcaas/compute/script.sh index c85dcc3d..dbf6fc79 100644 --- a/tools/image-builder/packer/hpcaas/compute/script.sh +++ b/tools/image-builder/packer/hpcaas/compute/script.sh @@ -83,57 +83,26 @@ curl -fsSL https://clis.cloud.ibm.com/install/linux | sh pip3 install ibm-vpc==0.10.0 pip3 install ibm-cloud-networking-services ibm-cloud-sdk-core selinux ibmcloud plugin install vpc-infrastructure DNS -echo 'LS_Standard 10.1 () () () () 18b1928f13939bd17bf25e09a2dd8459f238028f' > ${LSF_PACKAGES_PATH}/ls.entitlement -echo 'LSF_Standard 10.1 () () () pa 3f08e215230ffe4608213630cd5ef1d8c9b4dfea' > ${LSF_PACKAGES_PATH}/lsf.entitlement echo "======================Installation of IBMCloud Plugins completed=====================" -hostname lsfservers +hostnamectl +hostnamectl set-hostname lsfservers # Installation of LSF base packages on compute node cd "${LSF_PACKAGES_PATH}" || exit -zcat lsf*lsfinstall_linux_x86_64.tar.Z | tar xvf - -cd lsf*_lsfinstall || exit -sed -e '/show_copyright/ s/^#*/#/' -i lsfinstall -cat <> install.config -LSF_TOP="/opt/ibm/lsf" -LSF_ADMINS="lsfadmin" -LSF_CLUSTER_NAME="HPCCluster" -LSF_MASTER_LIST="lsfservers" -LSF_ENTITLEMENT_FILE="${LSF_PACKAGES_PATH}/lsf.entitlement" -CONFIGURATION_TEMPLATE="DEFAULT" -ENABLE_DYNAMIC_HOSTS="Y" -ENABLE_EGO="N" -ACCEPT_LICENSE="Y" -SILENT_INSTALL="Y" -LSF_SILENT_INSTALL_TARLIST="ALL" -EOT -bash lsfinstall -f install.config -echo $? -cat Install.log +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-conf-10.1.0.15-25050119.noarch.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-man-pages-10.1.0.15-25050119.noarch.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-client-10.1.0.15-25050119.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-server-10.1.0.15-25050119.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-integrations-10.1.0.15-25050118.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-ego-server-10.1.0.15-25050118.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-devel-10.1.0.15-25050119.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-data-mgr-10.1.0.15-25050119.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-ls-client-10.1.0.15-25050119.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/ibm-jre-1.8.0-25041010.x86_64.rpm +yum install -y --nogpgcheck "${LSF_PACKAGES_PATH}"/lsf-pm-client-10.2.0.15-25050118.x86_64.rpm echo "========================LSF 10.1 installation completed=====================" - -hostname lsfservers -# Installation of Resource connector configuration on compute nodes -cd "${LSF_PACKAGES_PATH}" || exit -cd lsf*_lsfinstall || exit -cat <> server.config -LSF_TOP="/opt/ibm/lsf_worker" -LSF_ADMINS="lsfadmin" -LSF_ENTITLEMENT_FILE="${LSF_PACKAGES_PATH}/lsf.entitlement" -LSF_SERVER_HOSTS="lsfservers" -LSF_LOCAL_RESOURCES="[resource cloudhpchost]" -ACCEPT_LICENSE="Y" -SILENT_INSTALL="Y" -EOT -bash lsfinstall -s -f server.config -echo $? -cat Install.log -rm -rf /opt/ibm/lsf_worker/10.1 -ln -s /opt/ibm/lsf/10.1 /opt/ibm/lsf_worker -echo "==================LSF 10.1 Resource connector installation completed===============" - - # Installation Of OpenMPI cd "${LSF_PACKAGES_PATH}" || exit wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.0.tar.gz @@ -194,6 +163,18 @@ else echo "INSTALL_SYDIG is set as false and the sysdig agent is not installed on compute node image" fi +#Cloud Log Agent Installation +echo "Cloud logs agent installation started" +pwd +wget https://logs-router-agent-install-packages.s3.us.cloud-object-storage.appdomain.cloud/logs-router-agent-rhel8-1.3.1.rpm.sha256 +wget https://logs-router-agent-install-packages.s3.us.cloud-object-storage.appdomain.cloud/logs-router-agent-rhel8-1.3.1.rpm +sha256sum -c logs-router-agent-rhel8-1.3.1.rpm.sha256 +rpm -ivh logs-router-agent-rhel8-1.3.1.rpm +rpm -qa | grep logs-router-agent +wget -O /root/post-config.sh https://logs-router-agent-config.s3.us.cloud-object-storage.appdomain.cloud/post-config.sh +ls -a /root +echo "Cloud logs agent installated" + # Security approach to delete unwanted ssh keys and host file entries rm -rf "${LSF_PACKAGES_PATH}" if grep -q 'ID="rhel"' /etc/os-release || grep -q 'ID="rocky"' /etc/os-release; then diff --git a/tools/image-builder/template_files.tf b/tools/image-builder/template_files.tf index c112055c..7c950a12 100644 --- a/tools/image-builder/template_files.tf +++ b/tools/image-builder/template_files.tf @@ -14,7 +14,6 @@ data "template_file" "packer_user_data" { target_dir = "/var" prefix = var.prefix cluster_name = var.cluster_name - reservation_id = var.reservation_id catalog_validate_ssh_key = var.ssh_keys[0] zones = join(",", var.zones) existing_resource_group = var.existing_resource_group diff --git a/tools/image-builder/templates/packer_user_data.tpl b/tools/image-builder/templates/packer_user_data.tpl index cd0e8bd0..e0b4fc64 100644 --- a/tools/image-builder/templates/packer_user_data.tpl +++ b/tools/image-builder/templates/packer_user_data.tpl @@ -125,16 +125,3 @@ if [ ! -d "$(pwd)/go" ]; then echo "export GOROOT=$(pwd)/go" >> ~/.bashrc source ~/.bashrc fi - -echo "========== Executing Go function to validate the image through HPC deployment =========" -export TF_VAR_ibmcloud_api_key=${ibm_api_key} - -if [ "${private_catalog_id}" ]; then - PREFIX=${prefix} CLUSTER_NAME=${cluster_name} RESERVATION_ID=${reservation_id} SSH_FILE_PATH="/HPCaaS/artifacts/.ssh/id_rsa" REMOTE_ALLOWED_IPS=$PACKER_FIP SSH_KEYS=$CICD_SSH_KEY CATALOG_VALIDATE_SSH_KEY=${catalog_validate_ssh_key} ZONES=${zones} EXISTING_RESOURCE_GROUP=${existing_resource_group} COMPUTE_IMAGE_NAME=${image_name} PRIVATE_CATALOG_ID=${private_catalog_id} VPC_ID=${vpc_id} SUBNET_ID=${vpc_subnet_id} SOURCE_IMAGE_NAME=${source_image_name} go test -v -timeout 900m -parallel 4 -run "TestRunHpcDeploymentForCustomImageBuilder" | tee hpc_log_$(date +%d-%m-%Y-%H-%M-%S).log -else - PREFIX=${prefix} CLUSTER_NAME=${cluster_name} RESERVATION_ID=${reservation_id} SSH_FILE_PATH="/HPCaaS/artifacts/.ssh/id_rsa" REMOTE_ALLOWED_IPS=$PACKER_FIP SSH_KEYS=$CICD_SSH_KEY ZONES=${zones} EXISTING_RESOURCE_GROUP=${existing_resource_group} COMPUTE_IMAGE_NAME=${image_name} SOURCE_IMAGE_NAME=${source_image_name} go test -v -timeout 900m -parallel 4 -run "TestRunHpcDeploymentForCustomImageBuilder" | tee hpc_log_$(date +%d-%m-%Y-%H-%M-%S).log -fi - -echo "========== Deleting the SSH key =========" - -ibmcloud is key-delete $CICD_SSH_KEY -f diff --git a/tools/image-builder/variables.tf b/tools/image-builder/variables.tf index e044ede0..d8d14505 100644 --- a/tools/image-builder/variables.tf +++ b/tools/image-builder/variables.tf @@ -21,7 +21,7 @@ variable "existing_resource_group" { type = string default = "Default" validation { - condition = var.resource_group != null + condition = var.existing_resource_group != null error_message = "If you want to provide null for resource_group variable, it should be within double quotes." } } @@ -197,17 +197,6 @@ variable "cluster_name" { } } -# tflint-ignore: terraform_unused_declarations -variable "reservation_id" { - type = string - sensitive = true - description = "Ensure that you have received the reservation ID from IBM technical sales. Reservation ID is a unique identifier to distinguish different IBM Cloud HPC service agreements. It must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_)." - validation { - condition = can(regex("^[a-zA-Z][a-zA-Z0-9-_]*$", var.reservation_id)) - error_message = "Reservation ID must start with a letter and can only contain letters, numbers, hyphens (-), or underscores (_)." - } -} - # tflint-ignore: terraform_unused_declarations variable "private_catalog_id" { type = string diff --git a/tools/image-builder/version.tf b/tools/image-builder/version.tf index 0fa51187..cfef57e6 100644 --- a/tools/image-builder/version.tf +++ b/tools/image-builder/version.tf @@ -3,7 +3,7 @@ terraform { required_providers { ibm = { source = "IBM-Cloud/ibm" - version = "1.69.2" + version = ">= 1.77.0, < 2.0.0" } null = { source = "hashicorp/null"