diff --git a/cli/cmd/launch.py b/cli/cmd/launch.py index aa39b35..0b0f767 100644 --- a/cli/cmd/launch.py +++ b/cli/cmd/launch.py @@ -9,6 +9,7 @@ import cli.partition.partition as partition from cli.storage.storage_exception import StorageError import cli.storage.storage as storage +import storage.virtual_storage as vstorage import cli.storage.vopt_storage as vopt import cli.utils.command_util as command_util import cli.utils.common as common @@ -124,74 +125,89 @@ def _launch(config, cookies, sys_uuid, vios_uuids): logger.info("Installation ISOs attached to the partition") logger.info("Setting up storage") - # Re-run scenario: Check if physical disk is already attached + setup_storage(config, cookies, active_vios_servers, sys_uuid, partition_uuid) + + lpar_state = activation.check_lpar_status(config, cookies, partition_uuid) + if lpar_state != "running": + logger.debug("Setting partition bootstring as 'cd/dvd-all'") + partition_payload = partition.get_partition_details( + config, cookies, sys_uuid, partition_uuid) + partition.set_partition_boot_string( + config, cookies, sys_uuid, partition_uuid, partition_payload, "cd/dvd-all") + + logger.info("Activating the partition") + activation.activate_partition(config, cookies, partition_uuid) + logger.info("Partition activated") + + logger.info("Monitoring boot process, this will take a while") + monitor_util.monitor_bootstrap_boot(config) + monitor_util.monitor_pim(config) + except (AIAppError, AuthError, NetworkError, PartitionError, StorageError, VIOSError, paramiko.SSHException, Exception) as e: + raise e + +def setup_storage(config, cookies, active_vios, sys_uuid, lpar_id): + try: + # Re-run scenario: Check if physical disk/virtualdisk/SAN storage(VFC) is already attached to partition storage_attached = False - for a_vios_uuid, a_vios in active_vios_servers.items(): + for a_vios_uuid, a_vios in active_vios.items(): logger.debug(f"Checking for existing physical disk attachment in VIOS '{a_vios_uuid}'") physical_disk_found, _ = storage.check_if_storage_attached( - a_vios, partition_uuid) + a_vios, lpar_id) if physical_disk_found: logger.info( - f"Physical disk is already attached to lpar '{partition_uuid}'") + f"Physical disk is already attached to lpar '{lpar_id}'") storage_attached = True + if not storage_attached: + vdisk_found, vdisk_name = vstorage.check_if_vdisk_attached(a_vios, lpar_id) + if vdisk_found: + logger.info(f"Virtual disk '{vdisk_name}' is already attached to lpar '{lpar_id}'") + storage_attached = True if not storage_attached: vfc_disk_found, _ = storage.check_if_vfc_disk_attached( - a_vios, partition_uuid) + a_vios, lpar_id) if vfc_disk_found: logger.info( - f"SAN storage(VFC) disk is already attached to lpar '{partition_uuid}'") + f"SAN storage(VFC) disk is already attached to lpar '{lpar_id}'") storage_attached = True - # Enable below code block when virtual disk support is added - ''' - use_vdisk = util.use_virtual_disk(config) - if use_vdisk: - vios_storage_uuid = vios_storage_list[0][0] - updated_vios_payload = get_vios_details(config, cookies, sys_uuid, vios_storage_uuid) - use_existing_vd = util.use_existing_vd(config) - if use_existing_vd: - vstorage.attach_virtualdisk(updated_vios_payload, config, cookies, partition_uuid, sys_uuid, vios_storage_uuid) - else: - # Create volume group, virtual disk and attach storage - use_existing_vg = util.use_existing_vg(config) - if not use_existing_vg: - # Create volume group - vg_id = vstorage.create_volumegroup(config, cookies, vios_storage_uuid) - else: - vg_id = get_volume_group(config, cookies, vios_storage_uuid, util.get_volume_group(config)) - vstorage.create_virtualdisk(config, cookies, vios_storage_uuid, vg_id) - time.sleep(60) - updated_vios_payload = get_vios_details(config, cookies, sys_uuid, vios_storage_uuid) - vstorage.attach_virtualdisk(updated_vios_payload, config, cookies, partition_uuid, sys_uuid, vios_storage_uuid) - diskname = util.get_virtual_disk_name(config) - ''' - - if not storage_attached: + vdisk_attached = False + if util.use_logical_volume(config) and not storage_attached: + handle_virtual_disk(config, cookies, active_vios, sys_uuid, lpar_id) + vdisk_attached = True + + if not storage_attached and not vdisk_attached: vios_storage_list = vios_operation.get_vios_with_physical_storage( - config, active_vios_servers) + config, active_vios) if len(vios_storage_list) == 0: logger.error( "failed to find physical volume for the partition") raise StorageError( "failed to find physical volume for the partition") storage.attach_physical_storage( - config, cookies, sys_uuid, partition_uuid, vios_storage_list) - - - lpar_state = activation.check_lpar_status(config, cookies, partition_uuid) - if lpar_state != "running": - logger.debug("Setting partition bootstring as 'cd/dvd-all'") - partition_payload = partition.get_partition_details( - config, cookies, sys_uuid, partition_uuid) - partition.set_partition_boot_string( - config, cookies, sys_uuid, partition_uuid, partition_payload, "cd/dvd-all") - - logger.info("Activating the partition") - activation.activate_partition(config, cookies, partition_uuid) - logger.info("Partition activated") + config, cookies, sys_uuid, lpar_id, vios_storage_list) + except (StorageError, VIOSError, Exception) as e: + raise e - logger.info("Monitoring boot process, this will take a while") - monitor_util.monitor_bootstrap_boot(config) - monitor_util.monitor_pim(config) - except (AIAppError, AuthError, NetworkError, PartitionError, StorageError, VIOSError, paramiko.SSHException, Exception) as e: +def handle_virtual_disk(config, cookies, active_vios, sys_uuid, lpar_id): + try: + vios_storage_list = vios_operation.get_vios_with_physical_storage( + config, active_vios) + if len(vios_storage_list) == 0: + logger.error("failed to find physical volume for the partition") + raise StorageError("failed to find physical volume for the partition") + vios_storage_uuid = vios_storage_list[0][0] + updated_vios_payload = vios_operation.get_vios_details(config, cookies, sys_uuid, vios_storage_uuid) + + # Check if virtual storage already exists under the given volumegroup + vg_id = vstorage.get_volume_group_id(config, cookies, vios_storage_uuid, util.get_volume_group_name(config)) + found, _, _ = vstorage.check_if_vdisk_exists(config, cookies, vios_storage_uuid, vg_id, util.get_virtual_disk_name(config)) + # Create virtualdisk only if it doesn't exist under volumegroup + if not found: + vstorage.create_virtualdisk(config, cookies, vios_storage_uuid, vg_id) + updated_vios_payload = vios_operation.get_vios_details(config, cookies, sys_uuid, vios_storage_uuid) + + vstorage.attach_virtualdisk(updated_vios_payload, config, cookies, lpar_id, sys_uuid, vios_storage_uuid) + except (StorageError, VIOSError, Exception) as e: raise e + logger.info(f"virtualdisk is attached to partition '{lpar_id}' successfully") + return diff --git a/cli/storage/virtual_storage.py b/cli/storage/virtual_storage.py index c04c3cc..e752e80 100644 --- a/cli/storage/virtual_storage.py +++ b/cli/storage/virtual_storage.py @@ -1,8 +1,11 @@ import requests from bs4 import BeautifulSoup +import cli.utils.common as common +from cli.storage.storage_exception import StorageError import cli.utils.string_util as util -import cli.storage.storage as storage + +logger = common.get_logger("virtual-storage") def get_vg_payload(physical_vol_name, volume_group): return f''' @@ -40,13 +43,13 @@ def get_vdisk_payload(config, vg_payload): payload = str(vg_soup) return payload -def get_vdisk_vios_payload(vios_payload, config, hmc_host, partition_uuid, system_uuid): +def get_vdisk_vios_payload(vios, config, partition_uuid, system_uuid): vdisk_mapping = f''' - + @@ -59,29 +62,35 @@ def get_vdisk_vios_payload(vios_payload, config, hmc_host, partition_uuid, syste ''' - vios_bs = BeautifulSoup(vios_payload, 'xml') + soup = BeautifulSoup(vios, 'xml') vdisk_bs = BeautifulSoup(vdisk_mapping, 'xml') - scsi_mappings = vios_bs.find('VirtualSCSIMappings') + scsi_mappings = soup.find('VirtualSCSIMappings') scsi_mappings.append(vdisk_bs) - payload = str(vios_bs) - return payload + return str(soup) -def create_volumegroup(config, cookies, vios_uuid): +def get_volume_group_id(config, cookies, vios_uuid, vg_name): uri = f"/rest/api/uom/VirtualIOServer/{vios_uuid}/VolumeGroup" hmc_host = util.get_host_address(config) url = "https://" + hmc_host + uri - physical_vol_name = util.get_physical_volume_name(config) - vg_name = util.get_volume_group(config) - payload = get_vg_payload(physical_vol_name, vg_name) headers = {"x-api-key": util.get_session_key(config), "Content-Type": "application/vnd.ibm.powervm.uom+xml; type=VolumeGroup"} - response = requests.put(url, headers=headers, cookies=cookies, data=payload, verify=False) - - if response.status_code != 200: - print("Failed to create volume group", response.text) - exit() + try: + response = requests.get(url, headers=headers, cookies=cookies, verify=False) + if response.status_code != 200: + logger.error(f"failed to get volume group: '{response.text}'") + raise Exception("failed to get volume group") + except Exception as e: + raise e soup = BeautifulSoup(response.text, 'xml') - vg_id = soup.find('id').text + entries = soup.find_all("entry") + vg_id = None + for entry in entries: + vol_group = entry.find("VolumeGroup") + if vol_group.find("GroupName").text == vg_name: + vg_id = entry.find("id").text + if vg_id is None: + logger.error(f"failed to find volumegroup id corresponding to volumegroup name '{vg_name}'") + raise StorageError(f"failed to find volumegroup id corresponding to volumegroup name '{vg_name}'") return vg_id def get_volume_group_details(config, cookies, vios_uuid, vg_id): @@ -89,15 +98,75 @@ def get_volume_group_details(config, cookies, vios_uuid, vg_id): hmc_host = util.get_host_address(config) url = "https://" + hmc_host + uri headers = {"x-api-key": util.get_session_key(config), "Content-Type": "application/vnd.ibm.powervm.uom+xml; type=VolumeGroup"} - response = requests.get(url, headers=headers, cookies=cookies, verify=False) - - if response.status_code != 200: - print("Failed to create volume group", response.text) - exit() + try: + response = requests.get(url, headers=headers, cookies=cookies, verify=False) + if response.status_code != 200: + logger.error(f"failed to get volume group details: '{response.text}'") + raise Exception("failed to get volume group details") + except Exception as e: + raise e soup = BeautifulSoup(response.text, 'xml') vol_group_details = str(soup.find("VolumeGroup")) return vol_group_details +def check_if_vdisk_attached(vios, partition_uuid): + found = False + vdisk = "" + if partition_uuid == "": + return found, vdisk + + try: + soup = BeautifulSoup(vios, 'xml') + scsi_mappings = soup.find_all('VirtualSCSIMapping') + # Iterate over all SCSI mappings and look for Storage followed by VirtualDisk XML tags + for scsi in scsi_mappings: + lpar_link = scsi.find("AssociatedLogicalPartition") + if lpar_link is not None and partition_uuid in lpar_link.attrs["href"]: + storage = scsi.find("Storage") + if storage is not None: + logical_vol = storage.find("VirtualDisk") + if logical_vol is not None: + logger.debug( + f"Found virtual disk SCSI mapping for partition '{partition_uuid}' in VIOS") + found = True + vdisk = logical_vol.find("DiskName").text + break + except Exception as e: + logger.error( + "failed to check if storage SCSI mapping is present in VIOS") + raise e + return found, vdisk + +# Checks if virtualdisk is created under a given volumegroup +def check_if_vdisk_exists(config, cookies, vios_uuid, vg_id, vdisk): + try: + url = f"https://{util.get_host_address(config)}/rest/api/uom/VirtualIOServer/{vios_uuid}/VolumeGroup/{vg_id}" + headers = {"x-api-key": util.get_session_key( + config), "Content-Type": "application/vnd.ibm.powervm.uom+xml; type=VolumeGroup"} + response = requests.get(url, headers=headers, + cookies=cookies, verify=False) + if response.status_code != 200: + logger.error( + f"failed to get volumegroup details, error: '{response.text}'") + raise Exception( + f"failed to get volumegroup details, error: '{response.text}'") + soup = BeautifulSoup(response.text, 'xml') + + # remove virtual disk + volume_group = soup.find("VolumeGroup") + vdisks = soup.find_all("VirtualDisk") + present = False + virt_disk_xml = None + for disk in vdisks: + disk_name = disk.find("DiskName") + if disk_name is not None and disk_name.text == vdisk: + present = True + virt_disk_xml = disk + break + except Exception as e: + raise e + return present, virt_disk_xml, volume_group + def create_virtualdisk(config, cookies, vios_uuid, vg_id): # Get Volume group details vg_details = get_volume_group_details(config, cookies, vios_uuid, vg_id) @@ -106,24 +175,26 @@ def create_virtualdisk(config, cookies, vios_uuid, vg_id): hmc_host = util.get_host_address(config) url = "https://" + hmc_host + uri headers = {"x-api-key": util.get_session_key(config), "Content-Type": "application/vnd.ibm.powervm.uom+xml; type=VolumeGroup"} - payload = get_vdisk_payload(config, vg_details) - response = requests.post(url, headers=headers, cookies=cookies, data=payload, verify=False) - if response.status_code != 200: - print("Failed to create virtual disk", response.text) - exit() - - print("Successfully created virtual disk") + try: + payload = get_vdisk_payload(config, vg_details) + response = requests.post(url, headers=headers, cookies=cookies, data=payload, verify=False) + if response.status_code != 200: + logger.error(f"failed to create virtual disk: '{response.text}'") + raise StorageError(f"failed to create virtual disk") + except Exception as e: + raise e + logger.info("Successfully created virtual disk") def attach_virtualdisk(vios_payload, config, cookies, partition_uuid, system_uuid, vios_uuid): uri = f"/rest/api/uom/ManagedSystem/{system_uuid}/VirtualIOServer/{vios_uuid}" - hmc_host = util.get_host_address(config) - url = "https://" + hmc_host + uri - payload = get_vdisk_vios_payload(vios_payload, config, hmc_host, partition_uuid, system_uuid) - headers = {"x-api-key": util.get_session_key(config), "Content-Type": storage.CONTENT_TYPE} - response = requests.post(url, headers=headers, cookies=cookies, data=payload, verify=False) - - if response.status_code != 200: - print("Failed to attach virtual storage to the partition ", response.text) - exit() - - print("Successfully attached virtual disk") + url = "https://" + util.get_host_address(config) + uri + payload = get_vdisk_vios_payload(vios_payload, config, partition_uuid, system_uuid) + headers = {"x-api-key": util.get_session_key(config), "Content-Type": "application/vnd.ibm.powervm.uom+xml; Type=VirtualIOServer"} + try: + response = requests.post(url, headers=headers, cookies=cookies, data=payload, verify=False) + if response.status_code != 200: + logger.error(f"failed to attach virtual storage to the partition: '{response.text}'") + raise StorageError(f"failed to attach virtual storage to the partition") + except Exception as e: + raise e + logger.info("Successfully attached virtual disk") diff --git a/cli/utils/command_util.py b/cli/utils/command_util.py index 2fe5fc3..df7ea75 100644 --- a/cli/utils/command_util.py +++ b/cli/utils/command_util.py @@ -3,12 +3,14 @@ from bs4 import BeautifulSoup import cli.auth.auth as auth +import cli.storage.storage as storage +import cli.storage.virtual_storage as vstorage import cli.utils.common as common import cli.utils.validator as validator import cli.utils.string_util as util import cli.utils.iso_util as iso_util import cli.vios.vios as vios_operation -import cli.storage.storage as storage + logger = common.get_logger("command-util") @@ -76,6 +78,35 @@ def remove_vopt_device(config, cookies, vios, vopt_name): raise e return + +def remove_virtual_disk(config, cookies, vios_uuid, vg_id, vdisk_name): + try: + found, vdisk, vg = vstorage.check_if_vdisk_exists(config, cookies, vios_uuid, vg_id, vdisk_name) + if not found: + logger.debug(f"No virtualdisk '{vdisk}' is found under volumegroup") + return + # drop virtualdisk xml block + vdisk.decompose() + + uri = f"/rest/api/uom/VirtualIOServer/{vios_uuid}/VolumeGroup/{vg_id}" + hmc_host = util.get_host_address(config) + url = "https://" + hmc_host + uri + headers = {"x-api-key": util.get_session_key( + config), "Content-Type": "application/vnd.ibm.powervm.uom+xml; type=VolumeGroup"} + response = requests.post(url, data=str(vg), + headers=headers, cookies=cookies, verify=False) + if response.status_code != 200: + logger.error( + f"failed to update volume group after deleting virtual disk, error: '{response.text}'") + return + + logger.debug( + f"Virtualdisk '{vdisk_name}' has been deleted successfully") + except Exception as e: + raise e + return + + def check_if_scsi_mapping_exist(partition_uuid, vios, media_dev_name): found = False vscsi = None diff --git a/cli/utils/string_util.py b/cli/utils/string_util.py index 582b186..0113333 100644 --- a/cli/utils/string_util.py +++ b/cli/utils/string_util.py @@ -112,23 +112,17 @@ def get_ssh_pub_key(config): return config["ssh"]["pub-key-file"] # storage related Getters -def get_physical_volume_name(config): - return config.get("STORAGE", "physical_volume_name").strip('"') - -def get_volume_group(config): - return config.get("STORAGE", "vg_name").strip('"') +def get_volume_group_name(config): + return config["virtual-disk"]["vg_name"] def get_virtual_disk_name(config): - return config.get("STORAGE", "vdisk_name").strip('"') - -def use_virtual_disk(config): - return False + return config["virtual-disk"]["vdisk_name"] -def use_existing_vd(config): - return config.getboolean("STORAGE", "use_existing_vd") +def use_logical_volume(config): + return config["virtual-disk"].as_bool("use_logical_volume") -def use_existing_vg(config): - return config.getboolean("STORAGE", "use_existing_vg") +def get_virtual_disk_size(config): + return config["virtual-disk"]["vdisk_size"] def get_required_disk_size(config): return config["partition"]["storage"]["size"] diff --git a/cli/vios/vios.py b/cli/vios/vios.py index 29d904d..9b522aa 100644 --- a/cli/vios/vios.py +++ b/cli/vios/vios.py @@ -4,6 +4,7 @@ from bs4 import BeautifulSoup import cli.storage.storage as storage +import cli.storage.virtual_storage as vstorage import cli.utils.common as common import cli.utils.command_util as command_util import cli.utils.string_util as util @@ -143,6 +144,36 @@ def get_volume_group(config, cookies, vios_uuid, vg_name): vg_id = vol_group.find("AtomID").text return vg_id +def cleanup_logical_volume(config, cookies, vios, vios_uuid, sys_uuid, partition_uuid): + vdisk_found, vdisk = vstorage.check_if_vdisk_attached(vios, partition_uuid) + if vdisk_found: + logger.info( + f"Removing SCSI mapping for virtual disk '{vdisk}'") + command_util.remove_scsi_mappings( + config, cookies, sys_uuid, partition_uuid, vios_uuid, vios, vdisk) + + # delete associated virtual disk + vg_id = get_volume_group(config, cookies, vios_uuid, util.get_volume_group_name(config)) + command_util.remove_virtual_disk(config, cookies, vios_uuid, vg_id, util.get_virtual_disk_name(config)) + logger.info(f"Deleted virtualdisk '{util.get_virtual_disk_name(config)}' associated to partition '{util.get_partition_name(config)}'") + return + +def cleanup_storage(config, cookies, vios, vios_uuid, sys_uuid, partition_uuid): + storage_cleaned = False + # remove SCSI mappings from VIOS + phys_disk_found, phys_disk = storage.check_if_storage_attached( + vios, partition_uuid) + if phys_disk_found and not storage_cleaned: + logger.info( + f"Removing SCSI mapping for physical disk '{phys_disk}'") + command_util.remove_scsi_mappings( + config, cookies, sys_uuid, partition_uuid, vios_uuid, vios, phys_disk) + storage_cleaned = True + # Check if attached disk is virtual disk + if not phys_disk_found and not storage_cleaned: + cleanup_logical_volume(config, cookies, vios, vios_uuid, sys_uuid, partition_uuid) + storage_cleaned = True + return storage_cleaned def cleanup_vios(config, cookies, sys_uuid, partition_uuid, vios_uuid_list): vopt_list = [util.get_bootstrap_iso( @@ -159,21 +190,11 @@ def cleanup_vios(config, cookies, sys_uuid, partition_uuid, vios_uuid_list): continue vios = get_vios_details(config, cookies, sys_uuid, vios_uuid) - # remove SCSI mappings from VIOS - found, phys_disk = storage.check_if_storage_attached( - vios, partition_uuid) - if found and not storage_cleaned: - logger.info( - f"Removing SCSI mapping for physical disk '{phys_disk}'") - command_util.remove_scsi_mappings( - config, cookies, sys_uuid, partition_uuid, vios_uuid, vios, phys_disk) - storage_cleaned = True - + storage_cleaned = cleanup_storage(config, cookies, vios, vios_uuid, sys_uuid, partition_uuid) vios = get_vios_details(config, cookies, sys_uuid, vios_uuid) logger.info(f"Removing SCSI mapping for vOPT device '{vopt}'") command_util.remove_scsi_mappings( config, cookies, sys_uuid, partition_uuid, vios_uuid, vios, vopt) - # TODO: delete virtual disk, volumegroup if created by the script during launch vios = get_vios_details(config, cookies, sys_uuid, vios_uuid) @@ -187,15 +208,8 @@ def cleanup_vios(config, cookies, sys_uuid, partition_uuid, vios_uuid_list): if not storage_cleaned: for vios_uuid in vios_uuid_list: if vios_uuid not in processed_vios_list: - vios = get_vios_details( - config, cookies, sys_uuid, vios_uuid) - found, phys_disk = storage.check_if_storage_attached( - vios, partition_uuid) - if found: - logger.debug( - f"Removing SCSI mapping for physical disk '{phys_disk}'") - command_util.remove_scsi_mappings( - config, cookies, sys_uuid, partition_uuid, vios_uuid, vios, phys_disk) + vios = get_vios_details(config, cookies, sys_uuid, vios_uuid) + _ = cleanup_storage(config, cookies, vios, vios_uuid, sys_uuid, partition_uuid) except Exception as e: logger.error(f"failed to clean up VIOS, error: {e}") diff --git a/config.ini b/config.ini index 0e7e570..284e069 100644 --- a/config.ini +++ b/config.ini @@ -79,6 +79,12 @@ max-memory = 32 desired-memory = 32 min-memory = 10 +[virtual-disk] + # set below attribute to 'true' and populate below parameters if virtual disk needs to be created and attached to PIM partition + use_logical_volume = false + vdisk_name = "" + vdisk_size = "120" + vg_name = "" [ssh] user-name = "pim" pub-key-file = ""