From 813bfb1cfadf26b12ee480960165efcbeed1bad3 Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Tue, 26 Mar 2024 12:47:02 -0600 Subject: [PATCH 01/10] Create last_update_export.py This script is designed for use in a "disconnect" environment. The host server is to do daily exports based on last build of a channel. This script looks at the list of channels, gets the ChannelLastBuildByID and checks if the date is the current date or prior and then exports that channel. Logging is provided as well to show which channels were updated in that export. The target server can then rsync the `/mnt/export` directory and run another script to execute the inter-server-sync import of the exported data. --- uyuni-tools/last_update_export.py | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 uyuni-tools/last_update_export.py diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py new file mode 100644 index 0000000..362ce8e --- /dev/null +++ b/uyuni-tools/last_update_export.py @@ -0,0 +1,42 @@ +#! /usr/bin/python3 + +import os +import subprocess +import datetime +from xmlrpc.client import ServerProxy +import ssl +import socket + +SUMA_FQDN = socket.getfqdn() +MANAGER_LOGIN = "admin" +MANAGER_PASSWORD = "" + +MANAGER_URL = "https://" + SUMA_FQDN + "/rpc/api" + +context = ssl.create_default_context() +client = ServerProxy(MANAGER_URL, context=context) +key = client.auth.login(MANAGER_LOGIN, MANAGER_PASSWORD) + +today = datetime.date.today() +yesterday = today - datetime.timedelta(days=1) +log_file_path = f"/mnt/logs/{today}-daily_export.log" +output_dir = "/mnt/export/updates" + +subprocess.run(f"rm -rf {output_dir}/*", shell=True, check=True) + +channel_list = client.channel.listVendorChannels(key) + +for channel in channel_list: + build_date, channel_label = client.channel.software.getChannelLastBuildById(key, channel["id"]).split()[0], channel["label"] + build_date = datetime.datetime.strptime(build_date, "%Y-%m-%d").date() + + if build_date in [today, yesterday]: + channel_output_dir = os.path.join(output_dir, channel_label) + os.makedirs(channel_output_dir, exist_ok=True) + options = f"--outputDir='{channel_output_dir}' --orgLimit=2 --packagesOnlyAfter={yesterday}" + command = f"inter-server-sync export --channels='{channel_label}' {options}" + with open(log_file_path, "a") as log_file: + subprocess.run(command, shell=True, stdout=log_file, stderr=subprocess.STDOUT) + current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + completion_message = f"{current_time} Export for channel {channel_label} completed.\n" + log_file.write(completion_message) From 49361047a0af14d47cf5edd745431317400f2d2f Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Fri, 12 Apr 2024 10:41:37 -0600 Subject: [PATCH 02/10] Update last_update_export.py Updates from the source since uploading to uyuni-tools, multiple iterations to streamline have occured. Updates include fixes from comments and further use, which include... 1) script header 2) better path management and configurable parameters 3) use of shutil.rmtree avoiding extra calls to subprocess 4) configurable days for export 5) configurable user/group 6) added the use of mgr-sycn credentials file instead of using username/password in the script. --- uyuni-tools/last_update_export.py | 111 +++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 25 deletions(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index 362ce8e..3cf65b8 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -1,4 +1,29 @@ #! /usr/bin/python3 +""" +Script Name: Last Update Export + +Description: +This script automates the export of software channels using an XML-RPC client to interface with a server for use in Airgapped Environments or Disconnected SUMA. +It is designed to operate on a scheduled basis (daily), removing the previous day's export data, and writing new export logs into designated directories. +The concept behind the script is to use the API's to determine which channels have been updated within the defined dates, and export only those channels, as such, +this creates a more streamlined export/import for Disconnected or Airgapped systems. + +Instructions: +1. Ensure Python 3.x is installed on your system. +2. This script must be run with root privileges to manage file permissions and perform system-level operations. +3. Before running the script, update the `/root/.mgr-sync` configuration file with the correct manager login credentials using `mgr-sync -s refresh` to create the credentials file. +4. Customize the directory path, by default `/mnt` was chosen but this could be any location you want, ensure the location has ample free space. +5. Customize the 'rsync_user' and 'rsync_group' in the script to match the user and group names on your system. +6. Schedule this script using a cron job or another scheduler for daily execution. + +Intended Usage: +- Scheduled daily exports of software channel data. +- Logging of export operations in '/mnt/logs'. +- Exports are stored in '/mnt/export/updates'. +- This script is intended for systems administrators managing software channel updates for Airgapped Environments. + +Ensure the server and paths are correctly configured and accessible before running the script. +""" import os import subprocess @@ -6,37 +31,73 @@ from xmlrpc.client import ServerProxy import ssl import socket +import configparser +import shutil -SUMA_FQDN = socket.getfqdn() -MANAGER_LOGIN = "admin" -MANAGER_PASSWORD = "" +def setup_directories(base_path, output_path, log_path): + if not os.path.exists(base_path): + os.makedirs(base_path, exist_ok=True) + if not os.path.exists(log_path): + os.makedirs(log_path, exist_ok=True) + if os.path.exists(output_path): + shutil.rmtree(output_path) + os.makedirs(output_path, exist_ok=True) -MANAGER_URL = "https://" + SUMA_FQDN + "/rpc/api" +def setup_logging(log_dir, today): + log_file_path = os.path.join(log_dir, f"{today}-daily_export.log") + return log_file_path -context = ssl.create_default_context() -client = ServerProxy(MANAGER_URL, context=context) -key = client.auth.login(MANAGER_LOGIN, MANAGER_PASSWORD) +def create_client(): + config_path = os.path.expanduser('/root/.mgr-sync') + config = configparser.ConfigParser() + with open(config_path, 'r') as f: + config.read_string('[DEFAULT]\n' + f.read()) + MANAGER_LOGIN = config.get('DEFAULT', 'mgrsync.user') + MANAGER_PASSWORD = config.get('DEFAULT', 'mgrsync.password') + SUMA_FQDN = socket.getfqdn() + MANAGER_URL = f"https://{SUMA_FQDN}/rpc/api" + context = ssl.create_default_context() + client = ServerProxy(MANAGER_URL, context=context) + return client, client.auth.login(MANAGER_LOGIN, MANAGER_PASSWORD) +# Configuration Variables +base_dir = "/mnt" # Define base directory where the exports will be. +output_dir = os.path.join(base_dir, "export/updates") +log_dir = os.path.join(base_dir, "logs") today = datetime.date.today() -yesterday = today - datetime.timedelta(days=1) -log_file_path = f"/mnt/logs/{today}-daily_export.log" -output_dir = "/mnt/export/updates" +target_date = today - datetime.timedelta(days=1) # Define the number of days back for export, 1 day by default +rsync_user = "rsyncuser" # Define rsync user +rsync_group = "users" # Define rsync group -subprocess.run(f"rm -rf {output_dir}/*", shell=True, check=True) +# Setup Directories and Logging +setup_directories(base_dir, output_dir, log_dir) +log_file_path = setup_logging(log_dir, today) -channel_list = client.channel.listVendorChannels(key) +# Create XML-RPC Client +client, key = create_client() +# Process channels +channel_list = client.channel.listVendorChannels(key) for channel in channel_list: - build_date, channel_label = client.channel.software.getChannelLastBuildById(key, channel["id"]).split()[0], channel["label"] - build_date = datetime.datetime.strptime(build_date, "%Y-%m-%d").date() - - if build_date in [today, yesterday]: - channel_output_dir = os.path.join(output_dir, channel_label) - os.makedirs(channel_output_dir, exist_ok=True) - options = f"--outputDir='{channel_output_dir}' --orgLimit=2 --packagesOnlyAfter={yesterday}" - command = f"inter-server-sync export --channels='{channel_label}' {options}" - with open(log_file_path, "a") as log_file: - subprocess.run(command, shell=True, stdout=log_file, stderr=subprocess.STDOUT) - current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") - completion_message = f"{current_time} Export for channel {channel_label} completed.\n" - log_file.write(completion_message) + build_date, channel_label = client.channel.software.getChannelLastBuildById(key, channel["id"]).split()[0], channel["label"] + build_date = datetime.datetime.strptime(build_date, "%Y-%m-%d").date() + + if build_date in [today, target_date]: + channel_output_dir = os.path.join(output_dir, channel_label) + os.makedirs(channel_output_dir, exist_ok=True) + options_dict = { + "outputDir": channel_output_dir, + "orgLimit": "2", # Define the default Organization, 2 by default assuming there is only 1, if multiple set this to the one you assign for exporting. + "logLevel": "error", # Set as 'error' by default but change to 'debug' for detailed logging + "packagesOnlyAfter": target_date.strftime('%Y-%m-%d') + } + options = ' '.join([f"--{opt}='{val}'" for opt, val in options_dict.items()]) + command = f"inter-server-sync export --channels='{channel_label}' {options}" + with open(log_file_path, "a") as log_file: + subprocess.run(command, shell=True, stdout=log_file, stderr=subprocess.STDOUT, check=True) + current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + completion_message = f"{current_time} Export for channel {channel_label} completed.\n" + log_file.write(completion_message) + +# Change ownership of the output directory +subprocess.run(["chown", "-R", f"{rsync_user}:{rsync_group}", output_dir], check=True) From bea72fd4625fb086fbcd25ebcb2b98e37704404a Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Mon, 15 Apr 2024 11:44:15 -0600 Subject: [PATCH 03/10] Update last_update_export.py updated the constants and variables, and changed the order they appear in the script --- uyuni-tools/last_update_export.py | 52 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index 3cf65b8..48ffe23 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -13,7 +13,7 @@ 2. This script must be run with root privileges to manage file permissions and perform system-level operations. 3. Before running the script, update the `/root/.mgr-sync` configuration file with the correct manager login credentials using `mgr-sync -s refresh` to create the credentials file. 4. Customize the directory path, by default `/mnt` was chosen but this could be any location you want, ensure the location has ample free space. -5. Customize the 'rsync_user' and 'rsync_group' in the script to match the user and group names on your system. +5. Customize the 'RSYNC_USER' and 'RSYNC_GROUP' in the script to match the user and group names on your system. 6. Schedule this script using a cron job or another scheduler for daily execution. Intended Usage: @@ -34,6 +34,15 @@ import configparser import shutil +# Configuration Variables +BASE_DIR = "/mnt" # Define base directory where the exports will be. +OUTPUT_DIR = os.path.join(BASE_DIR, "export/updates") +LOG_DIR = os.path.join(BASE_DIR, "logs") +TODAY = datetime.date.TODAY() +TARGET_DATE = TODAY - datetime.timedelta(days=1) # Define the number of days back for export, 1 day by default +RSYNC_USER = "rsyncuser" # Define rsync user +RSYNC_GROUP = "users" # Define rsync group + def setup_directories(base_path, output_path, log_path): if not os.path.exists(base_path): os.makedirs(base_path, exist_ok=True) @@ -43,8 +52,8 @@ def setup_directories(base_path, output_path, log_path): shutil.rmtree(output_path) os.makedirs(output_path, exist_ok=True) -def setup_logging(log_dir, today): - log_file_path = os.path.join(log_dir, f"{today}-daily_export.log") +def setup_logging(LOG_DIR, TODAY): + log_file_path = os.path.join(LOG_DIR, f"{TODAY}-daily_export.log") return log_file_path def create_client(): @@ -52,26 +61,17 @@ def create_client(): config = configparser.ConfigParser() with open(config_path, 'r') as f: config.read_string('[DEFAULT]\n' + f.read()) - MANAGER_LOGIN = config.get('DEFAULT', 'mgrsync.user') - MANAGER_PASSWORD = config.get('DEFAULT', 'mgrsync.password') - SUMA_FQDN = socket.getfqdn() - MANAGER_URL = f"https://{SUMA_FQDN}/rpc/api" + manager_login = config.get('DEFAULT', 'mgrsync.user') + manager_password = config.get('DEFAULT', 'mgrsync.password') + suma_fqdn = socket.getfqdn() + manager_url = f"https://{suma_fqdn}/rpc/api" context = ssl.create_default_context() - client = ServerProxy(MANAGER_URL, context=context) - return client, client.auth.login(MANAGER_LOGIN, MANAGER_PASSWORD) - -# Configuration Variables -base_dir = "/mnt" # Define base directory where the exports will be. -output_dir = os.path.join(base_dir, "export/updates") -log_dir = os.path.join(base_dir, "logs") -today = datetime.date.today() -target_date = today - datetime.timedelta(days=1) # Define the number of days back for export, 1 day by default -rsync_user = "rsyncuser" # Define rsync user -rsync_group = "users" # Define rsync group + client = ServerProxy(manager_url, context=context) + return client, client.auth.login(manager_login, manager_password) # Setup Directories and Logging -setup_directories(base_dir, output_dir, log_dir) -log_file_path = setup_logging(log_dir, today) +setup_directories(BASE_DIR, OUTPUT_DIR, LOG_DIR) +log_file_path = setup_logging(LOG_DIR, TODAY) # Create XML-RPC Client client, key = create_client() @@ -82,14 +82,14 @@ def create_client(): build_date, channel_label = client.channel.software.getChannelLastBuildById(key, channel["id"]).split()[0], channel["label"] build_date = datetime.datetime.strptime(build_date, "%Y-%m-%d").date() - if build_date in [today, target_date]: - channel_output_dir = os.path.join(output_dir, channel_label) - os.makedirs(channel_output_dir, exist_ok=True) + if build_date in [TODAY, TARGET_DATE]: + channel_OUTPUT_DIR = os.path.join(OUTPUT_DIR, channel_label) + os.makedirs(channel_OUTPUT_DIR, exist_ok=True) options_dict = { - "outputDir": channel_output_dir, + "outputDir": channel_OUTPUT_DIR, "orgLimit": "2", # Define the default Organization, 2 by default assuming there is only 1, if multiple set this to the one you assign for exporting. "logLevel": "error", # Set as 'error' by default but change to 'debug' for detailed logging - "packagesOnlyAfter": target_date.strftime('%Y-%m-%d') + "packagesOnlyAfter": TARGET_DATE.strftime('%Y-%m-%d') } options = ' '.join([f"--{opt}='{val}'" for opt, val in options_dict.items()]) command = f"inter-server-sync export --channels='{channel_label}' {options}" @@ -100,4 +100,4 @@ def create_client(): log_file.write(completion_message) # Change ownership of the output directory -subprocess.run(["chown", "-R", f"{rsync_user}:{rsync_group}", output_dir], check=True) +subprocess.run(["chown", "-R", f"{RSYNC_USER}:{RSYNC_GROUP}", OUTPUT_DIR], check=True) From cb0cd4754abb57f712a608da8c1d9cb869733603 Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Mon, 15 Apr 2024 11:50:56 -0600 Subject: [PATCH 04/10] Update last_update_export.py caught the datetime.date.today() --- uyuni-tools/last_update_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index 48ffe23..e2b2c71 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -38,7 +38,7 @@ BASE_DIR = "/mnt" # Define base directory where the exports will be. OUTPUT_DIR = os.path.join(BASE_DIR, "export/updates") LOG_DIR = os.path.join(BASE_DIR, "logs") -TODAY = datetime.date.TODAY() +TODAY = datetime.date.today() TARGET_DATE = TODAY - datetime.timedelta(days=1) # Define the number of days back for export, 1 day by default RSYNC_USER = "rsyncuser" # Define rsync user RSYNC_GROUP = "users" # Define rsync group From 8279d5670226ec447c156658ab4b32b041d7b1bd Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Fri, 19 Apr 2024 07:09:19 -0600 Subject: [PATCH 05/10] Update uyuni-tools/last_update_export.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Pablo Suárez Hernández --- uyuni-tools/last_update_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index e2b2c71..7ead93c 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -82,7 +82,7 @@ def create_client(): build_date, channel_label = client.channel.software.getChannelLastBuildById(key, channel["id"]).split()[0], channel["label"] build_date = datetime.datetime.strptime(build_date, "%Y-%m-%d").date() - if build_date in [TODAY, TARGET_DATE]: + if TARGET_DATE <= build_date <= TODAY: channel_OUTPUT_DIR = os.path.join(OUTPUT_DIR, channel_label) os.makedirs(channel_OUTPUT_DIR, exist_ok=True) options_dict = { From 63cf30f19e1f977e4796e14832590236ad6bf79c Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Fri, 19 Apr 2024 07:19:08 -0600 Subject: [PATCH 06/10] Update last_update_export.py removed the shell=True from the subprocess.run command, missed that from the last commit after updating the command --- uyuni-tools/last_update_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index 7ead93c..c82d784 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -94,7 +94,7 @@ def create_client(): options = ' '.join([f"--{opt}='{val}'" for opt, val in options_dict.items()]) command = f"inter-server-sync export --channels='{channel_label}' {options}" with open(log_file_path, "a") as log_file: - subprocess.run(command, shell=True, stdout=log_file, stderr=subprocess.STDOUT, check=True) + subprocess.run(command, stdout=log_file, stderr=subprocess.STDOUT, check=True) current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") completion_message = f"{current_time} Export for channel {channel_label} completed.\n" log_file.write(completion_message) From d55f838bd081524b66f324dfc975117a2e84a287 Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Fri, 19 Apr 2024 08:28:58 -0600 Subject: [PATCH 07/10] Fixed the shebang line Fixed the shebang line --- uyuni-tools/last_update_export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index c82d784..0a9df53 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -1,4 +1,4 @@ -#! /usr/bin/python3 +#!/usr/bin/env python3 """ Script Name: Last Update Export From 366949d8c7c15380883de41e6e9d4ae49a3d5126 Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Mon, 22 Apr 2024 07:22:21 -0600 Subject: [PATCH 08/10] Update last_update_export.py fixed the command execution --- uyuni-tools/last_update_export.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index 0a9df53..956a48c 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -33,6 +33,7 @@ import socket import configparser import shutil +import shlex # Configuration Variables BASE_DIR = "/mnt" # Define base directory where the exports will be. @@ -93,8 +94,9 @@ def create_client(): } options = ' '.join([f"--{opt}='{val}'" for opt, val in options_dict.items()]) command = f"inter-server-sync export --channels='{channel_label}' {options}" + command_args = shlex.split(command) with open(log_file_path, "a") as log_file: - subprocess.run(command, stdout=log_file, stderr=subprocess.STDOUT, check=True) + subprocess.run(command_args, stdout=log_file, stderr=subprocess.STDOUT, check=True) current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") completion_message = f"{current_time} Export for channel {channel_label} completed.\n" log_file.write(completion_message) From f70f8b959fd42265fa1dcb980eb7bac0457fed29 Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Mon, 22 Apr 2024 07:27:40 -0600 Subject: [PATCH 09/10] Update last_update_export.py script header updated script header to be better formatted and easier to read. --- uyuni-tools/last_update_export.py | 45 +++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/uyuni-tools/last_update_export.py b/uyuni-tools/last_update_export.py index 956a48c..e1149d2 100644 --- a/uyuni-tools/last_update_export.py +++ b/uyuni-tools/last_update_export.py @@ -1,26 +1,43 @@ #!/usr/bin/env python3 + """ Script Name: Last Update Export Description: -This script automates the export of software channels using an XML-RPC client to interface with a server for use in Airgapped Environments or Disconnected SUMA. -It is designed to operate on a scheduled basis (daily), removing the previous day's export data, and writing new export logs into designated directories. -The concept behind the script is to use the API's to determine which channels have been updated within the defined dates, and export only those channels, as such, -this creates a more streamlined export/import for Disconnected or Airgapped systems. + This script automates the export of software channels using an XML-RPC + client to interface with a server for use in Airgapped Environments or + Disconnected SUMA. + + It is designed to operate on a scheduled basis (daily), removing the + previous day's export data, and writing new export logs into designated + directories. + + The concept behind the script is to use the API's to determine which + channels have been updated within the defined dates, and export only + those channels, as such, this creates a more streamlined export/import + for Disconnected or Airgapped systems. Instructions: -1. Ensure Python 3.x is installed on your system. -2. This script must be run with root privileges to manage file permissions and perform system-level operations. -3. Before running the script, update the `/root/.mgr-sync` configuration file with the correct manager login credentials using `mgr-sync -s refresh` to create the credentials file. -4. Customize the directory path, by default `/mnt` was chosen but this could be any location you want, ensure the location has ample free space. -5. Customize the 'RSYNC_USER' and 'RSYNC_GROUP' in the script to match the user and group names on your system. -6. Schedule this script using a cron job or another scheduler for daily execution. + 1. Ensure Python 3.x is installed on your system. + 2. This script must be run with root privileges to manage file + permissions and perform system-level operations. + 3. Before running the script, update the `/root/.mgr-sync` configuration + file with the correct manager login credentials using + `mgr-sync -s refresh` to create the credentials file. + 4. Customize the directory path, by default `/mnt` was chosen but this + could be any location you want, ensure the location has ample free + space. + 5. Customize the 'RSYNC_USER' and 'RSYNC_GROUP' in the script to match + the user and group names on your system. + 6. Schedule this script using a cron job or another scheduler for daily + execution. Intended Usage: -- Scheduled daily exports of software channel data. -- Logging of export operations in '/mnt/logs'. -- Exports are stored in '/mnt/export/updates'. -- This script is intended for systems administrators managing software channel updates for Airgapped Environments. + - Scheduled daily exports of software channel data. + - Logging of export operations in '/mnt/logs'. + - Exports are stored in '/mnt/export/updates'. + - This script is intended for systems administrators managing software + channel updates for Airgapped Environments. Ensure the server and paths are correctly configured and accessible before running the script. """ From 5b3e496b126d876d41c64aa48aaf14f4d02f28ae Mon Sep 17 00:00:00 2001 From: Puppet Pirate Date: Tue, 23 Apr 2024 22:20:09 -0600 Subject: [PATCH 10/10] Create initial_export.py This file is part of the last_update_export.py script put forward. This file is for the initial export using the ISSv2 used in a disconnected or air-gapped environment, to do a full export instead of an incremental export as the last_update_export.py, it can also be used to add a new product channel to an air-gapped system. --- uyuni-tools/initial_export.py | 123 ++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 uyuni-tools/initial_export.py diff --git a/uyuni-tools/initial_export.py b/uyuni-tools/initial_export.py new file mode 100644 index 0000000..20c06ad --- /dev/null +++ b/uyuni-tools/initial_export.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 + +""" +Description: This script is used for the initial export of channels for the + customer. This means if they are a new to SUMA, or an existing + user and we don't know when they got their last update from the + SCC, or they are getting a new channel for i.e. a new Service Pack. + + This script automates the process of exporting channel data from a + specified directory, clearing the directory before use, and + handling file and user permissions. It reads channel names from a + specified text file and executes a sync/export command for each. + + The script logs each export operation to a daily log file and + changes the ownership of the exported files to a specific user and + group. + +Constants: + BASE_DIR - Base directory from which channels are exported. + RSYNC_USER - Username owning the exported files. + RSYNC_GROUP - Group owning the exported files. + LOG_DIR - Directory where logs are stored. + CHANNELS_FILE - Path to the file containing channel names, one + channel per line. + TODAY - Today's date, used for naming the log file. + +Instructions: + 1. THIS SCRIPT REQUIRES THE USE OF THE `mgr-sync -s refresh` + CREDENTIALS FILE! + 2. Ensure paths for BASE_DIR, LOG_DIR, and CHANNELS_FILE are + correctly set according to your system configuration. + 3. Adjust RSYNC_USER and RSYNC_GROUP to match appropriate user + and group on your system. + 4. Ensure 'channels.txt' contains the list of channels to be + exported, one per line. + 5. Run this script with sufficient permissions to access and modify + the specified directories and files. +""" + +import os +import shutil +import datetime +import subprocess +import configparser +import ssl +import socket +from xmlrpc.client import ServerProxy +import shlex + +# Constants +BASE_DIR = "/mnt" +OUTPUT_DIR = os.path.join(BASE_DIR, "export/initial") +LOG_DIR = os.path.join(BASE_DIR, "logs") +SCRIPTS = os.path.join(BASE_DIR, "scripts") +RSYNC_USER = "rsyncuser" +RSYNC_GROUP = "users" +CHANNELS_FILE = os.path.join(SCRIPTS, "channels.txt") +TODAY = datetime.date.today().strftime("%Y-%m-%d") + +def create_client(): + config_path = os.path.expanduser('/root/.mgr-sync') + config = configparser.ConfigParser() + with open(config_path, 'r') as f: + config.read_string('[DEFAULT]\n' + f.read()) + manager_login = config.get('DEFAULT', 'mgrsync.user') + manager_password = config.get('DEFAULT', 'mgrsync.password') + suma_fqdn = socket.getfqdn() + manager_url = f"https://{suma_fqdn}/rpc/api" + context = ssl.create_default_context() + client = ServerProxy(manager_url, context=context) + key = client.auth.login(manager_login, manager_password) + return client, key + +def setup_directories(): + os.makedirs(LOG_DIR, exist_ok=True) + if os.path.exists(OUTPUT_DIR): + shutil.rmtree(OUTPUT_DIR) + os.makedirs(OUTPUT_DIR, exist_ok=True) + +def setup_logging(): + log_file_path = os.path.join(LOG_DIR, f"{TODAY}-initial-export.log") + return log_file_path + +def get_channels(): + """ + Check if channels.txt exists and is not empty, or use API call. + """ + if os.path.exists(CHANNELS_FILE) and os.path.getsize(CHANNELS_FILE) > 0: + with open(CHANNELS_FILE, 'r') as file: + channels = [line.strip() for line in file if line.strip()] + else: + client, key = create_client() + channel_data = client.channel.listVendorChannels(key) + channels = [channel['label'] for channel in channel_data if 'label' in channel] + return channels + +# Initialize directories and logging +setup_directories() +log_file_path = setup_logging() + +# Initialized channels list +channels = get_channels() + +# Process channel exports +for channel in channels: + product_dir = os.path.join(OUTPUT_DIR, channel) + os.makedirs(product_dir, exist_ok=True) + options_dict = { + "outputDir": product_dir, + "logLevel": "error", + "orgLimit": "2" + } + options = ' '.join([f"--{opt}='{val}'" for opt, val in options_dict.items()]) + command = f"inter-server-sync export --channels='{channel}' {options}" + command_args = shlex.split(command) + with open(log_file_path, "a") as log_file: + subprocess.run(command_args, stdout=log_file, stderr=subprocess.STDOUT) + current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + completion_message = f"{current_time} Export for channel {channel} completed.\n" + log_file.write(completion_message) + +# Change ownership of the base directory recursively +subprocess.run(['chown', '-R', f'{RSYNC_USER}.{RSYNC_GROUP}', BASE_DIR], check=True)