diff --git a/client/bindir/cloud-setup-management.in b/client/bindir/cloud-setup-management.in index 84c87ae2e442..b4fe76cc8d8a 100755 --- a/client/bindir/cloud-setup-management.in +++ b/client/bindir/cloud-setup-management.in @@ -36,6 +36,106 @@ from cloudutils.cloudException import CloudRuntimeException, CloudInternalExcept from cloudutils.globalEnv import globalEnv from cloudutils.serviceConfigServer import cloudManagementConfig from optparse import OptionParser +import urllib.request +import configparser +import hashlib + +SYSTEMVM_TEMPLATES_PATH = "/usr/share/cloudstack-management/templates/systemvm" +SYSTEMVM_TEMPLATES_METADATA_FILE = SYSTEMVM_TEMPLATES_PATH + "/metadata.ini" + +def verify_sha512_checksum(file_path, expected_checksum): + sha512 = hashlib.sha512() + try: + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + sha512.update(chunk) + return sha512.hexdigest().lower() == expected_checksum.lower() + except Exception as e: + print(f"Failed to verify checksum for {file_path}: {e}") + return False + +def download_file(url, dest_path, chunk_size=8 * 1024 * 1024): + """ + Downloads a file from the given URL to the specified destination path in chunks. + """ + try: + with urllib.request.urlopen(url) as response: + total_size = response.length if response.length else None + downloaded = 0 + try: + with open(dest_path, 'wb') as out_file: + while True: + chunk = response.read(chunk_size) + if not chunk: + break + out_file.write(chunk) + downloaded += len(chunk) + if total_size: + print(f"Downloaded {downloaded / (1024 * 1024):.2f}MB of {total_size / (1024 * 1024):.2f}MB", end='\r') + except PermissionError as pe: + print(f"Permission denied: {dest_path}") + raise + print(f"\nDownloaded file from {url} to {dest_path}") + except Exception as e: + print(f"Failed to download file: {e}") + raise + +def download_template_if_needed(template, url, filename, checksum): + dest_path = os.path.join(SYSTEMVM_TEMPLATES_PATH, filename) + if os.path.exists(dest_path): + if checksum and verify_sha512_checksum(dest_path, checksum): + print(f"{template} System VM template already exists at {dest_path} with valid checksum, skipping download.") + return + else: + print(f"{template} System VM template at {dest_path} has invalid or missing checksum, re-downloading...") + else: + print(f"Downloading {template} System VM template from {url} to {dest_path}...") + try: + download_file(url, dest_path) + #After download, verify checksum if provided + if checksum: + if verify_sha512_checksum(dest_path, checksum): + print(f"{template} System VM template downloaded and verified successfully.") + else: + print(f"ERROR: Checksum verification failed for {template} System VM template after download.") + except Exception as e: + print(f"ERROR: Failed to download {template} System VM template: {e}") + +def collect_template_metadata(selected_templates, options): + template_metadata_list = [] + if not os.path.exists(SYSTEMVM_TEMPLATES_METADATA_FILE): + print(f"ERROR: System VM templates metadata file not found at {SYSTEMVM_TEMPLATES_METADATA_FILE}, cannot download templates.") + sys.exit(1) + config = configparser.ConfigParser() + config.read(SYSTEMVM_TEMPLATES_METADATA_FILE) + template_repo_url = None + if options.systemvm_templates_repository: + if "default" in config and "downloadrepository" in config["default"]: + template_repo_url = config["default"]["downloadrepository"].strip() + if not template_repo_url: + print("ERROR: downloadrepository value is empty in metadata file, cannot use --systemvm-template-repository option.") + sys.exit(1) + for template in selected_templates: + if template in config: + url = config[template].get("downloadurl") + filename = config[template].get("filename") + checksum = config[template].get("checksum") + if url and filename: + if template_repo_url: + url = url.replace(template_repo_url, options.systemvm_templates_repository) + template_metadata_list.append({ + "template": template, + "url": url, + "filename": filename, + "checksum": checksum + }) + else: + print(f"ERROR: URL or filename not found for {template} System VM template in metadata.") + sys.exit(1) + else: + print(f"ERROR: No metadata found for {template} System VM template.") + sys.exit(1) + return template_metadata_list if __name__ == '__main__': initLoging("@MSLOGDIR@/setupManagement.log") @@ -45,6 +145,16 @@ if __name__ == '__main__': parser.add_option("--https", action="store_true", dest="https", help="Enable HTTPs connection of management server") parser.add_option("--tomcat7", action="store_true", dest="tomcat7", help="Depreciated option, don't use it") parser.add_option("--no-start", action="store_true", dest="nostart", help="Do not start management server after successful configuration") + parser.add_option( + "--systemvm-templates", + dest="systemvm_templates", + help="Specify System VM templates to download: all, kvm-aarch64, kvm-x86_64, xenserver, vmware or comma-separated list of hypervisor combinations (e.g., kvm-x86_64,xenserver). Default is kvm-x86_64.", + ) + parser.add_option( + "--systemvm-templates-repository", + dest="systemvm_templates_repository", + help="Specify the URL to download System VM templates from." + ) (options, args) = parser.parse_args() if options.https: glbEnv.svrMode = "HttpsServer" @@ -53,6 +163,34 @@ if __name__ == '__main__': if options.nostart: glbEnv.noStart = True + available_templates = ["kvm-aarch64", "kvm-x86_64", "xenserver", "vmware"] + templates_arg = options.systemvm_templates + + selected_templates = ["kvm-x86_64"] + if templates_arg: + templates_list = [t.strip().lower() for t in templates_arg.split(",")] + if "all" in templates_list: + if len(templates_list) > 1: + print("WARNING: 'all' specified for System VM templates, ignoring other specified templates.") + selected_templates = available_templates + else: + invalid_templates = [] + for t in templates_list: + if t in available_templates: + if t not in selected_templates: + selected_templates.append(t) + else: + if t not in invalid_templates: + invalid_templates.append(t) + if invalid_templates: + print(f"ERROR: Invalid System VM template names provided: {', '.join(invalid_templates)}") + sys.exit(1) + print(f"Selected systemvm templates to download: {', '.join(selected_templates) if selected_templates else 'None'}") + + template_metadata_list = [] + if selected_templates: + template_metadata_list = collect_template_metadata(selected_templates, options) + glbEnv.mode = "Server" print("Starting to configure CloudStack Management Server:") @@ -74,3 +212,6 @@ if __name__ == '__main__': syscfg.restore() except: pass + + for meta in template_metadata_list: + download_template_if_needed(meta["template"], meta["url"], meta["filename"], meta["checksum"]) diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 5958486b4dff..10452c9b6572 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -62,3 +62,8 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # Thread pool configuration #threads.min=10 #threads.max=500 + +# The URL prefix for the system VM templates repository. When downloading system VM templates, the server replaces the +# `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL +# will be for download. +# system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/ diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 9b1420f22c37..4a57b7bd820e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -51,6 +51,7 @@ import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreDao; import org.apache.cloudstack.storage.datastore.db.TemplateDataStoreVO; import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -108,7 +109,11 @@ public class SystemVmTemplateRegistration { private static Integer LINUX_12_ID = 363; private static final Integer SCRIPT_TIMEOUT = 1800000; private static final Integer LOCK_WAIT_TIMEOUT = 1200; + protected static final String TEMPLATE_DOWNLOAD_URL_KEY = "downloadurl"; + protected static final String TEMPLATES_DOWNLOAD_REPOSITORY_KEY = "downloadrepository"; + protected static final String TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY = "system.vm.templates.download.repository"; protected static final List DOWNLOADABLE_TEMPLATE_ARCH_TYPES = Arrays.asList( + CPU.CPUArch.amd64, CPU.CPUArch.arm64 ); @@ -820,6 +825,11 @@ public static String parseMetadataFile() { LOGGER.error(errMsg); throw new CloudRuntimeException(errMsg); } + Ini.Section defaultSection = ini.get("default"); + String defaultDownloadRepository = defaultSection.get(TEMPLATES_DOWNLOAD_REPOSITORY_KEY); + String customDownloadRepository = ServerPropertiesUtil.getProperty(TEMPLATES_CUSTOM_DOWNLOAD_REPOSITORY_KEY); + boolean updateCustomDownloadRepository = StringUtils.isNotBlank(customDownloadRepository) && + StringUtils.isNotBlank(defaultDownloadRepository); for (Pair hypervisorType : hypervisorList) { String key = getHypervisorArchKey(hypervisorType.first(), hypervisorType.second()); Ini.Section section = ini.get(key); @@ -828,16 +838,21 @@ public static String parseMetadataFile() { key, metadataFilePath); continue; } + String url = section.get(TEMPLATE_DOWNLOAD_URL_KEY); + if (StringUtils.isNotBlank(url) && updateCustomDownloadRepository) { + url = url.replaceFirst(defaultDownloadRepository.trim(), + customDownloadRepository.trim()); + LOGGER.debug("Updated download URL for {} using custom repository to {}", key, url); + } NewTemplateMap.put(key, new MetadataTemplateDetails( hypervisorType.first(), section.get("templatename"), section.get("filename"), - section.get("downloadurl"), + url, section.get("checksum"), hypervisorType.second(), section.get("guestos"))); } - Ini.Section defaultSection = ini.get("default"); return defaultSection.get("version").trim(); } diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index dceb8e07b07d..b9b0cbcf0011 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -344,7 +344,7 @@ public void testIsTemplateFileChecksumDifferent_mismatch() { @Test(expected = CloudRuntimeException.class) public void testValidateTemplates_metadataTemplateFailure() { List> list = new ArrayList<>(); - list.add(new Pair<>(Hypervisor.HypervisorType.KVM, CPU.CPUArch.amd64)); + list.add(new Pair<>(Hypervisor.HypervisorType.VMware, CPU.CPUArch.arm64)); systemVmTemplateRegistration.validateTemplates(list); } diff --git a/engine/schema/templateConfig.sh b/engine/schema/templateConfig.sh index d6d1809c24d4..dcf3ecd382a1 100755 --- a/engine/schema/templateConfig.sh +++ b/engine/schema/templateConfig.sh @@ -27,6 +27,7 @@ function getTemplateVersion() { export CS_VERSION="${subversion1}"."${subversion2}" export CS_MINOR_VERSION="${minorversion}" export VERSION="${CS_VERSION}.${CS_MINOR_VERSION}" + export CS_SYSTEMTEMPLATE_REPO="https://download.cloudstack.org/systemvm/" } function getGenericName() { @@ -63,7 +64,7 @@ function getChecksum() { function createMetadataFile() { local fileData=$(cat $SOURCEFILE) - echo -e "["default"]\nversion = $VERSION.${securityversion}\n" >> $METADATAFILE + echo -e "["default"]\nversion = $VERSION.${securityversion}\ndownloadrepository = $CS_SYSTEMTEMPLATE_REPO\n" >> $METADATAFILE for template in "${templates[@]}" do section="${template%%:*}" @@ -82,13 +83,21 @@ function createMetadataFile() { declare -a templates getTemplateVersion $1 -templates=( "kvm-x86_64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "kvm-aarch64:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-aarch64-kvm.qcow2.bz2" - "vmware:https://download.cloudstack.org/systemvm/${CS_VERSION}/systemvmtemplate-$VERSION-x86_64-vmware.ova" - "xenserver:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-xen.vhd.bz2" - "hyperv:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-hyperv.vhd.zip" - "lxc:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-kvm.qcow2.bz2" - "ovm3:https://download.cloudstack.org/systemvm/$CS_VERSION/systemvmtemplate-$VERSION-x86_64-ovm.raw.bz2" ) +declare -A template_specs=( + [kvm-x86_64]="x86_64-kvm.qcow2.bz2" + [kvm-aarch64]="aarch64-kvm.qcow2.bz2" + [vmware]="x86_64-vmware.ova" + [xenserver]="x86_64-xen.vhd.bz2" + [hyperv]="x86_64-hyperv.vhd.zip" + [lxc]="x86_64-kvm.qcow2.bz2" + [ovm3]="x86_64-ovm.raw.bz2" +) + +templates=() +for key in "${!template_specs[@]}"; do + url="${CS_SYSTEMTEMPLATE_REPO}/${CS_VERSION}/systemvmtemplate-$VERSION-${template_specs[$key]}" + templates+=("$key:$url") +done PARENTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/dist/systemvm-templates/" mkdir -p $PARENTPATH diff --git a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java new file mode 100644 index 000000000000..14d24dbb6410 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.server; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloud.utils.PropertiesUtil; + +public class ServerPropertiesUtil { + private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); + protected static final String PROPERTIES_FILE = "server.properties"; + protected static final AtomicReference propertiesRef = new AtomicReference<>(); + + public static String getProperty(String name) { + Properties props = propertiesRef.get(); + if (props != null) { + return props.getProperty(name); + } + File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); + if (propsFile == null) { + logger.error("{} file not found", PROPERTIES_FILE); + return null; + } + Properties tempProps = new Properties(); + try (FileInputStream is = new FileInputStream(propsFile)) { + tempProps.load(is); + } catch (IOException e) { + logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); + return null; + } + if (!propertiesRef.compareAndSet(null, tempProps)) { + tempProps = propertiesRef.get(); + } + return tempProps.getProperty(name); + } +} diff --git a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java new file mode 100644 index 000000000000..2eece43e47b1 --- /dev/null +++ b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.server; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Properties; + +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.PropertiesUtil; + +@RunWith(MockitoJUnitRunner.class) +public class ServerPropertiesUtilTest { + + @After + public void clearCache() { + ServerPropertiesUtil.propertiesRef.set(null); + } + + @Test + public void returnsPropertyValueWhenPropertiesAreLoaded() { + Properties mockProperties = mock(Properties.class); + when(mockProperties.getProperty("key")).thenReturn("value"); + ServerPropertiesUtil.propertiesRef.set(mockProperties); + String result = ServerPropertiesUtil.getProperty("key"); + assertEquals("value", result); + } + + @Test + public void returnsNullWhenPropertyDoesNotExist() { + Properties mockProperties = mock(Properties.class); + ServerPropertiesUtil.propertiesRef.set(mockProperties); + assertNull(ServerPropertiesUtil.getProperty("nonexistentKey")); + } + + @Test + public void loadsPropertiesFromFileWhenNotCached() throws Exception { + File tempFile = Files.createTempFile("server", ".properties").toFile(); + tempFile.deleteOnExit(); + Files.writeString(tempFile.toPath(), "key=value\n"); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(tempFile); + assertEquals("value", ServerPropertiesUtil.getProperty("key")); + } + } + + @Test + public void returnsNullWhenPropertiesFileNotFound() { + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(null); + assertNull(ServerPropertiesUtil.getProperty("key")); + } + } + + @Test + public void returnsNullWhenIOExceptionOccurs() throws IOException { + File tempFile = Files.createTempFile("bad", ".properties").toFile(); + tempFile.deleteOnExit(); + Files.writeString(tempFile.toPath(), "\u0000\u0000\u0000"); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(tempFile); + assertNull(ServerPropertiesUtil.getProperty("key")); + } + } +}