Merge branch 'develop' into develop-tag-update

Author: himani2411

CHANGELOG.md

@@ -12,6 +12,8 @@ CHANGELOG
 - Add support for p6-b300 instances for all OSs except AL2.
 - Add alarm on missing clustermgtd heartbeat.
 - Support updates of `Tags` during cluster-updates.
+- Add `LaunchTemplateOverrides` to cluster config to allow network interfaces to be customized by overriding the launch template of a compute resource.
+  - This overrides the parallelcluster default using a shallow merge.
 
 **BUG FIXES**
 - Add validation to block updates that change tag order. Blocking such change prevents update failures.

cli/src/pcluster/aws/ec2.py

@@ -683,3 +683,27 @@ def get_instance_type_and_reservation_type_from_capacity_reservation(
                 reservation_type = reservation.reservation_type()
 
         return instance_type, reservation_type
+
+    @AWSExceptionHandler.handle_client_exception
+    def describe_launch_template_version(self, launch_template_id: str, version: str):
+        """
+        Describe a launch template version and return its data.
+
+        Args:
+            launch_template_id: The launch template ID (e.g., 'lt-0abc123def456')
+            version: The version number or '$Default'/'$Latest'
+
+        Returns:
+            dict: The LaunchTemplateData from the specified version
+        """
+        response = self._client.describe_launch_template_versions(
+            LaunchTemplateId=launch_template_id,
+            Versions=[str(version)],
+        )
+        versions = response.get("LaunchTemplateVersions", [])
+        if versions:
+            return versions[0].get("LaunchTemplateData", {})
+        raise AWSClientError(
+            function_name="describe_launch_template_versions",
+            message=f"Launch template {launch_template_id} version {version} not found",
+        )

cli/src/pcluster/config/cluster_config.py

@@ -188,6 +188,7 @@
     InstancesEFAValidator,
     InstancesMemorySchedulingWarningValidator,
     InstancesNetworkingValidator,
+    LaunchTemplateValidator,
 )
 from pcluster.validators.kms_validators import KmsKeyIdEncryptedValidator, KmsKeyValidator
 from pcluster.validators.monitoring_validators import DetailedMonitoringValidator, LogRotationValidator
@@ -878,6 +879,15 @@ def __init__(self, enabled: bool = None, gdr_support: bool = None, **kwargs):
         self.gdr_support = Resource.init_param(gdr_support, default=False)
 
 
+class LaunchTemplateOverrides(Resource):
+    """Represent the Launch Template Overrides configuration."""
+
+    def __init__(self, launch_template_id: str = None, version: int = None, **kwargs):
+        super().__init__(**kwargs)
+        self.launch_template_id = Resource.init_param(launch_template_id)
+        self.version = Resource.init_param(version)
+
+
 # ---------------------- Health Checks ---------------------- #
 
 
@@ -2230,6 +2240,7 @@ def __init__(
         tags: List[Tag] = None,
         static_node_priority: int = None,
         dynamic_node_priority: int = None,
+        launch_template_overrides: LaunchTemplateOverrides = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -2250,6 +2261,7 @@ def __init__(
         self.tags = tags
         self.static_node_priority = Resource.init_param(static_node_priority, default=1)
         self.dynamic_node_priority = Resource.init_param(dynamic_node_priority, default=1000)
+        self.launch_template_overrides = launch_template_overrides
 
     @abstractmethod
     def is_flexible(self) -> bool:
@@ -2357,6 +2369,13 @@ def _register_validators(self, context: ValidatorContext = None):
             ec2memory=min_memory,
             instance_type=smallest_type,
         )
+        if self.launch_template_overrides:
+            self._register_validator(
+                LaunchTemplateValidator,
+                compute_resource_name=self.name,
+                launch_template_id=self.launch_template_overrides.launch_template_id,
+                launch_template_version=self.launch_template_overrides.version,
+            )
 
     def is_flexible(self):
         """Return True because the ComputeResource can contain multiple instance types."""
@@ -2449,6 +2468,13 @@ def _register_validators(self, context: ValidatorContext = None):
             ec2memory=self._instance_type_info.ec2memory_size_in_mib(),
             instance_type=self.instance_type,
         )
+        if self.launch_template_overrides:
+            self._register_validator(
+                LaunchTemplateValidator,
+                compute_resource_name=self.name,
+                launch_template_id=self.launch_template_overrides.launch_template_id,
+                launch_template_version=self.launch_template_overrides.version,
+            )
 
     @property
     def architecture(self) -> str:

cli/src/pcluster/schemas/cluster_schema.py

@@ -64,6 +64,7 @@
     Image,
     Imds,
     IntelSoftware,
+    LaunchTemplateOverrides,
     LocalStorage,
     LoginNodes,
     LoginNodesIam,
@@ -829,6 +830,31 @@ def make_resource(self, data, **kwargs):
         return Efa(**data)
 
 
+class LaunchTemplateOverridesSchema(BaseSchema):
+    """Represent the schema of LaunchTemplateOverrides for a Compute Resource."""
+
+    launch_template_id = fields.Str(
+        required=True,
+        metadata={"update_policy": UpdatePolicy.QUEUE_UPDATE_STRATEGY},
+    )
+    version = fields.Int(
+        validate=validate.Range(min=1),
+        required=True,
+        metadata={"update_policy": UpdatePolicy.QUEUE_UPDATE_STRATEGY},
+    )
+
+    @validates_schema
+    def validate_version_required(self, data, **kwargs):
+        """Validate that version is required when launch_template_id is specified."""
+        if data.get("launch_template_id") and not data.get("version"):
+            raise ValidationError("Version is required when LaunchTemplateId is specified.", field_name="Version")
+
+    @post_load
+    def make_resource(self, data, **kwargs):
+        """Generate resource."""
+        return LaunchTemplateOverrides(**data)
+
+
 # ---------------------- Monitoring ---------------------- #
 
 
@@ -1585,6 +1611,9 @@ class SlurmComputeResourceSchema(_ComputeResourceSchema):
         validate=validate.Range(min=MIN_SLURM_NODE_PRIORITY, max=MAX_SLURM_NODE_PRIORITY),
         metadata={"update_policy": UpdatePolicy.SUPPORTED},
     )
+    launch_template_overrides = fields.Nested(
+        LaunchTemplateOverridesSchema, metadata={"update_policy": UpdatePolicy.QUEUE_UPDATE_STRATEGY}
+    )
 
     @validates_schema
     def no_coexist_instance_type_flexibility(self, data, **kwargs):

cli/src/pcluster/templates/queues_stack.py

@@ -37,6 +37,31 @@
 from pcluster.utils import get_attr, get_http_tokens_setting
 
 
+def _apply_launch_template_overrides(launch_template, compute_resource):
+    """
+    Apply user-specified launch template overrides to the generated launch template.
+
+    For each property in the user's launch template, applies it using add_property_override.
+    This replaces the specific property while CDK handles the rest normally.
+
+    Args:
+        launch_template: The CDK CfnLaunchTemplate construct
+        compute_resource: The compute resource configuration
+    """
+    launch_template_overrides = getattr(compute_resource, "launch_template_overrides", None)
+    if not launch_template_overrides or not launch_template_overrides.launch_template_id:
+        return
+
+    version = str(launch_template_overrides.version)
+    override_lt_data = AWSApi.instance().ec2.describe_launch_template_version(
+        launch_template_overrides.launch_template_id, version
+    )
+
+    # Apply each property from the override launch template
+    for key, value in override_lt_data.items():
+        launch_template.add_property_override(f"LaunchTemplateData.{key}", value)
+
+
 class QueuesStack(NestedStack):
     """Stack encapsulating a set of queues and the associated resources."""
 
@@ -364,6 +389,8 @@ def _add_compute_resource_launch_template(
             ),
         )
 
+        _apply_launch_template_overrides(launch_template, compute_resource)
+
         return launch_template
 
 

cli/src/pcluster/validators/instances_validators.py

@@ -11,6 +11,7 @@
 from enum import Enum
 from typing import Callable, Dict
 
+from pcluster.aws.aws_api import AWSApi
 from pcluster.aws.aws_resources import InstanceTypeInfo
 from pcluster.config import cluster_config
 from pcluster.constants import MIN_MEMORY_ABSOLUTE_DIFFERENCE, MIN_MEMORY_PRECENTAGE_DIFFERENCE
@@ -335,3 +336,42 @@ def _min_max_memory(self, instance_types_info: Dict[str, InstanceTypeInfo]):
         # EC2 API should return valid values, but since it's really cheap better add a check
         available_memory = [value for value in available_memory if value is not None]
         return min(available_memory), max(available_memory)
+
+
+class LaunchTemplateValidator(Validator):
+    """Validate that the specified launch template exists, is accessible, and only contains allowed properties."""
+
+    # Only these properties are allowed in the launch template
+    ALLOWED_PROPERTIES = {"NetworkInterfaces"}
+
+    def _validate(
+        self,
+        compute_resource_name: str,
+        launch_template_id: str,
+        launch_template_version: int,
+        **kwargs,
+    ):
+        """Check if the launch template exists, is valid, and only contains NetworkInterfaces."""
+        if not launch_template_id:
+            return
+
+        version = str(launch_template_version)
+        try:
+            lt_data = AWSApi.instance().ec2.describe_launch_template_version(launch_template_id, version)
+        except Exception as e:
+            self._add_failure(
+                f"Launch template '{launch_template_id}' version '{version}' specified in Compute Resource "
+                f"'{compute_resource_name}' could not be found or accessed: {e}",
+                FailureLevel.ERROR,
+            )
+            return
+
+        # Check for disallowed properties
+        disallowed_properties = set(lt_data.keys()) - self.ALLOWED_PROPERTIES
+        if disallowed_properties:
+            self._add_failure(
+                f"Launch template '{launch_template_id}' in Compute Resource '{compute_resource_name}' contains "
+                f"properties that are not allowed: {', '.join(sorted(disallowed_properties))}. "
+                f"Only NetworkInterfaces is allowed.",
+                FailureLevel.ERROR,
+            )

cli/tests/pcluster/aws/dummy_aws_api.py

@@ -136,6 +136,10 @@ def __init__(self):
         }
         self.security_groups_cache = {}
 
+    def describe_launch_template_version(self, launch_template_id, version):
+        """Return mock launch template data."""
+        return {"NetworkInterfaces": [{"DeviceIndex": 0, "SubnetId": "subnet-123"}]}
+
     def get_official_image_id(self, os, architecture, filters=None):
         return "dummy-ami-id"
 
@@ -430,5 +434,9 @@ def mock_aws_api(mocker, mock_instance_type_info=True):
         "pcluster.aws.ec2.Ec2Client.describe_image",
         return_value=ImageInfo({"BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 35}}]}),
     )
+    mocker.patch(
+        "pcluster.aws.ec2.Ec2Client.describe_launch_template_version",
+        return_value={"NetworkInterfaces": [{"DeviceIndex": 0, "SubnetId": "subnet-123"}]},
+    )
     if mock_instance_type_info:
         mocker.patch("pcluster.aws.ec2.Ec2Client.get_instance_type_info", side_effect=_DummyInstanceTypeInfo)

cli/tests/pcluster/example_configs/slurm.full.yaml

@@ -188,6 +188,9 @@ Scheduling:
           Efa:
             Enabled: true
             GdrSupport: false
+          LaunchTemplateOverrides:
+            LaunchTemplateId: lt-01234567890abcdef
+            Version: 1
       Iam:
         InstanceProfile: arn:aws:iam::aws:instance-profile/CustomNodeInstanceProfile
       Image: