From 0cd84be4a47651d4fdab6501ec61018b9d516c3c Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 24 Jun 2025 10:56:23 +0200 Subject: [PATCH 01/41] wip --- api/v1alpha1/karpentermachinepool_types.go | 144 ++++++++ api/v1alpha1/zz_generated.deepcopy.go | 234 ++++++++++++- ...luster.x-k8s.io_karpentermachinepools.yaml | 179 ++++++++++ .../karpentermachinepool_controller.go | 328 +++++++++++++++++- .../karpentermachinepool_controller_test.go | 32 +- ...luster.x-k8s.io_karpentermachinepools.yaml | 179 ++++++++++ pkg/conditions/conditions.go | 18 + 7 files changed, 1107 insertions(+), 7 deletions(-) diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index f1c8a793..9707607b 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -17,15 +17,139 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capi "sigs.k8s.io/cluster-api/api/v1beta1" ) +// NodePoolSpec defines the configuration for a Karpenter NodePool +type NodePoolSpec struct { + // Disruption specifies the disruption behavior for the node pool + // +optional + Disruption *DisruptionSpec `json:"disruption,omitempty"` + + // Limits specifies the limits for the node pool + // +optional + Limits *LimitsSpec `json:"limits,omitempty"` + + // Requirements specifies the requirements for the node pool + // +optional + Requirements []RequirementSpec `json:"requirements,omitempty"` + + // Taints specifies the taints to apply to nodes in this pool + // +optional + Taints []TaintSpec `json:"taints,omitempty"` + + // Labels specifies the labels to apply to nodes in this pool + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Weight specifies the weight of this node pool + // +optional + Weight *int32 `json:"weight,omitempty"` +} + +// DisruptionSpec defines the disruption behavior for a NodePool +type DisruptionSpec struct { + // ConsolidateAfter specifies when to consolidate nodes + // +optional + ConsolidateAfter *metav1.Duration `json:"consolidateAfter,omitempty"` + + // ConsolidationPolicy specifies the consolidation policy + // +optional + ConsolidationPolicy *string `json:"consolidationPolicy,omitempty"` + + // ConsolidateUnder specifies when to consolidate under + // +optional + ConsolidateUnder *ConsolidateUnderSpec `json:"consolidateUnder,omitempty"` +} + +// ConsolidateUnderSpec defines when to consolidate under +type ConsolidateUnderSpec struct { + // CPUUtilization specifies the CPU utilization threshold + // +optional + CPUUtilization *string `json:"cpuUtilization,omitempty"` + + // MemoryUtilization specifies the memory utilization threshold + // +optional + MemoryUtilization *string `json:"memoryUtilization,omitempty"` +} + +// LimitsSpec defines the limits for a NodePool +type LimitsSpec struct { + // CPU specifies the CPU limit + // +optional + CPU *resource.Quantity `json:"cpu,omitempty"` + + // Memory specifies the memory limit + // +optional + Memory *resource.Quantity `json:"memory,omitempty"` +} + +// RequirementSpec defines a requirement for a NodePool +type RequirementSpec struct { + // Key specifies the requirement key + Key string `json:"key"` + + // Operator specifies the requirement operator + Operator string `json:"operator"` + + // Values specifies the requirement values + // +optional + Values []string `json:"values,omitempty"` +} + +// TaintSpec defines a taint for a NodePool +type TaintSpec struct { + // Key specifies the taint key + Key string `json:"key"` + + // Value specifies the taint value + // +optional + Value *string `json:"value,omitempty"` + + // Effect specifies the taint effect + Effect string `json:"effect"` +} + +// EC2NodeClassSpec defines the configuration for a Karpenter EC2NodeClass +type EC2NodeClassSpec struct { + // AMIID specifies the AMI ID to use + // +optional + AMIID *string `json:"amiId,omitempty"` + + // SecurityGroups specifies the security groups to use + // +optional + SecurityGroups []string `json:"securityGroups,omitempty"` + + // Subnets specifies the subnets to use + // +optional + Subnets []string `json:"subnets,omitempty"` + + // UserData specifies the user data to use + // +optional + UserData *string `json:"userData,omitempty"` + + // Tags specifies the tags to apply to EC2 instances + // +optional + Tags map[string]string `json:"tags,omitempty"` +} + // KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. type KarpenterMachinePoolSpec struct { // The name or the Amazon Resource Name (ARN) of the instance profile associated // with the IAM role for the instance. The instance profile contains the IAM // role. IamInstanceProfile string `json:"iamInstanceProfile,omitempty"` + + // NodePool specifies the configuration for the Karpenter NodePool + // +optional + NodePool *NodePoolSpec `json:"nodePool,omitempty"` + + // EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass + // +optional + EC2NodeClass *EC2NodeClassSpec `json:"ec2NodeClass,omitempty"` + // ProviderIDList are the identification IDs of machine instances provided by the provider. // This field must match the provider IDs as seen on the node objects corresponding to a machine pool's machine instances. // +optional @@ -41,6 +165,18 @@ type KarpenterMachinePoolStatus struct { // Replicas is the most recently observed number of replicas // +optional Replicas int32 `json:"replicas"` + + // Conditions defines current service state of the KarpenterMachinePool. + // +optional + Conditions capi.Conditions `json:"conditions,omitempty"` + + // NodePoolReady indicates whether the NodePool is ready + // +optional + NodePoolReady bool `json:"nodePoolReady"` + + // EC2NodeClassReady indicates whether the EC2NodeClass is ready + // +optional + EC2NodeClassReady bool `json:"ec2NodeClassReady"` } // +kubebuilder:object:root=true @@ -72,3 +208,11 @@ type KarpenterMachinePoolList struct { func init() { SchemeBuilder.Register(&KarpenterMachinePool{}, &KarpenterMachinePoolList{}) } + +func (in *KarpenterMachinePool) GetConditions() capi.Conditions { + return in.Status.Conditions +} + +func (in *KarpenterMachinePool) SetConditions(conditions capi.Conditions) { + in.Status.Conditions = conditions +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 42853805..1747ae85 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,16 +21,115 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/api/v1beta1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConsolidateUnderSpec) DeepCopyInto(out *ConsolidateUnderSpec) { + *out = *in + if in.CPUUtilization != nil { + in, out := &in.CPUUtilization, &out.CPUUtilization + *out = new(string) + **out = **in + } + if in.MemoryUtilization != nil { + in, out := &in.MemoryUtilization, &out.MemoryUtilization + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsolidateUnderSpec. +func (in *ConsolidateUnderSpec) DeepCopy() *ConsolidateUnderSpec { + if in == nil { + return nil + } + out := new(ConsolidateUnderSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DisruptionSpec) DeepCopyInto(out *DisruptionSpec) { + *out = *in + if in.ConsolidateAfter != nil { + in, out := &in.ConsolidateAfter, &out.ConsolidateAfter + *out = new(v1.Duration) + **out = **in + } + if in.ConsolidationPolicy != nil { + in, out := &in.ConsolidationPolicy, &out.ConsolidationPolicy + *out = new(string) + **out = **in + } + if in.ConsolidateUnder != nil { + in, out := &in.ConsolidateUnder, &out.ConsolidateUnder + *out = new(ConsolidateUnderSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DisruptionSpec. +func (in *DisruptionSpec) DeepCopy() *DisruptionSpec { + if in == nil { + return nil + } + out := new(DisruptionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { + *out = *in + if in.AMIID != nil { + in, out := &in.AMIID, &out.AMIID + *out = new(string) + **out = **in + } + if in.SecurityGroups != nil { + in, out := &in.SecurityGroups, &out.SecurityGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Subnets != nil { + in, out := &in.Subnets, &out.Subnets + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.UserData != nil { + in, out := &in.UserData, &out.UserData + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassSpec. +func (in *EC2NodeClassSpec) DeepCopy() *EC2NodeClassSpec { + if in == nil { + return nil + } + out := new(EC2NodeClassSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterMachinePool) DeepCopyInto(out *KarpenterMachinePool) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterMachinePool. @@ -86,6 +185,16 @@ func (in *KarpenterMachinePoolList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterMachinePoolSpec) DeepCopyInto(out *KarpenterMachinePoolSpec) { *out = *in + if in.NodePool != nil { + in, out := &in.NodePool, &out.NodePool + *out = new(NodePoolSpec) + (*in).DeepCopyInto(*out) + } + if in.EC2NodeClass != nil { + in, out := &in.EC2NodeClass, &out.EC2NodeClass + *out = new(EC2NodeClassSpec) + (*in).DeepCopyInto(*out) + } if in.ProviderIDList != nil { in, out := &in.ProviderIDList, &out.ProviderIDList *out = make([]string, len(*in)) @@ -106,6 +215,13 @@ func (in *KarpenterMachinePoolSpec) DeepCopy() *KarpenterMachinePoolSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterMachinePoolStatus) DeepCopyInto(out *KarpenterMachinePoolStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(v1beta1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterMachinePoolStatus. @@ -117,3 +233,119 @@ func (in *KarpenterMachinePoolStatus) DeepCopy() *KarpenterMachinePoolStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LimitsSpec) DeepCopyInto(out *LimitsSpec) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + x := (*in).DeepCopy() + *out = &x + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LimitsSpec. +func (in *LimitsSpec) DeepCopy() *LimitsSpec { + if in == nil { + return nil + } + out := new(LimitsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { + *out = *in + if in.Disruption != nil { + in, out := &in.Disruption, &out.Disruption + *out = new(DisruptionSpec) + (*in).DeepCopyInto(*out) + } + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = new(LimitsSpec) + (*in).DeepCopyInto(*out) + } + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]RequirementSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]TaintSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePoolSpec. +func (in *NodePoolSpec) DeepCopy() *NodePoolSpec { + if in == nil { + return nil + } + out := new(NodePoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RequirementSpec) DeepCopyInto(out *RequirementSpec) { + *out = *in + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequirementSpec. +func (in *RequirementSpec) DeepCopy() *RequirementSpec { + if in == nil { + return nil + } + out := new(RequirementSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaintSpec) DeepCopyInto(out *TaintSpec) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaintSpec. +func (in *TaintSpec) DeepCopy() *TaintSpec { + if in == nil { + return nil + } + out := new(TaintSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 69c3e422..6b83883d 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -50,12 +50,139 @@ spec: spec: description: KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. properties: + ec2NodeClass: + description: EC2NodeClass specifies the configuration for the Karpenter + EC2NodeClass + properties: + amiId: + description: AMIID specifies the AMI ID to use + type: string + securityGroups: + description: SecurityGroups specifies the security groups to use + items: + type: string + type: array + subnets: + description: Subnets specifies the subnets to use + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags specifies the tags to apply to EC2 instances + type: object + userData: + description: UserData specifies the user data to use + type: string + type: object iamInstanceProfile: description: |- The name or the Amazon Resource Name (ARN) of the instance profile associated with the IAM role for the instance. The instance profile contains the IAM role. type: string + nodePool: + description: NodePool specifies the configuration for the Karpenter + NodePool + properties: + disruption: + description: Disruption specifies the disruption behavior for + the node pool + properties: + consolidateAfter: + description: ConsolidateAfter specifies when to consolidate + nodes + type: string + consolidateUnder: + description: ConsolidateUnder specifies when to consolidate + under + properties: + cpuUtilization: + description: CPUUtilization specifies the CPU utilization + threshold + type: string + memoryUtilization: + description: MemoryUtilization specifies the memory utilization + threshold + type: string + type: object + consolidationPolicy: + description: ConsolidationPolicy specifies the consolidation + policy + type: string + type: object + labels: + additionalProperties: + type: string + description: Labels specifies the labels to apply to nodes in + this pool + type: object + limits: + description: Limits specifies the limits for the node pool + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU specifies the CPU limit + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory specifies the memory limit + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requirements: + description: Requirements specifies the requirements for the node + pool + items: + description: RequirementSpec defines a requirement for a NodePool + properties: + key: + description: Key specifies the requirement key + type: string + operator: + description: Operator specifies the requirement operator + type: string + values: + description: Values specifies the requirement values + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + taints: + description: Taints specifies the taints to apply to nodes in + this pool + items: + description: TaintSpec defines a taint for a NodePool + properties: + effect: + description: Effect specifies the taint effect + type: string + key: + description: Key specifies the taint key + type: string + value: + description: Value specifies the taint value + type: string + required: + - effect + - key + type: object + type: array + weight: + description: Weight specifies the weight of this node pool + format: int32 + type: integer + type: object providerIDList: description: |- ProviderIDList are the identification IDs of machine instances provided by the provider. @@ -68,6 +195,58 @@ spec: description: KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. properties: + conditions: + description: Conditions defines current service state of the KarpenterMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + ec2NodeClassReady: + description: EC2NodeClassReady indicates whether the EC2NodeClass + is ready + type: boolean + nodePoolReady: + description: NodePoolReady indicates whether the NodePool is ready + type: boolean ready: description: Ready is true when the provider resource is ready. type: boolean diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index b087f1a7..d72a50ef 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -3,6 +3,7 @@ package controllers import ( "context" "crypto/sha256" + "encoding/json" "errors" "fmt" "path" @@ -10,6 +11,7 @@ import ( "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" @@ -27,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/aws-resolver-rules-operator/api/v1alpha1" + "github.com/aws-resolver-rules-operator/pkg/conditions" "github.com/aws-resolver-rules-operator/pkg/resolver" ) @@ -38,6 +41,14 @@ const ( KarpenterNodePoolReadyCondition capi.ConditionType = "KarpenterNodePoolReadyCondition" // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. WaitingForBootstrapDataReason = "WaitingForBootstrapData" + // NodePoolCreatedReason indicates that the NodePool was successfully created + NodePoolCreatedReason = "NodePoolCreated" + // NodePoolCreationFailedReason indicates that the NodePool creation failed + NodePoolCreationFailedReason = "NodePoolCreationFailed" + // EC2NodeClassCreatedReason indicates that the EC2NodeClass was successfully created + EC2NodeClassCreatedReason = "EC2NodeClassCreated" + // EC2NodeClassCreationFailedReason indicates that the EC2NodeClass creation failed + EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" ) type KarpenterMachinePoolReconciler struct { @@ -165,6 +176,12 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } } + // Create or update Karpenter resources in the workload cluster + if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, bootstrapSecretValue); err != nil { + logger.Error(err, "failed to create or update Karpenter resources") + return reconcile.Result{}, err + } + providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) if err != nil { return reconcile.Result{}, err @@ -212,14 +229,11 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo } // Terminate EC2 instances with the karpenter.sh/nodepool tag matching the KarpenterMachinePool name - logger.Info("Terminating EC2 instances for KarpenterMachinePool", "karpenterMachinePoolName", karpenterMachinePool.Name) instanceIDs, err := ec2Client.TerminateInstancesByTag(ctx, logger, "karpenter.sh/nodepool", karpenterMachinePool.Name) if err != nil { return reconcile.Result{}, fmt.Errorf("failed to terminate EC2 instances: %w", err) } - logger.Info("Found instances", "instanceIDs", instanceIDs) - // Requeue if we find instances to terminate. Once there are no instances to terminate, we proceed to remove the finalizer. // We do this when the cluster is being deleted, to avoid removing the finalizer before karpenter launches a new instance that would be left over. if len(instanceIDs) > 0 { @@ -227,9 +241,16 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo } } + // Delete Karpenter resources from the workload cluster + if err := r.deleteKarpenterResources(ctx, logger, cluster, karpenterMachinePool); err != nil { + logger.Error(err, "failed to delete Karpenter resources") + return reconcile.Result{}, err + } + // Create deep copy of the reconciled object so we can change it karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() + logger.Info("Removing finalizer", "finalizer", KarpenterFinalizer) controllerutil.RemoveFinalizer(karpenterMachinePool, KarpenterFinalizer) if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { logger.Error(err, "failed to remove finalizer", "finalizer", KarpenterFinalizer) @@ -280,6 +301,307 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo return providerIDList, int32(len(nodeClaimList.Items)), nil } +// createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass resources in the workload cluster +func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) + if err != nil { + return fmt.Errorf("failed to get workload cluster client: %w", err) + } + + // Create or update EC2NodeClass + if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, cluster, awsCluster, karpenterMachinePool, bootstrapSecretValue); err != nil { + conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, EC2NodeClassCreationFailedReason, err.Error()) + return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) + } + + // Create or update NodePool + if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, karpenterMachinePool); err != nil { + conditions.MarkNodePoolNotReady(karpenterMachinePool, NodePoolCreationFailedReason, err.Error()) + return fmt.Errorf("failed to create or update NodePool: %w", err) + } + + // Mark both resources as ready + conditions.MarkEC2NodeClassReady(karpenterMachinePool) + conditions.MarkNodePoolReady(karpenterMachinePool) + + return nil +} + +// createOrUpdateEC2NodeClass creates or updates the EC2NodeClass resource in the workload cluster +func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { + ec2NodeClassGVR := schema.GroupVersionResource{ + Group: "karpenter.k8s.aws", + Version: "v1beta1", + Resource: "ec2nodeclasses", + } + + ec2NodeClass := &unstructured.Unstructured{} + ec2NodeClass.SetGroupVersionKind(ec2NodeClassGVR.GroupVersion().WithKind("EC2NodeClass")) + ec2NodeClass.SetName(karpenterMachinePool.Name) + ec2NodeClass.SetNamespace("default") + + // Check if the EC2NodeClass already exists + err := workloadClusterClient.Get(ctx, client.ObjectKey{Name: karpenterMachinePool.Name, Namespace: "default"}, ec2NodeClass) + if err != nil && !k8serrors.IsNotFound(err) { + return fmt.Errorf("failed to get existing EC2NodeClass: %w", err) + } + + // Generate user data for Ignition + userData := r.generateUserData(awsCluster.Spec.Region, cluster.Name, karpenterMachinePool.Name) + + // Build the EC2NodeClass spec + spec := map[string]interface{}{ + "amiFamily": "AL2", + "role": karpenterMachinePool.Spec.IamInstanceProfile, + "userData": userData, + } + + // Add AMI ID if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && karpenterMachinePool.Spec.EC2NodeClass.AMIID != nil { + spec["amiSelectorTerms"] = []map[string]interface{}{ + { + "id": *karpenterMachinePool.Spec.EC2NodeClass.AMIID, + }, + } + } + + // Add security groups if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { + spec["securityGroupSelectorTerms"] = []map[string]interface{}{ + { + "tags": map[string]string{ + "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now + }, + }, + } + } + + // Add subnets if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { + subnetSelectorTerms := []map[string]interface{}{} + for _, subnet := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { + subnetSelectorTerms = append(subnetSelectorTerms, map[string]interface{}{ + "id": subnet, + }) + } + spec["subnetSelectorTerms"] = subnetSelectorTerms + } + + // Add tags if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Tags) > 0 { + spec["tags"] = karpenterMachinePool.Spec.EC2NodeClass.Tags + } + + ec2NodeClass.Object["spec"] = spec + + // Create or update the EC2NodeClass + if k8serrors.IsNotFound(err) { + logger.Info("Creating EC2NodeClass", "name", karpenterMachinePool.Name) + if err := workloadClusterClient.Create(ctx, ec2NodeClass); err != nil { + return fmt.Errorf("failed to create EC2NodeClass: %w", err) + } + } else { + logger.Info("Updating EC2NodeClass", "name", karpenterMachinePool.Name) + if err := workloadClusterClient.Update(ctx, ec2NodeClass); err != nil { + return fmt.Errorf("failed to update EC2NodeClass: %w", err) + } + } + + return nil +} + +// createOrUpdateNodePool creates or updates the NodePool resource in the workload cluster +func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { + nodePoolGVR := schema.GroupVersionResource{ + Group: "karpenter.sh", + Version: "v1beta1", + Resource: "nodepools", + } + + nodePool := &unstructured.Unstructured{} + nodePool.SetGroupVersionKind(nodePoolGVR.GroupVersion().WithKind("NodePool")) + nodePool.SetName(karpenterMachinePool.Name) + nodePool.SetNamespace("default") + + // Check if the NodePool already exists + err := workloadClusterClient.Get(ctx, client.ObjectKey{Name: karpenterMachinePool.Name, Namespace: "default"}, nodePool) + if err != nil && !k8serrors.IsNotFound(err) { + return fmt.Errorf("failed to get existing NodePool: %w", err) + } + + // Build the NodePool spec + spec := map[string]interface{}{ + "disruption": map[string]interface{}{ + "consolidateAfter": "30s", + }, + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "nodeClassRef": map[string]interface{}{ + "apiVersion": "karpenter.k8s.aws/v1beta1", + "kind": "EC2NodeClass", + "name": karpenterMachinePool.Name, + }, + }, + }, + } + + // Add NodePool configuration if specified + if karpenterMachinePool.Spec.NodePool != nil { + if karpenterMachinePool.Spec.NodePool.Disruption != nil { + if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter != nil { + spec["disruption"].(map[string]interface{})["consolidateAfter"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter.Duration.String() + } + if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy != nil { + spec["disruption"].(map[string]interface{})["consolidationPolicy"] = *karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy + } + } + + if karpenterMachinePool.Spec.NodePool.Limits != nil { + limits := map[string]interface{}{} + if karpenterMachinePool.Spec.NodePool.Limits.CPU != nil { + limits["cpu"] = karpenterMachinePool.Spec.NodePool.Limits.CPU.String() + } + if karpenterMachinePool.Spec.NodePool.Limits.Memory != nil { + limits["memory"] = karpenterMachinePool.Spec.NodePool.Limits.Memory.String() + } + if len(limits) > 0 { + spec["limits"] = limits + } + } + + if len(karpenterMachinePool.Spec.NodePool.Requirements) > 0 { + requirements := []map[string]interface{}{} + for _, req := range karpenterMachinePool.Spec.NodePool.Requirements { + requirement := map[string]interface{}{ + "key": req.Key, + "operator": req.Operator, + } + if len(req.Values) > 0 { + requirement["values"] = req.Values + } + requirements = append(requirements, requirement) + } + spec["requirements"] = requirements + } + + if len(karpenterMachinePool.Spec.NodePool.Taints) > 0 { + taints := []map[string]interface{}{} + for _, taint := range karpenterMachinePool.Spec.NodePool.Taints { + taintMap := map[string]interface{}{ + "key": taint.Key, + "effect": taint.Effect, + } + if taint.Value != nil { + taintMap["value"] = *taint.Value + } + taints = append(taints, taintMap) + } + spec["taints"] = taints + } + + if len(karpenterMachinePool.Spec.NodePool.Labels) > 0 { + spec["labels"] = karpenterMachinePool.Spec.NodePool.Labels + } + + if karpenterMachinePool.Spec.NodePool.Weight != nil { + spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight + } + } + + nodePool.Object["spec"] = spec + + // Create or update the NodePool + if k8serrors.IsNotFound(err) { + logger.Info("Creating NodePool", "name", karpenterMachinePool.Name) + if err := workloadClusterClient.Create(ctx, nodePool); err != nil { + return fmt.Errorf("failed to create NodePool: %w", err) + } + } else { + logger.Info("Updating NodePool", "name", karpenterMachinePool.Name) + if err := workloadClusterClient.Update(ctx, nodePool); err != nil { + return fmt.Errorf("failed to update NodePool: %w", err) + } + } + + return nil +} + +// deleteKarpenterResources deletes the Karpenter NodePool and EC2NodeClass resources from the workload cluster +func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) + if err != nil { + return fmt.Errorf("failed to get workload cluster client: %w", err) + } + + // Delete NodePool + nodePoolGVR := schema.GroupVersionResource{ + Group: "karpenter.sh", + Version: "v1beta1", + Resource: "nodepools", + } + + nodePool := &unstructured.Unstructured{} + nodePool.SetGroupVersionKind(nodePoolGVR.GroupVersion().WithKind("NodePool")) + nodePool.SetName(karpenterMachinePool.Name) + nodePool.SetNamespace("default") + + if err := workloadClusterClient.Delete(ctx, nodePool); err != nil && !k8serrors.IsNotFound(err) { + logger.Error(err, "failed to delete NodePool", "name", karpenterMachinePool.Name) + return fmt.Errorf("failed to delete NodePool: %w", err) + } + + // Delete EC2NodeClass + ec2NodeClassGVR := schema.GroupVersionResource{ + Group: "karpenter.k8s.aws", + Version: "v1beta1", + Resource: "ec2nodeclasses", + } + + ec2NodeClass := &unstructured.Unstructured{} + ec2NodeClass.SetGroupVersionKind(ec2NodeClassGVR.GroupVersion().WithKind("EC2NodeClass")) + ec2NodeClass.SetName(karpenterMachinePool.Name) + ec2NodeClass.SetNamespace("default") + + if err := workloadClusterClient.Delete(ctx, ec2NodeClass); err != nil && !k8serrors.IsNotFound(err) { + logger.Error(err, "failed to delete EC2NodeClass", "name", karpenterMachinePool.Name) + return fmt.Errorf("failed to delete EC2NodeClass: %w", err) + } + + return nil +} + +// generateUserData generates the user data for Ignition configuration +func (r *KarpenterMachinePoolReconciler) generateUserData(region, clusterName, karpenterMachinePoolName string) string { + userData := map[string]interface{}{ + "ignition": map[string]interface{}{ + "config": map[string]interface{}{ + "merge": []map[string]interface{}{ + { + "source": fmt.Sprintf("s3://%s-capa-%s/%s/%s", region, clusterName, S3ObjectPrefix, karpenterMachinePoolName), + "verification": map[string]interface{}{}, + }, + }, + "replace": map[string]interface{}{ + "verification": map[string]interface{}{}, + }, + }, + "proxy": map[string]interface{}{}, + "security": map[string]interface{}{ + "tls": map[string]interface{}{}, + }, + "timeouts": map[string]interface{}{}, + "version": "3.4.0", + }, + "kernelArguments": map[string]interface{}{}, + "passwd": map[string]interface{}{}, + "storage": map[string]interface{}{}, + "systemd": map[string]interface{}{}, + } + + userDataBytes, _ := json.Marshal(userData) + return string(userDataBytes) +} + // SetupWithManager sets up the controller with the Manager. func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { logger := capalogger.FromContext(ctx).GetLogger() diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 4403aa5a..ac689108 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -219,7 +219,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ObjectMeta: metav1.ObjectMeta{ Name: "default", }, - Spec: capa.AWSClusterRoleIdentitySpec{}, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, } err = fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) Expect(err).NotTo(HaveOccurred()) @@ -530,7 +534,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ObjectMeta: metav1.ObjectMeta{ Name: "default", }, - Spec: capa.AWSClusterRoleIdentitySpec{}, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, } err := fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) Expect(err).NotTo(HaveOccurred()) @@ -779,7 +787,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ObjectMeta: metav1.ObjectMeta{ Name: "default", }, - Spec: capa.AWSClusterRoleIdentitySpec{}, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, } err = fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) Expect(err).NotTo(HaveOccurred()) @@ -799,4 +811,18 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(s3Client.PutCallCount()).To(Equal(0)) }) }) + + When("the KarpenterMachinePool has NodePool and EC2NodeClass configuration", func() { + It("should handle the configuration without errors", func() { + // This test verifies that the controller can handle KarpenterMachinePool + // with NodePool and EC2NodeClass configuration without panicking + // The actual resource creation is tested in integration tests + Expect(reconcileErr).NotTo(HaveOccurred()) + }) + }) }) + +// Helper function to create string pointers +func stringPtr(s string) *string { + return &s +} diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 69c3e422..6b83883d 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -50,12 +50,139 @@ spec: spec: description: KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. properties: + ec2NodeClass: + description: EC2NodeClass specifies the configuration for the Karpenter + EC2NodeClass + properties: + amiId: + description: AMIID specifies the AMI ID to use + type: string + securityGroups: + description: SecurityGroups specifies the security groups to use + items: + type: string + type: array + subnets: + description: Subnets specifies the subnets to use + items: + type: string + type: array + tags: + additionalProperties: + type: string + description: Tags specifies the tags to apply to EC2 instances + type: object + userData: + description: UserData specifies the user data to use + type: string + type: object iamInstanceProfile: description: |- The name or the Amazon Resource Name (ARN) of the instance profile associated with the IAM role for the instance. The instance profile contains the IAM role. type: string + nodePool: + description: NodePool specifies the configuration for the Karpenter + NodePool + properties: + disruption: + description: Disruption specifies the disruption behavior for + the node pool + properties: + consolidateAfter: + description: ConsolidateAfter specifies when to consolidate + nodes + type: string + consolidateUnder: + description: ConsolidateUnder specifies when to consolidate + under + properties: + cpuUtilization: + description: CPUUtilization specifies the CPU utilization + threshold + type: string + memoryUtilization: + description: MemoryUtilization specifies the memory utilization + threshold + type: string + type: object + consolidationPolicy: + description: ConsolidationPolicy specifies the consolidation + policy + type: string + type: object + labels: + additionalProperties: + type: string + description: Labels specifies the labels to apply to nodes in + this pool + type: object + limits: + description: Limits specifies the limits for the node pool + properties: + cpu: + anyOf: + - type: integer + - type: string + description: CPU specifies the CPU limit + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + memory: + anyOf: + - type: integer + - type: string + description: Memory specifies the memory limit + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requirements: + description: Requirements specifies the requirements for the node + pool + items: + description: RequirementSpec defines a requirement for a NodePool + properties: + key: + description: Key specifies the requirement key + type: string + operator: + description: Operator specifies the requirement operator + type: string + values: + description: Values specifies the requirement values + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + taints: + description: Taints specifies the taints to apply to nodes in + this pool + items: + description: TaintSpec defines a taint for a NodePool + properties: + effect: + description: Effect specifies the taint effect + type: string + key: + description: Key specifies the taint key + type: string + value: + description: Value specifies the taint value + type: string + required: + - effect + - key + type: object + type: array + weight: + description: Weight specifies the weight of this node pool + format: int32 + type: integer + type: object providerIDList: description: |- ProviderIDList are the identification IDs of machine instances provided by the provider. @@ -68,6 +195,58 @@ spec: description: KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. properties: + conditions: + description: Conditions defines current service state of the KarpenterMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + ec2NodeClassReady: + description: EC2NodeClassReady indicates whether the EC2NodeClass + is ready + type: boolean + nodePoolReady: + description: NodePoolReady indicates whether the NodePool is ready + type: boolean ready: description: Ready is true when the provider resource is ready. type: boolean diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 7773a3da..4f7e2f58 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -13,6 +13,8 @@ const ( TransitGatewayCreated capi.ConditionType = "TransitGatewayCreated" TransitGatewayAttached capi.ConditionType = "TransitGatewayAttached" PrefixListEntriesReady capi.ConditionType = "PrefixListEntriesReady" + NodePoolReady capi.ConditionType = "NodePoolReady" + EC2NodeClassReady capi.ConditionType = "EC2NodeClassReady" ) func MarkReady(setter capiconditions.Setter, condition capi.ConditionType) { @@ -42,3 +44,19 @@ func MarkIDNotProvided(cluster *capi.Cluster, id string) { "The %s ID is missing from the annotations", id, ) } + +func MarkNodePoolReady(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, NodePoolReady) +} + +func MarkNodePoolNotReady(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, NodePoolReady, reason, capi.ConditionSeverityError, message) +} + +func MarkEC2NodeClassReady(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, EC2NodeClassReady) +} + +func MarkEC2NodeClassNotReady(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, EC2NodeClassReady, reason, capi.ConditionSeverityError, message) +} From ef2c8dc3fd976be394f8b049b560db11622fca50 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Wed, 25 Jun 2025 19:01:20 +0200 Subject: [PATCH 02/41] wip2 --- .../karpentermachinepool_controller.go | 444 ++++++++++++------ .../karpentermachinepool_controller_test.go | 72 +++ .../templates/rbac.yaml | 1 + 3 files changed, 383 insertions(+), 134 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index d72a50ef..d76c4a13 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "path" + "strconv" + "strings" "time" "github.com/go-logr/logr" @@ -18,6 +20,7 @@ import ( capalogger "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" capi "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/remote" + capiexp "sigs.k8s.io/cluster-api/exp/api/v1beta1" capiutilexp "sigs.k8s.io/cluster-api/exp/util" capiutil "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" @@ -49,6 +52,8 @@ const ( EC2NodeClassCreatedReason = "EC2NodeClassCreated" // EC2NodeClassCreationFailedReason indicates that the EC2NodeClass creation failed EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" + // VersionSkewBlockedReason indicates that the update was blocked due to version skew policy + VersionSkewBlockedReason = "VersionSkewBlocked" ) type KarpenterMachinePoolReconciler struct { @@ -177,7 +182,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } // Create or update Karpenter resources in the workload cluster - if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, bootstrapSecretValue); err != nil { + if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool, bootstrapSecretValue); err != nil { logger.Error(err, "failed to create or update Karpenter resources") return reconcile.Result{}, err } @@ -301,8 +306,79 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo return providerIDList, int32(len(nodeClaimList.Items)), nil } +// getControlPlaneVersion retrieves the Kubernetes version from the control plane +func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Context, cluster *capi.Cluster) (string, error) { + if cluster.Spec.ControlPlaneRef == nil { + return "", fmt.Errorf("cluster has no control plane reference") + } + + // Parse the API version to get group and version + apiVersion := cluster.Spec.ControlPlaneRef.APIVersion + groupVersion, err := schema.ParseGroupVersion(apiVersion) + if err != nil { + return "", fmt.Errorf("failed to parse control plane API version %s: %w", apiVersion, err) + } + + // Create the GVR using the parsed group and version + controlPlaneGVR := schema.GroupVersionResource{ + Group: groupVersion.Group, + Version: groupVersion.Version, + Resource: strings.ToLower(cluster.Spec.ControlPlaneRef.Kind) + "s", // Convert Kind to resource name + } + + controlPlane := &unstructured.Unstructured{} + controlPlane.SetGroupVersionKind(controlPlaneGVR.GroupVersion().WithKind(cluster.Spec.ControlPlaneRef.Kind)) + controlPlane.SetName(cluster.Spec.ControlPlaneRef.Name) + controlPlane.SetNamespace(cluster.Spec.ControlPlaneRef.Namespace) + + if err := r.client.Get(ctx, client.ObjectKey{Name: cluster.Spec.ControlPlaneRef.Name, Namespace: cluster.Spec.ControlPlaneRef.Namespace}, controlPlane); err != nil { + return "", fmt.Errorf("failed to get control plane %s: %w", cluster.Spec.ControlPlaneRef.Kind, err) + } + + version, found, err := unstructured.NestedString(controlPlane.Object, "spec", "version") + if err != nil { + return "", fmt.Errorf("failed to get version from control plane: %w", err) + } + if !found { + return "", fmt.Errorf("version not found in control plane spec") + } + + return version, nil +} + // createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass resources in the workload cluster -func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { +func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool, bootstrapSecretValue []byte) error { + // Get the worker version from MachinePool + workerVersion := "" + if machinePool.Spec.Template.Spec.Version != nil { + workerVersion = *machinePool.Spec.Template.Spec.Version + } + + // Get control plane version and check version skew + if workerVersion != "" { + controlPlaneVersion, err := r.getControlPlaneVersion(ctx, cluster) + if err != nil { + logger.Error(err, "Failed to get control plane version, proceeding with update") + } else { + allowed, err := IsVersionSkewAllowed(controlPlaneVersion, workerVersion) + if err != nil { + logger.Error(err, "Failed to check version skew, proceeding with update") + } else if !allowed { + message := fmt.Sprintf("Version skew policy violation: control plane version %s is more than 2 minor versions ahead of worker version %s", controlPlaneVersion, workerVersion) + logger.Info("Blocking Karpenter resource update due to version skew policy", + "controlPlaneVersion", controlPlaneVersion, + "workerVersion", workerVersion, + "reason", message) + + // Mark resources as not ready due to version skew + conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) + conditions.MarkNodePoolNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) + + return fmt.Errorf("version skew policy violation: %s", message) + } + } + } + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { return fmt.Errorf("failed to get workload cluster client: %w", err) @@ -340,71 +416,66 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. ec2NodeClass.SetName(karpenterMachinePool.Name) ec2NodeClass.SetNamespace("default") - // Check if the EC2NodeClass already exists - err := workloadClusterClient.Get(ctx, client.ObjectKey{Name: karpenterMachinePool.Name, Namespace: "default"}, ec2NodeClass) - if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to get existing EC2NodeClass: %w", err) - } - // Generate user data for Ignition userData := r.generateUserData(awsCluster.Spec.Region, cluster.Name, karpenterMachinePool.Name) - // Build the EC2NodeClass spec - spec := map[string]interface{}{ - "amiFamily": "AL2", - "role": karpenterMachinePool.Spec.IamInstanceProfile, - "userData": userData, - } + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { + // Build the EC2NodeClass spec + spec := map[string]interface{}{ + "amiFamily": "AL2", + "role": karpenterMachinePool.Spec.IamInstanceProfile, + "userData": userData, + } - // Add AMI ID if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && karpenterMachinePool.Spec.EC2NodeClass.AMIID != nil { - spec["amiSelectorTerms"] = []map[string]interface{}{ - { - "id": *karpenterMachinePool.Spec.EC2NodeClass.AMIID, - }, + // Add AMI ID if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && karpenterMachinePool.Spec.EC2NodeClass.AMIID != nil { + spec["amiSelectorTerms"] = []map[string]interface{}{ + { + "id": *karpenterMachinePool.Spec.EC2NodeClass.AMIID, + }, + } } - } - // Add security groups if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { - spec["securityGroupSelectorTerms"] = []map[string]interface{}{ - { - "tags": map[string]string{ - "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now + // Add security groups if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { + spec["securityGroupSelectorTerms"] = []map[string]interface{}{ + { + "tags": map[string]string{ + "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now + }, }, - }, + } } - } - // Add subnets if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { - subnetSelectorTerms := []map[string]interface{}{} - for _, subnet := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { - subnetSelectorTerms = append(subnetSelectorTerms, map[string]interface{}{ - "id": subnet, - }) + // Add subnets if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { + subnetSelectorTerms := []map[string]interface{}{} + for _, subnet := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { + subnetSelectorTerms = append(subnetSelectorTerms, map[string]interface{}{ + "id": subnet, + }) + } + spec["subnetSelectorTerms"] = subnetSelectorTerms } - spec["subnetSelectorTerms"] = subnetSelectorTerms - } - // Add tags if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Tags) > 0 { - spec["tags"] = karpenterMachinePool.Spec.EC2NodeClass.Tags - } + // Add tags if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Tags) > 0 { + spec["tags"] = karpenterMachinePool.Spec.EC2NodeClass.Tags + } - ec2NodeClass.Object["spec"] = spec + ec2NodeClass.Object["spec"] = spec + return nil + }) - // Create or update the EC2NodeClass - if k8serrors.IsNotFound(err) { - logger.Info("Creating EC2NodeClass", "name", karpenterMachinePool.Name) - if err := workloadClusterClient.Create(ctx, ec2NodeClass); err != nil { - return fmt.Errorf("failed to create EC2NodeClass: %w", err) - } - } else { - logger.Info("Updating EC2NodeClass", "name", karpenterMachinePool.Name) - if err := workloadClusterClient.Update(ctx, ec2NodeClass); err != nil { - return fmt.Errorf("failed to update EC2NodeClass: %w", err) - } + if err != nil { + return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) + } + + switch operation { + case controllerutil.OperationResultCreated: + logger.Info("Created EC2NodeClass") + case controllerutil.OperationResultUpdated: + logger.Info("Updated EC2NodeClass") } return nil @@ -423,104 +494,99 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont nodePool.SetName(karpenterMachinePool.Name) nodePool.SetNamespace("default") - // Check if the NodePool already exists - err := workloadClusterClient.Get(ctx, client.ObjectKey{Name: karpenterMachinePool.Name, Namespace: "default"}, nodePool) - if err != nil && !k8serrors.IsNotFound(err) { - return fmt.Errorf("failed to get existing NodePool: %w", err) - } - - // Build the NodePool spec - spec := map[string]interface{}{ - "disruption": map[string]interface{}{ - "consolidateAfter": "30s", - }, - "template": map[string]interface{}{ - "spec": map[string]interface{}{ - "nodeClassRef": map[string]interface{}{ - "apiVersion": "karpenter.k8s.aws/v1beta1", - "kind": "EC2NodeClass", - "name": karpenterMachinePool.Name, + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, nodePool, func() error { + // Build the NodePool spec + spec := map[string]interface{}{ + "disruption": map[string]interface{}{ + "consolidateAfter": "30s", + }, + "template": map[string]interface{}{ + "spec": map[string]interface{}{ + "nodeClassRef": map[string]interface{}{ + "apiVersion": "karpenter.k8s.aws/v1beta1", + "kind": "EC2NodeClass", + "name": karpenterMachinePool.Name, + }, }, }, - }, - } - - // Add NodePool configuration if specified - if karpenterMachinePool.Spec.NodePool != nil { - if karpenterMachinePool.Spec.NodePool.Disruption != nil { - if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter != nil { - spec["disruption"].(map[string]interface{})["consolidateAfter"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter.Duration.String() - } - if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy != nil { - spec["disruption"].(map[string]interface{})["consolidationPolicy"] = *karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy - } } - if karpenterMachinePool.Spec.NodePool.Limits != nil { - limits := map[string]interface{}{} - if karpenterMachinePool.Spec.NodePool.Limits.CPU != nil { - limits["cpu"] = karpenterMachinePool.Spec.NodePool.Limits.CPU.String() - } - if karpenterMachinePool.Spec.NodePool.Limits.Memory != nil { - limits["memory"] = karpenterMachinePool.Spec.NodePool.Limits.Memory.String() - } - if len(limits) > 0 { - spec["limits"] = limits + // Add NodePool configuration if specified + if karpenterMachinePool.Spec.NodePool != nil { + if karpenterMachinePool.Spec.NodePool.Disruption != nil { + if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter != nil { + spec["disruption"].(map[string]interface{})["consolidateAfter"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter.Duration.String() + } + if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy != nil { + spec["disruption"].(map[string]interface{})["consolidationPolicy"] = *karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy + } } - } - if len(karpenterMachinePool.Spec.NodePool.Requirements) > 0 { - requirements := []map[string]interface{}{} - for _, req := range karpenterMachinePool.Spec.NodePool.Requirements { - requirement := map[string]interface{}{ - "key": req.Key, - "operator": req.Operator, + if karpenterMachinePool.Spec.NodePool.Limits != nil { + limits := map[string]interface{}{} + if karpenterMachinePool.Spec.NodePool.Limits.CPU != nil { + limits["cpu"] = karpenterMachinePool.Spec.NodePool.Limits.CPU.String() } - if len(req.Values) > 0 { - requirement["values"] = req.Values + if karpenterMachinePool.Spec.NodePool.Limits.Memory != nil { + limits["memory"] = karpenterMachinePool.Spec.NodePool.Limits.Memory.String() + } + if len(limits) > 0 { + spec["limits"] = limits } - requirements = append(requirements, requirement) } - spec["requirements"] = requirements - } - if len(karpenterMachinePool.Spec.NodePool.Taints) > 0 { - taints := []map[string]interface{}{} - for _, taint := range karpenterMachinePool.Spec.NodePool.Taints { - taintMap := map[string]interface{}{ - "key": taint.Key, - "effect": taint.Effect, + if len(karpenterMachinePool.Spec.NodePool.Requirements) > 0 { + requirements := []map[string]interface{}{} + for _, req := range karpenterMachinePool.Spec.NodePool.Requirements { + requirement := map[string]interface{}{ + "key": req.Key, + "operator": req.Operator, + } + if len(req.Values) > 0 { + requirement["values"] = req.Values + } + requirements = append(requirements, requirement) } - if taint.Value != nil { - taintMap["value"] = *taint.Value + spec["requirements"] = requirements + } + + if len(karpenterMachinePool.Spec.NodePool.Taints) > 0 { + taints := []map[string]interface{}{} + for _, taint := range karpenterMachinePool.Spec.NodePool.Taints { + taintMap := map[string]interface{}{ + "key": taint.Key, + "effect": taint.Effect, + } + if taint.Value != nil { + taintMap["value"] = *taint.Value + } + taints = append(taints, taintMap) } - taints = append(taints, taintMap) + spec["taints"] = taints } - spec["taints"] = taints - } - if len(karpenterMachinePool.Spec.NodePool.Labels) > 0 { - spec["labels"] = karpenterMachinePool.Spec.NodePool.Labels - } + if len(karpenterMachinePool.Spec.NodePool.Labels) > 0 { + spec["labels"] = karpenterMachinePool.Spec.NodePool.Labels + } - if karpenterMachinePool.Spec.NodePool.Weight != nil { - spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight + if karpenterMachinePool.Spec.NodePool.Weight != nil { + spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight + } } - } - nodePool.Object["spec"] = spec + nodePool.Object["spec"] = spec + return nil + }) - // Create or update the NodePool - if k8serrors.IsNotFound(err) { - logger.Info("Creating NodePool", "name", karpenterMachinePool.Name) - if err := workloadClusterClient.Create(ctx, nodePool); err != nil { - return fmt.Errorf("failed to create NodePool: %w", err) - } - } else { - logger.Info("Updating NodePool", "name", karpenterMachinePool.Name) - if err := workloadClusterClient.Update(ctx, nodePool); err != nil { - return fmt.Errorf("failed to update NodePool: %w", err) - } + if err != nil { + return fmt.Errorf("failed to create or update NodePool: %w", err) + } + + switch operation { + case controllerutil.OperationResultCreated: + logger.Info("Created NodePool") + case controllerutil.OperationResultUpdated: + logger.Info("Updated NodePool") } return nil @@ -612,3 +678,113 @@ func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, m WithEventFilter(predicates.ResourceNotPaused(logger)). Complete(r) } + +// CompareKubernetesVersions compares two Kubernetes versions and returns: +// -1 if version1 < version2 +// +// 0 if version1 == version2 +// +// +1 if version1 > version2 +func CompareKubernetesVersions(version1, version2 string) (int, error) { + // Remove 'v' prefix if present + v1 := strings.TrimPrefix(version1, "v") + v2 := strings.TrimPrefix(version2, "v") + + parts1 := strings.Split(v1, ".") + parts2 := strings.Split(v2, ".") + + if len(parts1) < 2 || len(parts2) < 2 { + return 0, fmt.Errorf("invalid version format: %s or %s", version1, version2) + } + + // Compare major version + major1, err := strconv.Atoi(parts1[0]) + if err != nil { + return 0, fmt.Errorf("invalid major version in %s: %w", version1, err) + } + major2, err := strconv.Atoi(parts2[0]) + if err != nil { + return 0, fmt.Errorf("invalid major version in %s: %w", version2, err) + } + + if major1 != major2 { + if major1 < major2 { + return -1, nil + } + return 1, nil + } + + // Compare minor version + minor1, err := strconv.Atoi(parts1[1]) + if err != nil { + return 0, fmt.Errorf("invalid minor version in %s: %w", version1, err) + } + minor2, err := strconv.Atoi(parts2[1]) + if err != nil { + return 0, fmt.Errorf("invalid minor version in %s: %w", version2, err) + } + + if minor1 < minor2 { + return -1, nil + } else if minor1 > minor2 { + return 1, nil + } + + // If major and minor are the same, compare patch version if available + if len(parts1) >= 3 && len(parts2) >= 3 { + patch1, err := strconv.Atoi(parts1[2]) + if err != nil { + return 0, fmt.Errorf("invalid patch version in %s: %w", version1, err) + } + patch2, err := strconv.Atoi(parts2[2]) + if err != nil { + return 0, fmt.Errorf("invalid patch version in %s: %w", version2, err) + } + + if patch1 < patch2 { + return -1, nil + } else if patch1 > patch2 { + return 1, nil + } + } + + return 0, nil +} + +// IsVersionSkewAllowed checks if the worker version can be updated based on the control plane version +// According to Kubernetes version skew policy, workers can be at most 2 minor versions behind the control plane +func IsVersionSkewAllowed(controlPlaneVersion, workerVersion string) (bool, error) { + comparison, err := CompareKubernetesVersions(controlPlaneVersion, workerVersion) + if err != nil { + return false, err + } + + // If control plane version is older than or equal to worker version, allow the update + if comparison <= 0 { + return true, nil + } + + // Parse versions to check minor version difference + v1 := strings.TrimPrefix(controlPlaneVersion, "v") + v2 := strings.TrimPrefix(workerVersion, "v") + + parts1 := strings.Split(v1, ".") + parts2 := strings.Split(v2, ".") + + if len(parts1) < 2 || len(parts2) < 2 { + return false, fmt.Errorf("invalid version format: %s or %s", controlPlaneVersion, workerVersion) + } + + minor1, err := strconv.Atoi(parts1[1]) + if err != nil { + return false, fmt.Errorf("invalid minor version in %s: %w", controlPlaneVersion, err) + } + minor2, err := strconv.Atoi(parts2[1]) + if err != nil { + return false, fmt.Errorf("invalid minor version in %s: %w", workerVersion, err) + } + + // Allow if the difference is at most 2 minor versions + versionDiff := minor1 - minor2 + return versionDiff <= 2, nil +} diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index ac689108..6d6c54e8 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -820,6 +820,78 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(reconcileErr).NotTo(HaveOccurred()) }) }) + + Describe("CompareKubernetesVersions", func() { + It("should correctly compare versions", func() { + // Test cases: (version1, version2, expected_result) + testCases := []struct { + v1, v2 string + want int + }{ + {"v1.20.0", "v1.20.0", 0}, + {"v1.20.0", "v1.21.0", -1}, + {"v1.21.0", "v1.20.0", 1}, + {"v1.20.1", "v1.20.0", 1}, + {"v1.20.0", "v1.20.1", -1}, + {"1.20.0", "v1.20.0", 0}, + {"v1.20.0", "1.20.0", 0}, + } + + for _, tc := range testCases { + result, err := controllers.CompareKubernetesVersions(tc.v1, tc.v2) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(tc.want), "comparing %s with %s", tc.v1, tc.v2) + } + }) + + It("should handle invalid version formats", func() { + _, err := controllers.CompareKubernetesVersions("invalid", "v1.20.0") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid version format")) + }) + }) + + Describe("IsVersionSkewAllowed", func() { + It("should allow updates when control plane is older or equal", func() { + testCases := []struct { + controlPlane, worker string + allowed bool + }{ + {"v1.20.0", "v1.20.0", true}, // Same version + {"v1.20.0", "v1.21.0", true}, // Worker newer + {"v1.20.0", "v1.22.0", true}, // Worker newer + } + + for _, tc := range testCases { + allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) + Expect(err).NotTo(HaveOccurred()) + Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) + } + }) + + It("should allow updates within 2 minor versions", func() { + testCases := []struct { + controlPlane, worker string + allowed bool + }{ + {"v1.22.0", "v1.20.0", true}, // 2 versions behind + {"v1.22.0", "v1.21.0", true}, // 1 version behind + {"v1.22.0", "v1.19.0", false}, // 3 versions behind + {"v1.23.0", "v1.20.0", false}, // 3 versions behind + } + + for _, tc := range testCases { + allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) + Expect(err).NotTo(HaveOccurred()) + Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) + } + }) + + It("should handle invalid version formats", func() { + _, err := controllers.IsVersionSkewAllowed("invalid", "v1.20.0") + Expect(err).To(HaveOccurred()) + }) + }) }) // Helper function to create string pointers diff --git a/helm/aws-resolver-rules-operator/templates/rbac.yaml b/helm/aws-resolver-rules-operator/templates/rbac.yaml index d3a5d75b..204ad885 100644 --- a/helm/aws-resolver-rules-operator/templates/rbac.yaml +++ b/helm/aws-resolver-rules-operator/templates/rbac.yaml @@ -24,6 +24,7 @@ rules: - controlplane.cluster.x-k8s.io resources: - awsmanagedcontrolplanes + - kubeadmcontrolplanes verbs: - get - list From ec6e6438412f69a3bce7c99d526aae62879bbea0 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Mon, 30 Jun 2025 15:17:37 +0200 Subject: [PATCH 03/41] wip3 --- Makefile.gen.app.mk | 1 - Makefile.gen.go.mk | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 2 +- .../karpentermachinepool_controller.go | 31 ++- .../karpentermachinepool_controller_test.go | 234 +++++++++++++----- go.mod | 8 +- pkg/conditions/conditions.go | 4 +- 7 files changed, 191 insertions(+), 91 deletions(-) diff --git a/Makefile.gen.app.mk b/Makefile.gen.app.mk index 3f8a89c9..26a4880a 100644 --- a/Makefile.gen.app.mk +++ b/Makefile.gen.app.mk @@ -22,7 +22,6 @@ lint-chart: check-env ## Runs ct against the default chart. rm -rf /tmp/$(APPLICATION)-test mkdir -p /tmp/$(APPLICATION)-test/helm cp -a ./helm/$(APPLICATION) /tmp/$(APPLICATION)-test/helm/ - architect helm template --dir /tmp/$(APPLICATION)-test/helm/$(APPLICATION) docker run -it --rm -v /tmp/$(APPLICATION)-test:/wd --workdir=/wd --name ct $(IMAGE) ct lint --validate-maintainers=false --charts="helm/$(APPLICATION)" rm -rf /tmp/$(APPLICATION)-test diff --git a/Makefile.gen.go.mk b/Makefile.gen.go.mk index a186ca71..b31216a2 100644 --- a/Makefile.gen.go.mk +++ b/Makefile.gen.go.mk @@ -11,7 +11,7 @@ GITSHA1 := $(shell git rev-parse --verify HEAD) MODULE := $(shell go list -m) OS := $(shell go env GOOS) SOURCES := $(shell find . -name '*.go') -VERSION := $(shell architect project version) + ifeq ($(OS), linux) EXTLDFLAGS := -static endif diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1747ae85..f470cb29 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index d76c4a13..70adbce8 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -44,12 +44,8 @@ const ( KarpenterNodePoolReadyCondition capi.ConditionType = "KarpenterNodePoolReadyCondition" // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. WaitingForBootstrapDataReason = "WaitingForBootstrapData" - // NodePoolCreatedReason indicates that the NodePool was successfully created - NodePoolCreatedReason = "NodePoolCreated" // NodePoolCreationFailedReason indicates that the NodePool creation failed NodePoolCreationFailedReason = "NodePoolCreationFailed" - // EC2NodeClassCreatedReason indicates that the EC2NodeClass was successfully created - EC2NodeClassCreatedReason = "EC2NodeClassCreated" // EC2NodeClassCreationFailedReason indicates that the EC2NodeClass creation failed EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" // VersionSkewBlockedReason indicates that the update was blocked due to version skew policy @@ -155,6 +151,12 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } } + // Create or update Karpenter resources in the workload cluster + if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool, bootstrapSecretValue); err != nil { + logger.Error(err, "failed to create or update Karpenter resources") + return reconcile.Result{}, err + } + bootstrapUserDataHash := fmt.Sprintf("%x", sha256.Sum256(bootstrapSecretValue)) previousHash, annotationHashExists := karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] if !annotationHashExists || previousHash != bootstrapUserDataHash { @@ -181,12 +183,6 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } } - // Create or update Karpenter resources in the workload cluster - if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool, bootstrapSecretValue); err != nil { - logger.Error(err, "failed to create or update Karpenter resources") - return reconcile.Result{}, err - } - providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) if err != nil { return reconcile.Result{}, err @@ -407,7 +403,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { ec2NodeClassGVR := schema.GroupVersionResource{ Group: "karpenter.k8s.aws", - Version: "v1beta1", + Version: "v1", Resource: "ec2nodeclasses", } @@ -485,7 +481,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { nodePoolGVR := schema.GroupVersionResource{ Group: "karpenter.sh", - Version: "v1beta1", + Version: "v1", Resource: "nodepools", } @@ -503,10 +499,12 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "template": map[string]interface{}{ "spec": map[string]interface{}{ "nodeClassRef": map[string]interface{}{ - "apiVersion": "karpenter.k8s.aws/v1beta1", + "apiVersion": "karpenter.k8s.aws/v1", + "group": "karpenter.k8s.aws", "kind": "EC2NodeClass", "name": karpenterMachinePool.Name, }, + "requirements": []interface{}{}, }, }, } @@ -547,7 +545,8 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont } requirements = append(requirements, requirement) } - spec["requirements"] = requirements + + spec["template"].(map[string]interface{})["spec"].(map[string]interface{})["requirements"] = requirements } if len(karpenterMachinePool.Spec.NodePool.Taints) > 0 { @@ -602,7 +601,7 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co // Delete NodePool nodePoolGVR := schema.GroupVersionResource{ Group: "karpenter.sh", - Version: "v1beta1", + Version: "v1", Resource: "nodepools", } @@ -619,7 +618,7 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co // Delete EC2NodeClass ec2NodeClassGVR := schema.GroupVersionResource{ Group: "karpenter.k8s.aws", - Version: "v1beta1", + Version: "v1", Resource: "ec2nodeclasses", } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 6d6c54e8..e9827616 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -241,6 +241,24 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { When("the owner cluster is also being deleted", func() { BeforeEach(func() { + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": KarpenterMachinePoolNamespace, + }, + "spec": map[string]interface{}{ + "version": "v1.21.2", + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err := fakeCtrlClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ Namespace: KarpenterMachinePoolNamespace, @@ -251,15 +269,22 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Finalizers: []string{"something-to-keep-it-around-when-deleting"}, }, Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: KarpenterMachinePoolNamespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", Namespace: KarpenterMachinePoolNamespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, + Topology: nil, }, } - err := fakeCtrlClient.Create(ctx, cluster) + err = fakeCtrlClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) err = fakeCtrlClient.Delete(ctx, cluster) @@ -322,6 +347,9 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + NodePool: &karpenterinfra.NodePoolSpec{}, + }, } err := fakeCtrlClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) @@ -434,6 +462,12 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capi.ClusterSpec{ Paused: true, + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: KarpenterMachinePoolNamespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", Namespace: KarpenterMachinePoolNamespace, @@ -445,6 +479,24 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err := fakeCtrlClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": KarpenterMachinePoolNamespace, + }, + "spec": map[string]interface{}{ + "version": "v1.21.2", + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = fakeCtrlClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ Namespace: KarpenterMachinePoolNamespace, @@ -470,6 +522,12 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: KarpenterMachinePoolNamespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", Namespace: KarpenterMachinePoolNamespace, @@ -480,6 +538,24 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { } err := fakeCtrlClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) + + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": KarpenterMachinePoolNamespace, + }, + "spec": map[string]interface{}{ + "version": "v1.21.2", + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = fakeCtrlClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) }) When("there is no AWSCluster", func() { It("returns an error", func() { @@ -577,6 +653,33 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err := fakeCtrlClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) + It("creates karpenter resources in the wc", func() { + Expect(reconcileErr).NotTo(HaveOccurred()) + + nodepoolList := &unstructured.UnstructuredList{} + nodepoolList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.sh", + Kind: "NodePoolList", + Version: "v1", + }) + + err := fakeCtrlClient.List(ctx, nodepoolList) + Expect(err).NotTo(HaveOccurred()) + Expect(nodepoolList.Items).To(HaveLen(1)) + Expect(nodepoolList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) + + ec2nodeclassList := &unstructured.UnstructuredList{} + ec2nodeclassList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.k8s.aws", + Kind: "EC2NodeClassList", + Version: "v1", + }) + + err = fakeCtrlClient.List(ctx, ec2nodeclassList) + Expect(err).NotTo(HaveOccurred()) + Expect(ec2nodeclassList.Items).To(HaveLen(1)) + Expect(ec2nodeclassList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) + }) It("adds the finalizer to the KarpenterMachinePool", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} @@ -821,80 +924,77 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) }) - Describe("CompareKubernetesVersions", func() { - It("should correctly compare versions", func() { - // Test cases: (version1, version2, expected_result) - testCases := []struct { - v1, v2 string - want int - }{ - {"v1.20.0", "v1.20.0", 0}, - {"v1.20.0", "v1.21.0", -1}, - {"v1.21.0", "v1.20.0", 1}, - {"v1.20.1", "v1.20.0", 1}, - {"v1.20.0", "v1.20.1", -1}, - {"1.20.0", "v1.20.0", 0}, - {"v1.20.0", "1.20.0", 0}, - } + Describe("Version comparison functions", func() { + Describe("CompareKubernetesVersions", func() { + It("should correctly compare versions", func() { + // Test cases: (version1, version2, expected_result) + testCases := []struct { + v1, v2 string + want int + }{ + {"v1.20.0", "v1.20.0", 0}, + {"v1.20.0", "v1.21.0", -1}, + {"v1.21.0", "v1.20.0", 1}, + {"v1.20.1", "v1.20.0", 1}, + {"v1.20.0", "v1.20.1", -1}, + {"1.20.0", "v1.20.0", 0}, + {"v1.20.0", "1.20.0", 0}, + } - for _, tc := range testCases { - result, err := controllers.CompareKubernetesVersions(tc.v1, tc.v2) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(tc.want), "comparing %s with %s", tc.v1, tc.v2) - } - }) + for _, tc := range testCases { + result, err := controllers.CompareKubernetesVersions(tc.v1, tc.v2) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(tc.want), "comparing %s with %s", tc.v1, tc.v2) + } + }) - It("should handle invalid version formats", func() { - _, err := controllers.CompareKubernetesVersions("invalid", "v1.20.0") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("invalid version format")) + It("should handle invalid version formats", func() { + _, err := controllers.CompareKubernetesVersions("invalid", "v1.20.0") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("invalid version format")) + }) }) - }) - Describe("IsVersionSkewAllowed", func() { - It("should allow updates when control plane is older or equal", func() { - testCases := []struct { - controlPlane, worker string - allowed bool - }{ - {"v1.20.0", "v1.20.0", true}, // Same version - {"v1.20.0", "v1.21.0", true}, // Worker newer - {"v1.20.0", "v1.22.0", true}, // Worker newer - } + Describe("IsVersionSkewAllowed", func() { + It("should allow updates when control plane is older or equal", func() { + testCases := []struct { + controlPlane, worker string + allowed bool + }{ + {"v1.20.0", "v1.20.0", true}, // Same version + {"v1.20.0", "v1.21.0", true}, // Worker newer + {"v1.20.0", "v1.22.0", true}, // Worker newer + } - for _, tc := range testCases { - allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) - Expect(err).NotTo(HaveOccurred()) - Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) - } - }) + for _, tc := range testCases { + allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) + Expect(err).NotTo(HaveOccurred()) + Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) + } + }) - It("should allow updates within 2 minor versions", func() { - testCases := []struct { - controlPlane, worker string - allowed bool - }{ - {"v1.22.0", "v1.20.0", true}, // 2 versions behind - {"v1.22.0", "v1.21.0", true}, // 1 version behind - {"v1.22.0", "v1.19.0", false}, // 3 versions behind - {"v1.23.0", "v1.20.0", false}, // 3 versions behind - } + It("should allow updates within 2 minor versions", func() { + testCases := []struct { + controlPlane, worker string + allowed bool + }{ + {"v1.22.0", "v1.20.0", true}, // 2 versions behind + {"v1.22.0", "v1.21.0", true}, // 1 version behind + {"v1.22.0", "v1.19.0", false}, // 3 versions behind + {"v1.23.0", "v1.20.0", false}, // 3 versions behind + } - for _, tc := range testCases { - allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) - Expect(err).NotTo(HaveOccurred()) - Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) - } - }) + for _, tc := range testCases { + allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) + Expect(err).NotTo(HaveOccurred()) + Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) + } + }) - It("should handle invalid version formats", func() { - _, err := controllers.IsVersionSkewAllowed("invalid", "v1.20.0") - Expect(err).To(HaveOccurred()) + It("should handle invalid version formats", func() { + _, err := controllers.IsVersionSkewAllowed("invalid", "v1.20.0") + Expect(err).To(HaveOccurred()) + }) }) }) }) - -// Helper function to create string pointers -func stringPtr(s string) *string { - return &s -} diff --git a/go.mod b/go.mod index 72011b42..10d59624 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/aws-resolver-rules-operator -go 1.23.0 +go 1.24.0 + +toolchain go1.24.3 require ( github.com/aws/aws-sdk-go v1.55.7 @@ -26,8 +28,6 @@ require ( sigs.k8s.io/yaml v1.5.0 ) -replace google.golang.org/protobuf v1.31.0 => google.golang.org/protobuf v1.33.0 - require ( github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -90,3 +90,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) + +replace google.golang.org/protobuf v1.31.0 => google.golang.org/protobuf v1.33.0 diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 4f7e2f58..a89132a5 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -50,7 +50,7 @@ func MarkNodePoolReady(setter capiconditions.Setter) { } func MarkNodePoolNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, NodePoolReady, reason, capi.ConditionSeverityError, message) + capiconditions.MarkFalse(setter, NodePoolReady, reason, capi.ConditionSeverityError, message, nil) } func MarkEC2NodeClassReady(setter capiconditions.Setter) { @@ -58,5 +58,5 @@ func MarkEC2NodeClassReady(setter capiconditions.Setter) { } func MarkEC2NodeClassNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, EC2NodeClassReady, reason, capi.ConditionSeverityError, message) + capiconditions.MarkFalse(setter, EC2NodeClassReady, reason, capi.ConditionSeverityError, message, nil) } From d8163feca903a4a72cd4cf4df4a34c34798141b0 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Wed, 9 Jul 2025 16:24:55 +0200 Subject: [PATCH 04/41] Use envtest k8s client --- Makefile.custom.mk | 8 - api/v1alpha1/karpentermachinepool_types.go | 11 +- api/v1alpha1/zz_generated.deepcopy.go | 5 - ...luster.x-k8s.io_karpentermachinepools.yaml | 11 +- controllers/controllers_suite_test.go | 5 + .../karpentermachinepool_controller.go | 77 +- .../karpentermachinepool_controller_test.go | 568 ++++++++---- controllers/route_controller_test.go | 6 +- go.mod | 1 - ...luster.x-k8s.io_karpentermachinepools.yaml | 11 +- .../karpenter.k8s.aws_ec2nodeclasses.yaml | 847 ++++++++++++++++++ .../crds/karpenter.sh_nodeclaims.yaml | 399 +++++++++ .../testdata/crds/karpenter.sh_nodepools.yaml | 525 +++++++++++ 13 files changed, 2229 insertions(+), 245 deletions(-) create mode 100644 tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml create mode 100644 tests/testdata/crds/karpenter.sh_nodeclaims.yaml create mode 100644 tests/testdata/crds/karpenter.sh_nodepools.yaml diff --git a/Makefile.custom.mk b/Makefile.custom.mk index ac676670..364cfe69 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -27,14 +27,6 @@ generate: controller-gen crds ## Generate code containing DeepCopy, DeepCopyInto go generate ./... $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - .PHONY: create-acceptance-cluster create-acceptance-cluster: kind KIND=$(KIND) CLUSTER=$(CLUSTER) IMG=$(IMG) MANAGEMENT_CLUSTER_NAMESPACE=$(MANAGEMENT_CLUSTER_NAMESPACE) ./scripts/ensure-kind-cluster.sh diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index 9707607b..c74474dc 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -114,9 +114,14 @@ type TaintSpec struct { // EC2NodeClassSpec defines the configuration for a Karpenter EC2NodeClass type EC2NodeClassSpec struct { - // AMIID specifies the AMI ID to use - // +optional - AMIID *string `json:"amiId,omitempty"` + // Name is the ami name in EC2. + // This value is the name field, which is different from the name tag. + AMIName string `json:"amiName,omitempty"` + // Owner is the owner for the ami. + // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + AMIOwner string `json:"amiOwner,omitempty"` + // - name: flatcar-stable-{{ $.Values.baseOS }}-kube-{{ $.Values.k8sVersion }}-tooling-{{ $.Values.toolingVersion }}-gs + // // owner: {{ int64 $.Values.amiOwner | quote }} // SecurityGroups specifies the security groups to use // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index f470cb29..d0a5f151 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -84,11 +84,6 @@ func (in *DisruptionSpec) DeepCopy() *DisruptionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = *in - if in.AMIID != nil { - in, out := &in.AMIID, &out.AMIID - *out = new(string) - **out = **in - } if in.SecurityGroups != nil { in, out := &in.SecurityGroups, &out.SecurityGroups *out = make([]string, len(*in)) diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 6b83883d..0b11c968 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -54,8 +54,15 @@ spec: description: EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass properties: - amiId: - description: AMIID specifies the AMI ID to use + amiName: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + amiOwner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string securityGroups: description: SecurityGroups specifies the security groups to use diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index fa80df42..031d28f7 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -39,6 +39,7 @@ import ( capi "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -89,8 +90,10 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", fmt.Sprintf("cluster-api@%s", capiModule[0].Module.Version), "config", "crd", "bases"), + filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", fmt.Sprintf("cluster-api@%s", capiModule[0].Module.Version), "controlplane", "kubeadm", "config", "crd", "bases"), filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", "cluster-api-provider-aws", fmt.Sprintf("v2@%s", capaModule[0].Module.Version), "config", "crd", "bases"), crdPath, + filepath.Join("..", "config", "crd", "bases"), }, ErrorIfCRDPathMissing: true, } @@ -111,6 +114,8 @@ var _ = BeforeSuite(func() { k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) + + komega.SetClient(k8sClient) }) var _ = AfterSuite(func() { diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 70adbce8..98380ded 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -196,7 +196,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco karpenterMachinePool.Status.Replicas = numberOfNodeClaims karpenterMachinePool.Status.Ready = true - logger.Info("Found NodeClaims in workload cluster, patching KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims) + logger.Info("Found NodeClaims in workload cluster, patching KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { logger.Error(err, "failed to patch karpenterMachinePool.status.Replicas") @@ -293,6 +293,7 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo logger.Error(err, "error retrieving nodeClaim.status.providerID", "nodeClaim", nc.GetName()) continue } + logger.Info("nodeClaim.status.providerID", "nodeClaimName", nc.GetName(), "statusFieldFound", found, "nodeClaim", nc.Object) if found && providerID != "" { providerIDList = append(providerIDList, providerID) } @@ -413,46 +414,66 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. ec2NodeClass.SetNamespace("default") // Generate user data for Ignition - userData := r.generateUserData(awsCluster.Spec.Region, cluster.Name, karpenterMachinePool.Name) + userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, cluster.Name, karpenterMachinePool.Name) operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { // Build the EC2NodeClass spec spec := map[string]interface{}{ "amiFamily": "AL2", - "role": karpenterMachinePool.Spec.IamInstanceProfile, - "userData": userData, - } - - // Add AMI ID if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && karpenterMachinePool.Spec.EC2NodeClass.AMIID != nil { - spec["amiSelectorTerms"] = []map[string]interface{}{ + "amiSelectorTerms": []map[string]interface{}{ { - "id": *karpenterMachinePool.Spec.EC2NodeClass.AMIID, + "name": karpenterMachinePool.Spec.EC2NodeClass.AMIName, + "owner": karpenterMachinePool.Spec.EC2NodeClass.AMIOwner, }, - } - } - - // Add security groups if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { - spec["securityGroupSelectorTerms"] = []map[string]interface{}{ + }, + "instanceProfile": karpenterMachinePool.Spec.IamInstanceProfile, + "securityGroupSelectorTerms": []map[string]interface{}{ { "tags": map[string]string{ "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now }, }, - } + }, + "subnetSelectorTerms": []map[string]interface{}{ + { + "tags": map[string]string{ + "Name": karpenterMachinePool.Spec.EC2NodeClass.Subnets[0], // Using first security group for now + }, + }, + }, + "userData": userData, } + // Add AMI ID if specified + // if karpenterMachinePool.Spec.EC2NodeClass != nil && karpenterMachinePool.Spec.EC2NodeClass.AMIID != nil { + // spec["amiSelectorTerms"] = []map[string]interface{}{ + // { + // "id": *karpenterMachinePool.Spec.EC2NodeClass.AMIID, + // }, + // } + // } + + // Add security groups if specified + // if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { + // spec["securityGroupSelectorTerms"] = []map[string]interface{}{ + // { + // "tags": map[string]string{ + // "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now + // }, + // }, + // } + // } + // Add subnets if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { - subnetSelectorTerms := []map[string]interface{}{} - for _, subnet := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { - subnetSelectorTerms = append(subnetSelectorTerms, map[string]interface{}{ - "id": subnet, - }) - } - spec["subnetSelectorTerms"] = subnetSelectorTerms - } + // if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { + // subnetSelectorTerms := []map[string]interface{}{} + // for _, subnet := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { + // subnetSelectorTerms = append(subnetSelectorTerms, map[string]interface{}{ + // "id": subnet, + // }) + // } + // spec["subnetSelectorTerms"] = subnetSelectorTerms + // } // Add tags if specified if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Tags) > 0 { @@ -636,13 +657,13 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co } // generateUserData generates the user data for Ignition configuration -func (r *KarpenterMachinePoolReconciler) generateUserData(region, clusterName, karpenterMachinePoolName string) string { +func (r *KarpenterMachinePoolReconciler) generateUserData(s3bucketName, clusterName, karpenterMachinePoolName string) string { userData := map[string]interface{}{ "ignition": map[string]interface{}{ "config": map[string]interface{}{ "merge": []map[string]interface{}{ { - "source": fmt.Sprintf("s3://%s-capa-%s/%s/%s", region, clusterName, S3ObjectPrefix, karpenterMachinePoolName), + "source": fmt.Sprintf("s3://%s/%s/%s-%s", s3bucketName, S3ObjectPrefix, clusterName, karpenterMachinePoolName), "verification": map[string]interface{}{}, }, }, diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index e9827616..85f5ec39 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -17,11 +17,10 @@ import ( "k8s.io/kubectl/pkg/scheme" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" capi "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" capiexp "sigs.k8s.io/cluster-api/exp/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" "sigs.k8s.io/controller-runtime/pkg/reconcile" karpenterinfra "github.com/aws-resolver-rules-operator/api/v1alpha1" @@ -37,8 +36,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { dataSecretName string s3Client *resolverfakes.FakeS3Client ec2Client *resolverfakes.FakeEC2Client - fakeCtrlClient client.Client - fakeClusterClientGetter remote.ClusterClientGetter ctx context.Context reconciler *controllers.KarpenterMachinePoolReconciler reconcileErr error @@ -46,11 +43,16 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ) const ( + AMIName = "flatcar-stable-1234.5-kube-1.25.1-tooling-1.2.3-gs" + AMIOwner = "1234567890" + AWSRegion = "eu-west-1" ClusterName = "foo" AWSClusterBucketName = "my-awesome-bucket" DataSecretName = "foo-mp-12345" KarpenterMachinePoolName = "foo" - KarpenterMachinePoolNamespace = "org-bar" + KarpenterNodesInstanceProfile = "karpenter-iam-role" + KarpenterNodesSecurityGroup = "sg-12345678" + KarpenterNodesSubnets = "subnet-12345678" KubernetesVersion = "v1.29.1" ) @@ -72,15 +74,14 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err = karpenterinfra.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - fakeCtrlClient = fake.NewClientBuilder(). - WithScheme(scheme.Scheme). - WithStatusSubresource(&karpenterinfra.KarpenterMachinePool{}). - Build() + // fakeCtrlClient := fake.NewClientBuilder(). + // WithScheme(scheme.Scheme). + // // WithStatusSubresource(&karpenterinfra.KarpenterMachinePool{}). + // Build() - // Use the default fake cluster client getter - fakeClusterClientGetter = func(ctx context.Context, _ string, _ client.Client, _ client.ObjectKey) (client.Client, error) { + workloadClusterClientGetter := func(ctx context.Context, _ string, _ client.Client, _ client.ObjectKey) (client.Client, error) { // Return the same client that we're using for the test - return fakeCtrlClient, nil + return k8sClient, nil } s3Client = new(resolverfakes.FakeS3Client) @@ -91,13 +92,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { EC2Client: ec2Client, } - reconciler = controllers.NewKarpenterMachinepoolReconciler(fakeCtrlClient, fakeClusterClientGetter, clientsFactory) + reconciler = controllers.NewKarpenterMachinepoolReconciler(k8sClient, workloadClusterClientGetter, clientsFactory) }) JustBeforeEach(func() { request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, }, } @@ -114,11 +115,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, }, } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) + err := k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) It("does nothing", func() { @@ -128,34 +129,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { When("the KarpenterMachinePool is being deleted", func() { BeforeEach(func() { - karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: KarpenterMachinePoolName, - Labels: map[string]string{ - capi.ClusterNameLabel: ClusterName, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "MachinePool", - Name: KarpenterMachinePoolName, - }, - }, - Finalizers: []string{controllers.KarpenterFinalizer}, - }, - } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - - err = fakeCtrlClient.Delete(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - dataSecretName = DataSecretName version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -170,7 +148,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, @@ -178,7 +156,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -187,37 +165,63 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err = fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + Finalizers: []string{controllers.KarpenterFinalizer}, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Delete(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) awsCluster := &capa.AWSCluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, }, Spec: capa.AWSClusterSpec{ IdentityRef: &capa.AWSIdentityReference{ - Name: "default", + Name: "default-delete-test", Kind: capa.ClusterRoleIdentityKind, }, S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, }, } - err = fakeCtrlClient.Create(ctx, awsCluster) + err = k8sClient.Create(ctx, awsCluster) Expect(err).NotTo(HaveOccurred()) clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-kubeconfig", ClusterName), }, } - err = fakeCtrlClient.Create(ctx, clusterKubeconfigSecret) + err = k8sClient.Create(ctx, clusterKubeconfigSecret) Expect(err).NotTo(HaveOccurred()) awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ ObjectMeta: metav1.ObjectMeta{ - Name: "default", + Name: "default-delete-test", }, Spec: capa.AWSClusterRoleIdentitySpec{ AWSRoleSpec: capa.AWSRoleSpec{ @@ -225,17 +229,17 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err = fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) + err = k8sClient.Create(ctx, awsClusterRoleIdentity) Expect(err).NotTo(HaveOccurred()) bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"value": capiBootstrapSecretContent}, } - err = fakeCtrlClient.Create(ctx, bootstrapSecret) + err = k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) @@ -245,9 +249,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { kubeadmControlPlane.Object = map[string]interface{}{ "metadata": map[string]interface{}{ "name": ClusterName, - "namespace": KarpenterMachinePoolNamespace, + "namespace": namespace, }, "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, "version": "v1.21.2", }, } @@ -256,38 +264,38 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "KubeadmControlPlane", Version: "v1beta1", }) - err := fakeCtrlClient.Create(ctx, kubeadmControlPlane) + err := k8sClient.Create(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, }, - Finalizers: []string{"something-to-keep-it-around-when-deleting"}, + Finalizers: []string{"giantswarm.io/something-to-keep-it-around-when-deleting"}, }, Spec: capi.ClusterSpec{ ControlPlaneRef: &v1.ObjectReference{ Kind: "KubeadmControlPlane", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, Topology: nil, }, } - err = fakeCtrlClient.Create(ctx, cluster) + err = k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) - err = fakeCtrlClient.Delete(ctx, cluster) + err = k8sClient.Delete(ctx, cluster) Expect(err).NotTo(HaveOccurred()) }) // This test is a bit cumbersome because we are deleting CRs, so we can't use different `It` blocks or the CRs would be gone. @@ -306,7 +314,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(reconcileResult.RequeueAfter).To(Equal(30 * time.Second)) karpenterMachinePoolList := &karpenterinfra.KarpenterMachinePoolList{} - err := fakeCtrlClient.List(ctx, karpenterMachinePoolList) + err := k8sClient.List(ctx, karpenterMachinePoolList, client.InNamespace(namespace)) Expect(err).NotTo(HaveOccurred()) // Finalizer should be there blocking the deletion of the CR Expect(karpenterMachinePoolList.Items).To(HaveLen(1)) @@ -315,13 +323,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { reconcileResult, reconcileErr = reconciler.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, }, }) karpenterMachinePoolList = &karpenterinfra.KarpenterMachinePoolList{} - err = fakeCtrlClient.List(ctx, karpenterMachinePoolList) + err = k8sClient.List(ctx, karpenterMachinePoolList, client.InNamespace(namespace)) Expect(err).NotTo(HaveOccurred()) // Finalizer should've been removed and the CR should be gone Expect(karpenterMachinePoolList.Items).To(HaveLen(0)) @@ -331,30 +339,32 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) When("the KarpenterMachinePool exists and it has a MachinePool owner", func() { - BeforeEach(func() { - karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: KarpenterMachinePoolName, - Labels: map[string]string{ - capi.ClusterNameLabel: ClusterName, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "MachinePool", - Name: KarpenterMachinePoolName, + When("the referenced MachinePool does not exist", func() { + BeforeEach(func() { + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: "12345678-1234-1234-1234-123456789012", + }, }, }, - }, - Spec: karpenterinfra.KarpenterMachinePoolSpec{ - NodePool: &karpenterinfra.NodePoolSpec{}, - }, - } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - }) - When("the referenced MachinePool does not exist", func() { + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{}, + NodePool: &karpenterinfra.NodePoolSpec{}, + }, + } + err := k8sClient.Create(ctx, karpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + }) It("returns an error", func() { Expect(reconcileErr).To(MatchError(ContainSubstring("failed to get MachinePool owning the KarpenterMachinePool"))) }) @@ -364,7 +374,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -376,18 +386,18 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Template: capi.MachineTemplateSpec{ ObjectMeta: capi.ObjectMeta{}, Spec: capi.MachineSpec{ - ClusterName: "", + ClusterName: ClusterName, Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -396,7 +406,30 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err := fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{}, + } + err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) It("returns early", func() { @@ -409,7 +442,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -425,7 +458,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, @@ -433,7 +466,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -442,7 +475,41 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err := fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + AMIName: AMIName, + AMIOwner: AMIOwner, + SecurityGroups: []string{KarpenterNodesSecurityGroup}, + Subnets: []string{KarpenterNodesSubnets}, + // UserData: nil, + // Tags: nil, + }, + IamInstanceProfile: KarpenterNodesInstanceProfile, + NodePool: &karpenterinfra.NodePoolSpec{}, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) When("there is no Cluster owning the MachinePool", func() { @@ -454,7 +521,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -464,28 +531,32 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Paused: true, ControlPlaneRef: &v1.ObjectReference{ Kind: "KubeadmControlPlane", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, }, } - err := fakeCtrlClient.Create(ctx, cluster) + err := k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) kubeadmControlPlane := &unstructured.Unstructured{} kubeadmControlPlane.Object = map[string]interface{}{ "metadata": map[string]interface{}{ "name": ClusterName, - "namespace": KarpenterMachinePoolNamespace, + "namespace": namespace, }, "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, "version": "v1.21.2", }, } @@ -494,16 +565,16 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "KubeadmControlPlane", Version: "v1beta1", }) - err = fakeCtrlClient.Create(ctx, kubeadmControlPlane) + err = k8sClient.Create(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-kubeconfig", ClusterName), }, } - err = fakeCtrlClient.Create(ctx, clusterKubeconfigSecret) + err = k8sClient.Create(ctx, clusterKubeconfigSecret) Expect(err).NotTo(HaveOccurred()) }) It("returns early", func() { @@ -515,7 +586,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -524,28 +595,32 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Spec: capi.ClusterSpec{ ControlPlaneRef: &v1.ObjectReference{ Kind: "KubeadmControlPlane", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, }, } - err := fakeCtrlClient.Create(ctx, cluster) + err := k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) kubeadmControlPlane := &unstructured.Unstructured{} kubeadmControlPlane.Object = map[string]interface{}{ "metadata": map[string]interface{}{ "name": ClusterName, - "namespace": KarpenterMachinePoolNamespace, + "namespace": namespace, }, "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, "version": "v1.21.2", }, } @@ -554,7 +629,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "KubeadmControlPlane", Version: "v1beta1", }) - err = fakeCtrlClient.Create(ctx, kubeadmControlPlane) + err = k8sClient.Create(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) }) When("there is no AWSCluster", func() { @@ -566,7 +641,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { awsCluster := &capa.AWSCluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -574,7 +649,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capa.AWSClusterSpec{}, } - err := fakeCtrlClient.Create(ctx, awsCluster) + err := k8sClient.Create(ctx, awsCluster) Expect(err).NotTo(HaveOccurred()) }) It("returns an error", func() { @@ -582,30 +657,47 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) }) When("the AWSCluster exists and there is a S3 bucket defined on it", func() { - BeforeEach(func() { - awsCluster := &capa.AWSCluster{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: ClusterName, - }, - Spec: capa.AWSClusterSpec{ - IdentityRef: &capa.AWSIdentityReference{ - Name: "default", - Kind: capa.ClusterRoleIdentityKind, - }, - S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, - }, - } - err := fakeCtrlClient.Create(ctx, awsCluster) - Expect(err).NotTo(HaveOccurred()) - }) When("it can't find the identity used by the AWSCluster", func() { + BeforeEach(func() { + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + IdentityRef: &capa.AWSIdentityReference{ + Name: "not-referenced-by-test", + Kind: capa.ClusterRoleIdentityKind, + }, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err := k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + }) It("returns an error", func() { Expect(reconcileErr).To(MatchError(ContainSubstring("failed to get AWSClusterRoleIdentity referenced in AWSCluster"))) }) }) When("it finds the identity used by the AWSCluster", func() { BeforeEach(func() { + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + IdentityRef: &capa.AWSIdentityReference{ + Name: "default", + Kind: capa.ClusterRoleIdentityKind, + }, + Region: AWSRegion, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err := k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ ObjectMeta: metav1.ObjectMeta{ Name: "default", @@ -616,8 +708,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err := fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) - Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) }) When("the bootstrap secret referenced in the dataSecretName field does not exist", func() { @@ -629,12 +724,12 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"not-what-we-expect": capiBootstrapSecretContent}, } - err := fakeCtrlClient.Create(ctx, bootstrapSecret) + err := k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) It("returns an error", func() { @@ -645,12 +740,12 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"value": capiBootstrapSecretContent}, } - err := fakeCtrlClient.Create(ctx, bootstrapSecret) + err := k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) It("creates karpenter resources in the wc", func() { @@ -663,7 +758,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Version: "v1", }) - err := fakeCtrlClient.List(ctx, nodepoolList) + err := k8sClient.List(ctx, nodepoolList) Expect(err).NotTo(HaveOccurred()) Expect(nodepoolList.Items).To(HaveLen(1)) Expect(nodepoolList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) @@ -675,15 +770,69 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Version: "v1", }) - err = fakeCtrlClient.List(ctx, ec2nodeclassList) + err = k8sClient.List(ctx, ec2nodeclassList) Expect(err).NotTo(HaveOccurred()) Expect(ec2nodeclassList.Items).To(HaveLen(1)) Expect(ec2nodeclassList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) + amiSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "amiSelectorTerms") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(amiSelectorTerms).To(HaveLen(1)) + + // Let's make sure the amiSelectorTerms field is what we expect + term0, ok := amiSelectorTerms[0].(map[string]interface{}) + Expect(ok).To(BeTrue(), "expected amiSelectorTerms[0] to be a map") + // Assert the name field + nameVal, ok := term0["name"].(string) + Expect(ok).To(BeTrue(), "expected name to be a string") + Expect(nameVal).To(Equal(AMIName)) + // Assert the owner field + ownerF, ok := term0["owner"].(string) + Expect(ok).To(BeTrue(), "expected owner to be a number") + Expect(ownerF).To(Equal(AMIOwner)) + + // Assert security groups are the expected ones + securityGroupSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "securityGroupSelectorTerms") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(securityGroupSelectorTerms).To(HaveLen(1)) + // Let's make sure the securityGroupSelectorTerms field is what we expect + securityGroupSelectorTerm0, ok := securityGroupSelectorTerms[0].(map[string]interface{}) + Expect(ok).To(BeTrue(), "expected securityGroupSelectorTerms[0] to be a map") + // Assert the security group name field + securityGroupTags, ok := securityGroupSelectorTerm0["tags"].(map[string]interface{}) + Expect(ok).To(BeTrue(), "expected tags to be a map[string]string") + Expect(securityGroupTags["Name"]).To(Equal(KarpenterNodesSecurityGroup)) + + // Assert subnets are the expected ones + subnetSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "subnetSelectorTerms") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(subnetSelectorTerms).To(HaveLen(1)) + // Let's make sure the subnetSelectorTerms field is what we expect + subnetSelectorTerm0, ok := subnetSelectorTerms[0].(map[string]interface{}) + Expect(ok).To(BeTrue(), "expected subnetSelectorTerms[0] to be a map") + // Assert the security group name field + subnetTags, ok := subnetSelectorTerm0["tags"].(map[string]interface{}) + Expect(ok).To(BeTrue(), "expected tags to be a map[string]string") + Expect(subnetTags["Name"]).To(Equal(KarpenterNodesSubnets)) + + // Assert userdata is the expected one + userData, found, err := unstructured.NestedString(ec2nodeclassList.Items[0].Object, "spec", "userData") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(userData).To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s-%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, ClusterName, KarpenterMachinePoolName))) + + // Assert instance profile is the expected one + iamInstanceProfile, found, err := unstructured.NestedString(ec2nodeclassList.Items[0].Object, "spec", "instanceProfile") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(iamInstanceProfile).To(Equal(KarpenterNodesInstanceProfile)) }) It("adds the finalizer to the KarpenterMachinePool", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.GetFinalizers()).To(ContainElement(controllers.KarpenterFinalizer)) }) @@ -698,7 +847,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { It("writes annotation containing bootstrap data hash", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.Annotations).To(HaveKeyWithValue(controllers.BootstrapDataHashAnnotation, Equal(capiBootstrapSecretHash))) }) @@ -707,7 +856,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(reconcileErr).NotTo(HaveOccurred()) Expect(reconcileResult.RequeueAfter).To(Equal(1 * time.Minute)) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.Status.Ready).To(BeFalse()) Expect(updatedKarpenterMachinePool.Status.Replicas).To(BeZero()) @@ -721,9 +870,19 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { "metadata": map[string]interface{}{ "name": fmt.Sprintf("%s-z9y8x", KarpenterMachinePoolName), }, - "spec": map[string]interface{}{}, - "status": map[string]interface{}{ - "providerID": "aws:///us-west-2a/i-1234567890abcdef0", + "spec": map[string]interface{}{ + "nodeClassRef": map[string]interface{}{ + "group": "karpenter.k8s.aws", + "kind": "EC2NodeClass", + "name": "default", + }, + "requirements": []interface{}{ + map[string]interface{}{ + "key": "kubernetes.io/arch", + "operator": "In", + "values": []string{"amd64"}, + }, + }, }, } nodeClaim1.SetGroupVersionKind(schema.GroupVersionKind{ @@ -731,17 +890,30 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "NodeClaim", Version: "v1", }) - err := fakeCtrlClient.Create(ctx, nodeClaim1) + err := k8sClient.Create(ctx, nodeClaim1) Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(nodeClaim1.Object, map[string]interface{}{"providerID": "aws:///us-west-2a/i-1234567890abcdef0"}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, nodeClaim1) nodeClaim2 := &unstructured.Unstructured{} nodeClaim2.Object = map[string]interface{}{ "metadata": map[string]interface{}{ "name": fmt.Sprintf("%s-m0n1o", KarpenterMachinePoolName), }, - "spec": map[string]interface{}{}, - "status": map[string]interface{}{ - "providerID": "aws:///us-west-2a/i-09876543219fedcba", + "spec": map[string]interface{}{ + "nodeClassRef": map[string]interface{}{ + "group": "karpenter.k8s.aws", + "kind": "EC2NodeClass", + "name": "default", + }, + "requirements": []interface{}{ + map[string]interface{}{ + "key": "kubernetes.io/arch", + "operator": "In", + "values": []string{"amd64"}, + }, + }, }, } nodeClaim2.SetGroupVersionKind(schema.GroupVersionKind{ @@ -749,13 +921,16 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "NodeClaim", Version: "v1", }) - err = fakeCtrlClient.Create(ctx, nodeClaim2) + err = k8sClient.Create(ctx, nodeClaim2) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(nodeClaim2.Object, map[string]interface{}{"providerID": "aws:///us-west-2a/i-09876543219fedcba"}, "status") Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, nodeClaim2) }) It("updates the KarpenterMachinePool spec and status accordingly", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.Status.Ready).To(BeTrue()) Expect(updatedKarpenterMachinePool.Status.Replicas).To(Equal(int32(2))) @@ -779,33 +954,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) When("the KarpenterMachinePool exists with a hash annotation signaling unchanged bootstrap data", func() { BeforeEach(func() { - karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: KarpenterMachinePoolName, - Annotations: map[string]string{ - controllers.BootstrapDataHashAnnotation: capiBootstrapSecretHash, - }, - Labels: map[string]string{ - capi.ClusterNameLabel: ClusterName, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "MachinePool", - Name: KarpenterMachinePoolName, - }, - }, - }, - } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - dataSecretName := DataSecretName version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -821,7 +974,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, @@ -829,7 +982,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -838,12 +991,47 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err = fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Annotations: map[string]string{ + controllers.BootstrapDataHashAnnotation: capiBootstrapSecretHash, + }, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + SecurityGroups: []string{KarpenterNodesSecurityGroup}, + Subnets: []string{KarpenterNodesSubnets}, + // UserData: nil, + // Tags: nil, + }, + IamInstanceProfile: KarpenterNodesInstanceProfile, + NodePool: &karpenterinfra.NodePoolSpec{}, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -852,27 +1040,27 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Spec: capi.ClusterSpec{ InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, }, } - err = fakeCtrlClient.Create(ctx, cluster) + err = k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-kubeconfig", ClusterName), }, } - err = fakeCtrlClient.Create(ctx, clusterKubeconfigSecret) + err = k8sClient.Create(ctx, clusterKubeconfigSecret) Expect(err).NotTo(HaveOccurred()) awsCluster := &capa.AWSCluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, }, Spec: capa.AWSClusterSpec{ @@ -883,7 +1071,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, }, } - err = fakeCtrlClient.Create(ctx, awsCluster) + err = k8sClient.Create(ctx, awsCluster) Expect(err).NotTo(HaveOccurred()) awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ @@ -896,17 +1084,20 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err = fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) - Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"value": capiBootstrapSecretContent}, } - err = fakeCtrlClient.Create(ctx, bootstrapSecret) + err = k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) It("doesn't write the user data to S3 again", func() { @@ -915,15 +1106,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) }) - When("the KarpenterMachinePool has NodePool and EC2NodeClass configuration", func() { - It("should handle the configuration without errors", func() { - // This test verifies that the controller can handle KarpenterMachinePool - // with NodePool and EC2NodeClass configuration without panicking - // The actual resource creation is tested in integration tests - Expect(reconcileErr).NotTo(HaveOccurred()) - }) - }) - Describe("Version comparison functions", func() { Describe("CompareKubernetesVersions", func() { It("should correctly compare versions", func() { diff --git a/controllers/route_controller_test.go b/controllers/route_controller_test.go index 012d1c02..525f3cbd 100644 --- a/controllers/route_controller_test.go +++ b/controllers/route_controller_test.go @@ -40,7 +40,7 @@ var _ = Describe("RouteReconciler", func() { transitGatewayID = "tgw-019120b363d1e81e4" prefixListARN = fmt.Sprintf("arn:aws:ec2:eu-north-1:123456789012:prefix-list/%s", prefixlistID) transitGatewayARN = fmt.Sprintf("arn:aws:ec2:eu-north-1:123456789012:transit-gateway/%s", transitGatewayID) - //subnets []string + // subnets []string ) getActualCluster := func() *capa.AWSCluster { @@ -75,7 +75,7 @@ var _ = Describe("RouteReconciler", func() { prefixListARN, ) requestResourceName = awsCluster.Name - //subnets = []string{awsCluster.Spec.NetworkSpec.Subnets[0].ID} + // subnets = []string{awsCluster.Spec.NetworkSpec.Subnets[0].ID} }) JustBeforeEach(func() { @@ -136,7 +136,7 @@ var _ = Describe("RouteReconciler", func() { }) }) - //There is no difference between GiantswarmManaged and UserManaged mode + // There is no difference between GiantswarmManaged and UserManaged mode Describe("GiantswarmManaged Mode", func() { When("There is an error while adding routes", func() { BeforeEach(func() { diff --git a/go.mod b/go.mod index 10d59624..90a53429 100644 --- a/go.mod +++ b/go.mod @@ -77,7 +77,6 @@ require ( golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 6b83883d..0b11c968 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -54,8 +54,15 @@ spec: description: EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass properties: - amiId: - description: AMIID specifies the AMI ID to use + amiName: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + amiOwner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string securityGroups: description: SecurityGroups specifies the security groups to use diff --git a/tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml new file mode 100644 index 00000000..4504d1ca --- /dev/null +++ b/tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -0,0 +1,847 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: ec2nodeclasses.karpenter.k8s.aws +spec: + group: karpenter.k8s.aws + names: + categories: + - karpenter + kind: EC2NodeClass + listKind: EC2NodeClassList + plural: ec2nodeclasses + shortNames: + - ec2nc + - ec2ncs + singular: ec2nodeclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, or alias ARN) of the customer managed KMS key to use for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + capacityReservationSelectorTerms: + description: |- + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + deprecated: + description: Deprecation status of the AMI + type: boolean + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + capacityReservations: + description: |- + CapacityReservations contains the current capacity reservation values that are available to this NodeClass under the + CapacityReservation selectors. + items: + properties: + availabilityZone: + description: The availability zone the capacity reservation is available in. + type: string + endTime: + description: |- + The time at which the capacity reservation expires. Once expired, the reserved capacity is released and Karpenter + will no longer be able to launch instances into that reservation. + format: date-time + type: string + id: + description: The id for the capacity reservation. + pattern: ^cr-[0-9a-z]+$ + type: string + instanceMatchCriteria: + description: Indicates the type of instance launches the capacity reservation accepts. + enum: + - open + - targeted + type: string + instanceType: + description: The instance type for the capacity reservation. + type: string + ownerID: + description: The ID of the AWS account that owns the capacity reservation. + pattern: ^[0-9]{12}$ + type: string + reservationType: + default: default + description: The type of capacity reservation. + enum: + - default + - capacity-block + type: string + state: + default: active + description: |- + The state of the capacity reservation. A capacity reservation is considered to be expiring if it is within the EC2 + reclaimation window. Only capacity-block reservations may be in this state. + enum: + - active + - expiring + type: string + required: + - availabilityZone + - id + - instanceMatchCriteria + - instanceType + - ownerID + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current security group values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/tests/testdata/crds/karpenter.sh_nodeclaims.yaml b/tests/testdata/crds/karpenter.sh_nodeclaims.yaml new file mode 100644 index 00000000..c989950e --- /dev/null +++ b/tests/testdata/crds/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,399 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.imageID + name: ImageID + priority: 1 + type: string + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Drifted")].status + name: Drifted + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/tests/testdata/crds/karpenter.sh_nodepools.yaml b/tests/testdata/crds/karpenter.sh_nodepools.yaml new file mode 100644 index 00000000..1f87a6fe --- /dev/null +++ b/tests/testdata/crds/karpenter.sh_nodepools.yaml @@ -0,0 +1,525 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' + required: + - group + - kind + - name + type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodeClassObservedGeneration: + description: |- + NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match + the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset + format: int64 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} From 7599b88ff86d408bda98755b1fa2005734ae840f Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Wed, 9 Jul 2025 17:12:46 +0200 Subject: [PATCH 05/41] Only use security groups and subnets if defined --- Makefile.custom.mk | 2 +- ...luster.x-k8s.io_karpentermachinepools.yaml | 2 +- .../karpentermachinepool_controller.go | 51 +++++-------------- ...luster.x-k8s.io_karpentermachinepools.yaml | 2 +- 4 files changed, 17 insertions(+), 40 deletions(-) diff --git a/Makefile.custom.mk b/Makefile.custom.mk index 364cfe69..cd823a23 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -119,7 +119,7 @@ coverage-html: test-unit CONTROLLER_GEN = $(shell pwd)/bin/controller-gen .PHONY: controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.18.0) ENVTEST = $(shell pwd)/bin/setup-envtest .PHONY: envtest diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 0b11c968..f3be09c1 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 helm.sh/resource-policy: keep labels: cluster.x-k8s.io/v1beta1: v1alpha1 diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 98380ded..ca71a64f 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -427,54 +427,31 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. }, }, "instanceProfile": karpenterMachinePool.Spec.IamInstanceProfile, - "securityGroupSelectorTerms": []map[string]interface{}{ + "userData": userData, + } + + // Add security groups if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { + spec["securityGroupSelectorTerms"] = []map[string]interface{}{ { "tags": map[string]string{ "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now }, }, - }, - "subnetSelectorTerms": []map[string]interface{}{ + } + } + + // Add subnets if specified + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { + spec["subnetSelectorTerms"] = []map[string]interface{}{ { "tags": map[string]string{ - "Name": karpenterMachinePool.Spec.EC2NodeClass.Subnets[0], // Using first security group for now + "Name": karpenterMachinePool.Spec.EC2NodeClass.Subnets[0], // Using first subnet for now }, }, - }, - "userData": userData, + } } - // Add AMI ID if specified - // if karpenterMachinePool.Spec.EC2NodeClass != nil && karpenterMachinePool.Spec.EC2NodeClass.AMIID != nil { - // spec["amiSelectorTerms"] = []map[string]interface{}{ - // { - // "id": *karpenterMachinePool.Spec.EC2NodeClass.AMIID, - // }, - // } - // } - - // Add security groups if specified - // if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { - // spec["securityGroupSelectorTerms"] = []map[string]interface{}{ - // { - // "tags": map[string]string{ - // "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now - // }, - // }, - // } - // } - - // Add subnets if specified - // if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { - // subnetSelectorTerms := []map[string]interface{}{} - // for _, subnet := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { - // subnetSelectorTerms = append(subnetSelectorTerms, map[string]interface{}{ - // "id": subnet, - // }) - // } - // spec["subnetSelectorTerms"] = subnetSelectorTerms - // } - // Add tags if specified if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Tags) > 0 { spec["tags"] = karpenterMachinePool.Spec.EC2NodeClass.Tags diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 0b11c968..f3be09c1 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 helm.sh/resource-policy: keep labels: cluster.x-k8s.io/v1beta1: v1alpha1 From ff4804601a8d40ac9e7d031d1360a9ca5ec737c0 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Wed, 9 Jul 2025 23:44:27 +0200 Subject: [PATCH 06/41] Use map for subnet and security group selectors --- api/v1alpha1/karpentermachinepool_types.go | 10 +--- api/v1alpha1/zz_generated.deepcopy.go | 17 ++++--- ...luster.x-k8s.io_karpentermachinepools.yaml | 15 +++--- .../karpentermachinepool_controller.go | 48 +++++++++++-------- .../karpentermachinepool_controller_test.go | 14 +++--- ...luster.x-k8s.io_karpentermachinepools.yaml | 15 +++--- 6 files changed, 57 insertions(+), 62 deletions(-) diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index c74474dc..6ea43fe3 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -120,20 +120,14 @@ type EC2NodeClassSpec struct { // Owner is the owner for the ami. // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" AMIOwner string `json:"amiOwner,omitempty"` - // - name: flatcar-stable-{{ $.Values.baseOS }}-kube-{{ $.Values.k8sVersion }}-tooling-{{ $.Values.toolingVersion }}-gs - // // owner: {{ int64 $.Values.amiOwner | quote }} // SecurityGroups specifies the security groups to use // +optional - SecurityGroups []string `json:"securityGroups,omitempty"` + SecurityGroups map[string]string `json:"securityGroups,omitempty"` // Subnets specifies the subnets to use // +optional - Subnets []string `json:"subnets,omitempty"` - - // UserData specifies the user data to use - // +optional - UserData *string `json:"userData,omitempty"` + Subnets map[string]string `json:"subnets,omitempty"` // Tags specifies the tags to apply to EC2 instances // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d0a5f151..a7c00595 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -86,18 +86,17 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = *in if in.SecurityGroups != nil { in, out := &in.SecurityGroups, &out.SecurityGroups - *out = make([]string, len(*in)) - copy(*out, *in) + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } } if in.Subnets != nil { in, out := &in.Subnets, &out.Subnets - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.UserData != nil { - in, out := &in.UserData, &out.UserData - *out = new(string) - **out = **in + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } } if in.Tags != nil { in, out := &in.Tags, &out.Tags diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index f3be09c1..8b44c962 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -65,23 +65,20 @@ spec: You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string securityGroups: - description: SecurityGroups specifies the security groups to use - items: + additionalProperties: type: string - type: array + description: SecurityGroups specifies the security groups to use + type: object subnets: - description: Subnets specifies the subnets to use - items: + additionalProperties: type: string - type: array + description: Subnets specifies the subnets to use + type: object tags: additionalProperties: type: string description: Tags specifies the tags to apply to EC2 instances type: object - userData: - description: UserData specifies the user data to use - type: string type: object iamInstanceProfile: description: |- diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index ca71a64f..c3a1a3e9 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -416,6 +416,28 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. // Generate user data for Ignition userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, cluster.Name, karpenterMachinePool.Name) + // Add security groups tag selector if specified + securityGroupTagsSelector := map[string]string{ + fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", + "sigs.k8s.io/cluster-api-provider-aws/role": "node", + } + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { + for securityGroupTagKey, securityGroupTagValue := range karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups { + securityGroupTagsSelector[securityGroupTagKey] = securityGroupTagValue + } + } + + // Add subnet tag selector if specified + subnetTagsSelector := map[string]string{ + fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", + "giantswarm.io/role": "nodes", + } + if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { + for subnetTagKey, subnetTagValue := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { + subnetTagsSelector[subnetTagKey] = subnetTagValue + } + } + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { // Build the EC2NodeClass spec spec := map[string]interface{}{ @@ -427,29 +449,17 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. }, }, "instanceProfile": karpenterMachinePool.Spec.IamInstanceProfile, - "userData": userData, - } - - // Add security groups if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { - spec["securityGroupSelectorTerms"] = []map[string]interface{}{ + "securityGroupSelectorTerms": []map[string]interface{}{ { - "tags": map[string]string{ - "Name": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups[0], // Using first security group for now - }, + "tags": securityGroupTagsSelector, }, - } - } - - // Add subnets if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { - spec["subnetSelectorTerms"] = []map[string]interface{}{ + }, + "subnetSelectorTerms": []map[string]interface{}{ { - "tags": map[string]string{ - "Name": karpenterMachinePool.Spec.EC2NodeClass.Subnets[0], // Using first subnet for now - }, + "tags": subnetTagsSelector, }, - } + }, + "userData": userData, } // Add tags if specified diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 85f5ec39..36dceb89 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -500,9 +500,8 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ AMIName: AMIName, AMIOwner: AMIOwner, - SecurityGroups: []string{KarpenterNodesSecurityGroup}, - Subnets: []string{KarpenterNodesSubnets}, - // UserData: nil, + SecurityGroups: map[string]string{"my-target-sg": "is-this"}, + Subnets: map[string]string{"my-target-subnet": "is-that"}, // Tags: nil, }, IamInstanceProfile: KarpenterNodesInstanceProfile, @@ -802,7 +801,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { // Assert the security group name field securityGroupTags, ok := securityGroupSelectorTerm0["tags"].(map[string]interface{}) Expect(ok).To(BeTrue(), "expected tags to be a map[string]string") - Expect(securityGroupTags["Name"]).To(Equal(KarpenterNodesSecurityGroup)) + Expect(securityGroupTags["my-target-sg"]).To(Equal("is-this")) // Assert subnets are the expected ones subnetSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "subnetSelectorTerms") @@ -815,7 +814,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { // Assert the security group name field subnetTags, ok := subnetSelectorTerm0["tags"].(map[string]interface{}) Expect(ok).To(BeTrue(), "expected tags to be a map[string]string") - Expect(subnetTags["Name"]).To(Equal(KarpenterNodesSubnets)) + Expect(subnetTags["my-target-subnet"]).To(Equal("is-that")) // Assert userdata is the expected one userData, found, err := unstructured.NestedString(ec2nodeclassList.Items[0].Object, "spec", "userData") @@ -1017,9 +1016,8 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: karpenterinfra.KarpenterMachinePoolSpec{ EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ - SecurityGroups: []string{KarpenterNodesSecurityGroup}, - Subnets: []string{KarpenterNodesSubnets}, - // UserData: nil, + SecurityGroups: map[string]string{"my-target-sg": "is-this"}, + Subnets: map[string]string{"my-target-subnet": "is-that"}, // Tags: nil, }, IamInstanceProfile: KarpenterNodesInstanceProfile, diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index f3be09c1..8b44c962 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -65,23 +65,20 @@ spec: You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string securityGroups: - description: SecurityGroups specifies the security groups to use - items: + additionalProperties: type: string - type: array + description: SecurityGroups specifies the security groups to use + type: object subnets: - description: Subnets specifies the subnets to use - items: + additionalProperties: type: string - type: array + description: Subnets specifies the subnets to use + type: object tags: additionalProperties: type: string description: Tags specifies the tags to apply to EC2 instances type: object - userData: - description: UserData specifies the user data to use - type: string type: object iamInstanceProfile: description: |- From 810a7b378c68286bf76f132fbf1f7cf8914cd40f Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 10 Jul 2025 15:37:24 +0200 Subject: [PATCH 07/41] Take types from karpenter project --- api/v1alpha1/duration.go | 71 ++++ api/v1alpha1/karpentermachinepool_types.go | 206 ++++++++-- api/v1alpha1/zz_generated.deepcopy.go | 257 +++++++++--- ...luster.x-k8s.io_karpentermachinepools.yaml | 372 ++++++++++++++---- .../karpentermachinepool_controller.go | 206 ++++++---- .../karpentermachinepool_controller_test.go | 86 +++- go.mod | 1 + go.sum | 2 + ...luster.x-k8s.io_karpentermachinepools.yaml | 372 ++++++++++++++---- 9 files changed, 1239 insertions(+), 334 deletions(-) create mode 100644 api/v1alpha1/duration.go diff --git a/api/v1alpha1/duration.go b/api/v1alpha1/duration.go new file mode 100644 index 00000000..1a1ef120 --- /dev/null +++ b/api/v1alpha1/duration.go @@ -0,0 +1,71 @@ +package v1alpha1 + +import ( + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/samber/lo" +) + +const Never = "Never" + +// NillableDuration is a wrapper around time.Duration which supports correct +// marshaling to YAML and JSON. It uses the value "Never" to signify +// that the duration is disabled and sets the inner duration as nil +type NillableDuration struct { + *time.Duration + + // Raw is used to ensure we remarshal the NillableDuration in the same format it was specified. + // This ensures tools like Flux and ArgoCD don't mistakenly detect drift due to our conversion webhooks. + Raw []byte `hash:"ignore"` +} + +func MustParseNillableDuration(val string) NillableDuration { + nd := NillableDuration{} + // Use %q instead of %s to ensure that we unmarshal the value as a string and not an int + lo.Must0(json.Unmarshal([]byte(fmt.Sprintf("%q", val)), &nd)) + return nd +} + +// UnmarshalJSON implements the json.Unmarshaller interface. +func (d *NillableDuration) UnmarshalJSON(b []byte) error { + var str string + err := json.Unmarshal(b, &str) + if err != nil { + return err + } + if str == Never { + return nil + } + pd, err := time.ParseDuration(str) + if err != nil { + return err + } + d.Raw = slices.Clone(b) + d.Duration = &pd + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (d NillableDuration) MarshalJSON() ([]byte, error) { + if d.Raw != nil { + return d.Raw, nil + } + if d.Duration != nil { + return json.Marshal(d.Duration.String()) + } + return json.Marshal(Never) +} + +// ToUnstructured implements the value.UnstructuredConverter interface. +func (d NillableDuration) ToUnstructured() interface{} { + if d.Raw != nil { + return d.Raw + } + if d.Duration != nil { + return d.Duration.String() + } + return Never +} diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index 6ea43fe3..a5a587b2 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capi "sigs.k8s.io/cluster-api/api/v1beta1" @@ -24,44 +25,203 @@ import ( // NodePoolSpec defines the configuration for a Karpenter NodePool type NodePoolSpec struct { - // Disruption specifies the disruption behavior for the node pool - // +optional - Disruption *DisruptionSpec `json:"disruption,omitempty"` + // Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + // NodeClaims launched from this NodePool will often be further constrained than the template specifies. + // +required + Template NodeClaimTemplate `json:"template"` - // Limits specifies the limits for the node pool + // Disruption contains the parameters that relate to Karpenter's disruption logic + // +kubebuilder:default:={consolidateAfter: "0s"} // +optional - Limits *LimitsSpec `json:"limits,omitempty"` + Disruption Disruption `json:"disruption"` - // Requirements specifies the requirements for the node pool + // Limits define a set of bounds for provisioning capacity. // +optional - Requirements []RequirementSpec `json:"requirements,omitempty"` - - // Taints specifies the taints to apply to nodes in this pool + Limits Limits `json:"limits,omitempty"` + + // Weight is the priority given to the nodepool during scheduling. A higher + // numerical weight indicates that this nodepool will be ordered + // ahead of other nodepools with lower weights. A nodepool with no weight + // will be treated as if it is a nodepool with a weight of 0. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=100 // +optional - Taints []TaintSpec `json:"taints,omitempty"` + Weight *int32 `json:"weight,omitempty"` +} - // Labels specifies the labels to apply to nodes in this pool +type NodeClaimTemplate struct { + ObjectMeta `json:"metadata,omitempty"` + // +required + Spec NodeClaimTemplateSpec `json:"spec"` +} + +type ObjectMeta struct { + // Map of string keys and values that can be used to organize and categorize + // (scope and select) objects. May match selectors of replication controllers + // and services. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels // +optional Labels map[string]string `json:"labels,omitempty"` - // Weight specifies the weight of this node pool + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations // +optional - Weight *int32 `json:"weight,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` } -// DisruptionSpec defines the disruption behavior for a NodePool -type DisruptionSpec struct { - // ConsolidateAfter specifies when to consolidate nodes +// NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool +// NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since +// users are not able to set resource requests in the NodePool. +type NodeClaimTemplateSpec struct { + // Taints will be applied to the NodeClaim's node. // +optional - ConsolidateAfter *metav1.Duration `json:"consolidateAfter,omitempty"` + Taints []v1.Taint `json:"taints,omitempty"` + // StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + // within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + // daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + // purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + // +optional + StartupTaints []v1.Taint `json:"startupTaints,omitempty"` + // Requirements are layered with GetLabels and applied to every node. + // +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)" + // +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)" + // +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)" + // +kubebuilder:validation:MaxItems:=100 + // +required + Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"` + // TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + // + // Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + // + // This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + // When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + // + // Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + // If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + // that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + // + // The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + // If left undefined, the controller will wait indefinitely for pods to be drained. + // +kubebuilder:validation:Pattern=`^([0-9]+(s|m|h))+$` + // +kubebuilder:validation:Type="string" + // +optional + TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"` + // ExpireAfter is the duration the controller will wait + // before terminating a node, measured from when the node is created. This + // is useful to implement features like eventually consistent node upgrade, + // memory leak protection, and disruption testing. + // +kubebuilder:default:="720h" + // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` + // +kubebuilder:validation:Type="string" + // +kubebuilder:validation:Schemaless + // +optional + ExpireAfter NillableDuration `json:"expireAfter,omitempty"` +} - // ConsolidationPolicy specifies the consolidation policy +// A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values +// and minValues that represent the requirement to have at least that many values. +type NodeSelectorRequirementWithMinValues struct { + v1.NodeSelectorRequirement `json:",inline"` + // This field is ALPHA and can be dropped or replaced at any time + // MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=50 // +optional - ConsolidationPolicy *string `json:"consolidationPolicy,omitempty"` + MinValues *int `json:"minValues,omitempty"` +} + +type NodeClassReference struct { + // Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + // +kubebuilder:validation:XValidation:rule="self != ''",message="kind may not be empty" + // +required + Kind string `json:"kind"` + // Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names + // +kubebuilder:validation:XValidation:rule="self != ''",message="name may not be empty" + // +required + Name string `json:"name"` + // API version of the referent + // +kubebuilder:validation:XValidation:rule="self != ''",message="group may not be empty" + // +kubebuilder:validation:Pattern=`^[^/]*$` + // +required + Group string `json:"group"` +} - // ConsolidateUnder specifies when to consolidate under +type Limits v1.ResourceList + +type ConsolidationPolicy string + +const ( + ConsolidationPolicyWhenEmpty ConsolidationPolicy = "WhenEmpty" + ConsolidationPolicyWhenEmptyOrUnderutilized ConsolidationPolicy = "WhenEmptyOrUnderutilized" +) + +// DisruptionReason defines valid reasons for disruption budgets. +// +kubebuilder:validation:Enum={Underutilized,Empty,Drifted} +type DisruptionReason string + +// Budget defines when Karpenter will restrict the +// number of Node Claims that can be terminating simultaneously. +type Budget struct { + // Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + // Otherwise, this will apply to each reason defined. + // allowed reasons are Underutilized, Empty, and Drifted. + // +optional + Reasons []DisruptionReason `json:"reasons,omitempty"` + // Nodes dictates the maximum number of NodeClaims owned by this NodePool + // that can be terminating at once. This is calculated by counting nodes that + // have a deletion timestamp set, or are actively being deleted by Karpenter. + // This field is required when specifying a budget. + // This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + // checking for int nodes for IntOrString nodes. + // Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + // +kubebuilder:validation:Pattern:="^((100|[0-9]{1,2})%|[0-9]+)$" + // +kubebuilder:default:="10%" + Nodes string `json:"nodes" hash:"ignore"` + // Schedule specifies when a budget begins being active, following + // the upstream cronjob syntax. If omitted, the budget is always active. + // Timezones are not supported. + // This field is required if Duration is set. + // +kubebuilder:validation:Pattern:=`^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$` + // +optional + Schedule *string `json:"schedule,omitempty" hash:"ignore"` + // Duration determines how long a Budget is active since each Schedule hit. + // Only minutes and hours are accepted, as cron does not work in seconds. + // If omitted, the budget is always active. + // This is required if Schedule is set. + // This regex has an optional 0s at the end since the duration.String() always adds + // a 0s at the end. + // +kubebuilder:validation:Pattern=`^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$` + // +kubebuilder:validation:Type="string" // +optional - ConsolidateUnder *ConsolidateUnderSpec `json:"consolidateUnder,omitempty"` + Duration *metav1.Duration `json:"duration,omitempty" hash:"ignore"` +} + +type Disruption struct { + // ConsolidateAfter is the duration the controller will wait + // before attempting to terminate nodes that are underutilized. + // Refer to ConsolidationPolicy for how underutilization is considered. + // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` + // +kubebuilder:validation:Type="string" + // +kubebuilder:validation:Schemaless + // +required + ConsolidateAfter NillableDuration `json:"consolidateAfter"` + // ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + // algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + // +kubebuilder:default:="WhenEmptyOrUnderutilized" + // +kubebuilder:validation:Enum:={WhenEmpty,WhenEmptyOrUnderutilized} + // +optional + ConsolidationPolicy ConsolidationPolicy `json:"consolidationPolicy,omitempty"` + // Budgets is a list of Budgets. + // If there are multiple active budgets, Karpenter uses + // the most restrictive value. If left undefined, + // this will default to one budget with a value to 10%. + // +kubebuilder:validation:XValidation:message="'schedule' must be set with 'duration'",rule="self.all(x, has(x.schedule) == has(x.duration))" + // +kubebuilder:default:={{nodes: "10%"}} + // +kubebuilder:validation:MaxItems=50 + // +optional + Budgets []Budget `json:"budgets,omitempty" hash:"ignore"` } // ConsolidateUnderSpec defines when to consolidate under @@ -128,10 +288,6 @@ type EC2NodeClassSpec struct { // Subnets specifies the subnets to use // +optional Subnets map[string]string `json:"subnets,omitempty"` - - // Tags specifies the tags to apply to EC2 instances - // +optional - Tags map[string]string `json:"tags,omitempty"` } // KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a7c00595..6d1436fb 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,11 +21,44 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + timex "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Budget) DeepCopyInto(out *Budget) { + *out = *in + if in.Reasons != nil { + in, out := &in.Reasons, &out.Reasons + *out = make([]DisruptionReason, len(*in)) + copy(*out, *in) + } + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + if in.Duration != nil { + in, out := &in.Duration, &out.Duration + *out = new(metav1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Budget. +func (in *Budget) DeepCopy() *Budget { + if in == nil { + return nil + } + out := new(Budget) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConsolidateUnderSpec) DeepCopyInto(out *ConsolidateUnderSpec) { *out = *in @@ -52,31 +85,24 @@ func (in *ConsolidateUnderSpec) DeepCopy() *ConsolidateUnderSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DisruptionSpec) DeepCopyInto(out *DisruptionSpec) { +func (in *Disruption) DeepCopyInto(out *Disruption) { *out = *in - if in.ConsolidateAfter != nil { - in, out := &in.ConsolidateAfter, &out.ConsolidateAfter - *out = new(v1.Duration) - **out = **in - } - if in.ConsolidationPolicy != nil { - in, out := &in.ConsolidationPolicy, &out.ConsolidationPolicy - *out = new(string) - **out = **in - } - if in.ConsolidateUnder != nil { - in, out := &in.ConsolidateUnder, &out.ConsolidateUnder - *out = new(ConsolidateUnderSpec) - (*in).DeepCopyInto(*out) + in.ConsolidateAfter.DeepCopyInto(&out.ConsolidateAfter) + if in.Budgets != nil { + in, out := &in.Budgets, &out.Budgets + *out = make([]Budget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DisruptionSpec. -func (in *DisruptionSpec) DeepCopy() *DisruptionSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Disruption. +func (in *Disruption) DeepCopy() *Disruption { if in == nil { return nil } - out := new(DisruptionSpec) + out := new(Disruption) in.DeepCopyInto(out) return out } @@ -98,13 +124,6 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { (*out)[key] = val } } - if in.Tags != nil { - in, out := &in.Tags, &out.Tags - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassSpec. @@ -228,6 +247,27 @@ func (in *KarpenterMachinePoolStatus) DeepCopy() *KarpenterMachinePoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Limits) DeepCopyInto(out *Limits) { + { + in := &in + *out = make(Limits, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Limits. +func (in Limits) DeepCopy() Limits { + if in == nil { + return nil + } + out := new(Limits) + in.DeepCopyInto(out) + return *out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LimitsSpec) DeepCopyInto(out *LimitsSpec) { *out = *in @@ -254,37 +294,114 @@ func (in *LimitsSpec) DeepCopy() *LimitsSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { +func (in *NillableDuration) DeepCopyInto(out *NillableDuration) { *out = *in - if in.Disruption != nil { - in, out := &in.Disruption, &out.Disruption - *out = new(DisruptionSpec) - (*in).DeepCopyInto(*out) + if in.Duration != nil { + in, out := &in.Duration, &out.Duration + *out = new(timex.Duration) + **out = **in } - if in.Limits != nil { - in, out := &in.Limits, &out.Limits - *out = new(LimitsSpec) - (*in).DeepCopyInto(*out) + if in.Raw != nil { + in, out := &in.Raw, &out.Raw + *out = make([]byte, len(*in)) + copy(*out, *in) } - if in.Requirements != nil { - in, out := &in.Requirements, &out.Requirements - *out = make([]RequirementSpec, len(*in)) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NillableDuration. +func (in *NillableDuration) DeepCopy() *NillableDuration { + if in == nil { + return nil + } + out := new(NillableDuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimTemplate) DeepCopyInto(out *NodeClaimTemplate) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimTemplate. +func (in *NodeClaimTemplate) DeepCopy() *NodeClaimTemplate { + if in == nil { + return nil + } + out := new(NodeClaimTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimTemplateSpec) DeepCopyInto(out *NodeClaimTemplateSpec) { + *out = *in + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]v1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Taints != nil { - in, out := &in.Taints, &out.Taints - *out = make([]TaintSpec, len(*in)) + if in.StartupTaints != nil { + in, out := &in.StartupTaints, &out.StartupTaints + *out = make([]v1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]NodeSelectorRequirementWithMinValues, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TerminationGracePeriod != nil { + in, out := &in.TerminationGracePeriod, &out.TerminationGracePeriod + *out = new(metav1.Duration) + **out = **in + } + in.ExpireAfter.DeepCopyInto(&out.ExpireAfter) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimTemplateSpec. +func (in *NodeClaimTemplateSpec) DeepCopy() *NodeClaimTemplateSpec { + if in == nil { + return nil + } + out := new(NodeClaimTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClassReference) DeepCopyInto(out *NodeClassReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClassReference. +func (in *NodeClassReference) DeepCopy() *NodeClassReference { + if in == nil { + return nil + } + out := new(NodeClassReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + in.Disruption.DeepCopyInto(&out.Disruption) + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(Limits, len(*in)) for key, val := range *in { - (*out)[key] = val + (*out)[key] = val.DeepCopy() } } if in.Weight != nil { @@ -304,6 +421,56 @@ func (in *NodePoolSpec) DeepCopy() *NodePoolSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeSelectorRequirementWithMinValues) DeepCopyInto(out *NodeSelectorRequirementWithMinValues) { + *out = *in + in.NodeSelectorRequirement.DeepCopyInto(&out.NodeSelectorRequirement) + if in.MinValues != nil { + in, out := &in.MinValues, &out.MinValues + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeSelectorRequirementWithMinValues. +func (in *NodeSelectorRequirementWithMinValues) DeepCopy() *NodeSelectorRequirementWithMinValues { + if in == nil { + return nil + } + out := new(NodeSelectorRequirementWithMinValues) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectMeta) DeepCopyInto(out *ObjectMeta) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectMeta. +func (in *ObjectMeta) DeepCopy() *ObjectMeta { + if in == nil { + return nil + } + out := new(ObjectMeta) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RequirementSpec) DeepCopyInto(out *RequirementSpec) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 8b44c962..618768a2 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -74,11 +74,6 @@ spec: type: string description: Subnets specifies the subnets to use type: object - tags: - additionalProperties: - type: string - description: Tags specifies the tags to apply to EC2 instances - type: object type: object iamInstanceProfile: description: |- @@ -91,101 +86,304 @@ spec: NodePool properties: disruption: - description: Disruption specifies the disruption behavior for - the node pool + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to + Karpenter's disruption logic properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons + for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) consolidateAfter: - description: ConsolidateAfter specifies when to consolidate - nodes + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ type: string - consolidateUnder: - description: ConsolidateUnder specifies when to consolidate - under - properties: - cpuUtilization: - description: CPUUtilization specifies the CPU utilization - threshold - type: string - memoryUtilization: - description: MemoryUtilization specifies the memory utilization - threshold - type: string - type: object consolidationPolicy: - description: ConsolidationPolicy specifies the consolidation - policy + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized type: string + required: + - consolidateAfter type: object - labels: + limits: additionalProperties: - type: string - description: Labels specifies the labels to apply to nodes in - this pool + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. type: object - limits: - description: Limits specifies the limits for the node pool + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. properties: - cpu: - anyOf: - - type: integer - - type: string - description: CPU specifies the CPU limit - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - memory: - anyOf: - - type: integer - - type: string - description: Memory specifies the memory limit - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requirements: - description: Requirements specifies the requirements for the node - pool - items: - description: RequirementSpec defines a requirement for a NodePool - properties: - key: - description: Key specifies the requirement key - type: string - operator: - description: Operator specifies the requirement operator - type: string - values: - description: Values specifies the requirement values - items: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ type: string - type: array - required: - - key - - operator - type: object - type: array - taints: - description: Taints specifies the taints to apply to nodes in - this pool - items: - description: TaintSpec defines a taint for a NodePool - properties: - effect: - description: Effect specifies the taint effect - type: string - key: - description: Key specifies the taint key - type: string - value: - description: Value specifies the taint value - type: string - required: - - effect - - key - type: object - type: array + requirements: + description: Requirements are layered with GetLabels and + applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a + value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have + a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator + == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) + >= 0) : true)' + - message: requirements with 'minValues' must have at + least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's + node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - requirements + type: object + required: + - spec + type: object weight: - description: Weight specifies the weight of this node pool + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. format: int32 + maximum: 100 + minimum: 1 type: integer + required: + - template type: object providerIDList: description: |- diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index c3a1a3e9..06d9ab1d 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -388,7 +388,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } // Create or update NodePool - if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, karpenterMachinePool); err != nil { + if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { conditions.MarkNodePoolNotReady(karpenterMachinePool, NodePoolCreationFailedReason, err.Error()) return fmt.Errorf("failed to create or update NodePool: %w", err) } @@ -441,14 +441,49 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { // Build the EC2NodeClass spec spec := map[string]interface{}{ - "amiFamily": "AL2", + "amiFamily": "Custom", "amiSelectorTerms": []map[string]interface{}{ { "name": karpenterMachinePool.Spec.EC2NodeClass.AMIName, "owner": karpenterMachinePool.Spec.EC2NodeClass.AMIOwner, }, }, + "blockDeviceMappings": []map[string]interface{}{ + { + "deviceName": "/dev/xvda", + "rootVolume": true, + "ebs": map[string]interface{}{ + "volumeSize": "8Gi", + "volumeType": "gp3", + "deleteOnTermination": true, + }, + }, + { + "deviceName": "/dev/xvdd", + "ebs": map[string]interface{}{ + "encrypted": true, + "volumeSize": "120Gi", + "volumeType": "gp3", + "deleteOnTermination": true, + }, + }, + { + "deviceName": "/dev/xvde", + "ebs": map[string]interface{}{ + "encrypted": true, + "volumeSize": "30Gi", + "volumeType": "gp3", + "deleteOnTermination": true, + }, + }, + }, "instanceProfile": karpenterMachinePool.Spec.IamInstanceProfile, + "metadataOptions": map[string]interface{}{ + "httpEndpoint": "enabled", + "httpProtocolIPv6": "disabled", + "httpPutResponseHopLimit": 1, + "httpTokens": "required", + }, "securityGroupSelectorTerms": []map[string]interface{}{ { "tags": securityGroupTagsSelector, @@ -462,11 +497,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. "userData": userData, } - // Add tags if specified - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Tags) > 0 { - spec["tags"] = karpenterMachinePool.Spec.EC2NodeClass.Tags - } - ec2NodeClass.Object["spec"] = spec return nil }) @@ -486,7 +516,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. } // createOrUpdateNodePool creates or updates the NodePool resource in the workload cluster -func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { +func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { nodePoolGVR := schema.GroupVersionResource{ Group: "karpenter.sh", Version: "v1", @@ -498,90 +528,100 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont nodePool.SetName(karpenterMachinePool.Name) nodePool.SetNamespace("default") + // Set default requirements and overwrite with the user provided configuration + requirements := []map[string]interface{}{ + { + "key": "karpenter.k8s.aws/instance-family", + "operator": "NotIn", + "values": []string{"t3", "t3a", "t2"}, + }, + { + "key": "karpenter.k8s.aws/instance-cpu", + "operator": "In", + "values": []string{"4", "8", "16", "32"}, + }, + { + "key": "karpenter.k8s.aws/instance-hypervisor", + "operator": "In", + "values": []string{"nitro"}, + }, + { + "key": "kubernetes.io/arch", + "operator": "In", + "values": []string{"amd64"}, + }, + { + "key": "karpenter.sh/capacity-type", + "operator": "In", + "values": []string{"spot", "on-demand"}, + }, + { + "key": "kubernetes.io/os", + "operator": "In", + "values": []string{"linux"}, + }, + } + if len(karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements) > 0 { + requirements = []map[string]interface{}{} + for _, req := range karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements { + requirement := map[string]interface{}{ + "key": req.Key, + "operator": req.Operator, + "values": req.Values, + } + requirements = append(requirements, requirement) + } + } + + // Set default labels and overwrite with the user provided configuration + labels := map[string]string{ + "giantswarm.io/machine-pool": fmt.Sprintf("%s-%s", cluster.Name, karpenterMachinePool.Name), + } + if len(karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels) > 0 { + for labelKey, labelValue := range karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels { + labels[labelKey] = labelValue + } + } + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, nodePool, func() error { - // Build the NodePool spec - spec := map[string]interface{}{ - "disruption": map[string]interface{}{ - "consolidateAfter": "30s", - }, + nodePool.Object["spec"] = map[string]interface{}{ "template": map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": labels, + }, "spec": map[string]interface{}{ + "taints": karpenterMachinePool.Spec.NodePool.Template.Spec.Taints, + "startupTaints": []interface{}{ + map[string]interface{}{ + "effect": "NoExecute", + "key": "node.cilium.io/agent-not-ready", + "value": "true", + }, + map[string]interface{}{ + "effect": "NoExecute", + "key": "node.cluster.x-k8s.io/uninitialized", + "value": "true", + }, + }, + "requirements": requirements, "nodeClassRef": map[string]interface{}{ - "apiVersion": "karpenter.k8s.aws/v1", - "group": "karpenter.k8s.aws", - "kind": "EC2NodeClass", - "name": karpenterMachinePool.Name, + "group": "karpenter.k8s.aws", + "kind": "EC2NodeClass", + "name": karpenterMachinePool.Name, }, - "requirements": []interface{}{}, + "terminationGracePeriodSeconds": karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod, + "expireAfter": karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter, }, }, + "disruption": map[string]interface{}{ + "budgets": karpenterMachinePool.Spec.NodePool.Disruption.Budgets, + "consolidateAfter": karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter, + "consolidationPolicy": karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy, + }, + "limits": karpenterMachinePool.Spec.NodePool.Limits, + "weight": karpenterMachinePool.Spec.NodePool.Weight, } - // Add NodePool configuration if specified - if karpenterMachinePool.Spec.NodePool != nil { - if karpenterMachinePool.Spec.NodePool.Disruption != nil { - if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter != nil { - spec["disruption"].(map[string]interface{})["consolidateAfter"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter.Duration.String() - } - if karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy != nil { - spec["disruption"].(map[string]interface{})["consolidationPolicy"] = *karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy - } - } - - if karpenterMachinePool.Spec.NodePool.Limits != nil { - limits := map[string]interface{}{} - if karpenterMachinePool.Spec.NodePool.Limits.CPU != nil { - limits["cpu"] = karpenterMachinePool.Spec.NodePool.Limits.CPU.String() - } - if karpenterMachinePool.Spec.NodePool.Limits.Memory != nil { - limits["memory"] = karpenterMachinePool.Spec.NodePool.Limits.Memory.String() - } - if len(limits) > 0 { - spec["limits"] = limits - } - } - - if len(karpenterMachinePool.Spec.NodePool.Requirements) > 0 { - requirements := []map[string]interface{}{} - for _, req := range karpenterMachinePool.Spec.NodePool.Requirements { - requirement := map[string]interface{}{ - "key": req.Key, - "operator": req.Operator, - } - if len(req.Values) > 0 { - requirement["values"] = req.Values - } - requirements = append(requirements, requirement) - } - - spec["template"].(map[string]interface{})["spec"].(map[string]interface{})["requirements"] = requirements - } - - if len(karpenterMachinePool.Spec.NodePool.Taints) > 0 { - taints := []map[string]interface{}{} - for _, taint := range karpenterMachinePool.Spec.NodePool.Taints { - taintMap := map[string]interface{}{ - "key": taint.Key, - "effect": taint.Effect, - } - if taint.Value != nil { - taintMap["value"] = *taint.Value - } - taints = append(taints, taintMap) - } - spec["taints"] = taints - } - - if len(karpenterMachinePool.Spec.NodePool.Labels) > 0 { - spec["labels"] = karpenterMachinePool.Spec.NodePool.Labels - } - - if karpenterMachinePool.Spec.NodePool.Weight != nil { - spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight - } - } - - nodePool.Object["spec"] = spec return nil }) diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 36dceb89..a8719ac8 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -43,7 +43,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ) const ( - AMIName = "flatcar-stable-1234.5-kube-1.25.1-tooling-1.2.3-gs" + AMIName = "flatcar-stable-4152.2.3-kube-1.29.1-tooling-1.26.0-gs" AMIOwner = "1234567890" AWSRegion = "eu-west-1" ClusterName = "foo" @@ -359,7 +359,25 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: karpenterinfra.KarpenterMachinePoolSpec{ EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{}, - NodePool: &karpenterinfra.NodePoolSpec{}, + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + }, }, } err := k8sClient.Create(ctx, karpenterMachinePool) @@ -427,7 +445,27 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, }, - Spec: karpenterinfra.KarpenterMachinePoolSpec{}, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + }, + }, } err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) @@ -502,10 +540,27 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { AMIOwner: AMIOwner, SecurityGroups: map[string]string{"my-target-sg": "is-this"}, Subnets: map[string]string{"my-target-subnet": "is-that"}, - // Tags: nil, }, IamInstanceProfile: KarpenterNodesInstanceProfile, - NodePool: &karpenterinfra.NodePoolSpec{}, + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + }, }, } err = k8sClient.Create(ctx, karpenterMachinePool) @@ -1018,10 +1073,27 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ SecurityGroups: map[string]string{"my-target-sg": "is-this"}, Subnets: map[string]string{"my-target-subnet": "is-that"}, - // Tags: nil, }, IamInstanceProfile: KarpenterNodesInstanceProfile, - NodePool: &karpenterinfra.NodePoolSpec{}, + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + }, }, } err = k8sClient.Create(ctx, karpenterMachinePool) diff --git a/go.mod b/go.mod index 90a53429..af656c81 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.22.0 + github.com/samber/lo v1.51.0 go.uber.org/zap v1.27.0 golang.org/x/tools v0.35.0 k8s.io/api v0.31.2 diff --git a/go.sum b/go.sum index 7e4e95bd..78c71df5 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= +github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 8b44c962..618768a2 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -74,11 +74,6 @@ spec: type: string description: Subnets specifies the subnets to use type: object - tags: - additionalProperties: - type: string - description: Tags specifies the tags to apply to EC2 instances - type: object type: object iamInstanceProfile: description: |- @@ -91,101 +86,304 @@ spec: NodePool properties: disruption: - description: Disruption specifies the disruption behavior for - the node pool + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to + Karpenter's disruption logic properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons + for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) consolidateAfter: - description: ConsolidateAfter specifies when to consolidate - nodes + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ type: string - consolidateUnder: - description: ConsolidateUnder specifies when to consolidate - under - properties: - cpuUtilization: - description: CPUUtilization specifies the CPU utilization - threshold - type: string - memoryUtilization: - description: MemoryUtilization specifies the memory utilization - threshold - type: string - type: object consolidationPolicy: - description: ConsolidationPolicy specifies the consolidation - policy + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized type: string + required: + - consolidateAfter type: object - labels: + limits: additionalProperties: - type: string - description: Labels specifies the labels to apply to nodes in - this pool + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. type: object - limits: - description: Limits specifies the limits for the node pool + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. properties: - cpu: - anyOf: - - type: integer - - type: string - description: CPU specifies the CPU limit - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - memory: - anyOf: - - type: integer - - type: string - description: Memory specifies the memory limit - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - type: object - requirements: - description: Requirements specifies the requirements for the node - pool - items: - description: RequirementSpec defines a requirement for a NodePool - properties: - key: - description: Key specifies the requirement key - type: string - operator: - description: Operator specifies the requirement operator - type: string - values: - description: Values specifies the requirement values - items: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ type: string - type: array - required: - - key - - operator - type: object - type: array - taints: - description: Taints specifies the taints to apply to nodes in - this pool - items: - description: TaintSpec defines a taint for a NodePool - properties: - effect: - description: Effect specifies the taint effect - type: string - key: - description: Key specifies the taint key - type: string - value: - description: Value specifies the taint value - type: string - required: - - effect - - key - type: object - type: array + requirements: + description: Requirements are layered with GetLabels and + applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a + value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have + a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator + == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) + >= 0) : true)' + - message: requirements with 'minValues' must have at + least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's + node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - requirements + type: object + required: + - spec + type: object weight: - description: Weight specifies the weight of this node pool + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. format: int32 + maximum: 100 + minimum: 1 type: integer + required: + - template type: object providerIDList: description: |- From e87d502b82187844b8586d982f304f74ced9df97 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 10 Jul 2025 16:32:21 +0200 Subject: [PATCH 08/41] Add managed-by label to resources created by controller --- controllers/karpentermachinepool_controller.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 06d9ab1d..d1baec94 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -411,7 +411,8 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. ec2NodeClass := &unstructured.Unstructured{} ec2NodeClass.SetGroupVersionKind(ec2NodeClassGVR.GroupVersion().WithKind("EC2NodeClass")) ec2NodeClass.SetName(karpenterMachinePool.Name) - ec2NodeClass.SetNamespace("default") + ec2NodeClass.SetNamespace("") + ec2NodeClass.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) // Generate user data for Ignition userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, cluster.Name, karpenterMachinePool.Name) @@ -526,7 +527,8 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont nodePool := &unstructured.Unstructured{} nodePool.SetGroupVersionKind(nodePoolGVR.GroupVersion().WithKind("NodePool")) nodePool.SetName(karpenterMachinePool.Name) - nodePool.SetNamespace("default") + nodePool.SetNamespace("") + nodePool.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) // Set default requirements and overwrite with the user provided configuration requirements := []map[string]interface{}{ @@ -572,6 +574,8 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont requirements = append(requirements, requirement) } } + nodePool.SetNamespace("") + nodePool.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) // Set default labels and overwrite with the user provided configuration labels := map[string]string{ @@ -603,7 +607,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "value": "true", }, }, - "requirements": requirements, + "requirements": karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements, "nodeClassRef": map[string]interface{}{ "group": "karpenter.k8s.aws", "kind": "EC2NodeClass", From bdc953f5a7ddb09e1acc31344858045d427877bc Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 10 Jul 2025 17:04:02 +0200 Subject: [PATCH 09/41] Check if NodePool field is nil --- .../karpentermachinepool_controller.go | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index d1baec94..947108a4 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -14,6 +14,7 @@ import ( "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" @@ -581,20 +582,46 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont labels := map[string]string{ "giantswarm.io/machine-pool": fmt.Sprintf("%s-%s", cluster.Name, karpenterMachinePool.Name), } - if len(karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels) > 0 { + if karpenterMachinePool.Spec.NodePool != nil && len(karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels) > 0 { for labelKey, labelValue := range karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels { labels[labelKey] = labelValue } } + requirements := []v1alpha1.NodeSelectorRequirementWithMinValues{} + taints := []v1.Taint{} + expireAfter := v1alpha1.MustParseNillableDuration("720h") + budgets := []v1alpha1.Budget{} + consolidateAfter := v1alpha1.MustParseNillableDuration("0s") + consolidationPolicy := "WhenEmptyOrUnderutilized" + + if karpenterMachinePool.Spec.NodePool != nil { + requirements = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements + taints = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints + expireAfter = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter + budgets = karpenterMachinePool.Spec.NodePool.Disruption.Budgets + consolidateAfter = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter + consolidationPolicy = string(karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy) + } + + terminationGracePeriod := metav1.Duration{} + if karpenterMachinePool.Spec.NodePool != nil && karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod != nil { + terminationGracePeriod = *karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod + } + + weight := int32(1) + if karpenterMachinePool.Spec.NodePool != nil && karpenterMachinePool.Spec.NodePool.Weight != nil { + weight = *karpenterMachinePool.Spec.NodePool.Weight + } + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, nodePool, func() error { - nodePool.Object["spec"] = map[string]interface{}{ + spec := map[string]interface{}{ "template": map[string]interface{}{ "metadata": map[string]interface{}{ "labels": labels, }, "spec": map[string]interface{}{ - "taints": karpenterMachinePool.Spec.NodePool.Template.Spec.Taints, + "taints": taints, "startupTaints": []interface{}{ map[string]interface{}{ "effect": "NoExecute", @@ -607,25 +634,27 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "value": "true", }, }, - "requirements": karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements, + "requirements": requirements, "nodeClassRef": map[string]interface{}{ "group": "karpenter.k8s.aws", "kind": "EC2NodeClass", "name": karpenterMachinePool.Name, }, - "terminationGracePeriodSeconds": karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod, - "expireAfter": karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter, + "terminationGracePeriodSeconds": terminationGracePeriod, + "expireAfter": expireAfter, }, }, "disruption": map[string]interface{}{ - "budgets": karpenterMachinePool.Spec.NodePool.Disruption.Budgets, - "consolidateAfter": karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter, - "consolidationPolicy": karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy, + "budgets": budgets, + "consolidateAfter": consolidateAfter, + "consolidationPolicy": consolidationPolicy, }, "limits": karpenterMachinePool.Spec.NodePool.Limits, - "weight": karpenterMachinePool.Spec.NodePool.Weight, + "weight": weight, } + nodePool.Object["spec"] = spec + return nil }) From 60cd70edced94edc010a21ae94433209e21b77ca Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 10 Jul 2025 17:24:52 +0200 Subject: [PATCH 10/41] Avoid setting defaults --- .../karpentermachinepool_controller.go | 64 ++++++++----------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 947108a4..280ad6df 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -14,7 +14,6 @@ import ( "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" @@ -588,32 +587,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont } } - requirements := []v1alpha1.NodeSelectorRequirementWithMinValues{} - taints := []v1.Taint{} - expireAfter := v1alpha1.MustParseNillableDuration("720h") - budgets := []v1alpha1.Budget{} - consolidateAfter := v1alpha1.MustParseNillableDuration("0s") - consolidationPolicy := "WhenEmptyOrUnderutilized" - - if karpenterMachinePool.Spec.NodePool != nil { - requirements = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements - taints = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints - expireAfter = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter - budgets = karpenterMachinePool.Spec.NodePool.Disruption.Budgets - consolidateAfter = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter - consolidationPolicy = string(karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy) - } - - terminationGracePeriod := metav1.Duration{} - if karpenterMachinePool.Spec.NodePool != nil && karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod != nil { - terminationGracePeriod = *karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod - } - - weight := int32(1) - if karpenterMachinePool.Spec.NodePool != nil && karpenterMachinePool.Spec.NodePool.Weight != nil { - weight = *karpenterMachinePool.Spec.NodePool.Weight - } - operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, nodePool, func() error { spec := map[string]interface{}{ "template": map[string]interface{}{ @@ -621,7 +594,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "labels": labels, }, "spec": map[string]interface{}{ - "taints": taints, "startupTaints": []interface{}{ map[string]interface{}{ "effect": "NoExecute", @@ -634,23 +606,39 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "value": "true", }, }, - "requirements": requirements, "nodeClassRef": map[string]interface{}{ "group": "karpenter.k8s.aws", "kind": "EC2NodeClass", "name": karpenterMachinePool.Name, }, - "terminationGracePeriodSeconds": terminationGracePeriod, - "expireAfter": expireAfter, }, }, - "disruption": map[string]interface{}{ - "budgets": budgets, - "consolidateAfter": consolidateAfter, - "consolidationPolicy": consolidationPolicy, - }, - "limits": karpenterMachinePool.Spec.NodePool.Limits, - "weight": weight, + "disruption": map[string]interface{}{}, + } + + if karpenterMachinePool.Spec.NodePool != nil { + dis := spec["disruption"].(map[string]interface{}) + dis["budgets"] = karpenterMachinePool.Spec.NodePool.Disruption.Budgets + dis["consolidateAfter"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter + dis["consolidationPolicy"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy + + if karpenterMachinePool.Spec.NodePool.Limits != nil { + spec["limits"] = karpenterMachinePool.Spec.NodePool.Limits + } + + if karpenterMachinePool.Spec.NodePool.Weight != nil { + spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight + } + + tpl := spec["template"].(map[string]interface{})["spec"].(map[string]interface{}) + + tpl["taints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints + tpl["requirements"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements + tpl["expireAfter"] = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter + + if karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod != nil { + tpl["terminationGracePeriodSeconds"] = karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod + } } nodePool.Object["spec"] = spec From aa07033a4f98323db89f87b9ce764ec21877c1bd Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 10 Jul 2025 23:38:45 +0200 Subject: [PATCH 11/41] Run goimports in api files (including generated files) --- Makefile.custom.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.custom.mk b/Makefile.custom.mk index cd823a23..076c6eb2 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -26,6 +26,7 @@ crds: controller-gen ## Generate CustomResourceDefinition. generate: controller-gen crds ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. go generate ./... $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + @go run golang.org/x/tools/cmd/goimports -w ./api/v1alpha1 .PHONY: create-acceptance-cluster create-acceptance-cluster: kind From 3ae94b54b51ddaecbed61d94735f59afde2cf673 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Sun, 13 Jul 2025 19:07:32 +0200 Subject: [PATCH 12/41] Refactor tests --- .../karpentermachinepool_controller.go | 318 +++++------------- .../karpentermachinepool_controller_test.go | 279 +++++++-------- go.mod | 2 +- 3 files changed, 201 insertions(+), 398 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 280ad6df..6ff9d05f 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -7,10 +7,9 @@ import ( "errors" "fmt" "path" - "strconv" - "strings" "time" + "github.com/blang/semver/v4" "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -63,8 +62,10 @@ func NewKarpenterMachinepoolReconciler(client client.Client, clusterClientGetter return &KarpenterMachinePoolReconciler{awsClients: awsClients, client: client, clusterClientGetter: clusterClientGetter} } -// Reconcile will upload to S3 the Ignition configuration for the reconciled node pool. +// Reconcile reconciles KarpenterMachinePool, which represent cluster node pools that will be managed by karpenter. +// The controller will upload to S3 the Ignition configuration for the reconciled node pool. // It will also take care of deleting EC2 instances created by karpenter when the cluster is being removed. +// And lastly, it will create the karpenter CRs in the workload cluster. func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { logger := log.FromContext(ctx) logger.Info("Reconciling") @@ -125,11 +126,9 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, fmt.Errorf("failed to get AWSClusterRoleIdentity referenced in AWSCluster: %w", err) } - // Create deep copy of the reconciled object so we can change it + // Create a deep copy of the reconciled object so we can change it karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - // We only remove the finalizer after we've removed the EC2 instances created by karpenter. - // These are normally removed by Karpenter but when deleting a cluster, karpenter may not have enough time to clean them up. if !karpenterMachinePool.GetDeletionTimestamp().IsZero() { return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity) } @@ -151,9 +150,9 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } } - // Create or update Karpenter resources in the workload cluster + // Create or update Karpenter custom resources in the workload cluster. if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool, bootstrapSecretValue); err != nil { - logger.Error(err, "failed to create or update Karpenter resources") + logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") return reconcile.Result{}, err } @@ -188,11 +187,6 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, err } - if numberOfNodeClaims == 0 { - // Karpenter has not reacted yet, let's requeue - return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil - } - karpenterMachinePool.Status.Replicas = numberOfNodeClaims karpenterMachinePool.Status.Ready = true @@ -222,7 +216,11 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, nil } +// reconcileDelete deletes the karpenter custom resources from the workload cluster. func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, roleIdentity *capa.AWSClusterRoleIdentity) (reconcile.Result, error) { + // We check if the owner Cluster is also being deleted (on top of the `KarpenterMachinePool` being deleted). + // If the Cluster is being deleted, we terminate all the ec2 instances that karpenter may have launched. + // These are normally removed by Karpenter, but when deleting a cluster, karpenter may not have enough time to clean them up. if !cluster.GetDeletionTimestamp().IsZero() { ec2Client, err := r.awsClients.NewEC2Client(awsCluster.Spec.Region, roleIdentity.Spec.RoleArn) if err != nil { @@ -235,8 +233,8 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo return reconcile.Result{}, fmt.Errorf("failed to terminate EC2 instances: %w", err) } - // Requeue if we find instances to terminate. Once there are no instances to terminate, we proceed to remove the finalizer. - // We do this when the cluster is being deleted, to avoid removing the finalizer before karpenter launches a new instance that would be left over. + // Requeue if we find instances to terminate. On the next reconciliation, once there are no instances to terminate, we proceed to remove the finalizer. + // We don't want to remove the finalizer when there may be ec2 instances still around to be cleaned up. if len(instanceIDs) > 0 { return reconcile.Result{RequeueAfter: 30 * time.Second}, nil } @@ -248,7 +246,7 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo return reconcile.Result{}, err } - // Create deep copy of the reconciled object so we can change it + // Create a deep copy of the reconciled object so we can change it karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() logger.Info("Removing finalizer", "finalizer", KarpenterFinalizer) @@ -261,28 +259,28 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo return reconcile.Result{}, nil } -func getWorkloadClusterNodeClaims(ctx context.Context, ctrlClient client.Client) (*unstructured.UnstructuredList, error) { +func (r *KarpenterMachinePoolReconciler) getWorkloadClusterNodeClaims(ctx context.Context, cluster *capi.Cluster) (*unstructured.UnstructuredList, error) { + nodeClaimList := &unstructured.UnstructuredList{} + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) + if err != nil { + return nodeClaimList, err + } + nodeClaimGVR := schema.GroupVersionResource{ Group: "karpenter.sh", Version: "v1", Resource: "nodeclaims", } - nodeClaimList := &unstructured.UnstructuredList{} nodeClaimList.SetGroupVersionKind(nodeClaimGVR.GroupVersion().WithKind("NodeClaimList")) - err := ctrlClient.List(ctx, nodeClaimList) + err = workloadClusterClient.List(ctx, nodeClaimList) return nodeClaimList, err } func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx context.Context, logger logr.Logger, cluster *capi.Cluster) ([]string, int32, error) { var providerIDList []string - workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) - if err != nil { - return providerIDList, 0, err - } - - nodeClaimList, err := getWorkloadClusterNodeClaims(ctx, workloadClusterClient) + nodeClaimList, err := r.getWorkloadClusterNodeClaims(ctx, cluster) if err != nil { return providerIDList, 0, err } @@ -303,28 +301,19 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo return providerIDList, int32(len(nodeClaimList.Items)), nil } -// getControlPlaneVersion retrieves the Kubernetes version from the control plane +// getControlPlaneVersion retrieves the current Kubernetes version from the control plane func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Context, cluster *capi.Cluster) (string, error) { if cluster.Spec.ControlPlaneRef == nil { return "", fmt.Errorf("cluster has no control plane reference") } - // Parse the API version to get group and version - apiVersion := cluster.Spec.ControlPlaneRef.APIVersion - groupVersion, err := schema.ParseGroupVersion(apiVersion) - if err != nil { - return "", fmt.Errorf("failed to parse control plane API version %s: %w", apiVersion, err) - } - - // Create the GVR using the parsed group and version - controlPlaneGVR := schema.GroupVersionResource{ - Group: groupVersion.Group, - Version: groupVersion.Version, - Resource: strings.ToLower(cluster.Spec.ControlPlaneRef.Kind) + "s", // Convert Kind to resource name + groupVersionKind := schema.GroupVersionKind{ + Group: cluster.Spec.ControlPlaneRef.GroupVersionKind().Group, + Version: cluster.Spec.ControlPlaneRef.GroupVersionKind().Version, + Kind: cluster.Spec.ControlPlaneRef.GroupVersionKind().Kind, } - controlPlane := &unstructured.Unstructured{} - controlPlane.SetGroupVersionKind(controlPlaneGVR.GroupVersion().WithKind(cluster.Spec.ControlPlaneRef.Kind)) + controlPlane.SetGroupVersionKind(groupVersionKind) controlPlane.SetName(cluster.Spec.ControlPlaneRef.Name) controlPlane.SetNamespace(cluster.Spec.ControlPlaneRef.Namespace) @@ -332,9 +321,9 @@ func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Cont return "", fmt.Errorf("failed to get control plane %s: %w", cluster.Spec.ControlPlaneRef.Kind, err) } - version, found, err := unstructured.NestedString(controlPlane.Object, "spec", "version") + version, found, err := unstructured.NestedString(controlPlane.Object, "status", "version") if err != nil { - return "", fmt.Errorf("failed to get version from control plane: %w", err) + return "", fmt.Errorf("failed to get current k8s version from control plane: %w", err) } if !found { return "", fmt.Errorf("version not found in control plane spec") @@ -343,37 +332,25 @@ func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Cont return version, nil } -// createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass resources in the workload cluster +// createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass custom resources in the workload cluster func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool, bootstrapSecretValue []byte) error { - // Get the worker version from MachinePool - workerVersion := "" - if machinePool.Spec.Template.Spec.Version != nil { - workerVersion = *machinePool.Spec.Template.Spec.Version + allowed, controlPlaneCurrentVersion, nodePoolDesiredVersion, err := r.IsVersionSkewAllowed(ctx, cluster, machinePool) + if err != nil { + return err } - // Get control plane version and check version skew - if workerVersion != "" { - controlPlaneVersion, err := r.getControlPlaneVersion(ctx, cluster) - if err != nil { - logger.Error(err, "Failed to get control plane version, proceeding with update") - } else { - allowed, err := IsVersionSkewAllowed(controlPlaneVersion, workerVersion) - if err != nil { - logger.Error(err, "Failed to check version skew, proceeding with update") - } else if !allowed { - message := fmt.Sprintf("Version skew policy violation: control plane version %s is more than 2 minor versions ahead of worker version %s", controlPlaneVersion, workerVersion) - logger.Info("Blocking Karpenter resource update due to version skew policy", - "controlPlaneVersion", controlPlaneVersion, - "workerVersion", workerVersion, - "reason", message) - - // Mark resources as not ready due to version skew - conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) - conditions.MarkNodePoolNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) - - return fmt.Errorf("version skew policy violation: %s", message) - } - } + if !allowed { + message := fmt.Sprintf("Version skew policy violation: control plane version %s is older than node pool version %s", controlPlaneCurrentVersion, nodePoolDesiredVersion) + logger.Info("Blocking Karpenter custom resources update due to version skew policy", + "controlPlaneCurrentVersion", controlPlaneCurrentVersion, + "nodePoolDesiredVersion", nodePoolDesiredVersion, + "reason", message) + + // Mark resources as not ready due to version skew + conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) + conditions.MarkNodePoolNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) + + return fmt.Errorf("version skew policy violation: %s", message) } workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) @@ -418,10 +395,11 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, cluster.Name, karpenterMachinePool.Name) // Add security groups tag selector if specified - securityGroupTagsSelector := map[string]string{ - fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", - "sigs.k8s.io/cluster-api-provider-aws/role": "node", - } + securityGroupTagsSelector := map[string]string{} + // securityGroupTagsSelector := map[string]string{ + // fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", + // "sigs.k8s.io/cluster-api-provider-aws/role": "node", + // } if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { for securityGroupTagKey, securityGroupTagValue := range karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups { securityGroupTagsSelector[securityGroupTagKey] = securityGroupTagValue @@ -429,10 +407,11 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. } // Add subnet tag selector if specified - subnetTagsSelector := map[string]string{ - fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", - "giantswarm.io/role": "nodes", - } + subnetTagsSelector := map[string]string{} + // subnetTagsSelector := map[string]string{ + // fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", + // "giantswarm.io/role": "nodes", + // } if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { for subnetTagKey, subnetTagValue := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { subnetTagsSelector[subnetTagKey] = subnetTagValue @@ -530,69 +509,10 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont nodePool.SetNamespace("") nodePool.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) - // Set default requirements and overwrite with the user provided configuration - requirements := []map[string]interface{}{ - { - "key": "karpenter.k8s.aws/instance-family", - "operator": "NotIn", - "values": []string{"t3", "t3a", "t2"}, - }, - { - "key": "karpenter.k8s.aws/instance-cpu", - "operator": "In", - "values": []string{"4", "8", "16", "32"}, - }, - { - "key": "karpenter.k8s.aws/instance-hypervisor", - "operator": "In", - "values": []string{"nitro"}, - }, - { - "key": "kubernetes.io/arch", - "operator": "In", - "values": []string{"amd64"}, - }, - { - "key": "karpenter.sh/capacity-type", - "operator": "In", - "values": []string{"spot", "on-demand"}, - }, - { - "key": "kubernetes.io/os", - "operator": "In", - "values": []string{"linux"}, - }, - } - if len(karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements) > 0 { - requirements = []map[string]interface{}{} - for _, req := range karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements { - requirement := map[string]interface{}{ - "key": req.Key, - "operator": req.Operator, - "values": req.Values, - } - requirements = append(requirements, requirement) - } - } - nodePool.SetNamespace("") - nodePool.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) - - // Set default labels and overwrite with the user provided configuration - labels := map[string]string{ - "giantswarm.io/machine-pool": fmt.Sprintf("%s-%s", cluster.Name, karpenterMachinePool.Name), - } - if karpenterMachinePool.Spec.NodePool != nil && len(karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels) > 0 { - for labelKey, labelValue := range karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels { - labels[labelKey] = labelValue - } - } - operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, nodePool, func() error { spec := map[string]interface{}{ "template": map[string]interface{}{ - "metadata": map[string]interface{}{ - "labels": labels, - }, + "metadata": map[string]interface{}{}, "spec": map[string]interface{}{ "startupTaints": []interface{}{ map[string]interface{}{ @@ -630,14 +550,17 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight } - tpl := spec["template"].(map[string]interface{})["spec"].(map[string]interface{}) + templateMetadata := spec["template"].(map[string]interface{})["metadata"].(map[string]interface{}) + templateMetadata["labels"] = karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels + + templateSpec := spec["template"].(map[string]interface{})["spec"].(map[string]interface{}) - tpl["taints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints - tpl["requirements"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements - tpl["expireAfter"] = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter + templateSpec["taints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints + templateSpec["requirements"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements + templateSpec["expireAfter"] = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter if karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod != nil { - tpl["terminationGracePeriodSeconds"] = karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod + templateSpec["terminationGracePeriod"] = karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod } } @@ -747,112 +670,23 @@ func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, m Complete(r) } -// CompareKubernetesVersions compares two Kubernetes versions and returns: -// -1 if version1 < version2 -// -// 0 if version1 == version2 -// -// +1 if version1 > version2 -func CompareKubernetesVersions(version1, version2 string) (int, error) { - // Remove 'v' prefix if present - v1 := strings.TrimPrefix(version1, "v") - v2 := strings.TrimPrefix(version2, "v") - - parts1 := strings.Split(v1, ".") - parts2 := strings.Split(v2, ".") - - if len(parts1) < 2 || len(parts2) < 2 { - return 0, fmt.Errorf("invalid version format: %s or %s", version1, version2) - } - - // Compare major version - major1, err := strconv.Atoi(parts1[0]) - if err != nil { - return 0, fmt.Errorf("invalid major version in %s: %w", version1, err) - } - major2, err := strconv.Atoi(parts2[0]) +// IsVersionSkewAllowed checks if the worker version can be updated based on the control plane version. +// The workers can't use a newer k8s version than the one used by the control plane. +func (r *KarpenterMachinePoolReconciler) IsVersionSkewAllowed(ctx context.Context, cluster *capi.Cluster, machinePool *capiexp.MachinePool) (bool, string, string, error) { + controlPlaneVersion, err := r.getControlPlaneVersion(ctx, cluster) if err != nil { - return 0, fmt.Errorf("invalid major version in %s: %w", version2, err) + return true, "", "", fmt.Errorf("failed to get current Control Plane k8s version: %w", err) } - if major1 != major2 { - if major1 < major2 { - return -1, nil - } - return 1, nil - } - - // Compare minor version - minor1, err := strconv.Atoi(parts1[1]) - if err != nil { - return 0, fmt.Errorf("invalid minor version in %s: %w", version1, err) - } - minor2, err := strconv.Atoi(parts2[1]) + controlPlaneCurrentK8sVersion, err := semver.ParseTolerant(controlPlaneVersion) if err != nil { - return 0, fmt.Errorf("invalid minor version in %s: %w", version2, err) - } - - if minor1 < minor2 { - return -1, nil - } else if minor1 > minor2 { - return 1, nil + return true, "", "", fmt.Errorf("failed to parse current Control Plane k8s version: %w", err) } - // If major and minor are the same, compare patch version if available - if len(parts1) >= 3 && len(parts2) >= 3 { - patch1, err := strconv.Atoi(parts1[2]) - if err != nil { - return 0, fmt.Errorf("invalid patch version in %s: %w", version1, err) - } - patch2, err := strconv.Atoi(parts2[2]) - if err != nil { - return 0, fmt.Errorf("invalid patch version in %s: %w", version2, err) - } - - if patch1 < patch2 { - return -1, nil - } else if patch1 > patch2 { - return 1, nil - } - } - - return 0, nil -} - -// IsVersionSkewAllowed checks if the worker version can be updated based on the control plane version -// According to Kubernetes version skew policy, workers can be at most 2 minor versions behind the control plane -func IsVersionSkewAllowed(controlPlaneVersion, workerVersion string) (bool, error) { - comparison, err := CompareKubernetesVersions(controlPlaneVersion, workerVersion) - if err != nil { - return false, err - } - - // If control plane version is older than or equal to worker version, allow the update - if comparison <= 0 { - return true, nil - } - - // Parse versions to check minor version difference - v1 := strings.TrimPrefix(controlPlaneVersion, "v") - v2 := strings.TrimPrefix(workerVersion, "v") - - parts1 := strings.Split(v1, ".") - parts2 := strings.Split(v2, ".") - - if len(parts1) < 2 || len(parts2) < 2 { - return false, fmt.Errorf("invalid version format: %s or %s", controlPlaneVersion, workerVersion) - } - - minor1, err := strconv.Atoi(parts1[1]) - if err != nil { - return false, fmt.Errorf("invalid minor version in %s: %w", controlPlaneVersion, err) - } - minor2, err := strconv.Atoi(parts2[1]) + machinePoolDesiredK8sVersion, err := semver.ParseTolerant(*machinePool.Spec.Template.Spec.Version) if err != nil { - return false, fmt.Errorf("invalid minor version in %s: %w", workerVersion, err) + return true, controlPlaneVersion, "", fmt.Errorf("failed to parse node pool desired k8s version: %w", err) } - // Allow if the difference is at most 2 minor versions - versionDiff := minor1 - minor2 - return versionDiff <= 2, nil + return controlPlaneCurrentK8sVersion.GE(machinePoolDesiredK8sVersion), controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, nil } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index a8719ac8..ec4d7052 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -9,7 +9,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -51,9 +53,8 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { DataSecretName = "foo-mp-12345" KarpenterMachinePoolName = "foo" KarpenterNodesInstanceProfile = "karpenter-iam-role" - KarpenterNodesSecurityGroup = "sg-12345678" - KarpenterNodesSubnets = "subnet-12345678" KubernetesVersion = "v1.29.1" + NewerKubernetesVersion = "v1.29.1" ) BeforeEach(func() { @@ -266,6 +267,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) err := k8sClient.Create(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ @@ -518,6 +523,8 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + terminationGracePeriod := metav1.Duration{Duration: 30 * time.Second} + weight := int32(1) karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ ObjectMeta: ctrl.ObjectMeta{ Namespace: namespace, @@ -554,12 +561,19 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, }, + ExpireAfter: karpenterinfra.MustParseNillableDuration("24h"), + TerminationGracePeriod: &terminationGracePeriod, }, }, Disruption: karpenterinfra.Disruption{ ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("1000Mi"), + }, + Weight: &weight, }, }, } @@ -621,6 +635,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) err = k8sClient.Create(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ @@ -685,6 +703,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) err = k8sClient.Create(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) }) When("there is no AWSCluster", func() { It("returns an error", func() { @@ -802,21 +824,9 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err := k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) - It("creates karpenter resources in the wc", func() { + It("creates karpenter EC2NodeClass object in workload cluster", func() { Expect(reconcileErr).NotTo(HaveOccurred()) - nodepoolList := &unstructured.UnstructuredList{} - nodepoolList.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "karpenter.sh", - Kind: "NodePoolList", - Version: "v1", - }) - - err := k8sClient.List(ctx, nodepoolList) - Expect(err).NotTo(HaveOccurred()) - Expect(nodepoolList.Items).To(HaveLen(1)) - Expect(nodepoolList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) - ec2nodeclassList := &unstructured.UnstructuredList{} ec2nodeclassList.SetGroupVersionKind(schema.GroupVersionKind{ Group: "karpenter.k8s.aws", @@ -824,64 +834,68 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Version: "v1", }) - err = k8sClient.List(ctx, ec2nodeclassList) + err := k8sClient.List(ctx, ec2nodeclassList) Expect(err).NotTo(HaveOccurred()) Expect(ec2nodeclassList.Items).To(HaveLen(1)) Expect(ec2nodeclassList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) - amiSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "amiSelectorTerms") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(amiSelectorTerms).To(HaveLen(1)) - - // Let's make sure the amiSelectorTerms field is what we expect - term0, ok := amiSelectorTerms[0].(map[string]interface{}) - Expect(ok).To(BeTrue(), "expected amiSelectorTerms[0] to be a map") - // Assert the name field - nameVal, ok := term0["name"].(string) - Expect(ok).To(BeTrue(), "expected name to be a string") - Expect(nameVal).To(Equal(AMIName)) - // Assert the owner field - ownerF, ok := term0["owner"].(string) - Expect(ok).To(BeTrue(), "expected owner to be a number") - Expect(ownerF).To(Equal(AMIOwner)) - // Assert security groups are the expected ones - securityGroupSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "securityGroupSelectorTerms") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(securityGroupSelectorTerms).To(HaveLen(1)) - // Let's make sure the securityGroupSelectorTerms field is what we expect - securityGroupSelectorTerm0, ok := securityGroupSelectorTerms[0].(map[string]interface{}) - Expect(ok).To(BeTrue(), "expected securityGroupSelectorTerms[0] to be a map") - // Assert the security group name field - securityGroupTags, ok := securityGroupSelectorTerm0["tags"].(map[string]interface{}) - Expect(ok).To(BeTrue(), "expected tags to be a map[string]string") - Expect(securityGroupTags["my-target-sg"]).To(Equal("is-this")) - - // Assert subnets are the expected ones - subnetSelectorTerms, found, err := unstructured.NestedSlice(ec2nodeclassList.Items[0].Object, "spec", "subnetSelectorTerms") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(subnetSelectorTerms).To(HaveLen(1)) - // Let's make sure the subnetSelectorTerms field is what we expect - subnetSelectorTerm0, ok := subnetSelectorTerms[0].(map[string]interface{}) - Expect(ok).To(BeTrue(), "expected subnetSelectorTerms[0] to be a map") - // Assert the security group name field - subnetTags, ok := subnetSelectorTerm0["tags"].(map[string]interface{}) - Expect(ok).To(BeTrue(), "expected tags to be a map[string]string") - Expect(subnetTags["my-target-subnet"]).To(Equal("is-that")) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "userData").To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s-%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, ClusterName, KarpenterMachinePoolName))) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "instanceProfile").To(Equal(KarpenterNodesInstanceProfile)) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To( + ContainElement( // slice matcher: at least one element matches + gstruct.MatchAllKeys(gstruct.Keys{ // map matcher: all these keys must match exactly + "name": Equal(AMIName), + "owner": Equal(AMIOwner), + }), + ), + ) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "securityGroupSelectorTerms").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "securityGroupSelectorTerms").To( + ConsistOf( + gstruct.MatchAllKeys(gstruct.Keys{ + // the top-level map has a single "tags" field, + // whose value itself must be a map containing our SG name → value + "tags": gstruct.MatchAllKeys(gstruct.Keys{ + "my-target-sg": Equal("is-this"), + }), + }), + ), + ) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "subnetSelectorTerms").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "subnetSelectorTerms").To( + ConsistOf( + gstruct.MatchAllKeys(gstruct.Keys{ + "tags": gstruct.MatchAllKeys(gstruct.Keys{ + "my-target-subnet": Equal("is-that"), + }), + }), + ), + ) + }) + It("creates karpenter NodePool object in workload cluster", func() { + nodepoolList := &unstructured.UnstructuredList{} + nodepoolList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.sh", + Kind: "NodePoolList", + Version: "v1", + }) - // Assert userdata is the expected one - userData, found, err := unstructured.NestedString(ec2nodeclassList.Items[0].Object, "spec", "userData") + err := k8sClient.List(ctx, nodepoolList) Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(userData).To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s-%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, ClusterName, KarpenterMachinePoolName))) + Expect(nodepoolList.Items).To(HaveLen(1)) + Expect(nodepoolList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) - // Assert instance profile is the expected one - iamInstanceProfile, found, err := unstructured.NestedString(ec2nodeclassList.Items[0].Object, "spec", "instanceProfile") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(iamInstanceProfile).To(Equal(KarpenterNodesInstanceProfile)) + ExpectUnstructured(nodepoolList.Items[0], "spec", "disruption", "consolidateAfter").To(Equal("30s")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "disruption", "consolidationPolicy").To(BeEquivalentTo(karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized)) + ExpectUnstructured(nodepoolList.Items[0], "spec", "limits").To(HaveKeyWithValue("cpu", "1")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "limits").To(HaveKeyWithValue("memory", "1000Mi")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "weight").To(BeEquivalentTo(int64(1))) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "expireAfter").To(BeEquivalentTo("24h")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "terminationGracePeriod").To(BeEquivalentTo("30s")) }) It("adds the finalizer to the KarpenterMachinePool", func() { Expect(reconcileErr).NotTo(HaveOccurred()) @@ -905,18 +919,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.Annotations).To(HaveKeyWithValue(controllers.BootstrapDataHashAnnotation, Equal(capiBootstrapSecretHash))) }) - When("there are no NodeClaim in the workload cluster yet", func() { - It("requeues to try again soon", func() { - Expect(reconcileErr).NotTo(HaveOccurred()) - Expect(reconcileResult.RequeueAfter).To(Equal(1 * time.Minute)) - updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - Expect(updatedKarpenterMachinePool.Status.Ready).To(BeFalse()) - Expect(updatedKarpenterMachinePool.Status.Replicas).To(BeZero()) - Expect(updatedKarpenterMachinePool.Spec.ProviderIDList).To(BeEmpty()) - }) - }) When("there are NodeClaim resources in the workload cluster", func() { BeforeEach(func() { nodeClaim1 := &unstructured.Unstructured{} @@ -1009,7 +1011,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { When("the KarpenterMachinePool exists with a hash annotation signaling unchanged bootstrap data", func() { BeforeEach(func() { dataSecretName := DataSecretName - version := KubernetesVersion + kubernetesVersion := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ Namespace: namespace, @@ -1020,7 +1022,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capiexp.MachinePoolSpec{ ClusterName: ClusterName, - // Replicas: nil, Template: capi.MachineTemplateSpec{ ObjectMeta: capi.ObjectMeta{}, Spec: capi.MachineSpec{ @@ -1040,7 +1041,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, - Version: &version, + Version: &kubernetesVersion, }, }, }, @@ -1099,6 +1100,32 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": KubernetesVersion, + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ Namespace: namespace, @@ -1108,6 +1135,12 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", Namespace: namespace, @@ -1175,78 +1208,14 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(s3Client.PutCallCount()).To(Equal(0)) }) }) - - Describe("Version comparison functions", func() { - Describe("CompareKubernetesVersions", func() { - It("should correctly compare versions", func() { - // Test cases: (version1, version2, expected_result) - testCases := []struct { - v1, v2 string - want int - }{ - {"v1.20.0", "v1.20.0", 0}, - {"v1.20.0", "v1.21.0", -1}, - {"v1.21.0", "v1.20.0", 1}, - {"v1.20.1", "v1.20.0", 1}, - {"v1.20.0", "v1.20.1", -1}, - {"1.20.0", "v1.20.0", 0}, - {"v1.20.0", "1.20.0", 0}, - } - - for _, tc := range testCases { - result, err := controllers.CompareKubernetesVersions(tc.v1, tc.v2) - Expect(err).NotTo(HaveOccurred()) - Expect(result).To(Equal(tc.want), "comparing %s with %s", tc.v1, tc.v2) - } - }) - - It("should handle invalid version formats", func() { - _, err := controllers.CompareKubernetesVersions("invalid", "v1.20.0") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("invalid version format")) - }) - }) - - Describe("IsVersionSkewAllowed", func() { - It("should allow updates when control plane is older or equal", func() { - testCases := []struct { - controlPlane, worker string - allowed bool - }{ - {"v1.20.0", "v1.20.0", true}, // Same version - {"v1.20.0", "v1.21.0", true}, // Worker newer - {"v1.20.0", "v1.22.0", true}, // Worker newer - } - - for _, tc := range testCases { - allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) - Expect(err).NotTo(HaveOccurred()) - Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) - } - }) - - It("should allow updates within 2 minor versions", func() { - testCases := []struct { - controlPlane, worker string - allowed bool - }{ - {"v1.22.0", "v1.20.0", true}, // 2 versions behind - {"v1.22.0", "v1.21.0", true}, // 1 version behind - {"v1.22.0", "v1.19.0", false}, // 3 versions behind - {"v1.23.0", "v1.20.0", false}, // 3 versions behind - } - - for _, tc := range testCases { - allowed, err := controllers.IsVersionSkewAllowed(tc.controlPlane, tc.worker) - Expect(err).NotTo(HaveOccurred()) - Expect(allowed).To(Equal(tc.allowed), "control plane %s, worker %s", tc.controlPlane, tc.worker) - } - }) - - It("should handle invalid version formats", func() { - _, err := controllers.IsVersionSkewAllowed("invalid", "v1.20.0") - Expect(err).To(HaveOccurred()) - }) - }) - }) }) + +// ExpectUnstructured digs into u.Object at the given path, +// asserts that it was found and error‐free, and returns +// a GomegaAssertion on the raw interface{} value. +func ExpectUnstructured(u unstructured.Unstructured, fields ...string) Assertion { + v, found, err := unstructured.NestedFieldNoCopy(u.Object, fields...) + Expect(found).To(BeTrue(), "expected to find field %v", fields) + Expect(err).NotTo(HaveOccurred(), "error retrieving %v: %v", fields, err) + return Expect(v) +} diff --git a/go.mod b/go.mod index af656c81..43a55472 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.24.3 require ( github.com/aws/aws-sdk-go v1.55.7 + github.com/blang/semver/v4 v4.0.0 github.com/giantswarm/k8smetadata v0.25.0 github.com/go-logr/logr v1.4.3 github.com/google/go-cmp v0.7.0 @@ -32,7 +33,6 @@ require ( require ( github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect From 9fe6d2f95d9a90f0ba2c00f6237c0233132344ad Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 15 Jul 2025 16:46:55 +0200 Subject: [PATCH 13/41] Fix user data s3 key --- controllers/karpentermachinepool_controller.go | 6 +++--- controllers/karpentermachinepool_controller_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 6ff9d05f..f860676f 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -392,7 +392,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. ec2NodeClass.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) // Generate user data for Ignition - userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, cluster.Name, karpenterMachinePool.Name) + userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, karpenterMachinePool.Name) // Add security groups tag selector if specified securityGroupTagsSelector := map[string]string{} @@ -628,13 +628,13 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co } // generateUserData generates the user data for Ignition configuration -func (r *KarpenterMachinePoolReconciler) generateUserData(s3bucketName, clusterName, karpenterMachinePoolName string) string { +func (r *KarpenterMachinePoolReconciler) generateUserData(s3bucketName, karpenterMachinePoolName string) string { userData := map[string]interface{}{ "ignition": map[string]interface{}{ "config": map[string]interface{}{ "merge": []map[string]interface{}{ { - "source": fmt.Sprintf("s3://%s/%s/%s-%s", s3bucketName, S3ObjectPrefix, clusterName, karpenterMachinePoolName), + "source": fmt.Sprintf("s3://%s/%s/%s", s3bucketName, S3ObjectPrefix, karpenterMachinePoolName), "verification": map[string]interface{}{}, }, }, diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index ec4d7052..e37c20a7 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -839,7 +839,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(ec2nodeclassList.Items).To(HaveLen(1)) Expect(ec2nodeclassList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) - ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "userData").To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s-%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, ClusterName, KarpenterMachinePoolName))) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "userData").To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, KarpenterMachinePoolName))) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "instanceProfile").To(Equal(KarpenterNodesInstanceProfile)) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To(HaveLen(1)) From fc9987286ec0a86b3ba3493b703f52c966bda8b1 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Wed, 16 Jul 2025 10:31:40 +0200 Subject: [PATCH 14/41] Ignore 'no matches for kind' errors when deleting karpenter CRs --- .../karpentermachinepool_controller.go | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index f860676f..98533e4e 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -13,6 +13,7 @@ import ( "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" @@ -37,12 +38,9 @@ import ( const ( BootstrapDataHashAnnotation = "giantswarm.io/userdata-hash" + EC2NodeClassAPIGroup = "karpenter.k8s.aws" KarpenterFinalizer = "capa-operator.finalizers.giantswarm.io/karpenter-controller" S3ObjectPrefix = "karpenter-machine-pool" - // KarpenterNodePoolReadyCondition reports on current status of the autoscaling group. Ready indicates the group is provisioned. - KarpenterNodePoolReadyCondition capi.ConditionType = "KarpenterNodePoolReadyCondition" - // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. - WaitingForBootstrapDataReason = "WaitingForBootstrapData" // NodePoolCreationFailedReason indicates that the NodePool creation failed NodePoolCreationFailedReason = "NodePoolCreationFailed" // EC2NodeClassCreationFailedReason indicates that the EC2NodeClass creation failed @@ -380,7 +378,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co // createOrUpdateEC2NodeClass creates or updates the EC2NodeClass resource in the workload cluster func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { ec2NodeClassGVR := schema.GroupVersionResource{ - Group: "karpenter.k8s.aws", + Group: EC2NodeClassAPIGroup, Version: "v1", Resource: "ec2nodeclasses", } @@ -396,10 +394,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. // Add security groups tag selector if specified securityGroupTagsSelector := map[string]string{} - // securityGroupTagsSelector := map[string]string{ - // fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", - // "sigs.k8s.io/cluster-api-provider-aws/role": "node", - // } if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { for securityGroupTagKey, securityGroupTagValue := range karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups { securityGroupTagsSelector[securityGroupTagKey] = securityGroupTagValue @@ -408,10 +402,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. // Add subnet tag selector if specified subnetTagsSelector := map[string]string{} - // subnetTagsSelector := map[string]string{ - // fmt.Sprintf("sigs.k8s.io/cluster-api-provider-aws/cluster/%s", cluster.Name): "owned", - // "giantswarm.io/role": "nodes", - // } if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { for subnetTagKey, subnetTagValue := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { subnetTagsSelector[subnetTagKey] = subnetTagValue @@ -527,7 +517,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont }, }, "nodeClassRef": map[string]interface{}{ - "group": "karpenter.k8s.aws", + "group": EC2NodeClassAPIGroup, "kind": "EC2NodeClass", "name": karpenterMachinePool.Name, }, @@ -602,14 +592,14 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co nodePool.SetName(karpenterMachinePool.Name) nodePool.SetNamespace("default") - if err := workloadClusterClient.Delete(ctx, nodePool); err != nil && !k8serrors.IsNotFound(err) { + if err := workloadClusterClient.Delete(ctx, nodePool); err != nil && !k8serrors.IsNotFound(err) && !meta.IsNoMatchError(err) { logger.Error(err, "failed to delete NodePool", "name", karpenterMachinePool.Name) return fmt.Errorf("failed to delete NodePool: %w", err) } // Delete EC2NodeClass ec2NodeClassGVR := schema.GroupVersionResource{ - Group: "karpenter.k8s.aws", + Group: EC2NodeClassAPIGroup, Version: "v1", Resource: "ec2nodeclasses", } @@ -619,7 +609,7 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co ec2NodeClass.SetName(karpenterMachinePool.Name) ec2NodeClass.SetNamespace("default") - if err := workloadClusterClient.Delete(ctx, ec2NodeClass); err != nil && !k8serrors.IsNotFound(err) { + if err := workloadClusterClient.Delete(ctx, ec2NodeClass); err != nil && !k8serrors.IsNotFound(err) && !meta.IsNoMatchError(err) { logger.Error(err, "failed to delete EC2NodeClass", "name", karpenterMachinePool.Name) return fmt.Errorf("failed to delete EC2NodeClass: %w", err) } From 04829045d015845a35d14e5466a8db15506b3025 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 17 Jul 2025 16:00:27 +0200 Subject: [PATCH 15/41] Mimic upstream ec2nodeclass --- CHANGELOG.md | 4 + api/v1alpha1/ec2nodeclass.go | 451 +++++++++++++ api/v1alpha1/karpentermachinepool_types.go | 274 -------- api/v1alpha1/nodepool.go | 256 ++++++++ api/v1alpha1/zz_generated.deepcopy.go | 396 +++++++++++- ...luster.x-k8s.io_karpentermachinepools.yaml | 610 +++++++++++++++++- .../karpentermachinepool_controller.go | 76 +-- .../karpentermachinepool_controller_test.go | 119 +++- ...luster.x-k8s.io_karpentermachinepools.yaml | 610 +++++++++++++++++- 9 files changed, 2383 insertions(+), 413 deletions(-) create mode 100644 api/v1alpha1/ec2nodeclass.go create mode 100644 api/v1alpha1/nodepool.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 67895d3b..f3f4b940 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Create karpenter custom resources in workload clusters. + ## [0.20.0] - 2025-06-23 ### Changed diff --git a/api/v1alpha1/ec2nodeclass.go b/api/v1alpha1/ec2nodeclass.go new file mode 100644 index 00000000..f5f70322 --- /dev/null +++ b/api/v1alpha1/ec2nodeclass.go @@ -0,0 +1,451 @@ +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. +// This will contain configuration necessary to launch instances in AWS. +type EC2NodeClassSpec struct { + // SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="subnetSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term",rule="!self.all(x, has(x.id) && has(x.tags))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SubnetSelectorTerms []SubnetSelectorTerm `json:"subnetSelectorTerms" hash:"ignore"` + // SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="securityGroupSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))" + // +kubebuilder:validation:XValidation:message="'name' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term",rule="!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` + // CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + // determine the set of eligible capacity reservations. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))" + // +kubebuilder:validation:MaxItems:=30 + // +optional + CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms" hash:"ignore"` + // AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + // +optional + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` + // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameter']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameter))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms",rule="!(self.exists(x, has(x.alias)) && self.size() != 1)" + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=30 + // +required + AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms" hash:"ignore"` + // AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + // This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + // family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + // alias is specified, this field is required. + // NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + // the AMIFamily() helper function + // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Custom,Windows2019,Windows2022} + // +optional + AMIFamily *string `json:"amiFamily,omitempty" hash:"ignore"` + // UserData to be applied to the provisioned nodes. + // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + // this UserData to ensure nodes are being provisioned with the correct configuration. + // +optional + UserData *string `json:"userData,omitempty"` + // Role is the AWS identity that nodes use. This field is immutable. + // This field is mutually exclusive from instanceProfile. + // Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + // This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + // for the old instance profiles on an update. + // +kubebuilder:validation:XValidation:rule="self != ''",message="role cannot be empty" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="immutable field changed" + // +optional + Role string `json:"role,omitempty"` + // InstanceProfile is the AWS entity that instances use. + // This field is mutually exclusive from role. + // The instance profile should already have a role assigned to it that Karpenter + // has PassRole permission on for instance launch using this instanceProfile to succeed. + // +kubebuilder:validation:XValidation:rule="self != ''",message="instanceProfile cannot be empty" + // +optional + InstanceProfile *string `json:"instanceProfile,omitempty"` + // Tags to be applied on ec2 resources like instances and launch templates. + // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching eks:eks-cluster-name",rule="self.all(k, k !='eks:eks-cluster-name')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodeclaim",rule="self.all(k, k !='karpenter.sh/nodeclaim')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass",rule="self.all(k, k !='karpenter.k8s.aws/ec2nodeclass')" + // +optional + Tags map[string]string `json:"tags,omitempty"` + // Kubelet defines args to be used when configuring kubelet on provisioned nodes. + // They are a subset of the upstream types, recognizing not all options may be supported. + // Wherever possible, the types and names should reflect the upstream kubelet types. + // +kubebuilder:validation:XValidation:message="imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent",rule="has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true" + // +kubebuilder:validation:XValidation:message="evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod",rule="has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true" + // +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true" + // +optional + Kubelet *KubeletConfiguration `json:"kubelet,omitempty"` + // BlockDeviceMappings to be applied to provisioned nodes. + // +kubebuilder:validation:XValidation:message="must have only one blockDeviceMappings with rootVolume",rule="self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1" + // +kubebuilder:validation:MaxItems:=50 + // +optional + BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` + // InstanceStorePolicy specifies how to handle instance-store disks. + // +optional + InstanceStorePolicy *InstanceStorePolicy `json:"instanceStorePolicy,omitempty"` + // DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + // +optional + DetailedMonitoring *bool `json:"detailedMonitoring,omitempty"` + // MetadataOptions for the generated launch template of provisioned nodes. + // + // This specifies the exposure of the Instance Metadata Service to + // provisioned EC2 nodes. For more information, + // see Instance Metadata and User Data + // (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + // in the Amazon Elastic Compute Cloud User Guide. + // + // Refer to recommended, security best practices + // (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + // for limiting exposure of Instance Metadata and User Data to pods. + // If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + // disabled, with httpPutResponseLimit of 1, and with httpTokens + // required. + // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":1,"httpTokens":"required"} + // +optional + MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // Context is a Reserved field in EC2 APIs + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + // +optional + Context *string `json:"context,omitempty"` +} + +// SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SubnetSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the subnet id in EC2 + // +kubebuilder:validation:Pattern="subnet-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` +} + +// SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SecurityGroupSelectorTerm struct { + // Tags is a map of key/value tags used to select security groups. + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the security group id in EC2 + // +kubebuilder:validation:Pattern:="sg-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the security group name in EC2. + // This value is the name field, which is different from the name tag. + Name string `json:"name,omitempty"` +} + +type CapacityReservationSelectorTerm struct { + // Tags is a map of key/value tags used to select capacity reservations. + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the capacity reservation id in EC2 + // +kubebuilder:validation:Pattern:="^cr-[0-9a-z]+$" + // +optional + ID string `json:"id,omitempty"` + // Owner is the owner id for the ami. + // +kubebuilder:validation:Pattern:="^[0-9]{12}$" + // +optional + OwnerID string `json:"ownerID,omitempty"` +} + +// AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type AMISelectorTerm struct { + // Alias specifies which EKS optimized AMI to select. + // Each alias consists of a family and an AMI version, specified as "family@version". + // Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + // The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + // The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + // Note: The Windows families do **not** support version pinning, and only latest may be used. + // +kubebuilder:validation:XValidation:message="'alias' is improperly formatted, must match the format 'family@version'",rule="self.matches('^[a-zA-Z0-9]+@.+$')" + // +kubebuilder:validation:XValidation:message="family is not supported, must be one of the following: 'al2', 'al2023', 'bottlerocket', 'windows2019', 'windows2022'",rule="self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022']" + // +kubebuilder:validation:XValidation:message="windows families may only specify version 'latest'",rule="self.split('@')[0] in ['windows2019','windows2022'] ? self.split('@')[1] == 'latest' : true" + // +kubebuilder:validation:MaxLength=30 + // +optional + Alias string `json:"alias,omitempty"` + // Tags is a map of key/value tags used to select amis. + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the ami id in EC2 + // +kubebuilder:validation:Pattern:="ami-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the ami name in EC2. + // This value is the name field, which is different from the name tag. + // +optional + Name string `json:"name,omitempty"` + // Owner is the owner for the ami. + // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + // +optional + Owner string `json:"owner,omitempty"` + // SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. + // +optional + SSMParameter string `json:"ssmParameter,omitempty"` +} + +// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. +// They are a subset of the upstream types, recognizing not all options may be supported. +// Wherever possible, the types and names should reflect the upstream kubelet types. +// https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration +// https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53 +type KubeletConfiguration struct { + // clusterDNS is a list of IP addresses for the cluster DNS server. + // Note that not all providers may use all addresses. + // +optional + ClusterDNS []string `json:"clusterDNS,omitempty"` + // MaxPods is an override for the maximum number of pods that can run on + // a worker node instance. + // +kubebuilder:validation:Minimum:=0 + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + // PodsPerCore is an override for the number of pods that can run on a worker node + // instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + // MaxPods is a lower value, that value will be used. + // +kubebuilder:validation:Minimum:=0 + // +optional + PodsPerCore *int32 `json:"podsPerCore,omitempty"` + // SystemReserved contains resources reserved for OS system daemons and kernel memory. + // +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // KubeReserved contains resources reserved for Kubernetes system components. + // +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // EvictionHard is the map of signal names to quantities that define hard eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionHard map[string]string `json:"evictionHard,omitempty"` + // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoftGracePeriod map[string]metav1.Duration `json:"evictionSoftGracePeriod,omitempty"` + // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + // response to soft eviction thresholds being met. + // +optional + EvictionMaxPodGracePeriod *int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // ImageGCHighThresholdPercent is the percent of disk usage after which image + // garbage collection is always run. The percent is calculated by dividing this + // field value by 100, so this field must be between 0 and 100, inclusive. + // When specified, the value must be greater than ImageGCLowThresholdPercent. + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` + // ImageGCLowThresholdPercent is the percent of disk usage before which image + // garbage collection is never run. Lowest disk usage to garbage collect to. + // The percent is calculated by dividing this field value by 100, + // so the field value must be between 0 and 100, inclusive. + // When specified, the value must be less than imageGCHighThresholdPercent + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` + // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + // +optional + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` +} + +// MetadataOptions contains parameters for specifying the exposure of the +// Instance Metadata Service to provisioned EC2 nodes. +type MetadataOptions struct { + // HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + // nodes. If metadata options is non-nil, but this parameter is not specified, + // the default state is "enabled". + // + // If you specify a value of "disabled", instance metadata will not be accessible + // on the node. + // +kubebuilder:default=enabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPEndpoint *string `json:"httpEndpoint,omitempty"` + // HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + // service on provisioned nodes. If metadata options is non-nil, but this parameter + // is not specified, the default state is "disabled". + // +kubebuilder:default=disabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPProtocolIPv6 *string `json:"httpProtocolIPv6,omitempty"` + // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + // instance metadata requests. The larger the number, the further instance + // metadata requests can travel. Possible values are integers from 1 to 64. + // If metadata options is non-nil, but this parameter is not specified, the + // default value is 1. + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=64 + // +optional + HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"` + // HTTPTokens determines the state of token usage for instance metadata + // requests. If metadata options is non-nil, but this parameter is not + // specified, the default state is "required". + // + // If the state is optional, one can choose to retrieve instance metadata with + // or without a signed token header on the request. If one retrieves the IAM + // role credentials without a token, the version 1.0 role credentials are + // returned. If one retrieves the IAM role credentials using a valid signed + // token, the version 2.0 role credentials are returned. + // + // If the state is "required", one must send a signed token header with any + // instance metadata retrieval requests. In this state, retrieving the IAM + // role credentials always returns the version 2.0 credentials; the version + // 1.0 credentials are not available. + // +kubebuilder:default=required + // +kubebuilder:validation:Enum:={required,optional} + // +optional + HTTPTokens *string `json:"httpTokens,omitempty"` +} + +type BlockDeviceMapping struct { + // The device name (for example, /dev/sdh or xvdh). + // +optional + DeviceName *string `json:"deviceName,omitempty"` + // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + // +kubebuilder:validation:XValidation:message="snapshotID or volumeSize must be defined",rule="has(self.snapshotID) || has(self.volumeSize)" + // +kubebuilder:validation:XValidation:message="snapshotID must be set when volumeInitializationRate is set",rule="!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '')" + // +optional + EBS *BlockDevice `json:"ebs,omitempty"` + // RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + // configure at most one root volume in BlockDeviceMappings. + // +optional + RootVolume bool `json:"rootVolume,omitempty"` +} + +type BlockDevice struct { + // DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + // +optional + DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"` + // Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + // be attached to instances that support Amazon EBS encryption. If you are creating + // a volume from a snapshot, you can't specify an encryption value. + // +optional + Encrypted *bool `json:"encrypted,omitempty"` + // IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + // this represents the number of IOPS that are provisioned for the volume. For + // gp2 volumes, this represents the baseline performance of the volume and the + // rate at which the volume accumulates I/O credits for bursting. + // + // The following are the supported values for each volume type: + // + // * gp3: 3,000-16,000 IOPS + // + // * io1: 100-64,000 IOPS + // + // * io2: 100-64,000 IOPS + // + // For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + // on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + // Other instance families guarantee performance up to 32,000 IOPS. + // + // This parameter is supported for io1, io2, and gp3 volumes only. This parameter + // is not supported for gp2, st1, sc1, or standard volumes. + // +optional + IOPS *int64 `json:"iops,omitempty"` + // Identifier (key ID, key alias, key ARN, or alias ARN) of the customer managed KMS key to use for EBS encryption. + // +optional + KMSKeyID *string `json:"kmsKeyID,omitempty"` + // SnapshotID is the ID of an EBS snapshot + // +optional + SnapshotID *string `json:"snapshotID,omitempty"` + // Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + // Valid Range: Minimum value of 125. Maximum value of 1000. + // +optional + Throughput *int64 `json:"throughput,omitempty"` + // VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + // in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + // initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + // predictable and consistent rate after creation. Only allowed if SnapshotID is set. + // Valid Range: Minimum value of 100. Maximum value of 300. + // +kubebuilder:validation:Minimum:=100 + // +kubebuilder:validation:Maximum:=300 + // +optional + VolumeInitializationRate *int32 `json:"volumeInitializationRate,omitempty"` + // VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + // a volume size. The following are the supported volumes sizes for each volume + // type: + // + // * gp2 and gp3: 1-16,384 + // + // * io1 and io2: 4-16,384 + // + // * st1 and sc1: 125-16,384 + // + // * standard: 1-1,024 + // + TODO: Add the CEL resources.quantity type after k8s 1.29 + // + https://github.com/kubernetes/apiserver/commit/b137c256373aec1c5d5810afbabb8932a19ecd2a#diff-838176caa5882465c9d6061febd456397a3e2b40fb423ed36f0cabb1847ecb4dR190 + // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type:=string + // +optional + VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` + // VolumeType of the block device. + // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + // in the Amazon Elastic Compute Cloud User Guide. + // +kubebuilder:validation:Enum:={standard,io1,io2,gp2,sc1,st1,gp3} + // +optional + VolumeType *string `json:"volumeType,omitempty"` +} + +// InstanceStorePolicy enumerates options for configuring instance store disks. +// +kubebuilder:validation:Enum={RAID0} +type InstanceStorePolicy string + +const ( + // InstanceStorePolicyRAID0 configures a RAID-0 array that includes all ephemeral NVMe instance storage disks. + // The containerd and kubelet state directories (`/var/lib/containerd` and `/var/lib/kubelet`) will then use the + // ephemeral storage for more and faster node ephemeral-storage. The node's ephemeral storage can be shared among + // pods that request ephemeral storage and container images that are downloaded to the node. + InstanceStorePolicyRAID0 InstanceStorePolicy = "RAID0" +) + +// // EC2NodeClassSpec defines the configuration for a Karpenter EC2NodeClass +// type EC2NodeClassSpec struct { +// // Name is the ami name in EC2. +// // This value is the name field, which is different from the name tag. +// AMIName string `json:"amiName,omitempty"` +// // Owner is the owner for the ami. +// // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" +// AMIOwner string `json:"amiOwner,omitempty"` +// +// // SecurityGroups specifies the security groups to use +// // +optional +// SecurityGroups map[string]string `json:"securityGroups,omitempty"` +// +// // Subnets specifies the subnets to use +// // +optional +// Subnets map[string]string `json:"subnets,omitempty"` +// } diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index a5a587b2..5fc898dc 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -17,286 +17,12 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" capi "sigs.k8s.io/cluster-api/api/v1beta1" ) -// NodePoolSpec defines the configuration for a Karpenter NodePool -type NodePoolSpec struct { - // Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. - // NodeClaims launched from this NodePool will often be further constrained than the template specifies. - // +required - Template NodeClaimTemplate `json:"template"` - - // Disruption contains the parameters that relate to Karpenter's disruption logic - // +kubebuilder:default:={consolidateAfter: "0s"} - // +optional - Disruption Disruption `json:"disruption"` - - // Limits define a set of bounds for provisioning capacity. - // +optional - Limits Limits `json:"limits,omitempty"` - - // Weight is the priority given to the nodepool during scheduling. A higher - // numerical weight indicates that this nodepool will be ordered - // ahead of other nodepools with lower weights. A nodepool with no weight - // will be treated as if it is a nodepool with a weight of 0. - // +kubebuilder:validation:Minimum:=1 - // +kubebuilder:validation:Maximum:=100 - // +optional - Weight *int32 `json:"weight,omitempty"` -} - -type NodeClaimTemplate struct { - ObjectMeta `json:"metadata,omitempty"` - // +required - Spec NodeClaimTemplateSpec `json:"spec"` -} - -type ObjectMeta struct { - // Map of string keys and values that can be used to organize and categorize - // (scope and select) objects. May match selectors of replication controllers - // and services. - // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels - // +optional - Labels map[string]string `json:"labels,omitempty"` - - // Annotations is an unstructured key value map stored with a resource that may be - // set by external tools to store and retrieve arbitrary metadata. They are not - // queryable and should be preserved when modifying objects. - // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations - // +optional - Annotations map[string]string `json:"annotations,omitempty"` -} - -// NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool -// NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since -// users are not able to set resource requests in the NodePool. -type NodeClaimTemplateSpec struct { - // Taints will be applied to the NodeClaim's node. - // +optional - Taints []v1.Taint `json:"taints,omitempty"` - // StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically - // within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by - // daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning - // purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. - // +optional - StartupTaints []v1.Taint `json:"startupTaints,omitempty"` - // Requirements are layered with GetLabels and applied to every node. - // +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)" - // +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)" - // +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)" - // +kubebuilder:validation:MaxItems:=100 - // +required - Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"` - // TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. - // - // Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. - // - // This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. - // When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. - // - // Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. - // If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, - // that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. - // - // The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. - // If left undefined, the controller will wait indefinitely for pods to be drained. - // +kubebuilder:validation:Pattern=`^([0-9]+(s|m|h))+$` - // +kubebuilder:validation:Type="string" - // +optional - TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"` - // ExpireAfter is the duration the controller will wait - // before terminating a node, measured from when the node is created. This - // is useful to implement features like eventually consistent node upgrade, - // memory leak protection, and disruption testing. - // +kubebuilder:default:="720h" - // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` - // +kubebuilder:validation:Type="string" - // +kubebuilder:validation:Schemaless - // +optional - ExpireAfter NillableDuration `json:"expireAfter,omitempty"` -} - -// A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values -// and minValues that represent the requirement to have at least that many values. -type NodeSelectorRequirementWithMinValues struct { - v1.NodeSelectorRequirement `json:",inline"` - // This field is ALPHA and can be dropped or replaced at any time - // MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. - // +kubebuilder:validation:Minimum:=1 - // +kubebuilder:validation:Maximum:=50 - // +optional - MinValues *int `json:"minValues,omitempty"` -} - -type NodeClassReference struct { - // Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - // +kubebuilder:validation:XValidation:rule="self != ''",message="kind may not be empty" - // +required - Kind string `json:"kind"` - // Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names - // +kubebuilder:validation:XValidation:rule="self != ''",message="name may not be empty" - // +required - Name string `json:"name"` - // API version of the referent - // +kubebuilder:validation:XValidation:rule="self != ''",message="group may not be empty" - // +kubebuilder:validation:Pattern=`^[^/]*$` - // +required - Group string `json:"group"` -} - -type Limits v1.ResourceList - -type ConsolidationPolicy string - -const ( - ConsolidationPolicyWhenEmpty ConsolidationPolicy = "WhenEmpty" - ConsolidationPolicyWhenEmptyOrUnderutilized ConsolidationPolicy = "WhenEmptyOrUnderutilized" -) - -// DisruptionReason defines valid reasons for disruption budgets. -// +kubebuilder:validation:Enum={Underutilized,Empty,Drifted} -type DisruptionReason string - -// Budget defines when Karpenter will restrict the -// number of Node Claims that can be terminating simultaneously. -type Budget struct { - // Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. - // Otherwise, this will apply to each reason defined. - // allowed reasons are Underutilized, Empty, and Drifted. - // +optional - Reasons []DisruptionReason `json:"reasons,omitempty"` - // Nodes dictates the maximum number of NodeClaims owned by this NodePool - // that can be terminating at once. This is calculated by counting nodes that - // have a deletion timestamp set, or are actively being deleted by Karpenter. - // This field is required when specifying a budget. - // This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern - // checking for int nodes for IntOrString nodes. - // Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 - // +kubebuilder:validation:Pattern:="^((100|[0-9]{1,2})%|[0-9]+)$" - // +kubebuilder:default:="10%" - Nodes string `json:"nodes" hash:"ignore"` - // Schedule specifies when a budget begins being active, following - // the upstream cronjob syntax. If omitted, the budget is always active. - // Timezones are not supported. - // This field is required if Duration is set. - // +kubebuilder:validation:Pattern:=`^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$` - // +optional - Schedule *string `json:"schedule,omitempty" hash:"ignore"` - // Duration determines how long a Budget is active since each Schedule hit. - // Only minutes and hours are accepted, as cron does not work in seconds. - // If omitted, the budget is always active. - // This is required if Schedule is set. - // This regex has an optional 0s at the end since the duration.String() always adds - // a 0s at the end. - // +kubebuilder:validation:Pattern=`^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$` - // +kubebuilder:validation:Type="string" - // +optional - Duration *metav1.Duration `json:"duration,omitempty" hash:"ignore"` -} - -type Disruption struct { - // ConsolidateAfter is the duration the controller will wait - // before attempting to terminate nodes that are underutilized. - // Refer to ConsolidationPolicy for how underutilization is considered. - // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` - // +kubebuilder:validation:Type="string" - // +kubebuilder:validation:Schemaless - // +required - ConsolidateAfter NillableDuration `json:"consolidateAfter"` - // ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation - // algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified - // +kubebuilder:default:="WhenEmptyOrUnderutilized" - // +kubebuilder:validation:Enum:={WhenEmpty,WhenEmptyOrUnderutilized} - // +optional - ConsolidationPolicy ConsolidationPolicy `json:"consolidationPolicy,omitempty"` - // Budgets is a list of Budgets. - // If there are multiple active budgets, Karpenter uses - // the most restrictive value. If left undefined, - // this will default to one budget with a value to 10%. - // +kubebuilder:validation:XValidation:message="'schedule' must be set with 'duration'",rule="self.all(x, has(x.schedule) == has(x.duration))" - // +kubebuilder:default:={{nodes: "10%"}} - // +kubebuilder:validation:MaxItems=50 - // +optional - Budgets []Budget `json:"budgets,omitempty" hash:"ignore"` -} - -// ConsolidateUnderSpec defines when to consolidate under -type ConsolidateUnderSpec struct { - // CPUUtilization specifies the CPU utilization threshold - // +optional - CPUUtilization *string `json:"cpuUtilization,omitempty"` - - // MemoryUtilization specifies the memory utilization threshold - // +optional - MemoryUtilization *string `json:"memoryUtilization,omitempty"` -} - -// LimitsSpec defines the limits for a NodePool -type LimitsSpec struct { - // CPU specifies the CPU limit - // +optional - CPU *resource.Quantity `json:"cpu,omitempty"` - - // Memory specifies the memory limit - // +optional - Memory *resource.Quantity `json:"memory,omitempty"` -} - -// RequirementSpec defines a requirement for a NodePool -type RequirementSpec struct { - // Key specifies the requirement key - Key string `json:"key"` - - // Operator specifies the requirement operator - Operator string `json:"operator"` - - // Values specifies the requirement values - // +optional - Values []string `json:"values,omitempty"` -} - -// TaintSpec defines a taint for a NodePool -type TaintSpec struct { - // Key specifies the taint key - Key string `json:"key"` - - // Value specifies the taint value - // +optional - Value *string `json:"value,omitempty"` - - // Effect specifies the taint effect - Effect string `json:"effect"` -} - -// EC2NodeClassSpec defines the configuration for a Karpenter EC2NodeClass -type EC2NodeClassSpec struct { - // Name is the ami name in EC2. - // This value is the name field, which is different from the name tag. - AMIName string `json:"amiName,omitempty"` - // Owner is the owner for the ami. - // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" - AMIOwner string `json:"amiOwner,omitempty"` - - // SecurityGroups specifies the security groups to use - // +optional - SecurityGroups map[string]string `json:"securityGroups,omitempty"` - - // Subnets specifies the subnets to use - // +optional - Subnets map[string]string `json:"subnets,omitempty"` -} - // KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. type KarpenterMachinePoolSpec struct { - // The name or the Amazon Resource Name (ARN) of the instance profile associated - // with the IAM role for the instance. The instance profile contains the IAM - // role. - IamInstanceProfile string `json:"iamInstanceProfile,omitempty"` - // NodePool specifies the configuration for the Karpenter NodePool // +optional NodePool *NodePoolSpec `json:"nodePool,omitempty"` diff --git a/api/v1alpha1/nodepool.go b/api/v1alpha1/nodepool.go new file mode 100644 index 00000000..6ad9f961 --- /dev/null +++ b/api/v1alpha1/nodepool.go @@ -0,0 +1,256 @@ +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodePoolSpec defines the configuration for a Karpenter NodePool +type NodePoolSpec struct { + // Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + // NodeClaims launched from this NodePool will often be further constrained than the template specifies. + // +required + Template NodeClaimTemplate `json:"template"` + + // Disruption contains the parameters that relate to Karpenter's disruption logic + // +kubebuilder:default:={consolidateAfter: "0s"} + // +optional + Disruption Disruption `json:"disruption"` + + // Limits define a set of bounds for provisioning capacity. + // +optional + Limits Limits `json:"limits,omitempty"` + + // Weight is the priority given to the nodepool during scheduling. A higher + // numerical weight indicates that this nodepool will be ordered + // ahead of other nodepools with lower weights. A nodepool with no weight + // will be treated as if it is a nodepool with a weight of 0. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=100 + // +optional + Weight *int32 `json:"weight,omitempty"` +} + +type NodeClaimTemplate struct { + ObjectMeta `json:"metadata,omitempty"` + // +required + Spec NodeClaimTemplateSpec `json:"spec"` +} + +type ObjectMeta struct { + // Map of string keys and values that can be used to organize and categorize + // (scope and select) objects. May match selectors of replication controllers + // and services. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool +// NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since +// users are not able to set resource requests in the NodePool. +type NodeClaimTemplateSpec struct { + // Taints will be applied to the NodeClaim's node. + // +optional + Taints []v1.Taint `json:"taints,omitempty"` + // StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + // within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + // daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + // purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + // +optional + StartupTaints []v1.Taint `json:"startupTaints,omitempty"` + // Requirements are layered with GetLabels and applied to every node. + // +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)" + // +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)" + // +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)" + // +kubebuilder:validation:MaxItems:=100 + // +required + Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"` + // TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + // + // Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + // + // This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + // When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + // + // Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + // If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + // that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + // + // The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + // If left undefined, the controller will wait indefinitely for pods to be drained. + // +kubebuilder:validation:Pattern=`^([0-9]+(s|m|h))+$` + // +kubebuilder:validation:Type="string" + // +optional + TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"` + // ExpireAfter is the duration the controller will wait + // before terminating a node, measured from when the node is created. This + // is useful to implement features like eventually consistent node upgrade, + // memory leak protection, and disruption testing. + // +kubebuilder:default:="720h" + // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` + // +kubebuilder:validation:Type="string" + // +kubebuilder:validation:Schemaless + // +optional + ExpireAfter NillableDuration `json:"expireAfter,omitempty"` +} + +// A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values +// and minValues that represent the requirement to have at least that many values. +type NodeSelectorRequirementWithMinValues struct { + v1.NodeSelectorRequirement `json:",inline"` + // This field is ALPHA and can be dropped or replaced at any time + // MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=50 + // +optional + MinValues *int `json:"minValues,omitempty"` +} + +type NodeClassReference struct { + // Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + // +kubebuilder:validation:XValidation:rule="self != ''",message="kind may not be empty" + // +required + Kind string `json:"kind"` + // Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names + // +kubebuilder:validation:XValidation:rule="self != ''",message="name may not be empty" + // +required + Name string `json:"name"` + // API version of the referent + // +kubebuilder:validation:XValidation:rule="self != ''",message="group may not be empty" + // +kubebuilder:validation:Pattern=`^[^/]*$` + // +required + Group string `json:"group"` +} + +type Limits v1.ResourceList + +type ConsolidationPolicy string + +const ( + ConsolidationPolicyWhenEmpty ConsolidationPolicy = "WhenEmpty" + ConsolidationPolicyWhenEmptyOrUnderutilized ConsolidationPolicy = "WhenEmptyOrUnderutilized" +) + +// DisruptionReason defines valid reasons for disruption budgets. +// +kubebuilder:validation:Enum={Underutilized,Empty,Drifted} +type DisruptionReason string + +// Budget defines when Karpenter will restrict the +// number of Node Claims that can be terminating simultaneously. +type Budget struct { + // Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + // Otherwise, this will apply to each reason defined. + // allowed reasons are Underutilized, Empty, and Drifted. + // +optional + Reasons []DisruptionReason `json:"reasons,omitempty"` + // Nodes dictates the maximum number of NodeClaims owned by this NodePool + // that can be terminating at once. This is calculated by counting nodes that + // have a deletion timestamp set, or are actively being deleted by Karpenter. + // This field is required when specifying a budget. + // This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + // checking for int nodes for IntOrString nodes. + // Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + // +kubebuilder:validation:Pattern:="^((100|[0-9]{1,2})%|[0-9]+)$" + // +kubebuilder:default:="10%" + Nodes string `json:"nodes" hash:"ignore"` + // Schedule specifies when a budget begins being active, following + // the upstream cronjob syntax. If omitted, the budget is always active. + // Timezones are not supported. + // This field is required if Duration is set. + // +kubebuilder:validation:Pattern:=`^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$` + // +optional + Schedule *string `json:"schedule,omitempty" hash:"ignore"` + // Duration determines how long a Budget is active since each Schedule hit. + // Only minutes and hours are accepted, as cron does not work in seconds. + // If omitted, the budget is always active. + // This is required if Schedule is set. + // This regex has an optional 0s at the end since the duration.String() always adds + // a 0s at the end. + // +kubebuilder:validation:Pattern=`^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$` + // +kubebuilder:validation:Type="string" + // +optional + Duration *metav1.Duration `json:"duration,omitempty" hash:"ignore"` +} + +type Disruption struct { + // ConsolidateAfter is the duration the controller will wait + // before attempting to terminate nodes that are underutilized. + // Refer to ConsolidationPolicy for how underutilization is considered. + // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` + // +kubebuilder:validation:Type="string" + // +kubebuilder:validation:Schemaless + // +required + ConsolidateAfter NillableDuration `json:"consolidateAfter"` + // ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + // algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + // +kubebuilder:default:="WhenEmptyOrUnderutilized" + // +kubebuilder:validation:Enum:={WhenEmpty,WhenEmptyOrUnderutilized} + // +optional + ConsolidationPolicy ConsolidationPolicy `json:"consolidationPolicy,omitempty"` + // Budgets is a list of Budgets. + // If there are multiple active budgets, Karpenter uses + // the most restrictive value. If left undefined, + // this will default to one budget with a value to 10%. + // +kubebuilder:validation:XValidation:message="'schedule' must be set with 'duration'",rule="self.all(x, has(x.schedule) == has(x.duration))" + // +kubebuilder:default:={{nodes: "10%"}} + // +kubebuilder:validation:MaxItems=50 + // +optional + Budgets []Budget `json:"budgets,omitempty" hash:"ignore"` +} + +// ConsolidateUnderSpec defines when to consolidate under +type ConsolidateUnderSpec struct { + // CPUUtilization specifies the CPU utilization threshold + // +optional + CPUUtilization *string `json:"cpuUtilization,omitempty"` + + // MemoryUtilization specifies the memory utilization threshold + // +optional + MemoryUtilization *string `json:"memoryUtilization,omitempty"` +} + +// LimitsSpec defines the limits for a NodePool +type LimitsSpec struct { + // CPU specifies the CPU limit + // +optional + CPU *resource.Quantity `json:"cpu,omitempty"` + + // Memory specifies the memory limit + // +optional + Memory *resource.Quantity `json:"memory,omitempty"` +} + +// RequirementSpec defines a requirement for a NodePool +type RequirementSpec struct { + // Key specifies the requirement key + Key string `json:"key"` + + // Operator specifies the requirement operator + Operator string `json:"operator"` + + // Values specifies the requirement values + // +optional + Values []string `json:"values,omitempty"` +} + +// TaintSpec defines a taint for a NodePool +type TaintSpec struct { + // Key specifies the taint key + Key string `json:"key"` + + // Value specifies the taint value + // +optional + Value *string `json:"value,omitempty"` + + // Effect specifies the taint effect + Effect string `json:"effect"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6d1436fb..8bb0928b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -23,12 +23,119 @@ package v1alpha1 import ( timex "time" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/cluster-api/api/v1beta1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMISelectorTerm) DeepCopyInto(out *AMISelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMISelectorTerm. +func (in *AMISelectorTerm) DeepCopy() *AMISelectorTerm { + if in == nil { + return nil + } + out := new(AMISelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDevice) DeepCopyInto(out *BlockDevice) { + *out = *in + if in.DeleteOnTermination != nil { + in, out := &in.DeleteOnTermination, &out.DeleteOnTermination + *out = new(bool) + **out = **in + } + if in.Encrypted != nil { + in, out := &in.Encrypted, &out.Encrypted + *out = new(bool) + **out = **in + } + if in.IOPS != nil { + in, out := &in.IOPS, &out.IOPS + *out = new(int64) + **out = **in + } + if in.KMSKeyID != nil { + in, out := &in.KMSKeyID, &out.KMSKeyID + *out = new(string) + **out = **in + } + if in.SnapshotID != nil { + in, out := &in.SnapshotID, &out.SnapshotID + *out = new(string) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } + if in.VolumeInitializationRate != nil { + in, out := &in.VolumeInitializationRate, &out.VolumeInitializationRate + *out = new(int32) + **out = **in + } + if in.VolumeSize != nil { + in, out := &in.VolumeSize, &out.VolumeSize + x := (*in).DeepCopy() + *out = &x + } + if in.VolumeType != nil { + in, out := &in.VolumeType, &out.VolumeType + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDevice. +func (in *BlockDevice) DeepCopy() *BlockDevice { + if in == nil { + return nil + } + out := new(BlockDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDeviceMapping) DeepCopyInto(out *BlockDeviceMapping) { + *out = *in + if in.DeviceName != nil { + in, out := &in.DeviceName, &out.DeviceName + *out = new(string) + **out = **in + } + if in.EBS != nil { + in, out := &in.EBS, &out.EBS + *out = new(BlockDevice) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDeviceMapping. +func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { + if in == nil { + return nil + } + out := new(BlockDeviceMapping) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Budget) DeepCopyInto(out *Budget) { *out = *in @@ -44,7 +151,7 @@ func (in *Budget) DeepCopyInto(out *Budget) { } if in.Duration != nil { in, out := &in.Duration, &out.Duration - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } } @@ -59,6 +166,28 @@ func (in *Budget) DeepCopy() *Budget { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservationSelectorTerm) DeepCopyInto(out *CapacityReservationSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservationSelectorTerm. +func (in *CapacityReservationSelectorTerm) DeepCopy() *CapacityReservationSelectorTerm { + if in == nil { + return nil + } + out := new(CapacityReservationSelectorTerm) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConsolidateUnderSpec) DeepCopyInto(out *ConsolidateUnderSpec) { *out = *in @@ -110,20 +239,97 @@ func (in *Disruption) DeepCopy() *Disruption { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { *out = *in - if in.SecurityGroups != nil { - in, out := &in.SecurityGroups, &out.SecurityGroups - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val + if in.SubnetSelectorTerms != nil { + in, out := &in.SubnetSelectorTerms, &out.SubnetSelectorTerms + *out = make([]SubnetSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecurityGroupSelectorTerms != nil { + in, out := &in.SecurityGroupSelectorTerms, &out.SecurityGroupSelectorTerms + *out = make([]SecurityGroupSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.CapacityReservationSelectorTerms != nil { + in, out := &in.CapacityReservationSelectorTerms, &out.CapacityReservationSelectorTerms + *out = make([]CapacityReservationSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.Subnets != nil { - in, out := &in.Subnets, &out.Subnets + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } + if in.AMISelectorTerms != nil { + in, out := &in.AMISelectorTerms, &out.AMISelectorTerms + *out = make([]AMISelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } + if in.UserData != nil { + in, out := &in.UserData, &out.UserData + *out = new(string) + **out = **in + } + if in.InstanceProfile != nil { + in, out := &in.InstanceProfile, &out.InstanceProfile + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags *out = make(map[string]string, len(*in)) for key, val := range *in { (*out)[key] = val } } + if in.Kubelet != nil { + in, out := &in.Kubelet, &out.Kubelet + *out = new(KubeletConfiguration) + (*in).DeepCopyInto(*out) + } + if in.BlockDeviceMappings != nil { + in, out := &in.BlockDeviceMappings, &out.BlockDeviceMappings + *out = make([]*BlockDeviceMapping, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockDeviceMapping) + (*in).DeepCopyInto(*out) + } + } + } + if in.InstanceStorePolicy != nil { + in, out := &in.InstanceStorePolicy, &out.InstanceStorePolicy + *out = new(InstanceStorePolicy) + **out = **in + } + if in.DetailedMonitoring != nil { + in, out := &in.DetailedMonitoring, &out.DetailedMonitoring + *out = new(bool) + **out = **in + } + if in.MetadataOptions != nil { + in, out := &in.MetadataOptions, &out.MetadataOptions + *out = new(MetadataOptions) + (*in).DeepCopyInto(*out) + } + if in.Context != nil { + in, out := &in.Context, &out.Context + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassSpec. @@ -247,6 +453,91 @@ func (in *KarpenterMachinePoolStatus) DeepCopy() *KarpenterMachinePoolStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { + *out = *in + if in.ClusterDNS != nil { + in, out := &in.ClusterDNS, &out.ClusterDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } + if in.PodsPerCore != nil { + in, out := &in.PodsPerCore, &out.PodsPerCore + *out = new(int32) + **out = **in + } + if in.SystemReserved != nil { + in, out := &in.SystemReserved, &out.SystemReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.KubeReserved != nil { + in, out := &in.KubeReserved, &out.KubeReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionHard != nil { + in, out := &in.EvictionHard, &out.EvictionHard + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoft != nil { + in, out := &in.EvictionSoft, &out.EvictionSoft + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoftGracePeriod != nil { + in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod + *out = make(map[string]v1.Duration, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionMaxPodGracePeriod != nil { + in, out := &in.EvictionMaxPodGracePeriod, &out.EvictionMaxPodGracePeriod + *out = new(int32) + **out = **in + } + if in.ImageGCHighThresholdPercent != nil { + in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent + *out = new(int32) + **out = **in + } + if in.ImageGCLowThresholdPercent != nil { + in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent + *out = new(int32) + **out = **in + } + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. +func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { + if in == nil { + return nil + } + out := new(KubeletConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Limits) DeepCopyInto(out *Limits) { { @@ -293,6 +584,41 @@ func (in *LimitsSpec) DeepCopy() *LimitsSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetadataOptions) DeepCopyInto(out *MetadataOptions) { + *out = *in + if in.HTTPEndpoint != nil { + in, out := &in.HTTPEndpoint, &out.HTTPEndpoint + *out = new(string) + **out = **in + } + if in.HTTPProtocolIPv6 != nil { + in, out := &in.HTTPProtocolIPv6, &out.HTTPProtocolIPv6 + *out = new(string) + **out = **in + } + if in.HTTPPutResponseHopLimit != nil { + in, out := &in.HTTPPutResponseHopLimit, &out.HTTPPutResponseHopLimit + *out = new(int64) + **out = **in + } + if in.HTTPTokens != nil { + in, out := &in.HTTPTokens, &out.HTTPTokens + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetadataOptions. +func (in *MetadataOptions) DeepCopy() *MetadataOptions { + if in == nil { + return nil + } + out := new(MetadataOptions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NillableDuration) DeepCopyInto(out *NillableDuration) { *out = *in @@ -340,14 +666,14 @@ func (in *NodeClaimTemplateSpec) DeepCopyInto(out *NodeClaimTemplateSpec) { *out = *in if in.Taints != nil { in, out := &in.Taints, &out.Taints - *out = make([]v1.Taint, len(*in)) + *out = make([]corev1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } } if in.StartupTaints != nil { in, out := &in.StartupTaints, &out.StartupTaints - *out = make([]v1.Taint, len(*in)) + *out = make([]corev1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -361,7 +687,7 @@ func (in *NodeClaimTemplateSpec) DeepCopyInto(out *NodeClaimTemplateSpec) { } if in.TerminationGracePeriod != nil { in, out := &in.TerminationGracePeriod, &out.TerminationGracePeriod - *out = new(metav1.Duration) + *out = new(v1.Duration) **out = **in } in.ExpireAfter.DeepCopyInto(&out.ExpireAfter) @@ -491,6 +817,50 @@ func (in *RequirementSpec) DeepCopy() *RequirementSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroupSelectorTerm) DeepCopyInto(out *SecurityGroupSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroupSelectorTerm. +func (in *SecurityGroupSelectorTerm) DeepCopy() *SecurityGroupSelectorTerm { + if in == nil { + return nil + } + out := new(SecurityGroupSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSelectorTerm) DeepCopyInto(out *SubnetSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSelectorTerm. +func (in *SubnetSelectorTerm) DeepCopy() *SubnetSelectorTerm { + if in == nil { + return nil + } + out := new(SubnetSelectorTerm) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TaintSpec) DeepCopyInto(out *TaintSpec) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 618768a2..9b750f38 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -54,33 +54,607 @@ spec: description: EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass properties: - amiName: + amiFamily: description: |- - Name is the ami name in EC2. - This value is the name field, which is different from the name tag. + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 type: string - amiOwner: + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match + the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the + following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', + ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] + ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM + parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', + 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || + has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned + nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically + set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the + EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, + or alias ARN) of the customer managed KMS key to use + for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate + is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) + && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 + capacityReservationSelectorTerms: description: |- - Owner is the owner for the ami. - You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along + with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string - securityGroups: - additionalProperties: - type: string - description: SecurityGroups specifies the security groups to use + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring + is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal + names to quantities that define grace periods for each eviction + signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + description: KubeReserved contains resources reserved for + Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + description: SystemReserved contains resources reserved for + OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in + self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string type: object - subnets: + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security + group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with + a combination of other fields in a security group selector + term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector + terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: additionalProperties: type: string - description: Subnets specifies the subnets to use + description: Tags to be applied on ec2 resources like instances + and launch templates. type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms type: object - iamInstanceProfile: - description: |- - The name or the Amazon Resource Name (ARN) of the instance profile associated - with the IAM role for the instance. The instance profile contains the IAM - role. - type: string nodePool: description: NodePool specifies the configuration for the Karpenter NodePool diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 98533e4e..b23720e3 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -124,9 +124,6 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, fmt.Errorf("failed to get AWSClusterRoleIdentity referenced in AWSCluster: %w", err) } - // Create a deep copy of the reconciled object so we can change it - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - if !karpenterMachinePool.GetDeletionTimestamp().IsZero() { return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity) } @@ -141,6 +138,8 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, errors.New("error retrieving bootstrap data: secret value key is missing") } + // Create a deep copy of the reconciled object so we can change it + karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() updated := controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) if updated { if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { @@ -392,79 +391,22 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. // Generate user data for Ignition userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, karpenterMachinePool.Name) - // Add security groups tag selector if specified - securityGroupTagsSelector := map[string]string{} - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups) > 0 { - for securityGroupTagKey, securityGroupTagValue := range karpenterMachinePool.Spec.EC2NodeClass.SecurityGroups { - securityGroupTagsSelector[securityGroupTagKey] = securityGroupTagValue - } - } - - // Add subnet tag selector if specified - subnetTagsSelector := map[string]string{} - if karpenterMachinePool.Spec.EC2NodeClass != nil && len(karpenterMachinePool.Spec.EC2NodeClass.Subnets) > 0 { - for subnetTagKey, subnetTagValue := range karpenterMachinePool.Spec.EC2NodeClass.Subnets { - subnetTagsSelector[subnetTagKey] = subnetTagValue - } - } - operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { // Build the EC2NodeClass spec spec := map[string]interface{}{ - "amiFamily": "Custom", - "amiSelectorTerms": []map[string]interface{}{ - { - "name": karpenterMachinePool.Spec.EC2NodeClass.AMIName, - "owner": karpenterMachinePool.Spec.EC2NodeClass.AMIOwner, - }, - }, - "blockDeviceMappings": []map[string]interface{}{ - { - "deviceName": "/dev/xvda", - "rootVolume": true, - "ebs": map[string]interface{}{ - "volumeSize": "8Gi", - "volumeType": "gp3", - "deleteOnTermination": true, - }, - }, - { - "deviceName": "/dev/xvdd", - "ebs": map[string]interface{}{ - "encrypted": true, - "volumeSize": "120Gi", - "volumeType": "gp3", - "deleteOnTermination": true, - }, - }, - { - "deviceName": "/dev/xvde", - "ebs": map[string]interface{}{ - "encrypted": true, - "volumeSize": "30Gi", - "volumeType": "gp3", - "deleteOnTermination": true, - }, - }, - }, - "instanceProfile": karpenterMachinePool.Spec.IamInstanceProfile, + "amiFamily": "Custom", + "amiSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.AMISelectorTerms, + "blockDeviceMappings": karpenterMachinePool.Spec.EC2NodeClass.BlockDeviceMappings, + "instanceProfile": karpenterMachinePool.Spec.EC2NodeClass.InstanceProfile, "metadataOptions": map[string]interface{}{ "httpEndpoint": "enabled", "httpProtocolIPv6": "disabled", "httpPutResponseHopLimit": 1, "httpTokens": "required", }, - "securityGroupSelectorTerms": []map[string]interface{}{ - { - "tags": securityGroupTagsSelector, - }, - }, - "subnetSelectorTerms": []map[string]interface{}{ - { - "tags": subnetTagsSelector, - }, - }, - "userData": userData, + "securityGroupSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroupSelectorTerms, + "subnetSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SubnetSelectorTerms, + "userData": userData, } ec2NodeClass.Object["spec"] = spec diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index e37c20a7..d8bf7107 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -31,6 +31,18 @@ import ( "github.com/aws-resolver-rules-operator/pkg/resolver/resolverfakes" ) +const ( + AMIName = "flatcar-stable-4152.2.3-kube-1.29.1-tooling-1.26.0-gs" + AMIOwner = "1234567890" + AWSRegion = "eu-west-1" + ClusterName = "foo" + AWSClusterBucketName = "my-awesome-bucket" + DataSecretName = "foo-mp-12345" + KarpenterMachinePoolName = "foo" + KarpenterNodesInstanceProfile = "karpenter-iam-role" + KubernetesVersion = "v1.29.1" +) + var _ = Describe("KarpenterMachinePool reconciler", func() { var ( capiBootstrapSecretContent []byte @@ -39,24 +51,12 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { s3Client *resolverfakes.FakeS3Client ec2Client *resolverfakes.FakeEC2Client ctx context.Context + instanceProfile = KarpenterNodesInstanceProfile reconciler *controllers.KarpenterMachinePoolReconciler reconcileErr error reconcileResult reconcile.Result ) - const ( - AMIName = "flatcar-stable-4152.2.3-kube-1.29.1-tooling-1.26.0-gs" - AMIOwner = "1234567890" - AWSRegion = "eu-west-1" - ClusterName = "foo" - AWSClusterBucketName = "my-awesome-bucket" - DataSecretName = "foo-mp-12345" - KarpenterMachinePoolName = "foo" - KarpenterNodesInstanceProfile = "karpenter-iam-role" - KubernetesVersion = "v1.29.1" - NewerKubernetesVersion = "v1.29.1" - ) - BeforeEach(func() { ctx = context.Background() @@ -363,7 +363,24 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, Spec: karpenterinfra.KarpenterMachinePoolSpec{ - EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{}, + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Name: AMIName, + Owner: AMIOwner, + }, + }, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"my-target-sg": "is-this"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"my-target-subnet": "is-that"}, + }, + }, + }, NodePool: &karpenterinfra.NodePoolSpec{ Template: karpenterinfra.NodeClaimTemplate{ Spec: karpenterinfra.NodeClaimTemplateSpec{ @@ -525,6 +542,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { terminationGracePeriod := metav1.Duration{Duration: 30 * time.Second} weight := int32(1) + deviceName := "/dev/xvda" + volumeSize := resource.MustParse("8Gi") + volumeTypeGp3 := "gp3" + deleteOnTerminationTrue := true karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ ObjectMeta: ctrl.ObjectMeta{ Namespace: namespace, @@ -543,12 +564,35 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: karpenterinfra.KarpenterMachinePoolSpec{ EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ - AMIName: AMIName, - AMIOwner: AMIOwner, - SecurityGroups: map[string]string{"my-target-sg": "is-this"}, - Subnets: map[string]string{"my-target-subnet": "is-that"}, + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Name: AMIName, + Owner: AMIOwner, + }, + }, + BlockDeviceMappings: []*karpenterinfra.BlockDeviceMapping{ + { + DeviceName: &deviceName, + EBS: &karpenterinfra.BlockDevice{ + DeleteOnTermination: &deleteOnTerminationTrue, + VolumeSize: &volumeSize, + VolumeType: &volumeTypeGp3, + }, + RootVolume: true, + }, + }, + InstanceProfile: &instanceProfile, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"my-target-sg": "is-this"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"my-target-subnet": "is-that"}, + }, + }, }, - IamInstanceProfile: KarpenterNodesInstanceProfile, NodePool: &karpenterinfra.NodePoolSpec{ Template: karpenterinfra.NodeClaimTemplate{ Spec: karpenterinfra.NodeClaimTemplateSpec{ @@ -829,7 +873,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ec2nodeclassList := &unstructured.UnstructuredList{} ec2nodeclassList.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "karpenter.k8s.aws", + Group: controllers.EC2NodeClassAPIGroup, Kind: "EC2NodeClassList", Version: "v1", }) @@ -842,6 +886,21 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "userData").To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, KarpenterMachinePoolName))) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "instanceProfile").To(Equal(KarpenterNodesInstanceProfile)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "blockDeviceMappings").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "blockDeviceMappings").To( + ContainElement( // slice matcher: at least one element matches + gstruct.MatchAllKeys(gstruct.Keys{ // map matcher: all these keys must match exactly + "deviceName": Equal("/dev/xvda"), + "rootVolume": BeTrue(), + "ebs": gstruct.MatchAllKeys(gstruct.Keys{ + "deleteOnTermination": BeTrue(), + "volumeSize": Equal("8Gi"), + "volumeType": Equal("gp3"), + }), + }), + ), + ) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To(HaveLen(1)) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To( ContainElement( // slice matcher: at least one element matches @@ -1072,10 +1131,24 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: karpenterinfra.KarpenterMachinePoolSpec{ EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ - SecurityGroups: map[string]string{"my-target-sg": "is-this"}, - Subnets: map[string]string{"my-target-subnet": "is-that"}, + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Name: AMIName, + Owner: AMIOwner, + }, + }, + InstanceProfile: &instanceProfile, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"my-target-sg": "is-this"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"my-target-subnet": "is-that"}, + }, + }, }, - IamInstanceProfile: KarpenterNodesInstanceProfile, NodePool: &karpenterinfra.NodePoolSpec{ Template: karpenterinfra.NodeClaimTemplate{ Spec: karpenterinfra.NodeClaimTemplateSpec{ diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 618768a2..9b750f38 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -54,33 +54,607 @@ spec: description: EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass properties: - amiName: + amiFamily: description: |- - Name is the ami name in EC2. - This value is the name field, which is different from the name tag. + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 type: string - amiOwner: + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match + the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the + following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', + ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] + ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM + parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', + 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || + has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned + nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically + set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the + EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, + or alias ARN) of the customer managed KMS key to use + for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate + is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) + && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 + capacityReservationSelectorTerms: description: |- - Owner is the owner for the ami. - You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along + with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string - securityGroups: - additionalProperties: - type: string - description: SecurityGroups specifies the security groups to use + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring + is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal + names to quantities that define grace periods for each eviction + signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + description: KubeReserved contains resources reserved for + Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + description: SystemReserved contains resources reserved for + OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in + self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string type: object - subnets: + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security + group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with + a combination of other fields in a security group selector + term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector + terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: additionalProperties: type: string - description: Subnets specifies the subnets to use + description: Tags to be applied on ec2 resources like instances + and launch templates. type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms type: object - iamInstanceProfile: - description: |- - The name or the Amazon Resource Name (ARN) of the instance profile associated - with the IAM role for the instance. The instance profile contains the IAM - role. - type: string nodePool: description: NodePool specifies the configuration for the Karpenter NodePool From cc1af9980269062854efb595887560ae7ec7717c Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Mon, 21 Jul 2025 09:38:44 +0200 Subject: [PATCH 16/41] Add CAPA additional tags to karpenter resources --- .../fake_awscluster_client.go | 20 ---------------- .../controllersfakes/fake_cluster_client.go | 24 ------------------- .../karpentermachinepool_controller.go | 15 ++++++++++-- .../karpentermachinepool_controller_test.go | 18 ++++++++------ pkg/resolver/resolverfakes/fake_ec2client.go | 6 ----- .../resolverfakes/fake_prefix_list_client.go | 8 ------- pkg/resolver/resolverfakes/fake_ramclient.go | 4 ---- .../resolverfakes/fake_resolver_client.go | 14 ----------- .../resolverfakes/fake_route53client.go | 16 ------------- .../resolverfakes/fake_route_table_client.go | 4 ---- pkg/resolver/resolverfakes/fake_s3client.go | 2 -- .../fake_transit_gateway_client.go | 8 ------- 12 files changed, 24 insertions(+), 115 deletions(-) diff --git a/controllers/controllersfakes/fake_awscluster_client.go b/controllers/controllersfakes/fake_awscluster_client.go index b670ad9d..76442c27 100644 --- a/controllers/controllersfakes/fake_awscluster_client.go +++ b/controllers/controllersfakes/fake_awscluster_client.go @@ -795,26 +795,6 @@ func (fake *FakeAWSClusterClient) UpdateStatusReturnsOnCall(i int, result1 error func (fake *FakeAWSClusterClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addFinalizerMutex.RLock() - defer fake.addFinalizerMutex.RUnlock() - fake.getAWSClusterMutex.RLock() - defer fake.getAWSClusterMutex.RUnlock() - fake.getClusterMutex.RLock() - defer fake.getClusterMutex.RUnlock() - fake.getIdentityMutex.RLock() - defer fake.getIdentityMutex.RUnlock() - fake.getOwnerMutex.RLock() - defer fake.getOwnerMutex.RUnlock() - fake.markConditionTrueMutex.RLock() - defer fake.markConditionTrueMutex.RUnlock() - fake.patchClusterMutex.RLock() - defer fake.patchClusterMutex.RUnlock() - fake.removeFinalizerMutex.RLock() - defer fake.removeFinalizerMutex.RUnlock() - fake.unpauseMutex.RLock() - defer fake.unpauseMutex.RUnlock() - fake.updateStatusMutex.RLock() - defer fake.updateStatusMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/controllers/controllersfakes/fake_cluster_client.go b/controllers/controllersfakes/fake_cluster_client.go index 5db00477..03fb5c8d 100644 --- a/controllers/controllersfakes/fake_cluster_client.go +++ b/controllers/controllersfakes/fake_cluster_client.go @@ -944,30 +944,6 @@ func (fake *FakeClusterClient) UnpauseReturnsOnCall(i int, result1 error) { func (fake *FakeClusterClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addAWSClusterFinalizerMutex.RLock() - defer fake.addAWSClusterFinalizerMutex.RUnlock() - fake.addAWSManagedControlPlaneFinalizerMutex.RLock() - defer fake.addAWSManagedControlPlaneFinalizerMutex.RUnlock() - fake.addClusterFinalizerMutex.RLock() - defer fake.addClusterFinalizerMutex.RUnlock() - fake.getAWSClusterMutex.RLock() - defer fake.getAWSClusterMutex.RUnlock() - fake.getAWSManagedControlPlaneMutex.RLock() - defer fake.getAWSManagedControlPlaneMutex.RUnlock() - fake.getClusterMutex.RLock() - defer fake.getClusterMutex.RUnlock() - fake.getIdentityMutex.RLock() - defer fake.getIdentityMutex.RUnlock() - fake.markConditionTrueMutex.RLock() - defer fake.markConditionTrueMutex.RUnlock() - fake.removeAWSClusterFinalizerMutex.RLock() - defer fake.removeAWSClusterFinalizerMutex.RUnlock() - fake.removeAWSManagedControlPlaneFinalizerMutex.RLock() - defer fake.removeAWSManagedControlPlaneFinalizerMutex.RUnlock() - fake.removeClusterFinalizerMutex.RLock() - defer fake.removeClusterFinalizerMutex.RUnlock() - fake.unpauseMutex.RLock() - defer fake.unpauseMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index b23720e3..ffb9f7e3 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -356,7 +356,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } // Create or update EC2NodeClass - if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, cluster, awsCluster, karpenterMachinePool, bootstrapSecretValue); err != nil { + if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, awsCluster, karpenterMachinePool, bootstrapSecretValue); err != nil { conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, EC2NodeClassCreationFailedReason, err.Error()) return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) } @@ -375,7 +375,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } // createOrUpdateEC2NodeClass creates or updates the EC2NodeClass resource in the workload cluster -func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { +func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { ec2NodeClassGVR := schema.GroupVersionResource{ Group: EC2NodeClassAPIGroup, Version: "v1", @@ -407,6 +407,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. "securityGroupSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroupSelectorTerms, "subnetSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SubnetSelectorTerms, "userData": userData, + "tags": mergeMaps(awsCluster.Spec.AdditionalTags, karpenterMachinePool.Spec.EC2NodeClass.Tags), } ec2NodeClass.Object["spec"] = spec @@ -427,6 +428,16 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. return nil } +func mergeMaps[A comparable, B any](maps ...map[A]B) map[A]B { + result := make(map[A]B) + for _, m := range maps { + for k, v := range m { + result[k] = v + } + } + return result +} + // createOrUpdateNodePool creates or updates the NodePool resource in the workload cluster func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { nodePoolGVR := schema.GroupVersionResource{ diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index d8bf7107..23b0b836 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -75,11 +75,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err = karpenterinfra.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - // fakeCtrlClient := fake.NewClientBuilder(). - // WithScheme(scheme.Scheme). - // // WithStatusSubresource(&karpenterinfra.KarpenterMachinePool{}). - // Build() - workloadClusterClientGetter := func(ctx context.Context, _ string, _ client.Client, _ client.ObjectKey) (client.Client, error) { // Return the same client that we're using for the test return k8sClient, nil @@ -592,6 +587,9 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Tags: map[string]string{"my-target-subnet": "is-that"}, }, }, + Tags: map[string]string{ + "one-tag": "only-for-karpenter", + }, }, NodePool: &karpenterinfra.NodePoolSpec{ Template: karpenterinfra.NodeClaimTemplate{ @@ -807,6 +805,9 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Name: ClusterName, }, Spec: capa.AWSClusterSpec{ + AdditionalTags: map[string]string{ + "additional-tag-for-all-resources": "custom-tag", + }, IdentityRef: &capa.AWSIdentityReference{ Name: "default", Kind: capa.ClusterRoleIdentityKind, @@ -885,6 +886,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "userData").To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, KarpenterMachinePoolName))) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "instanceProfile").To(Equal(KarpenterNodesInstanceProfile)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "tags"). + To(HaveKeyWithValue("additional-tag-for-all-resources", "custom-tag")) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "tags"). + To(HaveKeyWithValue("one-tag", "only-for-karpenter")) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "blockDeviceMappings").To(HaveLen(1)) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "blockDeviceMappings").To( @@ -1283,8 +1288,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) }) -// ExpectUnstructured digs into u.Object at the given path, -// asserts that it was found and error‐free, and returns +// ExpectUnstructured digs into u.Object at the given path, asserts that it was found and error‐free, and returns // a GomegaAssertion on the raw interface{} value. func ExpectUnstructured(u unstructured.Unstructured, fields ...string) Assertion { v, found, err := unstructured.NestedFieldNoCopy(u.Object, fields...) diff --git a/pkg/resolver/resolverfakes/fake_ec2client.go b/pkg/resolver/resolverfakes/fake_ec2client.go index 290f44af..50fba80a 100644 --- a/pkg/resolver/resolverfakes/fake_ec2client.go +++ b/pkg/resolver/resolverfakes/fake_ec2client.go @@ -261,12 +261,6 @@ func (fake *FakeEC2Client) TerminateInstancesByTagReturnsOnCall(i int, result1 [ func (fake *FakeEC2Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.createSecurityGroupForResolverEndpointsMutex.RLock() - defer fake.createSecurityGroupForResolverEndpointsMutex.RUnlock() - fake.deleteSecurityGroupForResolverEndpointsMutex.RLock() - defer fake.deleteSecurityGroupForResolverEndpointsMutex.RUnlock() - fake.terminateInstancesByTagMutex.RLock() - defer fake.terminateInstancesByTagMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_prefix_list_client.go b/pkg/resolver/resolverfakes/fake_prefix_list_client.go index f85f8adc..11b350f4 100644 --- a/pkg/resolver/resolverfakes/fake_prefix_list_client.go +++ b/pkg/resolver/resolverfakes/fake_prefix_list_client.go @@ -319,14 +319,6 @@ func (fake *FakePrefixListClient) DeleteEntryReturnsOnCall(i int, result1 error) func (fake *FakePrefixListClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.applyMutex.RLock() - defer fake.applyMutex.RUnlock() - fake.applyEntryMutex.RLock() - defer fake.applyEntryMutex.RUnlock() - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - fake.deleteEntryMutex.RLock() - defer fake.deleteEntryMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_ramclient.go b/pkg/resolver/resolverfakes/fake_ramclient.go index 3f30b598..a1a2cc86 100644 --- a/pkg/resolver/resolverfakes/fake_ramclient.go +++ b/pkg/resolver/resolverfakes/fake_ramclient.go @@ -164,10 +164,6 @@ func (fake *FakeRAMClient) DeleteResourceShareReturnsOnCall(i int, result1 error func (fake *FakeRAMClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.applyResourceShareMutex.RLock() - defer fake.applyResourceShareMutex.RUnlock() - fake.deleteResourceShareMutex.RLock() - defer fake.deleteResourceShareMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_resolver_client.go b/pkg/resolver/resolverfakes/fake_resolver_client.go index 5fcff0cd..2a5c6938 100644 --- a/pkg/resolver/resolverfakes/fake_resolver_client.go +++ b/pkg/resolver/resolverfakes/fake_resolver_client.go @@ -583,20 +583,6 @@ func (fake *FakeResolverClient) GetResolverRuleByNameReturnsOnCall(i int, result func (fake *FakeResolverClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.associateResolverRuleWithContextMutex.RLock() - defer fake.associateResolverRuleWithContextMutex.RUnlock() - fake.createResolverRuleMutex.RLock() - defer fake.createResolverRuleMutex.RUnlock() - fake.deleteResolverRuleMutex.RLock() - defer fake.deleteResolverRuleMutex.RUnlock() - fake.disassociateResolverRuleWithContextMutex.RLock() - defer fake.disassociateResolverRuleWithContextMutex.RUnlock() - fake.findResolverRuleIdsAssociatedWithVPCIdMutex.RLock() - defer fake.findResolverRuleIdsAssociatedWithVPCIdMutex.RUnlock() - fake.findResolverRulesByAWSAccountIdMutex.RLock() - defer fake.findResolverRulesByAWSAccountIdMutex.RUnlock() - fake.getResolverRuleByNameMutex.RLock() - defer fake.getResolverRuleByNameMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_route53client.go b/pkg/resolver/resolverfakes/fake_route53client.go index 8f2dbd3a..8cc7ab47 100644 --- a/pkg/resolver/resolverfakes/fake_route53client.go +++ b/pkg/resolver/resolverfakes/fake_route53client.go @@ -653,22 +653,6 @@ func (fake *FakeRoute53Client) GetHostedZoneNSRecordReturnsOnCall(i int, result1 func (fake *FakeRoute53Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addDelegationToParentZoneMutex.RLock() - defer fake.addDelegationToParentZoneMutex.RUnlock() - fake.addDnsRecordsToHostedZoneMutex.RLock() - defer fake.addDnsRecordsToHostedZoneMutex.RUnlock() - fake.createHostedZoneMutex.RLock() - defer fake.createHostedZoneMutex.RUnlock() - fake.deleteDelegationFromParentZoneMutex.RLock() - defer fake.deleteDelegationFromParentZoneMutex.RUnlock() - fake.deleteDnsRecordsFromHostedZoneMutex.RLock() - defer fake.deleteDnsRecordsFromHostedZoneMutex.RUnlock() - fake.deleteHostedZoneMutex.RLock() - defer fake.deleteHostedZoneMutex.RUnlock() - fake.getHostedZoneIdByNameMutex.RLock() - defer fake.getHostedZoneIdByNameMutex.RUnlock() - fake.getHostedZoneNSRecordMutex.RLock() - defer fake.getHostedZoneNSRecordMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_route_table_client.go b/pkg/resolver/resolverfakes/fake_route_table_client.go index 84cef529..c63e7ef3 100644 --- a/pkg/resolver/resolverfakes/fake_route_table_client.go +++ b/pkg/resolver/resolverfakes/fake_route_table_client.go @@ -168,10 +168,6 @@ func (fake *FakeRouteTableClient) RemoveRoutesReturnsOnCall(i int, result1 error func (fake *FakeRouteTableClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addRoutesMutex.RLock() - defer fake.addRoutesMutex.RUnlock() - fake.removeRoutesMutex.RLock() - defer fake.removeRoutesMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_s3client.go b/pkg/resolver/resolverfakes/fake_s3client.go index 2bccf200..dbbe3ae8 100644 --- a/pkg/resolver/resolverfakes/fake_s3client.go +++ b/pkg/resolver/resolverfakes/fake_s3client.go @@ -99,8 +99,6 @@ func (fake *FakeS3Client) PutReturnsOnCall(i int, result1 error) { func (fake *FakeS3Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.putMutex.RLock() - defer fake.putMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_transit_gateway_client.go b/pkg/resolver/resolverfakes/fake_transit_gateway_client.go index ac9e828a..696dbaa7 100644 --- a/pkg/resolver/resolverfakes/fake_transit_gateway_client.go +++ b/pkg/resolver/resolverfakes/fake_transit_gateway_client.go @@ -319,14 +319,6 @@ func (fake *FakeTransitGatewayClient) DetachReturnsOnCall(i int, result1 error) func (fake *FakeTransitGatewayClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.applyMutex.RLock() - defer fake.applyMutex.RUnlock() - fake.applyAttachmentMutex.RLock() - defer fake.applyAttachmentMutex.RUnlock() - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - fake.detachMutex.RLock() - defer fake.detachMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value From 22a8cab5b404cdf65314f3ac540bd0e9fe89293f Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 22 Jul 2025 23:07:01 +0200 Subject: [PATCH 17/41] Refactor --- .../karpentermachinepool_controller.go | 166 ++++++++++++------ 1 file changed, 115 insertions(+), 51 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index ffb9f7e3..c9658c2e 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -37,10 +37,23 @@ import ( ) const ( + // BootstrapDataHashAnnotation stores the SHA256 hash of the bootstrap data + // to detect when userdata changes and needs to be re-uploaded to S3 BootstrapDataHashAnnotation = "giantswarm.io/userdata-hash" - EC2NodeClassAPIGroup = "karpenter.k8s.aws" - KarpenterFinalizer = "capa-operator.finalizers.giantswarm.io/karpenter-controller" - S3ObjectPrefix = "karpenter-machine-pool" + + // EC2NodeClassAPIGroup is the API group for Karpenter EC2NodeClass resources + EC2NodeClassAPIGroup = "karpenter.k8s.aws" + + // KarpenterFinalizer ensures proper cleanup of Karpenter resources and EC2 instances + // before allowing the KarpenterMachinePool to be deleted + KarpenterFinalizer = "capa-operator.finalizers.giantswarm.io/karpenter-controller" + + // S3ObjectPrefix is the S3 path prefix where bootstrap data is stored + // Format: s3://// + S3ObjectPrefix = "karpenter-machine-pool" + + // Condition reasons for tracking resource creation states + // NodePoolCreationFailedReason indicates that the NodePool creation failed NodePoolCreationFailedReason = "NodePoolCreationFailed" // EC2NodeClassCreationFailedReason indicates that the EC2NodeClass creation failed @@ -52,7 +65,7 @@ const ( type KarpenterMachinePoolReconciler struct { awsClients resolver.AWSClients client client.Client - // clusterClientGetter is used to create a client targeting the workload cluster + // clusterClientGetter is used to create a kubernetes client targeting the workload cluster clusterClientGetter remote.ClusterClientGetter } @@ -60,10 +73,12 @@ func NewKarpenterMachinepoolReconciler(client client.Client, clusterClientGetter return &KarpenterMachinePoolReconciler{awsClients: awsClients, client: client, clusterClientGetter: clusterClientGetter} } -// Reconcile reconciles KarpenterMachinePool, which represent cluster node pools that will be managed by karpenter. -// The controller will upload to S3 the Ignition configuration for the reconciled node pool. -// It will also take care of deleting EC2 instances created by karpenter when the cluster is being removed. -// And lastly, it will create the karpenter CRs in the workload cluster. +// Reconcile reconciles KarpenterMachinePool CRs, which represent cluster node pools that will be managed by karpenter. +// KarpenterMachinePoolReconciler reconciles KarpenterMachinePool objects by: +// 1. Creating Karpenter NodePool and EC2NodeClass resources in workload clusters +// 2. Managing bootstrap data upload to S3 for node initialization +// 3. Enforcing Kubernetes version skew policies +// 4. Cleaning up EC2 instances when clusters are deleted func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { logger := log.FromContext(ctx) logger.Info("Reconciling") @@ -86,6 +101,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } logger = logger.WithValues("machinePool", machinePool.Name) + // Bootstrap data must be available before we can proceed with creating Karpenter resources if machinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil { logger.Info("Bootstrap data secret reference is not yet available") return reconcile.Result{RequeueAfter: time.Duration(10) * time.Second}, nil @@ -115,31 +131,26 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return ctrl.Result{}, nil } + // S3 bucket is required for storing bootstrap data that Karpenter nodes will fetch if awsCluster.Spec.S3Bucket == nil { return reconcile.Result{}, errors.New("a cluster wide object storage configured at `AWSCluster.spec.s3Bucket` is required") } + // Get AWS credentials for S3 and EC2 operations roleIdentity := &capa.AWSClusterRoleIdentity{} if err = r.client.Get(ctx, client.ObjectKey{Name: awsCluster.Spec.IdentityRef.Name}, roleIdentity); err != nil { return reconcile.Result{}, fmt.Errorf("failed to get AWSClusterRoleIdentity referenced in AWSCluster: %w", err) } + // Handle deletion: cleanup EC2 instances and Karpenter resources if !karpenterMachinePool.GetDeletionTimestamp().IsZero() { return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity) } - bootstrapSecret := &v1.Secret{} - if err := r.client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName}, bootstrapSecret); err != nil { - return reconcile.Result{}, fmt.Errorf("bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName is not found: %w", err) - } - - bootstrapSecretValue, ok := bootstrapSecret.Data["value"] - if !ok { - return reconcile.Result{}, errors.New("error retrieving bootstrap data: secret value key is missing") - } - // Create a deep copy of the reconciled object so we can change it karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() + + // Add finalizer to ensure proper cleanup sequence updated := controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) if updated { if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { @@ -148,37 +159,17 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } // Create or update Karpenter custom resources in the workload cluster. - if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool, bootstrapSecretValue); err != nil { + if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") return reconcile.Result{}, err } - bootstrapUserDataHash := fmt.Sprintf("%x", sha256.Sum256(bootstrapSecretValue)) - previousHash, annotationHashExists := karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] - if !annotationHashExists || previousHash != bootstrapUserDataHash { - s3Client, err := r.awsClients.NewS3Client(awsCluster.Spec.Region, roleIdentity.Spec.RoleArn) - if err != nil { - return reconcile.Result{}, err - } - - key := path.Join(S3ObjectPrefix, req.Name) - - logger.Info("Writing userdata to S3", "bucket", awsCluster.Spec.S3Bucket.Name, "key", key) - if err = s3Client.Put(ctx, awsCluster.Spec.S3Bucket.Name, key, bootstrapSecretValue); err != nil { - return reconcile.Result{}, err - } - - if karpenterMachinePool.Annotations == nil { - karpenterMachinePool.Annotations = make(map[string]string) - } - karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] = bootstrapUserDataHash - - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.annotations with user data hash", "annotation", BootstrapDataHashAnnotation) - return reconcile.Result{}, err - } + // Reconcile bootstrap data - fetch secret and upload to S3 if changed + if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { + return reconcile.Result{}, err } + // Update status with current node information from the workload cluster providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) if err != nil { return reconcile.Result{}, err @@ -201,6 +192,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, err } + // Update the parent MachinePool replica count to match actual node claims if machinePool.Spec.Replicas == nil || *machinePool.Spec.Replicas != numberOfNodeClaims { machinePoolCopy := machinePool.DeepCopy() machinePool.Spec.Replicas = &numberOfNodeClaims @@ -213,7 +205,58 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, nil } +// reconcileMachinePoolBootstrapUserData handles the bootstrap user data reconciliation process. +// It fetches the bootstrap secret, checks if the data has changed, and uploads it to S3 if needed. +// It also updates the hash annotation to track the current bootstrap data version. +func (r *KarpenterMachinePoolReconciler) reconcileMachinePoolBootstrapUserData(ctx context.Context, logger logr.Logger, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, dataSecretName string, roleIdentity *capa.AWSClusterRoleIdentity) error { + // Get the bootstrap secret containing userdata for node initialization + bootstrapSecret := &v1.Secret{} + if err := r.client.Get(ctx, client.ObjectKey{Namespace: karpenterMachinePool.Namespace, Name: dataSecretName}, bootstrapSecret); err != nil { + return fmt.Errorf("bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName is not found: %w", err) + } + + bootstrapSecretValue, ok := bootstrapSecret.Data["value"] + if !ok { + return errors.New("error retrieving bootstrap data: secret value key is missing") + } + + // Check if bootstrap data has changed and needs to be re-uploaded to S3 + bootstrapUserDataHash := fmt.Sprintf("%x", sha256.Sum256(bootstrapSecretValue)) + previousHash, annotationHashExists := karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] + if !annotationHashExists || previousHash != bootstrapUserDataHash { + s3Client, err := r.awsClients.NewS3Client(awsCluster.Spec.Region, roleIdentity.Spec.RoleArn) + if err != nil { + return err + } + + key := path.Join(S3ObjectPrefix, karpenterMachinePool.Name) + + logger.Info("Writing userdata to S3", "bucket", awsCluster.Spec.S3Bucket.Name, "key", key) + if err = s3Client.Put(ctx, awsCluster.Spec.S3Bucket.Name, key, bootstrapSecretValue); err != nil { + return err + } + + // Create copy for patching annotations + karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() + + // Update the hash annotation to track the current bootstrap data version + if karpenterMachinePool.Annotations == nil { + karpenterMachinePool.Annotations = make(map[string]string) + } + karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] = bootstrapUserDataHash + + if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { + logger.Error(err, "failed to patch karpenterMachinePool.annotations with user data hash", "annotation", BootstrapDataHashAnnotation) + return err + } + } + + return nil +} + // reconcileDelete deletes the karpenter custom resources from the workload cluster. +// When the cluster itself is being deleted, it also terminates all EC2 instances +// created by Karpenter to prevent orphaned resources. func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, roleIdentity *capa.AWSClusterRoleIdentity) (reconcile.Result, error) { // We check if the owner Cluster is also being deleted (on top of the `KarpenterMachinePool` being deleted). // If the Cluster is being deleted, we terminate all the ec2 instances that karpenter may have launched. @@ -256,6 +299,8 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo return reconcile.Result{}, nil } +// getWorkloadClusterNodeClaims retrieves all NodeClaim resources from the workload cluster. +// NodeClaims represent actual compute resources provisioned by Karpenter. func (r *KarpenterMachinePoolReconciler) getWorkloadClusterNodeClaims(ctx context.Context, cluster *capi.Cluster) (*unstructured.UnstructuredList, error) { nodeClaimList := &unstructured.UnstructuredList{} workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) @@ -274,6 +319,9 @@ func (r *KarpenterMachinePoolReconciler) getWorkloadClusterNodeClaims(ctx contex return nodeClaimList, err } +// computeProviderIDListFromNodeClaimsInWorkloadCluster extracts provider IDs from NodeClaims +// and returns both the list of provider IDs and the total count of node claims. +// Provider IDs are AWS-specific identifiers like "aws:///us-west-2a/i-1234567890abcdef0" func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx context.Context, logger logr.Logger, cluster *capi.Cluster) ([]string, int32, error) { var providerIDList []string @@ -298,7 +346,9 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo return providerIDList, int32(len(nodeClaimList.Items)), nil } -// getControlPlaneVersion retrieves the current Kubernetes version from the control plane +// getControlPlaneVersion retrieves the current Kubernetes version from the control plane. +// This is used for version skew validation to ensure workers don't run newer versions +// than the control plane, as defined in the version skew policy https://kubernetes.io/releases/version-skew-policy/. func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Context, cluster *capi.Cluster) (string, error) { if cluster.Spec.ControlPlaneRef == nil { return "", fmt.Errorf("cluster has no control plane reference") @@ -329,8 +379,10 @@ func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Cont return version, nil } -// createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass custom resources in the workload cluster -func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool, bootstrapSecretValue []byte) error { +// createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass custom resources in the workload cluster. +// This method enforces version skew policies and sets appropriate conditions based on success/failure states. +func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { + // Validate version skew: ensure worker nodes don't use newer Kubernetes versions than control plane allowed, controlPlaneCurrentVersion, nodePoolDesiredVersion, err := r.IsVersionSkewAllowed(ctx, cluster, machinePool) if err != nil { return err @@ -356,7 +408,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } // Create or update EC2NodeClass - if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, awsCluster, karpenterMachinePool, bootstrapSecretValue); err != nil { + if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, awsCluster, karpenterMachinePool); err != nil { conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, EC2NodeClassCreationFailedReason, err.Error()) return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) } @@ -375,7 +427,9 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } // createOrUpdateEC2NodeClass creates or updates the EC2NodeClass resource in the workload cluster -func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, bootstrapSecretValue []byte) error { +// EC2NodeClass defines the EC2-specific configuration for nodes that Karpenter will provision, +// including AMI selection, instance profiles, security groups, and user data. +func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { ec2NodeClassGVR := schema.GroupVersionResource{ Group: EC2NodeClassAPIGroup, Version: "v1", @@ -428,6 +482,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. return nil } +// mergeMaps combines multiple maps, with later maps taking precedence for duplicate keys func mergeMaps[A comparable, B any](maps ...map[A]B) map[A]B { result := make(map[A]B) for _, m := range maps { @@ -438,7 +493,9 @@ func mergeMaps[A comparable, B any](maps ...map[A]B) map[A]B { return result } -// createOrUpdateNodePool creates or updates the NodePool resource in the workload cluster +// createOrUpdateNodePool creates or updates the NodePool resource in the workload cluster. +// NodePool defines the desired state and constraints for nodes that Karpenter should provision, +// including resource limits, disruption policies, and node requirements. func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { nodePoolGVR := schema.GroupVersionResource{ Group: "karpenter.sh", @@ -479,6 +536,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "disruption": map[string]interface{}{}, } + // Apply user-defined NodePool configuration if provided if karpenterMachinePool.Spec.NodePool != nil { dis := spec["disruption"].(map[string]interface{}) dis["budgets"] = karpenterMachinePool.Spec.NodePool.Disruption.Budgets @@ -526,7 +584,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont return nil } -// deleteKarpenterResources deletes the Karpenter NodePool and EC2NodeClass resources from the workload cluster +// deleteKarpenterResources deletes the Karpenter NodePool and EC2NodeClass resources from the workload cluster. func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { @@ -570,7 +628,7 @@ func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Co return nil } -// generateUserData generates the user data for Ignition configuration +// generateUserData generates the user data for Ignition configuration. func (r *KarpenterMachinePoolReconciler) generateUserData(s3bucketName, karpenterMachinePoolName string) string { userData := map[string]interface{}{ "ignition": map[string]interface{}{ @@ -615,12 +673,17 @@ func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, m // IsVersionSkewAllowed checks if the worker version can be updated based on the control plane version. // The workers can't use a newer k8s version than the one used by the control plane. +// +// This implements Kubernetes version skew policy https://kubernetes.io/releases/version-skew-policy/ +// +// Returns: (allowed bool, controlPlaneVersion string, desiredWorkerVersion string, error) func (r *KarpenterMachinePoolReconciler) IsVersionSkewAllowed(ctx context.Context, cluster *capi.Cluster, machinePool *capiexp.MachinePool) (bool, string, string, error) { controlPlaneVersion, err := r.getControlPlaneVersion(ctx, cluster) if err != nil { return true, "", "", fmt.Errorf("failed to get current Control Plane k8s version: %w", err) } + // Parse versions using semantic versioning for proper comparison controlPlaneCurrentK8sVersion, err := semver.ParseTolerant(controlPlaneVersion) if err != nil { return true, "", "", fmt.Errorf("failed to parse current Control Plane k8s version: %w", err) @@ -631,5 +694,6 @@ func (r *KarpenterMachinePoolReconciler) IsVersionSkewAllowed(ctx context.Contex return true, controlPlaneVersion, "", fmt.Errorf("failed to parse node pool desired k8s version: %w", err) } + // Allow if control plane version >= desired worker version return controlPlaneCurrentK8sVersion.GE(machinePoolDesiredK8sVersion), controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, nil } From 2d227a4822a5c4f14810b3065e2a7da691331dd0 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 09:47:49 +0200 Subject: [PATCH 18/41] Use conditions properly --- api/v1alpha1/karpentermachinepool_types.go | 12 - ...luster.x-k8s.io_karpentermachinepools.yaml | 10 - .../karpentermachinepool_controller.go | 92 +++++-- .../karpentermachinepool_controller_test.go | 224 +++++++++++++++++- ...luster.x-k8s.io_karpentermachinepools.yaml | 10 - pkg/conditions/conditions.go | 84 ++++++- 6 files changed, 364 insertions(+), 68 deletions(-) diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index 5fc898dc..c10e4b6b 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -39,10 +39,6 @@ type KarpenterMachinePoolSpec struct { // KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. type KarpenterMachinePoolStatus struct { - // Ready is true when the provider resource is ready. - // +optional - Ready bool `json:"ready"` - // Replicas is the most recently observed number of replicas // +optional Replicas int32 `json:"replicas"` @@ -50,14 +46,6 @@ type KarpenterMachinePoolStatus struct { // Conditions defines current service state of the KarpenterMachinePool. // +optional Conditions capi.Conditions `json:"conditions,omitempty"` - - // NodePoolReady indicates whether the NodePool is ready - // +optional - NodePoolReady bool `json:"nodePoolReady"` - - // EC2NodeClassReady indicates whether the EC2NodeClass is ready - // +optional - EC2NodeClassReady bool `json:"ec2NodeClassReady"` } // +kubebuilder:object:root=true diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 9b750f38..27f0c8fd 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -1016,16 +1016,6 @@ spec: - type type: object type: array - ec2NodeClassReady: - description: EC2NodeClassReady indicates whether the EC2NodeClass - is ready - type: boolean - nodePoolReady: - description: NodePoolReady indicates whether the NodePool is ready - type: boolean - ready: - description: Ready is true when the provider resource is ready. - type: boolean replicas: description: Replicas is the most recently observed number of replicas format: int32 diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index c9658c2e..4ceb4032 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -51,15 +51,6 @@ const ( // S3ObjectPrefix is the S3 path prefix where bootstrap data is stored // Format: s3://// S3ObjectPrefix = "karpenter-machine-pool" - - // Condition reasons for tracking resource creation states - - // NodePoolCreationFailedReason indicates that the NodePool creation failed - NodePoolCreationFailedReason = "NodePoolCreationFailed" - // EC2NodeClassCreationFailedReason indicates that the EC2NodeClass creation failed - EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" - // VersionSkewBlockedReason indicates that the update was blocked due to version skew policy - VersionSkewBlockedReason = "VersionSkewBlocked" ) type KarpenterMachinePoolReconciler struct { @@ -150,6 +141,9 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // Create a deep copy of the reconciled object so we can change it karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() + // Initialize conditions - mark as initializing until all steps complete + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.InitializingReason, "KarpenterMachinePool is being initialized") + // Add finalizer to ensure proper cleanup sequence updated := controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) if updated { @@ -161,35 +155,75 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // Create or update Karpenter custom resources in the workload cluster. if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") + + // Ensure conditions are persisted even when errors occur + if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { + logger.Error(statusErr, "failed to patch karpenterMachinePool status with error conditions") + } + return reconcile.Result{}, err } // Reconcile bootstrap data - fetch secret and upload to S3 if changed if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) + + // Ensure conditions are persisted even when errors occur + if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { + logger.Error(statusErr, "failed to patch karpenterMachinePool status with bootstrap error conditions") + } + return reconcile.Result{}, err } + conditions.MarkBootstrapDataReady(karpenterMachinePool) // Update status with current node information from the workload cluster + if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { + return reconcile.Result{}, err + } + + // Mark the KarpenterMachinePool as ready when all conditions are satisfied + conditions.MarkKarpenterMachinePoolReady(karpenterMachinePool) + + // Update the status to persist the Ready condition + if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { + logger.Error(err, "failed to patch karpenterMachinePool status with Ready condition") + return reconcile.Result{}, err + } + + return reconcile.Result{}, nil +} + +// saveKarpenterInstancesToStatus updates the KarpenterMachinePool and parent MachinePool with current node information +// from the workload cluster, including replica counts and provider ID lists. +func (r *KarpenterMachinePoolReconciler) saveKarpenterInstancesToStatus(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { + // Create a copy for tracking changes + karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() + + // Get current node information from the workload cluster providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) if err != nil { - return reconcile.Result{}, err + return err } + // Update KarpenterMachinePool status with current replica count karpenterMachinePool.Status.Replicas = numberOfNodeClaims - karpenterMachinePool.Status.Ready = true logger.Info("Found NodeClaims in workload cluster, patching KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) + // Patch the status with the updated replica count if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { logger.Error(err, "failed to patch karpenterMachinePool.status.Replicas") - return reconcile.Result{}, err + return err } + // Update KarpenterMachinePool spec with current provider ID list karpenterMachinePool.Spec.ProviderIDList = providerIDList + // Patch the spec with the updated provider ID list if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { logger.Error(err, "failed to patch karpenterMachinePool.spec.providerIDList") - return reconcile.Result{}, err + return err } // Update the parent MachinePool replica count to match actual node claims @@ -198,11 +232,11 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco machinePool.Spec.Replicas = &numberOfNodeClaims if err := r.client.Patch(ctx, machinePool, client.MergeFrom(machinePoolCopy), client.FieldOwner("karpenter-machinepool-controller")); err != nil { logger.Error(err, "failed to patch MachinePool.spec.replicas") - return reconcile.Result{}, err + return err } } - return reconcile.Result{}, nil + return nil } // reconcileMachinePoolBootstrapUserData handles the bootstrap user data reconciliation process. @@ -212,11 +246,17 @@ func (r *KarpenterMachinePoolReconciler) reconcileMachinePoolBootstrapUserData(c // Get the bootstrap secret containing userdata for node initialization bootstrapSecret := &v1.Secret{} if err := r.client.Get(ctx, client.ObjectKey{Namespace: karpenterMachinePool.Namespace, Name: dataSecretName}, bootstrapSecret); err != nil { - return fmt.Errorf("bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName is not found: %w", err) + if k8serrors.IsNotFound(err) { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretNotFoundReason, fmt.Sprintf("Bootstrap secret %s not found", dataSecretName)) + } else { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to get bootstrap secret %s: %v", dataSecretName, err)) + } + return fmt.Errorf("failed to get bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName: %w", err) } bootstrapSecretValue, ok := bootstrapSecret.Data["value"] if !ok { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretInvalidReason, "Bootstrap secret value key is missing") return errors.New("error retrieving bootstrap data: secret value key is missing") } @@ -395,13 +435,17 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co "nodePoolDesiredVersion", nodePoolDesiredVersion, "reason", message) - // Mark resources as not ready due to version skew - conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) - conditions.MarkNodePoolNotReady(karpenterMachinePool, VersionSkewBlockedReason, message) + // Mark version skew as invalid and resources as not ready + conditions.MarkVersionSkewInvalid(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) return fmt.Errorf("version skew policy violation: %s", message) } + // Mark version skew as valid + conditions.MarkVersionSkewValid(karpenterMachinePool) + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { return fmt.Errorf("failed to get workload cluster client: %w", err) @@ -409,19 +453,19 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co // Create or update EC2NodeClass if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, awsCluster, karpenterMachinePool); err != nil { - conditions.MarkEC2NodeClassNotReady(karpenterMachinePool, EC2NodeClassCreationFailedReason, err.Error()) + conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.EC2NodeClassCreationFailedReason, err.Error()) return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) } // Create or update NodePool if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { - conditions.MarkNodePoolNotReady(karpenterMachinePool, NodePoolCreationFailedReason, err.Error()) + conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, err.Error()) return fmt.Errorf("failed to create or update NodePool: %w", err) } - // Mark both resources as ready - conditions.MarkEC2NodeClassReady(karpenterMachinePool) - conditions.MarkNodePoolReady(karpenterMachinePool) + // Mark both resources as successfully created + conditions.MarkEC2NodeClassCreated(karpenterMachinePool) + conditions.MarkNodePoolCreated(karpenterMachinePool) return nil } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 23b0b836..9d436a47 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -43,6 +43,16 @@ const ( KubernetesVersion = "v1.29.1" ) +// findCondition returns the condition with the given type from the list of conditions. +func findCondition(conditions capi.Conditions, conditionType string) *capi.Condition { + for i := range conditions { + if conditions[i].Type == capi.ConditionType(conditionType) { + return &conditions[i] + } + } + return nil +} + var _ = Describe("KarpenterMachinePool reconciler", func() { var ( capiBootstrapSecretContent []byte @@ -838,7 +848,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { When("the bootstrap secret referenced in the dataSecretName field does not exist", func() { It("returns an error", func() { - Expect(reconcileErr).To(MatchError(ContainSubstring("bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName is not found"))) + Expect(reconcileErr).To(MatchError(ContainSubstring("failed to get bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName"))) }) }) When("the bootstrap secret exists but it does not contain the 'value' key", func() { @@ -1046,13 +1056,19 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err = unstructured.SetNestedField(nodeClaim2.Object, map[string]interface{}{"providerID": "aws:///us-west-2a/i-09876543219fedcba"}, "status") Expect(err).NotTo(HaveOccurred()) err = k8sClient.Status().Update(ctx, nodeClaim2) + Expect(err).NotTo(HaveOccurred()) }) It("updates the KarpenterMachinePool spec and status accordingly", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) - Expect(updatedKarpenterMachinePool.Status.Ready).To(BeTrue()) + + // Check that the Ready condition is True + readyCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "Ready") + Expect(readyCondition).NotTo(BeNil()) + Expect(string(readyCondition.Status)).To(Equal("True")) + Expect(updatedKarpenterMachinePool.Status.Replicas).To(Equal(int32(2))) Expect(updatedKarpenterMachinePool.Spec.ProviderIDList).To(ContainElements("aws:///us-west-2a/i-1234567890abcdef0", "aws:///us-west-2a/i-09876543219fedcba")) }) @@ -1286,6 +1302,210 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(s3Client.PutCallCount()).To(Equal(0)) }) }) + + When("version skew validation fails (node pool version newer than control plane)", func() { + var controlPlaneVersion = "v1.29.0" + var nodePoolVersion = "v1.30.0" // Newer than control plane - should violate version skew policy + + BeforeEach(func() { + dataSecretName = DataSecretName + machinePool := &capiexp.MachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capiexp.MachinePoolSpec{ + ClusterName: ClusterName, + Template: capi.MachineTemplateSpec{ + ObjectMeta: capi.ObjectMeta{}, + Spec: capi.MachineSpec{ + ClusterName: ClusterName, + Bootstrap: capi.Bootstrap{ + ConfigRef: &v1.ObjectReference{ + Kind: "KubeadmConfig", + Namespace: namespace, + Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), + APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", + }, + DataSecretName: &dataSecretName, + }, + InfrastructureRef: v1.ObjectReference{ + Kind: "KarpenterMachinePool", + Namespace: namespace, + Name: KarpenterMachinePoolName, + APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", + }, + Version: &nodePoolVersion, // Newer version than control plane + }, + }, + }, + } + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + cluster := &capi.Cluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, + InfrastructureRef: &v1.ObjectReference{ + Kind: "AWSCluster", + Namespace: namespace, + Name: ClusterName, + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + }, + }, + } + err = k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + // Create control plane with OLDER version than node pool + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": "v1.21.2", // Irrelevant for this test + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + + // Set control plane status with OLDER version than node pool + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": controlPlaneVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capa.AWSClusterSpec{ + Region: AWSRegion, + S3Bucket: &capa.S3Bucket{ + Name: AWSClusterBucketName, + }, + IdentityRef: &capa.AWSIdentityReference{ + Kind: "AWSClusterRoleIdentity", + Name: "aws-cluster-role-identity", + }, + AdditionalTags: map[string]string{ + "additional-tag-for-all-resources": "custom-tag", + }, + }, + } + err = k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + + awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ + ObjectMeta: ctrl.ObjectMeta{ + Name: "aws-cluster-role-identity", + }, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, + } + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) + + // Create bootstrap secret + bootstrapSecret := &v1.Secret{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: DataSecretName, + }, + Data: map[string][]byte{"value": capiBootstrapSecretContent}, + } + err = k8sClient.Create(ctx, bootstrapSecret) + Expect(err).NotTo(HaveOccurred()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{}, + } + err = k8sClient.Create(ctx, karpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + }) + + It("returns a version skew error", func() { + Expect(reconcileErr).To(MatchError(ContainSubstring("version skew policy violation"))) + Expect(reconcileErr).To(MatchError(ContainSubstring("control plane version v1.29.0 is older than node pool version v1.30.0"))) + }) + + It("persists the version skew conditions to the Kubernetes API", func() { + // This test verifies that conditions ARE saved even when errors occur + updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + // Verify that version skew condition was persisted with the correct state + versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewValid") + Expect(versionSkewCondition).NotTo(BeNil()) + Expect(string(versionSkewCondition.Status)).To(Equal("False")) + Expect(versionSkewCondition.Reason).To(Equal("VersionSkewBlocked")) + Expect(versionSkewCondition.Message).To(ContainSubstring("control plane version v1.29.0 is older than node pool version v1.30.0")) + + // Verify that EC2NodeClass condition was persisted with error state + ec2NodeClassCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "EC2NodeClassCreated") + Expect(ec2NodeClassCondition).NotTo(BeNil()) + Expect(string(ec2NodeClassCondition.Status)).To(Equal("False")) + Expect(ec2NodeClassCondition.Reason).To(Equal("VersionSkewBlocked")) + + // Verify that NodePool condition was persisted with error state + nodePoolCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "NodePoolCreated") + Expect(nodePoolCondition).NotTo(BeNil()) + Expect(string(nodePoolCondition.Status)).To(Equal("False")) + Expect(nodePoolCondition.Reason).To(Equal("VersionSkewBlocked")) + }) + }) }) // ExpectUnstructured digs into u.Object at the given path, asserts that it was found and error‐free, and returns diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 9b750f38..27f0c8fd 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -1016,16 +1016,6 @@ spec: - type type: object type: array - ec2NodeClassReady: - description: EC2NodeClassReady indicates whether the EC2NodeClass - is ready - type: boolean - nodePoolReady: - description: NodePoolReady indicates whether the NodePool is ready - type: boolean - ready: - description: Ready is true when the provider resource is ready. - type: boolean replicas: description: Replicas is the most recently observed number of replicas format: int32 diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index a89132a5..287d7c0a 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -13,8 +13,48 @@ const ( TransitGatewayCreated capi.ConditionType = "TransitGatewayCreated" TransitGatewayAttached capi.ConditionType = "TransitGatewayAttached" PrefixListEntriesReady capi.ConditionType = "PrefixListEntriesReady" - NodePoolReady capi.ConditionType = "NodePoolReady" - EC2NodeClassReady capi.ConditionType = "EC2NodeClassReady" + + // NodePoolCreatedCondition indicates whether the NodePool resource has been successfully + // created or updated in the workload cluster. This doesn't mean the NodePool is ready + // to provision nodes, just that the resource exists. + NodePoolCreatedCondition capi.ConditionType = "NodePoolCreated" + + // EC2NodeClassCreatedCondition indicates whether the EC2NodeClass resource has been + // successfully created or updated in the workload cluster. This doesn't mean the + // EC2NodeClass is ready for use, just that the resource exists. + EC2NodeClassCreatedCondition capi.ConditionType = "EC2NodeClassCreated" + + // BootstrapDataReadyCondition indicates whether the bootstrap user data has been + // successfully uploaded to S3 and is ready for use by Karpenter nodes. + BootstrapDataReadyCondition capi.ConditionType = "BootstrapDataReady" + + // VersionSkewValidCondition indicates whether the Kubernetes version skew policy + // is satisfied (worker nodes don't use newer versions than control plane). + VersionSkewValidCondition capi.ConditionType = "VersionSkewValid" + + // ReadyCondition indicates the overall readiness of the KarpenterMachinePool. + // This is True when all necessary Karpenter resources are created and configured. + ReadyCondition capi.ConditionType = "Ready" +) + +// Condition reasons used by various controllers +const ( + // Generic reasons used across controllers + InitializingReason = "Initializing" + ReadyReason = "Ready" + NotReadyReason = "NotReady" + + // KarpenterMachinePool controller reasons + NodePoolCreationFailedReason = "NodePoolCreationFailed" + NodePoolCreationSucceededReason = "NodePoolCreated" + EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" + EC2NodeClassCreationSucceededReason = "EC2NodeClassCreated" + BootstrapDataUploadFailedReason = "BootstrapDataUploadFailed" + BootstrapDataSecretNotFoundReason = "BootstrapDataSecretNotFound" + BootstrapDataSecretInvalidReason = "BootstrapDataSecretInvalid" + BootstrapDataUploadSucceededReason = "BootstrapDataUploaded" + VersionSkewBlockedReason = "VersionSkewBlocked" + VersionSkewValidReason = "VersionSkewValid" ) func MarkReady(setter capiconditions.Setter, condition capi.ConditionType) { @@ -45,18 +85,42 @@ func MarkIDNotProvided(cluster *capi.Cluster, id string) { ) } -func MarkNodePoolReady(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, NodePoolReady) +func MarkNodePoolCreated(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, NodePoolCreatedCondition) +} + +func MarkNodePoolNotCreated(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, NodePoolCreatedCondition, reason, capi.ConditionSeverityError, message, nil) +} + +func MarkEC2NodeClassCreated(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, EC2NodeClassCreatedCondition) +} + +func MarkEC2NodeClassNotCreated(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, EC2NodeClassCreatedCondition, reason, capi.ConditionSeverityError, message, nil) +} + +func MarkBootstrapDataReady(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, BootstrapDataReadyCondition) +} + +func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, message, nil) +} + +func MarkVersionSkewValid(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, VersionSkewValidCondition) } -func MarkNodePoolNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, NodePoolReady, reason, capi.ConditionSeverityError, message, nil) +func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, VersionSkewValidCondition, reason, capi.ConditionSeverityError, message, nil) } -func MarkEC2NodeClassReady(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, EC2NodeClassReady) +func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { + capiconditions.MarkTrue(setter, ReadyCondition) } -func MarkEC2NodeClassNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, EC2NodeClassReady, reason, capi.ConditionSeverityError, message, nil) +func MarkKarpenterMachinePoolNotReady(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, ReadyCondition, reason, capi.ConditionSeverityError, message, nil) } From e866ede76320067b7aea19beb271a25fa244a55b Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 12:54:57 +0200 Subject: [PATCH 19/41] Fix condition message formatting --- controllers/karpentermachinepool_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 4ceb4032..2e166075 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -453,13 +453,13 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co // Create or update EC2NodeClass if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, awsCluster, karpenterMachinePool); err != nil { - conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.EC2NodeClassCreationFailedReason, err.Error()) + conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.EC2NodeClassCreationFailedReason, fmt.Sprintf("%v", err)) return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) } // Create or update NodePool if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { - conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, err.Error()) + conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, fmt.Sprintf("%v", err)) return fmt.Errorf("failed to create or update NodePool: %w", err) } From 1f8fb8db8864250fd19d1191900aa878756e594a Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 14:31:28 +0200 Subject: [PATCH 20/41] Go mod tidy --- go.mod | 1 - 1 file changed, 1 deletion(-) diff --git a/go.mod b/go.mod index 42d31dcd..b3bf238d 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,6 @@ require ( golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect From bd9def17aa47de779edfca675462be0cb82d4e36 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 15:15:44 +0200 Subject: [PATCH 21/41] Use different copies for the different patch operations --- .../karpentermachinepool_controller.go | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 2e166075..a6bc3e7d 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -139,7 +139,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } // Create a deep copy of the reconciled object so we can change it - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() + karpenterMachinePoolFinalizerCopy := karpenterMachinePool.DeepCopy() // Initialize conditions - mark as initializing until all steps complete conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.InitializingReason, "KarpenterMachinePool is being initialized") @@ -147,17 +147,18 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // Add finalizer to ensure proper cleanup sequence updated := controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) if updated { - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { + if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolFinalizerCopy)); err != nil { return reconcile.Result{}, fmt.Errorf("failed to add finalizer to KarpenterMachinePool: %w", err) } } // Create or update Karpenter custom resources in the workload cluster. + karpenterMachinePoolConditionsCopy := karpenterMachinePool.DeepCopy() if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { + if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolConditionsCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { logger.Error(statusErr, "failed to patch karpenterMachinePool status with error conditions") } @@ -165,11 +166,12 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } // Reconcile bootstrap data - fetch secret and upload to S3 if changed + karpenterMachinePoolBootstrapConditionsCopy := karpenterMachinePool.DeepCopy() if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { + if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolBootstrapConditionsCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { logger.Error(statusErr, "failed to patch karpenterMachinePool status with bootstrap error conditions") } @@ -178,7 +180,15 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco conditions.MarkBootstrapDataReady(karpenterMachinePool) // Update status with current node information from the workload cluster + karpenterMachinePoolStatusCopy := karpenterMachinePool.DeepCopy() if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { + logger.Error(err, "failed to save Karpenter instances to status") + + // Ensure conditions are persisted even when errors occur + if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolStatusCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { + logger.Error(statusErr, "failed to patch karpenterMachinePool status with conditions before returning error") + } + return reconcile.Result{}, err } @@ -186,7 +196,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco conditions.MarkKarpenterMachinePoolReady(karpenterMachinePool) // Update the status to persist the Ready condition - if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { + if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolStatusCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { logger.Error(err, "failed to patch karpenterMachinePool status with Ready condition") return reconcile.Result{}, err } From 652793163796ffc8eee1f274d93b2a8c24e4d58b Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 16:38:20 +0200 Subject: [PATCH 22/41] Use update instead of patch for conditions --- .../karpentermachinepool_controller.go | 23 +- .../karpentermachinepool_controller_test.go | 229 ++++++++++++++++++ pkg/conditions/conditions.go | 30 ++- 3 files changed, 263 insertions(+), 19 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index a6bc3e7d..a1b458ff 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -153,26 +153,24 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } // Create or update Karpenter custom resources in the workload cluster. - karpenterMachinePoolConditionsCopy := karpenterMachinePool.DeepCopy() if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolConditionsCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { - logger.Error(statusErr, "failed to patch karpenterMachinePool status with error conditions") + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with error conditions") } return reconcile.Result{}, err } // Reconcile bootstrap data - fetch secret and upload to S3 if changed - karpenterMachinePoolBootstrapConditionsCopy := karpenterMachinePool.DeepCopy() if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolBootstrapConditionsCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { - logger.Error(statusErr, "failed to patch karpenterMachinePool status with bootstrap error conditions") + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with bootstrap error conditions") } return reconcile.Result{}, err @@ -180,13 +178,12 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco conditions.MarkBootstrapDataReady(karpenterMachinePool) // Update status with current node information from the workload cluster - karpenterMachinePoolStatusCopy := karpenterMachinePool.DeepCopy() if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to save Karpenter instances to status") // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolStatusCopy), client.FieldOwner("karpentermachinepool-controller")); statusErr != nil { - logger.Error(statusErr, "failed to patch karpenterMachinePool status with conditions before returning error") + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with conditions before returning error") } return reconcile.Result{}, err @@ -196,8 +193,8 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco conditions.MarkKarpenterMachinePoolReady(karpenterMachinePool) // Update the status to persist the Ready condition - if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolStatusCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool status with Ready condition") + if err := r.client.Status().Update(ctx, karpenterMachinePool); err != nil { + logger.Error(err, "failed to update karpenterMachinePool status with Ready condition") return reconcile.Result{}, err } @@ -466,15 +463,13 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.EC2NodeClassCreationFailedReason, fmt.Sprintf("%v", err)) return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) } + conditions.MarkEC2NodeClassCreated(karpenterMachinePool) // Create or update NodePool if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, fmt.Sprintf("%v", err)) return fmt.Errorf("failed to create or update NodePool: %w", err) } - - // Mark both resources as successfully created - conditions.MarkEC2NodeClassCreated(karpenterMachinePool) conditions.MarkNodePoolCreated(karpenterMachinePool) return nil diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 9d436a47..7532b582 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -1303,6 +1303,235 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) }) + When("creating the NodePool fails", func() { + BeforeEach(func() { + dataSecretName = DataSecretName + kubernetesVersion := KubernetesVersion + machinePool := &capiexp.MachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capiexp.MachinePoolSpec{ + ClusterName: ClusterName, + Template: capi.MachineTemplateSpec{ + Spec: capi.MachineSpec{ + ClusterName: ClusterName, + Version: &kubernetesVersion, + Bootstrap: capi.Bootstrap{ + DataSecretName: &dataSecretName, + }, + }, + }, + }, + } + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + // Get the created machinePool to access its UID + err = k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, machinePool) + Expect(err).NotTo(HaveOccurred()) + + deviceName := "/dev/xvda" + volumeType := "gp3" + deleteOnTermination := true + volumeSize, _ := resource.ParseQuantity("8Gi") + kmp := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.UID, + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + InstanceProfile: &instanceProfile, + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Alias: "al2@latest", + }, + }, + BlockDeviceMappings: []*karpenterinfra.BlockDeviceMapping{ + { + DeviceName: &deviceName, + RootVolume: true, + EBS: &karpenterinfra.BlockDevice{ + VolumeSize: &volumeSize, + VolumeType: &volumeType, + DeleteOnTermination: &deleteOnTermination, + }, + }, + }, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"Name": "foo"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"Name": "foo"}, + }, + }, + Tags: map[string]string{ + "one-tag": "only-for-karpenter", + }, + }, + // NodePool spec omitted to focus on testing condition persistence + // NodePool: nil, + }, + } + err = k8sClient.Create(ctx, kmp) + Expect(err).NotTo(HaveOccurred()) + + // Create Cluster resource + cluster := &capi.Cluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, + InfrastructureRef: &v1.ObjectReference{ + Kind: "AWSCluster", + Namespace: namespace, + Name: ClusterName, + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + }, + }, + } + err = k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + // Create AWSCluster resource + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + AdditionalTags: map[string]string{ + "additional-tag-for-all-resources": "custom-tag", + }, + IdentityRef: &capa.AWSIdentityReference{ + Name: "default", + Kind: capa.ClusterRoleIdentityKind, + }, + Region: AWSRegion, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err = k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + + // Create AWSClusterRoleIdentity resource + awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, + } + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) + + // Create bootstrap secret for successful reconciliation + bootstrapSecret := &v1.Secret{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: DataSecretName, + }, + Data: map[string][]byte{"value": capiBootstrapSecretContent}, + } + err = k8sClient.Create(ctx, bootstrapSecret) + Expect(err).NotTo(HaveOccurred()) + + // Create control plane with same version as machine pool for successful version skew validation + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": KubernetesVersion, + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + + reconcileResult, reconcileErr = reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + }, + }) + }) + + It("updates conditions even when reconciliation fails", func() { + Expect(reconcileErr).To(HaveOccurred()) + Expect(reconcileErr.Error()).To(ContainSubstring("failed to create or update NodePool")) + + // Get the updated KarpenterMachinePool + updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + // This condition should be properly persisted even though reconciliation failed + ec2NodeClassCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "EC2NodeClassCreated") + Expect(ec2NodeClassCondition).NotTo(BeNil(), "EC2NodeClass condition should be persisted even on error") + Expect(string(ec2NodeClassCondition.Status)).To(Equal("True"), "EC2NodeClass was created successfully") + Expect(ec2NodeClassCondition.Reason).To(Equal("EC2NodeClassCreated")) + + // Version skew should be valid since we use the same version + versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewValid") + Expect(versionSkewCondition).NotTo(BeNil(), "VersionSkew condition should be persisted even on error") + Expect(string(versionSkewCondition.Status)).To(Equal("True"), "Version skew should be valid") + Expect(versionSkewCondition.Reason).To(Equal("VersionSkewValid")) + + // NodePool should be False since creation failed + nodePoolCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "NodePoolCreated") + Expect(nodePoolCondition).NotTo(BeNil(), "NodePool condition should be persisted even on error") + Expect(string(nodePoolCondition.Status)).To(Equal("False"), "NodePool creation failed") + Expect(nodePoolCondition.Reason).To(Equal("NodePoolCreationFailed")) + }) + }) + When("version skew validation fails (node pool version newer than control plane)", func() { var controlPlaneVersion = "v1.29.0" var nodePoolVersion = "v1.30.0" // Newer than control plane - should violate version skew policy diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 287d7c0a..2a54a4a7 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -86,7 +86,11 @@ func MarkIDNotProvided(cluster *capi.Cluster, id string) { } func MarkNodePoolCreated(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, NodePoolCreatedCondition) + capiconditions.Set(setter, &capi.Condition{ + Type: NodePoolCreatedCondition, + Status: "True", + Reason: NodePoolCreationSucceededReason, + }) } func MarkNodePoolNotCreated(setter capiconditions.Setter, reason, message string) { @@ -94,7 +98,11 @@ func MarkNodePoolNotCreated(setter capiconditions.Setter, reason, message string } func MarkEC2NodeClassCreated(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, EC2NodeClassCreatedCondition) + capiconditions.Set(setter, &capi.Condition{ + Type: EC2NodeClassCreatedCondition, + Status: "True", + Reason: EC2NodeClassCreationSucceededReason, + }) } func MarkEC2NodeClassNotCreated(setter capiconditions.Setter, reason, message string) { @@ -102,7 +110,11 @@ func MarkEC2NodeClassNotCreated(setter capiconditions.Setter, reason, message st } func MarkBootstrapDataReady(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, BootstrapDataReadyCondition) + capiconditions.Set(setter, &capi.Condition{ + Type: BootstrapDataReadyCondition, + Status: "True", + Reason: ReadyReason, + }) } func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message string) { @@ -110,7 +122,11 @@ func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message str } func MarkVersionSkewValid(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, VersionSkewValidCondition) + capiconditions.Set(setter, &capi.Condition{ + Type: VersionSkewValidCondition, + Status: "True", + Reason: VersionSkewValidReason, + }) } func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string) { @@ -118,7 +134,11 @@ func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string } func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { - capiconditions.MarkTrue(setter, ReadyCondition) + capiconditions.Set(setter, &capi.Condition{ + Type: ReadyCondition, + Status: "True", + Reason: ReadyReason, + }) } func MarkKarpenterMachinePoolNotReady(setter capiconditions.Setter, reason, message string) { From 38c008c13381ed60b97fff157cd470f1e241603a Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 17:02:26 +0200 Subject: [PATCH 23/41] Rename policy skew condition --- controllers/karpentermachinepool_controller.go | 2 +- controllers/karpentermachinepool_controller_test.go | 4 ++-- helm/aws-resolver-rules-operator/templates/rbac.yaml | 1 + pkg/conditions/conditions.go | 10 +++++----- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index a1b458ff..ad4986ee 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -451,7 +451,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } // Mark version skew as valid - conditions.MarkVersionSkewValid(karpenterMachinePool) + conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 7532b582..518ef9f6 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -1519,7 +1519,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(ec2NodeClassCondition.Reason).To(Equal("EC2NodeClassCreated")) // Version skew should be valid since we use the same version - versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewValid") + versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewPolicySatisfied") Expect(versionSkewCondition).NotTo(BeNil(), "VersionSkew condition should be persisted even on error") Expect(string(versionSkewCondition.Status)).To(Equal("True"), "Version skew should be valid") Expect(versionSkewCondition.Reason).To(Equal("VersionSkewValid")) @@ -1716,7 +1716,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(err).NotTo(HaveOccurred()) // Verify that version skew condition was persisted with the correct state - versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewValid") + versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewPolicySatisfied") Expect(versionSkewCondition).NotTo(BeNil()) Expect(string(versionSkewCondition.Status)).To(Equal("False")) Expect(versionSkewCondition.Reason).To(Equal("VersionSkewBlocked")) diff --git a/helm/aws-resolver-rules-operator/templates/rbac.yaml b/helm/aws-resolver-rules-operator/templates/rbac.yaml index 204ad885..5abd5efd 100644 --- a/helm/aws-resolver-rules-operator/templates/rbac.yaml +++ b/helm/aws-resolver-rules-operator/templates/rbac.yaml @@ -20,6 +20,7 @@ rules: - list - patch - watch + - update - apiGroups: - controlplane.cluster.x-k8s.io resources: diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 2a54a4a7..15255acd 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -28,9 +28,9 @@ const ( // successfully uploaded to S3 and is ready for use by Karpenter nodes. BootstrapDataReadyCondition capi.ConditionType = "BootstrapDataReady" - // VersionSkewValidCondition indicates whether the Kubernetes version skew policy + // VersionSkewPolicySatisfiedCondition indicates whether the Kubernetes version skew policy // is satisfied (worker nodes don't use newer versions than control plane). - VersionSkewValidCondition capi.ConditionType = "VersionSkewValid" + VersionSkewPolicySatisfiedCondition capi.ConditionType = "VersionSkewPolicySatisfied" // ReadyCondition indicates the overall readiness of the KarpenterMachinePool. // This is True when all necessary Karpenter resources are created and configured. @@ -121,16 +121,16 @@ func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message str capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, message, nil) } -func MarkVersionSkewValid(setter capiconditions.Setter) { +func MarkVersionSkewPolicySatisfied(setter capiconditions.Setter) { capiconditions.Set(setter, &capi.Condition{ - Type: VersionSkewValidCondition, + Type: VersionSkewPolicySatisfiedCondition, Status: "True", Reason: VersionSkewValidReason, }) } func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, VersionSkewValidCondition, reason, capi.ConditionSeverityError, message, nil) + capiconditions.MarkFalse(setter, VersionSkewPolicySatisfiedCondition, reason, capi.ConditionSeverityError, message, nil) } func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { From 92d3ae47a83836f689676997f383e463dc13af68 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 17:16:50 +0200 Subject: [PATCH 24/41] Fix condition message formatting again --- pkg/conditions/conditions.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 15255acd..471f7a85 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -94,7 +94,7 @@ func MarkNodePoolCreated(setter capiconditions.Setter) { } func MarkNodePoolNotCreated(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, NodePoolCreatedCondition, reason, capi.ConditionSeverityError, message, nil) + capiconditions.MarkFalse(setter, NodePoolCreatedCondition, reason, capi.ConditionSeverityError, message, "") } func MarkEC2NodeClassCreated(setter capiconditions.Setter) { @@ -106,7 +106,7 @@ func MarkEC2NodeClassCreated(setter capiconditions.Setter) { } func MarkEC2NodeClassNotCreated(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, EC2NodeClassCreatedCondition, reason, capi.ConditionSeverityError, message, nil) + capiconditions.MarkFalse(setter, EC2NodeClassCreatedCondition, reason, capi.ConditionSeverityError, message, "") } func MarkBootstrapDataReady(setter capiconditions.Setter) { @@ -118,7 +118,7 @@ func MarkBootstrapDataReady(setter capiconditions.Setter) { } func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, message, nil) + capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, message, "") } func MarkVersionSkewPolicySatisfied(setter capiconditions.Setter) { @@ -130,7 +130,7 @@ func MarkVersionSkewPolicySatisfied(setter capiconditions.Setter) { } func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, VersionSkewPolicySatisfiedCondition, reason, capi.ConditionSeverityError, message, nil) + capiconditions.MarkFalse(setter, VersionSkewPolicySatisfiedCondition, reason, capi.ConditionSeverityError, message, "") } func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { @@ -142,5 +142,5 @@ func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { } func MarkKarpenterMachinePoolNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, ReadyCondition, reason, capi.ConditionSeverityError, message, nil) + capiconditions.MarkFalse(setter, ReadyCondition, reason, capi.ConditionSeverityError, message, "") } From d06eececae282c317f29a1b1971576b14f92d299 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Thu, 24 Jul 2025 17:37:37 +0200 Subject: [PATCH 25/41] Update conditions immediately --- .../karpentermachinepool_controller.go | 24 +++++++++++++++++++ pkg/conditions/conditions.go | 10 ++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index ad4986ee..aceeb33c 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -177,6 +177,12 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } conditions.MarkBootstrapDataReady(karpenterMachinePool) + // Persist bootstrap data success condition immediately + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with bootstrap data success condition") + return reconcile.Result{}, fmt.Errorf("failed to persist bootstrap data success condition: %w", statusErr) + } + // Update status with current node information from the workload cluster if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to save Karpenter instances to status") @@ -453,6 +459,12 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co // Mark version skew as valid conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) + // Persist version skew success condition immediately + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with version skew success condition") + return fmt.Errorf("failed to persist version skew success condition: %w", statusErr) + } + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { return fmt.Errorf("failed to get workload cluster client: %w", err) @@ -465,6 +477,12 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } conditions.MarkEC2NodeClassCreated(karpenterMachinePool) + // Persist EC2NodeClass success condition immediately + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with EC2NodeClass success condition") + return fmt.Errorf("failed to persist EC2NodeClass success condition: %w", statusErr) + } + // Create or update NodePool if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, fmt.Sprintf("%v", err)) @@ -472,6 +490,12 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } conditions.MarkNodePoolCreated(karpenterMachinePool) + // Persist NodePool success condition immediately + if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { + logger.Error(statusErr, "failed to update karpenterMachinePool status with NodePool success condition") + return fmt.Errorf("failed to persist NodePool success condition: %w", statusErr) + } + return nil } diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 471f7a85..9f47b2a5 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -94,7 +94,7 @@ func MarkNodePoolCreated(setter capiconditions.Setter) { } func MarkNodePoolNotCreated(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, NodePoolCreatedCondition, reason, capi.ConditionSeverityError, message, "") + capiconditions.MarkFalse(setter, NodePoolCreatedCondition, reason, capi.ConditionSeverityError, "%s", message) } func MarkEC2NodeClassCreated(setter capiconditions.Setter) { @@ -106,7 +106,7 @@ func MarkEC2NodeClassCreated(setter capiconditions.Setter) { } func MarkEC2NodeClassNotCreated(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, EC2NodeClassCreatedCondition, reason, capi.ConditionSeverityError, message, "") + capiconditions.MarkFalse(setter, EC2NodeClassCreatedCondition, reason, capi.ConditionSeverityError, "%s", message) } func MarkBootstrapDataReady(setter capiconditions.Setter) { @@ -118,7 +118,7 @@ func MarkBootstrapDataReady(setter capiconditions.Setter) { } func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, message, "") + capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, "%s", message) } func MarkVersionSkewPolicySatisfied(setter capiconditions.Setter) { @@ -130,7 +130,7 @@ func MarkVersionSkewPolicySatisfied(setter capiconditions.Setter) { } func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, VersionSkewPolicySatisfiedCondition, reason, capi.ConditionSeverityError, message, "") + capiconditions.MarkFalse(setter, VersionSkewPolicySatisfiedCondition, reason, capi.ConditionSeverityError, "%s", message) } func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { @@ -142,5 +142,5 @@ func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { } func MarkKarpenterMachinePoolNotReady(setter capiconditions.Setter, reason, message string) { - capiconditions.MarkFalse(setter, ReadyCondition, reason, capi.ConditionSeverityError, message, "") + capiconditions.MarkFalse(setter, ReadyCondition, reason, capi.ConditionSeverityError, "%s", message) } From b8f2c7bee5d51474c73816a6891ac978a44c923c Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Sun, 27 Jul 2025 12:46:40 +0200 Subject: [PATCH 26/41] Use patchHelper --- api/v1alpha1/duration.go | 11 +- .../karpentermachinepool_controller.go | 113 ++++-------------- .../karpentermachinepool_controller_test.go | 1 + 3 files changed, 34 insertions(+), 91 deletions(-) diff --git a/api/v1alpha1/duration.go b/api/v1alpha1/duration.go index 1a1ef120..b30d2a8a 100644 --- a/api/v1alpha1/duration.go +++ b/api/v1alpha1/duration.go @@ -62,7 +62,16 @@ func (d NillableDuration) MarshalJSON() ([]byte, error) { // ToUnstructured implements the value.UnstructuredConverter interface. func (d NillableDuration) ToUnstructured() interface{} { if d.Raw != nil { - return d.Raw + // Decode the JSON bytes to get the actual string value + var str string + if err := json.Unmarshal(d.Raw, &str); err == nil { + return str + } + // Fallback to string conversion if unmarshal fails + if d.Duration != nil { + return d.Duration.String() + } + return Never } if d.Duration != nil { return d.Duration.String() diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index aceeb33c..1f8c5214 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -24,6 +24,7 @@ import ( capiutilexp "sigs.k8s.io/cluster-api/exp/util" capiutil "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/cluster-api/util/predicates" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -80,6 +81,24 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, client.IgnoreNotFound(err) } + patchHelper, err := patch.NewHelper(karpenterMachinePool, r.client) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to init patch helper: %w", err) + } + defer func() { + if err := patchHelper.Patch(ctx, karpenterMachinePool, patch.WithOwnedConditions{ + Conditions: []capi.ConditionType{ + conditions.ReadyCondition, + conditions.NodePoolCreatedCondition, + conditions.EC2NodeClassCreatedCondition, + conditions.BootstrapDataReadyCondition, + conditions.VersionSkewPolicySatisfiedCondition, + }, + }, patch.WithForceOverwriteConditions{}); err != nil { + logger.Error(err, "failed to patch KarpenterMachinePool") + } + }() + machinePool, err := capiutilexp.GetOwnerMachinePool(ctx, r.client, karpenterMachinePool.ObjectMeta) if err != nil { return reconcile.Result{}, fmt.Errorf("failed to get MachinePool owning the KarpenterMachinePool: %w", err) @@ -135,84 +154,43 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // Handle deletion: cleanup EC2 instances and Karpenter resources if !karpenterMachinePool.GetDeletionTimestamp().IsZero() { - return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity) + return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity, patchHelper) } - // Create a deep copy of the reconciled object so we can change it - karpenterMachinePoolFinalizerCopy := karpenterMachinePool.DeepCopy() - // Initialize conditions - mark as initializing until all steps complete conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.InitializingReason, "KarpenterMachinePool is being initialized") // Add finalizer to ensure proper cleanup sequence - updated := controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) - if updated { - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolFinalizerCopy)); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to add finalizer to KarpenterMachinePool: %w", err) - } - } + controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) // Create or update Karpenter custom resources in the workload cluster. if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") - - // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with error conditions") - } - return reconcile.Result{}, err } // Reconcile bootstrap data - fetch secret and upload to S3 if changed if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) - - // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with bootstrap error conditions") - } - return reconcile.Result{}, err } conditions.MarkBootstrapDataReady(karpenterMachinePool) - // Persist bootstrap data success condition immediately - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with bootstrap data success condition") - return reconcile.Result{}, fmt.Errorf("failed to persist bootstrap data success condition: %w", statusErr) - } - // Update status with current node information from the workload cluster if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to save Karpenter instances to status") - - // Ensure conditions are persisted even when errors occur - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with conditions before returning error") - } - return reconcile.Result{}, err } // Mark the KarpenterMachinePool as ready when all conditions are satisfied conditions.MarkKarpenterMachinePoolReady(karpenterMachinePool) - // Update the status to persist the Ready condition - if err := r.client.Status().Update(ctx, karpenterMachinePool); err != nil { - logger.Error(err, "failed to update karpenterMachinePool status with Ready condition") - return reconcile.Result{}, err - } - return reconcile.Result{}, nil } // saveKarpenterInstancesToStatus updates the KarpenterMachinePool and parent MachinePool with current node information // from the workload cluster, including replica counts and provider ID lists. func (r *KarpenterMachinePoolReconciler) saveKarpenterInstancesToStatus(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { - // Create a copy for tracking changes - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - // Get current node information from the workload cluster providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) if err != nil { @@ -222,23 +200,11 @@ func (r *KarpenterMachinePoolReconciler) saveKarpenterInstancesToStatus(ctx cont // Update KarpenterMachinePool status with current replica count karpenterMachinePool.Status.Replicas = numberOfNodeClaims - logger.Info("Found NodeClaims in workload cluster, patching KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) - - // Patch the status with the updated replica count - if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.status.Replicas") - return err - } + logger.Info("Found NodeClaims in workload cluster, updating KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) // Update KarpenterMachinePool spec with current provider ID list karpenterMachinePool.Spec.ProviderIDList = providerIDList - // Patch the spec with the updated provider ID list - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.spec.providerIDList") - return err - } - // Update the parent MachinePool replica count to match actual node claims if machinePool.Spec.Replicas == nil || *machinePool.Spec.Replicas != numberOfNodeClaims { machinePoolCopy := machinePool.DeepCopy() @@ -289,19 +255,11 @@ func (r *KarpenterMachinePoolReconciler) reconcileMachinePoolBootstrapUserData(c return err } - // Create copy for patching annotations - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - // Update the hash annotation to track the current bootstrap data version if karpenterMachinePool.Annotations == nil { karpenterMachinePool.Annotations = make(map[string]string) } karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] = bootstrapUserDataHash - - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.annotations with user data hash", "annotation", BootstrapDataHashAnnotation) - return err - } } return nil @@ -310,7 +268,7 @@ func (r *KarpenterMachinePoolReconciler) reconcileMachinePoolBootstrapUserData(c // reconcileDelete deletes the karpenter custom resources from the workload cluster. // When the cluster itself is being deleted, it also terminates all EC2 instances // created by Karpenter to prevent orphaned resources. -func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, roleIdentity *capa.AWSClusterRoleIdentity) (reconcile.Result, error) { +func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, roleIdentity *capa.AWSClusterRoleIdentity, patchHelper *patch.Helper) (reconcile.Result, error) { // We check if the owner Cluster is also being deleted (on top of the `KarpenterMachinePool` being deleted). // If the Cluster is being deleted, we terminate all the ec2 instances that karpenter may have launched. // These are normally removed by Karpenter, but when deleting a cluster, karpenter may not have enough time to clean them up. @@ -339,15 +297,8 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo return reconcile.Result{}, err } - // Create a deep copy of the reconciled object so we can change it - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - logger.Info("Removing finalizer", "finalizer", KarpenterFinalizer) controllerutil.RemoveFinalizer(karpenterMachinePool, KarpenterFinalizer) - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { - logger.Error(err, "failed to remove finalizer", "finalizer", KarpenterFinalizer) - return reconcile.Result{}, err - } return reconcile.Result{}, nil } @@ -459,12 +410,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co // Mark version skew as valid conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) - // Persist version skew success condition immediately - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with version skew success condition") - return fmt.Errorf("failed to persist version skew success condition: %w", statusErr) - } - workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { return fmt.Errorf("failed to get workload cluster client: %w", err) @@ -477,12 +422,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } conditions.MarkEC2NodeClassCreated(karpenterMachinePool) - // Persist EC2NodeClass success condition immediately - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with EC2NodeClass success condition") - return fmt.Errorf("failed to persist EC2NodeClass success condition: %w", statusErr) - } - // Create or update NodePool if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, fmt.Sprintf("%v", err)) @@ -490,12 +429,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx co } conditions.MarkNodePoolCreated(karpenterMachinePool) - // Persist NodePool success condition immediately - if statusErr := r.client.Status().Update(ctx, karpenterMachinePool); statusErr != nil { - logger.Error(statusErr, "failed to update karpenterMachinePool status with NodePool success condition") - return fmt.Errorf("failed to persist NodePool success condition: %w", statusErr) - } - return nil } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 518ef9f6..dd08c359 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -1025,6 +1025,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err = unstructured.SetNestedField(nodeClaim1.Object, map[string]interface{}{"providerID": "aws:///us-west-2a/i-1234567890abcdef0"}, "status") Expect(err).NotTo(HaveOccurred()) err = k8sClient.Status().Update(ctx, nodeClaim1) + Expect(err).NotTo(HaveOccurred()) nodeClaim2 := &unstructured.Unstructured{} nodeClaim2.Object = map[string]interface{}{ From b40b8d2926837be9e87e23ded5038d9a62ce75c2 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Sun, 27 Jul 2025 13:50:05 +0200 Subject: [PATCH 27/41] Improve condition handling --- .../karpentermachinepool_controller.go | 13 +++- .../karpentermachinepool_controller_test.go | 71 +++++++++++-------- pkg/conditions/conditions.go | 26 +++---- 3 files changed, 63 insertions(+), 47 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 1f8c5214..b0b16bf9 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -101,6 +101,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco machinePool, err := capiutilexp.GetOwnerMachinePool(ctx, r.client, karpenterMachinePool.ObjectMeta) if err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get MachinePool owning the KarpenterMachinePool: %v", err)) return reconcile.Result{}, fmt.Errorf("failed to get MachinePool owning the KarpenterMachinePool: %w", err) } if machinePool == nil { @@ -113,6 +114,8 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // Bootstrap data must be available before we can proceed with creating Karpenter resources if machinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretMissingReferenceReason, "Bootstrap data secret reference is not yet available in MachinePool") + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, "Bootstrap data secret reference is not yet available in MachinePool") logger.Info("Bootstrap data secret reference is not yet available") return reconcile.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } @@ -121,6 +124,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco cluster, err := capiutil.GetClusterFromMetadata(ctx, r.client, machinePool.ObjectMeta) if err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get Cluster owning the MachinePool: %v", err)) return reconcile.Result{}, fmt.Errorf("failed to get Cluster owning the MachinePool that owns the KarpenterMachinePool: %w", err) } @@ -133,6 +137,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco awsCluster := &capa.AWSCluster{} if err := r.client.Get(ctx, client.ObjectKey{Namespace: cluster.Spec.InfrastructureRef.Namespace, Name: cluster.Spec.InfrastructureRef.Name}, awsCluster); err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get AWSCluster: %v", err)) return reconcile.Result{}, fmt.Errorf("failed to get AWSCluster referenced in Cluster.spec.infrastructureRef: %w", err) } @@ -143,12 +148,14 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // S3 bucket is required for storing bootstrap data that Karpenter nodes will fetch if awsCluster.Spec.S3Bucket == nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, "S3 bucket is required but not configured in AWSCluster.spec.s3Bucket") return reconcile.Result{}, errors.New("a cluster wide object storage configured at `AWSCluster.spec.s3Bucket` is required") } // Get AWS credentials for S3 and EC2 operations roleIdentity := &capa.AWSClusterRoleIdentity{} if err = r.client.Get(ctx, client.ObjectKey{Name: awsCluster.Spec.IdentityRef.Name}, roleIdentity); err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get AWSClusterRoleIdentity: %v", err)) return reconcile.Result{}, fmt.Errorf("failed to get AWSClusterRoleIdentity referenced in AWSCluster: %w", err) } @@ -157,21 +164,20 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity, patchHelper) } - // Initialize conditions - mark as initializing until all steps complete - conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.InitializingReason, "KarpenterMachinePool is being initialized") - // Add finalizer to ensure proper cleanup sequence controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) // Create or update Karpenter custom resources in the workload cluster. if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to create or update Karpenter resources: %v", err)) return reconcile.Result{}, err } // Reconcile bootstrap data - fetch secret and upload to S3 if changed if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) return reconcile.Result{}, err } conditions.MarkBootstrapDataReady(karpenterMachinePool) @@ -179,6 +185,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // Update status with current node information from the workload cluster if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to save Karpenter instances to status") + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to save Karpenter instances to status: %v", err)) return reconcile.Result{}, err } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index dd08c359..3231be2c 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -427,7 +428,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capiexp.MachinePoolSpec{ ClusterName: ClusterName, - // Replicas: nil, Template: capi.MachineTemplateSpec{ ObjectMeta: capi.ObjectMeta{}, Spec: capi.MachineSpec{ @@ -499,6 +499,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) It("returns early", func() { Expect(reconcileErr).NotTo(HaveOccurred()) + + updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("BootstrapDataReady", v1.ConditionFalse, "BootstrapDataSecretMissingReference", "Bootstrap data secret reference is not yet available in MachinePool")) }) }) When("the referenced MachinePool exists and MachinePool.spec.template.spec.bootstrap.dataSecretName is set", func() { @@ -1066,10 +1071,9 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(err).NotTo(HaveOccurred()) // Check that the Ready condition is True - readyCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "Ready") - Expect(readyCondition).NotTo(BeNil()) - Expect(string(readyCondition.Status)).To(Equal("True")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("Ready", v1.ConditionTrue, "Ready", "")) + // Check karpenter machine pool spec and status Expect(updatedKarpenterMachinePool.Status.Replicas).To(Equal(int32(2))) Expect(updatedKarpenterMachinePool.Spec.ProviderIDList).To(ContainElements("aws:///us-west-2a/i-1234567890abcdef0", "aws:///us-west-2a/i-09876543219fedcba")) }) @@ -1514,22 +1518,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(err).NotTo(HaveOccurred()) // This condition should be properly persisted even though reconciliation failed - ec2NodeClassCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "EC2NodeClassCreated") - Expect(ec2NodeClassCondition).NotTo(BeNil(), "EC2NodeClass condition should be persisted even on error") - Expect(string(ec2NodeClassCondition.Status)).To(Equal("True"), "EC2NodeClass was created successfully") - Expect(ec2NodeClassCondition.Reason).To(Equal("EC2NodeClassCreated")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("EC2NodeClassCreated", v1.ConditionTrue, "EC2NodeClassCreated", "")) // Version skew should be valid since we use the same version - versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewPolicySatisfied") - Expect(versionSkewCondition).NotTo(BeNil(), "VersionSkew condition should be persisted even on error") - Expect(string(versionSkewCondition.Status)).To(Equal("True"), "Version skew should be valid") - Expect(versionSkewCondition.Reason).To(Equal("VersionSkewValid")) - - // NodePool should be False since creation failed - nodePoolCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "NodePoolCreated") - Expect(nodePoolCondition).NotTo(BeNil(), "NodePool condition should be persisted even on error") - Expect(string(nodePoolCondition.Status)).To(Equal("False"), "NodePool creation failed") - Expect(nodePoolCondition.Reason).To(Equal("NodePoolCreationFailed")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("VersionSkewPolicySatisfied", v1.ConditionTrue, "VersionSkewValid", "")) + + // NodePoolCreated should be False since creation failed + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("NodePoolCreated", v1.ConditionFalse, "NodePoolCreationFailed", "failed to create or update NodePool")) }) }) @@ -1717,23 +1712,14 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(err).NotTo(HaveOccurred()) // Verify that version skew condition was persisted with the correct state - versionSkewCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "VersionSkewPolicySatisfied") - Expect(versionSkewCondition).NotTo(BeNil()) - Expect(string(versionSkewCondition.Status)).To(Equal("False")) - Expect(versionSkewCondition.Reason).To(Equal("VersionSkewBlocked")) - Expect(versionSkewCondition.Message).To(ContainSubstring("control plane version v1.29.0 is older than node pool version v1.30.0")) + // Expect(versionSkewCondition.Message).To(ContainSubstring("control plane version v1.29.0 is older than node pool version v1.30.0")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("VersionSkewPolicySatisfied", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) // Verify that EC2NodeClass condition was persisted with error state - ec2NodeClassCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "EC2NodeClassCreated") - Expect(ec2NodeClassCondition).NotTo(BeNil()) - Expect(string(ec2NodeClassCondition.Status)).To(Equal("False")) - Expect(ec2NodeClassCondition.Reason).To(Equal("VersionSkewBlocked")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("EC2NodeClassCreated", v1.ConditionFalse, "VersionSkewBlocked", "")) // Verify that NodePool condition was persisted with error state - nodePoolCondition := findCondition(updatedKarpenterMachinePool.Status.Conditions, "NodePoolCreated") - Expect(nodePoolCondition).NotTo(BeNil()) - Expect(string(nodePoolCondition.Status)).To(Equal("False")) - Expect(nodePoolCondition.Reason).To(Equal("VersionSkewBlocked")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("NodePoolCreated", v1.ConditionFalse, "VersionSkewBlocked", "")) }) }) }) @@ -1746,3 +1732,26 @@ func ExpectUnstructured(u unstructured.Unstructured, fields ...string) Assertion Expect(err).NotTo(HaveOccurred(), "error retrieving %v: %v", fields, err) return Expect(v) } + +// HaveCondition checks for a Condition with the given Type, Status, and Reason. +func HaveCondition(condType capi.ConditionType, status v1.ConditionStatus, reason, message string) gomegatypes.GomegaMatcher { + return WithTransform(func(conditions capi.Conditions) *capi.Condition { + for i := range conditions { + if conditions[i].Type == condType { + return &conditions[i] + } + } + return nil + }, And( + Not(BeNil()), + WithTransform(func(c *capi.Condition) v1.ConditionStatus { + return c.Status + }, Equal(status)), + WithTransform(func(c *capi.Condition) string { + return c.Reason + }, Equal(reason)), + WithTransform(func(c *capi.Condition) string { + return c.Message + }, ContainSubstring(message)), + )) +} diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 9f47b2a5..dbea1ad6 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -40,21 +40,21 @@ const ( // Condition reasons used by various controllers const ( // Generic reasons used across controllers - InitializingReason = "Initializing" - ReadyReason = "Ready" - NotReadyReason = "NotReady" + ReadyReason = "Ready" + NotReadyReason = "NotReady" // KarpenterMachinePool controller reasons - NodePoolCreationFailedReason = "NodePoolCreationFailed" - NodePoolCreationSucceededReason = "NodePoolCreated" - EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" - EC2NodeClassCreationSucceededReason = "EC2NodeClassCreated" - BootstrapDataUploadFailedReason = "BootstrapDataUploadFailed" - BootstrapDataSecretNotFoundReason = "BootstrapDataSecretNotFound" - BootstrapDataSecretInvalidReason = "BootstrapDataSecretInvalid" - BootstrapDataUploadSucceededReason = "BootstrapDataUploaded" - VersionSkewBlockedReason = "VersionSkewBlocked" - VersionSkewValidReason = "VersionSkewValid" + NodePoolCreationFailedReason = "NodePoolCreationFailed" + NodePoolCreationSucceededReason = "NodePoolCreated" + EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" + EC2NodeClassCreationSucceededReason = "EC2NodeClassCreated" + BootstrapDataUploadFailedReason = "BootstrapDataUploadFailed" + BootstrapDataSecretNotFoundReason = "BootstrapDataSecretNotFound" + BootstrapDataSecretInvalidReason = "BootstrapDataSecretInvalid" + BootstrapDataSecretMissingReferenceReason = "BootstrapDataSecretMissingReference" + BootstrapDataUploadSucceededReason = "BootstrapDataUploaded" + VersionSkewBlockedReason = "VersionSkewBlocked" + VersionSkewValidReason = "VersionSkewValid" ) func MarkReady(setter capiconditions.Setter, condition capi.ConditionType) { From 99941c1322d72e424475f684219c6e9e19200787 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Mon, 28 Jul 2025 14:49:46 +0200 Subject: [PATCH 28/41] Reconcile AWS metadata endpoint config --- controllers/karpentermachinepool_controller.go | 9 +++------ controllers/karpentermachinepool_controller_test.go | 11 +++++++++++ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index b0b16bf9..ed0fd710 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -455,21 +455,18 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. ec2NodeClass.SetNamespace("") ec2NodeClass.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) - // Generate user data for Ignition + // Generate Ignition user data userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, karpenterMachinePool.Name) operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { - // Build the EC2NodeClass spec spec := map[string]interface{}{ "amiFamily": "Custom", "amiSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.AMISelectorTerms, "blockDeviceMappings": karpenterMachinePool.Spec.EC2NodeClass.BlockDeviceMappings, "instanceProfile": karpenterMachinePool.Spec.EC2NodeClass.InstanceProfile, "metadataOptions": map[string]interface{}{ - "httpEndpoint": "enabled", - "httpProtocolIPv6": "disabled", - "httpPutResponseHopLimit": 1, - "httpTokens": "required", + "httpPutResponseHopLimit": karpenterMachinePool.Spec.EC2NodeClass.MetadataOptions.HTTPPutResponseHopLimit, + "httpTokens": karpenterMachinePool.Spec.EC2NodeClass.MetadataOptions.HTTPTokens, }, "securityGroupSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroupSelectorTerms, "subnetSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SubnetSelectorTerms, diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 3231be2c..82c2c729 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -556,6 +556,8 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { volumeSize := resource.MustParse("8Gi") volumeTypeGp3 := "gp3" deleteOnTerminationTrue := true + hopLimit := int64(5) + metadataOptionsRequired := "required" karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ ObjectMeta: ctrl.ObjectMeta{ Namespace: namespace, @@ -592,6 +594,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, InstanceProfile: &instanceProfile, + MetadataOptions: &karpenterinfra.MetadataOptions{ + HTTPPutResponseHopLimit: &hopLimit, + HTTPTokens: &metadataOptionsRequired, + }, SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ { Tags: map[string]string{"my-target-sg": "is-this"}, @@ -921,6 +927,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ), ) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "metadataOptions"). + To(HaveKeyWithValue("httpTokens", "required")) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "metadataOptions"). + To(HaveKeyWithValue("httpPutResponseHopLimit", int64(5))) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To(HaveLen(1)) ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To( ContainElement( // slice matcher: at least one element matches From daf0289b8aef306cfc14d22477ef1700c1d86977 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Mon, 28 Jul 2025 16:16:51 +0200 Subject: [PATCH 29/41] Bring back status.ready field --- api/v1alpha1/karpentermachinepool_types.go | 4 ++++ ...ructure.cluster.x-k8s.io_karpentermachinepools.yaml | 4 ++++ controllers/karpentermachinepool_controller.go | 10 ++++++++++ controllers/karpentermachinepool_controller_test.go | 1 + ...ructure.cluster.x-k8s.io_karpentermachinepools.yaml | 4 ++++ 5 files changed, 23 insertions(+) diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index c10e4b6b..1ae569fa 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -39,6 +39,10 @@ type KarpenterMachinePoolSpec struct { // KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. type KarpenterMachinePoolStatus struct { + // Ready denotes that the KarpenterMachinePool is ready and fulfilling the infrastructure contract. + // +optional + Ready bool `json:"ready"` + // Replicas is the most recently observed number of replicas // +optional Replicas int32 `json:"replicas"` diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 27f0c8fd..eeee82cf 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -1016,6 +1016,10 @@ spec: - type type: object type: array + ready: + description: Ready denotes that the KarpenterMachinePool is ready + and fulfilling the infrastructure contract. + type: boolean replicas: description: Replicas is the most recently observed number of replicas format: int32 diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index ed0fd710..a2459f97 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -102,6 +102,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco machinePool, err := capiutilexp.GetOwnerMachinePool(ctx, r.client, karpenterMachinePool.ObjectMeta) if err != nil { conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get MachinePool owning the KarpenterMachinePool: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get MachinePool owning the KarpenterMachinePool: %w", err) } if machinePool == nil { @@ -116,6 +117,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco if machinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil { conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretMissingReferenceReason, "Bootstrap data secret reference is not yet available in MachinePool") conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, "Bootstrap data secret reference is not yet available in MachinePool") + karpenterMachinePool.Status.Ready = false logger.Info("Bootstrap data secret reference is not yet available") return reconcile.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } @@ -125,6 +127,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco cluster, err := capiutil.GetClusterFromMetadata(ctx, r.client, machinePool.ObjectMeta) if err != nil { conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get Cluster owning the MachinePool: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get Cluster owning the MachinePool that owns the KarpenterMachinePool: %w", err) } @@ -138,6 +141,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco awsCluster := &capa.AWSCluster{} if err := r.client.Get(ctx, client.ObjectKey{Namespace: cluster.Spec.InfrastructureRef.Namespace, Name: cluster.Spec.InfrastructureRef.Name}, awsCluster); err != nil { conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get AWSCluster: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get AWSCluster referenced in Cluster.spec.infrastructureRef: %w", err) } @@ -149,6 +153,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // S3 bucket is required for storing bootstrap data that Karpenter nodes will fetch if awsCluster.Spec.S3Bucket == nil { conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, "S3 bucket is required but not configured in AWSCluster.spec.s3Bucket") + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, errors.New("a cluster wide object storage configured at `AWSCluster.spec.s3Bucket` is required") } @@ -156,6 +161,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco roleIdentity := &capa.AWSClusterRoleIdentity{} if err = r.client.Get(ctx, client.ObjectKey{Name: awsCluster.Spec.IdentityRef.Name}, roleIdentity); err != nil { conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get AWSClusterRoleIdentity: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get AWSClusterRoleIdentity referenced in AWSCluster: %w", err) } @@ -171,6 +177,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to create or update Karpenter resources: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, err } @@ -178,6 +185,7 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, err } conditions.MarkBootstrapDataReady(karpenterMachinePool) @@ -186,11 +194,13 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { logger.Error(err, "failed to save Karpenter instances to status") conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to save Karpenter instances to status: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, err } // Mark the KarpenterMachinePool as ready when all conditions are satisfied conditions.MarkKarpenterMachinePoolReady(karpenterMachinePool) + karpenterMachinePool.Status.Ready = true return reconcile.Result{}, nil } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 82c2c729..c743dad7 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -1083,6 +1083,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { // Check that the Ready condition is True Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("Ready", v1.ConditionTrue, "Ready", "")) + Expect(updatedKarpenterMachinePool.Status.Ready).To(BeTrue()) // Check karpenter machine pool spec and status Expect(updatedKarpenterMachinePool.Status.Replicas).To(Equal(int32(2))) diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 27f0c8fd..eeee82cf 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -1016,6 +1016,10 @@ spec: - type type: object type: array + ready: + description: Ready denotes that the KarpenterMachinePool is ready + and fulfilling the infrastructure contract. + type: boolean replicas: description: Replicas is the most recently observed number of replicas format: int32 From 23ead49ede28c77594ebbc6dbe5f659df22112b6 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 29 Jul 2025 00:27:35 +0200 Subject: [PATCH 30/41] Reconcile taints and startuptaints --- .../karpentermachinepool_controller.go | 1 + .../karpentermachinepool_controller_test.go | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index a2459f97..784efae0 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -577,6 +577,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont templateSpec := spec["template"].(map[string]interface{})["spec"].(map[string]interface{}) templateSpec["taints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints + templateSpec["startupTaints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.StartupTaints templateSpec["requirements"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements templateSpec["expireAfter"] = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index c743dad7..8da7f269 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -615,6 +615,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { NodePool: &karpenterinfra.NodePoolSpec{ Template: karpenterinfra.NodeClaimTemplate{ Spec: karpenterinfra.NodeClaimTemplateSpec{ + ExpireAfter: karpenterinfra.MustParseNillableDuration("24h"), Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ { NodeSelectorRequirement: v1.NodeSelectorRequirement{ @@ -624,7 +625,20 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, }, - ExpireAfter: karpenterinfra.MustParseNillableDuration("24h"), + StartupTaints: []v1.Taint{ + { + Key: "karpenter.sh/test-startup-taint", + Value: "test-taint-value", + Effect: v1.TaintEffectNoSchedule, + }, + }, + Taints: []v1.Taint{ + { + Key: "karpenter.sh/test-taint", + Value: "test-taint-value", + Effect: v1.TaintEffectNoSchedule, + }, + }, TerminationGracePeriod: &terminationGracePeriod, }, }, @@ -986,6 +1000,20 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { ExpectUnstructured(nodepoolList.Items[0], "spec", "weight").To(BeEquivalentTo(int64(1))) ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "expireAfter").To(BeEquivalentTo("24h")) ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "terminationGracePeriod").To(BeEquivalentTo("30s")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "startupTaints").To(BeEquivalentTo([]interface{}{ + map[string]interface{}{ + "key": "karpenter.sh/test-startup-taint", + "value": "test-taint-value", + "effect": "NoSchedule", + }, + })) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "taints").To(BeEquivalentTo([]interface{}{ + map[string]interface{}{ + "key": "karpenter.sh/test-taint", + "value": "test-taint-value", + "effect": "NoSchedule", + }, + })) }) It("adds the finalizer to the KarpenterMachinePool", func() { Expect(reconcileErr).NotTo(HaveOccurred()) From 810440b16bc927ea6aa56cd6c768495738517486 Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 29 Jul 2025 15:02:56 +0200 Subject: [PATCH 31/41] Extract version skew to its own package --- .../karpentermachinepool_controller.go | 35 +++---- pkg/versionskew/skew.go | 29 ++++++ pkg/versionskew/skew_test.go | 99 +++++++++++++++++++ 3 files changed, 141 insertions(+), 22 deletions(-) create mode 100644 pkg/versionskew/skew.go create mode 100644 pkg/versionskew/skew_test.go diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 784efae0..5bc67989 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -9,7 +9,6 @@ import ( "path" "time" - "github.com/blang/semver/v4" "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -35,6 +34,7 @@ import ( "github.com/aws-resolver-rules-operator/api/v1alpha1" "github.com/aws-resolver-rules-operator/pkg/conditions" "github.com/aws-resolver-rules-operator/pkg/resolver" + "github.com/aws-resolver-rules-operator/pkg/versionskew" ) const ( @@ -208,18 +208,14 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco // saveKarpenterInstancesToStatus updates the KarpenterMachinePool and parent MachinePool with current node information // from the workload cluster, including replica counts and provider ID lists. func (r *KarpenterMachinePoolReconciler) saveKarpenterInstancesToStatus(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { - // Get current node information from the workload cluster providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) if err != nil { return err } - // Update KarpenterMachinePool status with current replica count - karpenterMachinePool.Status.Replicas = numberOfNodeClaims - - logger.Info("Found NodeClaims in workload cluster, updating KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) + logger.Info("Updating MachinePool.spec.replicas, KarpenterMachinePool.spec.ProviderIDList and KarpenterMachinePool.status.Replicas", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) - // Update KarpenterMachinePool spec with current provider ID list + karpenterMachinePool.Status.Replicas = numberOfNodeClaims karpenterMachinePool.Spec.ProviderIDList = providerIDList // Update the parent MachinePool replica count to match actual node claims @@ -357,7 +353,7 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo logger.Error(err, "error retrieving nodeClaim.status.providerID", "nodeClaim", nc.GetName()) continue } - logger.Info("nodeClaim.status.providerID", "nodeClaimName", nc.GetName(), "statusFieldFound", found, "nodeClaim", nc.Object) + if found && providerID != "" { providerIDList = append(providerIDList, providerID) } @@ -494,9 +490,9 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context. switch operation { case controllerutil.OperationResultCreated: - logger.Info("Created EC2NodeClass") + logger.Info("Created EC2NodeClass in workload cluster") case controllerutil.OperationResultUpdated: - logger.Info("Updated EC2NodeClass") + logger.Info("Updated EC2NodeClass in workload cluster") } return nil @@ -556,7 +552,6 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont "disruption": map[string]interface{}{}, } - // Apply user-defined NodePool configuration if provided if karpenterMachinePool.Spec.NodePool != nil { dis := spec["disruption"].(map[string]interface{}) dis["budgets"] = karpenterMachinePool.Spec.NodePool.Disruption.Budgets @@ -597,9 +592,9 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont switch operation { case controllerutil.OperationResultCreated: - logger.Info("Created NodePool") + logger.Info("Created NodePool in workload cluster") case controllerutil.OperationResultUpdated: - logger.Info("Updated NodePool") + logger.Info("Updated NodePool in workload cluster") } return nil @@ -704,17 +699,13 @@ func (r *KarpenterMachinePoolReconciler) IsVersionSkewAllowed(ctx context.Contex return true, "", "", fmt.Errorf("failed to get current Control Plane k8s version: %w", err) } - // Parse versions using semantic versioning for proper comparison - controlPlaneCurrentK8sVersion, err := semver.ParseTolerant(controlPlaneVersion) - if err != nil { - return true, "", "", fmt.Errorf("failed to parse current Control Plane k8s version: %w", err) - } + desiredWorkerVersion := *machinePool.Spec.Template.Spec.Version - machinePoolDesiredK8sVersion, err := semver.ParseTolerant(*machinePool.Spec.Template.Spec.Version) + // Use the version package to check skew policy + allowed, err := versionskew.IsSkewAllowed(controlPlaneVersion, desiredWorkerVersion) if err != nil { - return true, controlPlaneVersion, "", fmt.Errorf("failed to parse node pool desired k8s version: %w", err) + return true, controlPlaneVersion, desiredWorkerVersion, fmt.Errorf("failed to validate version skew: %w", err) } - // Allow if control plane version >= desired worker version - return controlPlaneCurrentK8sVersion.GE(machinePoolDesiredK8sVersion), controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, nil + return allowed, controlPlaneVersion, desiredWorkerVersion, nil } diff --git a/pkg/versionskew/skew.go b/pkg/versionskew/skew.go new file mode 100644 index 00000000..abb83c33 --- /dev/null +++ b/pkg/versionskew/skew.go @@ -0,0 +1,29 @@ +package versionskew + +import ( + "fmt" + + "github.com/blang/semver/v4" +) + +// IsSkewAllowed checks if the worker version can be updated based on the control plane version. +// The workers can't use a newer k8s version than the one used by the control plane. +// +// This implements Kubernetes version skew policy https://kubernetes.io/releases/version-skew-policy/ +// +// Returns: (allowed bool, error) +func IsSkewAllowed(controlPlaneVersion, workerVersion string) (bool, error) { + // Parse versions using semantic versioning for proper comparison + controlPlaneCurrentK8sVersion, err := semver.ParseTolerant(controlPlaneVersion) + if err != nil { + return false, fmt.Errorf("failed to parse control plane k8s version %q: %w", controlPlaneVersion, err) + } + + workerDesiredK8sVersion, err := semver.ParseTolerant(workerVersion) + if err != nil { + return false, fmt.Errorf("failed to parse worker desired k8s version %q: %w", workerVersion, err) + } + + // Allow if control plane version >= desired worker version + return controlPlaneCurrentK8sVersion.GE(workerDesiredK8sVersion), nil +} diff --git a/pkg/versionskew/skew_test.go b/pkg/versionskew/skew_test.go new file mode 100644 index 00000000..65d2c21e --- /dev/null +++ b/pkg/versionskew/skew_test.go @@ -0,0 +1,99 @@ +package versionskew + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVersionSkew(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VersionSkew Suite") +} + +var _ = Describe("IsSkewAllowed", func() { + Context("when control plane version is higher than worker version", func() { + It("should allow the skew", func() { + allowed, err := IsSkewAllowed("1.28.0", "1.27.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + + It("should allow the skew with patch versions", func() { + allowed, err := IsSkewAllowed("1.28.5", "1.28.3") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + }) + + Context("when control plane version equals worker version", func() { + It("should allow the skew", func() { + allowed, err := IsSkewAllowed("1.28.0", "1.28.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + + It("should allow the skew with patch versions", func() { + allowed, err := IsSkewAllowed("1.28.5", "1.28.5") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + }) + + Context("when control plane version is lower than worker version", func() { + It("should not allow the skew", func() { + allowed, err := IsSkewAllowed("1.27.0", "1.28.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeFalse()) + }) + + It("should not allow the skew with patch versions", func() { + allowed, err := IsSkewAllowed("1.28.3", "1.28.5") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeFalse()) + }) + + It("should not allow the skew with minor version difference", func() { + allowed, err := IsSkewAllowed("1.27.10", "1.28.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeFalse()) + }) + }) + + Context("when parsing version strings with prefixes", func() { + It("should handle v prefixes correctly", func() { + allowed, err := IsSkewAllowed("v1.28.0", "v1.27.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + + It("should handle mixed prefixes correctly", func() { + allowed, err := IsSkewAllowed("v1.28.0", "1.27.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + }) + + Context("when version strings are invalid", func() { + It("should return error for invalid control plane version", func() { + _, err := IsSkewAllowed("invalid-version", "1.28.0") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to parse control plane k8s version")) + }) + + It("should return error for invalid worker version", func() { + _, err := IsSkewAllowed("1.28.0", "invalid-version") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to parse worker desired k8s version")) + }) + + It("should return error for empty versions", func() { + _, err := IsSkewAllowed("", "1.28.0") + Expect(err).To(HaveOccurred()) + + _, err = IsSkewAllowed("1.28.0", "") + Expect(err).To(HaveOccurred()) + }) + }) +}) From da0c949a54b88d529fc79e8f8a2bb3fc533b14ae Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 29 Jul 2025 15:42:14 +0200 Subject: [PATCH 32/41] Requeue instead of returning error when version skew policy does not allow workers upgrade --- .../karpentermachinepool_controller.go | 59 ++++++++----------- .../karpentermachinepool_controller_test.go | 7 +-- 2 files changed, 29 insertions(+), 37 deletions(-) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 5bc67989..57e2c577 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -170,6 +170,29 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity, patchHelper) } + // Validate version skew: ensure worker nodes don't use newer Kubernetes versions than control plane + allowed, controlPlaneCurrentVersion, nodePoolDesiredVersion, err := r.IsVersionSkewAllowed(ctx, cluster, machinePool) + if err != nil { + return reconcile.Result{}, err + } + + if !allowed { + message := fmt.Sprintf("Version skew policy violation: control plane version %s is older than node pool version %s", controlPlaneCurrentVersion, nodePoolDesiredVersion) + logger.Info("Blocking Karpenter custom resources update due to version skew policy", + "controlPlaneCurrentVersion", controlPlaneCurrentVersion, + "nodePoolDesiredVersion", nodePoolDesiredVersion, + "reason", message) + + // Mark version skew as invalid and resources as not ready + conditions.MarkVersionSkewInvalid(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + + return reconcile.Result{RequeueAfter: time.Duration(1) * time.Minute}, nil + } + + // Mark version skew as valid + conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) // Add finalizer to ensure proper cleanup sequence controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) @@ -397,32 +420,7 @@ func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Cont } // createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass custom resources in the workload cluster. -// This method enforces version skew policies and sets appropriate conditions based on success/failure states. func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { - // Validate version skew: ensure worker nodes don't use newer Kubernetes versions than control plane - allowed, controlPlaneCurrentVersion, nodePoolDesiredVersion, err := r.IsVersionSkewAllowed(ctx, cluster, machinePool) - if err != nil { - return err - } - - if !allowed { - message := fmt.Sprintf("Version skew policy violation: control plane version %s is older than node pool version %s", controlPlaneCurrentVersion, nodePoolDesiredVersion) - logger.Info("Blocking Karpenter custom resources update due to version skew policy", - "controlPlaneCurrentVersion", controlPlaneCurrentVersion, - "nodePoolDesiredVersion", nodePoolDesiredVersion, - "reason", message) - - // Mark version skew as invalid and resources as not ready - conditions.MarkVersionSkewInvalid(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) - conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) - conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) - - return fmt.Errorf("version skew policy violation: %s", message) - } - - // Mark version skew as valid - conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) - workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) if err != nil { return fmt.Errorf("failed to get workload cluster client: %w", err) @@ -691,21 +689,16 @@ func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, m // The workers can't use a newer k8s version than the one used by the control plane. // // This implements Kubernetes version skew policy https://kubernetes.io/releases/version-skew-policy/ -// -// Returns: (allowed bool, controlPlaneVersion string, desiredWorkerVersion string, error) func (r *KarpenterMachinePoolReconciler) IsVersionSkewAllowed(ctx context.Context, cluster *capi.Cluster, machinePool *capiexp.MachinePool) (bool, string, string, error) { controlPlaneVersion, err := r.getControlPlaneVersion(ctx, cluster) if err != nil { return true, "", "", fmt.Errorf("failed to get current Control Plane k8s version: %w", err) } - desiredWorkerVersion := *machinePool.Spec.Template.Spec.Version - - // Use the version package to check skew policy - allowed, err := versionskew.IsSkewAllowed(controlPlaneVersion, desiredWorkerVersion) + allowed, err := versionskew.IsSkewAllowed(controlPlaneVersion, *machinePool.Spec.Template.Spec.Version) if err != nil { - return true, controlPlaneVersion, desiredWorkerVersion, fmt.Errorf("failed to validate version skew: %w", err) + return true, controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, fmt.Errorf("failed to validate version skew: %w", err) } - return allowed, controlPlaneVersion, desiredWorkerVersion, nil + return allowed, controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, nil } diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 8da7f269..24e6b067 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -1741,8 +1741,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) It("returns a version skew error", func() { - Expect(reconcileErr).To(MatchError(ContainSubstring("version skew policy violation"))) - Expect(reconcileErr).To(MatchError(ContainSubstring("control plane version v1.29.0 is older than node pool version v1.30.0"))) + Expect(reconcileResult.RequeueAfter).To(Equal(60 * time.Second)) }) It("persists the version skew conditions to the Kubernetes API", func() { @@ -1756,10 +1755,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("VersionSkewPolicySatisfied", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) // Verify that EC2NodeClass condition was persisted with error state - Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("EC2NodeClassCreated", v1.ConditionFalse, "VersionSkewBlocked", "")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("EC2NodeClassCreated", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) // Verify that NodePool condition was persisted with error state - Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("NodePoolCreated", v1.ConditionFalse, "VersionSkewBlocked", "")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("NodePoolCreated", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) }) }) }) From df1bd017e877cfaea921e15891964d2fe967395a Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 29 Jul 2025 15:55:54 +0200 Subject: [PATCH 33/41] Add comment explaining goimports on generated files --- Makefile.custom.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.custom.mk b/Makefile.custom.mk index 076c6eb2..d09d3450 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -26,6 +26,7 @@ crds: controller-gen ## Generate CustomResourceDefinition. generate: controller-gen crds ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. go generate ./... $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." +# We need to run goimports after controller-gen to avoid CI complains about goimports in the generated files @go run golang.org/x/tools/cmd/goimports -w ./api/v1alpha1 .PHONY: create-acceptance-cluster From 5447edb66dbe2c166a5ff778b7269fc8e6485ad4 Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Fri, 22 Aug 2025 15:54:52 +0200 Subject: [PATCH 34/41] linting fixes --- api/v1alpha1/duration.go | 6 +++--- controllers/karpentermachinepool_controller.go | 2 +- controllers/karpentermachinepool_controller_test.go | 10 ---------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/api/v1alpha1/duration.go b/api/v1alpha1/duration.go index b30d2a8a..19a3508b 100644 --- a/api/v1alpha1/duration.go +++ b/api/v1alpha1/duration.go @@ -54,7 +54,7 @@ func (d NillableDuration) MarshalJSON() ([]byte, error) { return d.Raw, nil } if d.Duration != nil { - return json.Marshal(d.Duration.String()) + return json.Marshal((*d.Duration).String()) } return json.Marshal(Never) } @@ -69,12 +69,12 @@ func (d NillableDuration) ToUnstructured() interface{} { } // Fallback to string conversion if unmarshal fails if d.Duration != nil { - return d.Duration.String() + return (*d.Duration).String() } return Never } if d.Duration != nil { - return d.Duration.String() + return (*d.Duration).String() } return Never } diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 57e2c577..4ffce82c 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -565,7 +565,7 @@ func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Cont } templateMetadata := spec["template"].(map[string]interface{})["metadata"].(map[string]interface{}) - templateMetadata["labels"] = karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels + templateMetadata["labels"] = karpenterMachinePool.Spec.NodePool.Template.Labels templateSpec := spec["template"].(map[string]interface{})["spec"].(map[string]interface{}) diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 24e6b067..3c585810 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -44,16 +44,6 @@ const ( KubernetesVersion = "v1.29.1" ) -// findCondition returns the condition with the given type from the list of conditions. -func findCondition(conditions capi.Conditions, conditionType string) *capi.Condition { - for i := range conditions { - if conditions[i].Type == capi.ConditionType(conditionType) { - return &conditions[i] - } - } - return nil -} - var _ = Describe("KarpenterMachinePool reconciler", func() { var ( capiBootstrapSecretContent []byte From 5250cb3099d8c89324676d459f52ba7915471c83 Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Wed, 27 Aug 2025 10:03:14 +0200 Subject: [PATCH 35/41] revert makegen --- Makefile.gen.app.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.gen.app.mk b/Makefile.gen.app.mk index 26a4880a..3f8a89c9 100644 --- a/Makefile.gen.app.mk +++ b/Makefile.gen.app.mk @@ -22,6 +22,7 @@ lint-chart: check-env ## Runs ct against the default chart. rm -rf /tmp/$(APPLICATION)-test mkdir -p /tmp/$(APPLICATION)-test/helm cp -a ./helm/$(APPLICATION) /tmp/$(APPLICATION)-test/helm/ + architect helm template --dir /tmp/$(APPLICATION)-test/helm/$(APPLICATION) docker run -it --rm -v /tmp/$(APPLICATION)-test:/wd --workdir=/wd --name ct $(IMAGE) ct lint --validate-maintainers=false --charts="helm/$(APPLICATION)" rm -rf /tmp/$(APPLICATION)-test From f5865e734793c85d42a59ee3dbdb9121c968d8db Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Wed, 27 Aug 2025 10:03:48 +0200 Subject: [PATCH 36/41] revert makegen --- Makefile.gen.go.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile.gen.go.mk b/Makefile.gen.go.mk index b31216a2..bf02495a 100644 --- a/Makefile.gen.go.mk +++ b/Makefile.gen.go.mk @@ -11,6 +11,7 @@ GITSHA1 := $(shell git rev-parse --verify HEAD) MODULE := $(shell go list -m) OS := $(shell go env GOOS) SOURCES := $(shell find . -name '*.go') +VERSION := $(shell architect project version) ifeq ($(OS), linux) EXTLDFLAGS := -static From 31d711fb56762f9f95c650870ae4929056714ce1 Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Wed, 27 Aug 2025 10:16:10 +0200 Subject: [PATCH 37/41] file origins --- api/v1alpha1/ec2nodeclass.go | 10 ++++------ api/v1alpha1/nodepool.go | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/ec2nodeclass.go b/api/v1alpha1/ec2nodeclass.go index f5f70322..c0245b29 100644 --- a/api/v1alpha1/ec2nodeclass.go +++ b/api/v1alpha1/ec2nodeclass.go @@ -1,5 +1,7 @@ package v1alpha1 +// Taken from https://github.com/aws/karpenter-provider-aws/blob/main/pkg/apis/v1/ec2nodeclass.go + import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -56,13 +58,9 @@ type EC2NodeClassSpec struct { // this UserData to ensure nodes are being provisioned with the correct configuration. // +optional UserData *string `json:"userData,omitempty"` - // Role is the AWS identity that nodes use. This field is immutable. + // Role is the AWS identity that nodes use. // This field is mutually exclusive from instanceProfile. - // Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. - // This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented - // for the old instance profiles on an update. // +kubebuilder:validation:XValidation:rule="self != ''",message="role cannot be empty" - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="immutable field changed" // +optional Role string `json:"role,omitempty"` // InstanceProfile is the AWS entity that instances use. @@ -206,7 +204,7 @@ type AMISelectorTerm struct { // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" // +optional Owner string `json:"owner,omitempty"` - // SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. + //SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. // +optional SSMParameter string `json:"ssmParameter,omitempty"` } diff --git a/api/v1alpha1/nodepool.go b/api/v1alpha1/nodepool.go index 6ad9f961..6a0f794b 100644 --- a/api/v1alpha1/nodepool.go +++ b/api/v1alpha1/nodepool.go @@ -1,5 +1,7 @@ package v1alpha1 +// Crafted by hand from https://github.com/kubernetes-sigs/karpenter + import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" From 004ed32c9878ae74cc8924d4af9819454c1c9e9b Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Wed, 27 Aug 2025 10:16:31 +0200 Subject: [PATCH 38/41] not needed --- api/v1alpha1/ec2nodeclass.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/api/v1alpha1/ec2nodeclass.go b/api/v1alpha1/ec2nodeclass.go index c0245b29..46bcd5cb 100644 --- a/api/v1alpha1/ec2nodeclass.go +++ b/api/v1alpha1/ec2nodeclass.go @@ -429,21 +429,3 @@ const ( // pods that request ephemeral storage and container images that are downloaded to the node. InstanceStorePolicyRAID0 InstanceStorePolicy = "RAID0" ) - -// // EC2NodeClassSpec defines the configuration for a Karpenter EC2NodeClass -// type EC2NodeClassSpec struct { -// // Name is the ami name in EC2. -// // This value is the name field, which is different from the name tag. -// AMIName string `json:"amiName,omitempty"` -// // Owner is the owner for the ami. -// // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" -// AMIOwner string `json:"amiOwner,omitempty"` -// -// // SecurityGroups specifies the security groups to use -// // +optional -// SecurityGroups map[string]string `json:"securityGroups,omitempty"` -// -// // Subnets specifies the subnets to use -// // +optional -// Subnets map[string]string `json:"subnets,omitempty"` -// } From a2945e59dd0bee04279f084965340d69b32cf656 Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Wed, 27 Aug 2025 10:19:48 +0200 Subject: [PATCH 39/41] patch KMP --- controllers/karpentermachinepool_controller.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index 4ffce82c..7ab773d6 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -195,6 +195,10 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) // Add finalizer to ensure proper cleanup sequence controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) + // Patching the object to the API is required to ensure the finalizer is added. + if err := patchHelper.Patch(ctx, karpenterMachinePool); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to patch KarpenterMachinePool: %w", err) + } // Create or update Karpenter custom resources in the workload cluster. if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { From b57a3d85136b12d307246a5c5fec7c0e547ca07d Mon Sep 17 00:00:00 2001 From: Jose Armesto Date: Tue, 2 Sep 2025 15:14:23 +0200 Subject: [PATCH 40/41] Limit rbac permissions --- helm/aws-resolver-rules-operator/templates/rbac.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/helm/aws-resolver-rules-operator/templates/rbac.yaml b/helm/aws-resolver-rules-operator/templates/rbac.yaml index 5abd5efd..9e139a34 100644 --- a/helm/aws-resolver-rules-operator/templates/rbac.yaml +++ b/helm/aws-resolver-rules-operator/templates/rbac.yaml @@ -20,14 +20,18 @@ rules: - list - patch - watch - - update - apiGroups: - controlplane.cluster.x-k8s.io resources: - - awsmanagedcontrolplanes - kubeadmcontrolplanes verbs: - get + - apiGroups: + - controlplane.cluster.x-k8s.io + resources: + - awsmanagedcontrolplanes + verbs: + - get - list - patch - watch From 9e80301e751e66aab8308ece8ab979448946871f Mon Sep 17 00:00:00 2001 From: Pau Rosello Date: Tue, 2 Sep 2025 15:28:54 +0200 Subject: [PATCH 41/41] remove hold from build --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b8bb3c0b..b1b23519 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -74,8 +74,6 @@ workflows: only: /^v.*/ - architect/go-build: - requires: - - hold context: architect name: go-build binary: aws-resolver-rules-operator