diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go index 6b7d1633309f..b4470c01376a 100644 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go +++ b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go @@ -17,15 +17,17 @@ limitations under the License. package aws import ( + "context" "fmt" "reflect" "strings" "sync" "time" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" "k8s.io/autoscaler/cluster-autoscaler/config/dynamic" klog "k8s.io/klog/v2" ) @@ -41,7 +43,7 @@ type asgCache struct { asgToInstances map[AwsRef][]AwsInstanceRef instanceToAsg map[AwsInstanceRef]*asg instanceStatus map[AwsInstanceRef]*string - instanceLifecycle map[AwsInstanceRef]*string + instanceLifecycle map[AwsInstanceRef]autoscalingtypes.LifecycleState asgInstanceTypeCache *instanceTypeExpirationStore mutex sync.Mutex awsService *awsWrapper @@ -60,8 +62,8 @@ type launchTemplate struct { type mixedInstancesPolicy struct { launchTemplate *launchTemplate instanceTypesOverrides []string - instanceRequirementsOverrides *autoscaling.InstanceRequirements - instanceRequirements *ec2.InstanceRequirements + instanceRequirementsOverrides *autoscalingtypes.InstanceRequirements + instanceRequirements *ec2types.InstanceRequirements } type asg struct { @@ -76,7 +78,7 @@ type asg struct { LaunchConfigurationName string LaunchTemplate *launchTemplate MixedInstancesPolicy *mixedInstancesPolicy - Tags []*autoscaling.TagDescription + Tags []autoscalingtypes.TagDescription } func newASGCache(awsService *awsWrapper, explicitSpecs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) (*asgCache, error) { @@ -86,7 +88,7 @@ func newASGCache(awsService *awsWrapper, explicitSpecs []string, autoDiscoverySp asgToInstances: make(map[AwsRef][]AwsInstanceRef), instanceToAsg: make(map[AwsInstanceRef]*asg), instanceStatus: make(map[AwsInstanceRef]*string), - instanceLifecycle: make(map[AwsInstanceRef]*string), + instanceLifecycle: make(map[AwsInstanceRef]autoscalingtypes.LifecycleState), asgInstanceTypeCache: newAsgInstanceTypeCache(awsService), interrupt: make(chan struct{}), asgAutoDiscoverySpecs: autoDiscoverySpecs, @@ -243,12 +245,12 @@ func (m *asgCache) InstanceStatus(ref AwsInstanceRef) (*string, error) { return nil, fmt.Errorf("could not find instance %v", ref) } -func (m *asgCache) findInstanceLifecycle(ref AwsInstanceRef) (*string, error) { +func (m *asgCache) findInstanceLifecycle(ref AwsInstanceRef) (autoscalingtypes.LifecycleState, error) { if lifecycle, found := m.instanceLifecycle[ref]; found { return lifecycle, nil } - return nil, fmt.Errorf("could not find instance %v", ref) + return "", fmt.Errorf("could not find instance %v", ref) } func (m *asgCache) SetAsgSize(asg *asg, size int) error { @@ -261,12 +263,12 @@ func (m *asgCache) SetAsgSize(asg *asg, size int) error { func (m *asgCache) setAsgSizeNoLock(asg *asg, size int) error { params := &autoscaling.SetDesiredCapacityInput{ AutoScalingGroupName: aws.String(asg.Name), - DesiredCapacity: aws.Int64(int64(size)), + DesiredCapacity: aws.Int32(int32(size)), HonorCooldown: aws.Bool(false), } klog.V(0).Infof("Setting asg %s size to %d", asg.Name, size) start := time.Now() - _, err := m.awsService.SetDesiredCapacity(params) + _, err := m.awsService.SetDesiredCapacity(context.Background(), params) observeAWSRequest("SetDesiredCapacity", err, start) if err != nil { return err @@ -356,12 +358,11 @@ func (m *asgCache) DeleteInstances(instances []*AwsInstanceRef) error { return err } - if lifecycle != nil && - *lifecycle == autoscaling.LifecycleStateTerminated || - *lifecycle == autoscaling.LifecycleStateTerminating || - *lifecycle == autoscaling.LifecycleStateTerminatingWait || - *lifecycle == autoscaling.LifecycleStateTerminatingProceed { - klog.V(2).Infof("instance %s is already terminating in state %s, will skip instead", instance.Name, *lifecycle) + if lifecycle == autoscalingtypes.LifecycleStateTerminated || + lifecycle == autoscalingtypes.LifecycleStateTerminating || + lifecycle == autoscalingtypes.LifecycleStateTerminatingWait || + lifecycle == autoscalingtypes.LifecycleStateTerminatingProceed { + klog.V(2).Infof("instance %s is already terminating in state %s, will skip instead", instance.Name, lifecycle) continue } @@ -370,12 +371,12 @@ func (m *asgCache) DeleteInstances(instances []*AwsInstanceRef) error { ShouldDecrementDesiredCapacity: aws.Bool(true), } start := time.Now() - resp, err := m.awsService.TerminateInstanceInAutoScalingGroup(params) + resp, err := m.awsService.TerminateInstanceInAutoScalingGroup(context.Background(), params) observeAWSRequest("TerminateInstanceInAutoScalingGroup", err, start) if err != nil { return err } - klog.V(4).Infof(*resp.Activity.Description) + klog.V(4).Infof("Terminated instance %s in autoscaling group: %s", instance.Name, aws.ToString(resp.Activity.Description)) // Proactively decrement the size so autoscaler makes better decisions commonAsg.curSize-- @@ -421,7 +422,7 @@ func (m *asgCache) regenerate() error { newInstanceToAsgCache := make(map[AwsInstanceRef]*asg) newAsgToInstancesCache := make(map[AwsRef][]AwsInstanceRef) newInstanceStatusMap := make(map[AwsInstanceRef]*string) - newInstanceLifecycleMap := make(map[AwsInstanceRef]*string) + newInstanceLifecycleMap := make(map[AwsInstanceRef]autoscalingtypes.LifecycleState) // Fetch details of all ASGs refreshNames := m.buildAsgNames() @@ -448,7 +449,7 @@ func (m *asgCache) regenerate() error { // Register or update ASGs exists := make(map[AwsRef]bool) for _, group := range groups { - asg, err := m.buildAsgFromAWS(group) + asg, err := m.buildAsgFromAWS(&group) if err != nil { return err } @@ -497,11 +498,13 @@ func (m *asgCache) regenerate() error { return nil } -func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*autoscaling.Group) []*autoscaling.Group { +func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []autoscalingtypes.AutoScalingGroup) []autoscalingtypes.AutoScalingGroup { + var updatedGroups []autoscalingtypes.AutoScalingGroup for _, g := range groups { desired := *g.DesiredCapacity - realInstances := int64(len(g.Instances)) + realInstances := int32(len(g.Instances)) if desired <= realInstances { + updatedGroups = append(updatedGroups, g) continue } @@ -509,7 +512,7 @@ func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*aut "Creating placeholder instances.", *g.AutoScalingGroupName, realInstances, desired) healthStatus := "" - isAvailable, err := m.isNodeGroupAvailable(g) + isAvailable, err := m.isNodeGroupAvailable(&g) if err != nil { klog.V(4).Infof("Could not check instance availability, creating placeholder node anyways: %v", err) } else if !isAvailable { @@ -519,23 +522,24 @@ func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*aut for i := realInstances; i < desired; i++ { id := fmt.Sprintf("%s-%s-%d", placeholderInstanceNamePrefix, *g.AutoScalingGroupName, i) - g.Instances = append(g.Instances, &autoscaling.Instance{ + g.Instances = append(g.Instances, autoscalingtypes.Instance{ InstanceId: &id, - AvailabilityZone: g.AvailabilityZones[0], + AvailabilityZone: aws.String(g.AvailabilityZones[0]), HealthStatus: &healthStatus, }) } + updatedGroups = append(updatedGroups, g) } - return groups + return updatedGroups } -func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) { +func (m *asgCache) isNodeGroupAvailable(group *autoscalingtypes.AutoScalingGroup) (bool, error) { input := &autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: group.AutoScalingGroupName, } start := time.Now() - response, err := m.awsService.DescribeScalingActivities(input) + response, err := m.awsService.DescribeScalingActivities(context.Background(), input) observeAWSRequest("DescribeScalingActivities", err, start) if err != nil { return true, err // If we can't describe the scaling activities we assume the node group is available @@ -547,8 +551,8 @@ func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) lut := a.lastUpdateTime if activity.StartTime.Before(lut) { break - } else if *activity.StatusCode == "Failed" { - klog.Warningf("ASG %s scaling failed with %s", asgRef.Name, *activity) + } else if activity.StatusCode == autoscalingtypes.ScalingActivityStatusCodeFailed { + klog.Warningf("ASG %s scaling failed with description: %s", asgRef.Name, aws.ToString(activity.Description)) return false, nil } } else { @@ -558,11 +562,11 @@ func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) return true, nil } -func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { +func (m *asgCache) buildAsgFromAWS(g *autoscalingtypes.AutoScalingGroup) (*asg, error) { spec := dynamic.NodeGroupSpec{ - Name: aws.StringValue(g.AutoScalingGroupName), - MinSize: int(aws.Int64Value(g.MinSize)), - MaxSize: int(aws.Int64Value(g.MaxSize)), + Name: aws.ToString(g.AutoScalingGroupName), + MinSize: int(aws.ToInt32(g.MinSize)), + MaxSize: int(aws.ToInt32(g.MaxSize)), SupportScaleToZero: scaleToZeroSupported, } @@ -575,9 +579,9 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { minSize: spec.MinSize, maxSize: spec.MaxSize, - curSize: int(aws.Int64Value(g.DesiredCapacity)), - AvailabilityZones: aws.StringValueSlice(g.AvailabilityZones), - LaunchConfigurationName: aws.StringValue(g.LaunchConfigurationName), + curSize: int(aws.ToInt32(g.DesiredCapacity)), + AvailabilityZones: g.AvailabilityZones, + LaunchConfigurationName: aws.ToString(g.LaunchConfigurationName), Tags: g.Tags, } @@ -586,8 +590,8 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { } if g.MixedInstancesPolicy != nil { - getInstanceTypes := func(overrides []*autoscaling.LaunchTemplateOverrides) []string { - res := []string{} + getInstanceTypes := func(overrides []autoscalingtypes.LaunchTemplateOverrides) []string { + var res []string for _, override := range overrides { if override.InstanceType != nil { res = append(res, *override.InstanceType) @@ -596,7 +600,7 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { return res } - getInstanceTypeRequirements := func(overrides []*autoscaling.LaunchTemplateOverrides) *autoscaling.InstanceRequirements { + getInstanceTypeRequirements := func(overrides []autoscalingtypes.LaunchTemplateOverrides) *autoscalingtypes.InstanceRequirements { if len(overrides) == 1 && overrides[0].InstanceRequirements != nil { return overrides[0].InstanceRequirements } @@ -625,8 +629,8 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { return asg, nil } -func (m *asgCache) getInstanceRequirementsFromMixedInstancesPolicy(policy *mixedInstancesPolicy) (*ec2.InstanceRequirements, error) { - instanceRequirements := &ec2.InstanceRequirements{} +func (m *asgCache) getInstanceRequirementsFromMixedInstancesPolicy(policy *mixedInstancesPolicy) (*ec2types.InstanceRequirements, error) { + instanceRequirements := &ec2types.InstanceRequirements{} if policy.instanceRequirementsOverrides != nil { var err error instanceRequirements, err = m.awsService.getEC2RequirementsFromAutoscaling(policy.instanceRequirementsOverrides) @@ -646,11 +650,11 @@ func (m *asgCache) getInstanceRequirementsFromMixedInstancesPolicy(policy *mixed return instanceRequirements, nil } -func (m *asgCache) buildInstanceRefFromAWS(instance *autoscaling.Instance) AwsInstanceRef { - providerID := fmt.Sprintf("aws:///%s/%s", aws.StringValue(instance.AvailabilityZone), aws.StringValue(instance.InstanceId)) +func (m *asgCache) buildInstanceRefFromAWS(instance autoscalingtypes.Instance) AwsInstanceRef { + providerID := fmt.Sprintf("aws:///%s/%s", aws.ToString(instance.AvailabilityZone), aws.ToString(instance.InstanceId)) return AwsInstanceRef{ ProviderID: providerID, - Name: aws.StringValue(instance.InstanceId), + Name: aws.ToString(instance.InstanceId), } } diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go index b536194fdf64..d5d552661814 100644 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go +++ b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go @@ -22,8 +22,10 @@ import ( "time" "github.com/stretchr/testify/assert" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" + "github.com/stretchr/testify/mock" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" ) func TestBuildAsg(t *testing.T) { @@ -57,31 +59,31 @@ func TestCreatePlaceholders(t *testing.T) { cases := []struct { name string - desiredCapacity *int64 - activities []*autoscaling.Activity + desiredCapacity *int32 + activities []autoscalingtypes.Activity groupLastUpdateTime time.Time describeErr error asgToCheck *string }{ { name: "add placeholders successful", - desiredCapacity: aws.Int64(10), + desiredCapacity: aws.Int32(10), }, { name: "no placeholders needed", - desiredCapacity: aws.Int64(0), + desiredCapacity: aws.Int32(0), }, { name: "DescribeScalingActivities failed", - desiredCapacity: aws.Int64(1), + desiredCapacity: aws.Int32(1), describeErr: errors.New("timeout"), }, { name: "early abort if AWS scaling up fails", - desiredCapacity: aws.Int64(1), - activities: []*autoscaling.Activity{ + desiredCapacity: aws.Int32(1), + activities: []autoscalingtypes.Activity{ { - StatusCode: aws.String("Failed"), + StatusCode: autoscalingtypes.ScalingActivityStatusCodeFailed, StartTime: aws.Time(time.Unix(10, 0)), }, }, @@ -89,10 +91,10 @@ func TestCreatePlaceholders(t *testing.T) { }, { name: "AWS scaling failed event before CA scale_up", - desiredCapacity: aws.Int64(1), - activities: []*autoscaling.Activity{ + desiredCapacity: aws.Int32(1), + activities: []autoscalingtypes.Activity{ { - StatusCode: aws.String("Failed"), + StatusCode: autoscalingtypes.ScalingActivityStatusCodeFailed, StartTime: aws.Time(time.Unix(9, 0)), }, }, @@ -100,10 +102,10 @@ func TestCreatePlaceholders(t *testing.T) { }, { name: "asg not registered", - desiredCapacity: aws.Int64(10), - activities: []*autoscaling.Activity{ + desiredCapacity: aws.Int32(10), + activities: []autoscalingtypes.Activity{ { - StatusCode: aws.String("Failed"), + StatusCode: autoscalingtypes.ScalingActivityStatusCodeFailed, StartTime: aws.Time(time.Unix(10, 0)), }, }, @@ -115,7 +117,7 @@ func TestCreatePlaceholders(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { shouldCallDescribeScalingActivities := true - if *tc.desiredCapacity == int64(0) { + if *tc.desiredCapacity == int32(0) { shouldCallDescribeScalingActivities = false } @@ -126,9 +128,12 @@ func TestCreatePlaceholders(t *testing.T) { a := &autoScalingMock{} if shouldCallDescribeScalingActivities { - a.On("DescribeScalingActivities", &autoscaling.DescribeScalingActivitiesInput{ - AutoScalingGroupName: asgName, - }).Return( + a.On("DescribeScalingActivities", + mock.Anything, + &autoscaling.DescribeScalingActivitiesInput{ + AutoScalingGroupName: asgName, + }, + ).Return( &autoscaling.DescribeScalingActivitiesOutput{Activities: tc.activities}, tc.describeErr, ).Once() @@ -147,17 +152,17 @@ func TestCreatePlaceholders(t *testing.T) { }, } - groups := []*autoscaling.Group{ + groups := []autoscalingtypes.AutoScalingGroup{ { AutoScalingGroupName: asgName, - AvailabilityZones: []*string{aws.String("westeros-1a")}, + AvailabilityZones: []string{"westeros-1a"}, DesiredCapacity: tc.desiredCapacity, - Instances: []*autoscaling.Instance{}, + Instances: []autoscalingtypes.Instance{}, }, } - asgCache.createPlaceholdersForDesiredNonStartedInstances(groups) - assert.Equal(t, int64(len(groups[0].Instances)), *tc.desiredCapacity) - if tc.activities != nil && *tc.activities[0].StatusCode == "Failed" && tc.activities[0].StartTime.After(tc.groupLastUpdateTime) && asgName == registeredAsgName { + groups = asgCache.createPlaceholdersForDesiredNonStartedInstances(groups) + assert.Equal(t, int32(len(groups[0].Instances)), *tc.desiredCapacity) + if tc.activities != nil && tc.activities[0].StatusCode == autoscalingtypes.ScalingActivityStatusCodeFailed && tc.activities[0].StartTime.After(tc.groupLastUpdateTime) && asgName == registeredAsgName { assert.Equal(t, *groups[0].Instances[0].HealthStatus, placeholderUnfulfillableStatus) } else if len(groups[0].Instances) > 0 { assert.Equal(t, *groups[0].Instances[0].HealthStatus, "") diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go index 3117b1c73654..9344b39ec534 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go @@ -436,7 +436,7 @@ func BuildAWS(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover if opts.AWSUseStaticInstanceList { klog.Warningf("Using static EC2 Instance Types, this list could be outdated. Last update time: %s", lastUpdateTime) } else { - generatedInstanceTypes, err := GenerateEC2InstanceTypes(sdkProvider.session) + generatedInstanceTypes, err := GenerateEC2InstanceTypes(sdkProvider.cfg) if err != nil { klog.Errorf("Failed to generate AWS EC2 Instance Types: %v, falling back to static list with last update time: %s", err, lastUpdateTime) } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go index 1910f8752fd5..ea8dff0eaf5e 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go @@ -25,8 +25,9 @@ import ( apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" "k8s.io/autoscaler/cluster-autoscaler/config" ) @@ -76,33 +77,33 @@ func newTestAwsManagerWithAutoAsgs(t *testing.T, mockAutoScaling autoScalingI, m return m } -func testNamedDescribeAutoScalingGroupsOutput(groupName string, desiredCap int64, instanceIds ...string) *autoscaling.DescribeAutoScalingGroupsOutput { - instances := []*autoscaling.Instance{} +func testNamedDescribeAutoScalingGroupsOutput(groupName string, desiredCap int32, instanceIds ...string) *autoscaling.DescribeAutoScalingGroupsOutput { + var instances []autoscalingtypes.Instance for _, id := range instanceIds { - instances = append(instances, &autoscaling.Instance{ + instances = append(instances, autoscalingtypes.Instance{ InstanceId: aws.String(id), AvailabilityZone: aws.String("us-east-1a"), - LifecycleState: aws.String(autoscaling.LifecycleStateInService), + LifecycleState: autoscalingtypes.LifecycleStateInService, }) } return &autoscaling.DescribeAutoScalingGroupsOutput{ - AutoScalingGroups: []*autoscaling.Group{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ { AutoScalingGroupName: aws.String(groupName), - DesiredCapacity: aws.Int64(desiredCap), - MinSize: aws.Int64(1), - MaxSize: aws.Int64(5), + DesiredCapacity: aws.Int32(desiredCap), + MinSize: aws.Int32(1), + MaxSize: aws.Int32(5), Instances: instances, - AvailabilityZones: aws.StringSlice([]string{"us-east-1a"}), + AvailabilityZones: []string{"us-east-1a"}, }, }, } } -func testSetASGInstanceLifecycle(asg *autoscaling.DescribeAutoScalingGroupsOutput, lifecycleState string) *autoscaling.DescribeAutoScalingGroupsOutput { +func testSetASGInstanceLifecycle(asg *autoscaling.DescribeAutoScalingGroupsOutput, lifecycleState autoscalingtypes.LifecycleState) *autoscaling.DescribeAutoScalingGroupsOutput { for _, asg := range asg.AutoScalingGroups { - for _, instance := range asg.Instances { - instance.LifecycleState = aws.String(lifecycleState) + for i := range asg.Instances { + asg.Instances[i].LifecycleState = lifecycleState } } return asg @@ -166,18 +167,15 @@ func TestAutoDiscoveredNodeGroups(t *testing.T) { }, })) - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - Filters: []*autoscaling.Filter{ - {Name: aws.String("tag-key"), Values: aws.StringSlice([]string{"test"})}, + Filters: []autoscalingtypes.Filter{ + {Name: aws.String("tag-key"), Values: []string{"test"}}, }, - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("auto-asg", 1, "test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("auto-asg", 1, "test-instance-id"), nil) provider.Refresh() @@ -197,16 +195,13 @@ func TestNodeGroupForNode(t *testing.T) { a := &autoScalingMock{} provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", 1, "test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", 1, "test-instance-id"), nil) provider.Refresh() @@ -222,7 +217,7 @@ func TestNodeGroupForNode(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []cloudprovider.Instance{{Id: "aws:///us-east-1a/test-instance-id"}}, nodes) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) // test node in cluster that is not in a group managed by cluster autoscaler nodeNotInGroup := &apiv1.Node{ @@ -235,7 +230,7 @@ func TestNodeGroupForNode(t *testing.T) { assert.NoError(t, err) assert.Nil(t, group) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) } func TestNodeGroupForNodeWithNoProviderId(t *testing.T) { @@ -324,16 +319,13 @@ func TestTargetSize(t *testing.T) { provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", 2, "test-instance-id", "second-test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", 2, "test-instance-id", "second-test-instance-id"), nil) provider.Refresh() @@ -341,7 +333,7 @@ func TestTargetSize(t *testing.T) { assert.Equal(t, targetSize, 2) assert.NoError(t, err) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) } func TestIncreaseSize(t *testing.T) { @@ -349,22 +341,20 @@ func TestIncreaseSize(t *testing.T) { provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - a.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ - AutoScalingGroupName: aws.String(asgs[0].Id()), - DesiredCapacity: aws.Int64(3), - HonorCooldown: aws.Bool(false), - }).Return(&autoscaling.SetDesiredCapacityOutput{}) + a.On("SetDesiredCapacity", mock.Anything, + &autoscaling.SetDesiredCapacityInput{ + AutoScalingGroupName: aws.String(asgs[0].Id()), + DesiredCapacity: aws.Int32(3), + HonorCooldown: aws.Bool(false), + }).Return(&autoscaling.SetDesiredCapacityOutput{}, nil) - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", 2, "test-instance-id", "second-test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", 2, "test-instance-id", "second-test-instance-id"), nil) provider.Refresh() @@ -375,7 +365,7 @@ func TestIncreaseSize(t *testing.T) { err = asgs[0].IncreaseSize(1) assert.NoError(t, err) a.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -387,16 +377,13 @@ func TestBelongs(t *testing.T) { provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{asgs[0].Id()}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{asgs[0].Id()}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", 1, "test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", 1, "test-instance-id"), nil) provider.Refresh() @@ -407,7 +394,7 @@ func TestBelongs(t *testing.T) { } _, err := asgs[0].(*AwsNodeGroup).Belongs(invalidNode) assert.Error(t, err) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) validNode := &apiv1.Node{ Spec: apiv1.NodeSpec{ @@ -418,37 +405,38 @@ func TestBelongs(t *testing.T) { assert.Equal(t, belongs, true) assert.NoError(t, err) // As "test-instance-id" is already known to be managed by test-asg since - // the first `Belongs` call, no additional DescribAutoScalingGroupsPages + // the first `Belongs` call, no additional DescribAutoScalingGroups // call is made. - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) } func TestDeleteNodes(t *testing.T) { a := &autoScalingMock{} provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() + var expectedInstancesCount int32 = 2 - a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ - InstanceId: aws.String("test-instance-id"), - ShouldDecrementDesiredCapacity: aws.Bool(true), - }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ - Activity: &autoscaling.Activity{Description: aws.String("Deleted instance")}, - }) + a.On("TerminateInstanceInAutoScalingGroup", + mock.Anything, + &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: aws.String("test-instance-id"), + ShouldDecrementDesiredCapacity: aws.Bool(true), + }, + ).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ + Activity: &autoscalingtypes.Activity{Description: aws.String("Deleted instance")}, + }, nil) // Look up the current number of instances... - var expectedInstancesCount int64 = 2 - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id", "second-test-instance-id"), false) - // we expect the instance count to be 1 after the call to DeleteNodes expectedInstancesCount = 1 - }).Return(nil) + }, + ).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id", "second-test-instance-id"), nil) provider.Refresh() @@ -464,7 +452,7 @@ func TestDeleteNodes(t *testing.T) { err = asgs[0].DeleteNodes([]*apiv1.Node{node}) assert.NoError(t, err) a.AssertNumberOfCalls(t, "TerminateInstanceInAutoScalingGroup", 1) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -476,27 +464,31 @@ func TestDeleteNodesTerminatingInstances(t *testing.T) { provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ - InstanceId: aws.String("test-instance-id"), - ShouldDecrementDesiredCapacity: aws.Bool(true), - }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ - Activity: &autoscaling.Activity{Description: aws.String("Deleted instance")}, - }) + a.On("TerminateInstanceInAutoScalingGroup", + mock.Anything, + &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: aws.String("test-instance-id"), + ShouldDecrementDesiredCapacity: aws.Bool(true), + }, + ).Return( + &autoscaling.TerminateInstanceInAutoScalingGroupOutput{ + Activity: &autoscalingtypes.Activity{Description: aws.String("Deleted instance")}, + }, + nil, + ) // Look up the current number of instances... - var expectedInstancesCount int64 = 2 - a.On("DescribeAutoScalingGroupsPages", + var expectedInstancesCount int32 = 2 + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testSetASGInstanceLifecycle(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id", "second-test-instance-id"), autoscaling.LifecycleStateTerminatingWait), false) // we expect the instance count to be 1 after the call to DeleteNodes expectedInstancesCount = 1 - }).Return(nil) + }).Return(testSetASGInstanceLifecycle(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id", "second-test-instance-id"), autoscalingtypes.LifecycleStateTerminatingWait), nil) provider.Refresh() @@ -512,7 +504,7 @@ func TestDeleteNodesTerminatingInstances(t *testing.T) { err = asgs[0].DeleteNodes([]*apiv1.Node{node}) assert.NoError(t, err) a.AssertNumberOfCalls(t, "TerminateInstanceInAutoScalingGroup", 0) // instances which are terminating don't need to be terminated again - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -524,31 +516,34 @@ func TestDeleteNodesTerminatedInstances(t *testing.T) { provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ - InstanceId: aws.String("test-instance-id"), - ShouldDecrementDesiredCapacity: aws.Bool(true), - }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ - Activity: &autoscaling.Activity{Description: aws.String("Deleted instance")}, - }) + a.On("TerminateInstanceInAutoScalingGroup", + mock.Anything, + &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: aws.String("test-instance-id"), + ShouldDecrementDesiredCapacity: aws.Bool(true), + }, + ).Return( + &autoscaling.TerminateInstanceInAutoScalingGroupOutput{ + Activity: &autoscalingtypes.Activity{Description: aws.String("Deleted instance")}, + }, + nil, + ) - expectedInstancesCount := 2 - a.On("DescribeAutoScalingGroupsPages", + var expectedInstancesCount int32 = 2 + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testSetASGInstanceLifecycle(testNamedDescribeAutoScalingGroupsOutput("test-asg", int64(expectedInstancesCount), "test-instance-id", "second-test-instance-id"), autoscaling.LifecycleStateTerminated), false) - }).Return(nil) + ).Return(testSetASGInstanceLifecycle(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id", "second-test-instance-id"), autoscalingtypes.LifecycleStateTerminated), nil) // load ASG state into cache provider.Refresh() initialSize, err := asgs[0].TargetSize() assert.NoError(t, err) - assert.Equal(t, expectedInstancesCount, initialSize) + assert.Equal(t, expectedInstancesCount, int32(initialSize)) // try deleting a node, but all of them are already in a // Terminated state, so we should see no calls to Terminate. @@ -562,7 +557,7 @@ func TestDeleteNodesTerminatedInstances(t *testing.T) { // we expect no calls to TerminateInstanceInAutoScalingGroup, // because the Node we tried to Delete was already terminating. a.AssertNumberOfCalls(t, "TerminateInstanceInAutoScalingGroup", 0) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -577,32 +572,37 @@ func TestDeleteNodesWithPlaceholder(t *testing.T) { provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - a.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ - AutoScalingGroupName: aws.String(asgs[0].Id()), - DesiredCapacity: aws.Int64(1), - HonorCooldown: aws.Bool(false), - }).Return(&autoscaling.SetDesiredCapacityOutput{}) + a.On("SetDesiredCapacity", + mock.Anything, + &autoscaling.SetDesiredCapacityInput{ + AutoScalingGroupName: aws.String(asgs[0].Id()), + DesiredCapacity: aws.Int32(1), + HonorCooldown: aws.Bool(false), + }, + ).Return(&autoscaling.SetDesiredCapacityOutput{}, nil) // Look up the current number of instances... - var expectedInstancesCount int64 = 2 - a.On("DescribeAutoScalingGroupsPages", + var expectedInstancesCount int32 = 2 + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id"), false) // we expect the instance count to be 1 after the call to DeleteNodes expectedInstancesCount = 1 - }).Return(nil) + }).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "test-instance-id"), nil) a.On("DescribeScalingActivities", + mock.Anything, &autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: aws.String("test-asg"), }, - ).Return(&autoscaling.DescribeScalingActivitiesOutput{}, nil) + ).Return( + &autoscaling.DescribeScalingActivitiesOutput{}, + nil, + ) provider.Refresh() @@ -618,7 +618,7 @@ func TestDeleteNodesWithPlaceholder(t *testing.T) { err = asgs[0].DeleteNodes([]*apiv1.Node{node}) assert.NoError(t, err) a.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 2) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 2) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -631,24 +631,27 @@ func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) { provider := testProvider(t, manager) asgs := provider.NodeGroups() - a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ - InstanceId: aws.String("test-instance-id"), - ShouldDecrementDesiredCapacity: aws.Bool(true), - }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ - Activity: &autoscaling.Activity{Description: aws.String("Deleted instance")}, - }) + a.On("TerminateInstanceInAutoScalingGroup", + mock.Anything, + &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: aws.String("test-instance-id"), + ShouldDecrementDesiredCapacity: aws.Bool(true), + }, + ).Return( + &autoscaling.TerminateInstanceInAutoScalingGroupOutput{ + Activity: &autoscalingtypes.Activity{Description: aws.String("Deleted instance")}, + }, + nil, + ) // Look up the current number of instances... - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", 2, "test-instance-id", "second-test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", 2, "test-instance-id", "second-test-instance-id"), nil) provider.Refresh() // Call the manager directly as otherwise the call would be a noop as its within less then 60s @@ -771,28 +774,29 @@ func TestDeleteNodesWithPlaceholderAndStaleCache(t *testing.T) { } // desired capacity will be set as 6 as ASG has 4 placeholders - a.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ - AutoScalingGroupName: aws.String(asgs[0].Id()), - DesiredCapacity: aws.Int64(6), - HonorCooldown: aws.Bool(false), - }).Return(&autoscaling.SetDesiredCapacityOutput{}) + a.On("SetDesiredCapacity", + mock.Anything, + &autoscaling.SetDesiredCapacityInput{ + AutoScalingGroupName: aws.String(asgs[0].Id()), + DesiredCapacity: aws.Int32(6), + HonorCooldown: aws.Bool(false), + }, + ).Return(&autoscaling.SetDesiredCapacityOutput{}, nil) // Look up the current number of instances... - var expectedInstancesCount int64 = 10 - a.On("DescribeAutoScalingGroupsPages", + var expectedInstancesCount int32 = 10 + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"test-asg"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "i-0000", "i-0001", "i-0002", "i-0003", "i-0004", "i-0005"), false) - expectedInstancesCount = 4 - }).Return(nil) + }).Return(testNamedDescribeAutoScalingGroupsOutput("test-asg", expectedInstancesCount, "i-0000", "i-0001", "i-0002", "i-0003", "i-0004", "i-0005"), nil) a.On("DescribeScalingActivities", + mock.Anything, &autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: aws.String("test-asg"), }, @@ -834,12 +838,17 @@ func TestDeleteNodesWithPlaceholderAndStaleCache(t *testing.T) { // only setting 2 instances to be terminated out of 3 active instances if i < 2 { nodes = append(nodes, node) - a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ - InstanceId: aws.String(fmt.Sprintf("i-000%d", i)), - ShouldDecrementDesiredCapacity: aws.Bool(true), - }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ - Activity: &autoscaling.Activity{Description: aws.String("Deleted instance")}, - }) + a.On("TerminateInstanceInAutoScalingGroup", + mock.Anything, + &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: aws.String(fmt.Sprintf("i-000%d", i)), + ShouldDecrementDesiredCapacity: aws.Bool(true), + }, + ).Return( + &autoscaling.TerminateInstanceInAutoScalingGroupOutput{ + Activity: &autoscalingtypes.Activity{Description: aws.String("Deleted instance")}, + }, nil, + ) } awsInstanceRef := AwsInstanceRef{ ProviderID: providerId, @@ -857,7 +866,7 @@ func TestDeleteNodesWithPlaceholderAndStaleCache(t *testing.T) { err = asgs[0].DeleteNodes(nodes) assert.NoError(t, err) a.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) - a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 2) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroups", 2) // This ensures only 2 instances are terminated which are mocked in this unit test a.AssertNumberOfCalls(t, "TerminateInstanceInAutoScalingGroup", 2) diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager.go b/cluster-autoscaler/cloudprovider/aws/aws_manager.go index 6a649a5860a3..bafd9daf34dc 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager.go @@ -33,10 +33,12 @@ import ( "k8s.io/klog/v2" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks" "k8s.io/autoscaler/cluster-autoscaler/config" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" ) @@ -66,7 +68,7 @@ type asgTemplate struct { InstanceType *InstanceType Region string Zone string - Tags []*autoscaling.TagDescription + Tags []autoscalingtypes.TagDescription } // createAwsManagerInternal allows for custom objects to be passed in by tests @@ -79,8 +81,11 @@ func createAWSManagerInternal( klog.Infof("AWS SDK Version: %s", aws.SDKVersion) if awsService == nil { - sess := awsSDKProvider.session - awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess), eks.New(sess)} + awsService = &awsWrapper{ + autoScalingI: autoscaling.NewFromConfig(awsSDKProvider.cfg, autoscaling.WithEndpointResolver(newAutoscalingOverrideResolver(awsSDKProvider.cloudConfig))), + ec2I: ec2.NewFromConfig(awsSDKProvider.cfg, ec2.WithEndpointResolver(newEc2OverrideResolver(awsSDKProvider.cloudConfig))), + eksI: eks.NewFromConfig(awsSDKProvider.cfg, eks.WithEndpointResolver(newEksOverrideResolver(awsSDKProvider.cloudConfig))), + } } specs, err := parseASGAutoDiscoverySpecs(discoveryOpts) @@ -363,18 +368,18 @@ func (m *AwsManager) updateCapacityWithRequirementsOverrides(capacity *apiv1.Res instanceRequirements := policy.instanceRequirements if instanceRequirements.VCpuCount != nil && instanceRequirements.VCpuCount.Min != nil { - (*capacity)[apiv1.ResourceCPU] = *resource.NewQuantity(*instanceRequirements.VCpuCount.Min, resource.DecimalSI) + (*capacity)[apiv1.ResourceCPU] = *resource.NewQuantity(int64(*instanceRequirements.VCpuCount.Min), resource.DecimalSI) } if instanceRequirements.MemoryMiB != nil && instanceRequirements.MemoryMiB.Min != nil { - (*capacity)[apiv1.ResourceMemory] = *resource.NewQuantity(*instanceRequirements.MemoryMiB.Min*1024*1024, resource.DecimalSI) + (*capacity)[apiv1.ResourceMemory] = *resource.NewQuantity(int64(*instanceRequirements.MemoryMiB.Min*1024*1024), resource.DecimalSI) } for _, manufacturer := range instanceRequirements.AcceleratorManufacturers { - if *manufacturer == autoscaling.AcceleratorManufacturerNvidia { + if manufacturer == ec2types.AcceleratorManufacturerNvidia { for _, acceleratorType := range instanceRequirements.AcceleratorTypes { - if *acceleratorType == autoscaling.AcceleratorTypeGpu { - (*capacity)[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(*instanceRequirements.AcceleratorCount.Min, resource.DecimalSI) + if acceleratorType == ec2types.AcceleratorTypeGpu { + (*capacity)[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(int64(*instanceRequirements.AcceleratorCount.Min), resource.DecimalSI) } } } @@ -396,7 +401,7 @@ func buildGenericLabels(template *asgTemplate, nodeName string) map[string]strin return result } -func extractLabelsFromAsg(tags []*autoscaling.TagDescription) map[string]string { +func extractLabelsFromAsg(tags []autoscalingtypes.TagDescription) map[string]string { result := make(map[string]string) for _, tag := range tags { @@ -418,22 +423,22 @@ func extractLabelsFromAsg(tags []*autoscaling.TagDescription) map[string]string return result } -func extractAutoscalingOptionsFromTags(tags []*autoscaling.TagDescription) map[string]string { +func extractAutoscalingOptionsFromTags(tags []autoscalingtypes.TagDescription) map[string]string { options := make(map[string]string) for _, tag := range tags { - if !strings.HasPrefix(aws.StringValue(tag.Key), optionsTagsPrefix) { + if !strings.HasPrefix(aws.ToString(tag.Key), optionsTagsPrefix) { continue } - splits := strings.Split(aws.StringValue(tag.Key), optionsTagsPrefix) + splits := strings.Split(aws.ToString(tag.Key), optionsTagsPrefix) if len(splits) != 2 || splits[1] == "" { continue } - options[splits[1]] = aws.StringValue(tag.Value) + options[splits[1]] = aws.ToString(tag.Value) } return options } -func extractAllocatableResourcesFromAsg(tags []*autoscaling.TagDescription) map[string]*resource.Quantity { +func extractAllocatableResourcesFromAsg(tags []autoscalingtypes.TagDescription) map[string]*resource.Quantity { result := make(map[string]*resource.Quantity) for _, tag := range tags { @@ -476,7 +481,7 @@ func extractAllocatableResourcesFromTags(tags map[string]string) map[string]*res return result } -func extractTaintsFromAsg(tags []*autoscaling.TagDescription) []apiv1.Taint { +func extractTaintsFromAsg(tags []autoscalingtypes.TagDescription) []apiv1.Taint { taints := make([]apiv1.Taint, 0) for _, tag := range tags { diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go index 08c7f4fbc773..19192bbf1138 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go @@ -31,9 +31,11 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" "k8s.io/autoscaler/cluster-autoscaler/config" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" ) @@ -77,7 +79,7 @@ func TestBuildGenericLabels(t *testing.T) { } func TestExtractAllocatableResourcesFromAsg(t *testing.T) { - tags := []*autoscaling.TagDescription{ + tags := []autoscalingtypes.TagDescription{ { Key: aws.String("k8s.io/cluster-autoscaler/node-template/resources/cpu"), Value: aws.String("100m"), @@ -275,7 +277,7 @@ func TestBuildNodeFromTemplateWithManagedNodegroup(t *testing.T) { // Node with EKS labels observedNode, observedErr := awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String("eks:nodegroup-name"), Value: aws.String(ngNameLabelValue), @@ -342,7 +344,7 @@ func TestBuildNodeFromTemplateWithManagedNodegroupNoLabelsOrTaints(t *testing.T) // Node with EKS labels observedNode, observedErr := awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String("eks:nodegroup-name"), Value: aws.String(ngNameLabelValue), @@ -389,7 +391,7 @@ func TestBuildNodeFromTemplateWithManagedNodegroupNilLabelsOrTaints(t *testing.T // Node with EKS labels observedNode, observedErr := awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String("eks:nodegroup-name"), Value: aws.String(ngNameLabelValue), @@ -429,7 +431,7 @@ func TestBuildNodeFromTemplate(t *testing.T) { vpcIPKey := "vpc.amazonaws.com/PrivateIPv4Address" observedNode, observedErr := awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String(fmt.Sprintf("k8s.io/cluster-autoscaler/node-template/resources/%s", ephemeralStorageKey)), Value: aws.String(strconv.FormatInt(ephemeralStorageValue, 10)), @@ -454,7 +456,7 @@ func TestBuildNodeFromTemplate(t *testing.T) { GPULabelValue := "nvidia-telsa-v100" observedNode, observedErr = awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String(fmt.Sprintf("k8s.io/cluster-autoscaler/node-template/label/%s", GPULabel)), Value: aws.String(GPULabelValue), @@ -470,7 +472,7 @@ func TestBuildNodeFromTemplate(t *testing.T) { ngNameLabelValue := "nodegroup-1" observedNode, observedErr = awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String("eks:nodegroup-name"), Value: aws.String(ngNameLabelValue), @@ -490,7 +492,7 @@ func TestBuildNodeFromTemplate(t *testing.T) { } observedNode, observedErr = awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, - Tags: []*autoscaling.TagDescription{ + Tags: []autoscalingtypes.TagDescription{ { Key: aws.String(fmt.Sprintf("k8s.io/cluster-autoscaler/node-template/taint/%s", gpuTaint.Key)), Value: aws.String(fmt.Sprintf("%s:%s", gpuTaint.Value, gpuTaint.Effect)), @@ -505,20 +507,20 @@ func TestBuildNodeFromTemplate(t *testing.T) { // Node with instance requirements asg.MixedInstancesPolicy = &mixedInstancesPolicy{ - instanceRequirements: &ec2.InstanceRequirements{ - VCpuCount: &ec2.VCpuCountRange{ - Min: aws.Int64(4), - Max: aws.Int64(8), + instanceRequirements: &ec2types.InstanceRequirements{ + VCpuCount: &ec2types.VCpuCountRange{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, - MemoryMiB: &ec2.MemoryMiB{ - Min: aws.Int64(4), - Max: aws.Int64(8), + MemoryMiB: &ec2types.MemoryMiB{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, - AcceleratorTypes: []*string{aws.String(autoscaling.AcceleratorTypeGpu)}, - AcceleratorManufacturers: []*string{aws.String(autoscaling.AcceleratorManufacturerNvidia)}, - AcceleratorCount: &ec2.AcceleratorCount{ - Min: aws.Int64(4), - Max: aws.Int64(8), + AcceleratorTypes: []ec2types.AcceleratorType{ec2types.AcceleratorTypeGpu}, + AcceleratorManufacturers: []ec2types.AcceleratorManufacturer{ec2types.AcceleratorManufacturerNvidia}, + AcceleratorCount: &ec2types.AcceleratorCount{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, }, } @@ -536,7 +538,7 @@ func TestBuildNodeFromTemplate(t *testing.T) { } func TestExtractLabelsFromAsg(t *testing.T) { - tags := []*autoscaling.TagDescription{ + tags := []autoscalingtypes.TagDescription{ { Key: aws.String("k8s.io/cluster-autoscaler/node-template/label/foo"), Value: aws.String("bar"), @@ -564,7 +566,7 @@ func TestExtractLabelsFromAsg(t *testing.T) { } func TestExtractTaintsFromAsg(t *testing.T) { - tags := []*autoscaling.TagDescription{ + tags := []autoscalingtypes.TagDescription{ { Key: aws.String("k8s.io/cluster-autoscaler/node-template/taint/dedicated"), Value: aws.String("foo:NoSchedule"), @@ -627,37 +629,43 @@ func TestFetchExplicitAsgs(t *testing.T) { asgRef := AwsRef{Name: groupname} a := &autoScalingMock{} - a.On("DescribeAutoScalingGroups", &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: []*string{aws.String(groupname)}, - MaxRecords: aws.Int64(1), - }).Return(&autoscaling.DescribeAutoScalingGroupsOutput{ - AutoScalingGroups: []*autoscaling.Group{ - {AutoScalingGroupName: aws.String(groupname)}, + a.On("DescribeAutoScalingGroups", + mock.Anything, + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []string{groupname}, + MaxRecords: aws.Int32(1), }, - }) + ).Return( + &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + {AutoScalingGroupName: aws.String(groupname)}, + }, + }, + nil, + ) - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{groupname}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), - }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - zone := "test-1a" - fn(&autoscaling.DescribeAutoScalingGroupsOutput{ - AutoScalingGroups: []*autoscaling.Group{ + AutoScalingGroupNames: []string{groupname}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), + }, + ).Return( + &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ { - AvailabilityZones: []*string{&zone}, + AvailabilityZones: []string{"test-1a"}, AutoScalingGroupName: aws.String(groupname), - MinSize: aws.Int64(int64(min)), - MaxSize: aws.Int64(int64(max)), - DesiredCapacity: aws.Int64(int64(min)), + MinSize: aws.Int32(int32(min)), + MaxSize: aws.Int32(int32(max)), + DesiredCapacity: aws.Int32(int32(min)), }, - }}, false) - }).Return(nil) + }}, + nil, + ) a.On("DescribeScalingActivities", + mock.Anything, &autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: aws.String("coolasg"), }, @@ -694,7 +702,7 @@ func TestGetASGTemplate(t *testing.T) { asgRef := AwsRef{Name: asgName} - tags := []*autoscaling.TagDescription{ + tags := []autoscalingtypes.TagDescription{ { Key: aws.String("k8s.io/cluster-autoscaler/node-template/taint/dedicated"), Value: aws.String("foo:NoSchedule"), @@ -722,12 +730,12 @@ func TestGetASGTemplate(t *testing.T) { e := &ec2Mock{} e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ LaunchTemplateName: aws.String(ltName), - Versions: []*string{aws.String(ltVersion)}, + Versions: []string{ltVersion}, }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - InstanceType: aws.String(test.instanceType), + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + InstanceType: ec2types.InstanceType(test.instanceType), }, }, }, @@ -779,49 +787,48 @@ func TestFetchAutoAsgs(t *testing.T) { // Describe the group to register it, then again to generate the instance // cache. - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{groupname}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), - }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - zone := "test-1a" - fn(&autoscaling.DescribeAutoScalingGroupsOutput{ - AutoScalingGroups: []*autoscaling.Group{{ - AvailabilityZones: []*string{&zone}, + AutoScalingGroupNames: []string{groupname}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), + }, + ).Return( + &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{{ + AvailabilityZones: []string{"test-1a"}, AutoScalingGroupName: aws.String(groupname), - MinSize: aws.Int64(int64(min)), - MaxSize: aws.Int64(int64(max)), - DesiredCapacity: aws.Int64(int64(min)), - }}}, false) - }).Return(nil).Once() + MinSize: aws.Int32(int32(min)), + MaxSize: aws.Int32(int32(max)), + DesiredCapacity: aws.Int32(int32(min)), + }}}, + nil, + ).Once() expectedGroupsInputWithTags := &autoscaling.DescribeAutoScalingGroupsInput{ - Filters: []*autoscaling.Filter{ - {Name: aws.String("tag-key"), Values: aws.StringSlice([]string{tags[0]})}, - {Name: aws.String("tag-key"), Values: aws.StringSlice([]string{tags[1]})}, + Filters: []autoscalingtypes.Filter{ + {Name: aws.String("tag-key"), Values: []string{tags[0]}}, + {Name: aws.String("tag-key"), Values: []string{tags[1]}}, }, - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), } - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, mock.MatchedBy(tagsMatcher(expectedGroupsInputWithTags)), - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - zone := "test-1a" - fn(&autoscaling.DescribeAutoScalingGroupsOutput{ - AutoScalingGroups: []*autoscaling.Group{{ - AvailabilityZones: []*string{&zone}, + ).Return( + &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{{ + AvailabilityZones: []string{"test-1a"}, AutoScalingGroupName: aws.String(groupname), - MinSize: aws.Int64(int64(min)), - MaxSize: aws.Int64(int64(max)), - DesiredCapacity: aws.Int64(int64(min)), - }}}, false) - }).Return(nil).Once() + MinSize: aws.Int32(int32(min)), + MaxSize: aws.Int32(int32(max)), + DesiredCapacity: aws.Int32(int32(min)), + }}}, + nil, + ).Once() a.On("DescribeScalingActivities", + mock.Anything, &autoscaling.DescribeScalingActivitiesInput{ AutoScalingGroupName: aws.String("coolasg"), }, @@ -842,14 +849,14 @@ func TestFetchAutoAsgs(t *testing.T) { validateAsg(t, asgs[asgRef], groupname, min, max) // Simulate the previously discovered ASG disappearing - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, mock.MatchedBy(tagsMatcher(expectedGroupsInputWithTags)), - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(&autoscaling.DescribeAutoScalingGroupsOutput{ - AutoScalingGroups: []*autoscaling.Group{}}, false) - }).Return(nil).Once() + ).Return( + &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{}}, + nil, + ).Once() err = m.asgCache.regenerate() assert.NoError(t, err) @@ -872,10 +879,10 @@ func tagsMatcher(expected *autoscaling.DescribeAutoScalingGroupsInput) func(*aut } } -func flatTagSlice(filters []*autoscaling.Filter) []string { - tags := []string{} +func flatTagSlice(filters []autoscalingtypes.Filter) []string { + var tags []string for _, filter := range filters { - tags = append(tags, aws.StringValueSlice(filter.Values)...) + tags = append(tags, filter.Values...) } // Sort slice for compare sort.Strings(tags) diff --git a/cluster-autoscaler/cloudprovider/aws/aws_metrics.go b/cluster-autoscaler/cloudprovider/aws/aws_metrics.go index 7d38d17e0acf..96629f61d096 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_metrics.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_metrics.go @@ -17,9 +17,10 @@ limitations under the License. package aws import ( + "errors" "time" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/awserr" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/smithy-go" k8smetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/legacyregistry" ) @@ -49,11 +50,16 @@ func RegisterMetrics() { func observeAWSRequest(endpoint string, err error, start time.Time) { duration := time.Since(start).Seconds() status := "success" + if err != nil { status = "error" - if awsErr, ok := err.(awserr.Error); ok { - status = awsErr.Code() + + // Try to extract a meaningful AWS error code + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + status = apiErr.ErrorCode() } } + requestSummary.WithLabelValues(endpoint, status).Observe(duration) } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go b/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go index b1f33442f71b..e9968102429c 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider.go @@ -17,6 +17,7 @@ limitations under the License. package aws import ( + "context" "fmt" "io" "os" @@ -24,11 +25,14 @@ import ( "strings" "gopkg.in/gcfg.v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/ec2metadata" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/endpoints" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/request" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/session" + + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws/middleware" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/config" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks" + smithymiddleware "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/smithy-go/middleware" "k8s.io/autoscaler/cluster-autoscaler/version" provider_aws "k8s.io/cloud-provider-aws/pkg/providers/v1" "k8s.io/klog/v2" @@ -42,59 +46,64 @@ import ( // // t.Setenv("AWS_REGION", "fanghorn") func createAWSSDKProvider(configReader io.Reader) (*awsSDKProvider, error) { - cfg, err := readAWSCloudConfig(configReader) + cloudConfig, err := readAWSCloudConfig(configReader) if err != nil { klog.Errorf("Couldn't read config: %v", err) return nil, err } - if err = validateOverrides(cfg); err != nil { + if err = validateOverrides(cloudConfig); err != nil { klog.Errorf("Unable to validate custom endpoint overrides: %v", err) return nil, err } - config := aws.NewConfig(). - WithRegion(getRegion()). - WithEndpointResolver(getResolver(cfg)) - - config, err = setMaxRetriesFromEnv(config) - if err != nil { - return nil, err + // Configure all options before building the config + loadOpts := []func(*config.LoadOptions) error{ + config.WithRegion(getRegion()), + config.WithAPIOptions([]func(*smithymiddleware.Stack) error{ + // add cluster-autoscaler to the user-agent to make it easier to identify + middleware.AddUserAgentKeyValue("cluster-autoscaler", version.ClusterAutoscalerVersion), + }), + } + if maxRetries, isSet, err := getMaxRetriesFromEnv(); isSet { + if err != nil { + klog.Errorf("Error getting max retries from env: %v", err) + return nil, err + } + loadOpts = append(loadOpts, config.WithRetryMaxAttempts(maxRetries)) } - sess, err := session.NewSession(config) + awsConfig, err := config.LoadDefaultConfig(context.Background(), loadOpts...) if err != nil { + klog.Errorf("Unable to load default aws config: %v", err) return nil, err } - // add cluster-autoscaler to the user-agent to make it easier to identify - agent := fmt.Sprintf("cluster-autoscaler/v%s", version.ClusterAutoscalerVersion) - sess.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler(agent)) - provider := &awsSDKProvider{ - session: sess, + cfg: awsConfig, + cloudConfig: cloudConfig, } - return provider, nil } -// setMaxRetriesFromEnv sets aws config MaxRetries by reading AWS_MAX_ATTEMPTS +// getMaxRetriesFromEnv retrieves the MaxRetries configuration by reading AWS_MAX_ATTEMPTS // aws sdk does not auto-set these so instead of having more config options we can reuse what the aws cli // does and read AWS_MAX_ATTEMPTS from the env https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html -func setMaxRetriesFromEnv(config *aws.Config) (*aws.Config, error) { +func getMaxRetriesFromEnv() (int, bool, error) { maxRetries := os.Getenv("AWS_MAX_ATTEMPTS") if maxRetries != "" { num, err := strconv.Atoi(maxRetries) if err != nil { - return nil, err + return 0, true, err } - config = config.WithMaxRetries(num) + return num, true, nil } - return config, nil + return 0, false, nil } type awsSDKProvider struct { - session *session.Session + cfg aws.Config + cloudConfig *provider_aws.CloudConfig } // readAWSCloudConfig reads an instance of AWSCloudConfig from config reader. @@ -150,45 +159,99 @@ func validateOverrides(cfg *provider_aws.CloudConfig) error { return nil } -func getResolver(cfg *provider_aws.CloudConfig) endpoints.ResolverFunc { - defaultResolver := endpoints.DefaultResolver() - defaultResolverFn := func(service, region string, - optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { - return defaultResolver.EndpointFor(service, region, optFns...) +type ec2OverrideResolver struct { + defaultResolver ec2.EndpointResolver + cloudConfig *provider_aws.CloudConfig +} + +var _ ec2.EndpointResolver = &ec2OverrideResolver{} + +func newEc2OverrideResolver(cloudConfig *provider_aws.CloudConfig) *ec2OverrideResolver { + return &ec2OverrideResolver{ + defaultResolver: ec2.NewDefaultEndpointResolver(), + cloudConfig: cloudConfig, } - if len(cfg.ServiceOverride) == 0 { - return defaultResolverFn - } - - return func(service, region string, - optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) { - for _, override := range cfg.ServiceOverride { - if override.Service == service && override.Region == region { - return endpoints.ResolvedEndpoint{ - URL: override.URL, - SigningRegion: override.SigningRegion, - SigningMethod: override.SigningMethod, - SigningName: override.SigningName, - }, nil - } +} + +func (r *ec2OverrideResolver) ResolveEndpoint(region string, options ec2.EndpointResolverOptions) (aws.Endpoint, error) { + for _, override := range r.cloudConfig.ServiceOverride { + if override.Service == "ec2" && override.Region == region { + return aws.Endpoint{ + URL: override.URL, + SigningRegion: override.SigningRegion, + SigningMethod: override.SigningMethod, + SigningName: override.SigningName, + }, nil + } + } + return r.defaultResolver.ResolveEndpoint(region, options) +} + +type autoscalingOverrideResolver struct { + defaultResolver autoscaling.EndpointResolver + cloudConfig *provider_aws.CloudConfig +} + +var _ autoscaling.EndpointResolver = &autoscalingOverrideResolver{} + +func newAutoscalingOverrideResolver(cloudConfig *provider_aws.CloudConfig) *autoscalingOverrideResolver { + return &autoscalingOverrideResolver{ + defaultResolver: autoscaling.NewDefaultEndpointResolver(), + cloudConfig: cloudConfig, + } +} + +func (r *autoscalingOverrideResolver) ResolveEndpoint(region string, options autoscaling.EndpointResolverOptions) (aws.Endpoint, error) { + for _, override := range r.cloudConfig.ServiceOverride { + if override.Service == "autoscaling" && override.Region == region { + return aws.Endpoint{ + URL: override.URL, + SigningRegion: override.SigningRegion, + SigningMethod: override.SigningMethod, + SigningName: override.SigningName, + }, nil + } + } + return r.defaultResolver.ResolveEndpoint(region, options) +} + +type eksOverrideResolver struct { + defaultResolver eks.EndpointResolver + cloudConfig *provider_aws.CloudConfig +} + +var _ eks.EndpointResolver = &eksOverrideResolver{} + +func newEksOverrideResolver(cloudConfig *provider_aws.CloudConfig) *eksOverrideResolver { + return &eksOverrideResolver{ + defaultResolver: eks.NewDefaultEndpointResolver(), + cloudConfig: cloudConfig, + } +} + +func (r *eksOverrideResolver) ResolveEndpoint(region string, options eks.EndpointResolverOptions) (aws.Endpoint, error) { + for _, override := range r.cloudConfig.ServiceOverride { + if override.Service == "eks" && override.Region == region { + return aws.Endpoint{ + URL: override.URL, + SigningRegion: override.SigningRegion, + SigningMethod: override.SigningMethod, + SigningName: override.SigningName, + }, nil } - return defaultResolver.EndpointFor(service, region, optFns...) } + return r.defaultResolver.ResolveEndpoint(region, options) } // getRegion deduces the current AWS Region. -func getRegion(cfg ...*aws.Config) string { +func getRegion() string { region, present := os.LookupEnv("AWS_REGION") if !present { - sess, err := session.NewSession() + cfg, err := config.LoadDefaultConfig(context.Background()) if err != nil { - klog.Errorf("Error getting AWS session while retrieving region: %v", err) - } else { - svc := ec2metadata.New(sess, cfg...) - if r, err := svc.Region(); err == nil { - region = r - } + klog.Errorf("Unable to load default aws config: %v", err) } + region = cfg.Region } return region } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider_test.go b/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider_test.go index 1e45343f3538..c0ca154d173a 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_sdk_provider_test.go @@ -17,37 +17,25 @@ limitations under the License. package aws import ( - "encoding/json" "fmt" "github.com/stretchr/testify/assert" "io" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/ec2metadata" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks" provider_aws "k8s.io/cloud-provider-aws/pkg/providers/v1" - "net/http" - "net/http/httptest" - "os" "strings" "testing" ) // TestGetRegion ensures correct source supplies AWS Region. -func TestGetRegion(t *testing.T) { +func TestGetRegionFromEnvironmentVariable(t *testing.T) { key := "AWS_REGION" // Ensure environment variable retains precedence. expected1 := "the-shire-1" t.Setenv(key, expected1) assert.Equal(t, expected1, getRegion()) - // Ensure without environment variable, EC2 Metadata is used. - expected2 := "mordor-2" - expectedjson := ec2metadata.EC2InstanceIdentityDocument{Region: expected2} - js, _ := json.Marshal(expectedjson) - os.Unsetenv(key) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write(js) - })) - cfg := aws.NewConfig().WithEndpoint(server.URL) - assert.Equal(t, expected2, getRegion(cfg)) } func TestOverridesActiveConfig(t *testing.T) { @@ -76,7 +64,7 @@ func TestOverridesActiveConfig(t *testing.T) { [global] [ServiceOverride "1"] Region=sregion - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion SigningMethod = sign `), @@ -89,8 +77,8 @@ func TestOverridesActiveConfig(t *testing.T) { strings.NewReader(` [global] [ServiceOverride "1"] - Service=s3 - URL=https://s3.foo.bar + Service=eks + URL=https://eks.foo.bar SigningRegion=sregion SigningMethod = sign `), @@ -103,7 +91,7 @@ func TestOverridesActiveConfig(t *testing.T) { strings.NewReader(` [global] [ServiceOverride "1"] - Service="s3" + Service="eks" Region=sregion SigningRegion=sregion SigningMethod = sign @@ -117,9 +105,9 @@ func TestOverridesActiveConfig(t *testing.T) { strings.NewReader(` [global] [ServiceOverride "1"] - Service=s3 + Service=eks Region=sregion - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningMethod = sign `), nil, @@ -131,15 +119,15 @@ func TestOverridesActiveConfig(t *testing.T) { strings.NewReader(` [Global] [ServiceOverride "1"] - Service = "s3 " + Service = "eks " Region = sregion - URL = https://s3.foo.bar + URL = https://eks.foo.bar SigningRegion = sregion SigningMethod = v4 `), nil, false, true, - []ServiceDescriptor{{name: "s3", region: "sregion", signingRegion: "sregion", signingMethod: "v4"}}, + []ServiceDescriptor{{name: "eks", region: "sregion", signingRegion: "sregion", signingMethod: "v4"}}, }, { "Multiple Overridden Services", @@ -147,9 +135,9 @@ func TestOverridesActiveConfig(t *testing.T) { [Global] vpc = vpc-abc1234567 [ServiceOverride "1"] - Service=s3 + Service=eks Region=sregion1 - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion1 SigningMethod = v4 [ServiceOverride "2"] @@ -161,7 +149,7 @@ func TestOverridesActiveConfig(t *testing.T) { `), nil, false, true, - []ServiceDescriptor{{name: "s3", region: "sregion1", signingRegion: "sregion1", signingMethod: "v4"}, + []ServiceDescriptor{{name: "eks", region: "sregion1", signingRegion: "sregion1", signingMethod: "v4"}, {name: "ec2", region: "sregion2", signingRegion: "sregion2", signingMethod: "v4"}}, }, { @@ -170,15 +158,15 @@ func TestOverridesActiveConfig(t *testing.T) { [Global] vpc = vpc-abc1234567 [ServiceOverride "1"] - Service=s3 + Service=eks Region=sregion1 - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion SigningMethod = sign [ServiceOverride "2"] - Service=s3 + Service=eks Region=sregion1 - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion SigningMethod = sign `), @@ -191,9 +179,9 @@ func TestOverridesActiveConfig(t *testing.T) { strings.NewReader(` [global] [ServiceOverride "1"] - Service=s3 + Service=eks Region=region1 - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion1 [ServiceOverride "2"] Service=ec2 @@ -204,7 +192,7 @@ func TestOverridesActiveConfig(t *testing.T) { `), nil, false, true, - []ServiceDescriptor{{name: "s3", region: "region1", signingRegion: "sregion1", signingMethod: ""}, + []ServiceDescriptor{{name: "eks", region: "region1", signingRegion: "sregion1", signingMethod: ""}, {name: "ec2", region: "region2", signingRegion: "sregion", signingMethod: "v4"}}, }, { @@ -212,23 +200,23 @@ func TestOverridesActiveConfig(t *testing.T) { strings.NewReader(` [global] [ServiceOverride "1"] - Service=s3 + Service=eks Region=region1 - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion1 SigningMethod = v3 [ServiceOverride "2"] - Service=s3 + Service=eks Region=region2 - URL=https://s3.foo.bar + URL=https://eks.foo.bar SigningRegion=sregion1 SigningMethod = v4 SigningName = "name" `), nil, false, true, - []ServiceDescriptor{{name: "s3", region: "region1", signingRegion: "sregion1", signingMethod: "v3"}, - {name: "s3", region: "region2", signingRegion: "sregion1", signingMethod: "v4", signingName: "name"}}, + []ServiceDescriptor{{name: "eks", region: "region1", signingRegion: "sregion1", signingMethod: "v3"}, + {name: "eks", region: "region2", signingRegion: "sregion1", signingMethod: "v4", signingName: "name"}}, }, } @@ -288,25 +276,35 @@ func TestOverridesActiveConfig(t *testing.T) { sd.signingName, found.SigningName, test.name) } - fn := getResolver(cfg) - ep1, e := fn(sd.name, sd.region, nil) - if e != nil { - t.Errorf("Expected a valid endpoint for %s in case %s", - sd.name, test.name) - } else { - targetName := fmt.Sprintf("https://%s.foo.bar", sd.name) - if ep1.URL != targetName { - t.Errorf("Expected endpoint url: %s, received %s in case %s", - targetName, ep1.URL, test.name) - } - if ep1.SigningRegion != sd.signingRegion { - t.Errorf("Expected signing region '%s', received '%s' in case %s", - sd.signingRegion, ep1.SigningRegion, test.name) - } - if ep1.SigningMethod != sd.signingMethod { - t.Errorf("Expected signing method '%s', received '%s' in case %s", - sd.signingMethod, ep1.SigningRegion, test.name) - } + var endpoint aws.Endpoint + switch sd.name { + case "autoscaling": + resolver := newAutoscalingOverrideResolver(cfg) + endpoint, err = resolver.ResolveEndpoint(sd.region, autoscaling.EndpointResolverOptions{}) + case "ec2": + resolver := newEc2OverrideResolver(cfg) + endpoint, err = resolver.ResolveEndpoint(sd.region, ec2.EndpointResolverOptions{}) + case "eks": + resolver := newEksOverrideResolver(cfg) + endpoint, err = resolver.ResolveEndpoint(sd.region, eks.EndpointResolverOptions{}) + default: + t.Errorf("Unexpected service %s not supported", sd.name) + } + if err != nil { + t.Errorf("Failed to resolve endpoint for service %s in case %s", sd.name, test.name) + } + + if endpoint.URL != targetName { + t.Errorf("Expected endpoint url: %s, received %s in case %s", + targetName, endpoint.URL, test.name) + } + if endpoint.SigningRegion != sd.signingRegion { + t.Errorf("Expected signing region '%s', received '%s' in case %s", + sd.signingRegion, endpoint.SigningRegion, test.name) + } + if endpoint.SigningMethod != sd.signingMethod { + t.Errorf("Expected signing method '%s', received '%s' in case %s", + sd.signingMethod, endpoint.SigningRegion, test.name) } } } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util.go b/cluster-autoscaler/cloudprovider/aws/aws_util.go index b5b34cf07248..c9e10450280d 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_util.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_util.go @@ -17,33 +17,33 @@ limitations under the License. package aws import ( + "context" "errors" "fmt" + "k8s.io/klog/v2" "os" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/ec2metadata" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/session" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" -) - -var ( - ec2MetaDataServiceUrl = "http://169.254.169.254" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/config" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" ) // GenerateEC2InstanceTypes returns a map of ec2 resources -func GenerateEC2InstanceTypes(sess *session.Session) (map[string]*InstanceType, error) { - ec2Client := ec2.New(sess) +func GenerateEC2InstanceTypes(cfg aws.Config) (map[string]*InstanceType, error) { + ec2Client := ec2.NewFromConfig(cfg) input := ec2.DescribeInstanceTypesInput{} instanceTypes := make(map[string]*InstanceType) - if err := ec2Client.DescribeInstanceTypesPages(&input, func(page *ec2.DescribeInstanceTypesOutput, isLastPage bool) bool { + paginator := ec2.NewDescribeInstanceTypesPaginator(ec2Client, &input) + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("failed to describe instance types: %w", err) + } for _, rawInstanceType := range page.InstanceTypes { - instanceTypes[*rawInstanceType.InstanceType] = transformInstanceType(rawInstanceType) + instanceTypes[string(rawInstanceType.InstanceType)] = transformInstanceType(&rawInstanceType) } - return !isLastPage - }); err != nil { - return nil, err } if len(instanceTypes) == 0 { @@ -58,27 +58,27 @@ func GetStaticEC2InstanceTypes() (map[string]*InstanceType, string) { return InstanceTypes, StaticListLastUpdateTime } -func transformInstanceType(rawInstanceType *ec2.InstanceTypeInfo) *InstanceType { +func transformInstanceType(rawInstanceType *ec2types.InstanceTypeInfo) *InstanceType { instanceType := &InstanceType{ - InstanceType: *rawInstanceType.InstanceType, + InstanceType: string(rawInstanceType.InstanceType), } if rawInstanceType.MemoryInfo != nil && rawInstanceType.MemoryInfo.SizeInMiB != nil { instanceType.MemoryMb = *rawInstanceType.MemoryInfo.SizeInMiB } if rawInstanceType.VCpuInfo != nil && rawInstanceType.VCpuInfo.DefaultVCpus != nil { - instanceType.VCPU = *rawInstanceType.VCpuInfo.DefaultVCpus + instanceType.VCPU = int64(*rawInstanceType.VCpuInfo.DefaultVCpus) } if rawInstanceType.GpuInfo != nil && len(rawInstanceType.GpuInfo.Gpus) > 0 { - instanceType.GPU = getGpuCount(rawInstanceType.GpuInfo) + instanceType.GPU = int64(getGpuCount(rawInstanceType.GpuInfo)) } if rawInstanceType.ProcessorInfo != nil && len(rawInstanceType.ProcessorInfo.SupportedArchitectures) > 0 { - instanceType.Architecture = interpretEc2SupportedArchitecure(*rawInstanceType.ProcessorInfo.SupportedArchitectures[0]) + instanceType.Architecture = interpretEc2SupportedArchitecure(string(rawInstanceType.ProcessorInfo.SupportedArchitectures[0])) } return instanceType } -func getGpuCount(gpuInfo *ec2.GpuInfo) int64 { - var gpuCountSum int64 +func getGpuCount(gpuInfo *ec2types.GpuInfo) int32 { + var gpuCountSum int32 for _, gpu := range gpuInfo.Gpus { if gpu.Count != nil { gpuCountSum += *gpu.Count @@ -105,16 +105,15 @@ func interpretEc2SupportedArchitecure(archName string) string { // GetCurrentAwsRegion return region of current cluster without building awsManager func GetCurrentAwsRegion() (string, error) { region, present := os.LookupEnv("AWS_REGION") - if !present { - c := aws.NewConfig(). - WithEndpoint(ec2MetaDataServiceUrl) - sess, err := session.NewSession() + cfg, err := config.LoadDefaultConfig(context.Background()) if err != nil { - return "", fmt.Errorf("failed to create session") + klog.Errorf("Unable to load default aws config: %v", err) } - return ec2metadata.New(sess, c).Region() + region = cfg.Region + } + if region == "" { + return "", errors.New("failed to determine AWS region") } - return region, nil } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util_test.go b/cluster-autoscaler/cloudprovider/aws/aws_util_test.go index b93f963bcb8d..7b40ce7ea475 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_util_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_util_test.go @@ -17,14 +17,11 @@ limitations under the License. package aws import ( - "net/http" - "net/http/httptest" - "os" "testing" "github.com/stretchr/testify/assert" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" ) func TestGetStaticEC2InstanceTypes(t *testing.T) { @@ -33,15 +30,15 @@ func TestGetStaticEC2InstanceTypes(t *testing.T) { } func TestInstanceTypeTransform(t *testing.T) { - rawInstanceType := ec2.InstanceTypeInfo{ - InstanceType: aws.String("c4.xlarge"), - ProcessorInfo: &ec2.ProcessorInfo{ - SupportedArchitectures: []*string{aws.String("x86_64")}, + rawInstanceType := ec2types.InstanceTypeInfo{ + InstanceType: ec2types.InstanceType("c4.xlarge"), + ProcessorInfo: &ec2types.ProcessorInfo{ + SupportedArchitectures: []ec2types.ArchitectureType{ec2types.ArchitectureTypeX8664}, }, - VCpuInfo: &ec2.VCpuInfo{ - DefaultVCpus: aws.Int64(4), + VCpuInfo: &ec2types.VCpuInfo{ + DefaultVCpus: aws.Int32(4), }, - MemoryInfo: &ec2.MemoryInfo{ + MemoryInfo: &ec2types.MemoryInfo{ SizeInMiB: aws.Int64(7680), }, } @@ -89,36 +86,15 @@ func TestInterpretEc2SupportedArchitecure(t *testing.T) { } func TestGetGpuCount(t *testing.T) { - gpuDeviceInfos := []*ec2.GpuDeviceInfo{ - {Count: aws.Int64(8)}, - {Count: aws.Int64(4)}, - {Count: aws.Int64(0)}, + gpuDeviceInfos := []ec2types.GpuDeviceInfo{ + {Count: aws.Int32(8)}, + {Count: aws.Int32(4)}, + {Count: aws.Int32(0)}, } - gpuInfo := ec2.GpuInfo{Gpus: gpuDeviceInfos} + gpuInfo := ec2types.GpuInfo{Gpus: gpuDeviceInfos} - assert.Equal(t, int64(12), getGpuCount(&gpuInfo)) -} - -func TestGetCurrentAwsRegion(t *testing.T) { - region := "us-west-2" - if oldRegion, found := os.LookupEnv("AWS_REGION"); found { - os.Unsetenv("AWS_REGION") - defer os.Setenv("AWS_REGION", oldRegion) - } - - server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - rw.Write([]byte("{\"region\" : \"" + region + "\"}")) - })) - // Close the server when test finishes - defer server.Close() - - ec2MetaDataServiceUrl = server.URL - result, err := GetCurrentAwsRegion() - - assert.Nil(t, err) - assert.NotNil(t, result) - assert.Equal(t, region, result) + assert.Equal(t, int32(12), getGpuCount(&gpuInfo)) } func TestGetCurrentAwsRegionWithRegionEnv(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go b/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go index beba83acb32e..9f33d8058ffa 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go @@ -17,37 +17,48 @@ limitations under the License. package aws import ( + "context" "fmt" "strconv" "time" apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks" + ekstypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks/types" klog "k8s.io/klog/v2" ) // autoScalingI is the interface abstracting specific API calls of the auto-scaling service provided by AWS SDK for use in CA type autoScalingI interface { - DescribeAutoScalingGroupsPages(input *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error - DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) - DescribeScalingActivities(*autoscaling.DescribeScalingActivitiesInput) (*autoscaling.DescribeScalingActivitiesOutput, error) - SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) - TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) + autoscaling.DescribeAutoScalingGroupsAPIClient + autoscaling.DescribeLaunchConfigurationsAPIClient + autoscaling.DescribeScalingActivitiesAPIClient + //DescribeAutoScalingGroupsPages(input *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error + //DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) + //DescribeScalingActivities(*autoscaling.DescribeScalingActivitiesInput) (*autoscaling.DescribeScalingActivitiesOutput, error) + SetDesiredCapacity(ctx context.Context, input *autoscaling.SetDesiredCapacityInput, opts ...func(options *autoscaling.Options)) (*autoscaling.SetDesiredCapacityOutput, error) + TerminateInstanceInAutoScalingGroup(ctx context.Context, input *autoscaling.TerminateInstanceInAutoScalingGroupInput, opts ...func(options *autoscaling.Options)) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) } // ec2I is the interface abstracting specific API calls of the EC2 service provided by AWS SDK for use in CA type ec2I interface { - DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) - DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) - GetInstanceTypesFromInstanceRequirementsPages(input *ec2.GetInstanceTypesFromInstanceRequirementsInput, fn func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) error + ec2.DescribeImagesAPIClient + ec2.DescribeLaunchTemplateVersionsAPIClient + ec2.GetInstanceTypesFromInstanceRequirementsAPIClient + //DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) + //DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) + //GetInstanceTypesFromInstanceRequirementsPages(input *ec2.GetInstanceTypesFromInstanceRequirementsInput, fn func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) error } // eksI is the interface that represents a specific aspect of EKS (Elastic Kubernetes Service) which is provided by AWS SDK for use in CA type eksI interface { - DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) + eks.DescribeNodegroupAPIClient + //DescribeNodegroup(input *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) } // awsWrapper provides several utility methods over the services provided by the AWS SDK @@ -63,7 +74,7 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s NodegroupName: &nodegroupName, } start := time.Now() - r, err := m.DescribeNodegroup(params) + r, err := m.DescribeNodegroup(context.Background(), params) observeAWSRequest("DescribeNodegroup", err, start) if err != nil { return nil, nil, nil, err @@ -77,15 +88,15 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s // Labels will include diskSize, amiType, capacityType, version if r.Nodegroup.DiskSize != nil { - labels["diskSize"] = strconv.FormatInt(*r.Nodegroup.DiskSize, 10) + labels["diskSize"] = strconv.FormatInt(int64(*r.Nodegroup.DiskSize), 10) } - if r.Nodegroup.AmiType != nil && len(*r.Nodegroup.AmiType) > 0 { - labels["amiType"] = *r.Nodegroup.AmiType + if r.Nodegroup.AmiType != "" { + labels["amiType"] = string(r.Nodegroup.AmiType) } - if r.Nodegroup.CapacityType != nil && len(*r.Nodegroup.CapacityType) > 0 { - labels["eks.amazonaws.com/capacityType"] = *r.Nodegroup.CapacityType + if r.Nodegroup.CapacityType != "" { + labels["eks.amazonaws.com/capacityType"] = string(r.Nodegroup.CapacityType) } if r.Nodegroup.Version != nil && len(*r.Nodegroup.Version) > 0 { @@ -99,25 +110,21 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s if r.Nodegroup.Labels != nil && len(r.Nodegroup.Labels) > 0 { labelsMap := r.Nodegroup.Labels for k, v := range labelsMap { - if v != nil { - labels[k] = *v - } + labels[k] = v } } if r.Nodegroup.Tags != nil && len(r.Nodegroup.Tags) > 0 { tagsMap := r.Nodegroup.Tags for k, v := range tagsMap { - if v != nil { - tags[k] = *v - } + tags[k] = v } } if r.Nodegroup.Taints != nil && len(r.Nodegroup.Taints) > 0 { taintList := r.Nodegroup.Taints for _, taint := range taintList { - if taint != nil && taint.Effect != nil && taint.Key != nil && taint.Value != nil { + if taint.Key != nil && taint.Value != nil { formattedEffect, err := taintEksTranslator(taint) if err != nil { return nil, nil, nil, err @@ -134,7 +141,7 @@ func (m *awsWrapper) getManagedNodegroupInfo(nodegroupName string, clusterName s return taints, labels, tags, nil } -func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*string) (map[string]string, error) { +func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []string) (map[string]string, error) { launchConfigurationsToInstanceType := map[string]string{} for i := 0; i < len(launchConfigToQuery); i += 50 { @@ -145,10 +152,10 @@ func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*s } params := &autoscaling.DescribeLaunchConfigurationsInput{ LaunchConfigurationNames: launchConfigToQuery[i:end], - MaxRecords: aws.Int64(50), + MaxRecords: aws.Int32(50), } start := time.Now() - r, err := m.DescribeLaunchConfigurations(params) + r, err := m.DescribeLaunchConfigurations(context.Background(), params) observeAWSRequest("DescribeLaunchConfigurations", err, start) if err != nil { return nil, err @@ -160,8 +167,8 @@ func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*s return launchConfigurationsToInstanceType, nil } -func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling.Group, error) { - asgs := make([]*autoscaling.Group, 0) +func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]autoscalingtypes.AutoScalingGroup, error) { + asgs := make([]autoscalingtypes.AutoScalingGroup, 0) if len(names) == 0 { return asgs, nil } @@ -175,16 +182,20 @@ func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling } input := &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice(names[i:end]), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: names[i:end], + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), } start := time.Now() - err := m.DescribeAutoScalingGroupsPages(input, func(output *autoscaling.DescribeAutoScalingGroupsOutput, _ bool) bool { - asgs = append(asgs, output.AutoScalingGroups...) - // We return true while we want to be called with the next page of - // results, if any. - return true - }) + var err error + paginator := autoscaling.NewDescribeAutoScalingGroupsPaginator(m, input) + for paginator.HasMorePages() { + var page *autoscaling.DescribeAutoScalingGroupsOutput + page, err = paginator.NextPage(context.Background()) + if err != nil { + break + } + asgs = append(asgs, page.AutoScalingGroups...) + } observeAWSRequest("DescribeAutoScalingGroupsPages", err, start) if err != nil { return nil, err @@ -194,38 +205,42 @@ func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling return asgs, nil } -func (m *awsWrapper) getAutoscalingGroupsByTags(tags map[string]string) ([]*autoscaling.Group, error) { - asgs := make([]*autoscaling.Group, 0) +func (m *awsWrapper) getAutoscalingGroupsByTags(tags map[string]string) ([]autoscalingtypes.AutoScalingGroup, error) { + asgs := make([]autoscalingtypes.AutoScalingGroup, 0) if len(tags) == 0 { return asgs, nil } - filters := make([]*autoscaling.Filter, 0) + filters := make([]autoscalingtypes.Filter, 0) for key, value := range tags { if value != "" { - filters = append(filters, &autoscaling.Filter{ + filters = append(filters, autoscalingtypes.Filter{ Name: aws.String(fmt.Sprintf("tag:%s", key)), - Values: []*string{aws.String(value)}, + Values: []string{value}, }) } else { - filters = append(filters, &autoscaling.Filter{ + filters = append(filters, autoscalingtypes.Filter{ Name: aws.String("tag-key"), - Values: []*string{aws.String(key)}, + Values: []string{key}, }) } } input := &autoscaling.DescribeAutoScalingGroupsInput{ Filters: filters, - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), } start := time.Now() - err := m.DescribeAutoScalingGroupsPages(input, func(output *autoscaling.DescribeAutoScalingGroupsOutput, _ bool) bool { - asgs = append(asgs, output.AutoScalingGroups...) - // We return true while we want to be called with the next page of - // results, if any. - return true - }) + var err error + paginator := autoscaling.NewDescribeAutoScalingGroupsPaginator(m, input) + for paginator.HasMorePages() { + var page *autoscaling.DescribeAutoScalingGroupsOutput + page, err = paginator.NextPage(context.Background()) + if err != nil { + break + } + asgs = append(asgs, page.AutoScalingGroups...) + } observeAWSRequest("DescribeAutoScalingGroupsPages", err, start) if err != nil { return nil, err @@ -241,8 +256,8 @@ func (m *awsWrapper) getInstanceTypeByLaunchTemplate(launchTemplate *launchTempl } instanceType := "" - if templateData.InstanceType != nil { - instanceType = *templateData.InstanceType + if templateData.InstanceType != "" { + instanceType = string(templateData.InstanceType) } else if templateData.InstanceRequirements != nil && templateData.ImageId != nil { requirementsRequest, err := m.getRequirementsRequestFromEC2(templateData.InstanceRequirements) if err != nil { @@ -279,17 +294,17 @@ func (m *awsWrapper) getInstanceTypeFromRequirementsOverrides(policy *mixedInsta return "", err } - return instanceType, nil + return string(instanceType), nil } -func (m *awsWrapper) getLaunchTemplateData(templateName string, templateVersion string) (*ec2.ResponseLaunchTemplateData, error) { +func (m *awsWrapper) getLaunchTemplateData(templateName string, templateVersion string) (*ec2types.ResponseLaunchTemplateData, error) { describeTemplateInput := &ec2.DescribeLaunchTemplateVersionsInput{ LaunchTemplateName: aws.String(templateName), - Versions: []*string{aws.String(templateVersion)}, + Versions: []string{templateVersion}, } start := time.Now() - describeData, err := m.DescribeLaunchTemplateVersions(describeTemplateInput) + describeData, err := m.DescribeLaunchTemplateVersions(context.Background(), describeTemplateInput) observeAWSRequest("DescribeLaunchTemplateVersions", err, start) if err != nil { return nil, err @@ -304,22 +319,22 @@ func (m *awsWrapper) getLaunchTemplateData(templateName string, templateVersion return describeData.LaunchTemplateVersions[0].LaunchTemplateData, nil } -func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, requirementsRequest *ec2.InstanceRequirementsRequest) (string, error) { +func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, requirementsRequest *ec2types.InstanceRequirementsRequest) (string, error) { describeImagesInput := &ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String(imageId)}, + ImageIds: []string{imageId}, } start := time.Now() - describeImagesOutput, err := m.DescribeImages(describeImagesInput) + describeImagesOutput, err := m.DescribeImages(context.Background(), describeImagesInput) observeAWSRequest("DescribeImages", err, start) if err != nil { return "", err } - imageArchitectures := []*string{} - imageVirtualizationTypes := []*string{} + var imageArchitectures []ec2types.ArchitectureType + var imageVirtualizationTypes []ec2types.VirtualizationType for _, image := range describeImagesOutput.Images { - imageArchitectures = append(imageArchitectures, image.Architecture) + imageArchitectures = append(imageArchitectures, ec2types.ArchitectureType(image.Architecture)) imageVirtualizationTypes = append(imageVirtualizationTypes, image.VirtualizationType) } @@ -331,12 +346,16 @@ func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, req start = time.Now() var instanceTypes []string - err = m.GetInstanceTypesFromInstanceRequirementsPages(requirementsInput, func(page *ec2.GetInstanceTypesFromInstanceRequirementsOutput, isLastPage bool) bool { + paginator := ec2.NewGetInstanceTypesFromInstanceRequirementsPaginator(m, requirementsInput) + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.Background()) + if err != nil { + break + } for _, instanceType := range page.InstanceTypes { instanceTypes = append(instanceTypes, *instanceType.InstanceType) } - return !isLastPage - }) + } observeAWSRequest("GetInstanceTypesFromInstanceRequirements", err, start) if err != nil { return "", fmt.Errorf("unable to get instance types from requirements: %w", err) @@ -348,64 +367,80 @@ func (m *awsWrapper) getInstanceTypeFromInstanceRequirements(imageId string, req return instanceTypes[0], nil } -func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autoscaling.InstanceRequirements) (*ec2.InstanceRequirementsRequest, error) { - requirementsRequest := ec2.InstanceRequirementsRequest{} +func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autoscalingtypes.InstanceRequirements) (*ec2types.InstanceRequirementsRequest, error) { + requirementsRequest := ec2types.InstanceRequirementsRequest{} // required instance requirements - requirementsRequest.MemoryMiB = &ec2.MemoryMiBRequest{ + requirementsRequest.MemoryMiB = &ec2types.MemoryMiBRequest{ Min: requirements.MemoryMiB.Min, Max: requirements.MemoryMiB.Max, } - requirementsRequest.VCpuCount = &ec2.VCpuCountRangeRequest{ + requirementsRequest.VCpuCount = &ec2types.VCpuCountRangeRequest{ Min: requirements.VCpuCount.Min, Max: requirements.VCpuCount.Max, } // optional instance requirements if requirements.AcceleratorCount != nil { - requirementsRequest.AcceleratorCount = &ec2.AcceleratorCountRequest{ + requirementsRequest.AcceleratorCount = &ec2types.AcceleratorCountRequest{ Min: requirements.AcceleratorCount.Min, Max: requirements.AcceleratorCount.Max, } } if requirements.AcceleratorManufacturers != nil { - requirementsRequest.AcceleratorManufacturers = requirements.AcceleratorManufacturers + var acceleratorManufacturers []ec2types.AcceleratorManufacturer + for _, manufacturer := range requirements.AcceleratorManufacturers { + acceleratorManufacturers = append(acceleratorManufacturers, ec2types.AcceleratorManufacturer(manufacturer)) + } + requirementsRequest.AcceleratorManufacturers = acceleratorManufacturers } if requirements.AcceleratorNames != nil { - requirementsRequest.AcceleratorNames = requirements.AcceleratorNames + var acceleratorNames []ec2types.AcceleratorName + for _, name := range requirements.AcceleratorNames { + acceleratorNames = append(acceleratorNames, ec2types.AcceleratorName(name)) + } + requirementsRequest.AcceleratorNames = acceleratorNames } if requirements.AcceleratorTotalMemoryMiB != nil { - requirementsRequest.AcceleratorTotalMemoryMiB = &ec2.AcceleratorTotalMemoryMiBRequest{ + requirementsRequest.AcceleratorTotalMemoryMiB = &ec2types.AcceleratorTotalMemoryMiBRequest{ Min: requirements.AcceleratorTotalMemoryMiB.Min, Max: requirements.AcceleratorTotalMemoryMiB.Max, } } if requirements.AcceleratorTypes != nil { - requirementsRequest.AcceleratorTypes = requirements.AcceleratorTypes + var acceleratorTypes []ec2types.AcceleratorType + for _, acceleratorType := range requirements.AcceleratorTypes { + acceleratorTypes = append(acceleratorTypes, ec2types.AcceleratorType(acceleratorType)) + } + requirementsRequest.AcceleratorTypes = acceleratorTypes } - if requirements.BareMetal != nil { - requirementsRequest.BareMetal = requirements.BareMetal + if requirements.BareMetal != "" { + requirementsRequest.BareMetal = ec2types.BareMetal(requirements.BareMetal) } if requirements.BaselineEbsBandwidthMbps != nil { - requirementsRequest.BaselineEbsBandwidthMbps = &ec2.BaselineEbsBandwidthMbpsRequest{ + requirementsRequest.BaselineEbsBandwidthMbps = &ec2types.BaselineEbsBandwidthMbpsRequest{ Min: requirements.BaselineEbsBandwidthMbps.Min, Max: requirements.BaselineEbsBandwidthMbps.Max, } } - if requirements.BurstablePerformance != nil { - requirementsRequest.BurstablePerformance = requirements.BurstablePerformance + if requirements.BurstablePerformance != "" { + requirementsRequest.BurstablePerformance = ec2types.BurstablePerformance(requirements.BurstablePerformance) } if requirements.CpuManufacturers != nil { - requirementsRequest.CpuManufacturers = requirements.CpuManufacturers + var cpuManufacturers []ec2types.CpuManufacturer + for _, manufacturer := range requirements.CpuManufacturers { + cpuManufacturers = append(cpuManufacturers, ec2types.CpuManufacturer(manufacturer)) + } + requirementsRequest.CpuManufacturers = cpuManufacturers } if requirements.ExcludedInstanceTypes != nil { @@ -413,26 +448,34 @@ func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autosca } if requirements.InstanceGenerations != nil { - requirementsRequest.InstanceGenerations = requirements.InstanceGenerations + var instanceGenerations []ec2types.InstanceGeneration + for _, generation := range requirements.InstanceGenerations { + instanceGenerations = append(instanceGenerations, ec2types.InstanceGeneration(generation)) + } + requirementsRequest.InstanceGenerations = instanceGenerations } - if requirements.LocalStorage != nil { - requirementsRequest.LocalStorage = requirements.LocalStorage + if requirements.LocalStorage != "" { + requirementsRequest.LocalStorage = ec2types.LocalStorage(requirements.LocalStorage) } if requirements.LocalStorageTypes != nil { - requirementsRequest.LocalStorageTypes = requirements.LocalStorageTypes + var localStorageTypes []ec2types.LocalStorageType + for _, localStorageType := range requirements.LocalStorageTypes { + localStorageTypes = append(localStorageTypes, ec2types.LocalStorageType(localStorageType)) + } + requirementsRequest.LocalStorageTypes = localStorageTypes } if requirements.MemoryGiBPerVCpu != nil { - requirementsRequest.MemoryGiBPerVCpu = &ec2.MemoryGiBPerVCpuRequest{ + requirementsRequest.MemoryGiBPerVCpu = &ec2types.MemoryGiBPerVCpuRequest{ Min: requirements.MemoryGiBPerVCpu.Min, Max: requirements.MemoryGiBPerVCpu.Max, } } if requirements.NetworkInterfaceCount != nil { - requirementsRequest.NetworkInterfaceCount = &ec2.NetworkInterfaceCountRequest{ + requirementsRequest.NetworkInterfaceCount = &ec2types.NetworkInterfaceCountRequest{ Min: requirements.NetworkInterfaceCount.Min, Max: requirements.NetworkInterfaceCount.Max, } @@ -451,7 +494,7 @@ func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autosca } if requirements.TotalLocalStorageGB != nil { - requirementsRequest.TotalLocalStorageGB = &ec2.TotalLocalStorageGBRequest{ + requirementsRequest.TotalLocalStorageGB = &ec2types.TotalLocalStorageGBRequest{ Min: requirements.TotalLocalStorageGB.Min, Max: requirements.TotalLocalStorageGB.Max, } @@ -460,23 +503,23 @@ func (m *awsWrapper) getRequirementsRequestFromAutoscaling(requirements *autosca return &requirementsRequest, nil } -func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceRequirements) (*ec2.InstanceRequirementsRequest, error) { - requirementsRequest := ec2.InstanceRequirementsRequest{} +func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2types.InstanceRequirements) (*ec2types.InstanceRequirementsRequest, error) { + requirementsRequest := ec2types.InstanceRequirementsRequest{} // required instance requirements - requirementsRequest.MemoryMiB = &ec2.MemoryMiBRequest{ + requirementsRequest.MemoryMiB = &ec2types.MemoryMiBRequest{ Min: requirements.MemoryMiB.Min, Max: requirements.MemoryMiB.Max, } - requirementsRequest.VCpuCount = &ec2.VCpuCountRangeRequest{ + requirementsRequest.VCpuCount = &ec2types.VCpuCountRangeRequest{ Min: requirements.VCpuCount.Min, Max: requirements.VCpuCount.Max, } // optional instance requirements if requirements.AcceleratorCount != nil { - requirementsRequest.AcceleratorCount = &ec2.AcceleratorCountRequest{ + requirementsRequest.AcceleratorCount = &ec2types.AcceleratorCountRequest{ Min: requirements.AcceleratorCount.Min, Max: requirements.AcceleratorCount.Max, } @@ -491,7 +534,7 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq } if requirements.AcceleratorTotalMemoryMiB != nil { - requirementsRequest.AcceleratorTotalMemoryMiB = &ec2.AcceleratorTotalMemoryMiBRequest{ + requirementsRequest.AcceleratorTotalMemoryMiB = &ec2types.AcceleratorTotalMemoryMiBRequest{ Min: requirements.AcceleratorTotalMemoryMiB.Min, Max: requirements.AcceleratorTotalMemoryMiB.Max, } @@ -501,18 +544,18 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq requirementsRequest.AcceleratorTypes = requirements.AcceleratorTypes } - if requirements.BareMetal != nil { + if requirements.BareMetal != "" { requirementsRequest.BareMetal = requirements.BareMetal } if requirements.BaselineEbsBandwidthMbps != nil { - requirementsRequest.BaselineEbsBandwidthMbps = &ec2.BaselineEbsBandwidthMbpsRequest{ + requirementsRequest.BaselineEbsBandwidthMbps = &ec2types.BaselineEbsBandwidthMbpsRequest{ Min: requirements.BaselineEbsBandwidthMbps.Min, Max: requirements.BaselineEbsBandwidthMbps.Max, } } - if requirements.BurstablePerformance != nil { + if requirements.BurstablePerformance != "" { requirementsRequest.BurstablePerformance = requirements.BurstablePerformance } @@ -528,7 +571,7 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq requirementsRequest.InstanceGenerations = requirements.InstanceGenerations } - if requirements.LocalStorage != nil { + if requirements.LocalStorage != "" { requirementsRequest.LocalStorage = requirements.LocalStorage } @@ -537,14 +580,14 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq } if requirements.MemoryGiBPerVCpu != nil { - requirementsRequest.MemoryGiBPerVCpu = &ec2.MemoryGiBPerVCpuRequest{ + requirementsRequest.MemoryGiBPerVCpu = &ec2types.MemoryGiBPerVCpuRequest{ Min: requirements.MemoryGiBPerVCpu.Min, Max: requirements.MemoryGiBPerVCpu.Max, } } if requirements.NetworkInterfaceCount != nil { - requirementsRequest.NetworkInterfaceCount = &ec2.NetworkInterfaceCountRequest{ + requirementsRequest.NetworkInterfaceCount = &ec2types.NetworkInterfaceCountRequest{ Min: requirements.NetworkInterfaceCount.Min, Max: requirements.NetworkInterfaceCount.Max, } @@ -563,7 +606,7 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq } if requirements.TotalLocalStorageGB != nil { - requirementsRequest.TotalLocalStorageGB = &ec2.TotalLocalStorageGBRequest{ + requirementsRequest.TotalLocalStorageGB = &ec2types.TotalLocalStorageGBRequest{ Min: requirements.TotalLocalStorageGB.Min, Max: requirements.TotalLocalStorageGB.Max, } @@ -572,64 +615,80 @@ func (m *awsWrapper) getRequirementsRequestFromEC2(requirements *ec2.InstanceReq return &requirementsRequest, nil } -func (m *awsWrapper) getEC2RequirementsFromAutoscaling(autoscalingRequirements *autoscaling.InstanceRequirements) (*ec2.InstanceRequirements, error) { - ec2Requirements := ec2.InstanceRequirements{} +func (m *awsWrapper) getEC2RequirementsFromAutoscaling(autoscalingRequirements *autoscalingtypes.InstanceRequirements) (*ec2types.InstanceRequirements, error) { + ec2Requirements := ec2types.InstanceRequirements{} // required instance requirements - ec2Requirements.MemoryMiB = &ec2.MemoryMiB{ + ec2Requirements.MemoryMiB = &ec2types.MemoryMiB{ Min: autoscalingRequirements.MemoryMiB.Min, Max: autoscalingRequirements.MemoryMiB.Max, } - ec2Requirements.VCpuCount = &ec2.VCpuCountRange{ + ec2Requirements.VCpuCount = &ec2types.VCpuCountRange{ Min: autoscalingRequirements.VCpuCount.Min, Max: autoscalingRequirements.VCpuCount.Max, } // optional instance requirements if autoscalingRequirements.AcceleratorCount != nil { - ec2Requirements.AcceleratorCount = &ec2.AcceleratorCount{ + ec2Requirements.AcceleratorCount = &ec2types.AcceleratorCount{ Min: autoscalingRequirements.AcceleratorCount.Min, Max: autoscalingRequirements.AcceleratorCount.Max, } } if autoscalingRequirements.AcceleratorManufacturers != nil { - ec2Requirements.AcceleratorManufacturers = autoscalingRequirements.AcceleratorManufacturers + var acceleratorManufacturers []ec2types.AcceleratorManufacturer + for _, manufacturer := range autoscalingRequirements.AcceleratorManufacturers { + acceleratorManufacturers = append(acceleratorManufacturers, ec2types.AcceleratorManufacturer(manufacturer)) + } + ec2Requirements.AcceleratorManufacturers = acceleratorManufacturers } if autoscalingRequirements.AcceleratorNames != nil { - ec2Requirements.AcceleratorNames = autoscalingRequirements.AcceleratorNames + var acceleratorNames []ec2types.AcceleratorName + for _, acceleratorName := range autoscalingRequirements.AcceleratorNames { + acceleratorNames = append(acceleratorNames, ec2types.AcceleratorName(acceleratorName)) + } + ec2Requirements.AcceleratorNames = acceleratorNames } if autoscalingRequirements.AcceleratorTotalMemoryMiB != nil { - ec2Requirements.AcceleratorTotalMemoryMiB = &ec2.AcceleratorTotalMemoryMiB{ + ec2Requirements.AcceleratorTotalMemoryMiB = &ec2types.AcceleratorTotalMemoryMiB{ Min: autoscalingRequirements.AcceleratorTotalMemoryMiB.Min, Max: autoscalingRequirements.AcceleratorTotalMemoryMiB.Max, } } if autoscalingRequirements.AcceleratorTypes != nil { - ec2Requirements.AcceleratorTypes = autoscalingRequirements.AcceleratorTypes + var acceleratorTypes []ec2types.AcceleratorType + for _, acceleratorType := range autoscalingRequirements.AcceleratorTypes { + acceleratorTypes = append(acceleratorTypes, ec2types.AcceleratorType(acceleratorType)) + } + ec2Requirements.AcceleratorTypes = acceleratorTypes } - if autoscalingRequirements.BareMetal != nil { - ec2Requirements.BareMetal = autoscalingRequirements.BareMetal + if autoscalingRequirements.BareMetal != "" { + ec2Requirements.BareMetal = ec2types.BareMetal(autoscalingRequirements.BareMetal) } if autoscalingRequirements.BaselineEbsBandwidthMbps != nil { - ec2Requirements.BaselineEbsBandwidthMbps = &ec2.BaselineEbsBandwidthMbps{ + ec2Requirements.BaselineEbsBandwidthMbps = &ec2types.BaselineEbsBandwidthMbps{ Min: autoscalingRequirements.BaselineEbsBandwidthMbps.Min, Max: autoscalingRequirements.BaselineEbsBandwidthMbps.Max, } } - if autoscalingRequirements.BurstablePerformance != nil { - ec2Requirements.BurstablePerformance = autoscalingRequirements.BurstablePerformance + if autoscalingRequirements.BurstablePerformance != "" { + ec2Requirements.BurstablePerformance = ec2types.BurstablePerformance(autoscalingRequirements.BurstablePerformance) } if autoscalingRequirements.CpuManufacturers != nil { - ec2Requirements.CpuManufacturers = autoscalingRequirements.CpuManufacturers + var cpuManufacturers []ec2types.CpuManufacturer + for _, manufacturer := range autoscalingRequirements.CpuManufacturers { + cpuManufacturers = append(cpuManufacturers, ec2types.CpuManufacturer(manufacturer)) + } + ec2Requirements.CpuManufacturers = cpuManufacturers } if autoscalingRequirements.ExcludedInstanceTypes != nil { @@ -637,26 +696,34 @@ func (m *awsWrapper) getEC2RequirementsFromAutoscaling(autoscalingRequirements * } if autoscalingRequirements.InstanceGenerations != nil { - ec2Requirements.InstanceGenerations = autoscalingRequirements.InstanceGenerations + var instanceGenerations []ec2types.InstanceGeneration + for _, generation := range autoscalingRequirements.InstanceGenerations { + instanceGenerations = append(instanceGenerations, ec2types.InstanceGeneration(generation)) + } + ec2Requirements.InstanceGenerations = instanceGenerations } - if autoscalingRequirements.LocalStorage != nil { - ec2Requirements.LocalStorage = autoscalingRequirements.LocalStorage + if autoscalingRequirements.LocalStorage != "" { + ec2Requirements.LocalStorage = ec2types.LocalStorage(autoscalingRequirements.LocalStorage) } if autoscalingRequirements.LocalStorageTypes != nil { - ec2Requirements.LocalStorageTypes = autoscalingRequirements.LocalStorageTypes + var localStorageTypes []ec2types.LocalStorageType + for _, localStorageType := range autoscalingRequirements.LocalStorageTypes { + localStorageTypes = append(localStorageTypes, ec2types.LocalStorageType(localStorageType)) + } + ec2Requirements.LocalStorageTypes = localStorageTypes } if autoscalingRequirements.MemoryGiBPerVCpu != nil { - ec2Requirements.MemoryGiBPerVCpu = &ec2.MemoryGiBPerVCpu{ + ec2Requirements.MemoryGiBPerVCpu = &ec2types.MemoryGiBPerVCpu{ Min: autoscalingRequirements.MemoryGiBPerVCpu.Min, Max: autoscalingRequirements.MemoryGiBPerVCpu.Max, } } if autoscalingRequirements.NetworkInterfaceCount != nil { - ec2Requirements.NetworkInterfaceCount = &ec2.NetworkInterfaceCount{ + ec2Requirements.NetworkInterfaceCount = &ec2types.NetworkInterfaceCount{ Min: autoscalingRequirements.NetworkInterfaceCount.Min, Max: autoscalingRequirements.NetworkInterfaceCount.Max, } @@ -675,7 +742,7 @@ func (m *awsWrapper) getEC2RequirementsFromAutoscaling(autoscalingRequirements * } if autoscalingRequirements.TotalLocalStorageGB != nil { - ec2Requirements.TotalLocalStorageGB = &ec2.TotalLocalStorageGB{ + ec2Requirements.TotalLocalStorageGB = &ec2types.TotalLocalStorageGB{ Min: autoscalingRequirements.TotalLocalStorageGB.Min, Max: autoscalingRequirements.TotalLocalStorageGB.Max, } @@ -711,9 +778,9 @@ func (m *awsWrapper) getInstanceTypesForAsgs(asgs []*asg) (map[string]string, er klog.V(4).Infof("%d launch templates to query", len(launchTemplatesToQuery)) // Query these all at once to minimize AWS API calls - launchConfigNames := make([]*string, 0, len(launchConfigsToQuery)) + launchConfigNames := make([]string, 0, len(launchConfigsToQuery)) for _, cfgName := range launchConfigsToQuery { - launchConfigNames = append(launchConfigNames, aws.String(cfgName)) + launchConfigNames = append(launchConfigNames, cfgName) } launchConfigs, err := m.getInstanceTypeByLaunchConfigNames(launchConfigNames) if err != nil { @@ -755,7 +822,7 @@ func (m *awsWrapper) getInstanceTypesForAsgs(asgs []*asg) (map[string]string, er return results, nil } -func buildLaunchTemplateFromSpec(ltSpec *autoscaling.LaunchTemplateSpecification) *launchTemplate { +func buildLaunchTemplateFromSpec(ltSpec *autoscalingtypes.LaunchTemplateSpecification) *launchTemplate { // NOTE(jaypipes): The LaunchTemplateSpecification.Version is a pointer to // string. When the pointer is nil, EC2 AutoScaling API considers the value // to be "$Default", however aws.StringValue(ltSpec.Version) will return an @@ -777,26 +844,26 @@ func buildLaunchTemplateFromSpec(ltSpec *autoscaling.LaunchTemplateSpecification if ltSpec.Version == nil { version = "$Default" } else { - version = aws.StringValue(ltSpec.Version) + version = aws.ToString(ltSpec.Version) } return &launchTemplate{ - name: aws.StringValue(ltSpec.LaunchTemplateName), + name: aws.ToString(ltSpec.LaunchTemplateName), version: version, } } -func taintEksTranslator(t *eks.Taint) (apiv1.TaintEffect, error) { +func taintEksTranslator(t ekstypes.Taint) (apiv1.TaintEffect, error) { // Translation between AWS EKS and Kubernetes taints // // See: // // https://docs.aws.amazon.com/eks/latest/APIReference/API_Taint.html - switch effect := *t.Effect; effect { - case eks.TaintEffectNoSchedule: + switch effect := t.Effect; effect { + case ekstypes.TaintEffectNoSchedule: return apiv1.TaintEffectNoSchedule, nil - case eks.TaintEffectNoExecute: + case ekstypes.TaintEffectNoExecute: return apiv1.TaintEffectNoExecute, nil - case eks.TaintEffectPreferNoSchedule: + case ekstypes.TaintEffectPreferNoSchedule: return apiv1.TaintEffectPreferNoSchedule, nil default: return "", fmt.Errorf("couldn't translate EKS DescribeNodegroup response taint %s into Kubernetes format", effect) diff --git a/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go b/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go index 44b745014cf3..6faf415f4e1a 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go @@ -17,6 +17,7 @@ limitations under the License. package aws import ( + "context" "errors" "fmt" "strconv" @@ -25,38 +26,43 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/autoscaling" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/autoscaling/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks" + ekstypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks/types" ) type autoScalingMock struct { mock.Mock } -func (a *autoScalingMock) DescribeAutoScalingGroupsPages(i *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error { - args := a.Called(i, fn) - return args.Error(0) +var _ autoScalingI = &autoScalingMock{} + +func (a *autoScalingMock) DescribeAutoScalingGroups(ctx context.Context, i *autoscaling.DescribeAutoScalingGroupsInput, opts ...func(*autoscaling.Options)) (*autoscaling.DescribeAutoScalingGroupsOutput, error) { + args := a.Called(ctx, i) + return args.Get(0).(*autoscaling.DescribeAutoScalingGroupsOutput), nil } -func (a *autoScalingMock) DescribeLaunchConfigurations(i *autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) { - args := a.Called(i) +func (a *autoScalingMock) DescribeLaunchConfigurations(ctx context.Context, i *autoscaling.DescribeLaunchConfigurationsInput, opts ...func(options *autoscaling.Options)) (*autoscaling.DescribeLaunchConfigurationsOutput, error) { + args := a.Called(ctx, i) return args.Get(0).(*autoscaling.DescribeLaunchConfigurationsOutput), nil } -func (a *autoScalingMock) DescribeScalingActivities(i *autoscaling.DescribeScalingActivitiesInput) (*autoscaling.DescribeScalingActivitiesOutput, error) { - args := a.Called(i) +func (a *autoScalingMock) DescribeScalingActivities(ctx context.Context, i *autoscaling.DescribeScalingActivitiesInput, opts ...func(options *autoscaling.Options)) (*autoscaling.DescribeScalingActivitiesOutput, error) { + args := a.Called(ctx, i) return args.Get(0).(*autoscaling.DescribeScalingActivitiesOutput), args.Error(1) } -func (a *autoScalingMock) SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) { - args := a.Called(input) +func (a *autoScalingMock) SetDesiredCapacity(ctx context.Context, input *autoscaling.SetDesiredCapacityInput, opts ...func(options *autoscaling.Options)) (*autoscaling.SetDesiredCapacityOutput, error) { + args := a.Called(ctx, input) return args.Get(0).(*autoscaling.SetDesiredCapacityOutput), nil } -func (a *autoScalingMock) TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) { - args := a.Called(input) +func (a *autoScalingMock) TerminateInstanceInAutoScalingGroup(ctx context.Context, input *autoscaling.TerminateInstanceInAutoScalingGroupInput, opts ...func(options *autoscaling.Options)) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) { + args := a.Called(ctx, input) return args.Get(0).(*autoscaling.TerminateInstanceInAutoScalingGroupOutput), nil } @@ -64,27 +70,31 @@ type ec2Mock struct { mock.Mock } -func (e *ec2Mock) DescribeImages(input *ec2.DescribeImagesInput) (*ec2.DescribeImagesOutput, error) { - args := e.Called(input) +var _ ec2I = &ec2Mock{} + +func (e *ec2Mock) DescribeImages(ctx context.Context, i *ec2.DescribeImagesInput, opts ...func(options *ec2.Options)) (*ec2.DescribeImagesOutput, error) { + args := e.Called(ctx, i) return args.Get(0).(*ec2.DescribeImagesOutput), nil } -func (e *ec2Mock) DescribeLaunchTemplateVersions(i *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) { - args := e.Called(i) +func (e *ec2Mock) DescribeLaunchTemplateVersions(ctx context.Context, i *ec2.DescribeLaunchTemplateVersionsInput, opts ...func(options *ec2.Options)) (*ec2.DescribeLaunchTemplateVersionsOutput, error) { + args := e.Called(ctx, i) return args.Get(0).(*ec2.DescribeLaunchTemplateVersionsOutput), nil } -func (e *ec2Mock) GetInstanceTypesFromInstanceRequirementsPages(input *ec2.GetInstanceTypesFromInstanceRequirementsInput, fn func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) error { - args := e.Called(input, fn) - return args.Error(0) +func (e *ec2Mock) GetInstanceTypesFromInstanceRequirements(ctx context.Context, i *ec2.GetInstanceTypesFromInstanceRequirementsInput, opts ...func(options *ec2.Options)) (*ec2.GetInstanceTypesFromInstanceRequirementsOutput, error) { + args := e.Called(ctx, i) + return args.Get(0).(*ec2.GetInstanceTypesFromInstanceRequirementsOutput), nil } type eksMock struct { mock.Mock } -func (k *eksMock) DescribeNodegroup(i *eks.DescribeNodegroupInput) (*eks.DescribeNodegroupOutput, error) { - args := k.Called(i) +var _ eksI = &eksMock{} + +func (k *eksMock) DescribeNodegroup(ctx context.Context, i *eks.DescribeNodegroupInput, opts ...func(options *eks.Options)) (*eks.DescribeNodegroupOutput, error) { + args := k.Called(ctx, i) if args.Get(0) == nil && args.Get(1) == nil { return nil, nil @@ -114,28 +124,28 @@ func TestGetManagedNodegroup(t *testing.T) { nodegroupName := "testNodegroup" clusterName := "testCluster" - taintEffect1 := eks.TaintEffectNoSchedule + taintEffect1 := ekstypes.TaintEffectNoSchedule taintEffectTranslated1 := apiv1.TaintEffectNoSchedule taintKey1 := "key 1" taintValue1 := "value 1" - taint1 := eks.Taint{ - Effect: &taintEffect1, + taint1 := ekstypes.Taint{ + Effect: taintEffect1, Key: &taintKey1, Value: &taintValue1, } - taintEffect2 := eks.TaintEffectNoExecute + taintEffect2 := ekstypes.TaintEffectNoExecute taintEffectTranslated2 := apiv1.TaintEffectNoExecute taintKey2 := "key 2" taintValue2 := "value 2" - taint2 := eks.Taint{ - Effect: &taintEffect2, + taint2 := ekstypes.Taint{ + Effect: taintEffect2, Key: &taintKey2, Value: &taintValue2, } amiType := "testAmiType" - diskSize := int64(100) + diskSize := int32(100) capacityType := "testCapacityType" k8sVersion := "1.19" @@ -145,22 +155,25 @@ func TestGetManagedNodegroup(t *testing.T) { tagValue2 := "value 2" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, - Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2}, + Labels: map[string]string{labelKey1: labelValue1, labelKey2: labelValue2}, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, - Taints: []*eks.Taint{&taint1, &taint2}, - Tags: map[string]*string{tagKey1: &tagValue1, tagKey2: &tagValue2}, + Taints: []ekstypes.Taint{taint1, taint2}, + Tags: map[string]string{tagKey1: tagValue1, tagKey2: tagValue2}, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) taintList, labelMap, tagMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName) assert.Nil(t, err) @@ -174,7 +187,7 @@ func TestGetManagedNodegroup(t *testing.T) { assert.Equal(t, len(labelMap), 7) assert.Equal(t, labelMap[labelKey1], labelValue1) assert.Equal(t, labelMap[labelKey2], labelValue2) - assert.Equal(t, labelMap["diskSize"], strconv.FormatInt(diskSize, 10)) + assert.Equal(t, labelMap["diskSize"], strconv.FormatInt(int64(diskSize), 10)) assert.Equal(t, labelMap["amiType"], amiType) assert.Equal(t, labelMap["eks.amazonaws.com/capacityType"], capacityType) assert.Equal(t, labelMap["k8sVersion"], k8sVersion) @@ -200,22 +213,25 @@ func TestGetManagedNodegroupWithNilValues(t *testing.T) { k8sVersion := "1.19" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: nil, Labels: nil, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, Taints: nil, Tags: nil, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) taintList, labelMap, tagMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName) assert.Nil(t, err) @@ -244,22 +260,25 @@ func TestGetManagedNodegroupWithEmptyValues(t *testing.T) { k8sVersion := "1.19" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: nil, - Labels: make(map[string]*string), + Labels: make(map[string]string), NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, - Taints: make([]*eks.Taint, 0), - Tags: make(map[string]*string), + Taints: make([]ekstypes.Taint, 0), + Tags: make(map[string]string), } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) taintList, labelMap, tagMap, err := awsWrapper.getManagedNodegroupInfo(nodegroupName, clusterName) assert.Nil(t, err) @@ -287,28 +306,22 @@ func TestMoreThen100Groups(t *testing.T) { } // First batch, first 100 elements - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice(names[:100]), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: names[:100], + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("asg-1", 1, "test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("asg-1", 1, "test-instance-id"), nil) // Second batch, element 101 - a.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroups", + mock.Anything, &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"asg-100"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + AutoScalingGroupNames: []string{"asg-100"}, + MaxRecords: aws.Int32(maxRecordsReturnedByAPI), }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("asg-2", 1, "test-instance-id"), false) - }).Return(nil) + ).Return(testNamedDescribeAutoScalingGroupsOutput("asg-2", 1, "test-instance-id"), nil) asgs, err := awsWrapper.getAutoscalingGroupsByNames(names) assert.Nil(t, err) @@ -326,29 +339,41 @@ func TestGetInstanceTypesForAsgs(t *testing.T) { a := &autoScalingMock{} e := &ec2Mock{} - a.On("DescribeLaunchConfigurations", &autoscaling.DescribeLaunchConfigurationsInput{ - LaunchConfigurationNames: []*string{aws.String(ltName)}, - MaxRecords: aws.Int64(50), - }).Return(&autoscaling.DescribeLaunchConfigurationsOutput{ - LaunchConfigurations: []*autoscaling.LaunchConfiguration{ - { - LaunchConfigurationName: aws.String(ltName), - InstanceType: aws.String(instanceType), + a.On("DescribeLaunchConfigurations", + mock.Anything, + &autoscaling.DescribeLaunchConfigurationsInput{ + LaunchConfigurationNames: []string{ltName}, + MaxRecords: aws.Int32(50), + }, + ).Return( + &autoscaling.DescribeLaunchConfigurationsOutput{ + LaunchConfigurations: []autoscalingtypes.LaunchConfiguration{ + { + LaunchConfigurationName: aws.String(ltName), + InstanceType: aws.String(instanceType), + }, }, }, - }) - e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String(ltName), - Versions: []*string{aws.String(ltVersion)}, - }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ - { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - InstanceType: aws.String(instanceType), + nil, + ) + e.On("DescribeLaunchTemplateVersions", + mock.Anything, + &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String(ltName), + Versions: []string{ltVersion}, + }, + ).Return( + &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + InstanceType: ec2types.InstanceType(instanceType), + }, }, }, }, - }) + nil, + ) t.Setenv("AWS_REGION", "fanghorn") @@ -418,20 +443,20 @@ func TestGetInstanceTypesFromInstanceRequirementsOverrides(t *testing.T) { name: "launchTemplateName", version: "1", }, - instanceRequirementsOverrides: &autoscaling.InstanceRequirements{ - VCpuCount: &autoscaling.VCpuCountRequest{ - Min: aws.Int64(4), - Max: aws.Int64(8), + instanceRequirementsOverrides: &autoscalingtypes.InstanceRequirements{ + VCpuCount: &autoscalingtypes.VCpuCountRequest{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, - MemoryMiB: &autoscaling.MemoryMiBRequest{ - Min: aws.Int64(4), - Max: aws.Int64(8), + MemoryMiB: &autoscalingtypes.MemoryMiBRequest{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, - AcceleratorTypes: []*string{aws.String(autoscaling.AcceleratorTypeGpu)}, - AcceleratorManufacturers: []*string{aws.String(autoscaling.AcceleratorManufacturerNvidia)}, - AcceleratorCount: &autoscaling.AcceleratorCountRequest{ - Min: aws.Int64(4), - Max: aws.Int64(8), + AcceleratorTypes: []autoscalingtypes.AcceleratorType{autoscalingtypes.AcceleratorTypeGpu}, + AcceleratorManufacturers: []autoscalingtypes.AcceleratorManufacturer{autoscalingtypes.AcceleratorManufacturerNvidia}, + AcceleratorCount: &autoscalingtypes.AcceleratorCountRequest{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, }, } @@ -443,49 +468,61 @@ func TestGetInstanceTypesFromInstanceRequirementsOverrides(t *testing.T) { eksI: nil, } - e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String("launchTemplateName"), - Versions: []*string{aws.String("1")}, - }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ - { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - ImageId: aws.String("123"), + e.On("DescribeLaunchTemplateVersions", + mock.Anything, + &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String("launchTemplateName"), + Versions: []string{"1"}, + }, + ).Return( + &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + ImageId: aws.String("123"), + }, }, }, }, - }) + nil, + ) - e.On("DescribeImages", &ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String("123")}, - }).Return(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Architecture: aws.String("x86_64"), - VirtualizationType: aws.String("xen"), + e.On("DescribeImages", + mock.Anything, + &ec2.DescribeImagesInput{ + ImageIds: []string{"123"}, + }, + ).Return( + &ec2.DescribeImagesOutput{ + Images: []ec2types.Image{ + { + Architecture: ec2types.ArchitectureValuesX8664, + VirtualizationType: "xen", + }, }, }, - }) + nil, + ) requirements, err := awsWrapper.getRequirementsRequestFromAutoscaling(mixedInstancesPolicy.instanceRequirementsOverrides) assert.NoError(t, err) - e.On("GetInstanceTypesFromInstanceRequirementsPages", + e.On("GetInstanceTypesFromInstanceRequirements", + mock.Anything, &ec2.GetInstanceTypesFromInstanceRequirementsInput{ - ArchitectureTypes: []*string{aws.String("x86_64")}, + ArchitectureTypes: []ec2types.ArchitectureType{ec2types.ArchitectureTypeX8664}, InstanceRequirements: requirements, - VirtualizationTypes: []*string{aws.String("xen")}, + VirtualizationTypes: []ec2types.VirtualizationType{"xen"}, }, - mock.AnythingOfType("func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) - fn(&ec2.GetInstanceTypesFromInstanceRequirementsOutput{ - InstanceTypes: []*ec2.InstanceTypeInfoFromInstanceRequirements{ + ).Return( + &ec2.GetInstanceTypesFromInstanceRequirementsOutput{ + InstanceTypes: []ec2types.InstanceTypeInfoFromInstanceRequirements{ { InstanceType: aws.String("g4dn.xlarge"), }, }, - }, false) - }).Return(nil) + }, + nil, + ) result, err := awsWrapper.getInstanceTypeFromRequirementsOverrides(mixedInstancesPolicy) assert.NoError(t, err) @@ -505,67 +542,79 @@ func TestGetInstanceTypesFromInstanceRequirementsInLaunchTemplate(t *testing.T) eksI: nil, } - instanceRequirements := &ec2.InstanceRequirements{ - VCpuCount: &ec2.VCpuCountRange{ - Min: aws.Int64(4), - Max: aws.Int64(8), + instanceRequirements := &ec2types.InstanceRequirements{ + VCpuCount: &ec2types.VCpuCountRange{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, - MemoryMiB: &ec2.MemoryMiB{ - Min: aws.Int64(4), - Max: aws.Int64(8), + MemoryMiB: &ec2types.MemoryMiB{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, - AcceleratorTypes: []*string{aws.String(autoscaling.AcceleratorTypeGpu)}, - AcceleratorManufacturers: []*string{aws.String(autoscaling.AcceleratorManufacturerNvidia)}, - AcceleratorCount: &ec2.AcceleratorCount{ - Min: aws.Int64(4), - Max: aws.Int64(8), + AcceleratorTypes: []ec2types.AcceleratorType{ec2types.AcceleratorTypeGpu}, + AcceleratorManufacturers: []ec2types.AcceleratorManufacturer{ec2types.AcceleratorManufacturerNvidia}, + AcceleratorCount: &ec2types.AcceleratorCount{ + Min: aws.Int32(4), + Max: aws.Int32(8), }, } - e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String("launchTemplateName"), - Versions: []*string{aws.String("1")}, - }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ - { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - ImageId: aws.String("123"), - InstanceRequirements: instanceRequirements, + e.On("DescribeLaunchTemplateVersions", + mock.Anything, + &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String("launchTemplateName"), + Versions: []string{"1"}, + }, + ).Return( + &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + ImageId: aws.String("123"), + InstanceRequirements: instanceRequirements, + }, }, }, }, - }) + nil, + ) - e.On("DescribeImages", &ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String("123")}, - }).Return(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Architecture: aws.String("x86_64"), - VirtualizationType: aws.String("xen"), + e.On("DescribeImages", + mock.Anything, + &ec2.DescribeImagesInput{ + ImageIds: []string{"123"}, + }, + ).Return( + &ec2.DescribeImagesOutput{ + Images: []ec2types.Image{ + { + Architecture: ec2types.ArchitectureValuesX8664, + VirtualizationType: "xen", + }, }, }, - }) + nil, + ) requirements, err := awsWrapper.getRequirementsRequestFromEC2(instanceRequirements) assert.NoError(t, err) - e.On("GetInstanceTypesFromInstanceRequirementsPages", + e.On("GetInstanceTypesFromInstanceRequirements", + mock.Anything, &ec2.GetInstanceTypesFromInstanceRequirementsInput{ - ArchitectureTypes: []*string{aws.String("x86_64")}, + ArchitectureTypes: []ec2types.ArchitectureType{ec2types.ArchitectureTypeX8664}, InstanceRequirements: requirements, - VirtualizationTypes: []*string{aws.String("xen")}, + VirtualizationTypes: []ec2types.VirtualizationType{"xen"}, }, - mock.AnythingOfType("func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) - fn(&ec2.GetInstanceTypesFromInstanceRequirementsOutput{ - InstanceTypes: []*ec2.InstanceTypeInfoFromInstanceRequirements{ + ).Return( + &ec2.GetInstanceTypesFromInstanceRequirementsOutput{ + InstanceTypes: []ec2types.InstanceTypeInfoFromInstanceRequirements{ { InstanceType: aws.String("g4dn.xlarge"), }, }, - }, false) - }).Return(nil) + }, + nil, + ) result, err := awsWrapper.getInstanceTypeByLaunchTemplate(launchTemplate) assert.NoError(t, err) @@ -581,13 +630,13 @@ func TestGetLaunchTemplateData(t *testing.T) { testCases := []struct { testName string describeTemplateData *ec2.DescribeLaunchTemplateVersionsOutput - expectedData *ec2.ResponseLaunchTemplateData + expectedData *ec2types.ResponseLaunchTemplateData expectedErr error }{ { "no launch template version found", &ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{}, + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{}, }, nil, errors.New("unable to find template versions for launch template launchTemplateName"), @@ -595,7 +644,7 @@ func TestGetLaunchTemplateData(t *testing.T) { { "no data found for launch template", &ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ { LaunchTemplateName: aws.String("launchTemplateName"), LaunchTemplateData: nil, @@ -608,16 +657,16 @@ func TestGetLaunchTemplateData(t *testing.T) { { "launch template data found successfully", &ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ { LaunchTemplateName: aws.String("launchTemplateName"), - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ ImageId: aws.String("123"), }, }, }, }, - &ec2.ResponseLaunchTemplateData{ + &ec2types.ResponseLaunchTemplateData{ ImageId: aws.String("123"), }, nil, @@ -626,11 +675,17 @@ func TestGetLaunchTemplateData(t *testing.T) { describeTemplateInput := &ec2.DescribeLaunchTemplateVersionsInput{ LaunchTemplateName: aws.String("launchTemplateName"), - Versions: []*string{aws.String("1")}, + Versions: []string{"1"}, } for _, testCase := range testCases { - e.On("DescribeLaunchTemplateVersions", describeTemplateInput).Return(testCase.describeTemplateData).Once() + e.On("DescribeLaunchTemplateVersions", + mock.Anything, + describeTemplateInput, + ).Return( + testCase.describeTemplateData, + nil, + ).Once() describeData, err := awsWrapper.getLaunchTemplateData("launchTemplateName", "1") assert.Equal(t, testCase.expectedData, describeData) @@ -643,12 +698,12 @@ func TestBuildLaunchTemplateFromSpec(t *testing.T) { units := []struct { name string - in *autoscaling.LaunchTemplateSpecification + in *autoscalingtypes.LaunchTemplateSpecification exp *launchTemplate }{ { name: "non-default, specified version", - in: &autoscaling.LaunchTemplateSpecification{ + in: &autoscalingtypes.LaunchTemplateSpecification{ LaunchTemplateName: aws.String("foo"), Version: aws.String("1"), }, @@ -659,7 +714,7 @@ func TestBuildLaunchTemplateFromSpec(t *testing.T) { }, { name: "non-default, specified $Latest", - in: &autoscaling.LaunchTemplateSpecification{ + in: &autoscalingtypes.LaunchTemplateSpecification{ LaunchTemplateName: aws.String("foo"), Version: aws.String("$Latest"), }, @@ -670,7 +725,7 @@ func TestBuildLaunchTemplateFromSpec(t *testing.T) { }, { name: "specified $Default", - in: &autoscaling.LaunchTemplateSpecification{ + in: &autoscalingtypes.LaunchTemplateSpecification{ LaunchTemplateName: aws.String("foo"), Version: aws.String("$Default"), }, @@ -681,7 +736,7 @@ func TestBuildLaunchTemplateFromSpec(t *testing.T) { }, { name: "no version specified", - in: &autoscaling.LaunchTemplateSpecification{ + in: &autoscalingtypes.LaunchTemplateSpecification{ LaunchTemplateName: aws.String("foo"), Version: nil, }, @@ -705,31 +760,37 @@ func TestGetInstanceTypesFromInstanceRequirementsWithEmptyList(t *testing.T) { ec2I: e, eksI: nil, } - requirements := &ec2.InstanceRequirementsRequest{} + requirements := &ec2types.InstanceRequirementsRequest{} - e.On("DescribeImages", &ec2.DescribeImagesInput{ - ImageIds: []*string{aws.String("123")}, - }).Return(&ec2.DescribeImagesOutput{ - Images: []*ec2.Image{ - { - Architecture: aws.String("x86_64"), - VirtualizationType: aws.String("xen"), + e.On("DescribeImages", + mock.Anything, + &ec2.DescribeImagesInput{ + ImageIds: []string{"123"}, + }, + ).Return( + &ec2.DescribeImagesOutput{ + Images: []ec2types.Image{ + { + Architecture: ec2types.ArchitectureValuesX8664, + VirtualizationType: "xen", + }, }, }, - }) - e.On("GetInstanceTypesFromInstanceRequirementsPages", + nil, + ) + e.On("GetInstanceTypesFromInstanceRequirements", + mock.Anything, &ec2.GetInstanceTypesFromInstanceRequirementsInput{ - ArchitectureTypes: []*string{aws.String("x86_64")}, + ArchitectureTypes: []ec2types.ArchitectureType{ec2types.ArchitectureTypeX8664}, InstanceRequirements: requirements, - VirtualizationTypes: []*string{aws.String("xen")}, + VirtualizationTypes: []ec2types.VirtualizationType{"xen"}, + }, + ).Return( + &ec2.GetInstanceTypesFromInstanceRequirementsOutput{ + InstanceTypes: []ec2types.InstanceTypeInfoFromInstanceRequirements{}, }, - mock.AnythingOfType("func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*ec2.GetInstanceTypesFromInstanceRequirementsOutput, bool) bool) - fn(&ec2.GetInstanceTypesFromInstanceRequirementsOutput{ - InstanceTypes: []*ec2.InstanceTypeInfoFromInstanceRequirements{}, - }, false) - }).Return(nil) + nil, + ) result, err := awsWrapper.getInstanceTypeFromInstanceRequirements("123", requirements) assert.Error(t, err) @@ -742,46 +803,46 @@ func TestTaintEksTranslator(t *testing.T) { key := "key" value := "value" - taintEffect1 := eks.TaintEffectNoSchedule + taintEffect1 := ekstypes.TaintEffectNoSchedule taintEffectTranslated1 := apiv1.TaintEffectNoSchedule - taint1 := eks.Taint{ - Effect: &taintEffect1, + taint1 := ekstypes.Taint{ + Effect: taintEffect1, Key: &key, Value: &value, } - t1, err := taintEksTranslator(&taint1) + t1, err := taintEksTranslator(taint1) assert.Nil(t, err) assert.Equal(t, t1, taintEffectTranslated1) - taintEffect2 := eks.TaintEffectNoSchedule + taintEffect2 := ekstypes.TaintEffectNoSchedule taintEffectTranslated2 := apiv1.TaintEffectNoSchedule - taint2 := eks.Taint{ - Effect: &taintEffect2, + taint2 := ekstypes.Taint{ + Effect: taintEffect2, Key: &key, Value: &value, } - t2, err := taintEksTranslator(&taint2) + t2, err := taintEksTranslator(taint2) assert.Nil(t, err) assert.Equal(t, t2, taintEffectTranslated2) - taintEffect3 := eks.TaintEffectNoExecute + taintEffect3 := ekstypes.TaintEffectNoExecute taintEffectTranslated3 := apiv1.TaintEffectNoExecute - taint3 := eks.Taint{ - Effect: &taintEffect3, + taint3 := ekstypes.Taint{ + Effect: taintEffect3, Key: &key, Value: &value, } - t3, err := taintEksTranslator(&taint3) + t3, err := taintEksTranslator(taint3) assert.Nil(t, err) assert.Equal(t, t3, taintEffectTranslated3) taintEffect4 := "TAINT_NO_EXISTS" - taint4 := eks.Taint{ - Effect: &taintEffect4, + taint4 := ekstypes.Taint{ + Effect: ekstypes.TaintEffect(taintEffect4), Key: &key, Value: &value, } - _, err = taintEksTranslator(&taint4) + _, err = taintEksTranslator(taint4) assert.Error(t, err) } diff --git a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go index 61c46c2a04d0..88e5d99a9b39 100644 --- a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go +++ b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go @@ -20,16 +20,16 @@ limitations under the License. package main import ( + "context" "flag" "html/template" "os" "time" - "k8s.io/klog/v2" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws" - awssdk "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws/session" + awssdk "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/config" + "k8s.io/klog/v2" ) var packageTemplate = template.Must(template.New("").Parse(`/* @@ -83,20 +83,17 @@ var InstanceTypes = map[string]*InstanceType{ func main() { var region = flag.String("region", "", "aws region you'd like to generate instances from.") flag.Parse() - if awssdk.StringValue(region) == "" { + if awssdk.ToString(region) == "" { klog.Fatalf("Region is required to generate instance types") } - defer klog.Flush() - sess, err := session.NewSession(&awssdk.Config{ - Region: region, - }) + awsConfig, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(awssdk.ToString(region))) if err != nil { klog.Fatal(err) } - instanceTypes, err := aws.GenerateEC2InstanceTypes(sess) + instanceTypes, err := aws.GenerateEC2InstanceTypes(awsConfig) if err != nil { klog.Fatal(err) } diff --git a/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go b/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go index fb2d90183910..b0d36318c91d 100644 --- a/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go +++ b/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go @@ -21,9 +21,11 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/aws" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/ec2" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/aws" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2" + ec2types "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/ec2/types" "k8s.io/client-go/tools/cache" test_clock "k8s.io/utils/clock/testing" ) @@ -43,25 +45,30 @@ func TestInstanceTypeCache(t *testing.T) { func TestLTVersionChange(t *testing.T) { asgName, ltName := "testasg", "launcher" - ltVersions := []*string{aws.String("1"), aws.String("2")} - instanceTypes := []*string{aws.String("t2.large"), aws.String("m4.xlarge")} + ltVersions := []string{"1", "2"} + instanceTypes := []string{"t2.large", "m4.xlarge"} a := &autoScalingMock{} e := &ec2Mock{} for i := 0; i < 2; i++ { - e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String(ltName), - Versions: []*string{ltVersions[i]}, - }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ - { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - InstanceType: instanceTypes[i], + e.On("DescribeLaunchTemplateVersions", + mock.Anything, + &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String(ltName), + Versions: []string{ltVersions[i]}, + }, + ).Return( + &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + InstanceType: ec2types.InstanceType(instanceTypes[i]), + }, }, }, - }, - }) + }, nil, + ) } fakeClock := test_clock.NewFakeClock(time.Unix(0, 0)) @@ -85,7 +92,7 @@ func TestLTVersionChange(t *testing.T) { AwsRef: asgRef, LaunchTemplate: &launchTemplate{ name: ltName, - version: aws.StringValue(ltVersions[i]), + version: ltVersions[i], }, }, }) @@ -96,7 +103,7 @@ func TestLTVersionChange(t *testing.T) { assert.Truef(t, found, "%s did not find asg (iteration %d)", asgName, i) foundInstanceType := result.(instanceTypeCachedObject).instanceType - assert.Equalf(t, foundInstanceType, *instanceTypes[i], "%s had %s, expected %s (iteration %d)", asgName, foundInstanceType, *instanceTypes[i], i) + assert.Equalf(t, foundInstanceType, instanceTypes[i], "%s had %s, expected %s (iteration %d)", asgName, foundInstanceType, instanceTypes[i], i) // Expire the first instance fakeClock.SetTime(time.Now().Add(asgInstanceTypeCacheTTL + 10*time.Minute)) diff --git a/cluster-autoscaler/cloudprovider/aws/managed_nodegroup_cache_test.go b/cluster-autoscaler/cloudprovider/aws/managed_nodegroup_cache_test.go index 84e950adb4f0..cff009f63cb1 100644 --- a/cluster-autoscaler/cloudprovider/aws/managed_nodegroup_cache_test.go +++ b/cluster-autoscaler/cloudprovider/aws/managed_nodegroup_cache_test.go @@ -18,6 +18,7 @@ package aws import ( "errors" + "github.com/stretchr/testify/mock" "strconv" "testing" "time" @@ -25,7 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go/service/eks" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks" + ekstypes "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws/aws-sdk-go-v2/service/eks/types" "k8s.io/client-go/tools/cache" test_clock "k8s.io/utils/clock/testing" ) @@ -76,10 +78,13 @@ func TestGetManagedNodegroupWithError(t *testing.T) { nodegroupName := "testNodegroup" clusterName := "testCluster" - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(nil, errors.New("AccessDenied")) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(nil, errors.New("AccessDenied")) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -107,22 +112,25 @@ func TestGetManagedNodegroupNoTaintsOrLabels(t *testing.T) { k8sVersion := "1.19" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: nil, Labels: nil, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, Taints: nil, Tags: nil, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -145,7 +153,7 @@ func TestGetManagedNodegroupWithTaintsAndLabels(t *testing.T) { amiType := "testAmiType" capacityType := "testCapacityType" k8sVersion := "1.19" - diskSize := int64(100) + diskSize := int32(100) nodegroupName := "testNodegroup" clusterName := "testCluster" @@ -154,22 +162,22 @@ func TestGetManagedNodegroupWithTaintsAndLabels(t *testing.T) { labelValue1 := "testValue 1" labelValue2 := "testValue 2" - taintEffect1 := eks.TaintEffectNoSchedule + taintEffect1 := ekstypes.TaintEffectNoSchedule taintEffectTranslated1 := apiv1.TaintEffectNoSchedule taintKey1 := "key 1" taintValue1 := "value 1" - taint1 := eks.Taint{ - Effect: &taintEffect1, + taint1 := ekstypes.Taint{ + Effect: taintEffect1, Key: &taintKey1, Value: &taintValue1, } - taintEffect2 := eks.TaintEffectNoExecute + taintEffect2 := ekstypes.TaintEffectNoExecute taintEffectTranslated2 := apiv1.TaintEffectNoExecute taintKey2 := "key 2" taintValue2 := "value 2" - taint2 := eks.Taint{ - Effect: &taintEffect2, + taint2 := ekstypes.Taint{ + Effect: taintEffect2, Key: &taintKey2, Value: &taintValue2, } @@ -180,22 +188,25 @@ func TestGetManagedNodegroupWithTaintsAndLabels(t *testing.T) { tagValue2 := "tagValue 2" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, - Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2}, + Labels: map[string]string{labelKey1: labelValue1, labelKey2: labelValue2}, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, - Taints: []*eks.Taint{&taint1, &taint2}, - Tags: map[string]*string{tagKey1: &tagValue1, tagKey2: &tagValue2}, + Taints: []ekstypes.Taint{taint1, taint2}, + Tags: map[string]string{tagKey1: tagValue1, tagKey2: tagValue2}, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -213,7 +224,7 @@ func TestGetManagedNodegroupWithTaintsAndLabels(t *testing.T) { assert.Equal(t, len(cacheObj.labels), 7) assert.Equal(t, cacheObj.labels[labelKey1], labelValue1) assert.Equal(t, cacheObj.labels[labelKey2], labelValue2) - assert.Equal(t, cacheObj.labels["diskSize"], strconv.FormatInt(diskSize, 10)) + assert.Equal(t, cacheObj.labels["diskSize"], strconv.FormatInt(int64(diskSize), 10)) assert.Equal(t, cacheObj.labels["amiType"], amiType) assert.Equal(t, cacheObj.labels["eks.amazonaws.com/capacityType"], capacityType) assert.Equal(t, cacheObj.labels["k8sVersion"], k8sVersion) @@ -229,10 +240,13 @@ func TestGetManagedNodegroupInfoObjectWithError(t *testing.T) { nodegroupName := "testNodegroup" clusterName := "testCluster" - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(nil, errors.New("AccessDenied")) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(nil, errors.New("AccessDenied")) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -275,10 +289,13 @@ func TestGetManagedNodegroupInfoObjectWithCachedNodegroup(t *testing.T) { require.NoError(t, err) assert.Equal(t, len(mngInfoObject.labels), 1) assert.Equal(t, mngInfoObject.labels[labelKey], labelValue) - k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertNotCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupInfoObjectNoCachedNodegroup(t *testing.T) { @@ -289,7 +306,7 @@ func TestGetManagedNodegroupInfoObjectNoCachedNodegroup(t *testing.T) { amiType := "testAmiType" capacityType := "testCapacityType" k8sVersion := "1.19" - diskSize := int64(100) + diskSize := int32(100) labelKey1 := "labelKey 1" labelKey2 := "labelKey 2" @@ -302,22 +319,25 @@ func TestGetManagedNodegroupInfoObjectNoCachedNodegroup(t *testing.T) { tagValue2 := "tagValue 2" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, - Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2}, + Labels: map[string]string{labelKey1: labelValue1, labelKey2: labelValue2}, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, Taints: nil, - Tags: map[string]*string{tagKey1: &tagValue1, tagKey2: &tagValue2}, + Tags: map[string]string{tagKey1: tagValue1, tagKey2: tagValue2}, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -326,7 +346,7 @@ func TestGetManagedNodegroupInfoObjectNoCachedNodegroup(t *testing.T) { assert.Equal(t, len(mngInfoObject.labels), 7) assert.Equal(t, mngInfoObject.labels[labelKey1], labelValue1) assert.Equal(t, mngInfoObject.labels[labelKey2], labelValue2) - assert.Equal(t, mngInfoObject.labels["diskSize"], strconv.FormatInt(diskSize, 10)) + assert.Equal(t, mngInfoObject.labels["diskSize"], strconv.FormatInt(int64(diskSize), 10)) assert.Equal(t, mngInfoObject.labels["amiType"], amiType) assert.Equal(t, mngInfoObject.labels["eks.amazonaws.com/capacityType"], capacityType) assert.Equal(t, mngInfoObject.labels["k8sVersion"], k8sVersion) @@ -334,10 +354,13 @@ func TestGetManagedNodegroupInfoObjectNoCachedNodegroup(t *testing.T) { assert.Equal(t, len(mngInfoObject.tags), 2) assert.Equal(t, mngInfoObject.tags[tagKey1], tagValue1) assert.Equal(t, mngInfoObject.tags[tagKey2], tagValue2) - k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupLabelsWithCachedNodegroup(t *testing.T) { @@ -371,10 +394,13 @@ func TestGetManagedNodegroupLabelsWithCachedNodegroup(t *testing.T) { require.NoError(t, err) assert.Equal(t, len(labelsMap), 1) assert.Equal(t, labelsMap[labelKey], labelValue) - k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertNotCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupLabelsNoCachedNodegroup(t *testing.T) { @@ -385,7 +411,7 @@ func TestGetManagedNodegroupLabelsNoCachedNodegroup(t *testing.T) { amiType := "testAmiType" capacityType := "testCapacityType" k8sVersion := "1.19" - diskSize := int64(100) + diskSize := int32(100) labelKey1 := "labelKey 1" labelKey2 := "labelKey 2" @@ -393,22 +419,24 @@ func TestGetManagedNodegroupLabelsNoCachedNodegroup(t *testing.T) { labelValue2 := "testValue 2" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, - Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2}, + Labels: map[string]string{labelKey1: labelValue1, labelKey2: labelValue2}, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, Taints: nil, Tags: nil, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -417,15 +445,18 @@ func TestGetManagedNodegroupLabelsNoCachedNodegroup(t *testing.T) { assert.Equal(t, len(labelsMap), 7) assert.Equal(t, labelsMap[labelKey1], labelValue1) assert.Equal(t, labelsMap[labelKey2], labelValue2) - assert.Equal(t, labelsMap["diskSize"], strconv.FormatInt(diskSize, 10)) + assert.Equal(t, labelsMap["diskSize"], strconv.FormatInt(int64(diskSize), 10)) assert.Equal(t, labelsMap["amiType"], amiType) assert.Equal(t, labelsMap["eks.amazonaws.com/capacityType"], capacityType) assert.Equal(t, labelsMap["k8sVersion"], k8sVersion) assert.Equal(t, labelsMap["eks.amazonaws.com/nodegroup"], nodegroupName) - k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupLabelsWithCachedNodegroupThatExpires(t *testing.T) { @@ -436,7 +467,7 @@ func TestGetManagedNodegroupLabelsWithCachedNodegroupThatExpires(t *testing.T) { amiType := "testAmiType" capacityType := "testCapacityType" k8sVersion := "1.19" - diskSize := int64(100) + diskSize := int32(100) labelKey1 := "labelKey 1" labelKey2 := "labelKey 2" @@ -444,22 +475,25 @@ func TestGetManagedNodegroupLabelsWithCachedNodegroupThatExpires(t *testing.T) { labelValue2 := "testValue 2" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, - Labels: map[string]*string{labelKey1: &labelValue1, labelKey2: &labelValue2}, + Labels: map[string]string{labelKey1: labelValue1, labelKey2: labelValue2}, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, Taints: nil, Tags: nil, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) fakeClock := test_clock.NewFakeClock(time.Unix(0, 0)) fakeStore := cache.NewFakeExpirationStore( @@ -499,10 +533,13 @@ func TestGetManagedNodegroupLabelsWithCachedNodegroupThatExpires(t *testing.T) { require.NoError(t, err) assert.Equal(t, len(labelsMap), 1) assert.Equal(t, labelsMap[labelKey1], labelValue1) - k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertNotCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) // Expire nodegroup fakeClock.SetTime(time.Unix(0, 0).Add(managedNodegroupCachedTTL + 1*time.Minute)) @@ -513,15 +550,18 @@ func TestGetManagedNodegroupLabelsWithCachedNodegroupThatExpires(t *testing.T) { assert.Equal(t, len(newLabelsMap), 7) assert.Equal(t, newLabelsMap[labelKey1], labelValue1) assert.Equal(t, newLabelsMap[labelKey2], labelValue2) - assert.Equal(t, newLabelsMap["diskSize"], strconv.FormatInt(diskSize, 10)) + assert.Equal(t, newLabelsMap["diskSize"], strconv.FormatInt(int64(diskSize), 10)) assert.Equal(t, newLabelsMap["amiType"], amiType) assert.Equal(t, newLabelsMap["eks.amazonaws.com/capacityType"], capacityType) assert.Equal(t, newLabelsMap["k8sVersion"], k8sVersion) assert.Equal(t, newLabelsMap["eks.amazonaws.com/nodegroup"], nodegroupName) - k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupTaintsWithCachedNodegroup(t *testing.T) { @@ -554,10 +594,13 @@ func TestGetManagedNodegroupTaintsWithCachedNodegroup(t *testing.T) { assert.Equal(t, taintsList[0].Effect, apiv1.TaintEffect(taintEffect)) assert.Equal(t, taintsList[0].Key, taintKey) assert.Equal(t, taintsList[0].Value, taintValue) - k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertNotCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupTaintsNoCachedNodegroup(t *testing.T) { @@ -568,44 +611,47 @@ func TestGetManagedNodegroupTaintsNoCachedNodegroup(t *testing.T) { amiType := "testAmiType" capacityType := "testCapacityType" k8sVersion := "1.19" - diskSize := int64(100) + diskSize := int32(100) - taintEffect1 := eks.TaintEffectNoExecute + taintEffect1 := ekstypes.TaintEffectNoExecute taintEffectTranslated1 := apiv1.TaintEffectNoExecute taintKey1 := "key 1" taintValue1 := "value 1" - taint1 := eks.Taint{ - Effect: &taintEffect1, + taint1 := ekstypes.Taint{ + Effect: taintEffect1, Key: &taintKey1, Value: &taintValue1, } - taintEffect2 := eks.TaintEffectPreferNoSchedule + taintEffect2 := ekstypes.TaintEffectPreferNoSchedule taintEffectTranslated2 := apiv1.TaintEffectPreferNoSchedule taintKey2 := "key 2" taintValue2 := "value 2" - taint2 := eks.Taint{ - Effect: &taintEffect2, + taint2 := ekstypes.Taint{ + Effect: taintEffect2, Key: &taintKey2, Value: &taintValue2, } // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, Labels: nil, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, - Taints: []*eks.Taint{&taint1, &taint2}, + Taints: []ekstypes.Taint{taint1, taint2}, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -618,10 +664,13 @@ func TestGetManagedNodegroupTaintsNoCachedNodegroup(t *testing.T) { assert.Equal(t, taintsList[1].Effect, taintEffectTranslated2) assert.Equal(t, taintsList[1].Key, taintKey2) assert.Equal(t, taintsList[1].Value, taintValue2) - k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupTagsWithCachedNodegroup(t *testing.T) { @@ -655,10 +704,12 @@ func TestGetManagedNodegroupTagsWithCachedNodegroup(t *testing.T) { require.NoError(t, err) assert.Equal(t, len(tagsMap), 1) assert.Equal(t, tagsMap[tagKey], tagValue) - k.AssertNotCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertNotCalled(t, "DescribeNodegroup", mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) } func TestGetManagedNodegroupTagsNoCachedNodegroup(t *testing.T) { @@ -669,22 +720,22 @@ func TestGetManagedNodegroupTagsNoCachedNodegroup(t *testing.T) { amiType := "testAmiType" capacityType := "testCapacityType" k8sVersion := "1.19" - diskSize := int64(100) + diskSize := int32(100) - taintEffect1 := eks.TaintEffectNoSchedule + taintEffect1 := ekstypes.TaintEffectNoSchedule taintKey1 := "key 1" taintValue1 := "value 1" - taint1 := eks.Taint{ - Effect: &taintEffect1, + taint1 := ekstypes.Taint{ + Effect: taintEffect1, Key: &taintKey1, Value: &taintValue1, } - taintEffect2 := eks.TaintEffectPreferNoSchedule + taintEffect2 := ekstypes.TaintEffectPreferNoSchedule taintKey2 := "key 2" taintValue2 := "value 2" - taint2 := eks.Taint{ - Effect: &taintEffect2, + taint2 := ekstypes.Taint{ + Effect: taintEffect2, Key: &taintKey2, Value: &taintValue2, } @@ -695,22 +746,25 @@ func TestGetManagedNodegroupTagsNoCachedNodegroup(t *testing.T) { tagValue2 := "tagValue 2" // Create test nodegroup - testNodegroup := eks.Nodegroup{ - AmiType: &amiType, + testNodegroup := ekstypes.Nodegroup{ + AmiType: ekstypes.AMITypes(amiType), ClusterName: &clusterName, DiskSize: &diskSize, Labels: nil, NodegroupName: &nodegroupName, - CapacityType: &capacityType, + CapacityType: ekstypes.CapacityTypes(capacityType), Version: &k8sVersion, - Taints: []*eks.Taint{&taint1, &taint2}, - Tags: map[string]*string{tagKey1: &tagValue1, tagKey2: &tagValue2}, + Taints: []ekstypes.Taint{taint1, taint2}, + Tags: map[string]string{tagKey1: tagValue1, tagKey2: tagValue2}, } - k.On("DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) + k.On("DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ).Return(&eks.DescribeNodegroupOutput{Nodegroup: &testNodegroup}, nil) c := newManagedNodeGroupCache(&awsWrapper{nil, nil, k}) @@ -719,8 +773,11 @@ func TestGetManagedNodegroupTagsNoCachedNodegroup(t *testing.T) { assert.Equal(t, len(tagsMap), 2) assert.Equal(t, tagsMap[tagKey1], tagValue1) assert.Equal(t, tagsMap[tagKey2], tagValue2) - k.AssertCalled(t, "DescribeNodegroup", &eks.DescribeNodegroupInput{ - ClusterName: &clusterName, - NodegroupName: &nodegroupName, - }) + k.AssertCalled(t, "DescribeNodegroup", + mock.Anything, + &eks.DescribeNodegroupInput{ + ClusterName: &clusterName, + NodegroupName: &nodegroupName, + }, + ) }