diff --git a/aws/adapter.go b/aws/adapter.go index 19df8977..11b7da45 100644 --- a/aws/adapter.go +++ b/aws/adapter.go @@ -702,7 +702,7 @@ func (a *Adapter) UpdateTargetGroupsAndAutoScalingGroups(stacks []*Stack, proble // All the required resources (listeners and target group) are created in a // transactional fashion. // Failure to create the stack causes it to be deleted automatically. -func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool) (string, error) { +func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool, stackTags map[string]string) (string, error) { certARNs := make(map[string]time.Time, len(certificateARNs)) for _, arn := range certificateARNs { certARNs[arn] = time.Time{} @@ -754,7 +754,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o httpRedirectToHTTPS: a.httpRedirectToHTTPS, nlbCrossZone: a.nlbCrossZone, http2: http2, - tags: a.stackTags, + tags: mergeTags(a.stackTags, stackTags), internalDomains: a.internalDomains, denyInternalDomains: a.denyInternalDomains, denyInternalDomainsResponse: denyResp{ @@ -767,7 +767,7 @@ func (a *Adapter) CreateStack(certificateARNs []string, scheme, securityGroup, o return createStack(a.cloudformation, spec) } -func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time.Time, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool) (string, error) { +func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time.Time, scheme, securityGroup, owner, sslPolicy, ipAddressType, wafWebACLID string, cwAlarms CloudWatchAlarmList, loadBalancerType string, http2 bool, stackTags map[string]string) (string, error) { if _, ok := SSLPolicies[sslPolicy]; !ok { return "", fmt.Errorf("invalid SSLPolicy '%s' defined", sslPolicy) } @@ -810,7 +810,7 @@ func (a *Adapter) UpdateStack(stackName string, certificateARNs map[string]time. httpRedirectToHTTPS: a.httpRedirectToHTTPS, nlbCrossZone: a.nlbCrossZone, http2: http2, - tags: a.stackTags, + tags: mergeTags(a.stackTags, stackTags), internalDomains: a.internalDomains, denyInternalDomains: a.denyInternalDomains, denyInternalDomainsResponse: denyResp{ diff --git a/aws/cf.go b/aws/cf.go index c9ed72a7..d3b53d32 100644 --- a/aws/cf.go +++ b/aws/cf.go @@ -1,6 +1,7 @@ package aws import ( + "encoding/base64" "fmt" "strings" "time" @@ -8,6 +9,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudformation" "github.com/aws/aws-sdk-go/service/cloudformation/cloudformationiface" + + log "github.com/sirupsen/logrus" ) const ( @@ -15,6 +18,7 @@ const ( certificateARNTagPrefix = "ingress:certificate-arn/" ingressOwnerTag = "ingress:owner" cwAlarmConfigHashTag = "cloudwatch:alarm-config-hash" + targetGroupsArnsTag = "ingress:targetgroups" ) // Stack is a simple wrapper around a CloudFormation Stack. @@ -34,7 +38,7 @@ type Stack struct { TargetGroupARNs []string WAFWebACLID string CertificateARNs map[string]time.Time - tags map[string]string + Tags map[string]string } // IsComplete returns true if the stack status is a complete state. @@ -480,10 +484,28 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack { http2 = false } + tgARNs := outputs.targetGroupARNs() + + // If the stack is in rollback state, the outputs are not available. + // We need to store target group ARNs in the tags. + // To restore the ARNs, and keep sending traffic to the right target groups. + if aws.StringValue(stack.StackStatus) == cloudformation.StackStatusRollbackInProgress && len(tgARNs) == 0 { + if tgARNsTag, ok := tags[targetGroupsArnsTag]; ok { + values, err := base64.StdEncoding.DecodeString(tgARNsTag) + if err != nil { + log.Errorf("failed to decode target group ARNs from tags: %v", err) + } else { + tgARNs = strings.Split(string(values), ",") + } + } + } else if len(tgARNs) > 0 { + tags[targetGroupsArnsTag] = base64.StdEncoding.EncodeToString([]byte(strings.Join(tgARNs, ","))) + } + return &Stack{ Name: aws.StringValue(stack.StackName), DNSName: outputs.dnsName(), - TargetGroupARNs: outputs.targetGroupARNs(), + TargetGroupARNs: tgARNs, Scheme: parameters[parameterLoadBalancerSchemeParameter], SecurityGroup: parameters[parameterLoadBalancerSecurityGroupParameter], SSLPolicy: parameters[parameterListenerSslPolicyParameter], @@ -491,7 +513,7 @@ func mapToManagedStack(stack *cloudformation.Stack) *Stack { LoadBalancerType: parameters[parameterLoadBalancerTypeParameter], HTTP2: http2, CertificateARNs: certificateARNs, - tags: tags, + Tags: tags, OwnerIngress: ownerIngress, status: aws.StringValue(stack.StackStatus), statusReason: aws.StringValue(stack.StackStatusReason), diff --git a/aws/cf_test.go b/aws/cf_test.go index b0acad7e..b424aaaf 100644 --- a/aws/cf_test.go +++ b/aws/cf_test.go @@ -478,10 +478,11 @@ func TestFindManagedStacks(t *testing.T) { "cert-arn": {}, }, TargetGroupARNs: []string{"tg-arn"}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), + targetGroupsArnsTag: "dGctYXJu", // "tg-arn" }, status: cloudformation.StackStatusUpdateInProgress, HTTP2: true, @@ -493,10 +494,11 @@ func TestFindManagedStacks(t *testing.T) { "cert-arn": {}, }, TargetGroupARNs: []string{"tg-arn"}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), + targetGroupsArnsTag: "dGctYXJu", // "tg-arn" }, status: cloudformation.StackStatusCreateComplete, HTTP2: true, @@ -508,10 +510,11 @@ func TestFindManagedStacks(t *testing.T) { "cert-arn": {}, }, TargetGroupARNs: []string{"tg-arn", "http-tg-arn"}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), + targetGroupsArnsTag: "dGctYXJuLGh0dHAtdGctYXJu", // "tg-arn,http-tg-arn" }, status: cloudformation.StackStatusCreateComplete, HTTP2: true, @@ -519,7 +522,7 @@ func TestFindManagedStacks(t *testing.T) { { Name: "managed-stack-not-ready", CertificateARNs: map[string]time.Time{}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, }, @@ -542,6 +545,7 @@ func TestFindManagedStacks(t *testing.T) { cfTag(kubernetesCreatorTag, DefaultControllerID), cfTag(clusterIDTagPrefix+"test-cluster", resourceLifecycleOwned), cfTag(certificateARNTagPrefix+"cert-arn", time.Time{}.Format(time.RFC3339)), + cfTag(targetGroupsArnsTag, "YXJuOmF3czpzbnM6dXMtZWFzdC0xOnRhcmdldGdyb3VwczpsYi10YXJnZXQtZ3JvdXBzMSxhcm46YXdzOnNuczp1cy1lYXN0LTE6dGFyZ2V0Z3JvdXBzOmxiLXRhcmdldC1ncm91cHMy"), // "arn:aws:sns:us-east-1:targetgroups:lb-target-groups1", "arn:aws:sns:us-east-1:targetgroups:lb-target-groups2" }, Outputs: []*cloudformation.Output{}, }, @@ -554,7 +558,44 @@ func TestFindManagedStacks(t *testing.T) { CertificateARNs: map[string]time.Time{ "cert-arn": {}, }, - tags: map[string]string{ + Tags: map[string]string{ + kubernetesCreatorTag: DefaultControllerID, + clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, + certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), + targetGroupsArnsTag: "YXJuOmF3czpzbnM6dXMtZWFzdC0xOnRhcmdldGdyb3VwczpsYi10YXJnZXQtZ3JvdXBzMSxhcm46YXdzOnNuczp1cy1lYXN0LTE6dGFyZ2V0Z3JvdXBzOmxiLXRhcmdldC1ncm91cHMy", // "arn:aws:sns:us-east-1:targetgroups:lb-target-groups1", "arn:aws:sns:us-east-1:targetgroups:lb-target-groups2" + }, + TargetGroupARNs: []string{"arn:aws:sns:us-east-1:targetgroups:lb-target-groups1", "arn:aws:sns:us-east-1:targetgroups:lb-target-groups2"}, + status: cloudformation.StackStatusRollbackInProgress, + HTTP2: true, + }, + }, + }, + { + name: "successfull-call-with-rollback-status-and-no-tg-tag", + given: fake.CFOutputs{ + DescribeStackPages: fake.R(nil, nil), + DescribeStacks: fake.R(&cloudformation.DescribeStacksOutput{ + Stacks: []*cloudformation.Stack{ + { + StackName: aws.String("managed-stack-rolling-back"), + StackStatus: aws.String(cloudformation.StackStatusRollbackInProgress), + Tags: []*cloudformation.Tag{ + cfTag(kubernetesCreatorTag, DefaultControllerID), + cfTag(clusterIDTagPrefix+"test-cluster", resourceLifecycleOwned), + cfTag(certificateARNTagPrefix+"cert-arn", time.Time{}.Format(time.RFC3339)), + }, + Outputs: []*cloudformation.Output{}, + }, + }, + }, nil), + }, + want: []*Stack{ + { + Name: "managed-stack-rolling-back", + CertificateARNs: map[string]time.Time{ + "cert-arn": {}, + }, + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), @@ -603,9 +644,10 @@ func TestFindManagedStacks(t *testing.T) { DNSName: "example-notready.com", TargetGroupARNs: []string{"tg-arn"}, CertificateARNs: map[string]time.Time{}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, + targetGroupsArnsTag: "dGctYXJu", // "tg-arn" }, status: cloudformation.StackStatusReviewInProgress, HTTP2: true, @@ -615,9 +657,10 @@ func TestFindManagedStacks(t *testing.T) { DNSName: "example.com", TargetGroupARNs: []string{"tg-arn"}, CertificateARNs: map[string]time.Time{}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, + targetGroupsArnsTag: "dGctYXJu", // "tg-arn" }, status: cloudformation.StackStatusRollbackComplete, HTTP2: true, @@ -695,10 +738,11 @@ func TestGetStack(t *testing.T) { "cert-arn": {}, }, TargetGroupARNs: []string{"tg-arn"}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), + targetGroupsArnsTag: "dGctYXJu", // "tg-arn" }, status: cloudformation.StackStatusCreateComplete, HTTP2: true, @@ -735,10 +779,11 @@ func TestGetStack(t *testing.T) { "cert-arn": {}, }, TargetGroupARNs: []string{"tg-arn", "tg-http-arn"}, - tags: map[string]string{ + Tags: map[string]string{ kubernetesCreatorTag: DefaultControllerID, clusterIDTagPrefix + "test-cluster": resourceLifecycleOwned, certificateARNTagPrefix + "cert-arn": time.Time{}.Format(time.RFC3339), + targetGroupsArnsTag: "dGctYXJuLHRnLWh0dHAtYXJu", // "tg-arn,tg-http-arn" }, status: cloudformation.StackStatusCreateComplete, HTTP2: true, diff --git a/aws/fake/cf.go b/aws/fake/cf.go index 2713e143..8f82bda1 100644 --- a/aws/fake/cf.go +++ b/aws/fake/cf.go @@ -69,6 +69,7 @@ func (m *CFClient) DescribeStacks(in *cloudformation.DescribeStacksInput) (*clou } func (m *CFClient) CreateStack(params *cloudformation.CreateStackInput) (*cloudformation.CreateStackOutput, error) { + print("\n ======== CreateStack ======== \n") m.tagCreationHistory = append(m.tagCreationHistory, params.Tags) m.paramCreationHistory = append(m.paramCreationHistory, params.Parameters) m.templateCreationHistory = append(m.templateCreationHistory, *params.TemplateBody) diff --git a/worker.go b/worker.go index 86e9beea..11ae410a 100644 --- a/worker.go +++ b/worker.go @@ -534,7 +534,12 @@ func createStack(awsAdapter *aws.Adapter, lb *loadBalancer, problems *problem.Li log.Infof("Creating stack for certificates %q / ingress %q", certificates, lb.ingresses) - stackId, err := awsAdapter.CreateStack(certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2) + tags := make(map[string]string) + if lb.stack != nil && lb.stack.Tags != nil { + tags = lb.stack.Tags + } + + stackId, err := awsAdapter.CreateStack(certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2, tags) if err != nil { if isAlreadyExistsError(err) { lb.stack, err = awsAdapter.GetStack(stackId) @@ -554,7 +559,12 @@ func updateStack(awsAdapter *aws.Adapter, lb *loadBalancer, problems *problem.Li log.Infof("Updating %q stack for %d certificates / %d ingresses", lb.scheme, len(certificates), len(lb.ingresses)) - stackId, err := awsAdapter.UpdateStack(lb.stack.Name, certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2) + tags := make(map[string]string) + if lb.stack != nil && lb.stack.Tags != nil { + tags = lb.stack.Tags + } + + stackId, err := awsAdapter.UpdateStack(lb.stack.Name, certificates, lb.scheme, lb.securityGroup, lb.Owner(), lb.sslPolicy, lb.ipAddressType, lb.wafWebACLID, lb.cwAlarms, lb.loadBalancerType, lb.http2, tags) if isNoUpdatesToBePerformedError(err) { log.Debugf("Stack(%q) is already up to date", certificates) } else if err != nil { diff --git a/worker_test.go b/worker_test.go index bb0ff981..180e0ca6 100644 --- a/worker_test.go +++ b/worker_test.go @@ -4,6 +4,7 @@ import ( "context" "crypto/x509" "encoding/json" + "fmt" "io" "net/http/httptest" "os" @@ -527,9 +528,9 @@ func TestResourceConversionOneToOne(tt *testing.T) { t.Error(problems.Errors()) } - assert.Equal(t, len(clientCF.GetTagCreationHistory()), len(tags)) + assert.Equal(t, len(clientCF.GetTagCreationHistory()), len(tags), fmt.Sprintf("got %v, expected %v", tags, clientCF.GetTagCreationHistory())) assert.Equal(t, len(clientCF.GetParamCreationHistory()), len(params)) - assert.Equal(t, len(clientCF.GetTemplateCreationHistory()), len(templates)) + assert.Equal(t, len(clientCF.GetTemplateCreationHistory()), len(templates), fmt.Sprintf("got %v, expected %v", templates, clientCF.GetTemplateCreationHistory())) // This loop is necessary because assert.ElementsMatch only do set-style comparison // for the first level of the array. So for nested arrays it would not behave like expected.