Add allow-takeover-from annotation to enable automated VPC Lattice service takeover between clusters (#841)

* Add allow-takeover-from annotation to enable automated VPC Lattice service takeover between clusters

* address review comments

---------

Co-authored-by: vbedi <[email protected]>
Co-authored-by: Ryan Lymburner <[email protected]>

Author: 3 people

docs/guides/advanced-configurations.md

@@ -132,3 +132,42 @@ spec:
       port: 80
       targetPort: 8090
 ```
+
+### Blue/Green Multi-Cluster Migration with Service Takeover
+
+For blue/green cluster migrations, the controller supports automated takeover of VPC Lattice services using the `application-networking.k8s.aws/allow-takeover-from` annotation. The annotation value must match the value of the `application-networking.k8s.aws/ManagedBy` tag on the VPC Lattice Service, which has the format `{AWS_ACCOUNT_ID}/{CLUSTER_NAME}/{VPC_ID}` (e.g., "123456789012/blue-cluster/vpc-0abc123def456789"). This eliminates the need for manual ManagedBy tag changes during cluster migrations.
+
+#### Migration Workflow
+
+1. Blue cluster creates HTTPRoute
+2. Blue cluster exports service using ServiceExport (creates standalone target group for cross-cluster access)
+3. Green cluster imports blue service using ServiceImport (references the exported target group from blue cluster)
+4. Green cluster creates HTTPRoute with takeover annotation to claim the existing VPC Lattice service:
+
+```yaml
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: inventory-service
+  annotations:
+    application-networking.k8s.aws/allow-takeover-from: "123456789012/blue-cluster/vpc-0abc123def456789"
+spec:
+  parentRefs:
+  - name: my-gateway
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /inventory
+    backendRefs:
+    - name: inventory-ver1 
+      kind: ServiceImport
+      weight: 90
+    - name: inventory-ver2
+      kind: Service
+      port: 80
+      weight: 10
+```
+
+5. Controller takes over the VPC Lattice service and updates it to reflect traffic weights in green HTTPRoute
+6. Controller updates ManagedBy tag on service, service network service association, listeners, and rules to transfer ownership

pkg/aws/cloud.go

@@ -52,6 +52,9 @@ type Cloud interface {
 	// MergeTags creates a new tag map by merging baseTags and additionalTags.
 	// BaseTags will override additionalTags for any duplicate keys.
 	MergeTags(baseTags services.Tags, additionalTags services.Tags) services.Tags
+
+	// GetManagedByFromTags extracts the ManagedBy tag value from a tags map
+	GetManagedByFromTags(tags services.Tags) string
 }
 
 // NewCloud constructs new Cloud implementation.
@@ -168,7 +171,7 @@ func (c *defaultCloud) getTags(ctx context.Context, arn string) (services.Tags,
 	return resp.Tags, nil
 }
 
-func (c *defaultCloud) getManagedByFromTags(tags services.Tags) string {
+func (c *defaultCloud) GetManagedByFromTags(tags services.Tags) string {
 	tag, ok := tags[TagManagedBy]
 	if !ok || tag == nil {
 		return ""
@@ -181,7 +184,7 @@ func (c *defaultCloud) IsArnManaged(ctx context.Context, arn string) (bool, erro
 	if err != nil {
 		return false, err
 	}
-	return c.isOwner(c.getManagedByFromTags(tags)), nil
+	return c.isOwner(c.GetManagedByFromTags(tags)), nil
 }
 
 func (c *defaultCloud) TryOwn(ctx context.Context, arn string) (bool, error) {
@@ -195,7 +198,7 @@ func (c *defaultCloud) TryOwn(ctx context.Context, arn string) (bool, error) {
 
 func (c *defaultCloud) TryOwnFromTags(ctx context.Context, arn string, tags services.Tags) (bool, error) {
 	// For resources that need backwards compatibility - not having managedBy is considered as owned by controller.
-	managedBy := c.getManagedByFromTags(tags)
+	managedBy := c.GetManagedByFromTags(tags)
 	if managedBy == "" {
 		err := c.ownResource(ctx, arn)
 		if err != nil {

pkg/aws/cloud_mocks.go

@@ -37,7 +37,7 @@ type Tagging interface {
 	FindResourcesByTags(ctx context.Context, resourceType ResourceType, tags Tags) ([]string, error)
 
 	// Updates tags for a given resource ARN
-	UpdateTags(ctx context.Context, resourceArn string, newTags Tags) error
+	UpdateTags(ctx context.Context, resourceArn string, additionalTags Tags, awsManagedTags Tags) error
 }
 
 type defaultTagging struct {
@@ -170,14 +170,21 @@ func convertTagsToFilter(tags Tags) []*taggingapi.TagFilter {
 	return filters
 }
 
-func (t *defaultTagging) UpdateTags(ctx context.Context, resourceArn string, newTags Tags) error {
+func (t *defaultTagging) UpdateTags(ctx context.Context, resourceArn string, additionalTags Tags, awsManagedTags Tags) error {
 	existingTags, err := t.GetTagsForArns(ctx, []string{resourceArn})
 	if err != nil {
 		return fmt.Errorf("failed to get existing tags: %w", err)
 	}
 
 	currentTags := k8s.GetNonAWSManagedTags(existingTags[resourceArn])
-	filteredNewTags := k8s.GetNonAWSManagedTags(newTags)
+	filteredNewTags := k8s.GetNonAWSManagedTags(additionalTags)
+
+	for key, value := range awsManagedTags {
+		if existingValue, exists := existingTags[resourceArn][key]; exists {
+			currentTags[key] = existingValue
+		}
+		filteredNewTags[key] = value
+	}
 
 	tagsToAdd, tagsToRemove := k8s.CalculateTagDifference(currentTags, filteredNewTags)
 
@@ -204,7 +211,7 @@ func (t *defaultTagging) UpdateTags(ctx context.Context, resourceArn string, new
 	return nil
 }
 
-func (t *latticeTagging) UpdateTags(ctx context.Context, resourceArn string, newTags Tags) error {
+func (t *latticeTagging) UpdateTags(ctx context.Context, resourceArn string, additionalTags Tags, awsManagedTags Tags) error {
 	existingTags, err := t.ListTagsForResourceWithContext(ctx, &vpclattice.ListTagsForResourceInput{
 		ResourceArn: aws.String(resourceArn),
 	})
@@ -213,7 +220,14 @@ func (t *latticeTagging) UpdateTags(ctx context.Context, resourceArn string, new
 	}
 
 	currentTags := k8s.GetNonAWSManagedTags(existingTags.Tags)
-	filteredNewTags := k8s.GetNonAWSManagedTags(newTags)
+	filteredNewTags := k8s.GetNonAWSManagedTags(additionalTags)
+
+	for key, value := range awsManagedTags {
+		if existingValue, exists := existingTags.Tags[key]; exists {
+			currentTags[key] = existingValue
+		}
+		filteredNewTags[key] = value
+	}
 
 	tagsToAdd, tagsToRemove := k8s.CalculateTagDifference(currentTags, filteredNewTags)
 

pkg/aws/services/tagging.go

@@ -436,87 +436,118 @@ func TestLatticeTagging_UpdateTags(t *testing.T) {
 		name               string
 		resourceArn        string
 		existingTags       Tags
-		newTags            Tags
+		additionalTags     Tags
+		awsManagedTags     Tags
 		expectedTagCalls   int
 		expectedUntagCalls int
 		expectError        bool
 		description        string
 	}{
 		{
-			name:        "nil new tags removes all existing additional tags",
+			name:        "nil additional tags removes all existing additional tags",
 			resourceArn: "arn:aws:vpc-lattice:us-west-2:123456789:service/svc-123",
 			existingTags: Tags{
 				"Environment": aws.String("Dev"),
 				"Project":     aws.String("MyApp"),
 				"application-networking.k8s.aws/ManagedBy": aws.String("123456789/cluster/vpc-123"),
 			},
-			newTags:            nil,
+			additionalTags:     nil,
+			awsManagedTags:     nil,
 			expectedTagCalls:   0,
 			expectedUntagCalls: 1,
 			expectError:        false,
-			description:        "should remove all additional tags when newTags is nil",
+			description:        "should remove all additional tags when additionalTags is nil",
 		},
 		{
-			name:        "add new tags when no existing additional tags",
+			name:        "add new additional tags when no existing additional tags",
 			resourceArn: "arn:aws:vpc-lattice:us-west-2:123456789:service/svc-123",
 			existingTags: Tags{
 				"application-networking.k8s.aws/ManagedBy": aws.String("123456789/cluster/vpc-123"),
 			},
-			newTags: Tags{
+			additionalTags: Tags{
 				"Environment": aws.String("Dev"),
 				"Project":     aws.String("MyApp"),
 			},
+			awsManagedTags:     nil,
 			expectedTagCalls:   1,
 			expectedUntagCalls: 0,
 			expectError:        false,
-			description:        "should add new tags when no existing additional tags",
+			description:        "should add new additional tags when no existing additional tags",
 		},
 		{
-			name:        "update existing additional tags",
+			name:        "update AWS managed tags only",
+			resourceArn: "arn:aws:vpc-lattice:us-west-2:123456789:service/svc-123",
+			existingTags: Tags{
+				"Environment": aws.String("Dev"),
+				"application-networking.k8s.aws/ManagedBy": aws.String("old-cluster/old-vpc"),
+			},
+			additionalTags: Tags{
+				"Environment": aws.String("Dev"),
+			},
+			awsManagedTags: Tags{
+				"application-networking.k8s.aws/ManagedBy": aws.String("new-cluster/new-vpc"),
+			},
+			expectedTagCalls:   1,
+			expectedUntagCalls: 0,
+			expectError:        false,
+			description:        "should update AWS managed tags when provided",
+		},
+		{
+			name:        "update both additional and AWS managed tags",
 			resourceArn: "arn:aws:vpc-lattice:us-west-2:123456789:service/svc-123",
 			existingTags: Tags{
 				"Environment": aws.String("Dev"),
 				"Project":     aws.String("OldApp"),
-				"application-networking.k8s.aws/ManagedBy": aws.String("123456789/cluster/vpc-123"),
+				"application-networking.k8s.aws/ManagedBy": aws.String("old-cluster/old-vpc"),
 			},
-			newTags: Tags{
+			additionalTags: Tags{
 				"Environment": aws.String("Prod"),
 				"Project":     aws.String("NewApp"),
 			},
+			awsManagedTags: Tags{
+				"application-networking.k8s.aws/ManagedBy": aws.String("new-cluster/new-vpc"),
+			},
 			expectedTagCalls:   1,
 			expectedUntagCalls: 0,
 			expectError:        false,
-			description:        "should update changed additional tag values",
+			description:        "should update both additional and AWS managed tags",
 		},
 		{
-			name:        "no changes needed",
+			name:        "no changes needed with AWS managed tags",
 			resourceArn: "arn:aws:vpc-lattice:us-west-2:123456789:service/svc-123",
 			existingTags: Tags{
 				"Environment": aws.String("Dev"),
 				"Project":     aws.String("MyApp"),
 				"application-networking.k8s.aws/ManagedBy": aws.String("123456789/cluster/vpc-123"),
 			},
-			newTags: Tags{
+			additionalTags: Tags{
 				"Environment": aws.String("Dev"),
 				"Project":     aws.String("MyApp"),
 			},
+			awsManagedTags: Tags{
+				"application-networking.k8s.aws/ManagedBy": aws.String("123456789/cluster/vpc-123"),
+			},
 			expectedTagCalls:   0,
 			expectedUntagCalls: 0,
 			expectError:        false,
 			description:        "should not make API calls when no changes needed",
 		},
 		{
-			name:         "filters out AWS managed tags from new tags",
+			name:         "filters out AWS managed tags from additional tags",
 			resourceArn:  "arn:aws:vpc-lattice:us-west-2:123456789:service/svc-123",
 			existingTags: Tags{},
-			newTags: Tags{
+			additionalTags: Tags{
 				"application-networking.k8s.aws/ManagedBy": aws.String("test-override"),
 				"application-networking.k8s.aws/RouteType": aws.String("http"),
+				"Environment": aws.String("Dev"),
 			},
-			expectedTagCalls:   0,
+			awsManagedTags: Tags{
+				"application-networking.k8s.aws/ManagedBy": aws.String("correct-value"),
+			},
+			expectedTagCalls:   1,
 			expectedUntagCalls: 0,
 			expectError:        false,
-			description:        "should filter out AWS managed tags from new tags, resulting in no API calls",
+			description:        "should filter out AWS managed tags from additional tags but include them from awsManagedTags",
 		},
 	}
 
@@ -542,7 +573,7 @@ func TestLatticeTagging_UpdateTags(t *testing.T) {
 					Return(nil, nil).Times(tt.expectedTagCalls)
 			}
 
-			err := lt.UpdateTags(ctx, tt.resourceArn, tt.newTags)
+			err := lt.UpdateTags(ctx, tt.resourceArn, tt.additionalTags, tt.awsManagedTags)
 
 			if tt.expectError {
 				assert.Error(t, err, tt.description)

pkg/aws/services/tagging_mocks.go

@@ -0,0 +1,31 @@
+package predicates
+
+import (
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+
+	"github.com/aws/aws-application-networking-k8s/pkg/k8s"
+)
+
+var AllowTakeoverFromAnnotationChangedPredicate = predicate.Funcs{
+	UpdateFunc: func(e event.UpdateEvent) bool {
+		oldAnnotations := e.ObjectOld.GetAnnotations()
+		newAnnotations := e.ObjectNew.GetAnnotations()
+
+		oldAllowTakeoverFromAnnotation := getAllowTakeoverFromAnnotation(oldAnnotations)
+		newAllowTakeoverFromAnnotation := getAllowTakeoverFromAnnotation(newAnnotations)
+
+		return oldAllowTakeoverFromAnnotation != newAllowTakeoverFromAnnotation
+	},
+	CreateFunc: func(e event.CreateEvent) bool {
+		annotations := e.Object.GetAnnotations()
+		return getAllowTakeoverFromAnnotation(annotations) != ""
+	},
+}
+
+func getAllowTakeoverFromAnnotation(annotations map[string]string) string {
+	if annotations == nil {
+		return ""
+	}
+	return annotations[k8s.AllowTakeoverFromAnnotation]
+}